87 lines
3.6 KiB
Python
87 lines
3.6 KiB
Python
import threading
|
|
import time
|
|
from errors import SpiderTaskTypeError,SpiderNotFond,SpiderReFailed
|
|
import inspect
|
|
import ctypes
|
|
def _async_raise(tid, exctype):
|
|
"""raises the exception, performs cleanup if needed"""
|
|
tid = ctypes.c_long(tid)
|
|
if not inspect.isclass(exctype):
|
|
exctype = type(exctype)
|
|
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
|
|
if res == 0:
|
|
raise ValueError("invalid thread id")
|
|
elif res != 1:
|
|
# """if it returns a number greater than one, you're in trouble,
|
|
# and you should call it again with exc=NULL to revert the effect"""
|
|
ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
|
|
raise SystemError("PyThreadState_SetAsyncExc failed")
|
|
def stop_thread(thread):
|
|
_async_raise(thread.ident, RuntimeError)
|
|
class TimmerThread(threading.Thread):
|
|
def __init__(self, timmer, status, backend):
|
|
threading.Thread.__init__(self)
|
|
self._timmer = timmer
|
|
self._status = status
|
|
self._backend = backend
|
|
self._running = True
|
|
def terminate(self):
|
|
self._running = False
|
|
def timmer(self):
|
|
if self._running:
|
|
return self._timmer
|
|
else:
|
|
raise RuntimeError()
|
|
def status(self):
|
|
if self._running:
|
|
return self._status
|
|
else:
|
|
raise RuntimeError()
|
|
def backend(self):
|
|
if self._running:
|
|
return self._backend
|
|
else:
|
|
raise RuntimeError()
|
|
def run(self):
|
|
nowTime = 0.0
|
|
try:
|
|
while self._running and self.timmer().start:
|
|
if self.backend().getOffset() > self.timmer().maxoffset:
|
|
self.timmer().progress_totle.setProgress(self.backend().getOffset()/self.timmer().maxoffset, str(self.backend().getOffset()))
|
|
self.status().set("爬取结束")
|
|
self.timmer().startStop()
|
|
return
|
|
time.sleep(0.02)
|
|
if self.timmer().pause:
|
|
continue
|
|
nowTime += 0.02
|
|
try:
|
|
self.timmer().progress_each.setProgress(nowTime/self.timmer().sleeptime, str(round(nowTime,2)) + "s")
|
|
except ZeroDivisionError:
|
|
self.timmer().progress_each.setProgress(1, str(round(nowTime,2)) + "s")
|
|
if nowTime > self.timmer().sleeptime:
|
|
self.timmer().progress_totle.setProgress(self.backend().getOffset()/self.timmer().maxoffset, str(self.backend().getOffset()))
|
|
nowTime = 0.0
|
|
try:
|
|
res = self.backend().doOneStep()
|
|
self.status().set(res)
|
|
except SpiderTaskTypeError as e:
|
|
self.status().set(str(e))
|
|
self.timmer().startStop()
|
|
return
|
|
except SpiderReFailed as e:
|
|
self.status().set(str(e))
|
|
self.timmer().startStop()
|
|
return
|
|
except SpiderNotFond as e:
|
|
self.status().set(str(e))
|
|
except UnicodeEncodeError as e:
|
|
self.status().set("编码错误：\n" + str(e))
|
|
self.timmer().startStop()
|
|
return
|
|
except Exception as e:
|
|
self.status().set("未知错误：\n" + str(e))
|
|
self.timmer().startStop()
|
|
return
|
|
except RuntimeError:
|
|
return |