import threading import time from errors import SpiderTaskTypeError,SpiderNotFond,SpiderReFailed import inspect import ctypes def _async_raise(tid, exctype): """raises the exception, performs cleanup if needed""" tid = ctypes.c_long(tid) if not inspect.isclass(exctype): exctype = type(exctype) res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) if res == 0: raise ValueError("invalid thread id") elif res != 1: # """if it returns a number greater than one, you're in trouble, # and you should call it again with exc=NULL to revert the effect""" ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) raise SystemError("PyThreadState_SetAsyncExc failed") def stop_thread(thread): _async_raise(thread.ident, RuntimeError) class TimmerThread(threading.Thread): def __init__(self, timmer, status, backend): threading.Thread.__init__(self) self._timmer = timmer self._status = status self._backend = backend self._running = True def terminate(self): self._running = False def timmer(self): if self._running: return self._timmer else: raise RuntimeError() def status(self): if self._running: return self._status else: raise RuntimeError() def backend(self): if self._running: return self._backend else: raise RuntimeError() def run(self): nowTime = 0.0 try: while self._running and self.timmer().start: if self.backend().getOffset() > self.timmer().maxoffset: self.timmer().progress_totle.setProgress(self.backend().getOffset()/self.timmer().maxoffset, str(self.backend().getOffset())) self.status().set("爬取结束") self.timmer().startStop() return time.sleep(0.02) if self.timmer().pause: continue nowTime += 0.02 try: self.timmer().progress_each.setProgress(nowTime/self.timmer().sleeptime, str(round(nowTime,2)) + "s") except ZeroDivisionError: self.timmer().progress_each.setProgress(1, str(round(nowTime,2)) + "s") if nowTime > self.timmer().sleeptime: self.timmer().progress_totle.setProgress(self.backend().getOffset()/self.timmer().maxoffset, str(self.backend().getOffset())) nowTime = 0.0 try: res = self.backend().doOneStep() self.status().set(res) except SpiderTaskTypeError as e: self.status().set(str(e)) self.timmer().startStop() return except SpiderReFailed as e: self.status().set(str(e)) self.timmer().startStop() return except SpiderNotFond as e: self.status().set(str(e)) except UnicodeEncodeError as e: self.status().set("编码错误:\n" + str(e)) self.timmer().startStop() return except Exception as e: self.status().set("未知错误:\n" + str(e)) self.timmer().startStop() return except RuntimeError: return