NunoSempere/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improvements to task processing

Browse Source 
* Moves clean up to separate function for organization
* moves the `self.dequeued.append(item)` immediately after `self.queue.get()` so that we don't wait for it to show up

(cherry picked from commit bc9372e88f0c8855694431f811702d7fb899a97e)
This commit is contained in:
blitzmann 2020-08-29 12:24:12 -04:00
parent 4cb82ea9bd
commit e7eb5b6ea6
1 changed files with 21 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
cps/services/worker.py
View File

@ -82,16 +82,8 @@ class WorkerThread(threading.Thread):
tasks = self.queue.to_list() + self.dequeued tasks = self.queue.to_list() + self.dequeued
return sorted(tasks, key=lambda x: x.num) return sorted(tasks, key=lambda x: x.num)
def cleanup_tasks(self):
# Main thread loop starting the different tasks
def run(self):
main_thread = _get_main_thread()
while main_thread.is_alive():
item = self.queue.get()
with self.doLock: with self.doLock:
# once we hit our trigger, start cleaning up dead tasks
if len(self.dequeued) > TASK_CLEANUP_TRIGGER:
dead = [] dead = []
alive = [] alive = []
for x in self.dequeued: for x in self.dequeued:
@ -106,9 +98,21 @@ class WorkerThread(threading.Thread):
ret = sorted(dead, key=lambda x: x.task.end_time)[-TASK_CLEANUP_TRIGGER:] + alive ret = sorted(dead, key=lambda x: x.task.end_time)[-TASK_CLEANUP_TRIGGER:] + alive
self.dequeued = sorted(ret, key=lambda x: x.num) self.dequeued = sorted(ret, key=lambda x: x.num)
# Main thread loop starting the different tasks
def run(self):
main_thread = _get_main_thread()
while main_thread.is_alive():
# this blocks until something is available
item = self.queue.get()
with self.doLock:
# add to list so that in-progress tasks show up # add to list so that in-progress tasks show up
self.dequeued.append(item) self.dequeued.append(item)
# once we hit our trigger, start cleaning up dead tasks
if len(self.dequeued) > TASK_CLEANUP_TRIGGER:
self.cleanup_tasks()
# sometimes tasks (like Upload) don't actually have work to do and are created as already finished # sometimes tasks (like Upload) don't actually have work to do and are created as already finished
if item.task.stat is STAT_WAITING: if item.task.stat is STAT_WAITING:
# CalibreTask.start() should wrap all exceptions in it's own error handling # CalibreTask.start() should wrap all exceptions in it's own error handling