markil3/ComicDownload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes the way threading works

Browse Source
This commit is contained in:
Markil 3
2025-08-02 14:01:00 -06:00
parent 83ad521fea
commit 1f7a3e3d36
2 changed files with 26 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
__main__.py
View File

@@ -87,7 +87,7 @@ if __name__ == "__main__":
r = r[start - 1:end] r = r[start - 1:end]
for index in reversed(r): for index in reversed(r):
t = threading.Thread(name=str(comic) + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit)) t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
threads.append(t) threads.append(t)
t.start() t.start()

59
comic_download/comic_strip.py
View File

@@ -5,7 +5,7 @@ import logging.handlers
import argparse import argparse
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from datetime import datetime from datetime import datetime
from threading import Lock from threading import Lock, Condition
import bisect import bisect
import time import time
import re import re
@@ -102,9 +102,9 @@ class ImageRepo(ABC):
self._data_loaded = False self._data_loaded = False
"""A flag for when the data has been successfully loaded.""" """A flag for when the data has been successfully loaded."""
self._load_lock = Lock() self._load_lock = Condition(Lock())
"""A lock for enforcing thread safety when loading this repository.""" """A lock for enforcing thread safety when loading this repository."""
self._download_lock = Lock() self._download_lock = Condition(Lock())
"""A lock for enforcing thread safety when downloading secondary resources.""" """A lock for enforcing thread safety when downloading secondary resources."""
def get_identifier_string(self): def get_identifier_string(self):
@@ -122,7 +122,7 @@ class ImageRepo(ABC):
:returns: True if the data has been fully loaded, false otherwise. :returns: True if the data has been fully loaded, false otherwise.
""" """
return not self._load_lock.locked() and self._data_loaded return not self._load_lock._lock.locked() and self._data_loaded
def is_downloaded(self): def is_downloaded(self):
""" """
@@ -130,7 +130,7 @@ class ImageRepo(ABC):
:returns: True if the images have been fully downloaded, false otherwise. :returns: True if the images have been fully downloaded, false otherwise.
""" """
return not self.image_urls or (not self._download_lock.locked() and self.images) return not self.image_urls or (not self._download_lock._lock.locked() and self.images)
def load_data(self): def load_data(self):
""" """
@@ -147,12 +147,12 @@ class ImageRepo(ABC):
:see: #await_load :see: #await_load
:see: #_load_data :see: #_load_data
""" """
if not self._load_lock.locked() and not self._data_loaded: with self._load_lock:
if self._load_lock.acquire(): if not self._data_loaded:
self._load_data() self._load_data()
logging.info("Completed loading of %s", self.get_identifier_string()) logging.info("Completed loading of %s", self.get_identifier_string())
self._data_loaded = True self._data_loaded = True
self._load_lock.release() self._load_lock.notify_all()
def await_load(self): def await_load(self):
""" """
@@ -162,12 +162,8 @@ class ImageRepo(ABC):
:see: #load_data :see: #load_data
""" """
if self._load_lock.locked(): # Loads this ourselves
self._load_lock.acquire() self.load_data()
self._load_lock.release()
else:
# Loads this ourselves
self.load_data()
@abstractmethod @abstractmethod
@@ -200,12 +196,13 @@ class ImageRepo(ABC):
if not self.is_loaded(): if not self.is_loaded():
raise SequenceException("Cannot download images before first loading performed.") raise SequenceException("Cannot download images before first loading performed.")
# Checks to see if we need to download and that one is not already in progress with self._download_lock:
if not self._download_lock.locked() and self.image_urls and not self.images: # Checks to see if we need to download and that one is not already in progress
if self._download_lock.acquire(): if self.image_urls and not self.images:
logging.info("Downloading %s", self.get_identifier_string())
self._download_data() self._download_data()
logging.info("Completed downloading of %s", self.get_identifier_string()) logging.info("Completed downloading of %s", self.get_identifier_string())
self._download_lock.release() self._download_lock.notify_all()
def await_download(self): def await_download(self):
""" """
@@ -215,12 +212,7 @@ class ImageRepo(ABC):
:see: #download_data :see: #download_data
""" """
if self._download_lock.locked(): self.download_data()
self._download_lock.acquire()
self._download_lock.release()
else:
# Downloads this ourselves
self.download_data()
def _download_data(self): def _download_data(self):
""" """
@@ -346,7 +338,7 @@ class ComicStrip(ImageRepo, ABC):
"""A list of captions for each panel.""" """A list of captions for each panel."""
self.date = None self.date = None
"""The date that the comic was published.""" """The date that the comic was published."""
self._transform_lock = Lock() self._transform_lock = Condition(Lock())
"""A lock for enforcing thread safety when transforming resources.""" """A lock for enforcing thread safety when transforming resources."""
self.transformed_images = [] self.transformed_images = []
"""A list of NamedTemporaryFiles linking to the transformed versions of the image downloads.""" """A list of NamedTemporaryFiles linking to the transformed versions of the image downloads."""
@@ -438,21 +430,20 @@ class ComicStrip(ImageRepo, ABC):
if not self.is_downloaded(): if not self.is_downloaded():
raise SequenceException("Cannot package comic strip before strip data has been fully downloaded.") raise SequenceException("Cannot package comic strip before strip data has been fully downloaded.")
if not self.comic.is_loaded(): if not self.comic.is_loaded():
raise SequenceException("Cannot package comic before data has been fully loaded.") raise SequenceException("Cannot package comic strip before comic data has been fully loaded.")
if not self.comic.is_downloaded(): if not self.comic.is_downloaded():
raise SequenceException("Cannot package comic before data has been fully downloaded.") raise SequenceException("Cannot package comic strip before comic data has been fully downloaded.")
if not self.transformed_images: if not self.transformed_images:
if not self._transformed: if not self._transformed:
if self._transform_lock.locked(): if self._transform_lock._lock.locked():
# Await for the existing transform to complete # Await for the existing transform to complete
self._transform_lock.acquire() self._transform_lock.wait()
self._transform_lock.release()
else: else:
# Run the transformation outselves # Run the transformation outselves
self._transform_lock.acquire() with self._transform_lock:
self._transform_images() self._transform_images()
self._transformed = True self._transformed = True
self._transform_lock.release() self._transform_lock.notify_all()
with ZipFile(self.get_package_path(base_path), 'w') as comic_zip: with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
for i in range(len(self.transformed_images)): for i in range(len(self.transformed_images)):
image = self.transformed_images[i] image = self.transformed_images[i]