markil3/ComicDownload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moves the index outside of classes

Browse Source 
It makes more sense to have a universal index, rather than one per comic.
This commit is contained in:
Markil 3
2025-08-12 13:38:00 -06:00
parent b15913b4ca
commit 4ede8bcaa8
3 changed files with 97 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
comic_download/__init__.py
View File

@@ -14,4 +14,4 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class
from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present

8
comic_download/__main__.py
View File

@@ -105,7 +105,7 @@ def __main__():
comic_type = find_comic_class(url.netloc + url.path)
comic = comic_type.create_from_url(url)
comics.append(comic)
comic.read_index(base_path)
comic_download.read_index(base_path)
comic.await_load()
if args.latest:
r = [comic.get_latest_strip()]
@@ -118,19 +118,19 @@ def __main__():
try:
for index in reversed(r):
if not comic.is_strip_present(base_path, ".cbz", str(index)):
if not comic_download.is_strip_present(comic, base_path, ".cbz", str(index)):
t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
threads.append(t)
t.start()
finally:
comic.save_index(base_path)
comic_download.save_index(base_path)
try:
for t in threads:
t.join()
finally:
for comic in comics:
comic.save_index(base_path)
comic_download.save_index(base_path)
if __name__ == "__main__":
__main__()

164
comic_download/comic_strip.py
View File

@@ -93,6 +93,97 @@ class SequenceException(Exception):
"""
pass
strip_index = {}
"""
The strip index is used to map comic strip identifiers to the actual file name (sans stems). This
is needed since sometimes, the information used to name a file (like the title) is loaded from
the webpage. By caching this information, we can more easily check to see if we can skip a download.
"""
_index_loaded = False
"""
A flag for when the index has been successfully loaded.
"""
_index_lock = Condition(Lock())
"""
A lock for enforcing thread safety when loading the comic index file.
"""
def read_index(base_path: Path):
"""
Saves the strip index to the same base path we download comic strips to.
It is saved to "index.json."
:param base_path: The folder to save the index to.
"""
global strip_index
global _index_loaded
global _index_lock
with _index_lock:
if not _index_loaded:
index_path = Path(base_path, "index.json")
if index_path.exists():
try:
with open(Path(base_path, "index.json"), 'r') as index_fp:
strip_index = json.load(index_fp)
except json.JSONDecodeError as e:
logging.exception(e)
strip_index = {}
_index_loaded = True
_index_lock.notify_all()
def save_index(base_path: Path):
"""
Saves the strip index to the same base path we download comic strips to.
It is saved to "index.json."
:param base_path: The folder to save the index to.
"""
global strip_index
global _index_loaded
global _index_lock
with _index_lock:
with open(Path(base_path, "index.json"), 'w') as index_fp:
json.dump(strip_index, index_fp)
def index_strip(strip):
"""
Registers a comic to the index. This does not save the comic index.
:param strip: The comic strip to index
"""
global strip_index
global _index_loaded
global _index_lock
with _index_lock:
if not strip.comic.identifier in strip_index:
strip_index[strip.comic.get_identifier_string()] = {}
strip_index[strip.comic.get_identifier_string()][strip.identifier] = strip.get_filename()
def is_strip_present(comic, base_path: Path, suffix: str, identifier) -> bool:
"""
Checks to see if a given strip has been saved to the file.
:param comic: The comic object that the strip belongs to.
:param base_path: The folder we expect the strip to show up in.
:param suffix: The type suffix we want to check (i.e. ".cbz")
:param identifier: The identifier to check.
:returns: True if the strip file has been saved.
"""
global strip_index
global _index_loaded
global _index_lock
if not comic.get_identifier_string() in strip_index:
return False
if not identifier in strip_index[comic.get_identifier_string()]:
return False
path = Path(base_path, strip_index[comic.get_identifier_string()][identifier] + suffix)
return path.exists()
class ImageRepo(ABC):
"""
A generic representation of an online repository of data. This class allows for the
@@ -285,77 +376,6 @@ class Comic(ImageRepo, ABC):
self.author = None
"""The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role"."""
self.strip_index = {}
"""
The strip index maps comic identifiers to their filenames (sans stems). If we
cache this data, we can find out where each strip lives without having to
load data from the website, thus using less requests for already-downloaded
strips.
"""
self._index_loaded = False
"""A flag for when the index has been successfully loaded."""
self._index_lock = Condition(Lock())
"""A lock for enforcing thread safety when loading the comic index file."""
def read_index(self, base_path: Path):
"""
Saves the strip index to the same base path we download comic strips to.
It is saved to "index.json."
:param base_path: The folder to save the index to.
"""
with self._index_lock:
if not self._index_loaded:
index_path = Path(base_path, "index.json")
if index_path.exists():
try:
with open(Path(base_path, "index.json"), 'r') as index_fp:
self.strip_index = json.load(index_fp)
except json.JSONDecodeError as e:
logging.exception(e)
self.strip_index = {}
self._index_loaded = True
self._index_lock.notify_all()
def save_index(self, base_path: Path):
"""
Saves the strip index to the same base path we download comic strips to.
It is saved to "index.json."
:param base_path: The folder to save the index to.
"""
with self._index_lock:
with open(Path(base_path, "index.json"), 'w') as index_fp:
json.dump(self.strip_index, index_fp)
def index_strip(self, strip):
"""
Registers a comic to the index. This does not save the comic index.
:param strip: The comic strip to index
"""
with self._index_lock:
if not self.identifier in self.strip_index:
self.strip_index[self.get_identifier_string()] = {}
self.strip_index[self.get_identifier_string()][strip.identifier] = strip.get_filename()
def is_strip_present(self, base_path: Path, suffix: str, identifier) -> bool:
"""
Checks to see if a given strip has been saved to the file.
:param base_path: The folder we expect the strip to show up in.
:param suffix: The type suffix we want to check (i.e. ".cbz")
:param identifier: The identifier to check.
:returns: True if the strip file has been saved.
"""
if not self.get_identifier_string() in self.strip_index:
return False
if not identifier in self.strip_index[self.get_identifier_string()]:
return False
path = Path(base_path, self.strip_index[self.get_identifier_string()][identifier] + suffix)
return path.exists()
#@abstractmethod
@classmethod
def create_from_url(cls, url):
@@ -474,7 +494,7 @@ class ComicStrip(ImageRepo, ABC):
def load_data(self):
self.comic.load_data()
super().load_data()
self.comic.index_strip(self)
index_strip(self)
def download_data(self):
self.comic.download_data()