markil3/ComicDownload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Formats everything for proper packaging

Browse Source
This commit is contained in:
Markil 3
2025-08-03 13:10:00 -06:00
parent 771d1422d8
commit 514d119d4b
11 changed files with 951 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
comic_download/__init__.py
View File

@@ -1,2 +1,17 @@
# Copyright (c) 2025 Markil 3
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
# Copyright (C) 2025 Markil 3
# http://www.singlepilot.net
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class

136
comic_download/__main__.py Normal file
View File

@@ -0,0 +1,136 @@
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
# Copyright (C) 2025 Markil 3
# http://www.singlepilot.net
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import logging.handlers
import argparse
import datetime
import threading
import time
from urllib.parse import urlparse
from pathlib import Path
import importlib
import pkgutil
import comic_download
from comic_download import Comic, find_comic_class
def setup_args():
parser = argparse.ArgumentParser(
prog='xkcd',
description='Downloads the entire XKCD collection')
parser.add_argument('url', action='extend', nargs='+', type=urlparse, help="The URLs to load")
parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic. This option is only valid if there is a single URL.")
parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic. This option is only valid if there is a single URL.")
parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
return parser
def setup_logging():
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8')
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
ch.doRollover()
logger.addHandler(ch)
def load_plugins():
"""
Loads all registered extentions for Comic and ComicStrip. It is expected that they are all modules within the "comic_download" module.
"""
for finder, name, ispkg in pkgutil.iter_modules(comic_download.__path__, comic_download.__name__ + "."):
if name not in ("comic_download.comic_strip", "comic_download.__main__"):
logging.info("Loading plugin %s" % name)
importlib.import_module(name)
def load_strip(base_path, comic, identifier, plain, thread_limit):
"""
Threading function for downloading XKCD strips.
Arguments:
"""
with thread_limit:
strip = comic.create_strip(identifier)
strip.await_load()
if not strip.get_package_path(base_path).exists():
strip.await_download()
strip.package_data(base_path)
def __main__():
setup_logging()
parser = setup_args()
load_plugins()
args = parser.parse_args()
base_path = args.output
if not base_path.is_dir():
base_path.mkdir(parents=True)
logging.info("Beginning parsing")
threads = []
comics = []
thread_limit = threading.BoundedSemaphore(value=args.threads)
for url in args.url:
comic_type = find_comic_class(url.netloc + url.path)
comic = comic_type.create_from_url(url)
comics.append(comic)
comic.read_index(base_path)
comic.await_load()
if args.latest:
r = [comic.get_latest_strip()]
else:
r = comic.get_all_strips()
if r and type(r[0]) == int:
start = args.start or comic.get_first_strip()
end = args.end or comic.get_latest_strip()
r = r[start - 1:end]
try:
for index in reversed(r):
if not comic.is_strip_present(base_path, ".cbz", str(index)):
t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
threads.append(t)
t.start()
finally:
comic.save_index(base_path)
try:
for t in threads:
t.join()
finally:
for comic in comics:
comic.save_index(base_path)
if __name__ == "__main__":
__main__()

17
comic_download/bluesky.py
View File

@@ -1,6 +1,21 @@
#!/usr/bin/env python3
# Copyright (c) 2025 Markil 3
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
# Copyright (C) 2025 Markil 3
# http://www.singlepilot.net
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import datetime

18
comic_download/comic_strip.py
View File

@@ -1,5 +1,20 @@
#!/usr/bin/env python3
# Copyright (c) 2025 Markil 3
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
# Copyright (C) 2025 Markil 3
# http://www.singlepilot.net
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import logging.handlers
import argparse
@@ -653,7 +668,6 @@ def find_comic_class(url: str) -> type[Comic]:
:returns: A parser
"""
url_f = URLFormat(url)
logging.info(_registry_subdomains)
while len(url_f.domains) > 0:
i = bisect.bisect(_registry_subdomains, url_f)
if i == 0:

17
comic_download/existential_comics.py
View File

@@ -1,6 +1,21 @@
#!/usr/bin/env python3
# Copyright (c) 2025 Markil 3
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
# Copyright (C) 2025 Markil 3
# http://www.singlepilot.net
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import datetime

17
comic_download/xkcd.py
View File

@@ -1,6 +1,21 @@
#!/usr/bin/env python3
# Copyright (c) 2025 Markil 3
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
# Copyright (C) 2025 Markil 3
# http://www.singlepilot.net
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import logging
import datetime