Begins migration for a unified comic download package
This commit is contained in:
87
__main__.py
Normal file
87
__main__.py
Normal file
@@ -0,0 +1,87 @@
|
||||
# Copyright (c) 2025 Markil 3
|
||||
import logging
|
||||
import logging.handlers
|
||||
import argparse
|
||||
import datetime
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from comic_download.xkcd import XKCDComic, XKCDComicStrip
|
||||
|
||||
def setup_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='xkcd',
|
||||
description='Downloads the entire XKCD collection')
|
||||
|
||||
parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
|
||||
parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
|
||||
parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
|
||||
parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.")
|
||||
parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.")
|
||||
parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
|
||||
parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
|
||||
|
||||
return parser
|
||||
|
||||
def setup_logging():
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8')
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
ch.doRollover()
|
||||
logger.addHandler(ch)
|
||||
|
||||
def load_strip(base_path, index, plain, thread_limit):
|
||||
"""
|
||||
Threading function for downloading XKCD strips.
|
||||
|
||||
Arguments:
|
||||
|
||||
"""
|
||||
with thread_limit:
|
||||
strip = XKCDComicStrip(index)
|
||||
strip.await_load()
|
||||
if not strip.get_package_path(base_path).exists():
|
||||
strip.await_download()
|
||||
strip.package_data(base_path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
setup_logging()
|
||||
parser = setup_args()
|
||||
|
||||
args = parser.parse_args()
|
||||
base_path = args.output
|
||||
|
||||
if not base_path.is_dir():
|
||||
base_path.mkdir(parents=True)
|
||||
|
||||
logging.info("Beginning parsing")
|
||||
comic = XKCDComic()
|
||||
comic.await_load()
|
||||
logging.info("There are %d comics", comic.latest_identifier)
|
||||
if args.latest:
|
||||
r = range(comic.latest_identifier, comic.latest_identifier + 1)
|
||||
else:
|
||||
start = args.start or 1
|
||||
end = args.end or comic.latest_identifier
|
||||
r = range(end, start - 1, -1)
|
||||
|
||||
threads = []
|
||||
thread_limit = threading.BoundedSemaphore(value=args.threads)
|
||||
|
||||
for index in r:
|
||||
t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit))
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
Reference in New Issue
Block a user