Adds a comic format registry
There is the limitation that you still need to manually import all modules that register a new comic format, but this does give us a lot of flexibility
This commit is contained in:
45
__main__.py
45
__main__.py
@@ -5,19 +5,22 @@ import argparse
|
||||
import datetime
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
from pathlib import Path
|
||||
from comic_download.xkcd import XKCDComic, XKCDComicStrip
|
||||
from comic_download import Comic, find_comic_class
|
||||
import comic_download.xkcd
|
||||
|
||||
def setup_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='xkcd',
|
||||
description='Downloads the entire XKCD collection')
|
||||
|
||||
parser.add_argument('url', action='extend', nargs='+', type=urlparse, help="The URLs to load")
|
||||
parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
|
||||
parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
|
||||
parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
|
||||
parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.")
|
||||
parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.")
|
||||
parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic. This option is only valid if there is a single URL.")
|
||||
parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic. This option is only valid if there is a single URL.")
|
||||
parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
|
||||
parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
|
||||
|
||||
@@ -40,7 +43,7 @@ def setup_logging():
|
||||
ch.doRollover()
|
||||
logger.addHandler(ch)
|
||||
|
||||
def load_strip(base_path, index, plain, thread_limit):
|
||||
def load_strip(base_path, comic, identifier, plain, thread_limit):
|
||||
"""
|
||||
Threading function for downloading XKCD strips.
|
||||
|
||||
@@ -48,7 +51,7 @@ def load_strip(base_path, index, plain, thread_limit):
|
||||
|
||||
"""
|
||||
with thread_limit:
|
||||
strip = XKCDComicStrip(index)
|
||||
strip = comic.create_strip(identifier)
|
||||
strip.await_load()
|
||||
if not strip.get_package_path(base_path).exists():
|
||||
strip.await_download()
|
||||
@@ -65,23 +68,27 @@ if __name__ == "__main__":
|
||||
base_path.mkdir(parents=True)
|
||||
|
||||
logging.info("Beginning parsing")
|
||||
comic = XKCDComic()
|
||||
comic.await_load()
|
||||
logging.info("There are %d comics", comic.latest_identifier)
|
||||
if args.latest:
|
||||
r = range(comic.latest_identifier, comic.latest_identifier + 1)
|
||||
else:
|
||||
start = args.start or 1
|
||||
end = args.end or comic.latest_identifier
|
||||
r = range(end, start - 1, -1)
|
||||
|
||||
|
||||
threads = []
|
||||
thread_limit = threading.BoundedSemaphore(value=args.threads)
|
||||
|
||||
for index in r:
|
||||
t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit))
|
||||
threads.append(t)
|
||||
t.start()
|
||||
for url in args.url:
|
||||
comic_type = find_comic_class(url.netloc + url.path)
|
||||
comic = comic_type.create_from_url(url)
|
||||
comic.await_load()
|
||||
logging.info("There are %d comics", comic.latest_identifier)
|
||||
if args.latest:
|
||||
r = [comic.get_latest_strip()]
|
||||
else:
|
||||
r = comic.get_all_strips()
|
||||
start = args.start or 0
|
||||
end = args.end or (len(r) - 1)
|
||||
r = r[start:end + 1]
|
||||
|
||||
for index in reversed(r):
|
||||
t = threading.Thread(name=str(comic) + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
Reference in New Issue
Block a user