208 lines
8.4 KiB
Python
Executable File
208 lines
8.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Copyright (c) 2025 Markil 3
|
|
|
|
import logging
|
|
import logging.handlers
|
|
import argparse
|
|
import datetime
|
|
import threading
|
|
import time
|
|
import textwrap
|
|
from zipfile import ZipFile
|
|
from urllib.parse import urlparse
|
|
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from PIL import Image, ImageDraw, ImageFont
|
|
from bs4 import BeautifulSoup
|
|
|
|
class ComicStrip:
|
|
"""
|
|
A comic strip object represents a single XKCD strip.
|
|
"""
|
|
def __init__(self, index):
|
|
"""
|
|
Creates a comic strop object.
|
|
"""
|
|
self.index = index
|
|
self.url = f"https://xkcd.com/{index}"
|
|
self.data_url = f"{self.url}/info.0.json"
|
|
self.title = None
|
|
self.image_url = None
|
|
self.caption = None
|
|
self.transcript = None
|
|
|
|
def get_metadata_path(self, base_path: Path):
|
|
return Path(base_path, f"{self.index}.json")
|
|
|
|
def get_image_path(self, base_path: Path):
|
|
return Path(base_path, f"{self.index}.png")
|
|
|
|
def get_package_path(self, base_path: Path):
|
|
return Path(base_path, f"{self.index}.cbz")
|
|
|
|
def load_data(self):
|
|
logging.info("Loading data for comic %s from %s", self.index, self.url)
|
|
result = requests.get(self.data_url)
|
|
result.raise_for_status()
|
|
self.data = result.json()
|
|
self.date = datetime.date(int(self.data["year"]), int(self.data["month"]), int(self.data["day"]))
|
|
self.image_url = self.data["img"]
|
|
self.title = self.data["title"]
|
|
self.caption = self.data["alt"]
|
|
self.transcript = self.data["transcript"]
|
|
|
|
|
|
def download_data(self, base_path: Path):
|
|
"""
|
|
Downloads the raw image data.
|
|
"""
|
|
if self.image_url:
|
|
image_path = self.get_image_path(base_path)
|
|
if not image_path.exists():
|
|
with open(image_path, 'wb') as image_stream:
|
|
result = requests.get(self.image_url)
|
|
result.raise_for_status()
|
|
image_stream.write(result.content)
|
|
metadata_path = self.get_metadata_path(base_path)
|
|
if not metadata_path.exists():
|
|
with open(metadata_path, 'wb') as meta_stream:
|
|
result = requests.get(self.data_url)
|
|
result.raise_for_status()
|
|
meta_stream.write(result.content)
|
|
|
|
def transform_data(self, base_path: Path):
|
|
"""
|
|
Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent.
|
|
"""
|
|
image_path = self.get_image_path(base_path)
|
|
if image_path.exists():
|
|
f_width = 780
|
|
title_size = 21
|
|
caption_size = 12
|
|
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size, layout_engine=ImageFont.Layout.RAQM)
|
|
title_box = title_font.getbbox(self.title)
|
|
title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
|
|
|
|
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
|
caption = self.caption
|
|
try:
|
|
caption = textwrap.wrap(caption, width=(f_width - 20) / caption_size)
|
|
cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]])
|
|
caption_box = (cap_max, caption_size * len(caption))
|
|
caption = "\n".join(caption)
|
|
except Exception as e:
|
|
logging.exception("Unable to get caption bounding box", exc_info=e)
|
|
caption_box = caption_font.getbbox(caption)
|
|
caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1])
|
|
url_box = caption_font.getbbox(self.url)
|
|
url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1])
|
|
date_box = caption_font.getbbox(self.date.isoformat())
|
|
date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1])
|
|
img = Image.open(image_path)
|
|
f_img = Image.new("RGBA", (f_width, img.size[1] + caption_box[1] + title_box[1] + 40), (255, 255, 255, 255))
|
|
f_img.paste(img, ((f_width - img.size[0]) // 2, title_box[1] + 20))
|
|
|
|
draw = ImageDraw.Draw(f_img)
|
|
|
|
draw.text(((f_width - title_box[0]) / 2, 2), self.title.upper(), (0, 0, 0), font=title_font, features=["c2sc", "smcp"])
|
|
draw.multiline_text(((f_width - caption_box[0]) / 2, img.size[1] + title_box[1] + 22), caption, fill=(0, 0, 0), align="center", font=caption_font)
|
|
draw.text((10, f_img.size[1] - url_box[1] - 10), self.url, (0, 0, 0), align="left", font=caption_font)
|
|
draw.text((f_width - date_box[0] - 10, f_img.size[1] - date_box[1] - 10), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
|
|
f_img.save(image_path)
|
|
|
|
def package_data(self, base_path: Path):
|
|
"""
|
|
Compresses all the data into a cbz file and removes the source files
|
|
"""
|
|
with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
|
|
img_url = urlparse(self.image_url)
|
|
comic_zip.write(self.get_image_path(base_path), img_url.path.split('/')[-1])
|
|
if self.get_metadata_path(base_path).exists():
|
|
comic_zip.write(self.get_metadata_path(base_path), "info.0.json")
|
|
self.get_image_path(base_path).unlink()
|
|
self.get_metadata_path(base_path).unlink(missing_ok=True)
|
|
|
|
|
|
def setup_args():
|
|
parser = argparse.ArgumentParser(
|
|
prog='xkcd',
|
|
description='Downloads the entire XKCD collection')
|
|
|
|
parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
|
|
parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
|
|
parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
|
|
parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.")
|
|
parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.")
|
|
parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
|
|
parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
|
|
|
|
return parser
|
|
|
|
def setup_logging():
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
ch = logging.StreamHandler()
|
|
ch.setLevel(logging.INFO)
|
|
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
|
ch.setFormatter(formatter)
|
|
logger.addHandler(ch)
|
|
|
|
ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8')
|
|
ch.setLevel(logging.DEBUG)
|
|
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
ch.setFormatter(formatter)
|
|
ch.doRollover()
|
|
logger.addHandler(ch)
|
|
|
|
def load_strip(base_path, index, plain, thread_limit):
|
|
"""
|
|
Threading function for downloading XKCD strips.
|
|
|
|
Arguments:
|
|
|
|
"""
|
|
with thread_limit:
|
|
strip = ComicStrip(index)
|
|
if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists():
|
|
strip.load_data()
|
|
strip.download_data(base_path)
|
|
if not plain:
|
|
strip.transform_data(base_path)
|
|
strip.package_data(base_path)
|
|
|
|
if __name__ == "__main__":
|
|
setup_logging()
|
|
parser = setup_args()
|
|
|
|
args = parser.parse_args()
|
|
base_path = args.output
|
|
|
|
if not base_path.is_dir():
|
|
base_path.mkdir(parents=True)
|
|
|
|
logging.info("Beginning parsing")
|
|
latest = ComicStrip("")
|
|
latest.load_data()
|
|
logging.info("There are %d comics", latest.data["num"])
|
|
if args.latest:
|
|
r = range(latest.data["num"], latest.data["num"] + 1)
|
|
else:
|
|
start = args.start or 1
|
|
end = args.end or latest.data["num"]
|
|
r = range(end, start - 1, -1)
|
|
|
|
threads = []
|
|
thread_limit = threading.BoundedSemaphore(value=args.threads)
|
|
|
|
for index in r:
|
|
t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit))
|
|
threads.append(t)
|
|
t.start()
|
|
|
|
for t in threads:
|
|
t.join()
|