#!/usr/bin/env python3 # Copyright (c) 2025 Markil 3 import logging import logging.handlers import argparse import datetime import threading import time import textwrap from zipfile import ZipFile from urllib.parse import urlparse from pathlib import Path import requests from PIL import Image, ImageDraw, ImageFont from bs4 import BeautifulSoup class ComicStrip: """ A comic strip object represents a single XKCD strip. """ def __init__(self, index): """ Creates a comic strop object. """ self.index = index self.url = f"https://xkcd.com/{index}" self.data_url = f"{self.url}/info.0.json" self.title = None self.image_url = None self.caption = None self.transcript = None def get_metadata_path(self, base_path: Path): return Path(base_path, f"{self.index}.json") def get_image_path(self, base_path: Path): return Path(base_path, f"{self.index}.png") def get_package_path(self, base_path: Path): return Path(base_path, f"{self.index}.cbz") def load_data(self): logging.info("Loading data for comic %s from %s", self.index, self.url) result = requests.get(self.data_url) result.raise_for_status() self.data = result.json() self.date = datetime.date(int(self.data["year"]), int(self.data["month"]), int(self.data["day"])) self.image_url = self.data["img"] self.title = self.data["title"] self.caption = self.data["alt"] self.transcript = self.data["transcript"] def download_data(self, base_path: Path): """ Downloads the raw image data. """ if self.image_url: image_path = self.get_image_path(base_path) if not image_path.exists(): with open(image_path, 'wb') as image_stream: result = requests.get(self.image_url) result.raise_for_status() image_stream.write(result.content) metadata_path = self.get_metadata_path(base_path) if not metadata_path.exists(): with open(metadata_path, 'wb') as meta_stream: result = requests.get(self.data_url) result.raise_for_status() meta_stream.write(result.content) def transform_data(self, base_path: Path): """ Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent. """ image_path = self.get_image_path(base_path) if image_path.exists(): f_width = 780 title_size = 21 caption_size = 12 title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size, layout_engine=ImageFont.Layout.RAQM) title_box = title_font.getbbox(self.title) title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1]) caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size) caption = self.caption try: caption = textwrap.wrap(caption, width=(f_width - 20) / caption_size) cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]]) caption_box = (cap_max, caption_size * len(caption)) caption = "\n".join(caption) except Exception as e: logging.exception("Unable to get caption bounding box", exc_info=e) caption_box = caption_font.getbbox(caption) caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1]) url_box = caption_font.getbbox(self.url) url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1]) date_box = caption_font.getbbox(self.date.isoformat()) date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1]) img = Image.open(image_path) f_img = Image.new("RGBA", (f_width, img.size[1] + caption_box[1] + title_box[1] + 40), (255, 255, 255, 255)) f_img.paste(img, ((f_width - img.size[0]) // 2, title_box[1] + 20)) draw = ImageDraw.Draw(f_img) draw.text(((f_width - title_box[0]) / 2, 2), self.title.upper(), (0, 0, 0), font=title_font, features=["c2sc", "smcp"]) draw.multiline_text(((f_width - caption_box[0]) / 2, img.size[1] + title_box[1] + 22), caption, fill=(0, 0, 0), align="center", font=caption_font) draw.text((10, f_img.size[1] - url_box[1] - 10), self.url, (0, 0, 0), align="left", font=caption_font) draw.text((f_width - date_box[0] - 10, f_img.size[1] - date_box[1] - 10), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font) f_img.save(image_path) def package_data(self, base_path: Path): """ Compresses all the data into a cbz file and removes the source files """ with ZipFile(self.get_package_path(base_path), 'w') as comic_zip: img_url = urlparse(self.image_url) comic_zip.write(self.get_image_path(base_path), img_url.path.split('/')[-1]) if self.get_metadata_path(base_path).exists(): comic_zip.write(self.get_metadata_path(base_path), "info.0.json") self.get_image_path(base_path).unlink() self.get_metadata_path(base_path).unlink(missing_ok=True) def setup_args(): parser = argparse.ArgumentParser( prog='xkcd', description='Downloads the entire XKCD collection') parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.") parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request") parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once") parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.") parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.") parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded") parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.") return parser def setup_logging(): logger = logging.getLogger() logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.INFO) formatter = logging.Formatter('%(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8') ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) ch.doRollover() logger.addHandler(ch) def load_strip(base_path, index, plain, thread_limit): """ Threading function for downloading XKCD strips. Arguments: """ with thread_limit: strip = ComicStrip(index) if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists(): strip.load_data() strip.download_data(base_path) if not plain: strip.transform_data(base_path) strip.package_data(base_path) if __name__ == "__main__": setup_logging() parser = setup_args() args = parser.parse_args() base_path = args.output if not base_path.is_dir(): base_path.mkdir(parents=True) logging.info("Beginning parsing") latest = ComicStrip("") latest.load_data() logging.info("There are %d comics", latest.data["num"]) if args.latest: r = range(latest.data["num"], latest.data["num"] + 1) else: start = args.start or 1 end = args.end or latest.data["num"] r = range(end, start - 1, -1) threads = [] thread_limit = threading.BoundedSemaphore(value=args.threads) for index in r: t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit)) threads.append(t) t.start() for t in threads: t.join()