#!/usr/bin/env python3 # Copyright (c) 2025 Markil 3 import logging import logging.handlers import argparse from datetime import datetime import threading import time import re import textwrap import json from zipfile import ZipFile from urllib.parse import urlparse from pathlib import Path import requests from PIL import Image, ImageDraw, ImageFont from bs4 import BeautifulSoup from atproto import Client class ComicStrip: client = None def __init__(self, instance_url, handle, post): self.instance = instance_url self.handle = handle self.post = post self.title = None self.safe_title = None self.image_url = [] self.caption = [] self.date = None @classmethod def get_client(cls): if not cls.client: cls.client = Client() return cls.client def get_metadata_path(self, base_path: Path): return Path(base_path, f"{self.handle}-{self.post}.json") def get_image_path(self, base_path: Path, image_num: int = 1): return Path(base_path, f"{self.handle}-{self.post}-{image_num}.png") def get_package_path(self, base_path: Path): return Path(base_path, f"{self.date.date()}-{self.safe_title}.cbz") def load_data(self): logging.info("Loading data for post %s by %s", self.post, self.handle) post_data = self.get_client().get_post(self.post, self.handle) url = urlparse(post_data.uri) netloc = url.netloc self.title = post_data.value.text self.safe_title = re.sub('/', ' - ', self.title) self.safe_title = re.sub('&', 'and', self.safe_title) self.safe_title = re.sub(':', ' -', self.safe_title) self.safe_title = re.sub(r'\?', '', self.safe_title) self.safe_title = re.sub('\n', '', self.safe_title) if len(self.safe_title) > 70: self.safe_title = self.safe_title[0:70] if post_data.value.embed and post_data.value.embed.py_type == "app.bsky.embed.images": for i in range(len(post_data.value.embed.images)): image = post_data.value.embed.images[i] if image.py_type == "app.bsky.embed.images#image": image_link = image.image.ref.link image_link = f"https://cdn.{self.instance}/img/feed_thumbnail/plain/{netloc}/{image_link}@jpg" self.image_url.append(image_link) self.caption.append(image.alt) self.date = datetime.fromisoformat(post_data.value.created_at) def download_data(self, base_path: Path): if self.image_url: meta_path = self.get_metadata_path(base_path) if not meta_path.exists(): with open(meta_path, 'w') as meta_stream: json.dump({ "title": self.title, "image_url": self.image_url, "caption": self.caption, "date": self.date.isoformat(), }, meta_stream) for i in range(len(self.image_url)): logging.info(" Downloading %s-%s-%d", self.handle, self.post, i + 1) image_path = self.get_image_path(base_path, i + 1) if not image_path.exists(): result = requests.get(self.image_url[i]) result.raise_for_status() with open(image_path, 'wb') as image_stream: image_stream.write(result.content) def package_data(self, base_path: Path): """ Compresses all the data into a cbz file and removes the source files """ with ZipFile(self.get_package_path(base_path), 'w') as comic_zip: for i in range(len(self.image_url)): image_path = self.get_image_path(base_path, i + 1) comic_zip.write(image_path, str(i + 1) + image_path.suffix) if self.get_metadata_path(base_path).exists(): comic_zip.write(self.get_metadata_path(base_path), "info.0.json") for i in range(len(self.image_url)): self.get_image_path(base_path, i + 1).unlink() self.get_metadata_path(base_path).unlink(missing_ok=True) def setup_args(): parser = argparse.ArgumentParser( prog='xkcd', description='Downloads comics from BlueSky') parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.") parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request") parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once") parser.add_argument('--instance', type=str, default="bsky.app", help="The Bluesky instance to use (defaults to \"bsky.app\"") parser.add_argument('--handle', type=str, help="The Bluesky handle to use") parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.") parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.") parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded") parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.") return parser def setup_logging(): logger = logging.getLogger() logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.INFO) formatter = logging.Formatter('%(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) ch = logging.handlers.RotatingFileHandler("bluesky_download.log", encoding='utf-8') ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) ch.doRollover() logger.addHandler(ch) def load_strip(base_path, instance, handle, post, plain, thread_limit): """ Threading function for downloading XKCD strips. Arguments: """ with thread_limit: strip = ComicStrip(instance, handle, post) strip.load_data() if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists(): strip.download_data(base_path) #if not plain: # strip.transform_data(base_path) strip.package_data(base_path) def get_posts_from_rss(instance, handle): rss_url = f"https://{instance}/profile/{handle}/rss" result = requests.get(rss_url) result.raise_for_status() rss = BeautifulSoup(result.content, features="xml") for item in rss.channel.find_all("item"): logging.info(item) post_url = item.link.string.strip() post_url = urlparse(post_url) post_id = post_url.path.split("/")[-1] yield post_id posts = [ "3le2rltt2m22r", "3luzztwioxk2b", "3lusiuzh2nc2k", "3lungh2klv22y", "3luihmiqtus2x", "3luayf6sq5k2d", "3lu3tbja3ss23", "3ltwvnkf3g22b", "3ltsuox74es2e", "3ltsv6uplk227", "3ltpbisgqqk2d", "3ltk6o23wls2f", "3ltf6raqyj22q", "3lt6wgnn4kc2x", "3lt5karh7ms2j", "3lsyont4mk227", "3lstmfh2d5s2l", "3lsw7sos3uc2x", "3lsmpck6ims2v", "3lsm6yqgva22a", "3lsgxaqaut22z", "3lsbsgwppds2y", "3ls2hhf2zac2g", "3ls2hhjhgfk2g", "3lrwrenc7ms2c", "3lrvcuqr2d222", "3lrqdiff3zs2i", "3lriuf2gcoc2j", "3lrixbhfi3c2n", "3lrdtjetkoc2i", "3lr6qyppq4c2a", "3lqxainrmic2h", "3lr4cdocqyk2g", "3lqvc2io4722w", "3lqs74rjq3k2e", "3lqn5nrlqok2v", "3lqflgiz7x22g", "3lqaoqvftfk2h", "3lq3lxr4l4224", "3lpu3g25etk2x", "3lpp3a2fqtk2l", "3lpjz2at56k2a", "3lpckqwgqhk2q", "3lp5iihqikk2s", "3loyehjmovc22", "3loqrcugln22r", "3lolqbcgbds2r", "3logrbxvlis2m", "3lo75vdpk3c2x", "3lo27ziivr22q", "3lnv5skkeyc2i", "3lnnqzkknzs2r", "3lnikchunsk22", "3lndn7ur7ms2u", "3ln45awbjts2y", "3lmwywtb4e22s", "3lmrywp5m6s27", "3lmklt222ls27", "3lmffhpl3fc2q", "3lmaewixca22l", "3llytfxgcks2z", "3lltrv3fjuc2h", "3llrg2guy4s2v", "3lloujvsrbk2t", "3llmdjqgchs2c", "3llhc44fwkk2q", "3llc7u5r5cs2e", "3ll57ss2n7c25", "3lkx5alofos2a", "3lkvnh3if2k2a", "3lks3qgjmsu2f", "3lkqlcklufc2v", "3lkll2vqzfc27", "3lke5giatak2u", "3lk6yttystc24", "3lk4sw6af3222", "3lk4t2zaxe22h", "3lk2ke7x45k2w", "3ljso424eo22o", "3ljngjav23c2t", "3ljilhdtyqc2x", "3ljaz67awxs2b", "3lj6i36enqc2x", "3lj4bc5bag22d", "3lj3w5facqk2e", "3liwy6ljsqs26", "3lipewnxlpc2i", "3likfj5hrp22d", "3lihvotejc22v", "3lifde7c2lk2r", "3li5oqjc7fs2q", "3li3p4z2wdk2d", "3lhypepqcdc2a", "3lhtqmusffk2w", "3lhn7wct53s2l", "3lhma5544yc2g", "3lhhjtwwbgk23", "3lhh2vmmxtc2r", "3lhc3mw35ns2g", "3lh4zlga4xs23", "3lh2l2enao22q", "3lgxzc2zzps24", "3lgvfjbb5y22q", "3lgtkfpxdac2v", "3lgqj6naiuk2n", "3lgizbat2l22u", "3lgdxq2w3pc2q", "3lg74lxyb7k2t", "3lg6tvpdyic2v", "3lfxfbrmngs24", "3lfsdkdlym22m", "3lfq4ktiucc2l", "3lfnawmi6i22n", "3lffqudbdk22r", "3lfapkgbkms2p", "3lf3t22bh3c2y", "3lf3qkvvctc2j", "3leubskybes2f", "3lepbfjudgk2w", "3lemufveutk2w", "3lejym26k222f", "3lecttjatcs2g", "3lecltbuam22c", "3leaardk5uc2j", "3le56jdfhoc2r", "3le2rlwgijk2r", "3ldykvgpc622y", "3ldtcehgmzc2p", "3ldr2pc2qm22z", "3ldluyvf7jc2w", "3ldguofbiqk2e", "3ldfvg7irmc2j", "3ld7h6smups22", "3ld2g3qg32k2x", "3lcvi4dshrs2o", "3lcvb2p5asc2w", "3lcnrgclrpc2f", "3lcitywixr22q", "3lcdrbly4us2m", "3lc74s2lwhs2v", "3lbx2mppygs2b", "3lc4jsccz4225", "3lc45ylqyi22w", "3lbx2mppygs2b", "3lbvt5g2xfc2z", "3lbrynnzs4k26", "3lbnnmitbxk2p", "3lbkkx6luas22", "3lbgqnx7wu22q", "3lbfhgb4c522z", "3lbaf2ylktc2z", "3lb3yhibbjc25", "3laywzibuzk2l", "3latuve2bfc22", "3laoueyhivc2t", "3lahgfz7y6o2z", "3lacbmhxrm72z", "3la57pcazkc2d", "3l7vlhkfaw32l", "3l7tohca2wy2u", "3l7qila4qyg2a", "3l7o5grftsg2f", "3l7lhap7y4u2d", "3l7dzeyijg22d", "3l76wtw5zfx2d", "3l6zwmfshlh2h", "3l6sgqhic5o2z", "3l6nfwg5oqv2d", "3l6iga4gwdz2x", "3l6atb75y5223", "3l63otj4ure24", "3l5wsknqjlq24", "3l5p7esy64r26", "3l5kbkidgm32t", "3l5f75eev2v2i", "3l5cr4juokr23", "3l55irjwezr24", "3l4ymvvsuaq2y", "3l4toybkjg227", "3l4lyu6cfrs2b", "3l4gvipojkg2x", "3l4btvjgc6b2p", "3l42gpapcf72y", "3l3v74ouuhk2k", "3l3q73wb7jg2i", "3l3imu5jls622", "3l3duuldfiw2x", "3l36n3d67ss2m", "3l2x3ii4xqs23", "3l2s2fraxdj2z", "3l2mxjqpigt2f", "3l2fpbcq4ih2k", "3l2alzbnhtl2u", "3l23dane6eh2z", "3kztqzul4dt2u", "3kzp4j5vi3d2n", "3kzjvr6czhw2w", "3kzce6yxp5h2z", "3kz5edvageh2g", "3kyy7u47pca2a", "3kyqsamtnfz2y", "3kyloetc74k23", "3kygne33n332v", "3ky7c3z7ztp24", "3ky2bahurnl27", "3kxv4u74bhf2v", "3kxnltrvlg32r", "3kxijc74tm32u", "3kxde3ujnq72i", "3kx42evta732e", "3kwx4c6l4vz23", "3kwrytyspqd2t", "3kwk7xmo4nc23", "3kwffcat3sn2g", "3kwagbkberv2n", "3kvylvn6ebs2e", "3kvtt37sk7s2y", "3kvosflvzyk2k", "3kvh4c2jmln2u", "3kvc63r4v4d2b", "3kv4zwdfzfe2p", "3kv2i2qx7k22h", "3kuvhqlavpr2e", "3kuqecv6yzq2o", "3kullo7jigu2y", "3kudyfsy3bc2o", "3ku6si64pbs2g", "3ku6sozokgk2q", "3ktzv4yk2mc2z", "3ktuymwfthf2r", "3ktsgyxbhmk25", "3ktneffuh422p", "3ktiehoosac2p", "3ktapw575h22f", "3kt3uqywyv227", "3kswqis3lyd2u", "3ksp3mphugs25", "3kskcnww26c2g", "3kscnot6t622q", "3ks5tqnchuk2a", "3kryi3bppmq2y", "3krtjykszev22", "3krm4t6f3hd2w", "3krh2eg32ic23", "3krbzz4jxel23", "3kr26y757ol2f", "3kqy37huvv223", "3kqvajl2hkm2v", "3kqq6dfzztt2c", "3kqip35oxe22m", "3kqdkpkxrmy24", "3kq6kmhxtde2q", "3kpx6mwbyzx2q", "3kpsa2bztzw2e", "3kpn5tsxpxz2n", "3kpfolpzzbt27", "3kpaf5gww672s", "3kp6asc2k3h2w", "3kp3mq3v5rf2m", "3kou2xcx2m52k", "3koozplzpmb2c", "3kojvm6ehq42q", "3koccjnwyfg2w", "3ko5bc23nz224", "3kny7xsb7dt2b", "3knqofbpjju2g", "3knog34ey4t2b", "3knjbjipe5s2y", "3kngq72xjz42l", "3kn7bjwf2ol26", "3kn4p5fwrmk2f", "3kn2dr3vzwk2z", "3kmvdpyngrw2w", "3kmtcap2yxx2l", "3kmnkjrxwcp2s", "3kml7jfy2vw2i", "3kmj4j7esez2f", "3kmg474slz42d", "3kmemdt2ruu2d", "3kmdhojihqw22", "3km5p7t7yls2v", "3klzyhzyjvd2e", "3klxcthnt6222", "3kltk25g4l62k", "3kls5rchqlf25", "3kloifjxe4f2o", "3kllyc2fsh32r", "3klkx3kduwd25", "3klighxbzgd2y", "3klhz44tmw32l", "3klfhk4rurt2j", "3klcy3n6rak2u", "3klahec7c662c", "3kl66egnxn323", "3kkz64a6vx224", "3kkx2xdin472f", "3kkszqjc4xs2k", "3kg6mhlaxr52q", "3kg3srdu3jv2h", "3kfzhkxpciu2a", "3kfgle4bkw42w", "3kfeae6duti2r", "3kfbvgmsjhg2r", "3kf3ebk7tco2q", "3keyuh3dmer2i", "3ketpnc2xdt27", "3keh6776rog2e", "3keel4tcwks2b", "3keel3kyvcu24", "3ke7jhcpek427", "3ke3dq2v7742n", "3kdt5he6ctv2o", "3kdnywpvncg2z", "3kdl7nhx5qs2e", "3kdiy4fhmp22o", "3kddthn3jp225", "3kdbekg2bvz2n", "3kd6xewt3ic27", "3kd6vdwfcl62r", "3kd3rrah2yg26", "3kd2k4olmak2n", "3kckqafe5dy2n", "3kbwnfoelqx2x", "3kb7zr6olkp2e", "3kb7yjluo4k2o", "3kayfm2aois2l", "3kayfkv2jnv23", "3kavjsv6jxi2l", "3kagrbgquwe25", "3kacvtqswp72s", "3kaaqsygump2b", "3ka77otgehr2c", "3ka5xqef4af2c", "3ka3qeixvje25", "3ka252ljngl2q", "3ka24bi5em72c" ] if __name__ == "__main__": setup_logging() parser = setup_args() args = parser.parse_args() base_path = args.output if not base_path.is_dir(): base_path.mkdir(parents=True) logging.info("Beginning parsing") threads = [] thread_limit = threading.BoundedSemaphore(value=args.threads) for post_id in get_posts_from_rss(args.instance, args.handle): t = threading.Thread(name=f"{args.handle}-{post_id}", target=load_strip, args=(base_path, args.instance, args.handle, post_id, args.plain, thread_limit)) threads.append(t) t.start() for t in threads: t.join()