Adds Bluesky to the capabilities

bluesky.py

@@ -0,0 +1,537 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2025 Markil 3
+
+import logging
+import logging.handlers
+import argparse
+from datetime import datetime
+import threading
+import time
+import re
+import textwrap
+import json
+from zipfile import ZipFile
+from urllib.parse import urlparse
+
+from pathlib import Path
+
+import requests
+from PIL import Image, ImageDraw, ImageFont
+from bs4 import BeautifulSoup
+from atproto import Client
+
+class ComicStrip:
+    client = None
+    
+    def __init__(self, instance_url, handle, post):
+        self.instance = instance_url
+        self.handle = handle
+        self.post = post
+        self.title = None
+        self.safe_title = None
+        self.image_url = []
+        self.caption = []
+        self.date = None
+
+    @classmethod
+    def get_client(cls):
+        if not cls.client:
+            cls.client = Client()
+        return cls.client
+
+    def get_metadata_path(self, base_path: Path):
+        return Path(base_path, f"{self.handle}-{self.post}.json")
+
+    def get_image_path(self, base_path: Path, image_num: int = 1):
+        return Path(base_path, f"{self.handle}-{self.post}-{image_num}.png")
+
+    def get_package_path(self, base_path: Path):
+        return Path(base_path, f"{self.date.date()}-{self.safe_title}.cbz")
+
+    def load_data(self):
+        logging.info("Loading data for post %s by %s", self.post, self.handle)
+        post_data = self.get_client().get_post(self.post, self.handle)
+
+        url = urlparse(post_data.uri)
+        netloc = url.netloc
+        self.title = post_data.value.text
+        self.safe_title = re.sub('/', ' - ', self.title)
+        self.safe_title = re.sub('&', 'and', self.safe_title)
+        self.safe_title = re.sub(':', ' -', self.safe_title)
+        self.safe_title = re.sub(r'\?', '', self.safe_title)
+        self.safe_title = re.sub('\n', '', self.safe_title)
+        if len(self.safe_title) > 70:
+            self.safe_title = self.safe_title[0:70]
+        if post_data.value.embed and post_data.value.embed.py_type == "app.bsky.embed.images":
+            for i in range(len(post_data.value.embed.images)):
+                image = post_data.value.embed.images[i]
+                if image.py_type == "app.bsky.embed.images#image":
+                    image_link = image.image.ref.link
+                    image_link = f"https://cdn.{self.instance}/img/feed_thumbnail/plain/{netloc}/{image_link}@jpg"
+                    self.image_url.append(image_link)
+                    self.caption.append(image.alt)
+        self.date = datetime.fromisoformat(post_data.value.created_at)
+
+
+    def download_data(self, base_path: Path):
+        if self.image_url:
+            meta_path = self.get_metadata_path(base_path)
+            if not meta_path.exists():
+                with open(meta_path, 'w') as meta_stream:
+                    json.dump({
+                        "title": self.title,
+                        "image_url": self.image_url,
+                        "caption": self.caption,
+                        "date": self.date.isoformat(),
+                    }, meta_stream)
+            for i in range(len(self.image_url)):
+                logging.info("    Downloading %s-%s-%d", self.handle, self.post, i + 1)
+                image_path = self.get_image_path(base_path, i + 1)
+                if not image_path.exists():
+                    result = requests.get(self.image_url[i])
+                    result.raise_for_status()
+                    with open(image_path, 'wb') as image_stream:
+                        image_stream.write(result.content)
+
+    def package_data(self, base_path: Path):
+        """
+        Compresses all the data into a cbz file and removes the source files
+        """
+        with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
+            for i in range(len(self.image_url)):
+                image_path = self.get_image_path(base_path, i + 1)
+                comic_zip.write(image_path, str(i + 1) + image_path.suffix)
+            if self.get_metadata_path(base_path).exists():
+                comic_zip.write(self.get_metadata_path(base_path), "info.0.json")
+        for i in range(len(self.image_url)):
+            self.get_image_path(base_path, i + 1).unlink()
+        self.get_metadata_path(base_path).unlink(missing_ok=True)
+
+
+def setup_args():
+    parser = argparse.ArgumentParser(
+                    prog='xkcd',
+                    description='Downloads comics from BlueSky')
+
+    parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
+    parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
+    parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
+    parser.add_argument('--instance', type=str, default="bsky.app", help="The Bluesky instance to use (defaults to \"bsky.app\"")
+    parser.add_argument('--handle', type=str, help="The Bluesky handle to use")
+    parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.")
+    parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.")
+    parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
+    parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
+    
+    return parser
+
+def setup_logging():
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.INFO)
+    formatter = logging.Formatter('%(levelname)s - %(message)s')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    ch = logging.handlers.RotatingFileHandler("bluesky_download.log", encoding='utf-8')
+    ch.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    ch.setFormatter(formatter)
+    ch.doRollover()
+    logger.addHandler(ch)
+
+def load_strip(base_path, instance, handle, post, plain, thread_limit):
+    """
+    Threading function for downloading XKCD strips.
+    
+    Arguments:
+    
+    """
+    with thread_limit:
+        strip = ComicStrip(instance, handle, post)
+        strip.load_data()
+        if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists():
+            strip.download_data(base_path)
+            #if not plain:
+            #    strip.transform_data(base_path)
+            strip.package_data(base_path)
+
+def get_posts_from_rss(instance, handle):
+    rss_url = f"https://{instance}/profile/{handle}/rss"
+    result = requests.get(rss_url)
+    result.raise_for_status()
+    rss = BeautifulSoup(result.content, features="xml")
+
+    for item in rss.channel.find_all("item"):
+        logging.info(item)
+        post_url = item.link.string.strip()
+        post_url = urlparse(post_url)
+        post_id = post_url.path.split("/")[-1]
+        yield post_id
+
+posts = [
+  "3le2rltt2m22r",
+  "3luzztwioxk2b",
+  "3lusiuzh2nc2k",
+  "3lungh2klv22y",
+  "3luihmiqtus2x",
+  "3luayf6sq5k2d",
+  "3lu3tbja3ss23",
+  "3ltwvnkf3g22b",
+  "3ltsuox74es2e",
+  "3ltsv6uplk227",
+  "3ltpbisgqqk2d",
+  "3ltk6o23wls2f",
+  "3ltf6raqyj22q",
+  "3lt6wgnn4kc2x",
+  "3lt5karh7ms2j",
+  "3lsyont4mk227",
+  "3lstmfh2d5s2l",
+  "3lsw7sos3uc2x",
+  "3lsmpck6ims2v",
+  "3lsm6yqgva22a",
+  "3lsgxaqaut22z",
+  "3lsbsgwppds2y",
+  "3ls2hhf2zac2g",
+  "3ls2hhjhgfk2g",
+  "3lrwrenc7ms2c",
+  "3lrvcuqr2d222",
+  "3lrqdiff3zs2i",
+  "3lriuf2gcoc2j",
+  "3lrixbhfi3c2n",
+  "3lrdtjetkoc2i",
+  "3lr6qyppq4c2a",
+  "3lqxainrmic2h",
+  "3lr4cdocqyk2g",
+  "3lqvc2io4722w",
+  "3lqs74rjq3k2e",
+  "3lqn5nrlqok2v",
+  "3lqflgiz7x22g",
+  "3lqaoqvftfk2h",
+  "3lq3lxr4l4224",
+  "3lpu3g25etk2x",
+  "3lpp3a2fqtk2l",
+  "3lpjz2at56k2a",
+  "3lpckqwgqhk2q",
+  "3lp5iihqikk2s",
+  "3loyehjmovc22",
+  "3loqrcugln22r",
+  "3lolqbcgbds2r",
+  "3logrbxvlis2m",
+  "3lo75vdpk3c2x",
+  "3lo27ziivr22q",
+  "3lnv5skkeyc2i",
+  "3lnnqzkknzs2r",
+  "3lnikchunsk22",
+  "3lndn7ur7ms2u",
+  "3ln45awbjts2y",
+  "3lmwywtb4e22s",
+  "3lmrywp5m6s27",
+  "3lmklt222ls27",
+  "3lmffhpl3fc2q",
+  "3lmaewixca22l",
+  "3llytfxgcks2z",
+  "3lltrv3fjuc2h",
+  "3llrg2guy4s2v",
+  "3lloujvsrbk2t",
+  "3llmdjqgchs2c",
+  "3llhc44fwkk2q",
+  "3llc7u5r5cs2e",
+  "3ll57ss2n7c25",
+  "3lkx5alofos2a",
+  "3lkvnh3if2k2a",
+  "3lks3qgjmsu2f",
+  "3lkqlcklufc2v",
+  "3lkll2vqzfc27",
+  "3lke5giatak2u",
+  "3lk6yttystc24",
+  "3lk4sw6af3222",
+  "3lk4t2zaxe22h",
+  "3lk2ke7x45k2w",
+  "3ljso424eo22o",
+  "3ljngjav23c2t",
+  "3ljilhdtyqc2x",
+  "3ljaz67awxs2b",
+  "3lj6i36enqc2x",
+  "3lj4bc5bag22d",
+  "3lj3w5facqk2e",
+  "3liwy6ljsqs26",
+  "3lipewnxlpc2i",
+  "3likfj5hrp22d",
+  "3lihvotejc22v",
+  "3lifde7c2lk2r",
+  "3li5oqjc7fs2q",
+  "3li3p4z2wdk2d",
+  "3lhypepqcdc2a",
+  "3lhtqmusffk2w",
+  "3lhn7wct53s2l",
+  "3lhma5544yc2g",
+  "3lhhjtwwbgk23",
+  "3lhh2vmmxtc2r",
+  "3lhc3mw35ns2g",
+  "3lh4zlga4xs23",
+  "3lh2l2enao22q",
+  "3lgxzc2zzps24",
+  "3lgvfjbb5y22q",
+  "3lgtkfpxdac2v",
+  "3lgqj6naiuk2n",
+  "3lgizbat2l22u",
+  "3lgdxq2w3pc2q",
+  "3lg74lxyb7k2t",
+  "3lg6tvpdyic2v",
+  "3lfxfbrmngs24",
+  "3lfsdkdlym22m",
+  "3lfq4ktiucc2l",
+  "3lfnawmi6i22n",
+  "3lffqudbdk22r",
+  "3lfapkgbkms2p",
+  "3lf3t22bh3c2y",
+  "3lf3qkvvctc2j",
+  "3leubskybes2f",
+  "3lepbfjudgk2w",
+  "3lemufveutk2w",
+  "3lejym26k222f",
+  "3lecttjatcs2g",
+  "3lecltbuam22c",
+  "3leaardk5uc2j",
+  "3le56jdfhoc2r",
+  "3le2rlwgijk2r",
+  "3ldykvgpc622y",
+  "3ldtcehgmzc2p",
+  "3ldr2pc2qm22z",
+  "3ldluyvf7jc2w",
+  "3ldguofbiqk2e",
+  "3ldfvg7irmc2j",
+  "3ld7h6smups22",
+  "3ld2g3qg32k2x",
+  "3lcvi4dshrs2o",
+  "3lcvb2p5asc2w",
+  "3lcnrgclrpc2f",
+  "3lcitywixr22q",
+  "3lcdrbly4us2m",
+  "3lc74s2lwhs2v",
+  "3lbx2mppygs2b",
+  "3lc4jsccz4225",
+  "3lc45ylqyi22w",
+  "3lbx2mppygs2b",
+  "3lbvt5g2xfc2z",
+  "3lbrynnzs4k26",
+  "3lbnnmitbxk2p",
+  "3lbkkx6luas22",
+  "3lbgqnx7wu22q",
+  "3lbfhgb4c522z",
+  "3lbaf2ylktc2z",
+  "3lb3yhibbjc25",
+  "3laywzibuzk2l",
+  "3latuve2bfc22",
+  "3laoueyhivc2t",
+  "3lahgfz7y6o2z",
+  "3lacbmhxrm72z",
+  "3la57pcazkc2d",
+  "3l7vlhkfaw32l",
+  "3l7tohca2wy2u",
+  "3l7qila4qyg2a",
+  "3l7o5grftsg2f",
+  "3l7lhap7y4u2d",
+  "3l7dzeyijg22d",
+  "3l76wtw5zfx2d",
+  "3l6zwmfshlh2h",
+  "3l6sgqhic5o2z",
+  "3l6nfwg5oqv2d",
+  "3l6iga4gwdz2x",
+  "3l6atb75y5223",
+  "3l63otj4ure24",
+  "3l5wsknqjlq24",
+  "3l5p7esy64r26",
+  "3l5kbkidgm32t",
+  "3l5f75eev2v2i",
+  "3l5cr4juokr23",
+  "3l55irjwezr24",
+  "3l4ymvvsuaq2y",
+  "3l4toybkjg227",
+  "3l4lyu6cfrs2b",
+  "3l4gvipojkg2x",
+  "3l4btvjgc6b2p",
+  "3l42gpapcf72y",
+  "3l3v74ouuhk2k",
+  "3l3q73wb7jg2i",
+  "3l3imu5jls622",
+  "3l3duuldfiw2x",
+  "3l36n3d67ss2m",
+  "3l2x3ii4xqs23",
+  "3l2s2fraxdj2z",
+  "3l2mxjqpigt2f",
+  "3l2fpbcq4ih2k",
+  "3l2alzbnhtl2u",
+  "3l23dane6eh2z",
+  "3kztqzul4dt2u",
+  "3kzp4j5vi3d2n",
+  "3kzjvr6czhw2w",
+  "3kzce6yxp5h2z",
+  "3kz5edvageh2g",
+  "3kyy7u47pca2a",
+  "3kyqsamtnfz2y",
+  "3kyloetc74k23",
+  "3kygne33n332v",
+  "3ky7c3z7ztp24",
+  "3ky2bahurnl27",
+  "3kxv4u74bhf2v",
+  "3kxnltrvlg32r",
+  "3kxijc74tm32u",
+  "3kxde3ujnq72i",
+  "3kx42evta732e",
+  "3kwx4c6l4vz23",
+  "3kwrytyspqd2t",
+  "3kwk7xmo4nc23",
+  "3kwffcat3sn2g",
+  "3kwagbkberv2n",
+  "3kvylvn6ebs2e",
+  "3kvtt37sk7s2y",
+  "3kvosflvzyk2k",
+  "3kvh4c2jmln2u",
+  "3kvc63r4v4d2b",
+  "3kv4zwdfzfe2p",
+  "3kv2i2qx7k22h",
+  "3kuvhqlavpr2e",
+  "3kuqecv6yzq2o",
+  "3kullo7jigu2y",
+  "3kudyfsy3bc2o",
+  "3ku6si64pbs2g",
+  "3ku6sozokgk2q",
+  "3ktzv4yk2mc2z",
+  "3ktuymwfthf2r",
+  "3ktsgyxbhmk25",
+  "3ktneffuh422p",
+  "3ktiehoosac2p",
+  "3ktapw575h22f",
+  "3kt3uqywyv227",
+  "3kswqis3lyd2u",
+  "3ksp3mphugs25",
+  "3kskcnww26c2g",
+  "3kscnot6t622q",
+  "3ks5tqnchuk2a",
+  "3kryi3bppmq2y",
+  "3krtjykszev22",
+  "3krm4t6f3hd2w",
+  "3krh2eg32ic23",
+  "3krbzz4jxel23",
+  "3kr26y757ol2f",
+  "3kqy37huvv223",
+  "3kqvajl2hkm2v",
+  "3kqq6dfzztt2c",
+  "3kqip35oxe22m",
+  "3kqdkpkxrmy24",
+  "3kq6kmhxtde2q",
+  "3kpx6mwbyzx2q",
+  "3kpsa2bztzw2e",
+  "3kpn5tsxpxz2n",
+  "3kpfolpzzbt27",
+  "3kpaf5gww672s",
+  "3kp6asc2k3h2w",
+  "3kp3mq3v5rf2m",
+  "3kou2xcx2m52k",
+  "3koozplzpmb2c",
+  "3kojvm6ehq42q",
+  "3koccjnwyfg2w",
+  "3ko5bc23nz224",
+  "3kny7xsb7dt2b",
+  "3knqofbpjju2g",
+  "3knog34ey4t2b",
+  "3knjbjipe5s2y",
+  "3kngq72xjz42l",
+  "3kn7bjwf2ol26",
+  "3kn4p5fwrmk2f",
+  "3kn2dr3vzwk2z",
+  "3kmvdpyngrw2w",
+  "3kmtcap2yxx2l",
+  "3kmnkjrxwcp2s",
+  "3kml7jfy2vw2i",
+  "3kmj4j7esez2f",
+  "3kmg474slz42d",
+  "3kmemdt2ruu2d",
+  "3kmdhojihqw22",
+  "3km5p7t7yls2v",
+  "3klzyhzyjvd2e",
+  "3klxcthnt6222",
+  "3kltk25g4l62k",
+  "3kls5rchqlf25",
+  "3kloifjxe4f2o",
+  "3kllyc2fsh32r",
+  "3klkx3kduwd25",
+  "3klighxbzgd2y",
+  "3klhz44tmw32l",
+  "3klfhk4rurt2j",
+  "3klcy3n6rak2u",
+  "3klahec7c662c",
+  "3kl66egnxn323",
+  "3kkz64a6vx224",
+  "3kkx2xdin472f",
+  "3kkszqjc4xs2k",
+  "3kg6mhlaxr52q",
+  "3kg3srdu3jv2h",
+  "3kfzhkxpciu2a",
+  "3kfgle4bkw42w",
+  "3kfeae6duti2r",
+  "3kfbvgmsjhg2r",
+  "3kf3ebk7tco2q",
+  "3keyuh3dmer2i",
+  "3ketpnc2xdt27",
+  "3keh6776rog2e",
+  "3keel4tcwks2b",
+  "3keel3kyvcu24",
+  "3ke7jhcpek427",
+  "3ke3dq2v7742n",
+  "3kdt5he6ctv2o",
+  "3kdnywpvncg2z",
+  "3kdl7nhx5qs2e",
+  "3kdiy4fhmp22o",
+  "3kddthn3jp225",
+  "3kdbekg2bvz2n",
+  "3kd6xewt3ic27",
+  "3kd6vdwfcl62r",
+  "3kd3rrah2yg26",
+  "3kd2k4olmak2n",
+  "3kckqafe5dy2n",
+  "3kbwnfoelqx2x",
+  "3kb7zr6olkp2e",
+  "3kb7yjluo4k2o",
+  "3kayfm2aois2l",
+  "3kayfkv2jnv23",
+  "3kavjsv6jxi2l",
+  "3kagrbgquwe25",
+  "3kacvtqswp72s",
+  "3kaaqsygump2b",
+  "3ka77otgehr2c",
+  "3ka5xqef4af2c",
+  "3ka3qeixvje25",
+  "3ka252ljngl2q",
+  "3ka24bi5em72c"
+]
+    
+
+if __name__ == "__main__":
+    setup_logging()
+    parser = setup_args()
+
+    args = parser.parse_args()
+    base_path = args.output
+
+    if not base_path.is_dir():
+        base_path.mkdir(parents=True)
+
+    logging.info("Beginning parsing")
+
+    threads = []
+    thread_limit = threading.BoundedSemaphore(value=args.threads)
+
+    for post_id in get_posts_from_rss(args.instance, args.handle):
+        t = threading.Thread(name=f"{args.handle}-{post_id}", target=load_strip, args=(base_path, args.instance, args.handle, post_id, args.plain, thread_limit))
+        threads.append(t)
+        t.start()
+
+    for t in threads:
+        t.join()
+

download.sh

@@ -6,4 +6,4 @@ path=$(dirname $(realpath $0))
 source "$path/venv/bin/activate"
 "$path/xkcd.py" --output /mnt/MEDIA/Comics/xkcd -l
 "$path/existential_comics.py" --output /mnt/MEDIA/Comics/existential_comics -l
-
+"$path/bluesky.py" --handle fieldexplores.bsky.social --output /mnt/MEDIA/Comics/field_explores

requirements.txt

@@ -2,4 +2,4 @@ requests
 beautifulsoup4
 lxml
 pillow
-libraqm
+atproto