Moves the index outside of classes
It makes more sense to have a universal index, rather than one per comic.
This commit is contained in:
@@ -14,4 +14,4 @@
|
|||||||
#
|
#
|
||||||
# You should have received a copy of the GNU General Public License
|
# You should have received a copy of the GNU General Public License
|
||||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class
|
from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present
|
||||||
|
|||||||
@@ -105,7 +105,7 @@ def __main__():
|
|||||||
comic_type = find_comic_class(url.netloc + url.path)
|
comic_type = find_comic_class(url.netloc + url.path)
|
||||||
comic = comic_type.create_from_url(url)
|
comic = comic_type.create_from_url(url)
|
||||||
comics.append(comic)
|
comics.append(comic)
|
||||||
comic.read_index(base_path)
|
comic_download.read_index(base_path)
|
||||||
comic.await_load()
|
comic.await_load()
|
||||||
if args.latest:
|
if args.latest:
|
||||||
r = [comic.get_latest_strip()]
|
r = [comic.get_latest_strip()]
|
||||||
@@ -118,19 +118,19 @@ def __main__():
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
for index in reversed(r):
|
for index in reversed(r):
|
||||||
if not comic.is_strip_present(base_path, ".cbz", str(index)):
|
if not comic_download.is_strip_present(comic, base_path, ".cbz", str(index)):
|
||||||
t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
|
t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
|
||||||
threads.append(t)
|
threads.append(t)
|
||||||
t.start()
|
t.start()
|
||||||
finally:
|
finally:
|
||||||
comic.save_index(base_path)
|
comic_download.save_index(base_path)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for t in threads:
|
for t in threads:
|
||||||
t.join()
|
t.join()
|
||||||
finally:
|
finally:
|
||||||
for comic in comics:
|
for comic in comics:
|
||||||
comic.save_index(base_path)
|
comic_download.save_index(base_path)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
__main__()
|
__main__()
|
||||||
|
|||||||
@@ -93,6 +93,97 @@ class SequenceException(Exception):
|
|||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
strip_index = {}
|
||||||
|
"""
|
||||||
|
The strip index is used to map comic strip identifiers to the actual file name (sans stems). This
|
||||||
|
is needed since sometimes, the information used to name a file (like the title) is loaded from
|
||||||
|
the webpage. By caching this information, we can more easily check to see if we can skip a download.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_index_loaded = False
|
||||||
|
"""
|
||||||
|
A flag for when the index has been successfully loaded.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_index_lock = Condition(Lock())
|
||||||
|
"""
|
||||||
|
A lock for enforcing thread safety when loading the comic index file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def read_index(base_path: Path):
|
||||||
|
"""
|
||||||
|
Saves the strip index to the same base path we download comic strips to.
|
||||||
|
It is saved to "index.json."
|
||||||
|
|
||||||
|
:param base_path: The folder to save the index to.
|
||||||
|
"""
|
||||||
|
global strip_index
|
||||||
|
global _index_loaded
|
||||||
|
global _index_lock
|
||||||
|
with _index_lock:
|
||||||
|
if not _index_loaded:
|
||||||
|
index_path = Path(base_path, "index.json")
|
||||||
|
if index_path.exists():
|
||||||
|
try:
|
||||||
|
with open(Path(base_path, "index.json"), 'r') as index_fp:
|
||||||
|
strip_index = json.load(index_fp)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
logging.exception(e)
|
||||||
|
strip_index = {}
|
||||||
|
_index_loaded = True
|
||||||
|
_index_lock.notify_all()
|
||||||
|
|
||||||
|
def save_index(base_path: Path):
|
||||||
|
"""
|
||||||
|
Saves the strip index to the same base path we download comic strips to.
|
||||||
|
It is saved to "index.json."
|
||||||
|
|
||||||
|
:param base_path: The folder to save the index to.
|
||||||
|
"""
|
||||||
|
global strip_index
|
||||||
|
global _index_loaded
|
||||||
|
global _index_lock
|
||||||
|
with _index_lock:
|
||||||
|
with open(Path(base_path, "index.json"), 'w') as index_fp:
|
||||||
|
json.dump(strip_index, index_fp)
|
||||||
|
|
||||||
|
def index_strip(strip):
|
||||||
|
|
||||||
|
"""
|
||||||
|
Registers a comic to the index. This does not save the comic index.
|
||||||
|
|
||||||
|
:param strip: The comic strip to index
|
||||||
|
"""
|
||||||
|
global strip_index
|
||||||
|
global _index_loaded
|
||||||
|
global _index_lock
|
||||||
|
with _index_lock:
|
||||||
|
if not strip.comic.identifier in strip_index:
|
||||||
|
strip_index[strip.comic.get_identifier_string()] = {}
|
||||||
|
strip_index[strip.comic.get_identifier_string()][strip.identifier] = strip.get_filename()
|
||||||
|
|
||||||
|
def is_strip_present(comic, base_path: Path, suffix: str, identifier) -> bool:
|
||||||
|
"""
|
||||||
|
Checks to see if a given strip has been saved to the file.
|
||||||
|
|
||||||
|
:param comic: The comic object that the strip belongs to.
|
||||||
|
:param base_path: The folder we expect the strip to show up in.
|
||||||
|
:param suffix: The type suffix we want to check (i.e. ".cbz")
|
||||||
|
:param identifier: The identifier to check.
|
||||||
|
:returns: True if the strip file has been saved.
|
||||||
|
"""
|
||||||
|
global strip_index
|
||||||
|
global _index_loaded
|
||||||
|
global _index_lock
|
||||||
|
if not comic.get_identifier_string() in strip_index:
|
||||||
|
return False
|
||||||
|
if not identifier in strip_index[comic.get_identifier_string()]:
|
||||||
|
return False
|
||||||
|
path = Path(base_path, strip_index[comic.get_identifier_string()][identifier] + suffix)
|
||||||
|
return path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class ImageRepo(ABC):
|
class ImageRepo(ABC):
|
||||||
"""
|
"""
|
||||||
A generic representation of an online repository of data. This class allows for the
|
A generic representation of an online repository of data. This class allows for the
|
||||||
@@ -285,77 +376,6 @@ class Comic(ImageRepo, ABC):
|
|||||||
self.author = None
|
self.author = None
|
||||||
"""The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role"."""
|
"""The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role"."""
|
||||||
|
|
||||||
self.strip_index = {}
|
|
||||||
"""
|
|
||||||
The strip index maps comic identifiers to their filenames (sans stems). If we
|
|
||||||
cache this data, we can find out where each strip lives without having to
|
|
||||||
load data from the website, thus using less requests for already-downloaded
|
|
||||||
strips.
|
|
||||||
"""
|
|
||||||
self._index_loaded = False
|
|
||||||
"""A flag for when the index has been successfully loaded."""
|
|
||||||
self._index_lock = Condition(Lock())
|
|
||||||
"""A lock for enforcing thread safety when loading the comic index file."""
|
|
||||||
|
|
||||||
def read_index(self, base_path: Path):
|
|
||||||
"""
|
|
||||||
Saves the strip index to the same base path we download comic strips to.
|
|
||||||
It is saved to "index.json."
|
|
||||||
|
|
||||||
:param base_path: The folder to save the index to.
|
|
||||||
"""
|
|
||||||
with self._index_lock:
|
|
||||||
if not self._index_loaded:
|
|
||||||
index_path = Path(base_path, "index.json")
|
|
||||||
if index_path.exists():
|
|
||||||
try:
|
|
||||||
with open(Path(base_path, "index.json"), 'r') as index_fp:
|
|
||||||
self.strip_index = json.load(index_fp)
|
|
||||||
except json.JSONDecodeError as e:
|
|
||||||
logging.exception(e)
|
|
||||||
self.strip_index = {}
|
|
||||||
self._index_loaded = True
|
|
||||||
self._index_lock.notify_all()
|
|
||||||
|
|
||||||
def save_index(self, base_path: Path):
|
|
||||||
"""
|
|
||||||
Saves the strip index to the same base path we download comic strips to.
|
|
||||||
It is saved to "index.json."
|
|
||||||
|
|
||||||
:param base_path: The folder to save the index to.
|
|
||||||
"""
|
|
||||||
with self._index_lock:
|
|
||||||
with open(Path(base_path, "index.json"), 'w') as index_fp:
|
|
||||||
json.dump(self.strip_index, index_fp)
|
|
||||||
|
|
||||||
def index_strip(self, strip):
|
|
||||||
"""
|
|
||||||
Registers a comic to the index. This does not save the comic index.
|
|
||||||
|
|
||||||
:param strip: The comic strip to index
|
|
||||||
"""
|
|
||||||
with self._index_lock:
|
|
||||||
if not self.identifier in self.strip_index:
|
|
||||||
self.strip_index[self.get_identifier_string()] = {}
|
|
||||||
self.strip_index[self.get_identifier_string()][strip.identifier] = strip.get_filename()
|
|
||||||
|
|
||||||
def is_strip_present(self, base_path: Path, suffix: str, identifier) -> bool:
|
|
||||||
"""
|
|
||||||
Checks to see if a given strip has been saved to the file.
|
|
||||||
|
|
||||||
:param base_path: The folder we expect the strip to show up in.
|
|
||||||
:param suffix: The type suffix we want to check (i.e. ".cbz")
|
|
||||||
:param identifier: The identifier to check.
|
|
||||||
:returns: True if the strip file has been saved.
|
|
||||||
"""
|
|
||||||
if not self.get_identifier_string() in self.strip_index:
|
|
||||||
return False
|
|
||||||
if not identifier in self.strip_index[self.get_identifier_string()]:
|
|
||||||
return False
|
|
||||||
path = Path(base_path, self.strip_index[self.get_identifier_string()][identifier] + suffix)
|
|
||||||
return path.exists()
|
|
||||||
|
|
||||||
|
|
||||||
#@abstractmethod
|
#@abstractmethod
|
||||||
@classmethod
|
@classmethod
|
||||||
def create_from_url(cls, url):
|
def create_from_url(cls, url):
|
||||||
@@ -474,7 +494,7 @@ class ComicStrip(ImageRepo, ABC):
|
|||||||
def load_data(self):
|
def load_data(self):
|
||||||
self.comic.load_data()
|
self.comic.load_data()
|
||||||
super().load_data()
|
super().load_data()
|
||||||
self.comic.index_strip(self)
|
index_strip(self)
|
||||||
|
|
||||||
def download_data(self):
|
def download_data(self):
|
||||||
self.comic.download_data()
|
self.comic.download_data()
|
||||||
|
|||||||
Reference in New Issue
Block a user