support general archive format

Author: URenko

comiclib/main.py

@@ -3,11 +3,10 @@
 
 from .scan import watch, scannow
 from .config import settings
-from .utils import extract_thumbnail, convert_image
+from .utils import is_image, extract_thumbnail, convert_image, ArchiveFile
 from typing import Union, Annotated
 from enum import Enum
 from pathlib import Path
-from zipfile import ZipFile
 import re
 import base64
 import tempfile
@@ -267,11 +266,11 @@ def extract_archive(id: str, force: bool = True, db: Session = Depends(get_db)):
         return JSONResponse({"operation": "", "error": "This ID doesn't exist on the server.", "success": 0}, status.HTTP_400_BAD_REQUEST)
     path = Path(settings.content) / a.path
     if path.is_dir():
-        pages = [f"./api/archives/{id}/page?path="+quote(p.name, safe='') for p in sorted(path.iterdir()) if p.suffix != '.txt' and not p.name.startswith('.')]
-    elif path.suffix == '.zip':
-        with ZipFile(path) as z:
+        pages = [f"./api/archives/{id}/page?path="+quote(p.name, safe='') for p in sorted(path.iterdir()) if is_image(p)]
+    elif ArchiveFile.support_formats.fullmatch(path.name):
+        with ArchiveFile(path) as z:
             pages = [f"./api/archives/{id}/page?path="+quote(z_info.filename, safe='') for z_info in filter(
-                lambda z_info: not z_info.is_dir(), z.infolist())]
+                lambda z_info: not z_info.is_dir() and is_image(z_info.filename), z.infolist())]
     else:
         raise NotImplementedError
     return {"job": -1, "pages": pages}
@@ -292,15 +291,15 @@ def get_archive_page(request: Request, id: str, path: str, db: Session = Depends
             saveto.parent.mkdir(parents=True, exist_ok=True)
             if p.is_dir():
                 convert_image(p / path, saveto)
-            elif p.suffix == '.zip':
-                with ZipFile(p) as z, z.open(path) as f:
+            elif ArchiveFile.support_formats.fullmatch(path.name):
+                with ArchiveFile(p) as z, z.open(path) as f:
                     convert_image(f, saveto)
             else:
                 raise NotImplementedError
         return FileResponse(saveto)
-    if p.suffix == '.zip':
+    if ArchiveFile.support_formats.fullmatch(str(p)):
         def iterfile():
-            with ZipFile(p) as z, z.open(path) as f:
+            with ArchiveFile(p) as z, z.open(path) as f:
                 yield from f
         return StreamingResponse(iterfile())
     else:

comiclib/scanner/10-zip.py

@@ -1,7 +1,8 @@
 from pathlib import Path
-
 from zipfile import ZipFile
 
+from comiclib.utils import is_image
+
 import logging
 logger = logging.getLogger(__name__)
 
@@ -14,7 +15,7 @@ def scan(self, path: Path, id: str, metadata: dict, prev_scanners: list[str]) ->
             metadata["title"] = path.stem
             with ZipFile(path) as z:
                 metadata["pagecount"] = len(
-                    list(filter(lambda z_info: not z_info.is_dir(), z.infolist())))
+                    list(filter(lambda z_info: not z_info.is_dir() and is_image(z_info.filename), z.infolist())))
             assert metadata["pagecount"] > 0
             return True
         else:

comiclib/scanner/11-archive.py

@@ -0,0 +1,21 @@
+from pathlib import Path
+
+from comiclib.utils import is_image, ArchiveFile
+
+import logging
+logger = logging.getLogger(__name__)
+
+class Scanner:
+    '''Handle regular archived files, except zip, with the filename as the title.'''
+    
+    def scan(self, path: Path, id: str, metadata: dict, prev_scanners: list[str]) -> bool:
+        if '10-zip' not in prev_scanners and not ArchiveFile.support_formats.fullmatch(path.name) is None and not path.is_dir():
+            logger.info(f' <- {path}')
+            metadata["title"] = path.stem
+            with ArchiveFile(path) as z:
+                metadata["pagecount"] = len(
+                    list(filter(lambda z_info: not z_info.is_dir() and is_image(z_info.filename), z.infolist())))
+            assert metadata["pagecount"] > 0
+            return True
+        else:
+            return False

comiclib/utils.py

@@ -1,19 +1,31 @@
 from typing import Union
 from pathlib import Path
 from zipfile import ZipFile
-import asyncio
+import io
+import re
 import subprocess
 import tempfile
 import shutil
+import mimetypes
+import itertools
 import PIL
 from PIL import Image
 try:
     from jxlpy import JXLImagePlugin
 except ModuleNotFoundError:
     pass
 
+import logging
+logger = logging.getLogger(__name__)
+
 from .config import settings
 
+
+mimetypes.add_type('image/jxl', '.jxl')
+def is_image(p: Union[str, Path]):
+    mime = mimetypes.guess_type(p, strict=False)[0]
+    return False if mime is None else mime.partition('/')[0] == 'image'
+
 def convert_image(f_or_path, saveto: str, thumbnail=False):
     try:
         with Image.open(f_or_path) as im:
@@ -40,10 +52,10 @@ def extract_thumbnail(path: Union[str, Path], id: str, page: int, cache=False, c
         return saveto.relative_to(settings.thumb)
     saveto.parent.mkdir(parents=True, exist_ok=True)
     if path.is_dir():
-        convert_image(sorted(filter(lambda p:p.suffix != '.txt' and not p.name.startswith('.'), path.iterdir()))[page-1], saveto, thumbnail=True)
-    elif path.suffix == '.zip':
-        with ZipFile(path) as z:
-            with z.open(list(filter(lambda z_info: not z_info.is_dir(), z.infolist()))[page-1].filename) as f:
+        convert_image(sorted(filter(is_image, path.iterdir()))[page-1], saveto, thumbnail=True)
+    elif ArchiveFile.support_formats.fullmatch(path.name):
+        with ArchiveFile(path) as z:
+            with z.open(list(filter(lambda z_info: not z_info.is_dir() and is_image(z_info.filename), z.infolist()))[page-1].filename) as f:
                 convert_image(f, saveto, thumbnail=True)
     else:
         raise NotImplementedError
@@ -52,4 +64,75 @@ def extract_thumbnail(path: Union[str, Path], id: str, page: int, cache=False, c
         cover_path.parent.mkdir(parents=True, exist_ok=True)
         shutil.copy2(saveto, cover_path)
         return cover_path.name
-    return saveto.relative_to(settings.thumb)
+    return saveto.relative_to(settings.thumb)
+
+class ArchiveInfo:
+    def __init__(self, filename):
+        self.filename = filename
+    
+    def is_dir(self):
+        return self._is_dir
+
+class ArchiveFile:
+    support_formats = re.compile('.+\.(zip|rar|7z)$', re.IGNORECASE)
+    executable = None
+
+    def __init__(self, file):
+        self.file = Path(file)
+        if self.file.suffix.lower() == '.zip':
+            self.zipfile = ZipFile(file)
+            return
+        else:
+            self.zipfile = None
+        self._infolist = None
+        if self.executable is None:
+            sevenzip = ('7zzs', '7zz', '7z', '7za', '7zr')
+            for executable in itertools.chain(('./'+_s for _s in sevenzip), sevenzip):
+                try:
+                    p = subprocess.run([executable, 'i'], capture_output=True)
+                    if p.returncode == 0:
+                        self.executable = executable
+                    if b'Rar' not in p.stdout:
+                        logger.warning("Your version of 7-Zip does not support rar files, please download the correct version from https://7-zip.org/download.html.")
+                except FileNotFoundError:
+                    continue
+            if self.executable is None:
+                raise FileNotFoundError("We encountered a non-zip archive, this requires 7-Zip, but you do not have it installed, please download from https://7-zip.org/download.html and make sure 7zzs or 7zz or 7z is in the working directory or the directory indicated by PATH.")
+            logger.debug(f"Use 7-Zip {self.executable}")
+    
+    def infolist(self):
+        if not self._infolist is None:
+            return self._infolist
+        self._infolist = []
+        stdout = subprocess.run([self.executable, 'l', str(self.file)], check=True, capture_output=True, text=True).stdout
+        list_start = False
+        line_sep = '------------------- ----- ------------ ------------  ------------------------'
+        for line in stdout.splitlines():
+            if list_start:
+                if line == line_sep:
+                    break
+                m = re.match(r'\S+ \S+ ([\.D])\S{4} +\d+ +\d*  (.+)', line)
+                if m is None: raise NotImplementedError
+                archive_info = ArchiveInfo(m[2])
+                archive_info._is_dir = m[1] == 'D'
+                self._infolist.append(archive_info)
+            if line == line_sep:
+                list_start = True
+        return self._infolist
+    
+    def namelist(self):
+        return (archive_info.filename for archive_info in self.infolist())
+    
+    def open(self, name):
+        p = subprocess.run([self.executable, 'e', '-so', str(self.file), name], check=True, capture_output=True)
+        return io.BytesIO(p.stdout)
+    
+    def __enter__(self):
+        if self.zipfile is None:
+            return self
+        else:
+            return self.zipfile.__enter__()
+
+    def __exit__(self, *args):
+        if not self.zipfile is None:
+            self.zipfile.__exit__(*args)

docs/en/docs/supported-formats.md

@@ -3,6 +3,9 @@
 | File extension | Download method | Corresponding scanner |
 | -------------- | --------------- | ----------- |
 | zip | general | 10-zip.py |
+| rar, 7z [^1] | general | 11-archive.py |
 | zip | [ccloli/E-Hentai-Downloader](https://github.com/ccloli/E-Hentai-Downloader) | 20-ccloli.py |
 | folder | [Hentai@Home](https://ehwiki.org/wiki/Hentai@Home#H.40H_Downloader) | 21-hath.py |
-| folder | [EhViewer](https://github.com/seven332/EhViewer) | 22-ehviewer.py |
+| folder | [EhViewer](https://github.com/seven332/EhViewer) | 22-ehviewer.py |
+
+[^1]: They require 7-Zip, download it from [https://7-zip.org/download.html](https://7-zip.org/download.html) and make sure 7zzs or 7zz or 7z is in the working directory or the directory indicated by PATH. 7-Zip obtained from some other sources may not support rar files.

docs/zh/docs/supported-formats.md

@@ -3,6 +3,9 @@
 | 文件扩展名 | 下载方式 | 对应扫描脚本 |
 | --------- | ------- | ----------- |
 | zip | 一般 | 10-zip.py |
+| rar, 7z [^1] | 一般 | 11-archive.py |
 | zip | [ccloli/E-Hentai-Downloader](https://github.com/ccloli/E-Hentai-Downloader) | 20-ccloli.py |
 | 文件夹 | [Hentai@Home](https://ehwiki.org/wiki/Hentai@Home#H.40H_Downloader) | 21-hath.py |
-| 文件夹 | [EhViewer](https://github.com/seven332/EhViewer) | 22-ehviewer.py |
+| 文件夹 | [EhViewer](https://github.com/seven332/EhViewer) | 22-ehviewer.py |
+
+[^1]: 需要安装 7-Zip，请从 [https://7-zip.org/download.html](https://7-zip.org/download.html) 下载并确保 7zzs 或 7zz 或 7z 处在工作目录或 PATH 指示的目录内。从其他一些渠道获取的 7-Zip 可能不支持 rar 文件。