diff --git a/apps/parse/mangalib/list_parser/parser.py b/apps/parse/mangalib/list_parser/parser.py index 921156d1..da7cf628 100644 --- a/apps/parse/mangalib/list_parser/parser.py +++ b/apps/parse/mangalib/list_parser/parser.py @@ -34,7 +34,7 @@ async def get_list(self): self.logger.info("Starting list parser") self.logger.info("=====================") self.browser = await launch( - {"headless": True, "args": ["--no-sandbox", "--disable-setuid-sandbox"]} + {"headless": False, "args": ["--no-sandbox", "--disable-setuid-sandbox"]} ) workers = asyncio.gather( *[ @@ -78,25 +78,28 @@ async def parse_page(self, page: page.Page, page_num: int): self.logger.error(f"Page {full_url} returned 500 status") self.continue_parse = False - self.get_mangas_info(html_body) + self.get_mangas_info(html_body, page_num) def page_has_500_code(self, page_html, full_url): return bool(page_html.xpath(STATUS_CODE_TAG).extract_first("")) - def get_mangas_info(self, page_html): + def get_mangas_info(self, page_html, page_num): cards = page_html.xpath(CARDS_TAG).extract() - for html_card in cards: + offset = len(cards) * (page_num - 1) + for index, html_card in enumerate(cards, start=1): manga_card = HtmlResponse(url="", body=html_card, encoding="utf-8") title = manga_card.xpath(TITLE_TAG).extract_first("") source_url = manga_card.xpath(SOURCE_TAG).extract_first("") image = manga_card.xpath(IMAGE_TAG).extract_first("") + popularity = index + offset self.mangas.append( { "title": title, "image": MANGALIB_SOURCE + image, "thumbnail": MANGALIB_SOURCE + image, "source_url": source_url, + "popularity": popularity, } ) self.logger.info(f"Parsed manga `{title}`") diff --git a/apps/parse/migrations/0022_auto_20210824_1059.py b/apps/parse/migrations/0022_auto_20210824_1059.py new file mode 100644 index 00000000..3213b91a --- /dev/null +++ b/apps/parse/migrations/0022_auto_20210824_1059.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.3 on 2021-08-24 10:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parse', '0021_change_rating_scale'), + ] + + operations = [ + migrations.AlterModelOptions( + name='manga', + options={'ordering': ['popularity']}, + ), + migrations.AddField( + model_name='manga', + name='popularity', + field=models.IntegerField(default=0), + ), + ] diff --git a/apps/parse/models.py b/apps/parse/models.py index dc3a3ef6..e63bd2e7 100644 --- a/apps/parse/models.py +++ b/apps/parse/models.py @@ -83,10 +83,17 @@ class Manga(BaseModel): categories = ManyToManyField("Category", related_name="mangas", blank=True) updated_detail = models.DateTimeField(blank=True, null=True) updated_chapters = models.DateTimeField(blank=True, null=True) + # Popularity = index of the manga in the catalogue + popularity = models.IntegerField(default=0) people_related = ManyToManyField( "Person", through="PersonRelatedToManga", related_name="mangas" ) + class Meta: + ordering = [ + "popularity", + ] + @property def url_prefix(self) -> str: return re.match(r"(^http[s]?://(.*))/.*$", self.source_url).group(1) diff --git a/apps/parse/readmanga/list_parser/manga_spider.py b/apps/parse/readmanga/list_parser/manga_spider.py index a3d0016b..60e2a5cf 100644 --- a/apps/parse/readmanga/list_parser/manga_spider.py +++ b/apps/parse/readmanga/list_parser/manga_spider.py @@ -52,10 +52,11 @@ def start_requests(self): base_url = f"{READMANGA_URL}/list?&offset=" offsets = [offset for offset in range(0, maximum_offset, standart_offset)] - urls = [base_url + str(offset) for offset in offsets] - for url in urls: - yield scrapy.Request(url=url, callback=self.parse) + for offset in offsets: + yield scrapy.Request( + url=base_url + str(offset), callback=self.parse, cb_kwargs={"offset": offset} + ) def request_fallback(self, failure: Failure): self.logger.error( @@ -63,10 +64,10 @@ def request_fallback(self, failure: Failure): f"failed with status {failure.value.response.status}" ) - def parse(self, response): + def parse(self, response, offset): mangas = [] descriptions = response.xpath(MANGA_TILE_TAG).extract() - for description in descriptions: + for index, description in enumerate(descriptions, start=1): response = HtmlResponse(url="", body=description, encoding="utf-8") rating = parse_rating(response.xpath(STAR_RATE_TAG).extract_first("")) @@ -76,7 +77,7 @@ def parse(self, response): thumbnail = response.xpath(THUMBNAIL_IMG_URL_TAG).extract_first("") image = thumbnail.replace("_p", "") alt_title = response.xpath(ALT_TITLE_URL).extract_first("") - + popularity = index + offset mangas.append( { "rating": rating, @@ -86,6 +87,7 @@ def parse(self, response): "image": image, "genres": genres, "source_url": READMANGA_URL + source_url, + "popularity": popularity, } ) self.logger.info('Parsed manga "{}"'.format(title)) diff --git a/apps/parse/serializers.py b/apps/parse/serializers.py index 4ddbdb10..3814e8fb 100644 --- a/apps/parse/serializers.py +++ b/apps/parse/serializers.py @@ -34,6 +34,7 @@ class Meta: "year", "updated_chapters", "updated_detail", + "popularity", )