Created popularity field for Readmanga

apps/parse/mangalib/list_parser/parser.py

@@ -34,7 +34,7 @@ async def get_list(self):
         self.logger.info("Starting list parser")
         self.logger.info("=====================")
         self.browser = await launch(
-            {"headless": True, "args": ["--no-sandbox", "--disable-setuid-sandbox"]}
+            {"headless": False, "args": ["--no-sandbox", "--disable-setuid-sandbox"]}
         )
         workers = asyncio.gather(
             *[
@@ -78,25 +78,28 @@ async def parse_page(self, page: page.Page, page_num: int):
             self.logger.error(f"Page {full_url} returned 500 status")
             self.continue_parse = False
 
-        self.get_mangas_info(html_body)
+        self.get_mangas_info(html_body, page_num)
 
     def page_has_500_code(self, page_html, full_url):
         return bool(page_html.xpath(STATUS_CODE_TAG).extract_first(""))
 
-    def get_mangas_info(self, page_html):
+    def get_mangas_info(self, page_html, page_num):
         cards = page_html.xpath(CARDS_TAG).extract()
-        for html_card in cards:
+        offset = len(cards) * (page_num - 1)
+        for index, html_card in enumerate(cards, start=1):
             manga_card = HtmlResponse(url="", body=html_card, encoding="utf-8")
 
             title = manga_card.xpath(TITLE_TAG).extract_first("")
             source_url = manga_card.xpath(SOURCE_TAG).extract_first("")
             image = manga_card.xpath(IMAGE_TAG).extract_first("")
+            popularity = index + offset
             self.mangas.append(
                 {
                     "title": title,
                     "image": MANGALIB_SOURCE + image,
                     "thumbnail": MANGALIB_SOURCE + image,
                     "source_url": source_url,
+                    "popularity": popularity,
                 }
             )
             self.logger.info(f"Parsed manga `{title}`")

apps/parse/migrations/0022_auto_20210824_1059.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.3 on 2021-08-24 10:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('parse', '0021_change_rating_scale'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='manga',
+            options={'ordering': ['popularity']},
+        ),
+        migrations.AddField(
+            model_name='manga',
+            name='popularity',
+            field=models.IntegerField(default=0),
+        ),
+    ]

apps/parse/models.py

@@ -83,10 +83,17 @@ class Manga(BaseModel):
     categories = ManyToManyField("Category", related_name="mangas", blank=True)
     updated_detail = models.DateTimeField(blank=True, null=True)
     updated_chapters = models.DateTimeField(blank=True, null=True)
+    # Popularity = index of the manga in the catalogue
+    popularity = models.IntegerField(default=0)
     people_related = ManyToManyField(
         "Person", through="PersonRelatedToManga", related_name="mangas"
     )
 
+    class Meta:
+        ordering = [
+            "popularity",
+        ]
+
     @property
     def url_prefix(self) -> str:
         return re.match(r"(^http[s]?://(.*))/.*$", self.source_url).group(1)

apps/parse/readmanga/list_parser/manga_spider.py

@@ -52,21 +52,22 @@ def start_requests(self):
 
         base_url = f"{READMANGA_URL}/list?&offset="
         offsets = [offset for offset in range(0, maximum_offset, standart_offset)]
-        urls = [base_url + str(offset) for offset in offsets]
 
-        for url in urls:
-            yield scrapy.Request(url=url, callback=self.parse)
+        for offset in offsets:
+            yield scrapy.Request(
+                url=base_url + str(offset), callback=self.parse, cb_kwargs={"offset": offset}
+            )
 
     def request_fallback(self, failure: Failure):
         self.logger.error(
             f'Request for url "{failure.value.response.url}" '
             f"failed with status {failure.value.response.status}"
         )
 
-    def parse(self, response):
+    def parse(self, response, offset):
         mangas = []
         descriptions = response.xpath(MANGA_TILE_TAG).extract()
-        for description in descriptions:
+        for index, description in enumerate(descriptions, start=1):
             response = HtmlResponse(url="", body=description, encoding="utf-8")
 
             rating = parse_rating(response.xpath(STAR_RATE_TAG).extract_first(""))
@@ -76,7 +77,7 @@ def parse(self, response):
             thumbnail = response.xpath(THUMBNAIL_IMG_URL_TAG).extract_first("")
             image = thumbnail.replace("_p", "")
             alt_title = response.xpath(ALT_TITLE_URL).extract_first("")
-
+            popularity = index + offset
             mangas.append(
                 {
                     "rating": rating,
@@ -86,6 +87,7 @@ def parse(self, response):
                     "image": image,
                     "genres": genres,
                     "source_url": READMANGA_URL + source_url,
+                    "popularity": popularity,
                 }
             )
             self.logger.info('Parsed manga "{}"'.format(title))

apps/parse/serializers.py

@@ -34,6 +34,7 @@ class Meta:
             "year",
             "updated_chapters",
             "updated_detail",
+            "popularity",
         )