Mangabox: Chapter Retrieval and ID Changes (#156)

* fix(selectors): update selectors to reflect site changes.

Author and Chapter List selectors are now different on all MangaBox sites.

* refactor: switch to manga & chapter id matching and fix chapters not found.

MangaBox sites now pull Chapters via API request and are displayed via
js script. Due to this change, the move from full urls as IDs to now
Manga and Chapter IDs was done in-order to simplify interaction with
the API.

* chore: update base major version and reset minor & patch versions on sources.

Author: Batmeow

src/MangaBat/MangaBat.ts

@@ -12,7 +12,7 @@ import {
 const SITE_DOMAIN = 'https://www.mangabats.com'
 
 export const MangaBatInfo: SourceInfo = {
-    version: getExportVersion('4.0.2'),
+    version: getExportVersion('4.0.0'),
     name: 'MangaBat',
     icon: 'icon.png',
     author: 'Batmeow',

src/MangaBox.ts

@@ -30,7 +30,7 @@ import {
     resetSettings
 } from './MangaBoxSettings'
 
-const BASE_VERSION = '1.0.1'
+const BASE_VERSION = '2.0.0'
 export const getExportVersion = (EXTENSION_VERSION: string): string => {
     return BASE_VERSION.split('.').map((x, index) => Number(x) + Number(EXTENSION_VERSION.split('.')[index])).join('.')
 }
@@ -40,6 +40,14 @@ export interface HomeSectionsParams {
     values: [latest: string, newest: string, popular: string]
 }
 
+export interface APIChapter {
+    chapter_name: string
+    chapter_slug: string
+    chapter_num: number
+    updated_at: string
+    view: number
+}
+
 export abstract class MangaBox implements SearchResultsProviding, MangaProviding, ChapterProviding, HomePageSectionsProviding {
     // Website base URL. Eg. https://manganato.com
     abstract baseURL: string
@@ -93,7 +101,7 @@ export abstract class MangaBox implements SearchResultsProviding, MangaProviding
 
     // Selector for manga author.
     mangaAuthorSelector = 'div.story-info-right td:contains(Author) + td a,'
-        + 'ul.manga-info-text li:contains(Author) a'
+        + 'ul.manga-info-text li:contains(Author)'
 
     // Selector for manga description.
     mangaDescSelector = 'div.leftCol div#contentBox, div.chapter + div#contentBox, div#panel-story-info-description, div.manga-info-top + div#contentBox'
@@ -103,8 +111,8 @@ export abstract class MangaBox implements SearchResultsProviding, MangaProviding
         + 'ul.manga-info-text li:contains(Genres) a'
 
     // Selector for manga chapter list.
-    chapterListSelector = 'div.panel-story-chapter-list ul.row-content-chapter li,'
-        + 'div.manga-info-chapter div.chapter-list div.row'
+    chapterListSelector = 'div#chapter div.manga-info-chapter div#chapter-list-container div.chapter-list div.row,'
+        + 'div.panel-story-chapter-list ul.row-content-chapter li'
 
     // Selector for manga chapter time updated.
     chapterTimeSelector = 'span.chapter-time, span:last-of-type'
@@ -150,7 +158,7 @@ export abstract class MangaBox implements SearchResultsProviding, MangaProviding
         }))
     }
 
-    getMangaShareUrl(mangaId: string): string { return `${mangaId}` }
+    getMangaShareUrl(mangaId: string): string { return `${this.baseURL}/manga/${mangaId}/` }
 
     async getHomePageSections(sectionCallback: (section: HomeSection) => void): Promise<void> {
         const sections = [
@@ -223,7 +231,10 @@ export abstract class MangaBox implements SearchResultsProviding, MangaProviding
 
     async getMangaDetails(mangaId: string): Promise<SourceManga> {
         const request = App.createRequest({
-            url: `${mangaId}`,
+            url: new URLBuilder(this.baseURL)
+                .addPathComponent('manga')
+                .addPathComponent(mangaId)
+                .buildUrl(),
             method: 'GET'
         })
 
@@ -234,17 +245,44 @@ export abstract class MangaBox implements SearchResultsProviding, MangaProviding
         return this.parser.parseMangaDetails($, mangaId, this)
     }
 
-    async getChapters(mangaId: string): Promise<Chapter[]> {
+    async getChaptersAPI(mangaId: string, limit = 50, offset: number): Promise<any> {
         const request = App.createRequest({
-            url: `${mangaId}`,
+            url: new URLBuilder(this.baseURL)
+                .addPathComponent('api')
+                .addPathComponent('manga')
+                .addPathComponent(mangaId)
+                .addPathComponent('chapters')
+                .addQueryParameter('limit', limit.toString())
+                .addQueryParameter('offset', offset.toString())
+                .buildUrl(),
             method: 'GET'
         })
 
         const response = await this.requestManager.schedule(request, 1)
         this.checkResponseError(response)
 
-        const $ = this.cheerio.load(response.data as string)
-        return this.parser.parseChapters($, mangaId, this)
+        if (!response.data) throw new Error('No data received from Chapter API')
+        return JSON.parse(response.data)
+    }
+
+    async getChapters(mangaId: string): Promise<Chapter[]> {
+        const apiChapters: APIChapter[] = []
+        let offset = 0
+        let hasMore = true
+
+        while (hasMore) {
+            const chapters_api_data = await this.getChaptersAPI(mangaId, 50, offset)
+            if (!chapters_api_data.success) throw new Error('API did not return success for chapters request')
+            apiChapters.push(...chapters_api_data.data.chapters)
+
+            if (!chapters_api_data.data.pagination.has_more) {
+                hasMore = false
+                break
+            }
+            offset += 50
+        }
+
+        return this.parser.parseChapters(apiChapters, mangaId, this)
     }
 
     async getChapterDetails(mangaId: string, chapterId: string): Promise<ChapterDetails> {
@@ -253,7 +291,11 @@ export abstract class MangaBox implements SearchResultsProviding, MangaProviding
         const imageServer = await getImageServer(this.stateManager).then(value => value[0])
 
         const request = App.createRequest({
-            url: `${chapterId}`,
+            url: new URLBuilder(this.baseURL)
+                .addPathComponent('manga')
+                .addPathComponent(mangaId)
+                .addPathComponent(chapterId)
+                .buildUrl(),
             method: 'GET',
             cookies: [
                 App.createCookie({

src/MangaBoxParser.ts

@@ -9,7 +9,10 @@ import {
 
 import { decodeHTML } from 'entities'
 
-import { MangaBox } from './MangaBox'
+import {
+    APIChapter,
+    MangaBox
+} from './MangaBox'
 
 import { getImageServer } from './MangaBoxSettings'
 
@@ -19,7 +22,7 @@ export class MangaBoxParser {
         const collecedIds: string[] = []
 
         for (const manga of $(source.mangaListSelector).toArray()) {
-            const mangaId = $('a', manga).first().attr('href')
+            const mangaId = this.idCleaner($('a', manga).attr('href') ?? '')
             const image = $('img', manga).first().attr('src')?.trim() ?? ''
             const title = decodeHTML($('a', manga).first().attr('title')?.trim() ?? '')
             const subtitle = $(source.mangaSubtitleSelector, manga).first().text().trim() ?? ''
@@ -67,11 +70,7 @@ export class MangaBoxParser {
                 break
         }
 
-        const author = $(source.mangaAuthorSelector, mangaRootSelector)
-            .toArray()
-            .map(x => $(x).text().trim())
-            .join(', ') ?? ''
-
+        const author = $(source.mangaAuthorSelector, mangaRootSelector).first().text().replace('Author(s) :', '').trim()
         const desc = decodeHTML($(source.mangaDescSelector).first().children().remove().end().text().trim())
 
         const tags: Tag[] = []
@@ -103,23 +102,17 @@ export class MangaBoxParser {
         })
     }
 
-    parseChapters = ($: CheerioStatic, mangaId: string, source: MangaBox): Chapter[] => {
+    parseChapters = (apiChapters: APIChapter[], mangaId: string, source: MangaBox): Chapter[] => {
         const chapters: Chapter[] = []
         let sortingIndex = 0
 
-        for (const chapter of $(source.chapterListSelector).toArray()) {
-            const id = $('a', chapter).attr('href') ?? ''
+        for (const chapter of apiChapters) {
+            const id = chapter.chapter_slug ?? ''
             if (!id) continue
 
-            const name = decodeHTML($('a', chapter).text().trim())
-            const timeText = $(source.chapterTimeSelector, chapter).text().trim()
-            const time = this.parseDate(
-                (timeText.includes('a') ? timeText : $(source.chapterTimeSelector, chapter).attr('title')) ?? ''
-            )
-
-            let chapNum = 0
-            const chapRegex = id.match(/(?:chap.*)[-_](\d+\.?\d?)/)
-            if (chapRegex && chapRegex[1]) chapNum = Number(chapRegex[1].replace(/\\/g, '.'))
+            const name = decodeHTML(chapter.chapter_name?.trim() ?? '')
+            const time = new Date(chapter.updated_at?.trim() ?? '')
+            const chapNum = chapter.chapter_num ?? 0
 
             chapters.push({
                 id: id,
@@ -162,7 +155,6 @@ export class MangaBoxParser {
                     for (const url of cdns) image = image.replace(url, cdns[imageServer])
                 }
             }
-
             pages.push(image)
         }
 
@@ -200,26 +192,6 @@ export class MangaBoxParser {
         return TagSection
     }
 
-    parseDate = (date: string): Date => {
-        let time: Date
-        let number = Number((/\d*/.exec(date) ?? [])[0])
-        number = (number == 0 && date.includes('a')) ? 1 : number
-        date = date.toUpperCase()
-        if (date.includes('MINUTE') || date.includes('MINUTES') || date.includes('MINS')) {
-            time = new Date(Date.now() - (number * 60000))
-        } else if (date.includes('HOUR') || date.includes('HOURS')) {
-            time = new Date(Date.now() - (number * 3600000))
-        } else if (date.includes('DAY') || date.includes('DAYS')) {
-            time = new Date(Date.now() - (number * 86400000))
-        } else if (date.includes('YEAR') || date.includes('YEARS')) {
-            time = new Date(Date.now() - (number * 31556952000))
-        } else {
-            time = new Date(`${date} UTC`)
-        }
-
-        return time
-    }
-
     isLastPage = ($: CheerioStatic): boolean => {
         const currentPage = $('.page-select, .page_select').text()
         let totalPages = $('.page-last, .page_last').text()
@@ -231,4 +203,13 @@ export class MangaBoxParser {
 
         return true
     }
+
+    idCleaner(str: string): string {
+        let cleanId: string | null = str
+        cleanId = cleanId.replace(/\/$/, '')
+        cleanId = cleanId.split('/').pop() ?? null
+
+        if (!cleanId) throw new Error(`Unable to parse id for ${str}`) // Log to logger
+        return cleanId
+    }
 }

src/MangakakalotGG/MangakakalotGG.ts

@@ -12,7 +12,7 @@ import {
 const SITE_DOMAIN = 'https://www.mangakakalot.gg'
 
 export const MangakakalotGGInfo: SourceInfo = {
-    version: getExportVersion('0.1.4'),
+    version: getExportVersion('0.0.0'),
     name: 'MangakakalotGG',
     icon: 'icon.png',
     author: 'Batmeow',

src/Manganato/Manganato.ts

@@ -12,7 +12,7 @@ import {
 const SITE_DOMAIN = 'https://www.manganato.gg'
 
 export const ManganatoInfo: SourceInfo = {
-    version: getExportVersion('4.0.2'),
+    version: getExportVersion('4.0.0'),
     name: 'Manganato',
     icon: 'icon.png',
     author: 'Batmeow',

src/Natomanga/Natomanga.ts

@@ -12,7 +12,7 @@ import {
 const SITE_DOMAIN = 'https://www.natomanga.com'
 
 export const NatomangaInfo: SourceInfo = {
-    version: getExportVersion('0.1.4'),
+    version: getExportVersion('0.0.0'),
     name: 'Natomanga',
     icon: 'icon.png',
     author: 'Batmeow',