diff --git a/plugins/english/hennoveltranslations.ts b/plugins/english/hennoveltranslations.ts new file mode 100644 index 000000000..10dcb8925 --- /dev/null +++ b/plugins/english/hennoveltranslations.ts @@ -0,0 +1,226 @@ +import { Plugin } from '@/types/plugin'; +import { fetchApi } from '@libs/fetch'; +import { defaultCover } from '@libs/defaultCover'; +import { load } from 'cheerio'; + +function sanitizeChapter(html: string, base: string): string { + const $ = load(html, null, false); + $( + 'script,style,iframe,object,embed,svg,math,form,template,noscript', + ).remove(); + const tags = + 'p,div,span,h1,h2,h3,h4,h5,h6,br,hr,strong,em,b,i,u,s,del,small,sub,sup,blockquote,pre,code,ul,ol,li,table,thead,tbody,tfoot,tr,th,td,a,img,ruby,rt,rp'.split( + ',', + ); + $('*').each((_, element) => { + const node = $(element); + const tag = node.prop('tagName')?.toLowerCase() || ''; + if (!tags.includes(tag)) { + node.replaceWith(node.contents()); + return; + } + const allowed = ['title']; + if (tag === 'a') allowed.push('href'); + if (tag === 'img') allowed.push('src', 'alt'); + for (const attribute of Object.keys(node.attr() || {})) { + if (!allowed.includes(attribute)) node.removeAttr(attribute); + } + const attribute = tag === 'a' ? 'href' : tag === 'img' ? 'src' : undefined; + if (attribute && node.attr(attribute)) { + try { + const url = new URL(node.attr(attribute)!, base); + if (url.protocol !== 'https:' && url.protocol !== 'http:') + node.removeAttr(attribute); + else node.attr(attribute, url.href); + } catch { + node.removeAttr(attribute); + } + } + }); + return $.root().html() || ''; +} + +class HenNovelTranslations implements Plugin.PluginBase { + id = 'hennoveltranslations'; + name = 'Hen Novel Translations'; + site = 'https://hennoveltranslations.org'; + icon = 'src/en/hennoveltranslations/icon.png'; + version = '1.0.1'; + + private async html(path: string) { + const response = await fetchApi(this.resolveUrl(path)); + if (!response.ok) + throw Object.assign( + new Error(`Hen Novel Translations: HTTP ${response.status}.`), + { + status: response.status, + }, + ); + return response.text(); + } + + private async document(path: string) { + return load(await this.html(path)); + } + + async popularNovels(page: number): Promise { + if (page !== 1) return []; + const $ = await this.document('/'); + const novels: Plugin.NovelItem[] = []; + $('.book-item-feature').each((_, element) => { + const card = $(element); + const title = card.find('.book-title a'); + const href = title.attr('href'); + if (!href || !title.text().trim()) return; + const image = card.find('img').first(); + novels.push({ + name: title.text().trim(), + path: new URL(this.resolveUrl(href)).pathname, + cover: image.attr('data-src') || image.attr('src') || defaultCover, + }); + }); + if (!novels.length) + throw new Error( + 'Hen Novel Translations: the project list could not be read.', + ); + return novels; + } + + async parseNovel(path: string): Promise { + const html = await this.html(path); + const $ = load(html); + const name = $('.single-novel-title h1').text().trim(); + if (!name) + throw new Error('Hen Novel Translations: the novel could not be read.'); + const details = $('.custom-fields').first(); + const field = (label: string) => { + const paragraph = details + .find('p') + .filter( + (_, element) => + $(element).find('strong').first().text().trim() === label, + ) + .first() + .clone(); + paragraph.find('strong').remove(); + return paragraph.text().trim() || undefined; + }; + const summary = details.clone(); + summary.find('h2,.alternate-chapters,p').remove(); + const chapters: Plugin.ChapterItem[] = []; + const seen: Record = {}; + // The site's episode-list2 is its FREE CHAPTERS list; list1 is paid advance access. + const list = $('.episode-list2'); + const completeLists = html.match( + /]*\bclass\s*=\s*(["'])[^"']*\bepisode-list2\b[^"']*\1[^>]*>[\s\S]*?<\/ul\s*>/gi, + ); + const rows = list.children('li'); + const group = list.closest('.episode-group'); + if ( + list.length !== 1 || + completeLists?.length !== 1 || + !rows.length || + list.find('li').length !== rows.length || + list.find('a').length !== rows.length || + group.find('.pagination,a[rel=next],.load-more,[data-next-page]').length + ) + throw new Error( + 'Hen Novel Translations: the free chapter list is incomplete or has changed.', + ); + const declaredTotal = list.attr('data-total'); + if (declaredTotal !== undefined && Number(declaredTotal) !== rows.length) + throw new Error( + 'Hen Novel Translations: not all free chapters were returned.', + ); + rows.each((_, element) => { + const row = $(element); + const link = row.find('a[href]'); + if (link.length !== 1 || !link.text().trim()) + throw new Error( + 'Hen Novel Translations: a free chapter link is missing.', + ); + const url = new URL(this.resolveUrl(link.attr('href')!)); + const postId = url.searchParams.get('p'); + const chapterPath = url.pathname.startsWith('/episodes/') + ? url.pathname + : url.pathname === '/' && + url.searchParams.get('post_type') === 'episodes' && + postId && + /^\d+$/.test(postId) && + Number.isSafeInteger(Number(postId)) && + Number(postId) > 0 + ? `/?post_type=episodes&p=${Number(postId)}` + : undefined; + if (!chapterPath || seen[chapterPath]) + throw new Error( + 'Hen Novel Translations: an invalid or repeated free chapter was returned.', + ); + seen[chapterPath] = true; + const title = link.text().trim(); + const number = /(?:episode|chapter)\s+(\d+(?:\.\d+)?)/i.exec(title); + chapters.push({ + name: title, + path: chapterPath, + chapterNumber: number ? Number(number[1]) : undefined, + releaseTime: row.find('time').attr('datetime'), + }); + }); + chapters.reverse(); + if (chapters.length !== rows.length) + throw new Error( + 'Hen Novel Translations: the free chapter list is incomplete.', + ); + return { + name, + path, + cover: $('.novel-content img').first().attr('src') || defaultCover, + summary: summary.text().trim(), + author: field('Author:'), + genres: field('Genre:'), + status: field('Light Novel Status(Korean):'), + chapters, + }; + } + + async parseChapter(path: string): Promise { + const $ = await this.document(path); + const body = $('.episode-content').first(); + if ( + body.find('input[type=password]').length || + /^(?:This content is|Please log in|You must be logged)/i.test( + body.text().trim(), + ) + ) + throw new Error( + 'Hen Novel Translations: this chapter is not publicly readable.', + ); + body + .find('script,style,iframe,form,button,.episode-navigation,.adsbygoogle') + .remove(); + body.find('[style]').removeAttr('style'); + const chapter = sanitizeChapter(body.html() || '', this.resolveUrl(path)); + if (load(chapter).text().trim().length < 200) + throw new Error( + 'Hen Novel Translations: no readable public chapter was found.', + ); + return chapter; + } + + async searchNovels(term: string, page: number): Promise { + const query = term.trim().toLowerCase(); + return (await this.popularNovels(page)).filter(novel => + (novel.name + ' ' + novel.path.replace(/-/g, ' ')) + .toLowerCase() + .includes(query), + ); + } + + resolveUrl(path: string): string { + const url = new URL(path, this.site); + if (url.origin !== this.site) + throw new Error('Hen Novel Translations: invalid page address.'); + return url.href; + } +} + +export default new HenNovelTranslations(); diff --git a/plugins/english/maehwasup.ts b/plugins/english/maehwasup.ts new file mode 100644 index 000000000..5a11834e7 --- /dev/null +++ b/plugins/english/maehwasup.ts @@ -0,0 +1,158 @@ +import { Plugin } from '@/types/plugin'; +import { fetchApi } from '@libs/fetch'; +import { defaultCover } from '@libs/defaultCover'; +import { load } from 'cheerio'; + +function sanitizeChapter(html: string, base: string): string { + const $ = load(html, null, false); + $( + 'script,style,iframe,object,embed,svg,math,form,template,noscript', + ).remove(); + const tags = + 'p,div,span,h1,h2,h3,h4,h5,h6,br,hr,strong,em,b,i,u,s,del,small,sub,sup,blockquote,pre,code,ul,ol,li,table,thead,tbody,tfoot,tr,th,td,a,img,ruby,rt,rp'.split( + ',', + ); + $('*').each((_, element) => { + const node = $(element); + const tag = node.prop('tagName')?.toLowerCase() || ''; + if (!tags.includes(tag)) { + node.replaceWith(node.contents()); + return; + } + const allowed = ['title']; + if (tag === 'a') allowed.push('href'); + if (tag === 'img') allowed.push('src', 'alt'); + for (const attribute of Object.keys(node.attr() || {})) { + if (!allowed.includes(attribute)) node.removeAttr(attribute); + } + const attribute = tag === 'a' ? 'href' : tag === 'img' ? 'src' : undefined; + if (attribute && node.attr(attribute)) { + try { + const url = new URL(node.attr(attribute)!, base); + if (url.protocol !== 'https:' && url.protocol !== 'http:') + node.removeAttr(attribute); + else node.attr(attribute, url.href); + } catch { + node.removeAttr(attribute); + } + } + }); + return $.root().html() || ''; +} + +type Post = { URL: string; title: string; date: string }; +type Index = { found: number; posts: Post[] }; + +class Maehwasup implements Plugin.PluginBase { + id = 'maehwasup'; + name = 'Maehwasup'; + site = 'https://maehwasup.com'; + icon = 'src/en/maehwasup/icon.png'; + version = '1.0.1'; + + private async request(url: string) { + const response = await fetchApi(url); + if (!response.ok) + throw Object.assign(new Error(`Maehwasup: HTTP ${response.status}.`), { + status: response.status, + }); + return response; + } + + async popularNovels(page: number): Promise { + if (page !== 1) return []; + const $ = load(await (await this.request(this.site)).text()); + const name = $('.wp-block-site-title').first().text().trim(); + if (!name) throw new Error('Maehwasup: the novel title could not be read.'); + return [{ name, path: '/', cover: defaultCover }]; + } + + private async index(page: number): Promise { + const url = + 'https://public-api.wordpress.com/rest/v1.1/sites/maehwasup.com/posts/' + + `?number=100&page=${page}&order=ASC&order_by=date&fields=URL,title,date`; + const data: Index = await (await this.request(url)).json(); + if ( + !Number.isInteger(data.found) || + data.found < 0 || + !Array.isArray(data.posts) + ) + throw new Error('Maehwasup: the public chapter index could not be read.'); + return data; + } + + async parseNovel(path: string): Promise { + const novel = (await this.popularNovels(1))[0]; + const first = await this.index(1); + const pages = Math.ceil(first.found / 100); + if (pages > 100) + throw new Error('Maehwasup: the index is unexpectedly large.'); + const chapters: Plugin.ChapterItem[] = []; + const seen: Record = {}; + let received = 0; + for (let page = 1; page <= pages; page++) { + const data = page === 1 ? first : await this.index(page); + if (data.found !== first.found || !data.posts.length) + throw new Error( + 'Maehwasup: the index changed. Refresh the novel to retry.', + ); + received += data.posts.length; + for (const post of data.posts) { + const name = load(post.title).text().trim(); + if (!/chapter|side story|spinoff|prologue|epilogue/i.test(name)) + continue; + const url = new URL(post.URL); + if (url.origin !== this.site || seen[url.pathname]) + throw new Error( + 'Maehwasup: an invalid or repeated chapter was returned.', + ); + seen[url.pathname] = true; + const number = /^Chapter\s+(\d+(?:\.\d+)?)/i.exec(name); + chapters.push({ + name, + path: url.pathname, + releaseTime: post.date, + chapterNumber: number ? Number(number[1]) : undefined, + }); + } + } + if (received !== first.found || !chapters.length) + throw new Error( + 'Maehwasup: the chapter list is incomplete. Refresh to retry.', + ); + return { ...novel, path, chapters }; + } + + async parseChapter(path: string): Promise { + const $ = load(await (await this.request(this.resolveUrl(path))).text()); + const body = $('.entry-content.wp-block-post-content').first(); + body.find('script,style,iframe,.wordads-tag,.sharedaddy,form').remove(); + body.find('p').each((_, element) => { + if ( + /^(Please,?\s+subscribe\/donate|ROTMHS Glossary)/i.test( + $(element).text().trim(), + ) + ) + $(element).remove(); + }); + const chapter = sanitizeChapter(body.html() || '', this.resolveUrl(path)); + if (load(chapter).text().trim().length < 200) + throw new Error('Maehwasup: no readable public chapter was found.'); + return chapter; + } + + async searchNovels(term: string, page: number): Promise { + return (await this.popularNovels(page)).filter(novel => + novel.name.toLowerCase().includes(term.trim().toLowerCase()), + ); + } + + resolveUrl(path: string): string { + const url = new URL(path, this.site); + if (url.origin !== this.site) + throw new Error('Maehwasup: invalid page address.'); + return url.href; + } +} + +export default new Maehwasup(); diff --git a/plugins/english/mylasted.ts b/plugins/english/mylasted.ts new file mode 100644 index 000000000..580fc51a4 --- /dev/null +++ b/plugins/english/mylasted.ts @@ -0,0 +1,216 @@ +import { Plugin } from '@/types/plugin'; +import { fetchApi } from '@libs/fetch'; +import { defaultCover } from '@libs/defaultCover'; +import { load } from 'cheerio'; + +function sanitizeChapter(html: string, base: string): string { + const $ = load(html, null, false); + $( + 'script,style,iframe,object,embed,svg,math,form,template,noscript', + ).remove(); + const tags = + 'p,div,span,h1,h2,h3,h4,h5,h6,br,hr,strong,em,b,i,u,s,del,small,sub,sup,blockquote,pre,code,ul,ol,li,table,thead,tbody,tfoot,tr,th,td,a,img,ruby,rt,rp'.split( + ',', + ); + $('*').each((_, element) => { + const node = $(element); + const tag = node.prop('tagName')?.toLowerCase() || ''; + if (!tags.includes(tag)) { + node.replaceWith(node.contents()); + return; + } + const allowed = ['title']; + if (tag === 'a') allowed.push('href'); + if (tag === 'img') allowed.push('src', 'alt'); + for (const attribute of Object.keys(node.attr() || {})) { + if (!allowed.includes(attribute)) node.removeAttr(attribute); + } + const attribute = tag === 'a' ? 'href' : tag === 'img' ? 'src' : undefined; + if (attribute && node.attr(attribute)) { + try { + const url = new URL(node.attr(attribute)!, base); + if (url.protocol !== 'https:' && url.protocol !== 'http:') + node.removeAttr(attribute); + else node.attr(attribute, url.href); + } catch { + node.removeAttr(attribute); + } + } + }); + return $.root().html() || ''; +} + +type Entry = { + title: { $t: string }; + link: { rel: string; href: string }[]; + published: { $t: string }; + content?: { $t: string }; +}; +type Feed = { openSearch$totalResults: { $t: string }; entry?: Entry[] }; + +class MachineEditing implements Plugin.PluginBase { + id = 'mylasted'; + name = 'Machine Editing (MyLasted)'; + site = 'https://mylasted.blogspot.com'; + icon = 'src/en/mylasted/icon.png'; + version = '1.0.1'; + + private async request(url: string) { + const response = await fetchApi(url); + if (!response.ok) + throw Object.assign( + new Error(`Machine Editing: HTTP ${response.status}.`), + { + status: response.status, + }, + ); + return response; + } + + private async feed(label: string, start = 1, summary = false): Promise { + const url = `${this.site}/feeds/posts/${summary ? 'summary' : 'default'}/-/${encodeURIComponent(label)}?alt=json&max-results=150&start-index=${start}`; + const data = await (await this.request(url)).json(); + const feed: Feed = data.feed; + if (!feed || !Number.isInteger(Number(feed.openSearch$totalResults?.$t))) + throw new Error('Machine Editing: the public feed could not be read.'); + return feed; + } + + private project(entry: Entry): Plugin.NovelItem { + const href = entry.link.find(link => link.rel === 'alternate')?.href; + if (!href) throw new Error('Machine Editing: a series address is missing.'); + const $ = load(entry.content?.$t || ''); + return { + name: entry.title.$t, + path: new URL(this.resolveUrl(href)).pathname, + cover: $('img').first().attr('src') || defaultCover, + }; + } + + async popularNovels(page: number): Promise { + if (page !== 1) return []; + const feed = await this.feed('Series'); + const entries = feed.entry || []; + if (entries.length !== Number(feed.openSearch$totalResults.$t)) + throw new Error('Machine Editing: the series list is incomplete.'); + return entries.map(entry => this.project(entry)); + } + + async parseNovel(path: string): Promise { + const $ = load(await (await this.request(this.resolveUrl(path))).text()); + const label = /clwd\.run\(\s*['"]([^'"]+)['"]\s*\)/.exec( + $('#clwd').html() || '', + )?.[1]; + if (!label) + throw new Error( + 'Machine Editing: the novel chapter label could not be read.', + ); + const first = await this.feed(label, 1, true); + const total = Number(first.openSearch$totalResults.$t); + if (total < 1 || total > 15000) + throw new Error('Machine Editing: invalid chapter count.'); + let start = 1; + const chapters: Plugin.ChapterItem[] = []; + const seen: Record = {}; + while (start <= total) { + const feed = start === 1 ? first : await this.feed(label, start, true); + const entries = feed.entry || []; + if (Number(feed.openSearch$totalResults.$t) !== total || !entries.length) + throw new Error( + 'Machine Editing: the chapter feed changed. Refresh to retry.', + ); + for (const entry of entries) { + const href = entry.link.find(link => link.rel === 'alternate')?.href; + if (!href) + throw new Error('Machine Editing: a chapter address is missing.'); + const chapterPath = new URL(this.resolveUrl(href)).pathname; + if (seen[chapterPath]) + throw new Error('Machine Editing: repeated feed page.'); + seen[chapterPath] = true; + const number = /(?:chapter|episode)\s+(\d+(?:\.\d+)?)/i.exec( + entry.title.$t, + ); + if (!number && !/prologue|epilogue|side story/i.test(entry.title.$t)) + continue; + chapters.push({ + name: entry.title.$t, + path: chapterPath, + chapterNumber: number ? Number(number[1]) : undefined, + releaseTime: entry.published.$t, + }); + } + start += entries.length; + } + if (!chapters.length) + throw new Error('Machine Editing: no chapter entries were found.'); + // A single ordering tuple keeps numbered and unnumbered entries transitive. + const section = (chapter: Plugin.ChapterItem) => { + if (chapter.chapterNumber !== undefined) return 1; + if (/prologue/i.test(chapter.name)) return 0; + if (/epilogue/i.test(chapter.name)) return 3; + return 2; + }; + chapters.sort( + (a, b) => + section(a) - section(b) || + (a.chapterNumber ?? 0) - (b.chapterNumber ?? 0) || + (a.releaseTime || '').localeCompare(b.releaseTime || '') || + a.path.localeCompare(b.path), + ); + const field = (label: string) => + $('#extra-info dt') + .filter((_, element) => $(element).text().trim() === label) + .first() + .next('dd') + .text() + .trim() || undefined; + const name = + $('meta[property="og:title"]').attr('content') || + $('h1').first().text().trim(); + return { + name, + path, + cover: $('article img').first().attr('src') || defaultCover, + summary: $('#synopsis').text().trim(), + author: field('Author'), + genres: field('Tags'), + status: field('Chapter'), + chapters, + }; + } + + async parseChapter(path: string): Promise { + const $ = load(await (await this.request(this.resolveUrl(path))).text()); + const body = $('article.txt').first(); + if ($('#clwd', body).length) + throw new Error('Machine Editing: this is a series page, not a chapter.'); + body.find('script,style,iframe,form,.adsbygoogle,.separator').remove(); + body.find('[style]').removeAttr('style'); + body.find('div,p').each((_, element) => { + if ( + /^Consider supporting me by subscribing/i.test($(element).text().trim()) + ) + $(element).remove(); + }); + const chapter = sanitizeChapter(body.html() || '', this.resolveUrl(path)); + if (load(chapter).text().trim().length < 200) + throw new Error('Machine Editing: no readable public chapter was found.'); + return chapter; + } + + async searchNovels(term: string, page: number): Promise { + const query = term.trim().toLowerCase(); + return (await this.popularNovels(page)).filter(novel => + novel.name.toLowerCase().includes(query), + ); + } + + resolveUrl(path: string): string { + const url = new URL(path, this.site); + if (url.origin !== this.site) + throw new Error('Machine Editing: invalid page address.'); + return url.href; + } +} + +export default new MachineEditing(); diff --git a/public/static/src/en/hennoveltranslations/icon.png b/public/static/src/en/hennoveltranslations/icon.png new file mode 100644 index 000000000..28b3bce30 Binary files /dev/null and b/public/static/src/en/hennoveltranslations/icon.png differ diff --git a/public/static/src/en/maehwasup/icon.png b/public/static/src/en/maehwasup/icon.png new file mode 100644 index 000000000..be6ecf483 Binary files /dev/null and b/public/static/src/en/maehwasup/icon.png differ diff --git a/public/static/src/en/mylasted/icon.png b/public/static/src/en/mylasted/icon.png new file mode 100644 index 000000000..d491fe469 Binary files /dev/null and b/public/static/src/en/mylasted/icon.png differ