|
| 1 | +import fs from 'fs' |
| 2 | +import path from 'path' |
| 3 | +import { isMainThread, threadId } from 'worker_threads' |
| 4 | +import { mkdirp } from 'mkdirp' |
| 5 | +import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom' |
| 6 | + |
| 7 | +import Container from '../container.js' |
| 8 | +import event from '../event.js' |
| 9 | +import recorder from '../recorder.js' |
| 10 | +import store from '../store.js' |
| 11 | +import output from '../output.js' |
| 12 | +import { parsePluginArgs } from '../utils/pluginParser.js' |
| 13 | + |
| 14 | +const defaultConfig = { |
| 15 | + file: 'sitemap.xml', |
| 16 | + stripQuery: true, |
| 17 | +} |
| 18 | + |
| 19 | +const SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9' |
| 20 | +const PARTIAL_PREFIX = '.sitemap.' |
| 21 | + |
| 22 | +/** |
| 23 | + * Collects all pages visited while tests are running and saves them as a [sitemap](https://www.sitemaps.org/protocol.html) XML file. |
| 24 | + * |
| 25 | + * The plugin listens to browser navigation events, so it doesn't slow down test steps. |
| 26 | + * Page loads and SPA route changes (`history.pushState`) of the top-level page are recorded, iframes are ignored. |
| 27 | + * Every unique `http(s)` URL is added to the sitemap. URL fragments (`#...`) are dropped. |
| 28 | + * |
| 29 | + * Supported helpers: Playwright, Puppeteer, and WebDriver with BiDi protocol enabled (default). |
| 30 | + * The sitemap is written once all tests finish. |
| 31 | + * |
| 32 | + * #### Configuration |
| 33 | + * |
| 34 | + * ```js |
| 35 | + * "plugins": { |
| 36 | + * "sitemap": { |
| 37 | + * "enabled": true |
| 38 | + * } |
| 39 | + * } |
| 40 | + * ``` |
| 41 | + * |
| 42 | + * Possible config options: |
| 43 | + * |
| 44 | + * * `file`: path to the sitemap file, relative to the output directory. Default: `sitemap.xml`, saved as `output/sitemap.xml`. |
| 45 | + * * `stripQuery`: remove query strings so `/posts?page=1` and `/posts?page=2` become one `/posts` entry. Set to `false` to keep them. Default: true. |
| 46 | + * |
| 47 | + * CLI examples: |
| 48 | + * |
| 49 | + * ``` |
| 50 | + * npx codeceptjs run -p sitemap |
| 51 | + * npx codeceptjs run -p sitemap:file=visited.xml |
| 52 | + * npx codeceptjs run -p sitemap:stripQuery=false |
| 53 | + * ``` |
| 54 | + * |
| 55 | + * Works with `run-workers`: URLs from all workers are merged into a single sitemap. |
| 56 | + * |
| 57 | + * @param {*} config |
| 58 | + */ |
| 59 | +export default function (config = {}) { |
| 60 | + config = Object.assign({}, defaultConfig, config, parsePluginArgs(config._args)) |
| 61 | + |
| 62 | + const urls = new Set() |
| 63 | + |
| 64 | + const outputDir = () => store.outputDir || process.cwd() |
| 65 | + |
| 66 | + const addUrl = url => { |
| 67 | + if (!url) return |
| 68 | + let parsed |
| 69 | + try { |
| 70 | + parsed = new URL(url) |
| 71 | + } catch { |
| 72 | + return |
| 73 | + } |
| 74 | + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return |
| 75 | + parsed.hash = '' |
| 76 | + if (config.stripQuery) parsed.search = '' |
| 77 | + urls.add(parsed.toString()) |
| 78 | + } |
| 79 | + |
| 80 | + const writeSitemap = list => { |
| 81 | + if (!list.length) return |
| 82 | + |
| 83 | + const file = path.resolve(outputDir(), config.file) |
| 84 | + mkdirp.sync(path.dirname(file)) |
| 85 | + fs.writeFileSync(file, buildXml(list.sort())) |
| 86 | + output.plugin('sitemap', `Sitemap with ${list.length} pages saved to ${file}`) |
| 87 | + } |
| 88 | + |
| 89 | + const attached = new WeakSet() |
| 90 | + |
| 91 | + const onLoad = page => addUrl(page.url()) |
| 92 | + |
| 93 | + const onFrameNavigated = frame => { |
| 94 | + if (frame.parentFrame()) return |
| 95 | + addUrl(frame.url()) |
| 96 | + } |
| 97 | + |
| 98 | + const attachPage = page => { |
| 99 | + if (!page || attached.has(page)) return |
| 100 | + attached.add(page) |
| 101 | + addUrl(page.url()) |
| 102 | + page.on('load', onLoad) |
| 103 | + page.on('framenavigated', onFrameNavigated) |
| 104 | + } |
| 105 | + |
| 106 | + const attachContext = context => { |
| 107 | + attached.add(context) |
| 108 | + context.on('page', attachPage) |
| 109 | + context.pages().forEach(attachPage) |
| 110 | + } |
| 111 | + |
| 112 | + const attachPlaywright = browser => { |
| 113 | + if (!browser.contexts) return attachContext(browser) |
| 114 | + attached.add(browser) |
| 115 | + browser.on('context', attachContext) |
| 116 | + browser.contexts().forEach(attachContext) |
| 117 | + } |
| 118 | + |
| 119 | + const attachPuppeteer = async browser => { |
| 120 | + attached.add(browser) |
| 121 | + browser.on('targetcreated', async target => { |
| 122 | + if (target.type() !== 'page') return |
| 123 | + attachPage(await target.page()) |
| 124 | + }) |
| 125 | + for (const page of await browser.pages()) attachPage(page) |
| 126 | + } |
| 127 | + |
| 128 | + const subscribeBidi = async browser => { |
| 129 | + attached.add(browser) |
| 130 | + const frames = new Set() |
| 131 | + const onNavigation = params => { |
| 132 | + if (frames.has(params.context)) return |
| 133 | + addUrl(params.url) |
| 134 | + } |
| 135 | + browser.on('browsingContext.contextCreated', params => { |
| 136 | + if (params.parent) frames.add(params.context) |
| 137 | + }) |
| 138 | + browser.on('browsingContext.load', onNavigation) |
| 139 | + browser.on('browsingContext.historyUpdated', onNavigation) |
| 140 | + try { |
| 141 | + await browser.sessionSubscribe({ events: ['browsingContext.contextCreated', 'browsingContext.load'] }) |
| 142 | + } catch (err) { |
| 143 | + output.plugin('sitemap', `Can't subscribe to navigation events: ${err.message}`) |
| 144 | + } |
| 145 | + try { |
| 146 | + await browser.sessionSubscribe({ events: ['browsingContext.historyUpdated'] }) |
| 147 | + } catch {} |
| 148 | + } |
| 149 | + |
| 150 | + event.dispatcher.on(event.test.started, () => { |
| 151 | + const helpers = Container.helpers() |
| 152 | + |
| 153 | + if (helpers.Playwright) { |
| 154 | + const browser = helpers.Playwright.browser |
| 155 | + if (browser && !attached.has(browser)) attachPlaywright(browser) |
| 156 | + return |
| 157 | + } |
| 158 | + |
| 159 | + if (helpers.Puppeteer) { |
| 160 | + const browser = helpers.Puppeteer.browser |
| 161 | + if (!browser || attached.has(browser)) return |
| 162 | + recorder.add('sitemap: listen to navigation', () => attachPuppeteer(browser), true) |
| 163 | + return |
| 164 | + } |
| 165 | + |
| 166 | + if (helpers.WebDriver) { |
| 167 | + const browser = helpers.WebDriver.browser |
| 168 | + if (!browser || attached.has(browser)) return |
| 169 | + if (!browser.capabilities || !browser.capabilities.webSocketUrl) { |
| 170 | + attached.add(browser) |
| 171 | + output.plugin('sitemap', 'WebDriver BiDi protocol is disabled, visited pages are not collected') |
| 172 | + return |
| 173 | + } |
| 174 | + recorder.add('sitemap: subscribe to navigation', () => subscribeBidi(browser), true) |
| 175 | + } |
| 176 | + }) |
| 177 | + |
| 178 | + const partialFiles = dir => { |
| 179 | + if (!fs.existsSync(dir)) return [] |
| 180 | + return fs |
| 181 | + .readdirSync(dir) |
| 182 | + .filter(name => name.startsWith(PARTIAL_PREFIX) && name.endsWith('.json')) |
| 183 | + .map(name => path.join(dir, name)) |
| 184 | + } |
| 185 | + |
| 186 | + event.dispatcher.on(event.all.result, () => { |
| 187 | + if (isMainThread) { |
| 188 | + if (store.workerMode) return |
| 189 | + writeSitemap([...urls]) |
| 190 | + return |
| 191 | + } |
| 192 | + if (!urls.size) return |
| 193 | + const dir = outputDir() |
| 194 | + mkdirp.sync(dir) |
| 195 | + fs.writeFileSync(path.join(dir, `${PARTIAL_PREFIX}${process.pid}.${threadId}.json`), JSON.stringify([...urls])) |
| 196 | + }) |
| 197 | + |
| 198 | + event.dispatcher.on(event.workers.before, () => { |
| 199 | + for (const partial of partialFiles(outputDir())) fs.unlinkSync(partial) |
| 200 | + }) |
| 201 | + |
| 202 | + event.dispatcher.on(event.workers.result, () => { |
| 203 | + for (const partial of partialFiles(outputDir())) { |
| 204 | + try { |
| 205 | + for (const url of JSON.parse(fs.readFileSync(partial, 'utf8'))) urls.add(url) |
| 206 | + } catch {} |
| 207 | + fs.unlinkSync(partial) |
| 208 | + } |
| 209 | + writeSitemap([...urls]) |
| 210 | + }) |
| 211 | +} |
| 212 | + |
| 213 | +function buildXml(list) { |
| 214 | + const doc = new DOMImplementation().createDocument(SITEMAP_NS, 'urlset', null) |
| 215 | + const urlset = doc.documentElement |
| 216 | + for (const url of list) { |
| 217 | + const urlEl = doc.createElementNS(SITEMAP_NS, 'url') |
| 218 | + const locEl = doc.createElementNS(SITEMAP_NS, 'loc') |
| 219 | + locEl.appendChild(doc.createTextNode(url)) |
| 220 | + urlEl.appendChild(doc.createTextNode('\n ')) |
| 221 | + urlEl.appendChild(locEl) |
| 222 | + urlEl.appendChild(doc.createTextNode('\n ')) |
| 223 | + urlset.appendChild(doc.createTextNode('\n ')) |
| 224 | + urlset.appendChild(urlEl) |
| 225 | + } |
| 226 | + urlset.appendChild(doc.createTextNode('\n')) |
| 227 | + return `<?xml version="1.0" encoding="UTF-8"?>\n${new XMLSerializer().serializeToString(doc)}\n` |
| 228 | +} |
0 commit comments