diff --git a/README.md b/README.md index e2780ee..b7a9128 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ A failed lookup is reported on its own, does not hold back the other results and ## Config File -Configure via `updates.config.{ts,js,mjs,mts}` in your repo root. CLI arguments win over it, except `include`, `exclude` and `pin`, which merge. +Configure via `updates.config.{ts,js,mjs,mts}`. Each manifest uses the nearest config above it, and workspace members use the workspace root config. CLI arguments win over configured values, including `include`, `exclude` and `pin`. ```ts import type {Config} from "updates"; @@ -103,7 +103,7 @@ export default { ### Renovate config -A [Renovate](https://docs.renovatebot.com/) config is picked up automatically, inheriting `ignoreDeps`, `enabled` and `allowedVersions` in `packageRules` as `include`/`exclude`/`pin`. `minimumReleaseAge` is not inherited unless opted in: +A [Renovate](https://docs.renovatebot.com/) `renovate.json` is picked up automatically, inheriting `ignoreDeps`, `enabled` and `allowedVersions` from matching `packageRules` as `include`/`exclude`/`pin`. Exact-name `allowedVersions` ranges become pin ceilings that never downgrade. Configs using `extends` are rejected. `minimumReleaseAge` is not inherited unless opted in: ```ts export default { diff --git a/api.ts b/api.ts index 501d178..f7ffb9f 100755 --- a/api.ts +++ b/api.ts @@ -4,28 +4,29 @@ import {join, dirname, basename, resolve} from "node:path"; import {statSync, readdirSync, realpathSync, truncateSync, writeFileSync, accessSync, type Stats} from "node:fs"; import {readFile} from "node:fs/promises"; import {parseToml} from "./utils/toml.ts"; -import {githubActionsVersioning, validRange} from "./utils/semver.ts"; +import {coerce, githubActionsVersioning, satisfies, validRange} from "./utils/semver.ts"; import {timerel} from "timerel"; -import {npmTypes, uvTypes, goTypes, cargoTypes, cargoTargetTypes, expandDepTypes, parseUvDependencies, nonPackageEngines, parseDuration, parsePositiveInt, matchesAny, memoizeAsync, timestamp, forgeDirs, modeByFileName, pMap, pushTo, tryOrNull} from "./utils/utils.ts"; +import {npmTypes, uvTypes, goTypes, cargoTypes, cargoTargetTypes, expandDepTypes, parseUvDependencies, nonPackageEngines, parseDuration, parsePositiveInt, matchesAny, memoizeAsync, timestamp, forgeDirs, modeByFileName, pMap, tryOrNull} from "./utils/utils.ts"; import { type Dep, type Deps, type DepsByMode, type Limiter, type Output as ModeOutput, type ModeContext, - type PackageRepository, type PackageInfo, type TagEntry, + type PackageRepository, type TagEntry, fieldSep, normalizeUrl, fetchTimeout, goProbeTimeout, maxSockets, - doFetch, fetchActionTags, findVersion, findNewVersion, getInfoUrl, getGithubTokens, getLimiter, - passesCooldown, stripv, hashRe, isVersionLikeRef, defaultApiUrls, getExecFile, + doFetch, fetchActionTags, fetchForge, findVersion, findNewVersion, getInfoUrl, getGithubTokens, getLimiter, + passesCooldown, stripv, hashRe, isVersionLikeRef, defaultApiUrls, formatVersionPrecision, getExecFile, } from "./modes/shared.ts"; import {flushCacheWrites} from "./utils/fetchCache.ts"; -import {loadConfig, configMixedToRegexes, patternsToRegexSet, validatePin} from "./config.ts"; +import {cliBaseConfig, loadConfig, configMixedToRegexes, patternsToRegexSet, validatePin} from "./config.ts"; import type {Config, Override} from "./config.ts"; +import {matchesRenovateRule, testRenovateMatcher, type RenovateVersionRule} from "./utils/renovate.ts"; import { fetchNpmInfo, fetchNpmVersionInfo, fetchJsrInfo, isJsr, isLocalDep, isCatalogRef, parseJsrDependency, parseNpmAlias, - getNpmrc, updatePackageJson, updateVersionRange, normalizeRange, checkUrlDep, resolutionsBasePackage, selectorTypes, + updatePackageJson, updateVersionRange, normalizeRange, checkUrlDep, resolutionsBasePackage, selectorTypes, } from "./modes/npm.ts"; -import {fetchPypiInfo, updatePyprojectToml, updateRequirement} from "./modes/pypi.ts"; +import {fetchPypiInfo, pypiSatisfies, updatePyprojectToml, updateRequirement} from "./modes/pypi.ts"; import { resolveGoProxyChain, parseGoNoProxy, - parseGoMod, parseGoWork, fetchGoProxyInfo, updateGoMod, rewriteGoImports, - getGoInfoUrl, shortenGoVersion, shortenGoModule, + parseGoMod, parseGoWork, resolveGoWorkModule, fetchGoProxyInfo, updateGoMod, rewriteGoImports, + getGoInfoUrl, goModulePathForVersion, shortenGoVersion, shortenGoModule, } from "./modes/go.ts"; import { type ActionRef, @@ -35,22 +36,22 @@ import { } from "./modes/actions.ts"; import { type DockerImageRef, - parseDockerTag, extractDockerRefs, dockerImageNames, + parseDockerImageRef, parseDockerTag, extractDockerRefs, dockerImageNames, + fetchDockerTagDigest, getExtractionRegex, isDockerfile, isDockerFileName, dockerExactFileNames, fetchDockerInfo, findDockerVersion, getDockerInfoUrl, updateDockerfile, updateComposeFile, updateWorkflowDockerImages, - composeImageRe, workflowContainerRe, workflowDockerUsesRe, } from "./modes/docker.ts"; import { - type MakeRewrite, type MakeDockerImage, - type MakeUpdate, - type MakeDockerUpdate, isMakeFileName, makeExactFileNames, parseMakeGoInstalls, parseMakeDockerImages, - fetchMakeInfo, fetchMakeDockerInfo, formatMakeImageSpec, updateMakefile, + resolveGoModuleRoot, formatMakeImageSpec, updateMakefile, } from "./modes/make.ts"; import {fetchCratesIoInfo, updateCargoToml, updateCargoRange, cargoToNpmRange, parseCargoLock, findLockedVersion} from "./modes/cargo.ts"; -import {baseType, filterDepsForMember, resolveWorkspaceMembers, parsePnpmWorkspace, pnpmCatalogEntries, updatePnpmWorkspace, type WorkspaceMember} from "./utils/workspace.ts"; +import { + baseType, filterDepsForMember, resolveWorkspaceMembers, parsePnpmWorkspace, pnpmCatalogEntries, + updatePnpmWorkspace, type WorkspaceMember, +} from "./utils/workspace.ts"; /** A dependency whose lookup failed. Every other dependency is still resolved and written. */ export type DepError = { @@ -65,33 +66,21 @@ export type DepError = { export type Output = ModeOutput & {errors?: Array}; export type {Config, Override, Dep, Deps, DepsByMode}; +export {cliBaseConfig as cliConfigBaseDir}; -// Also the order rows print in, as file discovery finds a directory's manifests in readdir order and -// a walk up in modeByFileName order, which would otherwise print a project's modes either way round. -// The three that follow match a filename by pattern rather than by name, so the map does not hold them. const modeOrder = [...new Set(Object.values(modeByFileName)), "actions", "docker", "make"]; const defaultModes = new Set(modeOrder); -// One read-only Set per precision, shared by every dependency instead of built -// per name. Sharing the identity also lets findVersion's prerelease-variant -// cache hit across packages rather than recomputing a variant set per package. const semversByPrecision = { patch: new Set(["patch"]), minor: new Set(["patch", "minor"]), major: new Set(["patch", "minor", "major"]), }; -// Manifests that declare a workspace for their mode, and the plain manifest -// each supersedes in the same directory. Cargo is absent: it has no dedicated -// workspace filename, so it is detected by parsing Cargo.toml's content. -const workspaceManifests: Record = { - "go.work": {mode: "go", supersedes: "go.mod"}, - "pnpm-workspace.yaml": {mode: "npm", supersedes: "package.json"}, -}; +const workspaceManifests: Record = {"go.work": "go.mod", "pnpm-workspace.yaml": "package.json"}; -const apiUrl = (val: unknown, dflt: string | (() => string)) => typeof val === "string" ? normalizeUrl(val) : (typeof dflt === "function" ? dflt() : dflt); +const apiUrl = (value: unknown, fallback: string) => normalizeUrl(typeof value === "string" ? value : fallback); -// Splits a jsr specifier into its `npm:@jsr/pkg@` / `jsr:@scope/pkg@` prefix and version. const jsrSpecifierRe = /^(npm:@jsr\/[^@]+@|jsr:@[^@]+@)(.+)$/; function findUpSync(filenames: string[], dir: string): Map { @@ -110,32 +99,27 @@ function findUpSync(filenames: string[], dir: string): Map { return found; } -async function prefetchFiles(files: Iterable, concurrency: number): Promise> { - const entries = await pMap(files, async (file): Promise<[string, string]> => { - try { - return [file, await readFile(file, "utf8")]; - } catch (err) { - throw new Error(`Unable to open ${file}: ${(err as Error).message}`); - } - }, {concurrency}); - return new Map(entries); -} - -function setDepAge(dep: Dep, date: string): void { +function setDepAge(dep: Dep, date: string | null | undefined): void { if (date) { dep.date = date; dep.age = timerel(date, {noAffix: true, shortUnits: true}); } } -const depKey = (depType: string, typePrefix: string, name: string) => `${depType}${typePrefix}${fieldSep}${name}`; +const dependencyKey = (type: string, name: string, identity?: string) => + `${type}${fieldSep}${name}${identity === undefined ? "" : `${fieldSep}${identity}`}`; +const manifestDependencyKey = (depType: string, typePrefix: string, name: string, identity?: string) => + dependencyKey(`${depType}${typePrefix}`, name, identity); + +const depBelongsToMember = (key: string, memberPath: string): boolean => { + const type = key.split(fieldSep)[0]; + return type === (memberPath === "." ? baseType(type) : `${baseType(type)}|${memberPath}`); +}; const countDeps = (deps: DepsByMode) => Object.values(deps).reduce((num, modeDeps) => num + Object.keys(modeDeps).length, 0); const normalizePep503 = (name: string) => name.toLowerCase().replace(/[-_.]+/g, "-"); -// The spellings a dep answers to, so include/exclude patterns, overrides and pin keys -// all match on the same set regardless of which one the manifest happens to use. function depNames(name: string, kind: string): Array { if (kind === "go") return [name, shortenGoModule(name)]; if (kind === "docker") return dockerImageNames(name); @@ -143,28 +127,25 @@ function depNames(name: string, kind: string): Array { return [name]; } -const pinNameFor = (pin: Record, names: Array) => names.find(name => pin[name]); - -// `kind` selects the name spellings and defaults to `mode`. Make manifests hold both go -// and docker deps, so those call sites pass it rather than claiming to be another mode. -function canInclude(name: string, mode: string, include: Set, exclude: Set, depType: string, kind: string = mode): boolean { +function canInclude(name: string, mode: string, include: Set, exclude: Set, depType: string, kind: string = mode, packageName: string = name): boolean { if (depType === "engines" && nonPackageEngines.includes(name)) return false; if (mode === "pypi" && name === "python") return false; if (!include.size && !exclude.size) return true; - const names = depNames(name, kind); - for (const re of exclude) { - if (names.some(n => re.test(n))) return false; + const names = Array.from(new Set([...depNames(name, kind), ...depNames(packageName, kind)])); + const test = (matcher: RegExp, value: string) => testRenovateMatcher(matcher, value, packageName, name); + for (const matcher of exclude) { + if (names.some(value => test(matcher, value))) return false; } - for (const re of include) { - if (names.some(n => re.test(n))) return true; + for (const matcher of include) { + if (names.some(value => test(matcher, value))) return true; } return !include.size; } -function resolveFiles(filesArg: Set | false): Set { +function resolveFiles(filesArg: Array | undefined): Set { const resolvedFiles = new Set(); - if (filesArg) { + if (filesArg?.length) { for (const arg of filesArg) { let stat: Stats; try { @@ -172,10 +153,6 @@ function resolveFiles(filesArg: Set | false): Set { } catch (err) { throw new Error(`Unable to open ${arg}: ${(err as Error).message}`); } - // A symlink is the file it points at, which is also the spelling whose name selects a mode: - // an argument naming both collapses here rather than being collected twice, and `link.json` - // pointing at a `package.json` is the manifest it resolves to, as the auto-discovery branch - // below already treats a real path as the file's identity. let file = resolve(arg); try { file = realpathSync.native(arg); } catch {} @@ -206,8 +183,6 @@ function resolveFiles(filesArg: Set | false): Set { } else { const forgeDirSet = new Set(forgeDirs); const candidates = [...Object.keys(modeByFileName), ...dockerExactFileNames, ...makeExactFileNames, ...forgeDirs]; - // `Makefile` and `makefile` are both candidates and a case-insensitive filesystem opens - // either, so the real path's on-disk spelling is what stops one file being found twice. const realPaths = new Set(); for (const [filename, path] of findUpSync(candidates, cwd())) { if (forgeDirSet.has(filename)) { @@ -231,32 +206,33 @@ function resolveFiles(filesArg: Set | false): Set { } catch {} } - // A workspace manifest is processed before the plain manifests of its mode: a run started inside - // a member finds that member's own file first, which would then be collected a second time. const workspaceFiles: Array = []; for (const file of Array.from(resolvedFiles)) { const filename = basename(file); if (!Object.hasOwn(workspaceManifests, filename)) continue; workspaceFiles.push(file); - resolvedFiles.delete(join(dirname(file), workspaceManifests[filename].supersedes)); + resolvedFiles.delete(join(dirname(file), workspaceManifests[filename])); } return workspaceFiles.length ? new Set([...workspaceFiles, ...resolvedFiles]) : resolvedFiles; } -// preserve file metadata on windows function write(file: string, content: string): void { if (platform === "win32") truncateSync(file, 0); writeFileSync(file, content, platform === "win32" ? {flag: "r+"} : undefined); } -// `results` holds one entry per name, so a name a file references twice gets its authored ref appended. const rowId = (mode: string, key: string) => `${mode}${fieldSep}${key.split(fieldSep, 2).join(fieldSep)}`; +function displayKey(value: string): string { + if (!value.startsWith("[")) return value; + try { return (JSON.parse(value) as Array).join("."); } catch { return value; } +} + function buildOutput(deps: DepsByMode): Output { const output: Output = {results: {}}; const rowsPerName = new Map(); - const modes = Object.entries(deps).sort(([a], [b]) => modeOrder.indexOf(a) - modeOrder.indexOf(b)); + const modes = Object.entries(deps).sort(([left], [right]) => modeOrder.indexOf(left) - modeOrder.indexOf(right)); for (const [mode, modeDeps] of modes) { for (const key of Object.keys(modeDeps)) { const id = rowId(mode, key); @@ -271,7 +247,7 @@ function buildOutput(deps: DepsByMode): Output { props.old = mode === "go" ? shortenGoVersion(props.oldOrig) : props.oldOrig; } if (mode === "go") props.new = shortenGoVersion(props.new); - else if (mode === "actions") { + else if (mode === "actions" && !props.digestOnly) { props.old = stripv(props.old); props.new = stripv(props.new); } @@ -282,21 +258,47 @@ function buildOutput(deps: DepsByMode): Output { const [type, name, ref] = key.split(fieldSep); const label = ref && rowsPerName.get(rowId(mode, key))! > 1 ? - `${name}${mode === "actions" ? "@" : ":"}${ref}` : name; - const r = output.results[mode] ??= {}; - (r[type] ??= {})[label] = props; + `${name}${mode === "actions" ? "@" : ":"}${displayKey(ref)}` : name; + const modeResults = output.results[mode] ??= {}; + (modeResults[displayKey(type)] ??= {})[label] = props; } } - // Names sort within their type, as a reader scans for a name, not for the section it was authored in. - // By code unit: localeCompare would load ICU, and order by the machine's locale. for (const modeResults of Object.values(output.results)) { for (const [type, typeDeps] of Object.entries(modeResults)) { - modeResults[type] = Object.fromEntries(Object.entries(typeDeps).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)); + modeResults[type] = Object.fromEntries(Object.entries(typeDeps) + .sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)); } } return output; } +function versionAllowed(mode: string, version: string, allowedVersions: string | undefined): boolean { + if (!allowedVersions) return true; + const regex = /^(!?)\/(.*)\/(i?)$/.exec(allowedVersions); + if (regex) { + const matches = new RegExp(regex[2], regex[3]).test(version); + return regex[1] ? !matches : matches; + } + if (mode === "docker") return satisfies(coerce(parseDockerTag(version)?.version ?? "")?.version ?? "", allowedVersions); + if (mode === "pypi") return pypiSatisfies(version, allowedVersions); + return satisfies(version, allowedVersions); +} + +function filterVersionData(data: Record, mode: string, allowedVersions: string | undefined) { + if (!allowedVersions) return data; + const filterRecord = (record: Record) => Object.fromEntries(Object.entries(record) + .filter(([version]) => versionAllowed(mode, version, allowedVersions))); + if (mode === "go") return { + ...data, + ...(data.versions && {versions: filterRecord(data.versions)}), + new: typeof data.new === "string" && versionAllowed(mode, data.new, allowedVersions) ? data.new : "", + sameMajorNew: typeof data.sameMajorNew === "string" && versionAllowed(mode, data.sameMajorNew, allowedVersions) ? + data.sameMajorNew : "", + }; + const key = mode === "pypi" ? "releases" : mode === "docker" ? "tags" : "versions"; + return {...data, [key]: filterRecord(data[key] ?? {})}; +} + export type UpdatesOptions = Config & { /** Override GitHub/Gitea API URL (for testing) */ forgeapi?: string; @@ -312,31 +314,27 @@ export type UpdatesOptions = Config & { dockerapi?: string; }; -let dnsCacheEnabled = false; - export async function updates(opts: UpdatesOptions = {}): Promise { - if (!dnsCacheEnabled) { - const {enableDnsCache} = await import("./utils/dns.ts"); - enableDnsCache(); - dnsCacheEnabled = true; + const {enableDnsCache} = await import("./utils/dns.ts"); + const disposeDnsCache = enableDnsCache(); + try { + return await runUpdates(opts); + } finally { + disposeDnsCache(); } +} +async function runUpdates(opts: UpdatesOptions): Promise { const config: Config = {...opts}; if (typeof config.timeout === "number") config.timeout = parsePositiveInt(config.timeout, "timeout"); const concurrency = config.sockets ?? maxSockets; const userTimeout = config.timeout ?? 0; const forgeApiUrl = apiUrl(opts.forgeapi, defaultApiUrls.forgeapi); - const pypiApiUrl = apiUrl(opts.pypiapi, defaultApiUrls.pypiapi); - const jsrApiUrl = apiUrl(opts.jsrapi, defaultApiUrls.jsrapi); const goProxyChain = resolveGoProxyChain(opts.goproxy); - const goProxyUrl = goProxyChain[0].url; - const cratesIoUrl = apiUrl(opts.cargoapi, defaultApiUrls.cargoapi); - const dockerApiUrl = apiUrl(opts.dockerapi, defaultApiUrls.dockerapi); const goNoProxy = parseGoNoProxy(); const useVerboseColor = !config.noColor && (config.color || stderr.isTTY); - // validateStream drops the codes when stderr is not a TTY, which is exactly what -c overrides. const colorFn = (color: "magenta" | "green" | "red") => useVerboseColor ? (text: string | number) => styleText(color, String(text), {validateStream: false}) : String; const magenta = colorFn("magenta"); const vGreen = colorFn("green"); @@ -344,19 +342,17 @@ export async function updates(opts: UpdatesOptions = {}): Promise { let limit: Limiter | undefined; const ctx: ModeContext = { - execFile: async (file, args, opts) => (await getExecFile())(file, args, opts), // lazy, so child_process loads only if a mode shells out + execFile: async (file, args, execOpts) => (await getExecFile())(file, args, execOpts), fetchTimeout: userTimeout || fetchTimeout, goProbeTimeout: userTimeout ? Math.max(1, Math.floor(userTimeout / 2)) : goProbeTimeout, concurrency, forgeApiUrl, - pypiApiUrl, - jsrApiUrl, - goProxyUrl, + pypiApiUrl: apiUrl(opts.pypiapi, defaultApiUrls.pypiapi), + jsrApiUrl: apiUrl(opts.jsrapi, defaultApiUrls.jsrapi), + goProxyUrl: goProxyChain[0].url, goProxyChain, - cratesIoUrl, - dockerApiUrl, - // `--sockets` is one budget for the run: this slot covers the go major probes, which reach - // doFetch directly rather than through fetchWithRetry. + cratesIoUrl: apiUrl(opts.cargoapi, defaultApiUrls.cargoapi), + dockerApiUrl: apiUrl(opts.dockerapi, defaultApiUrls.dockerapi), doFetch: (url: string, fetchOpts?: RequestInit) => (limit ??= getLimiter(ctx))(async () => { if (config.verbose) console.error(`${timestamp()} ${magenta(fetchOpts?.method || "GET")} ${url}`); const res = await doFetch(url, fetchOpts); @@ -366,92 +362,103 @@ export async function updates(opts: UpdatesOptions = {}): Promise { noCache: Boolean(config.noCache), }; - const greatest = configMixedToRegexes(config.greatest); - const prerelease = configMixedToRegexes(config.prerelease); - const release = configMixedToRegexes(config.release); - const patch = configMixedToRegexes(config.patch); - const minor = configMixedToRegexes(config.minor); - const allowDowngrade = configMixedToRegexes(config.allowDowngrade); for (const mode of config.modes ?? []) { if (!defaultModes.has(mode)) throw new Error(`Invalid mode: ${mode}, expected one of: ${modeOrder.join(",")}`); } const enabledModes = config.modes?.length ? new Set(config.modes) : defaultModes; - type CompiledOverride = { - include?: Set, exclude?: Set, - greatest?: boolean, prerelease?: boolean, release?: boolean, - patch?: boolean, minor?: boolean, allowDowngrade?: boolean, cooldownDays?: number, - }; - const compiledOverrides: Array = (config.overrides ?? []).map(o => ({ - include: o.include?.length ? patternsToRegexSet(o.include) : undefined, - exclude: o.exclude?.length ? patternsToRegexSet(o.exclude) : undefined, - greatest: o.greatest, prerelease: o.prerelease, release: o.release, - patch: o.patch, minor: o.minor, allowDowngrade: o.allowDowngrade, - cooldownDays: o.cooldown !== undefined ? parseDuration(String(o.cooldown)) : undefined, - })); - const overrideMatches = (o: CompiledOverride, names: Array): boolean => { - if (o.include && names.every(n => !matchesAny(n, o.include!))) return false; - return !o.exclude || names.every(n => !matchesAny(n, o.exclude!)); + const compileVersionConfig = (source: Config) => { + const overrides = (source.overrides ?? []).map(override => ({ + include: override.include?.length ? patternsToRegexSet(override.include) : undefined, + exclude: override.exclude?.length ? patternsToRegexSet(override.exclude) : undefined, + greatest: override.greatest, prerelease: override.prerelease, release: override.release, + patch: override.patch, minor: override.minor, allowDowngrade: override.allowDowngrade, + cooldownDays: override.cooldown !== undefined ? parseDuration(String(override.cooldown)) : undefined, + })); + return { + greatest: configMixedToRegexes(source.greatest), + prerelease: configMixedToRegexes(source.prerelease), + release: configMixedToRegexes(source.release), + patch: configMixedToRegexes(source.patch), + minor: configMixedToRegexes(source.minor), + allowDowngrade: configMixedToRegexes(source.allowDowngrade), + overrides, + renovateVersionRules: (source as Config & {renovateVersionRules?: Array}).renovateVersionRules ?? [], + hasCooldownOverride: overrides.some(override => override.cooldownDays !== undefined), + }; }; - const overridesHaveCooldown = compiledOverrides.some(o => o.cooldownDays); - - // Kick off `gh auth token` early so the first forge request isn't blocked on a subprocess. + type VersionConfig = ReturnType; if (enabledModes.has("actions")) getGithubTokens(); - const versionOptsCache = new Map, useGreatest: boolean, usePre: boolean, useRel: boolean, semvers: Set, allowDowngrade: boolean, cooldownOverride: number | undefined}>(); + type ResolvedVersionOpts = { + names: Array, useGreatest: boolean, usePre: boolean, useRel: boolean, semvers: Set, + allowDowngrade: boolean, cooldownOverride: number | undefined, allowedVersions?: string, + }; + const versionOptsCache = new WeakMap>(); - // Resolve per-dependency options: start from the global flags, then apply - // every matching override in order so the last matching one wins. cooldown is - // returned as an override (undefined = no override) since its base differs per - // mode. patch wins over minor, matching the global precedence. - function getVersionOpts(kind: string, name: string) { - // Keyed by kind too: a docker `redis` and an npm `redis` resolve different overrides. - const cacheKey = `${kind}${fieldSep}${name}`; - let entry = versionOptsCache.get(cacheKey); + function getVersionOpts(versionConfig: VersionConfig, kind: string, packageName: string, depName: string = packageName) { + let cache = versionOptsCache.get(versionConfig); + if (!cache) versionOptsCache.set(versionConfig, cache = new Map()); + const cacheKey = `${kind}${fieldSep}${packageName}${fieldSep}${depName}`; + let entry = cache.get(cacheKey); if (!entry) { - const allNames = depNames(name, kind); - const anyMatches = (set: Set | boolean) => allNames.some(n => matchesAny(n, set)); - let useGreatest = anyMatches(greatest); - let usePre = anyMatches(prerelease); - let useRel = anyMatches(release); - let usePatch = anyMatches(patch); - let useMinor = anyMatches(minor); - let allowDown = anyMatches(allowDowngrade); + const allNames = Array.from(new Set([...depNames(packageName, kind), ...depNames(depName, kind)])); + const anyMatches = (set: Set | boolean) => allNames.some(name => matchesAny(name, set)); + let useGreatest = anyMatches(versionConfig.greatest); + let usePre = anyMatches(versionConfig.prerelease); + let useRel = anyMatches(versionConfig.release); + let usePatch = anyMatches(versionConfig.patch); + let useMinor = anyMatches(versionConfig.minor); + let allowDown = anyMatches(versionConfig.allowDowngrade); let cooldownOverride: number | undefined; - for (const o of compiledOverrides) { - if (!overrideMatches(o, allNames)) continue; - if (o.greatest !== undefined) useGreatest = o.greatest; - if (o.prerelease !== undefined) usePre = o.prerelease; - if (o.release !== undefined) useRel = o.release; - if (o.patch !== undefined) usePatch = o.patch; - if (o.minor !== undefined) useMinor = o.minor; - if (o.allowDowngrade !== undefined) allowDown = o.allowDowngrade; - if (o.cooldownDays !== undefined) cooldownOverride = o.cooldownDays; + for (const override of versionConfig.overrides) { + if (override.include && allNames.every(name => !matchesAny(name, override.include!)) || + override.exclude && allNames.some(name => matchesAny(name, override.exclude!))) continue; + if (override.greatest !== undefined) useGreatest = override.greatest; + if (override.prerelease !== undefined) usePre = override.prerelease; + if (override.release !== undefined) useRel = override.release; + if (override.patch !== undefined) usePatch = override.patch; + if (override.minor !== undefined) useMinor = override.minor; + if (override.allowDowngrade !== undefined) allowDown = override.allowDowngrade; + if (override.cooldownDays !== undefined) cooldownOverride = override.cooldownDays; } const semvers = usePatch ? semversByPrecision.patch : useMinor ? semversByPrecision.minor : semversByPrecision.major; + let allowedVersions: string | undefined; + for (const rule of versionConfig.renovateVersionRules) { + if (!matchesRenovateRule(rule, packageName, depName)) continue; + if (rule.allowedVersions !== undefined) allowedVersions = rule.allowedVersions; + if (rule.cooldownDays !== undefined) cooldownOverride = rule.cooldownDays; + } - entry = {names: allNames, useGreatest, usePre, useRel, semvers, allowDowngrade: allowDown, cooldownOverride}; - versionOptsCache.set(cacheKey, entry); + entry = {names: allNames, useGreatest, usePre, useRel, semvers, allowDowngrade: allowDown, cooldownOverride, allowedVersions}; + cache.set(cacheKey, entry); } return entry; } - const include = patternsToRegexSet(config.include ?? []); - const exclude = patternsToRegexSet(config.exclude ?? []); validatePin(config.pin); // the CLI validates on parse, the programmatic caller has not const globalPin: Record = config.pin ?? {}; - // A pin the user authored (CLI `-l`, the programmatic `pin`, `updates.config`) may move a - // dependency down into its range; one inherited from renovate's allowedVersions is a ceiling - // and only ever filters (utils/renovate.ts), so provenance decides per name. const resolvePin = (names: Array, filePin: Record, noDowngrade?: Config["pinNoDowngrade"]) => { - const authored = pinNameFor(globalPin, names); + const authored = names.find(name => globalPin[name]); if (authored) return {pinnedRange: globalPin[authored], pinNoDowngrade: false}; - const inherited = pinNameFor(filePin, names); + const inherited = names.find(name => filePin[name]); + const inheritedFromRenovate = Boolean(inherited && Array.isArray(noDowngrade) && noDowngrade.includes(inherited)); return { - pinnedRange: inherited ? filePin[inherited] : undefined, - pinNoDowngrade: Boolean(inherited && Array.isArray(noDowngrade) && noDowngrade.includes(inherited)), + pinnedRange: inherited && !inheritedFromRenovate ? filePin[inherited] : undefined, + pinNoDowngrade: inheritedFromRenovate, + }; + }; + const resolveVersionOpts = ( + versionConfig: VersionConfig, kind: string, packageName: string, depName: string, + filePin: Record, pinNoDowngrade: Config["pinNoDowngrade"], fileCooldownDays: number, + ) => { + const versionOpts = getVersionOpts(versionConfig, kind, packageName, depName); + return { + ...versionOpts, + ...resolvePin(versionOpts.names, filePin, pinNoDowngrade), + cooldownDays: versionOpts.cooldownOverride ?? fileCooldownDays, }; }; @@ -459,61 +466,53 @@ export async function updates(opts: UpdatesOptions = {}): Promise { const maybeUrlDeps: Deps = {}; const cargoCrates = new Map(); const npmAliases = new Map(); - // so a version the pypi writer would decline to write is dropped before it is reported + const npmPublishedNames = new Map(); + const pnpmWorkspaceOverrideKeys = new Map(); const pypiSpecs = new Map(); const errors: Array = []; - const addError = (mode: string, type: string, name: string, err: unknown) => { - errors.push({mode, type, name, error: (err as Error)?.message || String(err)}); - }; const addKeyError = (mode: string, key: string, err: unknown) => { const [type, name] = key.split(fieldSep); - addError(mode, type, name, err); + errors.push({mode, type, name, error: (err as Error)?.message || String(err)}); }; - type PlainFile = {absPath: string, content: string, memberPath: string, projectDir: string, modeConfig: Config, pin: Record, modeCooldownDays: number}; + const rejectDep = (mode: string, key: string, err: unknown) => { + delete deps[mode][key]; + addKeyError(mode, key, err); + }; + type PlainFile = {absPath: string, content: string, memberPath: string, projectDir: string}; const plainFiles: Record> = {}; const now = Date.now(); - const cooldownDaysFor = (local: Config["cooldown"]) => { - const raw = config.cooldown ?? local; - return raw ? parseDuration(String(raw)) : 0; - }; const cwdStr = cwd(); const toRelPath = (absPath: string) => absPath.replace(`${cwdStr}/`, "").replace(`${cwdStr}\\`, ""); const addDep = (mode: string, depType: string, typePrefix: string, name: string, old: string, oldOrig: string) => { - deps[mode][depKey(depType, typePrefix, name)] = {old, oldOrig} as Dep; + deps[mode][manifestDependencyKey(depType, typePrefix, name)] = {old, oldOrig} as Dep; }; - const addNpmDep = (depType: string, typePrefix: string, name: string, value: string) => { - // A catalog reference names a catalog, not a version: the range lives in pnpm-workspace.yaml, - // which is where it is reported and rewritten, so the member has nothing to resolve. + const addNpmDep = (key: string, name: string, value: string) => { if (isCatalogRef(value)) return; const alias = parseNpmAlias(value); if (isJsr(value)) { - addDep("npm", depType, typePrefix, name, parseJsrDependency(value, name).version, value); + deps.npm[key] = {old: parseJsrDependency(value, name).version, new: "", oldOrig: value}; } else if (validRange(value)) { - addDep("npm", depType, typePrefix, name, normalizeRange(value), value); + deps.npm[key] = {old: normalizeRange(value), new: "", oldOrig: value}; } else if (alias) { - npmAliases.set(depKey(depType, typePrefix, name), alias); - addDep("npm", depType, typePrefix, name, normalizeRange(alias.range), value); + npmAliases.set(key, alias); + deps.npm[key] = {old: normalizeRange(alias.range), new: "", oldOrig: value}; } else if (isLocalDep(value)) { - addDep("npm", depType, typePrefix, name, "0.0.0", value); + return; } else { - maybeUrlDeps[depKey(depType, typePrefix, name)] = {old: value} as Dep; + maybeUrlDeps[key] = {old: value} as Dep; } }; - // Only pypi has array-valued dep types; an array elsewhere is malformed, not indices to collect. const collectDeps = (mode: string, pkg: Record, typePrefix: string, depTypes: Array, modeInclude: Set, modeExclude: Set) => { - // uv resolves a dep with a source of its own from git, a url, the filesystem or another index, - // never from pypi.org, where the same name may well be someone else's package. The keys are - // matched PEP 503-normalized, as `Flask-SQLAlchemy` and `flask_sqlalchemy` are one project. const uvSources = new Set(Object.keys(pkg.tool?.uv?.sources ?? {}).map(normalizePep503)); const addUvDeps = (specs: Array, depType: string) => { for (const {name, version, spec} of parseUvDependencies(specs)) { if (uvSources.has(normalizePep503(name))) continue; if (canInclude(name, mode, modeInclude, modeExclude, depType)) { addDep(mode, depType, typePrefix, name, normalizeRange(version), version); - pypiSpecs.set(depKey(depType, typePrefix, name), spec); + pypiSpecs.set(manifestDependencyKey(depType, typePrefix, name), spec); } } }; @@ -529,26 +528,32 @@ export async function updates(opts: UpdatesOptions = {}): Promise { } } else { const entries = Object.entries(obj as Record); - // npm's nested `overrides` object carries no version of its own. Renovate recurses into it, - // which the writer's flat key scan cannot place a rewrite in, so it is skipped instead, and - // a name it shadows is skipped with it rather than rewritten inside the nested copy. - const nestedNames = new Set(); - if (mode === "npm") { - const collectNested = (value: any) => { - if (!value || typeof value !== "object") return; - for (const [key, inner] of Object.entries(value)) { - nestedNames.add(key); - collectNested(inner); + if (mode === "npm" && (depType === "overrides" || depType === "pnpm.overrides")) { + const root = depType === "overrides" ? ["overrides"] : ["pnpm", "overrides"]; + const collectOverrides = (child: Record, parents: Array) => { + for (const [selector, value] of Object.entries(child)) { + if (typeof value === "string") { + const name = resolutionsBasePackage(selector === "." ? parents.at(-1) ?? selector : selector); + const alias = parseNpmAlias(value); + if (!canInclude(name, mode, modeInclude, modeExclude, depType, mode, alias?.name ?? name)) continue; + const path = [...root, ...parents, selector]; + const identity = JSON.stringify(path); + const key = manifestDependencyKey(depType, typePrefix, name, identity); + addNpmDep(key, name, value); + } else if (value && typeof value === "object" && !Array.isArray(value)) { + collectOverrides(value, [...parents, selector]); + } } }; - for (const [, value] of entries) collectNested(value); + collectOverrides(obj as Record, []); + continue; } for (const [name, value] of entries) { - // An explicit `-t project.optional-dependencies` lands here, and its keys name groups. if (mode === "pypi" && Array.isArray(value)) { addUvDeps(value, `${depType}.${name}`); continue; } - if (typeof value !== "string" || nestedNames.has(name)) continue; - if (!canInclude(name, mode, modeInclude, modeExclude, depType)) continue; - if (mode === "npm") addNpmDep(depType, typePrefix, name, value); + if (typeof value !== "string") continue; + const alias = mode === "npm" ? parseNpmAlias(value) : null; + if (!canInclude(name, mode, modeInclude, modeExclude, depType, mode, alias?.name ?? name)) continue; + if (mode === "npm") addNpmDep(manifestDependencyKey(depType, typePrefix, name), name, value); else if (mode === "go") addDep(mode, depType, typePrefix, name, shortenGoVersion(value), stripv(value)); else if (validRange(value)) addDep(mode, depType, typePrefix, name, normalizeRange(value), value); } @@ -556,24 +561,27 @@ export async function updates(opts: UpdatesOptions = {}): Promise { } }; - const files = resolveFiles(config.files?.length ? new Set(config.files) : false); - const fileApplies = (file: string): boolean => { + const files = resolveFiles(config.files); + const fileContents = new Map(await pMap(Array.from(files).filter(file => { if (isWorkflowFile(file)) return enabledModes.has("actions") || enabledModes.has("docker"); const filename = basename(file); if (isDockerFileName(filename)) return enabledModes.has("docker"); if (isMakeFileName(filename)) return enabledModes.has("make"); - const mode = modeByFileName[filename]; - return Boolean(mode) && enabledModes.has(mode); - }; - const fileContents = await prefetchFiles(Array.from(files).filter(fileApplies), concurrency); + return enabledModes.has(modeByFileName[filename]); + }), async (file): Promise<[string, string]> => { + try { + return [file, await readFile(file, "utf8")]; + } catch (err) { + throw new Error(`Unable to open ${file}: ${(err as Error).message}`); + } + }, {concurrency})); - const wfData: Record = {}; - const dockerFileData: Record = {}; - const makeFileData: Record = {}; + const fileData: Record, + }> = {}; - type GoModFileInfo = {absPath: string, content: string, projectDir: string, memberPath: string}; - const goModFiles: GoModFileInfo[] = []; - let goWorkData: {file: string, content: string} | null = null; + const goModFiles: PlainFile[] = []; + const goWorkFiles: Array<{file: string, content: string, memberPath: string}> = []; const cargoMemberFiles: WorkspaceMember[] = []; const pnpmMemberFiles: WorkspaceMember[] = []; @@ -581,125 +589,143 @@ export async function updates(opts: UpdatesOptions = {}): Promise { type ActionDepInfo = ActionRef & { key: string, apiUrl: string, filePin: Record, filePinNoDowngrade: Config["pinNoDowngrade"], fileCooldownDays: number, - comment: string, // the version the line's trailing comment names, empty when it names none + versionConfig: VersionConfig, comment: string, }; const actionDepInfos: Array = []; type DockerDepInfo = { - key: string, fullImage: string, ref: DockerImageRef, filePin: Record, fileCooldownDays: number, + key: string, fullImage: string, ref: DockerImageRef, versionConfig: VersionConfig, + filePin: Record, fileCooldownDays: number, }; const dockerDepInfos: Array = []; type MakeDepBase = { key: string, name: string, oldSpec: string, projectDir: string, - filePin: Record, filePinNoDowngrade: Config["pinNoDowngrade"], fileCooldownDays: number, newSpec?: string, + versionConfig: VersionConfig, filePin: Record, filePinNoDowngrade: Config["pinNoDowngrade"], + fileCooldownDays: number, newSpec?: string, }; type MakeDepInfo = MakeDepBase & ( {kind: "go", installPath: string, version: string} | {kind: "docker", image: MakeDockerImage} ); const makeDepInfos: Array = []; - type ModeCtx = {modeConfig: Config, projectDir: string, pin: Record}; - const modeConfigs: Record = {}; - const presetFetch = {noCache: config.noCache, timeout: config.timeout || fetchTimeout}; - - // Load a directory's config and merge its include/exclude patterns onto the - // global ones. Callers differ only in how they use `pin` and `cooldown`. - // Memoized per directory: sibling manifests and workflow/Dockerfile/Makefile - // targets share a dir, and recompiling their regex sets per file is pure waste. + type ModeCtx = { + modeConfig: Config, versionConfig: VersionConfig, projectDir: string, pin: Record, + cooldownDays: number, + }; + const modeContextsBySuffix: Record> = {}; + const registerModeContext = (mode: string, memberPath: string, modeCtx: ModeCtx) => { + (modeContextsBySuffix[mode] ??= new Map()).set(memberPath === "." ? "" : `|${memberPath}`, modeCtx); + }; + const cliBase = (opts as {[cliBaseConfig]?: {fileConfig: Config, cliKeys: Array}})[cliBaseConfig]; + const cliKeys = cliBase && new Set(cliBase.cliKeys); + const configOverrides = Object.fromEntries(Object.entries(config).filter(([key, value]) => + value !== undefined && (!cliKeys || cliKeys.has(key)))); + const resolveDirConfig = memoizeAsync(async (dir: string) => { - const dirConfig = await loadConfig(dir, presetFetch); + const modeConfig = {...await loadConfig(dir), ...configOverrides}; return { - dirConfig, - include: dirConfig.include?.length ? patternsToRegexSet([...(config.include ?? []), ...dirConfig.include]) : include, - exclude: dirConfig.exclude?.length ? patternsToRegexSet([...(config.exclude ?? []), ...dirConfig.exclude]) : exclude, + modeConfig, + include: patternsToRegexSet(modeConfig.include ?? []), + exclude: patternsToRegexSet(modeConfig.exclude ?? []), + versionConfig: compileVersionConfig(modeConfig), + pin: modeConfig.pin ?? {}, + pinNoDowngrade: modeConfig.pinNoDowngrade, + cooldownDays: modeConfig.cooldown ? parseDuration(String(modeConfig.cooldown)) : 0, }; }); - async function resolveModeFilters(projectDir: string) { - const {dirConfig, include: modeInclude, exclude: modeExclude} = await resolveDirConfig(projectDir); - // The directory's own pins only; resolvePin consults globalPin first and by name. - return {modeConfig: dirConfig, modeInclude, modeExclude, pin: dirConfig.pin ?? {}}; - } - function resolveDepTypes(mode: string, modeConfig: Config): Array { - if (config.types?.length) return config.types; - if (modeConfig?.types?.length) return modeConfig.types; - if (mode === "npm") return npmTypes; - if (mode === "pypi") return uvTypes; - if (mode === "go") return config.indirect ? goTypes : goTypes.filter(t => t !== "indirect"); - // Target sections are only in the default list: an explicit `-t dependencies` asks for the - // plain table and must not be widened onto every `[target.*]` one. - if (mode === "cargo") return [...cargoTypes, ...cargoTargetTypes]; - return []; + if (modeConfig.types?.length) return modeConfig.types; + return mode === "npm" ? npmTypes : mode === "pypi" ? uvTypes : + mode === "go" ? modeConfig.indirect ? goTypes : goTypes.filter(type => type !== "indirect") : + mode === "cargo" ? [...cargoTypes, ...cargoTargetTypes] : []; } - type FileFilters = {include: Set, exclude: Set, pin: Record, pinNoDowngrade: Config["pinNoDowngrade"], cooldownDays: number}; + type FileFilters = Awaited>; + const modeCtx = (filters: Pick, projectDir: string): ModeCtx => ({ + modeConfig: filters.modeConfig, versionConfig: filters.versionConfig, projectDir, pin: filters.pin, + cooldownDays: filters.cooldownDays, + }); - function collectDockerRefs(content: string, relPath: string, regexes: Array, filters: FileFilters): void { - deps.docker ??= {}; - for (const regex of regexes) { - for (const {ref} of extractDockerRefs(content, regex)) { - if (!canInclude(ref.fullImage, "docker", filters.include, filters.exclude, "docker")) continue; - // The tag is part of the key: one image at two tags is two dependencies. - const key = `${relPath}${fieldSep}${ref.fullImage}${fieldSep}${ref.tag}`; - if (deps.docker[key]) continue; - const parsed = parseDockerTag(ref.tag); - if (!parsed) continue; - deps.docker[key] = {old: parsed.version, oldOrig: ref.tag} as Dep; - dockerDepInfos.push({ - key, fullImage: ref.fullImage, ref, filePin: filters.pin, fileCooldownDays: filters.cooldownDays, - }); - } + const pnpmWorkspaceOverrides = (content: string) => { + const entries: Array<{selector: string, value: string, lineNumber: number, valueIndex: number}> = []; + let sectionIndent = -1; + for (const [lineNumber, line] of content.split(/\r?\n/).entries()) { + const pair = /^(\s*)(?:"([^"]+)"|'([^']+)'|([^\s:#][^:#]*)):\s*(.*)$/.exec(line); + if (!pair) continue; + const indent = pair[1].length; + const key = (pair[2] ?? pair[3] ?? pair[4]).trim(); + if (indent === 0) { sectionIndent = key === "overrides" ? indent : -1; continue; } + if (sectionIndent === -1 || !pair[5]) continue; + const raw = pair[5].replace(/\s+#.*$/, "").trimEnd(); + const leading = pair[5].length - pair[5].trimStart().length; + const quoted = /^(['"])(.*)\1$/.exec(raw.trimStart()); + const value = quoted?.[2] ?? raw.trimStart(); + entries.push({selector: key, value, lineNumber, valueIndex: line.length - pair[5].length + leading + Number(Boolean(quoted))}); } - } + return entries; + }; - async function resolveFileConfig(fileDir: string): Promise { - const {dirConfig, include, exclude} = await resolveDirConfig(fileDir); - return { - include, exclude, pin: dirConfig.pin ?? {}, pinNoDowngrade: dirConfig.pinNoDowngrade, - cooldownDays: cooldownDaysFor(dirConfig.cooldown), - }; + function collectDockerRef(ref: DockerImageRef, relPath: string, filters: FileFilters): void { + deps.docker ??= {}; + if (!canInclude(ref.fullImage, "docker", filters.include, filters.exclude, "docker")) return; + const identity = `${ref.tag}${ref.digest ? `@${ref.digest}` : ""}`; + const key = dependencyKey(relPath, ref.fullImage, identity); + if (deps.docker[key]) return; + const parsed = parseDockerTag(ref.tag); + if (!parsed && !ref.digest) return; + deps.docker[key] = { + old: parsed?.version ?? ref.tag, oldOrig: ref.tag, + ...(ref.digest && {oldDigest: ref.digest}), ...(ref.digestOnly && {digestOnly: true}), + } as Dep; + dockerDepInfos.push({ + key, fullImage: ref.fullImage, ref, versionConfig: filters.versionConfig, + filePin: filters.pin, fileCooldownDays: filters.cooldownDays, + }); } - // A workspace manifest owns the empty dep-prefix for its mode, so plain ones must avoid the "." - // memberPath. Determined up front to stay independent of file order. - const workspaceModes = new Set(); - const parsedCargoToml = new Map>(); - // A cargo workspace root shares its filename with its members, so resolveFiles cannot hoist it. - // It has to run first all the same, or a member listed ahead of it is collected a second time. - const cargoWorkspaceFiles: Array = []; + const cargoWorkspaceFiles = new Map>(); + const npmWorkspaceFiles = new Map>(); for (const file of files) { const filename = basename(file); - if (Object.hasOwn(workspaceManifests, filename)) workspaceModes.add(workspaceManifests[filename].mode); - else if (filename === "Cargo.toml") { + if (filename === "package.json") { + const content = fileContents.get(file); + if (!content) continue; + try { + const parsed = JSON.parse(content); + const patterns = Array.isArray(parsed.workspaces) ? parsed.workspaces : parsed.workspaces?.packages; + if (Array.isArray(patterns) && patterns.some(pattern => typeof pattern === "string")) { + npmWorkspaceFiles.set(file, parsed); + } + } catch {} + } else if (filename === "Cargo.toml") { const content = fileContents.get(file); if (!content) continue; try { const parsed = parseToml(content); - parsedCargoToml.set(file, parsed); const members = (parsed.workspace as Record)?.members; if (Array.isArray(members) && members.length) { - workspaceModes.add("cargo"); - cargoWorkspaceFiles.push(file); + cargoWorkspaceFiles.set(file, parsed); } } catch {} } } - // Register a non-workspace manifest and return the type prefix its deps use. - // The first manifest of a mode keeps the "." memberPath (empty prefix) to preserve - // the single-manifest output shape and seeds the mode-level default context; later - // ones are disambiguated by their relative path so deps from distinct files never - // collide. - const addPlainFile = (mode: string, file: string, content: string, projectDir: string, modeConfig: Config, pin: Record): string => { + const workspaceRootCounts: Record = { + cargo: cargoWorkspaceFiles.size, + go: Array.from(files).filter(file => basename(file) === "go.work").length, + npm: new Set([...npmWorkspaceFiles.keys(), + ...Array.from(files).filter(file => basename(file) === "pnpm-workspace.yaml")]).size, + }; + + const addPlainFile = (mode: string, file: string, content: string, projectDir: string, filters: FileFilters): string => { const modeFiles = plainFiles[mode] ??= []; - const isFirstOfMode = !modeFiles.length && !workspaceModes.has(mode); + const isFirstOfMode = !modeFiles.length && !workspaceRootCounts[mode]; const memberPath = isFirstOfMode ? "." : toRelPath(file); - modeFiles.push({absPath: resolve(file), content, memberPath, projectDir, modeConfig, pin, modeCooldownDays: cooldownDaysFor(modeConfig.cooldown)}); - if (isFirstOfMode) modeConfigs[mode] = {modeConfig, projectDir, pin}; + modeFiles.push({absPath: resolve(file), content, memberPath, projectDir}); + registerModeContext(mode, memberPath, modeCtx(filters, projectDir)); return isFirstOfMode ? "" : `|${memberPath}`; }; - // Run a manifest parser, attributing a syntax error to its file. const parseFile = (file: string, parse: () => Record): Record => { try { return parse(); @@ -708,40 +734,75 @@ export async function updates(opts: UpdatesOptions = {}): Promise { } }; - // `fileContents` already holds exactly the files whose mode is enabled, in `files` order. - for (const file of cargoWorkspaceFiles.length ? new Set([...cargoWorkspaceFiles, ...fileContents.keys()]) : fileContents.keys()) { + const workspaceMemberPath = (mode: string, workspaceFile: string, memberPath: string) => + workspaceRootCounts[mode] > 1 ? + `${toRelPath(workspaceFile)}:${memberPath}` : memberPath; + const collectNpmWorkspaceMember = ( + workspaceFile: string, workspaceDir: string, member: WorkspaceMember, pkg: Record, + dependencyTypes: Array, filters: FileFilters, + ) => { + const memberPath = workspaceMemberPath("npm", workspaceFile, member.memberPath); + registerModeContext("npm", memberPath, modeCtx(filters, workspaceDir)); + pnpmMemberFiles.push({...member, memberPath}); + collectDeps("npm", pkg, memberPath === "." ? "" : `|${memberPath}`, dependencyTypes, filters.include, filters.exclude); + }; + for (const file of cargoWorkspaceFiles.size || npmWorkspaceFiles.size ? + new Set([...cargoWorkspaceFiles.keys(), ...npmWorkspaceFiles.keys(), ...fileContents.keys()]) : fileContents.keys()) { if (isWorkflowFile(file)) { const actionsEnabled = enabledModes.has("actions"); const dockerEnabled = enabledModes.has("docker"); const content = fileContents.get(file)!; const relPath = toRelPath(file); - const filters = await resolveFileConfig(dirname(file)); - wfData[relPath] = {absPath: file, content}; + const filters = await resolveDirConfig(dirname(file)); + const workflowLines = new Set(); + fileData[relPath] = {absPath: file, content, fileType: "workflow", workflowLines}; + const yamlPath: Array<{indent: number, key: string}> = []; - if (actionsEnabled) { - deps.actions ??= {}; - // The writer's parser, so a sha pin's trailing comment, its version, travels with the ref. - for (const line of content.split("\n")) { + for (const [lineNumber, line] of content.split("\n").entries()) { + if (actionsEnabled) { const parsed = parseUsesLine(line); const action = parsed && parseActionRef(parsed.value); - if (!action) continue; - if (!canInclude(action.name, "actions", filters.include, filters.exclude, "actions")) continue; - // The ref is part of the key: a workflow may pin one action twice. - const key = `${relPath}${fieldSep}${action.name}${fieldSep}${action.ref}`; - if (deps.actions[key]) continue; - deps.actions[key] = {old: action.ref} as Dep; - actionDepInfos.push({ - ...action, key, comment: parsed.pinnedVersion, - apiUrl: getForgeApiBaseUrl(action.host, forgeApiUrl), - filePin: filters.pin, filePinNoDowngrade: filters.pinNoDowngrade, fileCooldownDays: filters.cooldownDays, - }); + if (action && canInclude(action.name, "actions", filters.include, filters.exclude, "actions")) { + deps.actions ??= {}; + const comment = parsed.pinnedVersion || /^#\s*(\S+)\s*$/.exec(parsed.comment)?.[1] || ""; + const identity = action.isHash && comment ? `${comment}@${action.ref}` : action.ref; + const key = dependencyKey(relPath, action.name, identity); + if (!deps.actions[key]) { + deps.actions[key] = {old: action.ref} as Dep; + actionDepInfos.push({ + ...action, key, comment, + apiUrl: getForgeApiBaseUrl(action.host, forgeApiUrl), + versionConfig: filters.versionConfig, filePin: filters.pin, + filePinNoDowngrade: filters.pinNoDowngrade, fileCooldownDays: filters.cooldownDays, + }); + } + } } - } - if (dockerEnabled) { - dockerFileData[relPath] = {absPath: file, content, fileType: "workflow"}; - collectDockerRefs(content, relPath, [composeImageRe, workflowContainerRe, workflowDockerUsesRe], filters); + if (!dockerEnabled) continue; + const pair = /^(\s*)(?:-\s*)?(?:"([^"]+)"|'([^']+)'|([^\s:#][^:#]*)):\s*(.*)$/.exec(line); + if (!pair) continue; + const indent = pair[1].length; + while (yamlPath.length && yamlPath.at(-1)!.indent >= indent) yamlPath.pop(); + const key = (pair[2] ?? pair[3] ?? pair[4]).trim(); + const parents = yamlPath.map(entry => entry.key); + const container = key === "container" && parents[0] === "jobs" && parents.length === 2; + const image = key === "image" && parents[0] === "jobs" && ( + parents.length === 3 && parents[2] === "container" || + parents.length === 4 && parents[2] === "services" + ); + const uses = key === "uses" && ( + parents[0] === "jobs" && parents.length === 3 && parents[2] === "steps" || + parents[0] === "runs" && parents.length === 2 && parents[1] === "steps" + ); + if ((container || image || uses) && pair[5]) { + const value = pair[5].replace(/\s+#.*$/, "").replace(/^(['"])(.*)\1$/, "$2"); + const ref = parseDockerImageRef(uses ? value.replace(/^docker:\/\//, "") : value); + if (ref) { collectDockerRef(ref, relPath, filters); workflowLines.add(lineNumber); } + } + yamlPath.push({indent, key}); } + continue; } @@ -751,33 +812,35 @@ export async function updates(opts: UpdatesOptions = {}): Promise { const content = fileContents.get(file)!; const relPath = toRelPath(file); const fileType = isDockerfile(filename) ? "dockerfile" : "compose"; - const filters = await resolveFileConfig(dirname(file)); - dockerFileData[relPath] = {absPath: file, content, fileType}; - collectDockerRefs(content, relPath, [getExtractionRegex(filename)], filters); + const filters = await resolveDirConfig(dirname(file)); + fileData[relPath] = {absPath: file, content, fileType}; + for (const {ref} of extractDockerRefs(content, getExtractionRegex(filename))) { + collectDockerRef(ref, relPath, filters); + } continue; } if (isMakeFileName(filename)) { const content = fileContents.get(file)!; const relPath = toRelPath(file); - const filters = await resolveFileConfig(dirname(file)); - makeFileData[relPath] = {absPath: file, content}; + const filters = await resolveDirConfig(dirname(file)); + fileData[relPath] = {absPath: file, content, fileType: "make"}; deps.make ??= {}; const makeShared = { - projectDir: dirname(file), filePin: filters.pin, filePinNoDowngrade: filters.pinNoDowngrade, + projectDir: dirname(file), versionConfig: filters.versionConfig, + filePin: filters.pin, filePinNoDowngrade: filters.pinNoDowngrade, fileCooldownDays: filters.cooldownDays, }; for (const {installPath, version} of parseMakeGoInstalls(content)) { if (!canInclude(installPath, "make", filters.include, filters.exclude, "make", "go")) continue; - // The version is part of the key: a Makefile may install one tool at two versions. - const key = `${relPath}${fieldSep}${installPath}${fieldSep}${version}`; + const key = dependencyKey(relPath, installPath, version); if (deps.make[key]) continue; deps.make[key] = {old: stripv(version), oldOrig: version} as Dep; makeDepInfos.push({kind: "go", key, name: installPath, oldSpec: `${installPath}@${version}`, installPath, version, ...makeShared}); } for (const image of parseMakeDockerImages(content)) { if (!canInclude(image.writtenImage, "make", filters.include, filters.exclude, "make", "docker")) continue; - const key = `${relPath}${fieldSep}${image.writtenImage}${fieldSep}${image.ref.tag}`; + const key = dependencyKey(relPath, image.writtenImage, image.ref.tag); if (deps.make[key]) continue; const parsed = parseDockerTag(image.ref.tag); if (!parsed) continue; @@ -794,13 +857,13 @@ export async function updates(opts: UpdatesOptions = {}): Promise { deps[mode] ??= {}; const workspaceDir = dirname(resolve(file)); const workContent = fileContents.get(file)!; - goWorkData = {file, content: workContent}; const goWork = parseGoWork(workContent); - const [{modeConfig, modeInclude, modeExclude, pin}, useReads] = await Promise.all([ - resolveModeFilters(workspaceDir), + const [{modeConfig, include: modeInclude, exclude: modeExclude, versionConfig, pin, cooldownDays}, useReads] = await Promise.all([ + resolveDirConfig(workspaceDir), pMap(goWork.use, async (usePath) => { - const modPath = resolve(join(workspaceDir, usePath, "go.mod")); + const modPath = resolveGoWorkModule(workspaceDir, usePath); + if (!modPath) return null; try { return {usePath, modPath, content: await readFile(modPath, "utf8")}; } catch { @@ -809,48 +872,51 @@ export async function updates(opts: UpdatesOptions = {}): Promise { }, {concurrency}), ]); const dependencyTypes = resolveDepTypes(mode, modeConfig); - modeConfigs[mode] = {modeConfig, projectDir: workspaceDir, pin}; + const workspacePath = workspaceMemberPath(mode, file, "."); + const modeContext = modeCtx({modeConfig, versionConfig, pin, cooldownDays}, workspaceDir); + registerModeContext(mode, workspacePath, modeContext); + goWorkFiles.push({file, content: workContent, memberPath: workspacePath}); for (const entry of useReads) { if (!entry) continue; const {usePath, modPath, content: modContent} = entry; const parsed = parseGoMod(modContent); - const modProjectDir = dirname(modPath); - goModFiles.push({absPath: modPath, content: modContent, projectDir: modProjectDir, memberPath: usePath}); + const memberPath = workspaceMemberPath(mode, file, usePath); + registerModeContext(mode, memberPath, modeContext); + goModFiles.push({absPath: modPath, content: modContent, projectDir: dirname(modPath), memberPath}); - collectDeps(mode, parsed, usePath === "." ? "" : `|${usePath}`, dependencyTypes, modeInclude, modeExclude); + collectDeps(mode, parsed, memberPath === "." ? "" : `|${memberPath}`, dependencyTypes, modeInclude, modeExclude); } for (const [name, value] of Object.entries(goWork.replace)) { if (canInclude(name, mode, modeInclude, modeExclude, "replace")) { - addDep(mode, "replace", "", name, shortenGoVersion(value), stripv(value)); + addDep(mode, "replace", workspacePath === "." ? "" : `|${workspacePath}`, name, shortenGoVersion(value), stripv(value)); } } continue; } - // Skip only manifests already consumed as workspace members; unrelated ones (e.g. from a - // second `-f` directory) fall through to be processed as plain files. - if (filename === "go.mod" && goModFiles.some(m => m.absPath === resolve(file))) continue; - if (filename === "package.json" && pnpmMemberFiles.some(m => m.absPath === resolve(file))) continue; - if (filename === "Cargo.toml" && cargoMemberFiles.some(m => m.absPath === resolve(file))) continue; + if (filename === "go.mod" && goModFiles.some(member => member.absPath === resolve(file))) continue; + if (filename === "package.json" && pnpmMemberFiles.some(member => member.absPath === resolve(file))) continue; + if (filename === "Cargo.toml" && cargoMemberFiles.some(member => member.absPath === resolve(file))) continue; if (filename === "Cargo.toml") { deps[mode] ??= {}; const cargoContent = fileContents.get(file)!; - const cargoParsed = parsedCargoToml.get(file) ?? parseToml(cargoContent); + const cargoParsed = cargoWorkspaceFiles.get(file) ?? parseToml(cargoContent); const workspaceDir = dirname(resolve(file)); const lockPath = findUpSync(["Cargo.lock"], workspaceDir).get("Cargo.lock"); const wsMembers = (cargoParsed.workspace as Record)?.members; const isWorkspace = Array.isArray(wsMembers) && wsMembers.length; - const [{modeConfig, modeInclude, modeExclude, pin}, lockContent, members] = await Promise.all([ - resolveModeFilters(workspaceDir), + const [filters, lockContent, members] = await Promise.all([ + resolveDirConfig(workspaceDir), lockPath ? readFile(lockPath, "utf8") : Promise.resolve(null), isWorkspace ? resolveWorkspaceMembers(wsMembers, workspaceDir, "Cargo.toml", concurrency) : Promise.resolve([] as WorkspaceMember[]), ]); + const {modeConfig, include: modeInclude, exclude: modeExclude} = filters; const dependencyTypes = resolveDepTypes(mode, modeConfig); const lockedVersions = lockContent ? parseCargoLock(lockContent) : new Map(); @@ -860,14 +926,11 @@ export async function updates(opts: UpdatesOptions = {}): Promise { if (typeof obj !== "object" || Array.isArray(obj)) continue; for (const [name, value] of Object.entries(obj)) { if (!canInclude(name, mode, modeInclude, modeExclude, depType)) continue; - // `registry` joins `git` and `path` as a source this tool cannot resolve: crates.io - // would 404 on the name, or worse hit a same-named public crate. if (typeof value === "object" && value !== null && "version" in value && !("git" in value) && !("path" in value) && !("registry" in value)) { const versionStr = (value as Record).version; - // A renamed dep keeps the manifest key so the rewrite finds it, lookups use `package`. const crate = (value as Record).package || name; if (validRange(cargoToNpmRange(versionStr))) { - if (crate !== name) cargoCrates.set(depKey(depType, typePrefix, name), crate); + if (crate !== name) cargoCrates.set(manifestDependencyKey(depType, typePrefix, name), crate); addDep(mode, depType, typePrefix, name, findLockedVersion(lockedVersions, crate, versionStr) ?? normalizeRange(cargoToNpmRange(versionStr)), versionStr); } } else if (typeof value === "string" && validRange(cargoToNpmRange(value))) { @@ -878,22 +941,47 @@ export async function updates(opts: UpdatesOptions = {}): Promise { }; if (isWorkspace) { - modeConfigs[mode] = {modeConfig, projectDir: workspaceDir, pin}; - collectCargoDeps(cargoParsed, ""); - cargoMemberFiles.push({absPath: resolve(file), content: cargoContent, memberPath: "."}); + const workspacePath = workspaceMemberPath(mode, file, "."); + const modeContext = modeCtx(filters, workspaceDir); + registerModeContext(mode, workspacePath, modeContext); + collectCargoDeps(cargoParsed, workspacePath === "." ? "" : `|${workspacePath}`); + cargoMemberFiles.push({absPath: resolve(file), content: cargoContent, memberPath: workspacePath}); for (const member of members) { - cargoMemberFiles.push(member); - collectCargoDeps(parseFile(member.absPath, () => parseToml(member.content)), `|${member.memberPath}`); + const memberPath = workspaceMemberPath(mode, file, member.memberPath); + registerModeContext(mode, memberPath, modeContext); + cargoMemberFiles.push({...member, memberPath}); + collectCargoDeps(parseFile(member.absPath, () => parseToml(member.content)), `|${memberPath}`); } } else { - // Track each non-workspace Cargo.toml per file so several of them never - // overwrite each other. - collectCargoDeps(cargoParsed, addPlainFile(mode, file, cargoContent, workspaceDir, modeConfig, pin)); + collectCargoDeps(cargoParsed, addPlainFile(mode, file, cargoContent, workspaceDir, filters)); } continue; } + if (filename === "package.json" && npmWorkspaceFiles.has(file)) { + deps[mode] ??= {}; + const workspaceDir = dirname(resolve(file)); + const rootContent = fileContents.get(file)!; + const rootPkg = npmWorkspaceFiles.get(file)!; + const rawPackagePatterns = Array.isArray(rootPkg.workspaces) ? rootPkg.workspaces : rootPkg.workspaces?.packages; + const packagePatterns = Array.isArray(rawPackagePatterns) ? + rawPackagePatterns.filter((pattern: unknown): pattern is string => typeof pattern === "string") : []; + const [filters, members] = await Promise.all([ + resolveDirConfig(workspaceDir), + resolveWorkspaceMembers(packagePatterns, workspaceDir, "package.json", concurrency), + ]); + const dependencyTypes = resolveDepTypes(mode, filters.modeConfig); + collectNpmWorkspaceMember(file, workspaceDir, { + absPath: resolve(file), content: rootContent, memberPath: ".", + }, rootPkg, dependencyTypes, filters); + for (const member of members) { + collectNpmWorkspaceMember(file, workspaceDir, member, + parseFile(member.absPath, () => JSON.parse(member.content)), dependencyTypes, filters); + } + continue; + } + if (filename === "pnpm-workspace.yaml") { deps[mode] ??= {}; const workspaceDir = dirname(resolve(file)); @@ -901,29 +989,43 @@ export async function updates(opts: UpdatesOptions = {}): Promise { const packagePatterns = parsePnpmWorkspace(wsContent); const rootPkgPath = join(workspaceDir, "package.json"); - const [{modeConfig, modeInclude, modeExclude, pin}, rootContent, members] = await Promise.all([ - resolveModeFilters(workspaceDir), + const [filters, rootContent, members] = await Promise.all([ + resolveDirConfig(workspaceDir), tryOrNull(readFile(rootPkgPath, "utf8")), resolveWorkspaceMembers(packagePatterns, workspaceDir, "package.json", concurrency), ]); - const dependencyTypes = resolveDepTypes(mode, modeConfig); - modeConfigs[mode] = {modeConfig, projectDir: workspaceDir, pin}; + const dependencyTypes = resolveDepTypes(mode, filters.modeConfig); + const workspaceManifestPath = workspaceMemberPath(mode, file, filename); + registerModeContext(mode, workspaceManifestPath, modeCtx(filters, workspaceDir)); - pnpmCatalogFiles.push({absPath: resolve(file), content: wsContent, memberPath: filename}); + pnpmCatalogFiles.push({absPath: resolve(file), content: wsContent, memberPath: workspaceManifestPath}); for (const {type, name, value} of pnpmCatalogEntries(wsContent)) { - if (canInclude(name, mode, modeInclude, modeExclude, type)) addNpmDep(type, `|${filename}`, name, value); + if (canInclude(name, mode, filters.include, filters.exclude, type, mode, parseNpmAlias(value)?.name ?? name)) { + addNpmDep(manifestDependencyKey(type, `|${workspaceManifestPath}`, name), name, value); + } + } + for (const {selector, value} of pnpmWorkspaceOverrides(wsContent)) { + const packages = selector.match(/(?:^|>)(?:@[^/>\s]+\/[^@>\s]+|[^@>\s]+)(?=@|>|$)/g); + const packageName = packages?.at(-1)?.replace(/^>/, "") ?? selector; + const type = "pnpm-workspace.overrides"; + if (!canInclude(selector, mode, filters.include, filters.exclude, type, mode, packageName)) continue; + const identity = JSON.stringify(selector); + const key = manifestDependencyKey(type, `|${workspaceManifestPath}`, selector, identity); + npmPublishedNames.set(key, packageName); + pnpmWorkspaceOverrideKeys.set(key, selector); + addNpmDep(key, selector, value); } if (rootContent !== null) { const rootPkg = parseFile(rootPkgPath, () => JSON.parse(rootContent)); - pnpmMemberFiles.push({absPath: resolve(rootPkgPath), content: rootContent, memberPath: "."}); - collectDeps(mode, rootPkg, "", dependencyTypes, modeInclude, modeExclude); + collectNpmWorkspaceMember(file, workspaceDir, { + absPath: resolve(rootPkgPath), content: rootContent, memberPath: ".", + }, rootPkg, dependencyTypes, filters); } for (const member of members) { - const memberPkg = parseFile(member.absPath, () => JSON.parse(member.content)); - pnpmMemberFiles.push(member); - collectDeps(mode, memberPkg, `|${member.memberPath}`, dependencyTypes, modeInclude, modeExclude); + collectNpmWorkspaceMember(file, workspaceDir, member, + parseFile(member.absPath, () => JSON.parse(member.content)), dependencyTypes, filters); } continue; @@ -932,19 +1034,15 @@ export async function updates(opts: UpdatesOptions = {}): Promise { deps[mode] ??= {}; const projectDir = dirname(resolve(file)); - const {modeConfig, modeInclude, modeExclude, pin} = await resolveModeFilters(projectDir); + const filters = await resolveDirConfig(projectDir); + const {modeConfig, include: modeInclude, exclude: modeExclude} = filters; const dependencyTypes = resolveDepTypes(mode, modeConfig); const content = fileContents.get(file)!; - const typePrefix = addPlainFile(mode, file, content, projectDir, modeConfig, pin); - - const pkg = parseFile(file, () => { - if (mode === "npm") return JSON.parse(content); - if (mode === "pypi") return parseToml(content); - if (mode === "go") return parseGoMod(content); - return {}; - }); + const pkg = parseFile(file, () => mode === "npm" ? JSON.parse(content) : + mode === "pypi" ? parseToml(content) : parseGoMod(content)); + const typePrefix = addPlainFile(mode, file, content, projectDir, filters); collectDeps(mode, pkg, typePrefix, dependencyTypes, modeInclude, modeExclude); } @@ -954,90 +1052,43 @@ export async function updates(opts: UpdatesOptions = {}): Promise { } const fetchTasks: Array> = []; - // The abbreviated npm packument carries no publish dates, so cooldown needs the full one, - // which is roughly twice the size. Decided once per run because the doc is cached by URL - // and shared across every dep that reads it, but only from npm's own cooldown sources. - // The published name the lookup resolves options under, which the manifest key answers for in - // neither direction: an `npm:` alias names another package, and a selector key is no name at all. - const npmIdentity = (key: string, name: string) => npmAliases.get(key)?.name ?? + const npmIdentity = (key: string, name: string) => npmPublishedNames.get(key) ?? npmAliases.get(key)?.name ?? (selectorTypes.has(key.split(fieldSep)[0].split("|")[0]) ? resolutionsBasePackage(name) : name); - const npmNeedsDates = Boolean(cooldownDaysFor(modeConfigs.npm?.modeConfig.cooldown)) || - (plainFiles.npm ?? []).some(entry => entry.modeCooldownDays) || - (overridesHaveCooldown && Object.keys(deps.npm ?? {}).some(key => - getVersionOpts("npm", npmIdentity(key, key.split(fieldSep)[1])).cooldownOverride)); - const argsForNpm = {registry: config.registry, needsDates: npmNeedsDates}; - - for (const [mode, modeConfigEntry] of Object.entries(modeConfigs)) { - const hasDeps = deps[mode] && Object.keys(deps[mode]).length > 0; - const hasUrlDeps = mode === "npm" && Object.keys(maybeUrlDeps).length > 0; - if (!hasDeps && !hasUrlDeps) continue; - const {modeConfig: defaultModeConfig, projectDir: defaultProjectDir, pin: defaultPin} = modeConfigEntry; - const defaultCooldownDays = cooldownDaysFor(defaultModeConfig.cooldown); + const argsForNpm = {needsDates: (modeContextsBySuffix.npm?.values() ?? [][Symbol.iterator]()).some(entry => + entry.cooldownDays || entry.versionConfig.hasCooldownOverride)}; + + for (const [mode, modeContexts] of Object.entries(modeContextsBySuffix)) { + if (!Object.keys(deps[mode] ?? {}).length && (mode !== "npm" || !Object.keys(maybeUrlDeps).length)) continue; fetchTasks.push((async () => { - // Non-workspace manifests with a disambiguating `|memberPath` type suffix - // each carry their own config/projectDir/pin/cooldown; the empty-suffix - // case (single manifest or workspace root) uses the mode-level defaults. - const ctxBySuffix = new Map(); - for (const entry of plainFiles[mode] ?? []) { - if (entry.memberPath !== ".") ctxBySuffix.set(`|${entry.memberPath}`, entry); - } - const defaultCtx = {modeConfig: defaultModeConfig, projectDir: defaultProjectDir, pin: defaultPin, modeCooldownDays: defaultCooldownDays}; + const modeConfigEntry = modeContexts.values().next().value!; const ctxForType = (type: string) => { const barIdx = type.indexOf("|"); - return (barIdx !== -1 && ctxBySuffix.get(type.slice(barIdx))) || defaultCtx; + return modeContexts.get(barIdx === -1 ? "" : type.slice(barIdx)) ?? modeConfigEntry; }; const npmFollowUps = new Map}>(); - // Safety net for deps that bypass findNewVersion (URL tarballs, JSR - // follow-ups). findNewVersion's per-version cooldown filter handles the - // common case; this catches the rest. - const dropIfTooNew = (modeDeps: Deps) => { - for (const [k, {date}] of Object.entries(modeDeps)) { - if (!date) continue; - const [type, name] = k.split(fieldSep); - const {modeCooldownDays} = ctxForType(type); - if (!modeCooldownDays && !overridesHaveCooldown) continue; - const cd = getVersionOpts(mode, mode === "npm" ? npmIdentity(k, name) : name).cooldownOverride ?? modeCooldownDays; - if (cd && !passesCooldown(date, cd, now)) delete modeDeps[k]; - } - }; - const modeDeps = deps[mode]; const lookupDep = async (key: string, type: string, name: string) => { const baseT = baseType(type); - const {modeConfig, projectDir, pin, modeCooldownDays} = ctxForType(type); + const {modeConfig, versionConfig, projectDir, pin, cooldownDays} = ctxForType(type); const dep = modeDeps[key]; const npmAlias = npmAliases.get(key); - let info: PackageInfo; - if (mode === "npm") { - if (dep.oldOrig && isJsr(dep.oldOrig)) { - info = await fetchJsrInfo(name, ctx); - } else if (dep.oldOrig && isLocalDep(dep.oldOrig)) { - const localInfo = await tryOrNull(fetchNpmInfo(name, baseT, modeConfig, argsForNpm, ctx, projectDir)); - if (!localInfo) { delete modeDeps[key]; return; } - info = localInfo; - } else { - info = await fetchNpmInfo(npmAlias?.name ?? name, baseT, modeConfig, argsForNpm, ctx, projectDir, dep.old); - } - } else if (mode === "go") { - info = await fetchGoProxyInfo(name, baseT, dep.oldOrig || dep.old, projectDir, ctx, goNoProxy); - } else if (mode === "cargo") { - info = await fetchCratesIoInfo(cargoCrates.get(key) ?? name, ctx); - } else { - info = await fetchPypiInfo(name, ctx); - } + const info = mode === "npm" ? dep.oldOrig && isJsr(dep.oldOrig) ? fetchJsrInfo(name, ctx) : + fetchNpmInfo(npmIdentity(key, name), baseT, modeConfig, argsForNpm, ctx, projectDir, dep.old) : + mode === "go" ? fetchGoProxyInfo(name, type, dep.oldOrig || dep.old, projectDir, ctx, goNoProxy) : + mode === "cargo" ? fetchCratesIoInfo(cargoCrates.get(key) ?? name, ctx) : fetchPypiInfo(name, ctx); - const [data, registry] = info; + const [rawData, registry] = await info; + let data = rawData; if (data.error) throw new Error(data.error); - // A go module answers to its `/vN` short name too, which is what `-i`/`-e` accept. A - // `packageManager` names its own identity, so corepack's `yarn` keeps resolving as - // `@yarnpkg/cli` while options and pins stay on the `yarn` the manifest and the row show. const identity = baseT === "packageManager" ? name : data.name; - const {names, useGreatest, usePre, useRel, semvers, allowDowngrade: allowDown, cooldownOverride} = getVersionOpts(mode, identity); + const { + useGreatest, usePre, useRel, semvers, allowDowngrade: allowDown, allowedVersions, + pinnedRange, pinNoDowngrade, cooldownDays: depCooldownDays, + } = resolveVersionOpts(versionConfig, mode, identity, name, pin, modeConfig.pinNoDowngrade, cooldownDays); + data = filterVersionData(data, mode, allowedVersions); const {old: oldRange, oldOrig} = dep; - const {pinnedRange, pinNoDowngrade} = resolvePin(names, pin, modeConfig.pinNoDowngrade); - const depCooldownDays = cooldownOverride ?? modeCooldownDays; const newVersion = findNewVersion(data, { usePre, useRel, useGreatest, semvers, range: oldRange, mode, pinnedRange, pinNoDowngrade, allowDowngrade: allowDown, cooldownDays: depCooldownDays || undefined, now: depCooldownDays ? now : undefined, @@ -1049,27 +1100,20 @@ export async function updates(opts: UpdatesOptions = {}): Promise { } else if (mode === "cargo" && newVersion && oldOrig) { newRange = updateCargoRange(oldOrig, newVersion); } else if (newVersion) { - if (oldOrig && isLocalDep(oldOrig)) { - newRange = String(getNpmrc(projectDir)["save-exact"]) === "true" ? newVersion : `^${newVersion}`; - } else if (oldOrig && isJsr(oldOrig)) { + if (oldOrig && isJsr(oldOrig)) { const match = jsrSpecifierRe.exec(oldOrig); if (match) newRange = `${match[1]}${newVersion}`; else if (oldOrig.startsWith("jsr:")) newRange = `jsr:${newVersion}`; } else if (npmAlias) { - // Only the aliased package's range moves, and the `npm:@` prefix is written - // back with it so the manifest keeps aliasing the key it always did. newRange = `npm:${npmAlias.name}@${updateVersionRange(oldRange, newVersion, npmAlias.range, baseT)}`; } else { newRange = updateVersionRange(oldRange, newVersion, oldOrig, baseT); } } - // The pypi writer declines a rewrite leaving any specifier unsatisfied, so a version it - // would refuse must not be offered either. const spec = pypiSpecs.get(key); if (!newVersion || newVersion === oldRange || oldOrig && (oldOrig === newRange) || spec && !updateRequirement(spec, oldOrig || oldRange, newRange)) { - // Without this, a version no range could be written for reads as already current. if (config.verbose && newVersion && newVersion !== oldRange) { console.error(`${timestamp()} ${magenta("SKIP")} ${name}: ${oldOrig || oldRange} can not be rewritten to ${newVersion}`); } @@ -1078,21 +1122,15 @@ export async function updates(opts: UpdatesOptions = {}): Promise { } const date: string = (mode === "pypi" ? data.releases?.[newVersion]?.[0]?.upload_time_iso_8601 : - mode === "go" ? data.Time : - mode === "cargo" ? data.time?.[newVersion] : "") || ""; + mode === "go" ? data.Time : mode === "cargo" ? data.time?.[newVersion] : "") || ""; dep.new = newRange; if (oldOrig && isJsr(oldOrig)) dep.newPrint = newVersion; if (mode === "npm") { - npmFollowUps.set(key, {name: npmAlias?.name ?? name, promise: fetchNpmVersionInfo(data.name, newVersion, modeConfig, argsForNpm, ctx, projectDir)}); - } else if (mode === "pypi") { - dep.info = getInfoUrl(data, registry, data.info.name); - } else if (mode === "go") { - dep.info = getGoInfoUrl(data.newPath || name); - } else if (mode === "cargo") { - dep.info = `https://crates.io/crates/${data.name}`; - } + npmFollowUps.set(key, {name: npmIdentity(key, name), promise: fetchNpmVersionInfo(data.name, newVersion, modeConfig, argsForNpm, ctx, projectDir)}); + } else dep.info = mode === "pypi" ? getInfoUrl(data, registry, data.info.name) : + mode === "go" ? getGoInfoUrl(data.newPath || name) : `https://crates.io/crates/${data.name}`; setDepAge(dep, date); }; @@ -1102,8 +1140,7 @@ export async function updates(opts: UpdatesOptions = {}): Promise { try { await lookupDep(key, type, name); } catch (err) { - delete modeDeps[key]; - addError(mode, type, name, err); + rejectDep(mode, key, err); } }, {concurrency}); @@ -1112,7 +1149,7 @@ export async function updates(opts: UpdatesOptions = {}): Promise { const dep = modeDeps[key]; if (!dep) return; dep.info = getInfoUrl({repository: followUp.repository, homepage: followUp.homepage}, null, name); - if (followUp.date) setDepAge(dep, followUp.date); + setDepAge(dep, followUp.date); })); if (mode === "npm" && Object.keys(maybeUrlDeps).length) { @@ -1123,7 +1160,7 @@ export async function updates(opts: UpdatesOptions = {}): Promise { addKeyError("npm", key, err); return null; } - }, {concurrency})).filter(r => r !== null); + }, {concurrency})).filter(result => result !== null); for (const {key, newRange, user, repo, oldRef, newRef, newDate} of results) { const dep: Dep = modeDeps[key] = { @@ -1133,32 +1170,43 @@ export async function updates(opts: UpdatesOptions = {}): Promise { newPrint: hashRe.test(newRef) ? newRef.substring(0, 7) : newRef, info: `https://github.com/${user}/${repo}`, }; - if (newDate) setDepAge(dep, newDate); + setDepAge(dep, newDate); } } - dropIfTooNew(modeDeps); + for (const [key, {date}] of Object.entries(modeDeps)) { + if (!date) continue; + const [type, name] = key.split(fieldSep); + const {cooldownDays, versionConfig} = ctxForType(type); + if (!cooldownDays && !versionConfig.hasCooldownOverride) continue; + const identity = mode === "npm" ? npmIdentity(key, name) : name; + const effectiveCooldownDays = getVersionOpts(versionConfig, mode, identity, name).cooldownOverride ?? cooldownDays; + if (effectiveCooldownDays && !passesCooldown(date, effectiveCooldownDays, now)) delete modeDeps[key]; + } })()); } if (actionDepInfos.length) { fetchTasks.push((async () => { - const depsByRepo = Map.groupBy(actionDepInfos, info => `${info.apiUrl}/${info.owner}/${info.repo}`); - - await pMap(depsByRepo.values(), async (infos) => { + await pMap(Map.groupBy(actionDepInfos, info => `${info.apiUrl}/${info.owner}/${info.repo}`).values(), async (infos) => { const {apiUrl, owner, repo} = infos[0]; - let tags: Array; + const versionConsumers = infos.filter(info => info.isHash ? !info.comment : isVersionLikeRef(info.ref)); + const tagRefs = [ + ...versionConsumers.map(info => info.ref), + ...(apiUrl === defaultApiUrls.forgeapi ? [] : infos.filter(info => info.isHash && info.comment) + .map(info => info.comment)), + ]; + let tags: Array = []; try { - tags = await fetchActionTags(apiUrl, owner, repo, ctx, infos.map(info => info.ref)); + if (tagRefs.length) { + tags = await fetchActionTags(apiUrl, owner, repo, ctx, tagRefs, Boolean(versionConsumers.length)); + } } catch (err) { for (const info of infos) { - delete deps.actions[info.key]; - addKeyError("actions", info.key, err); + rejectDep("actions", info.key, err); } return; } - // Candidates are the versions tags parse to, never the tag text: a `+meta` or leading-zero - // tag would never map back to its own entry. const versions: string[] = []; const tagByVersion = new Map(); const entryByName = new Map(); @@ -1166,38 +1214,39 @@ export async function updates(opts: UpdatesOptions = {}): Promise { for (const tag of tags) { entryByName.set(tag.name, tag); const version = githubActionsVersioning.parse(tag.name)?.version; - if (version) { + if (version && tag.isStable !== false) { const existing = tagByVersion.get(version); - // `v3.19` and `v3.19.0` are the same version; the more precise tag names it. if (!existing) versions.push(version); if (!existing || tag.name.length > existing.length) tagByVersion.set(version, tag.name); } if (tag.commitSha) commitShaToTag.set(tag.commitSha, tag.name); } - // Caches the promise, so the several infos of one repo resolving to the - // same commit share a single request instead of racing their own. const getDate = memoizeAsync((commitSha: string) => fetchActionTagDate(apiUrl, owner, repo, commitSha, ctx)); + const getExactDigest = memoizeAsync(async (exactRef: string): Promise => { + const tagged = entryByName.get(exactRef)?.commitSha; + if (tagged) return tagged; + const path = apiUrl === defaultApiUrls.forgeapi ? "commits" : "branches"; + const url = `${apiUrl}/repos/${owner}/${repo}/${path}/${encodeURIComponent(exactRef)}`; + const response = await fetchForge(url, ctx); + if (response.status === 404) return ""; + if (!response.ok) throw new Error(`Unable to fetch ${owner}/${repo}@${exactRef}`); + const body = await response.json(); + return typeof body?.sha === "string" ? body.sha : typeof body?.id === "string" ? body.id : + typeof body?.commit?.sha === "string" ? body.commit.sha : typeof body?.commit?.id === "string" ? body.commit.id : ""; + }); - // Cooldown-aware selection: when cooldown is active, pick the highest - // version, fetch its commit date, and if it's too new, exclude it and - // retry. Bounded loop avoids pathological cases (e.g. all versions - // released within the cooldown window). - async function pickVersion(opts: Parameters[2]): Promise<{version: string, tag: string, commitSha: string, date: string} | null> { - // A tag's date costs a request, so findVersion has none to gate on and would reject - // every candidate under an active cooldown. The gate runs per pick below. + async function pickVersion(opts: Parameters[2], sourceVersions: Array): Promise<{version: string, tag: string, commitSha: string, date: string} | null> { const selectOpts = {...opts, cooldownDays: undefined, now: undefined}; const denylist = new Set(); for (let attempt = 0; attempt < 20; attempt++) { - const candidates = denylist.size ? versions.filter(v => !denylist.has(v)) : versions; + const candidates = denylist.size ? sourceVersions.filter(version => !denylist.has(version)) : sourceVersions; const picked = findVersion({}, candidates, selectOpts); if (!picked) return null; const tag = tagByVersion.get(picked)!; const commitSha = entryByName.get(tag)?.commitSha || ""; if (!opts.cooldownDays) return {version: picked, tag, commitSha, date: ""}; const date = commitSha ? await getDate(commitSha) : ""; - // An empty date is the commit carrying none, which holds the candidate back; not - // knowing is a failed run and must not read as either. if (date === undefined) throw new Error(`Unable to fetch the commit date for ${owner}/${repo}@${tag}`); if (passesCooldown(date, opts.cooldownDays, opts.now)) return {version: picked, tag, commitSha, date}; denylist.add(picked); @@ -1205,31 +1254,40 @@ export async function updates(opts: UpdatesOptions = {}): Promise { return null; } - const updateAction = async ({key, host, ref, comment, name: actionName, isHash, filePin, filePinNoDowngrade, fileCooldownDays}: ActionDepInfo) => { + const updateAction = async ({key, host, ref, comment, name: actionName, isHash, versionConfig, filePin, filePinNoDowngrade, fileCooldownDays}: ActionDepInfo) => { const dep = deps.actions[key]; const infoUrl = `https://${host || "github.com"}/${owner}/${repo}`; - const {pinnedRange: actionPin, pinNoDowngrade} = resolvePin([actionName], filePin, filePinNoDowngrade); + if (isHash && comment) { + const newDigest = await getExactDigest(comment); + if (!newDigest || newDigest.startsWith(ref) || ref.startsWith(newDigest)) { delete deps.actions[key]; return; } + dep.old = comment; + dep.new = comment; + dep.oldDigest = ref; + dep.newDigest = newDigest; + dep.digestOnly = true; + dep.info = infoUrl; + const newDate = await tryOrNull(getDate(newDigest)); + setDepAge(dep, newDate); + return; + } - // A sha pin's version is whatever its trailing comment names, failing that the tag - // carrying the commit; without one every candidate looks like an upgrade, an older commit - // included. A branch ref coerces to a version but must keep its text, or a release tag - // replaces the pin. let oldRef = ref; if (isHash) { - // abbreviated pins need a prefix scan, the map is keyed by full sha oldRef = comment || commitShaToTag.get(ref) || commitShaToTag.entries().find(([sha]) => sha.startsWith(ref))?.[1] || ""; } else if (!isVersionLikeRef(ref)) { oldRef = ""; } if (!oldRef) { delete deps.actions[key]; return; } - const {useGreatest, usePre, useRel, semvers, allowDowngrade: allowDown, cooldownOverride} = getVersionOpts("actions", actionName); - const actionCooldownDays = cooldownOverride ?? fileCooldownDays; + const { + useGreatest, usePre, useRel, semvers, allowDowngrade: allowDown, allowedVersions, + pinnedRange, pinNoDowngrade, cooldownDays: actionCooldownDays, + } = resolveVersionOpts(versionConfig, "actions", actionName, actionName, filePin, filePinNoDowngrade, fileCooldownDays); const result = await pickVersion({ range: oldRef, semvers, useGreatest, usePre, useRel, allowDowngrade: allowDown, versioning: githubActionsVersioning, - pinnedRange: actionPin, pinNoDowngrade, + pinnedRange, pinNoDowngrade, cooldownDays: actionCooldownDays || undefined, now: actionCooldownDays ? now : undefined, - }); + }, allowedVersions ? versions.filter(version => versionAllowed("actions", version, allowedVersions)) : versions); if (!result) { delete deps.actions[key]; return; } const {tag: newTag, commitSha: newCommitSha, date} = result; @@ -1250,18 +1308,15 @@ export async function updates(opts: UpdatesOptions = {}): Promise { } dep.info = infoUrl; - // Only a cooldown run has fetched the date already, otherwise it takes a request. - // The age is cosmetic, so an undeterminable date goes unprinted rather than dropping the update. const newDate = date || (newCommitSha ? await tryOrNull(getDate(newCommitSha)) : ""); - if (newDate) setDepAge(dep, newDate); + setDepAge(dep, newDate); }; await pMap(infos, async (info) => { try { await updateAction(info); } catch (err) { - delete deps.actions[info.key]; - addKeyError("actions", info.key, err); + rejectDep("actions", info.key, err); } }, {concurrency}); }, {concurrency}); @@ -1272,10 +1327,7 @@ export async function updates(opts: UpdatesOptions = {}): Promise { if (dockerDepInfos.length) { fetchTasks.push((async () => { - const depsByImage = Map.groupBy(dockerDepInfos, info => info.fullImage); - - await pMap(depsByImage.entries(), async ([fullImage, infos]) => { - // An image on a registry other than Docker Hub has no lookup to attempt yet. + await pMap(Map.groupBy(dockerDepInfos, info => info.fullImage).entries(), async ([fullImage, infos]) => { if (infos[0].ref.registry) { for (const info of infos) delete deps.docker[info.key]; return; @@ -1286,29 +1338,37 @@ export async function updates(opts: UpdatesOptions = {}): Promise { data = fetchedData; } catch (err) { for (const info of infos) { - delete deps.docker[info.key]; - addKeyError("docker", info.key, err); + rejectDep("docker", info.key, err); } return; } - const {names, semvers, usePre, useRel, cooldownOverride} = getVersionOpts("docker", fullImage); for (const info of infos) { const dep = deps.docker[info.key]; const oldTag = dep.oldOrig || dep.old; - // findDockerVersion only moves a tag up, so a renovate-derived pin has nothing to suppress. - const {pinnedRange} = resolvePin(names, info.filePin); - const dockerCooldownDays = cooldownOverride ?? info.fileCooldownDays; - const result = findDockerVersion( - data.tags, oldTag, semvers, + const {semvers, usePre, useRel, allowedVersions, pinnedRange, cooldownDays: dockerCooldownDays} = + resolveVersionOpts(info.versionConfig, "docker", fullImage, fullImage, info.filePin, undefined, info.fileCooldownDays); + const tags = filterVersionData(data, "docker", allowedVersions).tags; + const result = !info.ref.digestOnly && parseDockerTag(oldTag) ? findDockerVersion( + tags, oldTag, semvers, dockerCooldownDays || undefined, dockerCooldownDays ? now : undefined, pinnedRange, usePre, useRel, - ); - if (!result) { delete deps.docker[info.key]; continue; } + ) : null; + const newTag = result?.newTag ?? oldTag; + if (info.ref.digest) { + const newDigest = await fetchDockerTagDigest(info.ref.namespace, info.ref.repo, newTag, ctx); + if (!newDigest || newDigest === info.ref.digest && !result) { delete deps.docker[info.key]; continue; } + dep.oldDigest = info.ref.digest; + dep.newDigest = newDigest; + dep.digestOnly = info.ref.digestOnly; + } else if (!result) { + delete deps.docker[info.key]; + continue; + } - dep.new = result.newTag; + dep.new = newTag; dep.info = getDockerInfoUrl(info.ref); - setDepAge(dep, result.date); + setDepAge(dep, result?.date); } }, {concurrency}); @@ -1319,42 +1379,61 @@ export async function updates(opts: UpdatesOptions = {}): Promise { if (makeDepInfos.length) { fetchTasks.push((async () => { await pMap(makeDepInfos, async (info) => { - const {names, useGreatest, usePre, useRel, semvers, allowDowngrade: allowDown, cooldownOverride} = getVersionOpts(info.kind, info.name); - const {pinnedRange, pinNoDowngrade} = resolvePin(names, info.filePin, info.filePinNoDowngrade); - const makeCooldownDays = cooldownOverride ?? info.fileCooldownDays; + const { + useGreatest, usePre, useRel, semvers, allowDowngrade: allowDown, allowedVersions, + pinnedRange, pinNoDowngrade, cooldownDays: makeCooldownDays, + } = resolveVersionOpts( + info.versionConfig, info.kind, info.name, info.name, info.filePin, info.filePinNoDowngrade, + info.fileCooldownDays, + ); const opts = { semvers, useGreatest, usePre, useRel, allowDowngrade: allowDown, pinnedRange, pinNoDowngrade, cooldownDays: makeCooldownDays || undefined, now: makeCooldownDays ? now : undefined, }; const dep = deps.make[info.key]; try { - let update: MakeUpdate | MakeDockerUpdate; if (info.kind === "go") { - const goUpdate = await fetchMakeInfo(info.installPath, info.version, info.projectDir, ctx, goNoProxy, opts); - if (!goUpdate) { delete deps.make[info.key]; return; } - info.newSpec = `${goUpdate.newInstallPath}@${goUpdate.newVersion}`; - dep.new = goUpdate.newVersion; - update = goUpdate; + const modulePath = await resolveGoModuleRoot(info.installPath, info.projectDir, ctx, goNoProxy); + if (!modulePath) { delete deps.make[info.key]; return; } + const [rawData] = await fetchGoProxyInfo(modulePath, "tool", stripv(info.version), info.projectDir, ctx, goNoProxy); + const data = filterVersionData(rawData, "go", allowedVersions); + const newVersion = findNewVersion(data, {...opts, mode: "go", range: stripv(info.version)}); + if (!newVersion) { delete deps.make[info.key]; return; } + const newModulePath = data.newPath ?? goModulePathForVersion(modulePath, newVersion); + const newInstallPath = `${newModulePath}${info.installPath.slice(modulePath.length)}`; + const formattedVersion = formatVersionPrecision(newVersion, info.version); + if (newInstallPath === info.installPath && formattedVersion === info.version) { + delete deps.make[info.key]; return; + } + info.newSpec = `${newInstallPath}@${formattedVersion}`; + dep.new = formattedVersion; + dep.info = getGoInfoUrl(newModulePath); + setDepAge(dep, data.Time); } else { - const dockerUpdate = await fetchMakeDockerInfo(info.image, ctx, opts); + const [data] = await fetchDockerInfo(info.image.ref.fullImage, ctx); + const tags = filterVersionData(data, "docker", allowedVersions).tags; + const dockerUpdate = findDockerVersion( + tags, info.image.ref.tag, opts.semvers, opts.cooldownDays, opts.now, + opts.pinnedRange, opts.usePre, opts.useRel, + ); if (!dockerUpdate) { delete deps.make[info.key]; return; } - info.newSpec = formatMakeImageSpec(info.image.writtenImage, dockerUpdate.newTag, info.image.digest ? dockerUpdate.newDigest : null); + const newDigest = info.image.digest ? await fetchDockerTagDigest( + info.image.ref.namespace, info.image.ref.repo, dockerUpdate.newTag, ctx, + ) : null; + if (info.image.digest && !newDigest) { delete deps.make[info.key]; return; } + info.newSpec = formatMakeImageSpec(info.image.writtenImage, dockerUpdate.newTag, newDigest); dep.new = dockerUpdate.newTag; - update = dockerUpdate; + dep.info = getDockerInfoUrl(info.image.ref); + setDepAge(dep, dockerUpdate.date); } - dep.info = update.info; - if (update.date) setDepAge(dep, update.date); } catch (err) { - delete deps.make[info.key]; - addKeyError("make", info.key, err); + rejectDep("make", info.key, err); } }, {concurrency}); if (!Object.keys(deps.make).length) delete deps.make; })()); } - // Cache writes are detached from the fetch paths; settle them before - // returning so even an error exit cannot abandon in-flight writes. try { await Promise.all(fetchTasks); } finally { @@ -1362,84 +1441,86 @@ export async function updates(opts: UpdatesOptions = {}): Promise { } if (!countDeps(deps)) { - // A run that resolved nothing because everything failed is not an up-to-date one. return errors.length ? {results: {}, errors} : {results: {}, message: "All dependencies are up to date."}; } + const updatePnpmWorkspaceOverrideValues = (content: string, memberPath: string): string => { + const lines = content.split("\n"); + const entries = new Map(pnpmWorkspaceOverrides(content).map(entry => [entry.selector, entry])); + for (const [key, selector] of pnpmWorkspaceOverrideKeys) { + const dep = deps.npm?.[key]; + const entry = entries.get(selector); + if (!dep || !depBelongsToMember(key, memberPath) || !entry || entry.value !== (dep.oldOrig || dep.old)) continue; + const line = lines[entry.lineNumber]; + lines[entry.lineNumber] = `${line.slice(0, entry.valueIndex)}${dep.new}${line.slice(entry.valueIndex + entry.value.length)}`; + } + return lines.join("\n"); + }; + if (config.update) { - const updateMembers = (m: string, members: WorkspaceMember[], updateFn: (content: string, deps: Deps) => string) => { + const updateMembers = (mode: string, members: WorkspaceMember[], updateFn: (content: string, deps: Deps, member: WorkspaceMember) => string) => { for (const member of members) { - const localDeps = filterDepsForMember(deps[m], member.memberPath); + const localDeps = filterDepsForMember(deps[mode], member.memberPath); if (!Object.keys(localDeps).length) continue; - write(member.absPath, updateFn(member.content, localDeps)); + write(member.absPath, updateFn(member.content, localDeps, member)); } }; - // Group action and docker deps by their containing workflow/dockerfile so - // each file is rewritten once. buildOutput() (called after this block) - // mutates dep shape and must run after writes. - const actionsUpdatesByRelPath = new Map>(); - for (const [key, dep] of Object.entries(deps.actions ?? {})) { - const [relPath, name] = key.split(fieldSep); - // Sha pins keep the resolved tag in a trailing comment, which has to move along. - const newComment = hashRe.test(dep.old) ? dep.newPrint : undefined; - pushTo(actionsUpdatesByRelPath, relPath, {name, oldRef: dep.old, newRef: dep.new, newComment}); - } - - const dockerUpdatesByRelPath = new Map(); - for (const [key, dep] of Object.entries(deps.docker ?? {})) { - const [relPath] = key.split(fieldSep); - let map = dockerUpdatesByRelPath.get(relPath); - if (!map) dockerUpdatesByRelPath.set(relPath, map = {}); - map[key] = dep; - } - - const makeUpdatesByRelPath = new Map>(); - for (const info of makeDepInfos) { - if (!info.newSpec || !deps.make?.[info.key]) continue; - pushTo(makeUpdatesByRelPath, info.key.split(fieldSep)[0], {oldSpec: info.oldSpec, newSpec: info.newSpec}); - } - - // Process actions before docker: a workflow file may hold both an action and a - // docker-image update, and the actions branch syncs its rewrite into dockerFileData - // (one-way). Running docker first would overwrite the action edit on disk. - const orderedModes = Object.keys(deps).sort((a, b) => (a === "docker" ? 1 : 0) - (b === "docker" ? 1 : 0)); - for (const mode of orderedModes) { + const actionComments = new Map(actionDepInfos.map(info => [info.key, info.comment])); + const byRelPath = (entries: Array<[string, T]>) => Map.groupBy(entries, ([key]) => key.split(fieldSep)[0]); + const actionsUpdatesByRelPath = byRelPath(Object.entries(deps.actions ?? {})); + const dockerUpdatesByRelPath = byRelPath(Object.entries(deps.docker ?? {})); + const makeUpdatesByRelPath = Map.groupBy( + makeDepInfos.filter(info => info.newSpec && deps.make?.[info.key]), info => info.key.split(fieldSep)[0], + ); + + for (const mode of Object.keys(deps) + .sort((left, right) => (left === "docker" ? 1 : 0) - (right === "docker" ? 1 : 0))) { if (!Object.keys(deps[mode]).length) continue; if (mode === "actions") { - for (const [relPath, actionDeps] of actionsUpdatesByRelPath) { - const {absPath, content} = wfData[relPath] || {}; + for (const [relPath, entries] of actionsUpdatesByRelPath) { + const {absPath, content} = fileData[relPath] || {}; if (!absPath) continue; + const actionDeps = entries.map(([key, dep]) => { + const oldRef = dep.oldDigest ?? dep.old; + return { + name: key.split(fieldSep)[1], oldRef, + newRef: dep.newDigest ? dep.newDigest.substring(0, oldRef.length) : dep.new, + oldComment: actionComments.get(key) || undefined, + newComment: dep.digestOnly ? undefined : hashRe.test(dep.old) ? dep.newPrint : undefined, + }; + }); const updated = updateWorkflowFile(content, actionDeps); write(absPath, updated); - if (dockerFileData[relPath]) dockerFileData[relPath].content = updated; + fileData[relPath].content = updated; } continue; } if (mode === "docker") { - for (const [relPath, dockerDeps] of dockerUpdatesByRelPath) { - const fileInfo = dockerFileData[relPath]; + for (const [relPath, entries] of dockerUpdatesByRelPath) { + const fileInfo = fileData[relPath]; if (!fileInfo) continue; - const {absPath, content, fileType} = fileInfo; + const {absPath, content, fileType, workflowLines} = fileInfo; const updateFn = fileType === "dockerfile" ? updateDockerfile : - fileType === "compose" ? updateComposeFile : updateWorkflowDockerImages; - write(absPath, updateFn(content, dockerDeps)); + fileType === "compose" ? updateComposeFile : (workflow: string, workflowDeps: Deps) => + workflow.split("\n").map((line, lineNumber) => + workflowLines!.has(lineNumber) ? updateWorkflowDockerImages(line, workflowDeps) : line).join("\n"); + write(absPath, updateFn(content, Object.fromEntries(entries))); } continue; } if (mode === "make") { - for (const [relPath, rewrites] of makeUpdatesByRelPath) { - const fileInfo = makeFileData[relPath]; + for (const [relPath, infos] of makeUpdatesByRelPath) { + const fileInfo = fileData[relPath]; if (!fileInfo) continue; - write(fileInfo.absPath, updateMakefile(fileInfo.content, rewrites)); + write(fileInfo.absPath, updateMakefile(fileInfo.content, + infos.map(info => ({oldSpec: info.oldSpec, newSpec: info.newSpec!})))); } continue; } - // Workspace members and unrelated plain manifests of the same mode can coexist (e.g. a - // workspace dir plus a second `-f` directory), so both are written. if (mode === "go") { for (const goMod of [...goModFiles, ...(plainFiles.go ?? [])]) { const localDeps = filterDepsForMember(deps[mode], goMod.memberPath); @@ -1448,24 +1529,20 @@ export async function updates(opts: UpdatesOptions = {}): Promise { if (updatedContent !== goMod.content) write(goMod.absPath, updatedContent); rewriteGoImports(goMod.projectDir, majorVersionRewrites, write); } - if (goWorkData) { - const workDeps: Deps = {}; - for (const [key, dep] of Object.entries(deps[mode])) { - if (key.split(fieldSep)[0] === "replace") workDeps[key] = dep; - } + for (const goWork of goWorkFiles) { + const workDeps = Object.fromEntries(Object.entries(filterDepsForMember(deps[mode], goWork.memberPath)) + .filter(([key]) => baseType(key.split(fieldSep)[0]) === "replace")); if (Object.keys(workDeps).length) { - const [updatedWork] = updateGoMod(goWorkData.content, workDeps); - if (updatedWork !== goWorkData.content) write(goWorkData.file, updatedWork); + const [updatedWork] = updateGoMod(goWork.content, workDeps); + if (updatedWork !== goWork.content) write(goWork.file, updatedWork); } } } else if (mode === "cargo") { - // The member lists stay empty unless a workspace manifest was seen. - updateMembers(mode, cargoMemberFiles, updateCargoToml); - updateMembers(mode, plainFiles.cargo ?? [], updateCargoToml); + updateMembers(mode, [...cargoMemberFiles, ...(plainFiles.cargo ?? [])], updateCargoToml); } else if (mode === "npm") { - updateMembers(mode, pnpmCatalogFiles, updatePnpmWorkspace); - updateMembers(mode, pnpmMemberFiles, updatePackageJson); - updateMembers(mode, plainFiles.npm ?? [], updatePackageJson); + updateMembers(mode, pnpmCatalogFiles, (content, localDeps, member) => + updatePnpmWorkspaceOverrideValues(updatePnpmWorkspace(content, localDeps), member.memberPath)); + updateMembers(mode, [...pnpmMemberFiles, ...(plainFiles.npm ?? [])], updatePackageJson); } else { updateMembers(mode, plainFiles[mode] ?? [], updatePyprojectToml); } diff --git a/cli.test.ts b/cli.test.ts new file mode 100644 index 0000000..92b7009 --- /dev/null +++ b/cli.test.ts @@ -0,0 +1,17 @@ +import {parseCliArgs} from "./cli.ts"; + +test("recovers swallowed short option clusters", () => { + const single = parseCliArgs(["-T", "-u", "package.json"]); + expect(single.args).toMatchObject({timeout: true, update: true}); + expect(single.positionals).toEqual(["package.json"]); + + const clustered = parseCliArgs(["-T", "-uj", "package.json"]); + expect(clustered.args).toMatchObject({timeout: true, update: true, json: true}); + expect(clustered.positionals).toEqual(["package.json"]); + + const {args, positionals} = parseCliArgs(["-i", "-ug", "react", "package.json"]); + expect(args.include).toEqual([]); + expect(args.update).toBe(true); + expect(args.greatest).toEqual(["react"]); + expect(positionals).toEqual(["package.json"]); +}); diff --git a/cli.ts b/cli.ts index 9fd80ca..e5d11dc 100644 --- a/cli.ts +++ b/cli.ts @@ -2,8 +2,7 @@ import {cwd} from "node:process"; import {parseArgs} from "node:util"; import {dirname, isAbsolute, resolve} from "node:path"; import {statSync} from "node:fs"; -import {options, parseMixedArg, getOptionKey, parseArgList, parsePinArg, loadConfig} from "./config.ts"; -import {fetchTimeout} from "./modes/shared.ts"; +import {cliBaseConfig, options, parseMixedArg, getOptionKey, parseArgList, parsePinArg, loadConfig} from "./config.ts"; import {parsePositiveInt} from "./utils/utils.ts"; import type {Arg} from "./config.ts"; import type {UpdatesOptions} from "./api.ts"; @@ -27,16 +26,12 @@ function deriveStartDir(first: string | undefined): string { return isDir ? abs : dirname(abs); } -// Flatten -f/--file plus positionals into the target list, and derive the -// directory config discovery walks up from. Shared by the binary's prewarm -// path and resolveConfig, which both need it before any config is loaded. export function resolveFileArgs(args: Record, positionals: Array): {filesList: Array, startDir: string} { const fileSet = parseMixedArg(args.file); const filesList = [...(fileSet instanceof Set ? fileSet : []), ...positionals]; return {filesList, startDir: deriveStartDir(filesList[0])}; } -// Parse argv into option values, fixing the parseArgs "-a -b" → {a: "-b"} defect. export function parseCliArgs(argv?: Array): {args: Record, positionals: Array} { const result = parseArgs({ strict: false, @@ -51,97 +46,110 @@ export function parseCliArgs(argv?: Array): {args: Record, let positionalsSeen = 0; for (const [index, token] of result.tokens.entries()) { if (token.kind === "positional") positionalsSeen++; - // An inline value (`--exclude=-u`, `-i-g`) was written deliberately, so only a separately - // parsed one can be a flag parseArgs swallowed. if (token.kind !== "option" || token.inlineValue || !token.value?.startsWith("-")) continue; - const dashes = token.value.startsWith("--") ? 2 : 1; - const key = getOptionKey(token.value.substring(dashes)); - if (!key) continue; + const longOption = token.value.startsWith("--"); const next = result.tokens[index + 1]; - // The flag was wrongly swallowed as this option's value; drop only that bogus - // value (the dash-prefixed token.value, which may not be the last element) - // rather than discarding the whole accumulated array, so other repeats like - // `-i react -i -g -i vue` keep both `react` and `vue`. + const nextPositional = next?.kind === "positional" ? next.value : undefined; + const recoveredOptions: Array<{key: string, value: string | boolean}> = []; + const raw = token.value.substring(longOption ? 2 : 1); + let consumesPositional = false; + if (longOption) { + const key = getOptionKey(raw); + if (key) { + consumesPositional = options[key].type === "string" && nextPositional !== undefined; + recoveredOptions.push({key, value: consumesPositional ? nextPositional! : true}); + } + } else { + for (let offset = 0; offset < raw.length;) { + const key = getOptionKey(raw[offset]); + if (!key) { recoveredOptions.length = 0; break; } + if (options[key].type === "boolean") { + recoveredOptions.push({key, value: true}); + offset++; + } else { + const inlineValue = raw.substring(offset + 1); + consumesPositional = !inlineValue && nextPositional !== undefined; + recoveredOptions.push({ + key, + value: inlineValue || (consumesPositional ? nextPositional! : true), + }); + offset = raw.length; + } + } + } + if (!recoveredOptions.length) continue; const swallowed = values[token.name]; if (Array.isArray(swallowed)) { - const pos = swallowed.indexOf(token.value); - if (pos !== -1) swallowed.splice(pos, 1); + const position = swallowed.indexOf(token.value); + if (position !== -1) swallowed.splice(position, 1); } else { values[token.name] = true; } - const recovered = next?.kind === "positional" && next.value ? next.value : true; - // a recovered positional is that option's value, so it must not stay in the file list too - if (typeof recovered === "string") consumedPositionals.add(positionalsSeen); - if (options[key]?.multiple) { - const list = (values[key] ??= []) as Array; - list.push(recovered); - } else { - // non-multiple options expect a scalar; an array shape is rejected by the typeof string consumers - values[key] = recovered; + if (consumesPositional) consumedPositionals.add(positionalsSeen); + for (const {key, value} of recoveredOptions) { + if (options[key].multiple) { + const list = (values[key] ??= []) as Array; + list.push(value); + } else { + values[key] = value; + } } } return {args: values, positionals: result.positionals.filter((_val, index) => !consumedPositionals.has(index))}; } -// Overlay parsed CLI args onto the config file. Shared by the binary and tests. export async function resolveConfig( args: Record, positionals: Array, ): Promise { const {filesList, startDir} = resolveFileArgs(args, positionals); - const cliTimeout = typeof args.timeout === "string" ? parsePositiveInt(args.timeout, "timeout") : undefined; - - const fileConfig = await loadConfig(startDir, { - noCache: Boolean(args["no-cache"]), - timeout: cliTimeout ?? fetchTimeout, - }); - - // `pin` is dropped so it reaches the run as a per-directory pin rather than an authored one: - // a renovate-inherited ceiling in the global pin would gain the right to downgrade (api.ts). - const config: UpdatesOptions = {...fileConfig, pin: undefined}; - if (args.json) config.json = true; - if (args.verbose) config.verbose = true; - if (args["no-cache"]) config.noCache = true; - if (args.update) config.update = true; - if (args.indirect) config.indirect = true; - if (args["error-on-outdated"]) config.errorOnOutdated = true; - if (args["error-on-unchanged"]) config.errorOnUnchanged = true; - // each color flag clears the other so a CLI flag beats both file values, -n applied last so it wins - if (args.color) {config.color = true; config.noColor = false;} - if (args["no-color"]) {config.color = false; config.noColor = true;} - if (cliTimeout !== undefined) config.timeout = cliTimeout; - if (typeof args.sockets === "string") config.sockets = parsePositiveInt(args.sockets, "sockets"); - if (typeof args.registry === "string") config.registry = args.registry; - if (typeof args.cooldown === "string") config.cooldown = Number(args.cooldown) || args.cooldown; + const fileConfig = await loadConfig(startDir); + + const cliConfig: Partial = {}; + if (args.json) cliConfig.json = true; + if (args.verbose) cliConfig.verbose = true; + if (args["no-cache"]) cliConfig.noCache = true; + if (args.update) cliConfig.update = true; + if (args.indirect) cliConfig.indirect = true; + if (args["error-on-outdated"]) cliConfig.errorOnOutdated = true; + if (args["error-on-unchanged"]) cliConfig.errorOnUnchanged = true; + if (args.color) {cliConfig.color = true; cliConfig.noColor = false;} + if (args["no-color"]) {cliConfig.color = false; cliConfig.noColor = true;} + if (typeof args.timeout === "string") cliConfig.timeout = parsePositiveInt(args.timeout, "timeout"); + if (typeof args.sockets === "string") cliConfig.sockets = parsePositiveInt(args.sockets, "sockets"); + if (typeof args.registry === "string") cliConfig.registry = args.registry; + if (typeof args.cooldown === "string") cliConfig.cooldown = Number(args.cooldown) || args.cooldown; const cliInclude = parseArgList(args.include).map(cliPatternToRegex); const cliExclude = parseArgList(args.exclude).map(cliPatternToRegex); - if (cliInclude.length) config.include = cliInclude; - if (cliExclude.length) config.exclude = cliExclude; + if (cliInclude.length) cliConfig.include = cliInclude; + if (cliExclude.length) cliConfig.exclude = cliExclude; const cliTypes = parseArgList(args.types); - if (cliTypes.length) config.types = cliTypes; + if (cliTypes.length) cliConfig.types = cliTypes; const cliPin = parsePinArg(args.pin); - if (Object.keys(cliPin).length) config.pin = cliPin; + if (Object.keys(cliPin).length) cliConfig.pin = cliPin; const cliModes = parseMixedArg(args.modes); - if (cliModes instanceof Set) config.modes = Array.from(cliModes); + if (cliModes instanceof Set) cliConfig.modes = Array.from(cliModes); for (const key of ["greatest", "prerelease", "release", "patch", "minor"] as const) { const val = argToConfigMixed(args[key]); - if (val !== undefined) config[key] = val; + if (val !== undefined) cliConfig[key] = val; } const allowDowngrade = argToConfigMixed(args["allow-downgrade"]); - if (allowDowngrade !== undefined) config.allowDowngrade = allowDowngrade; + if (allowDowngrade !== undefined) cliConfig.allowDowngrade = allowDowngrade; - if (filesList.length) config.files = filesList; + if (filesList.length) cliConfig.files = filesList; for (const key of ["forgeapi", "pypiapi", "jsrapi", "goproxy", "cargoapi", "dockerapi"] as const) { - if (typeof args[key] === "string") config[key] = args[key]; + if (typeof args[key] === "string") cliConfig[key] = args[key]; } + const config: UpdatesOptions = {...fileConfig, pin: undefined, ...cliConfig}; + Object.defineProperty(config, cliBaseConfig, {value: {fileConfig, cliKeys: Object.keys(cliConfig)}}); return config; } diff --git a/config.test.ts b/config.test.ts new file mode 100644 index 0000000..da4efb8 --- /dev/null +++ b/config.test.ts @@ -0,0 +1,43 @@ +import {mkdtemp, mkdir, readFile, rm, writeFile} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; +import {cliConfigBaseDir} from "./api.ts"; +import {cliBaseConfig, loadConfig} from "./config.ts"; + +test("the package API preserves cliConfigBaseDir", () => { + expect(cliConfigBaseDir).toBe(cliBaseConfig); +}); + +test("config discovery starts at the target and loads only the highest-priority module", async () => { + const dir = await mkdtemp(join(tmpdir(), "updates-config-")); + const discoveryDir = join(dir, "discovery"); + const child = join(discoveryDir, "child"); + const priorityDir = join(dir, "priority"); + try { + await mkdir(child, {recursive: true}); + await mkdir(priorityDir); + await writeFile(join(discoveryDir, "updates.config.js"), "module.exports = {};\n"); + await writeFile(join(child, "renovate.json"), JSON.stringify({ignoreDeps: ["child-only"]})); + const [excluded] = (await loadConfig(child)).exclude!; + expect(excluded).toBeInstanceOf(RegExp); + expect((excluded as RegExp).test("child-only")).toBe(true); + + const marker = join(priorityDir, "loaded"); + await writeFile(join(priorityDir, "updates.config.js"), + `require("node:fs").appendFileSync(${JSON.stringify(marker)}, "js\\n"); module.exports = {exclude: ["js"]};\n`); + await writeFile(join(priorityDir, "updates.config.mjs"), + `import {appendFileSync} from "node:fs"; appendFileSync(${JSON.stringify(marker)}, "mjs\\n"); export default {exclude: ["mjs"]};\n`); + expect((await loadConfig(priorityDir)).exclude).toEqual(["js"]); + expect(await readFile(marker, "utf8")).toBe("js\n"); + + const brokenDir = join(priorityDir, "broken"); + await mkdir(brokenDir); + await writeFile(join(brokenDir, "updates.config.js"), "throw new Error('broken primary');\n"); + await writeFile(join(brokenDir, "updates.config.mjs"), + `import {appendFileSync} from "node:fs"; appendFileSync(${JSON.stringify(marker)}, "broken-mjs\\n"); export default {};\n`); + await expect(loadConfig(brokenDir)).rejects.toThrow(/broken primary/); + expect(await readFile(marker, "utf8")).toBe("js\n"); + } finally { + await rm(dir, {recursive: true, force: true}); + } +}); diff --git a/config.ts b/config.ts index a14d9da..d28866f 100644 --- a/config.ts +++ b/config.ts @@ -4,7 +4,8 @@ import {access} from "node:fs/promises"; import type {ParseArgsOptionsConfig} from "node:util"; import {validRange} from "./utils/semver.ts"; import {commaSeparatedToArray, patternToRegex, walkUp, memoizeAsync} from "./utils/utils.ts"; -import type {PresetFetchOptions, RenovateImportOptions} from "./utils/renovate.ts"; +import type {RenovateImportOptions} from "./utils/renovate.ts"; +import {loadRenovateConfig} from "./utils/renovate.ts"; export type Config = { /** Array of dependencies to include */ @@ -93,6 +94,7 @@ export type Override = { }; export type Arg = string | boolean | Array | undefined; +export const cliBaseConfig = Symbol("cliBaseConfig"); export const options: ParseArgsOptionsConfig = { "allow-downgrade": {short: "d", type: "string", multiple: true}, @@ -131,7 +133,7 @@ export const options: ParseArgsOptionsConfig = { }; export function parseMixedArg(arg: Arg): boolean | Set { - if (Array.isArray(arg) && arg.every(a => a === true)) { + if (Array.isArray(arg) && arg.every(val => val === true)) { return true; } else if (Array.isArray(arg)) { return new Set(arg.filter(val => typeof val === "string").flatMap(commaSeparatedToArray)); @@ -162,8 +164,6 @@ export function parseArgList(arg: Arg): Array { return []; } -// An unparsable range satisfies nothing, so dropping it would either discard the pin or freeze the -// dependency forever. Renovate likewise rejects an allowedVersions it cannot parse. export function validatePin(pin: Config["pin"]): void { for (const [pkg, range] of Object.entries(pin ?? {})) { if (!validRange(range)) throw new Error(`Invalid pin range for ${pkg}: ${range}`); @@ -173,7 +173,7 @@ export function validatePin(pin: Config["pin"]): void { export function parsePinArg(arg: Arg): Record { const result: Record = {}; for (const val of Array.isArray(arg) ? arg : [arg]) { - if (typeof val !== "string") continue; // a flag recovered from a swallowed value arrives as `true` + if (typeof val !== "string") continue; const eq = val.indexOf("="); if (eq < 1) throw new Error(`Invalid pin: ${val}, expected =`); result[val.slice(0, eq)] = val.slice(eq + 1); @@ -188,51 +188,34 @@ export function configMixedToRegexes(val: boolean | Array | und return patternsToRegexSet(val); } -type FoundConfig = {configDir: string, default: Config}; - -// Try to load any updates.config.* in dir. Returns the first that imports -// successfully. If none imports but at least one parsed-and-failed, throws -// the first parse error so a broken sibling next to a valid one does not -// block the valid one. -async function tryLoadInDir(dir: string): Promise { - const exts = ["js", "ts", "mjs", "mts"]; - const results = await Promise.all(exts.map(async (ext): Promise => { +const findConfigUp = memoizeAsync((startDir: string) => walkUp(startDir, async dir => { + for (const ext of ["js", "ts", "mjs", "mts"]) { const filename = `updates.config.${ext}`; const fullPath = join(dir, filename); try { await access(fullPath); - } catch { - return null; + } catch (err: any) { + if (err?.code === "ENOENT") continue; + throw new Error(`Unable to load config file ${filename}: ${err?.message ?? err}`); } try { const mod = await import(pathToFileURL(fullPath).href); - return {configDir: dir, default: mod.default ?? {}}; + return mod.default ?? {}; } catch (err: any) { - return new Error(`Unable to parse config file ${filename}: ${err?.message ?? err}`); + throw new Error(`Unable to parse config file ${filename}: ${err?.message ?? err}`); } - })); - for (const r of results) if (r && !(r instanceof Error)) return r; - for (const r of results) if (r instanceof Error) throw r; + } return null; -} - -const findConfigUp = memoizeAsync((startDir: string) => walkUp(startDir, tryLoadInDir)); +})); -export async function loadConfig(startDir: string, presetFetch: PresetFetchOptions = {}): Promise { - const found = await findConfigUp(startDir); - const raw: Config = found?.default ?? {}; - const {loadRenovateConfig, makePresetFetcher} = await import("./utils/renovate.ts"); - const fetchText = makePresetFetcher(presetFetch); - const renovateConfig = await loadRenovateConfig(found?.configDir ?? startDir, raw.inherit?.renovate, fetchText); +export async function loadConfig(startDir: string): Promise { + const raw = await findConfigUp(startDir) ?? {}; + const renovateConfig = await loadRenovateConfig(startDir, raw.inherit?.renovate); const config: Config = {...renovateConfig, ...raw}; - // `pin` merges per key, so an authored pin for one dependency keeps the ceilings inherited for - // the others. An authored entry may downgrade, so the marker keeps only the names renovate owns. if (renovateConfig.pin) { config.pin = {...renovateConfig.pin, ...raw.pin}; config.pinNoDowngrade = Object.keys(renovateConfig.pin).filter(name => !raw.pin?.[name]); } - // Overrides concatenate for the same reason, with the authored ones last so they win the - // last-match-wins pass in api.ts rather than discarding what renovate contributed. if (renovateConfig.overrides?.length) config.overrides = [...renovateConfig.overrides, ...(raw.overrides ?? [])]; validatePin(config.pin); return config; diff --git a/index.test.ts b/index.test.ts index 70b817e..f04be9b 100644 --- a/index.test.ts +++ b/index.test.ts @@ -47,7 +47,6 @@ const goPreFile = fileURLToPath(new URL("fixtures/go-prerelease/go.mod", import. const goPseudoFile = fileURLToPath(new URL("fixtures/go-pseudo/go.mod", import.meta.url)); const goPseudoUpdateFile = fileURLToPath(new URL("fixtures/go-pseudo-update/go.mod", import.meta.url)); const goWorkspaceDir = fileURLToPath(new URL("fixtures/go-workspace", import.meta.url)); -const goWorkspaceFile = fileURLToPath(new URL("fixtures/go-workspace/go.work", import.meta.url)); const invalidConfigFile = fileURLToPath(new URL("fixtures/invalid-config/package.json", import.meta.url)); const actionsDir = fileURLToPath(new URL("fixtures/actions/.github/workflows", import.meta.url)); const dockerfileFixture = fileURLToPath(new URL("fixtures/docker/Dockerfile", import.meta.url)); @@ -56,7 +55,6 @@ const dockerActionsDir = fileURLToPath(new URL("fixtures/docker-actions/.github/ const dockerDir = fileURLToPath(new URL("fixtures/docker", import.meta.url)); const cargoFile = fileURLToPath(new URL("fixtures/cargo/Cargo.toml", import.meta.url)); const cargoWorkspaceDir = fileURLToPath(new URL("fixtures/cargo-workspace", import.meta.url)); -const cargoWorkspaceFile = fileURLToPath(new URL("fixtures/cargo-workspace/Cargo.toml", import.meta.url)); const pnpmWorkspaceDir = fileURLToPath(new URL("fixtures/pnpm-workspace", import.meta.url)); const pnpmWorkspaceFile = fileURLToPath(new URL("fixtures/pnpm-workspace/pnpm-workspace.yaml", import.meta.url)); @@ -152,6 +150,7 @@ let jsrUrl: string; let goProxyUrl: string; let dockerUrl: string; let cargoUrl: string; +let localDependencyRequests = 0; beforeAll(async () => { npmServer = makeServer(defaultRoute); @@ -171,7 +170,6 @@ beforeAll(async () => { for (const pkgName of testPackages) { const name = (testPkg.resolutions[pkgName] ? resolutionsBasePackage(pkgName) : pkgName); const urlName = name.replace(/\//g, "%2f"); - // can not use file URLs because node stupidely throws on "%2f" in paths. const path = join(import.meta.dirname, `fixtures/npm/${urlName}.json`); npmFilesPromises.push((async () => ({urlName, data: await readFile(path, "utf8")}))()); } @@ -205,7 +203,6 @@ beforeAll(async () => { // `time` map, so serving one regardless would hide the full-packument fallback that fills them. const abbreviatedType = "application/vnd.npm.install-v1+json"; for (const {urlName} of npmFiles) { - // gzip lazily and per flavor: only a --cooldown run ever asks for the dated document const gzips = new Map(); npmServer.get(`/${urlName}`, (req, res) => { const flavor = req.headers.accept?.includes(abbreviatedType) ? "abbrev" : "full"; @@ -216,9 +213,20 @@ beforeAll(async () => { })); }); } + for (const name of ["local-file", "local-link"]) { + npmServer.get(`/${name}`, (_, res) => { + localDependencyRequests++; + res.statusCode = 500; + res.end(); + }); + } const gzipAll = await Promise.all([ - ...pypiFiles.map(async ({pkgName, data}) => ({type: "pypi" as const, key: `/pypi/${pkgName}/json`, gz: await gzipPromise(data)})), + ...pypiFiles.map(async ({pkgName, data}) => ({ + type: "pypi" as const, + key: `/pypi/${pkgName.toLowerCase().replace(/[-_.]+/g, "-")}/json`, + gz: await gzipPromise(data), + })), ...jsrFiles.map(async ({scope, name, data}) => ({type: "jsr" as const, key: `/@${scope}/${name}/meta.json`, gz: await gzipPromise(data)})), (async () => ({type: "github" as const, key: "/repos/silverwind/updates/commits", gz: await gzipPromise(commits)}))(), (async () => ({type: "github" as const, key: "/repos/silverwind/updates/tags", gz: await gzipPromise(tags)}))(), @@ -229,7 +237,6 @@ beforeAll(async () => { server.get(key, (_, res) => res.send(gz)); } - // Per-version follow-up routes; gzip lazily since tests hit only a few of ~6000. for (const [urlName, data] of npmParsed) { const versions = data.versions || {}; const time = data.time || {}; @@ -245,11 +252,9 @@ beforeAll(async () => { } } - // Override noty/3.1.4 to omit _npmOperationalInternal so the fallback to full packument is tested const notyVersionGz = await gzipPromise(JSON.stringify(npmParsed.get("noty").versions["3.1.4"])); npmServer.get("/noty/3.1.4", (_, res) => res.send(notyVersionGz)); - // Go proxy fixtures const goProxyRoutes: Array<{path: string, response: string}> = [ {path: "/github.com/google/uuid/@latest", response: JSON.stringify({Version: "v1.6.0", Time: "2024-06-13T02:52:04Z"})}, {path: "/github.com/google/go-github/v70/@latest", response: JSON.stringify({Version: "v70.0.0", Time: "2024-11-29T00:00:00Z"})}, @@ -260,10 +265,11 @@ beforeAll(async () => { {path: "/gitea.com/gitea/act/@latest", response: JSON.stringify({Version: "v0.261.7", Time: "2025-06-01T00:00:00Z"})}, {path: "/github.com/example/pseudopkg/@latest", response: JSON.stringify({Version: "v0.4.1", Time: "2023-06-01T00:00:00Z"})}, {path: "/github.com/example/pseudoupd/@latest", response: JSON.stringify({Version: "v1.5.0", Time: "2025-06-01T00:00:00Z"})}, - // `listonly` has no timestamps, forcing the follow-up `.info`; `listtime` carries them inline. {path: "/github.com/example/listonly/@v/list", response: "v1.0.0\nv1.2.0\nv1.3.0-rc.1\n"}, {path: "/github.com/example/listonly/@v/v1.2.0.info", response: JSON.stringify({Version: "v1.2.0", Time: "2025-03-01T00:00:00Z"})}, {path: "/github.com/example/listtime/@v/list", response: "v1.0.0 2024-01-01T00:00:00Z\nv1.1.0 2024-06-01T00:00:00Z\n"}, + {path: "/github.com/example/makeallowed/@latest", response: JSON.stringify({Version: "v1.1.0", Time: "2025-01-01T00:00:00Z"})}, + {path: "/github.com/example/makeallowed/v2/@latest", response: JSON.stringify({Version: "v2.0.0", Time: "2025-06-01T00:00:00Z"})}, ]; for (let v = 71; v <= 82; v++) { goProxyRoutes.push({ @@ -278,23 +284,24 @@ beforeAll(async () => { goProxyServer.get(path, (_, res) => res.send(gz)); } - // Actions fixtures for github server const actionsRoutes: Array<[string, string]> = [ ["/repos/actions/checkout/tags", "fixtures/github/actions-checkout-tags.json"], ["/repos/actions/setup-node/tags", "fixtures/github/actions-setup-node-tags.json"], ["/repos/actions/checkout/git/commits/cccc000000000000000000000000000000000011", "fixtures/github/actions-checkout-commit-v10.0.1.json"], ["/repos/actions/setup-node/git/commits/bbbb000000000000000000000000000000000010", "fixtures/github/actions-setup-node-commit-v10.json"], ]; - // Empty tags for tj-actions/changed-files (hash-pinned, no semver tags to resolve) const emptyTagsGz = await gzipPromise("[]"); githubServer.get("/repos/tj-actions/changed-files/tags", (_, res) => res.send(emptyTagsGz)); + githubServer.get("/repos/actions/checkout/branches/main", (_, res) => + res.send(gzipNow(JSON.stringify({commit: {sha: "aaaa000000000000000000000000000000000001"}})))); + githubServer.get("/repos/actions/checkout/branches/release", (_, res) => + res.send(gzipNow(JSON.stringify({commit: {sha: "bbbb000000000000000000000000000000000002"}})))); for (const [route, fixture] of actionsRoutes) { const data = await readFile(fileURLToPath(new URL(fixture, import.meta.url)), "utf8"); const gz = await gzipPromise(data); githubServer.get(route, (_, res) => res.send(gz)); } - // Docker Hub API fixtures const dockerFixtures: Array<[string, string]> = [ ["/v2/repositories/library/node/tags", "fixtures/docker/node-tags.json"], ["/v2/repositories/library/noty/tags", "fixtures/docker/node-tags.json"], // an image sharing an npm dep's name @@ -307,16 +314,22 @@ beforeAll(async () => { dockerServer.get(route, (_, res) => res.send(gz)); } - // Namespaced image + per-tag digest for make-mode docker images const makeImgTagsGz = await gzipPromise(JSON.stringify({count: 2, results: [ {name: "v0.11.0", tag_last_pushed: "2025-01-01T00:00:00Z"}, {name: "v0.12.0", tag_last_pushed: "2025-06-01T00:00:00Z"}, ]})); - const makeImgDigestGz = await gzipPromise(JSON.stringify({digest: `sha256:${"b".repeat(64)}`})); + const makeAllowedImgTagsGz = await gzipPromise(JSON.stringify({count: 3, results: [ + {name: "1.0", tag_last_pushed: "2025-01-01T00:00:00Z"}, + {name: "1.1", tag_last_pushed: "2025-03-01T00:00:00Z"}, + {name: "2.0", tag_last_pushed: "2025-06-01T00:00:00Z"}, + ]})); + const makeOldImgGz = await gzipPromise(JSON.stringify({digest: "sha256:list-old"})); + const makeNewImgGz = await gzipPromise(JSON.stringify({digest: "sha256:list-new"})); dockerServer.get("/v2/repositories/koalaman/shellcheck/tags", (_, res) => res.send(makeImgTagsGz)); - dockerServer.get("/v2/repositories/koalaman/shellcheck/tags/v0.12.0", (_, res) => res.send(makeImgDigestGz)); + dockerServer.get("/v2/repositories/example/makeallowed/tags", (_, res) => res.send(makeAllowedImgTagsGz)); + dockerServer.get("/v2/repositories/koalaman/shellcheck/tags/v0.11.0", (_, res) => res.send(makeOldImgGz)); + dockerServer.get("/v2/repositories/koalaman/shellcheck/tags/v0.12.0", (_, res) => res.send(makeNewImgGz)); - // The cargo sparse index serves NDJSON from a path sharded by name length: `1/a`, `ab/cd/name`. const serdeIndex = await readFile(fileURLToPath(new URL("fixtures/cargo/serde-index.ndjson", import.meta.url)), "utf8"); const serdeIndexGz = await gzipPromise(serdeIndex); cargoServer.get("/se/rd/serde", (_, res) => res.send(serdeIndexGz)); @@ -367,8 +380,6 @@ afterAll(async () => { ]); }); -// Run the CLI in-process. Takes [script, ...args] like execFileAsync and returns -// the JSON stdout the binary would print. noCache avoids per-test disk-cache churn. async function runCliExec(argvWithScript: Array): Promise<{stdout: string, stderr: string}> { const {args, positionals} = parseCliArgs(argvWithScript.slice(1)); const config = await resolveConfig(args, positionals); @@ -381,33 +392,37 @@ async function runCliExec(argvWithScript: Array): Promise<{stdout: strin return {stdout, stderr: ""}; } -function makeTest(args: string) { - return async () => { - const argv = args.split(/\s+/).filter(Boolean); - const hasFile = argv.includes("-f") || argv.includes("--file"); - const {stdout} = await runCliExec([ - script, ...argv, "-c", // -c absorbs a dangling optional-value flag (-g/-p/-R/-P) - ...apiArgs(), - ...(hasFile ? [] : ["-f", join(testDir, "package.json")]), - ]); - // undefined when the CLI printed {message} (nothing to update), else results - const {results} = JSON.parse(stdout); - - // Parse results, with custom validation for the dynamic "age" property - for (const mode of Object.keys(results || {})) { - for (const type of Object.keys(results[mode] || {})) { - for (const name of Object.keys(results[mode][type] || {})) { - delete results[mode][type][name].age; - } +async function makeTest(args: string) { + const argv = args.split(/\s+/).filter(Boolean); + const hasFile = argv.includes("-f") || argv.includes("--file"); + const {stdout} = await runCliExec([ + script, ...argv, "-c", + ...apiArgs(), + ...(hasFile ? [] : ["-f", join(testDir, "package.json")]), + ]); + const {results} = JSON.parse(stdout); + for (const mode of Object.keys(results || {})) { + for (const type of Object.keys(results[mode] || {})) { + for (const name of Object.keys(results[mode][type] || {})) { + delete results[mode][type][name].age; } } + } + return results; +} - return results; - }; +const dep = (info: string, newVersion: string, old: string) => ({info, new: newVersion, old}); + +function dependencyRows(results: Awaited>["results"]) { + const rows = Object.entries(results).flatMap(([mode, types]) => Object.entries(types).flatMap(([type, dependencies]) => + Object.entries(dependencies).map(([name, dependency]) => [mode, type, name, dependency] as const))); + return rows.sort((left, right) => { + const leftKey = `${left[0]}\0${left[1]}\0${left[2]}`; + const rightKey = `${right[0]}\0${right[1]}\0${right[2]}`; + return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0; + }); } -// Point every registry/API at its mock server. A repeated flag's later -// occurrence wins, so tests can override by appending (e.g. --registry). function apiArgs(): string[] { return [ "--registry", npmUrl, @@ -427,7 +442,7 @@ test("text output lists every dep, one row per version across sections", async ( expect(stderr).toEqual(""); expect(stdout).toContain("prismjs"); expect(stdout).toContain("https://github.com/silverwind/updates"); - expect(stdout.split("\n").filter(line => line.includes("@babel/preset-env"))).toHaveLength(2); + expect(stdout.split("\n").filter(line => line.includes("@babel/preset-env"))).toHaveLength(3); expect(stdout).toContain("~6.0.0 || ~7.11.5"); }); @@ -456,10 +471,8 @@ test("jsr", async ({expect = globalExpect}: any = {}) => { ]); expect(stderr).toEqual(""); const {results} = JSON.parse(stdout); - expect(results.npm.dependencies["@std/semver"]).toBeDefined(); expect(results.npm.dependencies["@std/semver"].old).toBe("1.0.5"); expect(results.npm.dependencies["@std/semver"].new).toBe("1.0.8"); - expect(results.npm.devDependencies["@std/path"]).toBeDefined(); expect(results.npm.devDependencies["@std/path"].old).toBe("1.0.0"); expect(results.npm.devDependencies["@std/path"].new).toBe("1.0.8"); }); @@ -469,16 +482,29 @@ test("npm alias resolves the aliased package, and a second run is a no-op", asyn mkdirSync(aliasDir, {recursive: true}); const pkgPath = join(aliasDir, "package.json"); await writeFile(pkgPath, JSON.stringify({dependencies: { - "sourcemaps": "npm:gulp-sourcemaps@^2.0.0", + "package-matched": "npm:gulp-sourcemaps@^2.0.0", + "dep-matched": "npm:gulp-sourcemaps@^2.0.0", "tagged": "npm:gulp-sourcemaps@latest", }}, null, 2)); + await writeFile(join(aliasDir, "renovate.json"), JSON.stringify({packageRules: [ + {matchPackageNames: ["gulp-sourcemaps"], allowedVersions: "<=2.5.2"}, + {matchDepNames: ["dep-matched"], allowedVersions: "<=2.4.1"}, + ]})); const run = () => updates({files: [pkgPath], registry: npmUrl, update: true, color: false, noCache: true}); const {results} = await run(); - expect(results.npm.dependencies["sourcemaps"]).toMatchObject({old: "npm:gulp-sourcemaps@^2.0.0", new: "npm:gulp-sourcemaps@^2.6.5"}); + expect(results.npm.dependencies["package-matched"]).toMatchObject({ + old: "npm:gulp-sourcemaps@^2.0.0", new: "npm:gulp-sourcemaps@^2.5.2", + }); + expect(results.npm.dependencies["dep-matched"]).toMatchObject({ + old: "npm:gulp-sourcemaps@^2.0.0", new: "npm:gulp-sourcemaps@^2.4.1", + }); expect(results.npm.dependencies["tagged"]).toBeUndefined(); const written = await readFile(pkgPath, "utf8"); - expect(JSON.parse(written).dependencies["sourcemaps"]).toBe("npm:gulp-sourcemaps@^2.6.5"); + expect(JSON.parse(written).dependencies).toMatchObject({ + "package-matched": "npm:gulp-sourcemaps@^2.5.2", + "dep-matched": "npm:gulp-sourcemaps@^2.4.1", + }); await run(); expect(await readFile(pkgPath, "utf8")).toBe(written); }); @@ -488,7 +514,6 @@ test("piped output stays colored with -c, on stdout and on -V stderr, and parsea mkdirSync(flagDir, {recursive: true}); const pkgPath = join(flagDir, "package.json"); await writeFile(pkgPath, JSON.stringify({dependencies: {prismjs: "1.0.0"}})); - // FORCE_COLOR is what a pipe would otherwise need, and -c has to stand on its own const env = {...process.env, FORCE_COLOR: "0"}; const {stdout: colored, stderr: verbose} = await execFileAsync(execPath, [script, "-c", "-V", ...apiArgs(), "-f", pkgPath], {env}); @@ -525,307 +550,59 @@ if (!versions.bun) { } +const latestRows: Array<[string, string, string, ReturnType]> = [ + ["npm", "dependencies", "@babel/preset-env", dep("https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", "7.11.5", "7.0.0")], + ["npm", "dependencies", "eslint-plugin-storybook", dep("https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", "10.0.0-beta.6", "10.0.0-beta.5")], + ["npm", "dependencies", "gulp-sourcemaps", dep("https://github.com/gulp-sourcemaps/gulp-sourcemaps", "2.6.5", "2.0.0")], + ["npm", "dependencies", "html-webpack-plugin", dep("https://github.com/jantimon/html-webpack-plugin", "4.0.0-beta.11", "4.0.0-alpha.2")], + ["npm", "dependencies", "jpeg-buffer-orientation", dep("https://github.com/fisker/jpeg-buffer-orientation", "2.0.3", "0.0.0")], + ["npm", "dependencies", "noty", dep("https://github.com/needim/noty", "3.1.4", "3.1.0")], + ["npm", "dependencies", "prismjs", dep("https://github.com/LeaVerou/prism", "1.17.1", "1.0.0")], + ["npm", "dependencies", "react", dep("https://github.com/facebook/react/tree/HEAD/packages/react", "18.2", "18.0")], + ["npm", "dependencies", "styled-components", dep("https://github.com/styled-components/styled-components", "4.4.1", "2.5.0-1")], + ["npm", "overrides", "noty", dep("https://github.com/needim/noty", "3.1.4", "3.1.0")], + ["npm", "overrides", "prismjs:overrides.@babel/preset-env.prismjs", dep("https://github.com/LeaVerou/prism", "1.17.1", "1.0.0")], + ["npm", "overrides", "prismjs:overrides.prismjs", dep("https://github.com/LeaVerou/prism", "1.17.1", "1.0.0")], + ["npm", "packageManager", "npm", dep("https://github.com/npm/cli", "11.6.2", "11.6.0")], + ["npm", "peerDependencies", "@babel/preset-env", dep("https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", "~6.0.0 || ~7.11.5", "~6.0.0")], + ["npm", "peerDependencies", "typescript", dep("https://github.com/Microsoft/TypeScript", "^4 || ^5", "^4")], + ["npm", "resolutions", "versions/updates", dep("https://github.com/silverwind/updates", "^10.0.0", "^1.0.0")], +]; + test("latest", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest("-j")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "@babel/preset-env": { - "info": "https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", - "new": "7.11.5", - "old": "7.0.0", - }, - "eslint-plugin-storybook": { - "info": "https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", - "new": "10.0.0-beta.6", - "old": "10.0.0-beta.5", - }, - "gulp-sourcemaps": { - "info": "https://github.com/gulp-sourcemaps/gulp-sourcemaps", - "new": "2.6.5", - "old": "2.0.0", - }, - "html-webpack-plugin": { - "info": "https://github.com/jantimon/html-webpack-plugin", - "new": "4.0.0-beta.11", - "old": "4.0.0-alpha.2", - }, - "jpeg-buffer-orientation": { - "info": "https://github.com/fisker/jpeg-buffer-orientation", - "new": "2.0.3", - "old": "0.0.0", - }, - "noty": { - "info": "https://github.com/needim/noty", - "new": "3.1.4", - "old": "3.1.0", - }, - "prismjs": { - "info": "https://github.com/LeaVerou/prism", - "new": "1.17.1", - "old": "1.0.0", - }, - "react": { - "info": "https://github.com/facebook/react/tree/HEAD/packages/react", - "new": "18.2", - "old": "18.0", - }, - "styled-components": { - "info": "https://github.com/styled-components/styled-components", - "new": "4.4.1", - "old": "2.5.0-1", - }, - "updates": { - "info": "https://github.com/silverwind/updates", - "new": "537ccb7", - "old": "6941e05", - }, - }, - "devDependencies": { - "prismjs": { - "info": "https://github.com/LeaVerou/prism", - "new": "^1.17.1", - "old": "link:../prismjs", - }, - "updates": { - "info": "https://github.com/silverwind/updates", - "new": "^10.0.0", - "old": "file:.", - }, - }, - "packageManager": { - "npm": { - "info": "https://github.com/npm/cli", - "new": "11.6.2", - "old": "11.6.0", - }, - }, - "peerDependencies": { - "@babel/preset-env": { - "info": "https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", - "new": "~6.0.0 || ~7.11.5", - "old": "~6.0.0", - }, - "typescript": { - "info": "https://github.com/Microsoft/TypeScript", - "new": "^4 || ^5", - "old": "^4", - }, - }, - "resolutions": { - "versions/updates": { - "info": "https://github.com/silverwind/updates", - "new": "^10.0.0", - "old": "^1.0.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest("-j"))).toEqual(latestRows); }); test("prerelease", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest("-j -g -p")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "@babel/preset-env": { - "info": "https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", - "new": "7.11.5", - "old": "7.0.0", - }, - "eslint-plugin-storybook": { - "info": "https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", - "new": "10.0.0-beta.6", - "old": "10.0.0-beta.5", - }, - "gulp-sourcemaps": { - "info": "https://github.com/gulp-sourcemaps/gulp-sourcemaps", - "new": "2.6.5", - "old": "2.0.0", - }, - "html-webpack-plugin": { - "info": "https://github.com/jantimon/html-webpack-plugin", - "new": "4.0.0-beta.11", - "old": "4.0.0-alpha.2", - }, - "jpeg-buffer-orientation": { - "info": "https://github.com/fisker/jpeg-buffer-orientation", - "new": "2.0.3", - "old": "0.0.0", - }, - "noty": { - "info": "https://github.com/needim/noty", - "new": "3.2.0-beta", - "old": "3.1.0", - }, - "prismjs": { - "info": "https://github.com/LeaVerou/prism", - "new": "1.17.1", - "old": "1.0.0", - }, - "react": { - "info": "https://github.com/facebook/react/tree/HEAD/packages/react", - "new": "18.3.0-next-fecc288b7-20221025", - "old": "18.0", - }, - "styled-components": { - "info": "https://github.com/styled-components/styled-components", - "new": "5.0.0-regexrehydrate", - "old": "2.5.0-1", - }, - "updates": { - "info": "https://github.com/silverwind/updates", - "new": "537ccb7", - "old": "6941e05", - }, - }, - "devDependencies": { - "prismjs": { - "info": "https://github.com/LeaVerou/prism", - "new": "^1.17.1", - "old": "link:../prismjs", - }, - "updates": { - "info": "https://github.com/silverwind/updates", - "new": "^10.0.0", - "old": "file:.", - }, - }, - "packageManager": { - "npm": { - "info": "https://github.com/npm/cli", - "new": "11.6.2", - "old": "11.6.0", - }, - }, - "peerDependencies": { - "@babel/preset-env": { - "info": "https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", - "new": "~6.0.0 || ~7.11.5", - "old": "~6.0.0", - }, - "noty": { - "info": "https://github.com/needim/noty", - "new": ">= 3.1 || >= 3.2.0-beta", - "old": ">= 3.1", - }, - "svgstore": { - "info": "https://github.com/svgstore/svgstore", - "new": "^1.0.0 || ^2.0.0 || ^3.0.0-2", - "old": "^1.0.0 || ^2.0.0", - }, - "typescript": { - "info": "https://github.com/Microsoft/TypeScript", - "new": "^4 || ^5.5.0-dev.20240601", - "old": "^4", - }, - }, - "resolutions": { - "versions/updates": { - "info": "https://github.com/silverwind/updates", - "new": "^10.0.0", - "old": "^1.0.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest("-j -g -p"))).toEqual([ + ["npm", "dependencies", "@babel/preset-env", dep("https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", "7.11.5", "7.0.0")], + ["npm", "dependencies", "eslint-plugin-storybook", dep("https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", "10.0.0-beta.6", "10.0.0-beta.5")], + ["npm", "dependencies", "gulp-sourcemaps", dep("https://github.com/gulp-sourcemaps/gulp-sourcemaps", "2.6.5", "2.0.0")], + ["npm", "dependencies", "html-webpack-plugin", dep("https://github.com/jantimon/html-webpack-plugin", "4.0.0-beta.11", "4.0.0-alpha.2")], + ["npm", "dependencies", "jpeg-buffer-orientation", dep("https://github.com/fisker/jpeg-buffer-orientation", "2.0.3", "0.0.0")], + ["npm", "dependencies", "noty", dep("https://github.com/needim/noty", "3.2.0-beta", "3.1.0")], + ["npm", "dependencies", "prismjs", dep("https://github.com/LeaVerou/prism", "1.17.1", "1.0.0")], + ["npm", "dependencies", "react", dep("https://github.com/facebook/react/tree/HEAD/packages/react", "18.3.0-next-fecc288b7-20221025", "18.0")], + ["npm", "dependencies", "styled-components", dep("https://github.com/styled-components/styled-components", "5.0.0-regexrehydrate", "2.5.0-1")], + ["npm", "overrides", "noty", dep("https://github.com/needim/noty", "3.2.0-beta", "3.1.0")], + ["npm", "overrides", "prismjs:overrides.@babel/preset-env.prismjs", dep("https://github.com/LeaVerou/prism", "1.17.1", "1.0.0")], + ["npm", "overrides", "prismjs:overrides.prismjs", dep("https://github.com/LeaVerou/prism", "1.17.1", "1.0.0")], + ["npm", "packageManager", "npm", dep("https://github.com/npm/cli", "11.6.2", "11.6.0")], + ["npm", "peerDependencies", "@babel/preset-env", dep("https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", "~6.0.0 || ~7.11.5", "~6.0.0")], + ["npm", "peerDependencies", "noty", dep("https://github.com/needim/noty", ">= 3.1 || >= 3.2.0-beta", ">= 3.1")], + ["npm", "peerDependencies", "svgstore", dep("https://github.com/svgstore/svgstore", "^1.0.0 || ^2.0.0 || ^3.0.0-2", "^1.0.0 || ^2.0.0")], + ["npm", "peerDependencies", "typescript", dep("https://github.com/Microsoft/TypeScript", "^4 || ^5.5.0-dev.20240601", "^4")], + ["npm", "resolutions", "versions/updates", dep("https://github.com/silverwind/updates", "^10.0.0", "^1.0.0")], + ]); }); test("release", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest("-j -R")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "@babel/preset-env": { - "info": "https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", - "new": "7.11.5", - "old": "7.0.0", - }, - "gulp-sourcemaps": { - "info": "https://github.com/gulp-sourcemaps/gulp-sourcemaps", - "new": "2.6.5", - "old": "2.0.0", - }, - "jpeg-buffer-orientation": { - "info": "https://github.com/fisker/jpeg-buffer-orientation", - "new": "2.0.3", - "old": "0.0.0", - }, - "noty": { - "info": "https://github.com/needim/noty", - "new": "3.1.4", - "old": "3.1.0", - }, - "prismjs": { - "info": "https://github.com/LeaVerou/prism", - "new": "1.17.1", - "old": "1.0.0", - }, - "react": { - "info": "https://github.com/facebook/react/tree/HEAD/packages/react", - "new": "18.2", - "old": "18.0", - }, - "styled-components": { - "info": "https://github.com/styled-components/styled-components", - "new": "4.4.1", - "old": "2.5.0-1", - }, - "updates": { - "info": "https://github.com/silverwind/updates", - "new": "537ccb7", - "old": "6941e05", - }, - }, - "devDependencies": { - "prismjs": { - "info": "https://github.com/LeaVerou/prism", - "new": "^1.17.1", - "old": "link:../prismjs", - }, - "updates": { - "info": "https://github.com/silverwind/updates", - "new": "^10.0.0", - "old": "file:.", - }, - }, - "packageManager": { - "npm": { - "info": "https://github.com/npm/cli", - "new": "11.6.2", - "old": "11.6.0", - }, - }, - "peerDependencies": { - "@babel/preset-env": { - "info": "https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env", - "new": "~6.0.0 || ~7.11.5", - "old": "~6.0.0", - }, - "typescript": { - "info": "https://github.com/Microsoft/TypeScript", - "new": "^4 || ^5", - "old": "^4", - }, - }, - "resolutions": { - "versions/updates": { - "info": "https://github.com/silverwind/updates", - "new": "^10.0.0", - "old": "^1.0.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest("-j -R"))).toEqual(latestRows.filter(row => + !["eslint-plugin-storybook", "html-webpack-plugin"].includes(row[2]))); }); -// --release only narrows the field to releases; stepping down off a prerelease train whose own -// release does not exist yet is --allow-downgrade's call. test("release with allow-downgrade", async ({expect = globalExpect}: any = {}) => { - const results = await makeTest("-j -R -d")(); + const results = await makeTest("-j -R -d"); expect(results.npm.dependencies["eslint-plugin-storybook"]).toEqual({ info: "https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", new: "9.1.7", @@ -834,70 +611,25 @@ test("release with allow-downgrade", async ({expect = globalExpect}: any = {}) = }); test("patch", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest("-j -P")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "eslint-plugin-storybook": { - "info": "https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", - "new": "10.0.0-beta.6", - "old": "10.0.0-beta.5", - }, - "gulp-sourcemaps": { - "info": "https://github.com/floridoo/gulp-sourcemaps", - "new": "2.0.1", - "old": "2.0.0", - }, - "html-webpack-plugin": { - "info": "https://github.com/jantimon/html-webpack-plugin", - "new": "4.0.0-beta.11", - "old": "4.0.0-alpha.2", - }, - "noty": { - "info": "https://github.com/needim/noty", - "new": "3.1.4", - "old": "3.1.0", - }, - "updates": { - "info": "https://github.com/silverwind/updates", - "new": "537ccb7", - "old": "6941e05", - }, - }, - "devDependencies": { - "prismjs": { - "info": "https://github.com/LeaVerou/prism", - "new": "^0.0.1", - "old": "link:../prismjs", - }, - }, - "packageManager": { - "npm": { - "info": "https://github.com/npm/cli", - "new": "11.6.2", - "old": "11.6.0", - }, - }, - "resolutions": { - "versions/updates": { - "info": "https://github.com/silverwind/updates", - "new": "^1.0.6", - "old": "^1.0.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest("-j -P"))).toEqual([ + ["npm", "dependencies", "eslint-plugin-storybook", dep("https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", "10.0.0-beta.6", "10.0.0-beta.5")], + ["npm", "dependencies", "gulp-sourcemaps", dep("https://github.com/floridoo/gulp-sourcemaps", "2.0.1", "2.0.0")], + ["npm", "dependencies", "html-webpack-plugin", dep("https://github.com/jantimon/html-webpack-plugin", "4.0.0-beta.11", "4.0.0-alpha.2")], + ["npm", "dependencies", "noty", dep("https://github.com/needim/noty", "3.1.4", "3.1.0")], + ["npm", "overrides", "noty", dep("https://github.com/needim/noty", "3.1.4", "3.1.0")], + ["npm", "packageManager", "npm", dep("https://github.com/npm/cli", "11.6.2", "11.6.0")], + ["npm", "resolutions", "versions/updates", dep("https://github.com/silverwind/updates", "^1.0.6", "^1.0.0")], + ]); }); -const notyResult = {npm: {dependencies: {noty: {info: "https://github.com/needim/noty", new: "3.1.4", old: "3.1.0"}}}}; +const notyDep = {info: "https://github.com/needim/noty", new: "3.1.4", old: "3.1.0"}; +const notyResult = {npm: {dependencies: {noty: notyDep}, overrides: {noty: notyDep}}}; -// Also covers preup: don't upgrade stable to prerelease (3.1.0 -> 3.1.4, not 3.2.0-beta from latest dist-tag). test.each([ ["include", "-j -i noty"], ["include 2", "-j -i /^noty/"], ])("%s", async (_name, args, {expect = globalExpect}: any = {}) => { - expect(await makeTest(args)()).toEqual(notyResult); + expect(await makeTest(args)).toEqual(notyResult); }); // Out of process, unlike its siblings: the in-process registry cache is keyed by URL alone, so @@ -908,110 +640,53 @@ test("cooldown duration", async ({expect = globalExpect}: any = {}) => { ]); const {results} = JSON.parse(stdout); delete results.npm.dependencies.noty.age; + delete results.npm.overrides.noty.age; expect(results).toEqual(notyResult); }); test("packageManager", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest("-j -i npm")()).toMatchInlineSnapshot(` - { - "npm": { - "packageManager": { - "npm": { - "info": "https://github.com/npm/cli", - "new": "11.6.2", - "old": "11.6.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest("-j -i npm"))).toEqual([ + ["npm", "packageManager", "npm", dep("https://github.com/npm/cli", "11.6.2", "11.6.0")], + ]); }); test("overrides type", async ({expect = globalExpect}: any = {}) => { - // the nested override carries no version of its own and is no dependency, and the top-level - // `prismjs` it shadows is skipped too, as a rewrite would land in the nested copy - expect(await makeTest("-j -t overrides")()).toEqual({npm: {overrides: notyResult.npm.dependencies}}); + expect(await makeTest("-j -t overrides")).toEqual({npm: {overrides: { + noty: notyDep, + "prismjs:overrides.@babel/preset-env.prismjs": { + info: "https://github.com/LeaVerou/prism", new: "1.17.1", old: "1.0.0", + }, + "prismjs:overrides.prismjs": { + info: "https://github.com/LeaVerou/prism", new: "1.17.1", old: "1.0.0", + }, + }}}); }); test("exclude", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest("-j -e gulp-sourcemaps -i /react/")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "react": { - "info": "https://github.com/facebook/react/tree/HEAD/packages/react", - "new": "18.2", - "old": "18.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest("-j -e gulp-sourcemaps -i /react/"))).toEqual([ + ["npm", "dependencies", "react", dep("https://github.com/facebook/react/tree/HEAD/packages/react", "18.2", "18.0")], + ]); }); test("exclude 2", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest("-j -i gulp*")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "gulp-sourcemaps": { - "info": "https://github.com/gulp-sourcemaps/gulp-sourcemaps", - "new": "2.6.5", - "old": "2.0.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest("-j -i gulp*"))).toEqual([ + ["npm", "dependencies", "gulp-sourcemaps", dep("https://github.com/gulp-sourcemaps/gulp-sourcemaps", "2.6.5", "2.0.0")], + ]); }); test("exclude 3", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest("-j -i /^gulp/ -P gulp*")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "gulp-sourcemaps": { - "info": "https://github.com/floridoo/gulp-sourcemaps", - "new": "2.0.1", - "old": "2.0.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest("-j -i /^gulp/ -P gulp*"))).toEqual([ + ["npm", "dependencies", "gulp-sourcemaps", dep("https://github.com/floridoo/gulp-sourcemaps", "2.0.1", "2.0.0")], + ]); }); test("uv", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest(`-j -f ${uvFile}`)()).toMatchInlineSnapshot(` - { - "pypi": { - "dependency-groups.dev": { - "PyYAML": { - "info": "https://github.com/yaml/pyyaml", - "new": "6.0", - "old": "1.0", - }, - "types-requests": { - "info": "https://github.com/python/typeshed", - "new": "2.32.4.20250611", - "old": "2.32.0.20240622", - }, - }, - "project.dependencies": { - "djlint": { - "info": "https://github.com/Riverside-Healthcare/djlint", - "new": "1.31.0", - "old": "1.30.0", - }, - "ty": { - "info": "https://github.com/astral-sh/ty", - "new": "0.0.1a19", - "old": "0.0.1a15", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest(`-j -f ${uvFile}`))).toEqual([ + ["pypi", "dependency-groups.dev", "PyYAML", dep("https://github.com/yaml/pyyaml", "6.0", "1.0")], + ["pypi", "dependency-groups.dev", "types-requests", dep("https://github.com/python/typeshed", "2.32.4.20250611", "2.32.0.20240622")], + ["pypi", "project.dependencies", "djlint", dep("https://github.com/Riverside-Healthcare/djlint", "1.31.0", "1.30.0")], + ["pypi", "project.dependencies", "ty", dep("https://github.com/astral-sh/ty", "0.0.1a19", "0.0.1a15")], + ]); }); test("invalid config", async ({expect = globalExpect}: any = {}) => { @@ -1027,85 +702,34 @@ test("invalid config", async ({expect = globalExpect}: any = {}) => { } }); -test("preup 1", async ({expect = globalExpect}: any = {}) => { - // Test that we DO upgrade to prerelease when explicitly requested with -p flag - // noty: 3.1.0 -> should suggest 3.2.0-beta (from latest dist-tag) when -p is used - expect(await makeTest("-j -i noty -p")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "noty": { - "info": "https://github.com/needim/noty", - "new": "3.2.0-beta", - "old": "3.1.0", - }, - }, - "peerDependencies": { - "noty": { - "info": "https://github.com/needim/noty", - "new": ">= 3.1 || >= 3.2.0-beta", - "old": ">= 3.1", - }, - }, - }, - } - `); -}); - -test("preup 2", async ({expect = globalExpect}: any = {}) => { - // Test that upgrading from prerelease to prerelease works without -p flag - // eslint-plugin-storybook: 10.0.0-beta.5 -> should allow upgrade to another prerelease - expect(await makeTest("-j -i eslint-plugin-storybook")()).toMatchInlineSnapshot(` - { - "npm": { - "dependencies": { - "eslint-plugin-storybook": { - "info": "https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", - "new": "10.0.0-beta.6", - "old": "10.0.0-beta.5", - }, - }, - }, - } - `); +test("prerelease selection", async ({expect = globalExpect}: any = {}) => { + expect(dependencyRows(await makeTest("-j -i noty -p"))).toEqual([ + ["npm", "dependencies", "noty", dep("https://github.com/needim/noty", "3.2.0-beta", "3.1.0")], + ["npm", "overrides", "noty", dep("https://github.com/needim/noty", "3.2.0-beta", "3.1.0")], + ["npm", "peerDependencies", "noty", dep("https://github.com/needim/noty", ">= 3.1 || >= 3.2.0-beta", ">= 3.1")], + ]); + expect(dependencyRows(await makeTest("-j -i eslint-plugin-storybook"))).toEqual([ + ["npm", "dependencies", "eslint-plugin-storybook", dep("https://github.com/storybookjs/storybook/tree/HEAD/code/lib/eslint-plugin", "10.0.0-beta.6", "10.0.0-beta.5")], + ]); }); test("go", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest(`-j -f ${goFile}`)()).toMatchInlineSnapshot(` - { - "go": { - "deps": { - "github.com/example/listonly": { - "info": "https://github.com/example/listonly", - "new": "1.2.0", - "old": "1.0.0", - }, - "github.com/example/listtime": { - "info": "https://github.com/example/listtime", - "new": "1.1.0", - "old": "1.0.0", - }, - "github.com/google/go-github/v70": { - "info": "https://github.com/google/go-github", - "new": "82.0.0", - "old": "70.0.0", - }, - "github.com/google/uuid": { - "info": "https://github.com/google/uuid", - "new": "1.6.0", - "old": "1.5.0", - }, - }, - }, - } - `); + const directRows = [ + ["go", "deps", "github.com/example/listonly", dep("https://github.com/example/listonly", "1.2.0", "1.0.0")], + ["go", "deps", "github.com/example/listtime", dep("https://github.com/example/listtime", "1.1.0", "1.0.0")], + ["go", "deps", "github.com/google/go-github/v70", dep("https://github.com/google/go-github", "82.0.0", "70.0.0")], + ["go", "deps", "github.com/google/uuid", dep("https://github.com/google/uuid", "2.0.0-2026021", "1.5.0")], + ]; + expect(dependencyRows(await makeTest(`-j -f ${goFile}`))).toEqual(directRows); + expect(dependencyRows(await makeTest(`-j -f ${goFile} -I`))).toEqual([ + ...directRows, + ["go", "indirect", "github.com/example/testpkg", dep("https://github.com/example/testpkg", "2.0.0", "0.9.0")], + ]); }); test("go @v/list fallback takes a date off the list line when .info is absent", async ({expect = globalExpect}: any = {}) => { const {stdout} = await runCliExec([script, "-j", "-f", goFile, "-c", "--goproxy", goProxyUrl]); const {deps} = JSON.parse(stdout).results.go; - expect(deps["github.com/example/listonly"].new).toBe("1.2.0"); - expect(deps["github.com/example/listtime"].new).toBe("1.1.0"); expect(deps["github.com/example/listtime"].age).toBeTruthy(); }); @@ -1133,71 +757,11 @@ test("color flags reach the config", async ({expect = globalExpect}: any = {}) = }); test("cargo", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest(`-j -f ${cargoFile}`)()).toMatchInlineSnapshot(` - { - "cargo": { - "dependencies": { - "tokio": { - "info": "https://crates.io/crates/tokio", - "new": "1.35", - "old": "1.0", - }, - }, - "dev-dependencies": { - "rand": { - "info": "https://crates.io/crates/rand", - "new": "0.9", - "old": "0.8", - }, - }, - "target.cfg(unix).dependencies": { - "rand": { - "info": "https://crates.io/crates/rand", - "new": "0.9", - "old": "0.8", - }, - }, - }, - } - `); -}); - -test("go indirect with -I flag", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest(`-j -f ${goFile} -I`)()).toMatchInlineSnapshot(` - { - "go": { - "deps": { - "github.com/example/listonly": { - "info": "https://github.com/example/listonly", - "new": "1.2.0", - "old": "1.0.0", - }, - "github.com/example/listtime": { - "info": "https://github.com/example/listtime", - "new": "1.1.0", - "old": "1.0.0", - }, - "github.com/google/go-github/v70": { - "info": "https://github.com/google/go-github", - "new": "82.0.0", - "old": "70.0.0", - }, - "github.com/google/uuid": { - "info": "https://github.com/google/uuid", - "new": "1.6.0", - "old": "1.5.0", - }, - }, - "indirect": { - "github.com/example/testpkg": { - "info": "https://github.com/example/testpkg", - "new": "1.0.0", - "old": "0.9.0", - }, - }, - }, - } - `); + expect(dependencyRows(await makeTest(`-j -f ${cargoFile}`))).toEqual([ + ["cargo", "dependencies", "tokio", dep("https://crates.io/crates/tokio", "1.35", "1.0")], + ["cargo", "dev-dependencies", "rand", dep("https://crates.io/crates/rand", "0.9", "0.8")], + ["cargo", "target.cfg(unix).dependencies", "rand", dep("https://crates.io/crates/rand", "0.9", "0.8")], + ]); }); test("go update", async ({expect = globalExpect}: any = {}) => { @@ -1219,12 +783,12 @@ test("go update", async ({expect = globalExpect}: any = {}) => { const updatedContent = await readFile(join(testGoModDir, "go.mod"), "utf8"); - expect(updatedContent).toContain("github.com/google/uuid v1.6.0"); + expect(updatedContent).toContain("github.com/google/uuid/v2 v2.0.0-20260217135312-8c5a7de9ffa1"); expect(updatedContent).not.toContain("uuid v1.5.0"); expect(updatedContent).not.toContain("go-github/v70"); expect(updatedContent).toMatch(/github\.com\/google\/go-github\/v\d+ v\d+\.\d+\.\d+/); - const matches = updatedContent.match(/github\.com\/google\/uuid v1\.6\.0/g); + const matches = updatedContent.match(/github\.com\/google\/uuid\/v2 v2\.0\.0-20260217135312-8c5a7de9ffa1/g); expect(matches).toBeTruthy(); expect(matches?.length).toBe(4); @@ -1258,37 +822,20 @@ test("go update v1 to v2", async ({expect = globalExpect}: any = {}) => { expect(updatedMain).not.toMatch(/"github\.com\/example\/testpkg"(?!\/v2)/); }); -test("go prerelease excluded by default", async ({expect = globalExpect}: any = {}) => { - // Without --prerelease, Go prerelease versions should not be offered - expect(await makeTest(`-j -f ${goPreFile}`)()).toMatchInlineSnapshot(`undefined`); -}); - -test("go prerelease with -p flag", async ({expect = globalExpect}: any = {}) => { - // With global --prerelease, Go prerelease versions should be offered - expect(await makeTest(`-j -f ${goPreFile} -p`)()).toMatchInlineSnapshot(` - { - "go": { - "deps": { - "github.com/example/prerelpkg": { - "info": "https://github.com/example/prerelpkg", - "new": "1.1.0-rc.1", - "old": "1.0.0", - }, - }, - }, - } - `); +test("go prerelease is excluded by default and enabled globally or per package", async ({expect = globalExpect}: any = {}) => { + expect(await makeTest(`-j -f ${goPreFile}`)).toBeUndefined(); + const expected = [["go", "deps", "github.com/example/prerelpkg", + dep("https://github.com/example/prerelpkg", "1.1.0-rc.1", "1.0.0")]]; + for (const flag of ["-p", "-p github.com/example/prerelpkg"]) { + expect(dependencyRows(await makeTest(`-j -f ${goPreFile} ${flag}`))).toEqual(expected); + } }); test("go pseudo-version no downgrade", async ({expect = globalExpect}: any = {}) => { - // A pseudo-version like v0.4.2-0.xxx should not be downgraded to a lower release like v0.4.1 - expect(await makeTest(`-j -f ${goPseudoFile}`)()).toMatchInlineSnapshot(`undefined`); + expect(await makeTest(`-j -f ${goPseudoFile}`)).toBeUndefined(); }); test("go pseudo-version update rewrites the full version", async ({expect = globalExpect}: any = {}) => { - // Shortened pseudo-version in dep.old would partial-match the full string in - // go.mod and leave the timestamp tail behind unless dep.oldOrig is preserved - // through the write. const testGoModDir = join(testDir, "test-go-pseudo-update"); mkdirSync(testGoModDir, {recursive: true}); const modPath = join(testGoModDir, "go.mod"); @@ -1324,11 +871,8 @@ test("make mode bumps go install versions and rewrites paths on major bumps", as await updates({files: [makePath], goproxy: goProxyUrl, update: true, color: false, noCache: true}); const updated = await readFile(makePath, "utf8"); - // same-major bump (its /v2 is a pseudo-version and must be skipped); path, operator and spacing preserved - expect(updated).toContain("UUID_PACKAGE ?= github.com/google/uuid@v1.6.0"); - // cross-major bump rewrites the /vN path segment; inline comment preserved + expect(updated).toContain("UUID_PACKAGE ?= github.com/google/uuid/v2@v2.0.0-20260217135312-8c5a7de9ffa1"); expect(updated).toContain("TESTPKG_PACKAGE := github.com/example/testpkg/v2@v2.0.0 # pinned tool"); - // commented-out install and non-install line untouched expect(updated).toContain("# DISABLED := github.com/example/testpkg@v0.5.0"); expect(updated).toContain("SOURCE := $(wildcard *.go)"); }); @@ -1349,7 +893,6 @@ test("make mode bumps docker image tags and re-resolves digests in Makefiles", a mkdirSync(makeDir, {recursive: true}); const makePath = join(makeDir, "Makefile"); const oldDigest = `sha256:${"a".repeat(64)}`; - const newDigest = `sha256:${"b".repeat(64)}`; await writeFile(makePath, [ `SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.11.0@${oldDigest} # renovate: datasource=docker`, "PLAIN := koalaman/shellcheck:v0.11.0", @@ -1360,14 +903,37 @@ test("make mode bumps docker image tags and re-resolves digests in Makefiles", a await updates({files: [makePath], dockerapi: dockerUrl, update: true, color: false, noCache: true}); const updated = await readFile(makePath, "utf8"); - // tag + digest bumped, registry prefix and renovate comment preserved - expect(updated).toContain(`SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.12.0@${newDigest} # renovate: datasource=docker`); - // plain image without a digest bumped, no digest introduced + expect(updated).toContain("SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.12.0@sha256:list-new # renovate: datasource=docker"); expect(updated).toContain("PLAIN := koalaman/shellcheck:v0.12.0"); - // host:port var left untouched expect(updated).toContain("TEST_MYSQL_HOST ?= mysql:3306"); }); +test("make allowedVersions falls back to the highest allowed candidate", async ({expect = globalExpect}: any = {}) => { + const dir = join(testDir, "test-make-allowed"); + mkdirSync(dir, {recursive: true}); + const file = join(dir, "Makefile"); + await writeFile(file, [ + "TOOL := github.com/example/makeallowed/cmd/tool@v1.0.0", + "IMAGE := example/makeallowed:1.0", + "", + ].join("\n")); + await writeFile(join(dir, "renovate.json"), JSON.stringify({packageRules: [ + {matchPackageNames: ["github.com/example/makeallowed/cmd/tool"], allowedVersions: "<2"}, + {matchPackageNames: ["example/makeallowed"], allowedVersions: "<2"}, + ]})); + + await updates({ + files: [file], modes: ["make"], goproxy: goProxyUrl, dockerapi: dockerUrl, + update: true, color: false, noCache: true, + }); + + expect(await readFile(file, "utf8")).toBe([ + "TOOL := github.com/example/makeallowed/cmd/tool@v1.1.0", + "IMAGE := example/makeallowed:1.1", + "", + ].join("\n")); +}); + test("docker image names match with and without the docker.io prefix", async ({expect = globalExpect}: any = {}) => { const makeDir = join(testDir, "test-docker-io-prefix"); mkdirSync(makeDir, {recursive: true}); @@ -1378,7 +944,6 @@ test("docker image names match with and without the docker.io prefix", async ({e "", ].join("\n")); - // a pin written without the registry must hold both spellings await updates({files: [makePath], dockerapi: dockerUrl, update: true, color: false, noCache: true, pin: {"koalaman/shellcheck": "0.11.x"}}); expect(await readFile(makePath, "utf8")).toBe([ "PREFIXED := docker.io/koalaman/shellcheck:v0.11.0", @@ -1386,7 +951,6 @@ test("docker image names match with and without the docker.io prefix", async ({e "", ].join("\n")); - // and so must one written with it await updates({files: [makePath], dockerapi: dockerUrl, update: true, color: false, noCache: true, exclude: ["docker.io/koalaman/shellcheck"]}); expect(await readFile(makePath, "utf8")).toBe([ "PREFIXED := docker.io/koalaman/shellcheck:v0.11.0", @@ -1394,7 +958,6 @@ test("docker image names match with and without the docker.io prefix", async ({e "", ].join("\n")); - // an npm dep of the same name resolves its options first, and answers for neither spelling const clashDir = join(testDir, "test-docker-npm-name-clash"); mkdirSync(clashDir, {recursive: true}); await writeFile(join(clashDir, "package.json"), JSON.stringify({dependencies: {noty: "3.1.0"}})); @@ -1407,83 +970,23 @@ test("docker image names match with and without the docker.io prefix", async ({e expect(clash.results.docker).toBeUndefined(); // 18 to 22 is no patch }); -test("go prerelease with -p per-package", async ({expect = globalExpect}: any = {}) => { - // With per-package --prerelease, Go prerelease versions should be offered for that package - expect(await makeTest(`-j -f ${goPreFile} -p github.com/example/prerelpkg`)()).toMatchInlineSnapshot(` - { - "go": { - "deps": { - "github.com/example/prerelpkg": { - "info": "https://github.com/example/prerelpkg", - "new": "1.1.0-rc.1", - "old": "1.0.0", - }, - }, - }, - } - `); -}); - -test("go replace", async ({expect = globalExpect}: any = {}) => { - expect(await makeTest(`-j -f ${goReplaceFile}`)()).toMatchInlineSnapshot(` - { - "go": { - "replace": { - "gitea.com/gitea/act": { - "info": "https://gitea.com/gitea/act", - "new": "0.261.7", - "old": "0.261.4", - }, - }, - }, - } - `); -}); - -test("go replace update", async ({expect = globalExpect}: any = {}) => { +test("go replace reports and writes the update", async ({expect = globalExpect}: any = {}) => { const testGoModDir = join(testDir, "test-go-replace"); mkdirSync(testGoModDir, {recursive: true}); - - const goReplaceContent = readFileSync(goReplaceFile, "utf8"); - await writeFile(join(testGoModDir, "go.mod"), goReplaceContent); - - await runCliExec([ - script, - "-u", - "-f", join(testGoModDir, "go.mod"), - "-c", - "--goproxy", goProxyUrl, + await writeFile(join(testGoModDir, "go.mod"), readFileSync(goReplaceFile, "utf8")); + const {stdout} = await runCliExec([ + script, "-j", "-u", "-f", join(testGoModDir, "go.mod"), "-c", "--goproxy", goProxyUrl, + ]); + expect(dependencyRows(JSON.parse(stdout).results)).toMatchObject([ + ["go", "replace", "gitea.com/gitea/act", dep("https://gitea.com/gitea/act", "0.261.7", "0.261.4")], ]); - const updatedContent = await readFile(join(testGoModDir, "go.mod"), "utf8"); - expect(updatedContent).toContain("gitea.com/gitea/act v0.261.7"); expect(updatedContent).not.toContain("gitea.com/gitea/act v0.261.4"); expect(updatedContent).toContain("replace"); }); -test("go workspace", async ({expect = globalExpect}: any = {}) => { - const result = await updates({ - files: [goWorkspaceFile], - goproxy: goProxyUrl, - color: false, - noCache: true, - }); - const {go} = result.results; - expect(go).toBeDefined(); - - // Should find deps from both workspace members - const appDeps = go["deps|./app"]; - const libDeps = go["deps|./lib"]; - expect(appDeps).toBeDefined(); - expect(libDeps).toBeDefined(); - expect(appDeps["github.com/google/uuid"]).toBeDefined(); - expect(libDeps["github.com/google/uuid"]).toBeDefined(); - expect(appDeps["github.com/google/uuid"].old).toBe("1.5.0"); - expect(libDeps["github.com/google/uuid"].old).toBe("1.5.0"); -}); - -test("go workspace update", async ({expect = globalExpect}: any = {}) => { +test("go workspace reports and writes member updates", async ({expect = globalExpect}: any = {}) => { const testGoWorkDir = join(testDir, "test-go-workspace"); mkdirSync(join(testGoWorkDir, "app"), {recursive: true}); mkdirSync(join(testGoWorkDir, "lib"), {recursive: true}); @@ -1493,54 +996,22 @@ test("go workspace update", async ({expect = globalExpect}: any = {}) => { writeFileSync(join(testGoWorkDir, "app", "main.go"), readFileSync(join(goWorkspaceDir, "app", "main.go"), "utf8")); writeFileSync(join(testGoWorkDir, "lib", "go.mod"), readFileSync(join(goWorkspaceDir, "lib", "go.mod"), "utf8")); - await runCliExec([ - script, - "-u", - "-f", join(testGoWorkDir, "go.work"), - "-c", - "--goproxy", goProxyUrl, - ]); + const {go} = (await updates({ + files: [join(testGoWorkDir, "go.work")], goproxy: goProxyUrl, update: true, color: false, noCache: true, + })).results; + expect(go["deps|./app"]["github.com/google/uuid"].old).toBe("1.5.0"); + expect(go["deps|./lib"]["github.com/google/uuid"].old).toBe("1.5.0"); const appMod = await readFile(join(testGoWorkDir, "app", "go.mod"), "utf8"); const libMod = await readFile(join(testGoWorkDir, "lib", "go.mod"), "utf8"); - expect(appMod).toContain("github.com/google/uuid v1.6.0"); + expect(appMod).toContain("github.com/google/uuid/v2 v2.0.0-20260217135312-8c5a7de9ffa1"); expect(appMod).not.toContain("uuid v1.5.0"); - expect(libMod).toContain("github.com/google/uuid v1.6.0"); + expect(libMod).toContain("github.com/google/uuid/v2 v2.0.0-20260217135312-8c5a7de9ffa1"); expect(libMod).not.toContain("uuid v1.5.0"); + expect(await readFile(join(testGoWorkDir, "app", "main.go"), "utf8")).toContain('"github.com/google/uuid/v2"'); }); -test("cargo workspace", async ({expect = globalExpect}: any = {}) => { - const result = await updates({ - files: [cargoWorkspaceFile], - cargoapi: cargoUrl, - color: false, - noCache: true, - }); - const {cargo} = result.results; - expect(cargo).toBeDefined(); - - // workspace.dependencies from root (no prefix) - expect(cargo["workspace.dependencies"]).toBeDefined(); - expect(cargo["workspace.dependencies"]["serde_json"]).toBeDefined(); - expect(cargo["workspace.dependencies"]["serde_json"].old).toBe("1.0.100"); - expect(cargo["workspace.dependencies"]["serde_json"].new).toBe("1.0.120"); - - // Member deps with prefixes - const crateADeps = cargo["dependencies|./crate-a"]; - const crateBDeps = cargo["dependencies|./crate-b"]; - const crateBDevDeps = cargo["dev-dependencies|./crate-b"]; - expect(crateADeps).toBeDefined(); - expect(crateBDeps).toBeDefined(); - expect(crateBDevDeps).toBeDefined(); - expect(crateADeps["serde"].old).toBe("1.0.100"); - expect(crateBDeps["tokio"].old).toBe("1.34.0"); - expect(crateBDevDeps["rand"].old).toBe("0.8.5"); - expect(crateADeps["serde"].new).toBe("1.0.200"); - expect(crateBDeps["tokio"].new).toBe("1.35.0"); - expect(crateBDevDeps["rand"].new).toBe("0.9.0"); -}); - -test("cargo workspace update", async ({expect = globalExpect}: any = {}) => { +test("cargo workspace reports and writes root and member updates", async ({expect = globalExpect}: any = {}) => { const testCargoWorkDir = join(testDir, "test-cargo-workspace"); mkdirSync(join(testCargoWorkDir, "crate-a"), {recursive: true}); mkdirSync(join(testCargoWorkDir, "crate-b"), {recursive: true}); @@ -1550,20 +1021,20 @@ test("cargo workspace update", async ({expect = globalExpect}: any = {}) => { writeFileSync(join(testCargoWorkDir, "crate-a", "Cargo.toml"), readFileSync(join(cargoWorkspaceDir, "crate-a", "Cargo.toml"), "utf8")); writeFileSync(join(testCargoWorkDir, "crate-b", "Cargo.toml"), readFileSync(join(cargoWorkspaceDir, "crate-b", "Cargo.toml"), "utf8")); - // a member named alongside its root, in either order, is no second file: the root supersedes it const both = await updates({ files: [join(testCargoWorkDir, "crate-a", "Cargo.toml"), join(testCargoWorkDir, "Cargo.toml")], cargoapi: cargoUrl, color: false, noCache: true, }); expect(Object.keys(both.results.cargo).filter(key => key.includes("crate-a"))).toHaveLength(1); - await runCliExec([ - script, - "-u", - "-f", join(testCargoWorkDir, "Cargo.toml"), - "-c", - "--cargoapi", cargoUrl, - ]); + const {cargo} = (await updates({ + files: [join(testCargoWorkDir, "Cargo.toml")], cargoapi: cargoUrl, + update: true, color: false, noCache: true, + })).results; + expect(cargo["workspace.dependencies"]["serde_json"]).toMatchObject({old: "1.0.100", new: "1.0.120"}); + expect(cargo["dependencies|./crate-a"].serde).toMatchObject({old: "1.0.100", new: "1.0.200"}); + expect(cargo["dependencies|./crate-b"].tokio).toMatchObject({old: "1.34.0", new: "1.35.0"}); + expect(cargo["dev-dependencies|./crate-b"].rand).toMatchObject({old: "0.8.5", new: "0.9.0"}); const rootToml = await readFile(join(testCargoWorkDir, "Cargo.toml"), "utf8"); const crateAToml = await readFile(join(testCargoWorkDir, "crate-a", "Cargo.toml"), "utf8"); @@ -1574,6 +1045,31 @@ test("cargo workspace update", async ({expect = globalExpect}: any = {}) => { expect(crateBToml).toContain('rand = "0.9.0"'); }); +test("multiple Cargo workspace roots keep member config identity", async ({expect = globalExpect}: any = {}) => { + const dir = mkdtempSync(join(tmpdir(), "updates-cargo-workspaces-")); + const roots = [join(dir, "one"), join(dir, "two")]; + try { + for (const [index, root] of roots.entries()) { + mkdirSync(join(root, "crates", "app"), {recursive: true}); + await writeFile(join(root, "Cargo.toml"), '[workspace]\nmembers = ["crates/*"]\n'); + await writeFile(join(root, "crates", "app", "Cargo.toml"), '[package]\nname = "app"\nversion = "0.1.0"\n\n[dependencies]\nserde = "1.0.0"\n'); + await writeFile(join(root, "renovate.json"), JSON.stringify({packageRules: [{ + matchPackageNames: ["serde"], allowedVersions: index === 0 ? "<=1.0.100" : "<=1.0.200", + }]})); + } + + await updates({ + files: roots.map(root => join(root, "Cargo.toml")), cargoapi: cargoUrl, + modes: ["cargo"], update: true, color: false, noCache: true, + }); + + expect(await readFile(join(roots[0], "crates", "app", "Cargo.toml"), "utf8")).toContain('serde = "1.0.100"'); + expect(await readFile(join(roots[1], "crates", "app", "Cargo.toml"), "utf8")).toContain('serde = "1.0.200"'); + } finally { + await rm(dir, {recursive: true, force: true, maxRetries: 10, retryDelay: 100}); + } +}); + test("pnpm workspace", async ({expect = globalExpect}: any = {}) => { const opts = { files: [pnpmWorkspaceFile], @@ -1584,26 +1080,17 @@ test("pnpm workspace", async ({expect = globalExpect}: any = {}) => { }; const result = await updates(opts); const {npm} = result.results; - expect(npm).toBeDefined(); - - // Root deps (no prefix) - expect(npm["devDependencies"]).toBeDefined(); - expect(npm["devDependencies"]["typescript"]).toBeDefined(); + expect(npm["devDependencies"]["typescript"].new).toBeTruthy(); - // Member deps with prefixes const appADeps = npm["dependencies|./packages/app-a"]; const libBDeps = npm["dependencies|./packages/lib-b"]; - expect(appADeps).toBeDefined(); - expect(libBDeps).toBeDefined(); - expect(appADeps["prismjs"]).toBeDefined(); - expect(libBDeps["react"]).toBeDefined(); - - expect(npm["catalog|pnpm-workspace.yaml"]["svgstore"]).toBeDefined(); - expect(npm["catalogs.build|pnpm-workspace.yaml"]["typescript"]).toBeDefined(); - // lib-b's `svgstore: "catalog:"` names the catalog and holds no range of its own + expect(appADeps["prismjs"].new).toBeTruthy(); + expect(libBDeps["react"].new).toBeTruthy(); + + expect(npm["catalog|pnpm-workspace.yaml"]["svgstore"].new).toBeTruthy(); + expect(npm["catalogs.build|pnpm-workspace.yaml"]["typescript"].new).toBeTruthy(); expect(libBDeps["svgstore"]).toBeUndefined(); - // a member's own manifest, listed first as a run started inside it would find it, is no second file const fromMember = await updates({ ...opts, files: [join(pnpmWorkspaceDir, "packages", "app-a", "package.json"), pnpmWorkspaceFile], @@ -1640,24 +1127,63 @@ test("pnpm workspace update, and a second run is a no-op", async ({expect = glob }); test("pnpm workspace alongside unrelated package.json", async ({expect = globalExpect}: any = {}) => { - // A pnpm workspace and an unrelated plain package.json (second -f path) must both be - // processed, regardless of order — the workspace no longer suppresses the plain manifest. for (const files of [[pnpmWorkspaceFile, testFile], [testFile, pnpmWorkspaceFile]]) { const result = await updates({files, registry: npmUrl, forgeapi: githubUrl, color: false, noCache: true}); const {npm} = result.results; - expect(npm).toBeDefined(); - - // workspace root + members still resolve - expect(npm["devDependencies"]?.["typescript"]).toBeDefined(); - expect(npm["dependencies|./packages/app-a"]?.["prismjs"]).toBeDefined(); - expect(npm["dependencies|./packages/lib-b"]?.["react"]).toBeDefined(); + expect(npm["devDependencies"]["typescript"].new).toBeTruthy(); + expect(npm["dependencies|./packages/app-a"]["prismjs"].new).toBeTruthy(); + expect(npm["dependencies|./packages/lib-b"]["react"].new).toBeTruthy(); - // the unrelated plain package.json (gulp-sourcemaps is unique to it) is no longer dropped const allNames = Object.values(npm).flatMap((group: any) => Object.keys(group)); expect(allNames).toContain("gulp-sourcemaps"); } }); +test("multiple npm workspace roots keep dependency and config identity", async ({expect = globalExpect}: any = {}) => { + const dir = join(testDir, "test-multiple-npm-workspaces"); + const roots = [join(dir, "array"), join(dir, "object")]; + for (const root of roots) mkdirSync(join(root, "packages", "app"), {recursive: true}); + await writeFile(join(roots[0], "package.json"), JSON.stringify({ + workspaces: ["packages/*"], dependencies: {react: "^17.0.0"}, + })); + await writeFile(join(roots[1], "package.json"), JSON.stringify({ + workspaces: {packages: ["packages/*"]}, dependencies: {react: "^17.0.0"}, + })); + for (const root of roots) { + await writeFile(join(root, "packages", "app", "package.json"), JSON.stringify({dependencies: {noty: "^3.1.0"}})); + } + await writeFile(join(roots[0], "renovate.json"), JSON.stringify({packageRules: [ + {matchPackageNames: ["react"], allowedVersions: "<=18.2.0"}, + {matchPackageNames: ["noty"], allowedVersions: "<=3.1.4"}, + ]})); + await writeFile(join(roots[1], "renovate.json"), JSON.stringify({packageRules: [ + {matchPackageNames: ["react"], allowedVersions: "<=18.1.0"}, + {matchPackageNames: ["noty"], allowedVersions: "<=3.1.3"}, + ]})); + + await updates(apiOpts({files: roots.map(root => join(root, "package.json")), modes: ["npm"], update: true})); + + expect(JSON.parse(await readFile(join(roots[0], "package.json"), "utf8")).dependencies.react).toBe("^18.2.0"); + expect(JSON.parse(await readFile(join(roots[1], "package.json"), "utf8")).dependencies.react).toBe("^18.1.0"); + expect(JSON.parse(await readFile(join(roots[0], "packages", "app", "package.json"), "utf8")).dependencies.noty).toBe("^3.1.4"); + expect(JSON.parse(await readFile(join(roots[1], "packages", "app", "package.json"), "utf8")).dependencies.noty).toBe("^3.1.3"); +}); + +test("local npm dependencies are neither requested nor rewritten", async ({expect = globalExpect}: any = {}) => { + const dir = join(testDir, "test-local-npm-dependencies"); + mkdirSync(dir, {recursive: true}); + const file = join(dir, "package.json"); + const content = `${JSON.stringify({dependencies: { + "local-file": "file:../local-file", + "local-link": "link:../local-link", + }}, null, 2)}\n`; + await writeFile(file, content); + const requestsBefore = localDependencyRequests; + await updates(apiOpts({files: [file], modes: ["npm"], update: true})); + expect(localDependencyRequests).toBe(requestsBefore); + expect(await readFile(file, "utf8")).toBe(content); +}); + test("pin holds the range and keeps the authored precision", async ({expect = globalExpect}: any = {}) => { const result = await updates({ files: [testFile], @@ -1670,7 +1196,6 @@ test("pin holds the range and keeps the authored precision", async ({expect = gl }); const {npm} = result.results; - // prismjs should be updated but only within the ^1.0.0 range expect(npm.dependencies.prismjs).toBeDefined(); expect(satisfies(npm.dependencies.prismjs.new, "^1.0.0")).toBe(true); @@ -1694,12 +1219,10 @@ test("a config-file pin and overrides merge with the renovate ones rather than r const output = await updates(apiOpts({files: [file]})); expect(output.results.npm.dependencies.noty.new).toBe("3.1.3"); - // the authored entry comes last, so api.ts's last-match-wins still gives it precedence const {args, positionals} = parseCliArgs(["-f", file]); - expect((await resolveConfig(args, positionals)).overrides).toEqual([ - {include: ["esbuild"], cooldown: 1}, - {include: ["gulp-sourcemaps"], greatest: true}, - ]); + const resolved = await resolveConfig(args, positionals) as UpdatesOptions & {renovateVersionRules: Array>}; + expect(resolved.overrides).toEqual([{include: ["gulp-sourcemaps"], greatest: true}]); + expect(resolved.renovateVersionRules).toContainEqual({matchPackageNames: ["esbuild"], cooldownDays: 1}); } finally { try { await rm(dir, {recursive: true, force: true, maxRetries: 10, retryDelay: 100}); @@ -1708,7 +1231,6 @@ test("a config-file pin and overrides merge with the renovate ones rather than r }); function actionsArgs(...extra: Array) { - // --dockerapi keeps the subprocess prewarm's docker HEAD on the mock. return [script, "-c", "--forgeapi", githubUrl, "--dockerapi", dockerUrl, "-M", "actions", "-f", actionsDir, ...extra]; } @@ -1717,6 +1239,26 @@ function getActionsDeps(results: any) { return results.actions[ciType!]; } +test("branch-only actions do not fetch forge metadata", async () => { + let requests = 0; + const server = makeServer((_req, res) => { + requests++; + res.statusCode = 500; + res.end(); + }); + const dir = mkdtempSync(join(tmpdir(), "updates-actions-branches-")); + const workflow = join(dir, ".github", "workflows", "ci.yml"); + mkdirSync(join(dir, ".github", "workflows"), {recursive: true}); + writeFileSync(workflow, "jobs:\n test:\n steps:\n - uses: one/repo@main\n - uses: two/repo@develop\n"); + await server.start(0); + try { + await updates({files: [workflow], modes: ["actions"], forgeapi: makeUrl(server), noCache: true, noColor: true}); + expect(requests).toBe(0); + } finally { + await Promise.all([server.close(), rm(dir, {recursive: true, force: true})]); + } +}); + test("actions basic", async ({expect = globalExpect}: any = {}) => { const {stdout, stderr} = await runCliExec(actionsArgs("-j")); expect(stderr).toEqual(""); @@ -1724,7 +1266,6 @@ test("actions basic", async ({expect = globalExpect}: any = {}) => { expect(output.results.actions).toBeDefined(); const actionsDeps = getActionsDeps(output.results); - // actions/checkout v2 -> v10 (v10 tag exists, precision preserved) expect(actionsDeps["actions/checkout"].old).toBe("2"); expect(actionsDeps["actions/checkout"].new).toBe("10"); expect(actionsDeps["actions/checkout"].info).toContain("actions/checkout"); @@ -1734,7 +1275,6 @@ test("actions basic", async ({expect = globalExpect}: any = {}) => { expect(actionsDeps["actions/setup-node@v1.0.0"].old).toBe("1.0.0"); expect(actionsDeps["actions/setup-node@v1.0.0"].new).toBe("10.0.0"); - // Docker, local, and hash-pinned without tags should be skipped expect(actionsDeps["tj-actions/changed-files"]).toBeUndefined(); }); @@ -1745,7 +1285,6 @@ test("actions include filter, with no false upgrade on the same major", async ({ expect(actionsDeps["actions/checkout"].old).toBe("2"); expect(actionsDeps["actions/checkout"].new).toBe("10"); expect(actionsDeps["actions/setup-node"]).toBeUndefined(); - // the `@v10` line resolves to v10.0.1, which formats back to the ref it was authored with expect(Object.keys(actionsDeps).filter(key => actionsDeps[key].old === "10")).toHaveLength(0); }); @@ -1788,7 +1327,7 @@ test("actions positional args", async ({expect = globalExpect}: any = {}) => { expect(actionsDeps["actions/setup-node@v1.0"].new).toBe("10.0.0"); }); -test("actions update rewrites a tag, a short-tag fallback and a sha pin with its comment", async ({expect = globalExpect}: any = {}) => { +test("actions update rewrites tags and keeps same-sha pin identities distinct", async ({expect = globalExpect}: any = {}) => { const tmpActionsDir = join(testDir, "actions-update-test/.github/workflows"); mkdirSync(tmpActionsDir, {recursive: true}); const wfPath = join(tmpActionsDir, "ci.yaml"); @@ -1801,7 +1340,9 @@ test("actions update rewrites a tag, a short-tag fallback and a sha pin with its " steps:", " - uses: actions/checkout@v2", " - uses: actions/setup-node@v1", - " - uses: actions/checkout@cccc000000000000000000000000000000000006 # v4.2.0", + " - uses: actions/checkout@dddd000000000000000000000000000000000000 # v4.2.0", + " - uses: actions/checkout@dddd000000000000000000000000000000000000 # main", + " - uses: actions/checkout@dddd000000000000000000000000000000000000 # release", "", ].join("\n")); @@ -1814,15 +1355,18 @@ test("actions update rewrites a tag, a short-tag fallback and a sha pin with its expect(updatedContent).toContain("actions/checkout@v10\n"); expect(updatedContent).not.toContain("actions/checkout@v2"); expect(updatedContent).toContain("actions/setup-node@v10.0.0"); - expect(updatedContent).toContain("actions/checkout@cccc000000000000000000000000000000000011 # v10.0.1"); - expect(updatedContent).not.toContain("cccc000000000000000000000000000000000006"); + expect(updatedContent).toContain("actions/checkout@cccc000000000000000000000000000000000006 # v4.2.0"); + expect(updatedContent).toContain("actions/checkout@aaaa000000000000000000000000000000000001 # main"); + expect(updatedContent).toContain("actions/checkout@bbbb000000000000000000000000000000000002 # release"); + expect(updatedContent).not.toContain("dddd000000000000000000000000000000000000"); }); test("actions hash-pinned", async ({expect = globalExpect}: any = {}) => { const tmpActionsDir = join(testDir, "actions-hash-test/.github/workflows"); mkdirSync(tmpActionsDir, {recursive: true}); const wfPath = join(tmpActionsDir, "ci.yaml"); - await writeFile(wfPath, "name: ci\non: push\njobs:\n ci:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@cccc000000000000000000000000000000000006 # v4.2.0\n"); + const oldDigest = "dddd000000000000000000000000000000000000"; + await writeFile(wfPath, `name: ci\non: push\njobs:\n ci:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@${oldDigest} # v4.2.0\n`); const {stdout, stderr} = await runCliExec([ script, "-j", "-c", "--forgeapi", githubUrl, "-M", "actions", @@ -1832,9 +1376,10 @@ test("actions hash-pinned", async ({expect = globalExpect}: any = {}) => { const output = JSON.parse(stdout); const ciKey = Object.keys(output.results.actions).find(t => t.endsWith("ci.yaml")); const actionsDeps = output.results.actions[ciKey!]; - expect(actionsDeps["actions/checkout"].old).toBe("4.2.0"); - expect(actionsDeps["actions/checkout"].new).toBe("10.0.1"); - expect(actionsDeps["actions/checkout"].age).toBeTruthy(); + expect(actionsDeps["actions/checkout"].old).toBe("v4.2.0"); + expect(actionsDeps["actions/checkout"].new).toBe("v4.2.0"); + expect(actionsDeps["actions/checkout"].oldDigest).toBe(oldDigest); + expect(actionsDeps["actions/checkout"].newDigest).toBe("cccc000000000000000000000000000000000006"); }); test("actions composite action discovery", async ({expect = globalExpect}: any = {}) => { @@ -1849,9 +1394,6 @@ test("actions composite action discovery", async ({expect = globalExpect}: any = const wfKey = Object.keys(results).find(k => k.endsWith("workflows/ci.yml")); const compKey = Object.keys(results).find(k => k.endsWith("my-action/action.yml")); const nestedKey = Object.keys(results).find(k => k.endsWith("nested/sub/action.yaml")); - expect(wfKey).toBeDefined(); - expect(compKey).toBeDefined(); - expect(nestedKey).toBeDefined(); expect(results[wfKey!]["actions/checkout"].new).toBe("10"); expect(results[compKey!]["actions/setup-node"].new).toBe("10.0.0"); expect(results[nestedKey!]["actions/checkout"].new).toBe("10"); @@ -1879,78 +1421,43 @@ test("actions composite action update, in every forge dir", async ({expect = glo } }); -// -- Docker tests -- function dockerArgs(...extra: Array) { return [script, "-c", "--dockerapi", dockerUrl, "-M", "docker", ...extra]; } -test("docker Dockerfile basic", async ({expect = globalExpect}: any = {}) => { - const {stdout, stderr} = await runCliExec(dockerArgs("-j", "-f", dockerfileFixture)); - expect(stderr).toEqual(""); - const output = JSON.parse(stdout); - expect(output.results.docker).toBeDefined(); - - const dockerfileKey = Object.keys(output.results.docker).find(t => t.endsWith("Dockerfile")); - expect(dockerfileKey).toBeDefined(); - const dockerDeps = output.results.docker[dockerfileKey!]; - - expect(dockerDeps["node:18"].old).toBe("18"); - expect(dockerDeps["node:18"].new).toBe("22"); - expect(dockerDeps["node:18"].info).toBe("https://hub.docker.com/_/node"); - expect(dockerDeps["node:20"].old).toBe("20"); - expect(dockerDeps["node:20"].new).toBe("22"); - - // postgres:15-alpine -> postgres:17-alpine (suffix preserved, oldOrig shown as old) - expect(dockerDeps.postgres.old).toBe("15-alpine"); - expect(dockerDeps.postgres.new).toBe("17-alpine"); - expect(dockerDeps.postgres.info).toBe("https://hub.docker.com/_/postgres"); -}); - -test("docker compose basic", async ({expect = globalExpect}: any = {}) => { - const {stdout, stderr} = await runCliExec(dockerArgs("-j", "-f", composeFixture)); +test.each([ + ["Dockerfile", dockerfileFixture, "Dockerfile", { + "node:18": {old: "18", new: "22", info: "https://hub.docker.com/_/node"}, + "node:20": {old: "20", new: "22"}, + postgres: {old: "15-alpine", new: "17-alpine", info: "https://hub.docker.com/_/postgres"}, + }], + ["compose", composeFixture, "docker-compose.yaml", { + node: {old: "18", new: "22"}, postgres: {old: "15-alpine", new: "17-alpine"}, redis: {old: "7", new: "8"}, + }], + ["workflow", dockerActionsDir, "ci.yaml", { + node: {old: "18", new: "22"}, postgres: {old: "15", new: "17"}, redis: {old: "7", new: "8"}, + }], +])("docker %s basic", async (_name, file, suffix, expected, {expect = globalExpect}: any = {}) => { + const {stdout, stderr} = await runCliExec(dockerArgs("-j", "-f", file)); expect(stderr).toEqual(""); - const output = JSON.parse(stdout); - expect(output.results.docker).toBeDefined(); - - const composeKey = Object.keys(output.results.docker).find(t => t.endsWith("docker-compose.yaml")); - expect(composeKey).toBeDefined(); - const dockerDeps = output.results.docker[composeKey!]; - - // node:18 -> node:22 - expect(dockerDeps.node.old).toBe("18"); - expect(dockerDeps.node.new).toBe("22"); - - // postgres:15-alpine -> postgres:17-alpine - expect(dockerDeps.postgres.old).toBe("15-alpine"); - expect(dockerDeps.postgres.new).toBe("17-alpine"); - - // redis:7 -> redis:8 - expect(dockerDeps.redis.old).toBe("7"); - expect(dockerDeps.redis.new).toBe("8"); + const docker = JSON.parse(stdout).results.docker; + expect(docker[Object.keys(docker).find(key => key.endsWith(suffix))!]).toMatchObject(expected); }); -test("docker workflow container/image", async ({expect = globalExpect}: any = {}) => { - const {stdout, stderr} = await runCliExec(dockerArgs("-j", "-f", dockerActionsDir)); - expect(stderr).toEqual(""); - const output = JSON.parse(stdout); - expect(output.results.docker).toBeDefined(); - - const ciKey = Object.keys(output.results.docker).find(t => t.endsWith("ci.yaml")); - expect(ciKey).toBeDefined(); - const dockerDeps = output.results.docker[ciKey!]; - - // node:18 -> node:22 (from container: and uses: docker://) - expect(dockerDeps.node.old).toBe("18"); - expect(dockerDeps.node.new).toBe("22"); +test("docker allowedVersions compares floating tags with Docker semantics", async ({expect = globalExpect}: any = {}) => { + const dir = join(testDir, "test-docker-allowed"); + mkdirSync(dir, {recursive: true}); + const file = join(dir, "Dockerfile"); + await writeFile(file, "FROM node:18\n"); + await writeFile(join(dir, "renovate.json"), JSON.stringify({packageRules: [ + {matchPackageNames: ["node"], allowedVersions: "<22"}, + ]})); - // postgres:15 -> postgres:17 (from services image:) - expect(dockerDeps.postgres.old).toBe("15"); - expect(dockerDeps.postgres.new).toBe("17"); + const output = await updates({files: [file], modes: ["docker"], dockerapi: dockerUrl, update: true, color: false, noCache: true}); - // redis:7 -> redis:8 (from container.image object form) - expect(dockerDeps.redis.old).toBe("7"); - expect(dockerDeps.redis.new).toBe("8"); + expect(Object.values(output.results.docker)[0].node.new).toBe("20"); + expect(await readFile(file, "utf8")).toBe("FROM node:20\n"); }); test("actions mode does not include docker from workflows", async ({expect = globalExpect}: any = {}) => { @@ -1960,26 +1467,14 @@ test("actions mode does not include docker from workflows", async ({expect = glo expect(output.results.docker).toBeUndefined(); }); -test("docker include filter", async ({expect = globalExpect}: any = {}) => { - const {stdout, stderr} = await runCliExec(dockerArgs("-j", "-f", composeFixture, "-i", "node")); - expect(stderr).toEqual(""); - const output = JSON.parse(stdout); - const composeKey = Object.keys(output.results.docker).find(t => t.endsWith("docker-compose.yaml")); - const dockerDeps = output.results.docker[composeKey!]; - expect(dockerDeps.node).toBeDefined(); - expect(dockerDeps.postgres).toBeUndefined(); - expect(dockerDeps.redis).toBeUndefined(); -}); - -test("docker exclude filter", async ({expect = globalExpect}: any = {}) => { - const {stdout, stderr} = await runCliExec(dockerArgs("-j", "-f", composeFixture, "-e", "node")); +test.each([ + ["include", "-i", ["node"]], + ["exclude", "-e", ["postgres", "redis"]], +])("docker %s filter", async (_name, flag, expected, {expect = globalExpect}: any = {}) => { + const {stdout, stderr} = await runCliExec(dockerArgs("-j", "-f", composeFixture, flag, "node")); expect(stderr).toEqual(""); - const output = JSON.parse(stdout); - const composeKey = Object.keys(output.results.docker).find(t => t.endsWith("docker-compose.yaml")); - const dockerDeps = output.results.docker[composeKey!]; - expect(dockerDeps.node).toBeUndefined(); - expect(dockerDeps.postgres).toBeDefined(); - expect(dockerDeps.redis).toBeDefined(); + const docker = JSON.parse(stdout).results.docker; + expect(Object.keys(docker[Object.keys(docker).find(key => key.endsWith("docker-compose.yaml"))!]).sort()).toEqual(expected); }); test("docker update rewrites Dockerfiles, compose files and workflows", async ({expect = globalExpect}: any = {}) => { @@ -2003,6 +1498,10 @@ test("docker update rewrites Dockerfiles, compose files and workflows", async ({ " image: postgres:15", " steps:", " - uses: docker://node:18", + " - run: |", + " cat >fragment.yml <<'EOF'", + " uses: docker://node:18", + " EOF", " test2:", " runs-on: ubuntu-latest", " container:", @@ -2024,7 +1523,7 @@ test("docker update rewrites Dockerfiles, compose files and workflows", async ({ expect(updatedWorkflow).toContain("image: postgres:17"); expect(updatedWorkflow).not.toContain("image: postgres:15"); expect(updatedWorkflow).toContain("docker://node:22"); - expect(updatedWorkflow).not.toContain("docker://node:18"); + expect(updatedWorkflow).toContain(" uses: docker://node:18"); expect(updatedWorkflow).toContain("image: redis:8"); expect(updatedWorkflow).not.toContain("image: redis:7"); }); @@ -2033,7 +1532,6 @@ test("docker directory discovery covers every recognized filename", async ({expe const {stdout, stderr} = await runCliExec(dockerArgs("-j", "-f", dockerDir)); expect(stderr).toEqual(""); const {docker} = JSON.parse(stdout).results; - // keys carry the platform separator, and the bare name needs one to tell it from docker-compose const byName = new Map(Object.entries(docker).map(([key, deps]) => [key.replace(/\\/g, "/"), deps as any])); const find = (suffix: string) => byName.keys().find(key => key.endsWith(suffix)); for (const suffix of ["Dockerfile", "Dockerfile.dev", "docker-compose.yaml", "docker-stack.yml", "/compose.yaml"]) { @@ -2043,14 +1541,11 @@ test("docker directory discovery covers every recognized filename", async ({expe expect(byName.get(find(suffix)!).node).toMatchObject({old: "18", new: "22"}); } - // a symlinked directory is the directory it points at, junction so win32 needs no privileges const linked = join(testDir, "test-docker-dir-symlink"); symlinkSync(dockerDir, linked, "junction"); const {stdout: linkedStdout} = await runCliExec(dockerArgs("-j", "-f", linked)); expect(Object.keys(JSON.parse(linkedStdout).results.docker)).toHaveLength(byName.size); - // a link is the file it points at, whose name is what selects the mode, and naming both is no - // second file however they are ordered const linkedFile = join(testDir, "test-docker-file-symlink"); const target = join(dockerDir, "Dockerfile.dev"); symlinkSync(target, linkedFile); @@ -2078,30 +1573,21 @@ test("fetch error includes URL and no stack trace", async ({expect = globalExpec }); test("repeated multi-value flag survives swallowed flag recovery", async ({expect = globalExpect}: any = {}) => { - // `-i react -i -p`: the second -i wrongly swallows -p; recovery must keep the - // earlier `react` include (not reset the array) and still enable prerelease. - const results = await makeTest("-j -i react -i -p")(); - // include `react` survived (without the fix the cleared array drops it and every - // outdated dep returns); the prerelease new version confirms `-p` still applied. + const results = await makeTest("-j -i react -i -p"); expect(Object.keys(results.npm.dependencies)).toEqual(["react"]); expect(results.npm.dependencies.react.new).toBe("18.3.0-next-fecc288b7-20221025"); - // `-i react -i -p -i gulp-sourcemaps`: the swallowed `-p` is NOT the last value; - // recovery must drop that specific bogus value, keeping both real includes. - const nonLast = await makeTest("-j -i react -i -p -i gulp-sourcemaps")(); + const nonLast = await makeTest("-j -i react -i -p -i gulp-sourcemaps"); const keys = Object.keys(nonLast.npm.dependencies); expect(keys).toContain("react"); expect(keys).toContain("gulp-sourcemaps"); expect(nonLast.npm.dependencies.react.new).toBe("18.3.0-next-fecc288b7-20221025"); - // an inline value was written deliberately, so `--exclude=-u` excludes `-u` rather than updating const {args} = parseCliArgs(["-i", "noty", "--exclude=-u"]); expect(args.exclude).toEqual(["-u"]); expect(args.update).toBeUndefined(); }); -// Config option tests — each test gets its own temp dir for concurrency safety. Only the exit -// code and the printed shape need the binary; `run` resolves the same config in-process. async function withConfigDir(config: string, fn: (dir: string) => Promise): Promise { const dir = mkdtempSync(join(tmpdir(), "updates-cfg-")); writeFileSync(join(dir, "package.json"), JSON.stringify(testPkg, null, 2)); @@ -2123,30 +1609,28 @@ function configTest(config: string, args: string): Promise<{stdout: string, stde ], {cwd: dir})); } -test("config errorOnOutdated", async ({expect = globalExpect}: any = {}) => { - try { - await configTest(`{ errorOnOutdated: true }`, "-j -i noty"); - throw new Error("Expected non-zero exit"); - } catch (err: any) { - expect(err?.stdout || err?.message).toContain("noty"); - expect(err?.code).toBe(2); +test("config exit-code options", async ({expect = globalExpect}: any = {}) => { + for (const [config, args, output] of [ + ["{ errorOnOutdated: true }", "-j -i noty", "noty"], + ["{ errorOnUnchanged: true }", "-j -i updates", "All dependencies are up to date."], + ]) { + try { + await configTest(config, args); + throw new Error("Expected non-zero exit"); + } catch (err: any) { + expect(err?.code).toBe(2); + expect(err?.stdout || err?.message).toContain(output); + } } }); -test("config errorOnUnchanged", async ({expect = globalExpect}: any = {}) => { - const {stdout} = await configTest(`{ errorOnUnchanged: true }`, "-j -i updates"); - expect(JSON.parse(stdout).results.npm).toBeDefined(); -}); - test("config cli overrides config", async ({expect = globalExpect}: any = {}) => { - // Config has minor (patch+minor), CLI -P overrides to patch-only const {stdout} = await withConfigDir(`{ minor: true }`, dir => runCliExec([script, "-j", "-i", "gulp-sourcemaps", "-P", "-c", ...apiArgs(), "-f", join(dir, "package.json")])); expect(JSON.parse(stdout).results.npm.dependencies["gulp-sourcemaps"].new).toBe("2.0.1"); }); test("config json yields JSON error output without -j flag", async ({expect = globalExpect}: any = {}) => { - // json comes from the config file, not the CLI; errors must still print as JSON. try { await configTest(`{ json: true }`, "-i noty --registry http://test.invalid -T 1000"); throw new Error("Expected non-zero exit"); @@ -2155,7 +1639,6 @@ test("config json yields JSON error output without -j flag", async ({expect = gl expect(errors[0].error).toContain("test.invalid"); } - // -j is honoured for an error raised before any config has loaded try { await configTest(`{}`, "-j -l foo"); throw new Error("Expected non-zero exit"); @@ -2165,16 +1648,15 @@ test("config json yields JSON error output without -j flag", async ({expect = gl }); test("a /regex/ cli value applies the flag to the packages it matches alone", async ({expect = globalExpect}: any = {}) => { - const greatest = await makeTest("-j -i gulp-sourcemaps,noty -g /^gulp/")(); + const greatest = await makeTest("-j -i gulp-sourcemaps,noty -g /^gulp/"); expect(greatest.npm.dependencies["gulp-sourcemaps"].new).toBe("2.6.5"); expect(greatest.npm.dependencies.noty.new).toBe("3.1.4"); - const prerelease = await makeTest("-j -i gulp-sourcemaps,noty -p /^noty/")(); + const prerelease = await makeTest("-j -i gulp-sourcemaps,noty -p /^noty/"); expect(prerelease.npm.dependencies.noty.new).toBe("3.2.0-beta"); expect(prerelease.npm.dependencies["gulp-sourcemaps"].new).toBe("2.6.5"); }); -// Direct API tests function apiOpts(overrides: UpdatesOptions = {}): UpdatesOptions { return { files: [testFile], @@ -2197,7 +1679,6 @@ test("api basic", async ({expect = globalExpect}: any = {}) => { expect(output.results.npm.dependencies.noty.new).toBe("3.1.4"); expect(output.results.npm.dependencies.noty.info).toBeTruthy(); - // a second call in the same process re-requests rather than answering from the finished run's cache let latest = "3.1.4"; const registry = makeServer((_, res) => res.send(gzipNow(JSON.stringify({ name: "noty", "dist-tags": {latest}, versions: {"3.1.0": {}, "3.1.4": {}, "3.2.1": {}}, @@ -2213,36 +1694,34 @@ test("api basic", async ({expect = globalExpect}: any = {}) => { } }); -test("api no deps", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({files: [emptyFile]})); +test("api messages, filters and mode validation", async ({expect = globalExpect}: any = {}) => { + let output = await updates(apiOpts({files: [emptyFile]})); expect(output.message).toBe("No dependencies found, nothing to do."); expect(Object.keys(output.results)).toHaveLength(0); -}); -test("api all up to date", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["updates"], cooldown: "999999d"})); + output = await updates(apiOpts({include: ["updates"], cooldown: "999999d"})); expect(output.message).toBe("All dependencies are up to date."); -}); -test("api include regex", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: [/^noty$/]})); - expect(output.results.npm.dependencies.noty).toBeDefined(); - const depNames = Object.keys(output.results.npm.dependencies); - expect(depNames).toEqual(["noty"]); -}); + output = await updates(apiOpts({include: [/^noty$/]})); + expect(Object.keys(output.results.npm.dependencies)).toEqual(["noty"]); -test("api exclude regex", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["noty", "gulp-sourcemaps"], exclude: [/sourcemaps/]})); - expect(output.results.npm.dependencies.noty).toBeDefined(); - expect(output.results.npm.dependencies["gulp-sourcemaps"]).toBeUndefined(); + output = await updates(apiOpts({include: ["noty", "gulp-sourcemaps"], exclude: [/sourcemaps/]})); + expect(Object.keys(output.results.npm.dependencies)).toEqual(["noty"]); + + output = await updates(apiOpts({include: ["noty"], modes: ["pypi"]})); + expect(output.message).toBe("No dependencies found, nothing to do."); + await expect(updates(apiOpts({modes: ["nope"]}))).rejects.toThrow("Invalid mode: nope"); }); -test("api greatest", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["gulp-sourcemaps"], greatest: true})); - expect(output.results.npm.dependencies["gulp-sourcemaps"].new).toBe("2.6.5"); +test.each([ + ["greatest", {greatest: true}, "2.6.5"], + ["patch", {patch: true}, "2.0.1"], + ["minor", {minor: true}, "2.6.5"], +])("api %s", async (_name, option, expected, {expect = globalExpect}: any = {}) => { + const output = await updates(apiOpts({include: ["gulp-sourcemaps"], ...option})); + expect(output.results.npm.dependencies["gulp-sourcemaps"].new).toBe(expected); }); -// each selector targets gulp-sourcemaps only, so noty keeps its default resolution test.each([ ["api greatest array", {greatest: ["gulp-sourcemaps"]}], ["api greatest regex", {greatest: [/^gulp/]}], @@ -2254,78 +1733,41 @@ test.each([ expect(output.results.npm.dependencies.noty.new).toBe("3.1.4"); }); -test("api patch", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["gulp-sourcemaps"], patch: true})); - expect(output.results.npm.dependencies["gulp-sourcemaps"].new).toBe("2.0.1"); -}); - -test("api minor", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["gulp-sourcemaps"], minor: true})); - expect(output.results.npm.dependencies["gulp-sourcemaps"].new).toBe("2.6.5"); -}); - -test("api overrides per-package cooldown", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["noty", "updates"], cooldown: "999999d", overrides: [{include: ["noty"], cooldown: 0}]})); +test("api cooldown overrides apply per package and last match wins", async ({expect = globalExpect}: any = {}) => { + let output = await updates(apiOpts({include: ["noty", "updates"], cooldown: "999999d", overrides: [{include: ["noty"], cooldown: 0}]})); expect(output.results.npm.dependencies.noty.new).toBe("3.1.4"); expect(output.results.npm.dependencies.updates).toBeUndefined(); -}); -test("api overrides last match wins", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["noty"], cooldown: "999999d", overrides: [{include: ["noty"], cooldown: "999999d"}, {include: ["noty"], cooldown: 0}]})); + output = await updates(apiOpts({include: ["noty"], cooldown: "999999d", overrides: [{include: ["noty"], cooldown: "999999d"}, {include: ["noty"], cooldown: 0}]})); expect(output.results.npm.dependencies.noty.new).toBe("3.1.4"); }); -test("api modes filter", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["noty"], modes: ["pypi"]})); - expect(output.message).toBe("No dependencies found, nothing to do."); - await expect(updates(apiOpts({modes: ["nope"]}))).rejects.toThrow("Invalid mode: nope"); -}); +test("pypi dotted group names are collected, a declined rewrite is not reported", async ({expect = globalExpect}: any = {}) => { + const file = join(testDir, "test-pypi-groups", "pyproject.toml"); + mkdirSync(join(testDir, "test-pypi-groups")); + await writeFile(file, [ + `[project]`, + `dependencies = ["djlint>=1.30.0,!=1.31.0"]`, + ``, + `[project.optional-dependencies]`, + `"extra.one" = ["PyYAML>=1.0"]`, + ``, + `[dependency-groups]`, + `"test.unit" = ["types-paramiko>=3.4.0.20240423"]`, + ``, + ].join("\n")); -test("api output structure", async ({expect = globalExpect}: any = {}) => { - const output = await updates(apiOpts({include: ["noty"]})); - expect(output).toHaveProperty("results"); - expect(output.results).toHaveProperty("npm"); - expect(output.results.npm).toHaveProperty("dependencies"); - const dep = output.results.npm.dependencies.noty; - expect(dep).toHaveProperty("old"); - expect(dep).toHaveProperty("new"); - expect(dep).toHaveProperty("info"); -}); + const {pypi} = (await updates(apiOpts({files: [file], modes: ["pypi"], update: true}))).results; + expect(pypi["project.optional-dependencies.extra.one"].PyYAML.new).toBe("6.0"); + expect(pypi["dependency-groups.test.unit"]["types-paramiko"].new).toBe("3.5.0.20250801"); + expect(pypi["project.dependencies"]).toBeUndefined(); -test("pypi dotted group names are collected, a declined rewrite is not reported", async ({expect = globalExpect}: any = {}) => { - const dir = mkdtempSync(join(tmpdir(), "updates-pypi-")); - try { - const file = join(dir, "pyproject.toml"); - await writeFile(file, [ - `[project]`, - `dependencies = ["djlint>=1.30.0,!=1.31.0"]`, - ``, - `[project.optional-dependencies]`, - `"extra.one" = ["PyYAML>=1.0"]`, - ``, - `[dependency-groups]`, - `"test.unit" = ["types-paramiko>=3.4.0.20240423"]`, - ``, - ].join("\n")); - - const {pypi} = (await updates(apiOpts({files: [file], modes: ["pypi"], update: true}))).results; - expect(pypi["project.optional-dependencies.extra.one"].PyYAML.new).toBe("6.0"); - expect(pypi["dependency-groups.test.unit"]["types-paramiko"].new).toBe("3.5.0.20250801"); - // 1.31.0 is the version `!=` excludes, so there is no update to report - expect(pypi["project.dependencies"]).toBeUndefined(); - - const written = await readFile(file, "utf8"); - expect(written).toContain(`"PyYAML>=6.0"`); - expect(written).toContain(`"types-paramiko>=3.5.0.20250801"`); - expect(written).toContain(`"djlint>=1.30.0,!=1.31.0"`); - } finally { - try { - await rm(dir, {recursive: true, force: true, maxRetries: 10, retryDelay: 100}); - } catch {} - } + const written = await readFile(file, "utf8"); + expect(written).toContain(`"PyYAML>=6.0"`); + expect(written).toContain(`"types-paramiko>=3.5.0.20250801"`); + expect(written).toContain(`"djlint>=1.30.0,!=1.31.0"`); }); -// Bug: no pin, override or per-dep flag ever matched a pypi dep. test("a pypi pin holds, keyed by the authored spelling or the normalized one", async ({expect = globalExpect}: any = {}) => { for (const key of ["PyYAML", "pyyaml"]) { const {pypi} = (await updates(apiOpts({files: [uvFile], modes: ["pypi"], include: ["PyYAML"], pin: {[key]: "<6.0"}}))).results; @@ -2333,57 +1775,23 @@ test("a pypi pin holds, keyed by the authored spelling or the normalized one", a } }); -// Bug: two non-workspace manifests of the same mode shared single mode-keyed -// slots, so only the last file was written and same name+type deps collided. -test("two non-workspace package.json: both updated, no dep loss", async ({expect = globalExpect}: any = {}) => { - const dir = mkdtempSync(join(tmpdir(), "updates-multi-")); - try { - const fileA = join(dir, "a", "package.json"); - const fileB = join(dir, "b", "package.json"); - mkdirSync(join(dir, "a"), {recursive: true}); - mkdirSync(join(dir, "b"), {recursive: true}); - await writeFile(fileA, `${JSON.stringify({dependencies: {noty: "3.1.0"}}, null, 2)}\n`); - await writeFile(fileB, `${JSON.stringify({dependencies: {"gulp-sourcemaps": "2.0.0"}}, null, 2)}\n`); - - const output = await updates(apiOpts({files: [fileA, fileB], update: true})); - - // Both deps survive (neither file's unique dep is lost to a shared slot). - const npm = output.results.npm; - const allDeps = Object.assign({}, ...Object.values(npm) as Array>); - expect(allDeps.noty?.new).toBe("3.1.4"); - expect(allDeps["gulp-sourcemaps"]?.new).toBe("2.6.5"); - - // Both files are written with their own update (not just the last one). - expect(await readFile(fileA, "utf8")).toContain(`"noty": "3.1.4"`); - expect(await readFile(fileB, "utf8")).toContain(`"gulp-sourcemaps": "2.6.5"`); - } finally { - try { - await rm(dir, {recursive: true, force: true, maxRetries: 10, retryDelay: 100}); - } catch {} - } -}); - -test("two non-workspace package.json with the same dep: both updated", async ({expect = globalExpect}: any = {}) => { - const dir = mkdtempSync(join(tmpdir(), "updates-multi-same-")); - try { - const fileA = join(dir, "a", "package.json"); - const fileB = join(dir, "b", "package.json"); - mkdirSync(join(dir, "a"), {recursive: true}); - mkdirSync(join(dir, "b"), {recursive: true}); - await writeFile(fileA, `${JSON.stringify({dependencies: {noty: "3.1.0"}}, null, 2)}\n`); - await writeFile(fileB, `${JSON.stringify({dependencies: {noty: "3.1.0"}}, null, 2)}\n`); - - const output = await updates(apiOpts({files: [fileA, fileB], update: true})); - - // Same name+type from both files must not collide into one dep entry. - const notyEntries = Object.values(output.results.npm).filter(section => "noty" in section); - expect(notyEntries.length).toBe(2); - - expect(await readFile(fileA, "utf8")).toContain(`"noty": "3.1.4"`); - expect(await readFile(fileB, "utf8")).toContain(`"noty": "3.1.4"`); - } finally { - try { - await rm(dir, {recursive: true, force: true, maxRetries: 10, retryDelay: 100}); - } catch {} +test("non-workspace manifests keep distinct dependencies and duplicate identities", async ({expect = globalExpect}: any = {}) => { + const dir = join(testDir, "test-multi-manifest"); + const manifests = [ + ["a", "noty", "3.1.0", "3.1.4"], + ["b", "noty", "3.1.0", "3.1.4"], + ["c", "gulp-sourcemaps", "2.0.0", "2.6.5"], + ] as const; + const files = await Promise.all(manifests.map(async ([subdir, name, old]) => { + mkdirSync(join(dir, subdir), {recursive: true}); + const file = join(dir, subdir, "package.json"); + await writeFile(file, `${JSON.stringify({dependencies: {[name]: old}}, null, 2)}\n`); + return file; + })); + + const {npm} = (await updates(apiOpts({files, update: true}))).results; + expect(Object.values(npm).filter(section => "noty" in section)).toHaveLength(2); + for (const [index, [, name, , expected]] of manifests.entries()) { + expect(await readFile(files[index], "utf8")).toContain(`"${name}": "${expected}"`); } }); diff --git a/index.ts b/index.ts index f28ecf1..dffae96 100755 --- a/index.ts +++ b/index.ts @@ -1,27 +1,79 @@ #!/usr/bin/env node -import {stdout, stderr, exit, platform, versions} from "node:process"; -import {stripVTControlCharacters, styleText} from "node:util"; -import {updates} from "./api.ts"; -import {parseCliArgs, resolveConfig, resolveFileArgs} from "./cli.ts"; -import {packageVersion, fetchTimeout, maxSockets} from "./modes/shared.ts"; -import {highlightDiff, textTable} from "./utils/utils.ts"; -import {shortenGoModule} from "./modes/go.ts"; -import {prewarmOrigins} from "./utils/prewarm.ts"; +import {argv, stdout, stderr, exit, platform, versions} from "node:process"; +import {readFileSync, statSync} from "node:fs"; +import {dirname, join, resolve} from "node:path"; +import {pathToFileURL} from "node:url"; import type {Output} from "./api.ts"; -const {args, positionals} = parseCliArgs(); +let red: (text: string | number) => string = String; +let green: (text: string | number) => string = String; +let jsonOutput = false; -if (!args.help && !args.version) { - for (const url of prewarmOrigins(resolveFileArgs(args, positionals).startDir, args)) { - (async () => { try { await fetch(url, {method: "HEAD"}); } catch {} })(); +const stringShortOptions = new Set("deflCpgPtmsTriM"); + +function hasFlag(args: Array, long: string, short: string): boolean { + if (args.includes(`--${long}`)) return true; + for (const arg of args) { + if (!/^-[^-]/.test(arg)) continue; + const options = arg.slice(1); + const index = options.indexOf(short); + if (index !== -1 && Array.from(options.slice(0, index)).every(option => !stringShortOptions.has(option))) return true; } + return false; } -let red: (text: string | number) => string = String; -let green: (text: string | number) => string = String; -// Seeded from the flag so an error raised before the config loads still honours -j, then widened -// to the effective setting below, so the config file's `json` reaches the error path too. -let jsonOutput = Boolean(args.json); +const valueOptions: Record = { + d: "allow-downgrade", e: "exclude", f: "file", l: "pin", C: "cooldown", p: "prerelease", R: "release", + g: "greatest", t: "types", P: "patch", m: "minor", s: "sockets", T: "timeout", r: "registry", i: "include", + M: "modes", forgeapi: "forgeapi", pypiapi: "pypiapi", jsrapi: "jsrapi", goproxy: "goproxy", + cargoapi: "cargoapi", dockerapi: "dockerapi", file: "file", modes: "modes", registry: "registry", +}; + +async function startPrewarm(rawArgs: Array): Promise { + const args: Record = {}; + let firstPositional: string | undefined; + for (let index = 0; index < rawArgs.length; index++) { + const arg = rawArgs[index]; + if (!arg.startsWith("-")) { + firstPositional ??= arg; + continue; + } + const long = /^--([^=]+)(?:=(.*))?$/.exec(arg); + const short = /^-([A-Za-z])(.*)$/.exec(arg); + const option = long ? valueOptions[long[1]] : short ? valueOptions[short[1]] : undefined; + if (!option) continue; + const inline = long ? long[2] : short?.[2]; + const value = inline || rawArgs[index + 1]?.startsWith("-") === false ? inline || rawArgs[++index] : undefined; + if (value === undefined) continue; + if (option === "file" || option === "modes") { + ((args[option] ??= []) as Array).push(...value.split(",")); + } else args[option] = value; + } + const first = (args.file as Array | undefined)?.[0] ?? firstPositional; + const firstPath = first ? resolve(first) : process.cwd(); + let startDir = first ? dirname(firstPath) : firstPath; + try { if (statSync(firstPath).isDirectory()) startDir = firstPath; } catch {} + + let config: Record = {}; + configSearch: + for (let dir = startDir; ; dir = dirname(dir)) { + for (const extension of ["js", "ts", "mjs", "mts"]) { + const path = join(dir, `updates.config.${extension}`); + try { + if (!statSync(path).isFile()) continue; + config = (await import(pathToFileURL(path).href)).default ?? {}; + break configSearch; + } catch {} + } + if (dirname(dir) === dir) break; + } + config = {...config, ...args}; + const files = Array.isArray(config.file) ? config.file : config.files; + const {prewarmOrigins} = await import("./utils/prewarm.ts"); + for (const origin of prewarmOrigins(startDir, {...config, files})) { + (async () => { try { await fetch(origin, {method: "HEAD"}); } catch {} })(); + } +} async function end(err?: Error | void, exitCode?: number): Promise { if (err) { @@ -45,7 +97,9 @@ async function main(): Promise { (stream as any)?._handle?.setBlocking?.(true); } - if (args.help) { + const rawArgs = argv.slice(2); + jsonOutput = hasFlag(rawArgs, "json", "j"); + if (hasFlag(rawArgs, "help", "h")) { stdout.write(`usage: updates [options] [files...] Options: @@ -63,8 +117,8 @@ async function main(): Promise { -P, --patch [] Consider only up to semver-patch -m, --minor [] Consider only up to semver-minor -d, --allow-downgrade [] Allow downgrading onto a lower latest tag - -s, --sockets Maximum number of parallel HTTP sockets opened. Default: ${maxSockets} - -T, --timeout Network request timeout in ms (go probes use half). Default: ${fetchTimeout} + -s, --sockets Maximum number of parallel HTTP sockets opened. Default: 25 + -T, --timeout Network request timeout in ms (go probes use half). Default: 5000 -r, --registry Override npm registry URL -I, --indirect Include indirect Go dependencies -E, --error-on-outdated Exit with code 2 when updates are available and 0 when not @@ -95,12 +149,23 @@ async function main(): Promise { await end(); } - if (args.version) { - console.info(packageVersion); + if (hasFlag(rawArgs, "version", "v")) { + let packageJson: string; + try { packageJson = readFileSync(new URL("../package.json", import.meta.url), "utf8"); } catch { + packageJson = readFileSync(new URL("package.json", import.meta.url), "utf8"); + } + console.info(JSON.parse(packageJson).version); await end(); } + try { await startPrewarm(rawArgs); } catch {} + const {parseCliArgs, resolveConfig} = await import("./cli.ts"); + const {args, positionals} = parseCliArgs(); const config = await resolveConfig(args, positionals); + const [{updates}, {highlightDiff, textTable}, {shortenGoModule}, {stripVTControlCharacters, styleText}] = + await Promise.all([ + import("./api.ts"), import("./utils/utils.ts"), import("./modes/go.ts"), import("node:util"), + ]); const useColor = !config.noColor && (config.color || stdout.isTTY); if (useColor) { @@ -124,36 +189,29 @@ async function main(): Promise { } else if (output.message) { console.info(output.message); } else if (hasResults) { - console.info(formatOutput(output)); + console.info(formatOutput(output, shortenGoModule, highlightDiff, textTable, stripVTControlCharacters)); } - if (config.update && hasResults && !config.json) { - for (const [mode, modeResults] of Object.entries(output.results)) { - if (Object.values(modeResults).some(deps => Object.keys(deps).length)) { - console.info(green(`✨ ${mode} updated`)); - } - } + if (config.update && !config.json) { + for (const mode of Object.keys(output.results)) console.info(green(`✨ ${mode} updated`)); } if (!config.json) { for (const {mode, name, error} of errors) console.info(red(`${mode} ${name}: ${error}`)); } - // A run that could not look everything up is neither outdated nor up to date, so -E/-U yield to it. - if (errors.length) { - await end(undefined, 1); - } else if (config.errorOnOutdated) { - await end(undefined, hasResults ? 2 : 0); - } else if (config.errorOnUnchanged) { - await end(undefined, hasResults ? 0 : 2); - } else { - await end(); - } + const exitCode = errors.length ? 1 : config.errorOnOutdated ? (hasResults ? 2 : 0) : + config.errorOnUnchanged ? (hasResults ? 0 : 2) : 0; + await end(undefined, exitCode); } -const ansiLen = (str: string): number => stripVTControlCharacters(str).length; - -function formatOutput(output: Output): string { +function formatOutput( + output: Output, + shortenGoModule: (value: string) => string, + highlightDiff: (left: string, right: string, colorFn: (text: string) => string) => string, + textTable: (rows: Array>, lengthFn: (value: string) => number) => string, + stripVTControlCharacters: (value: string) => string, +): string { const modes = Object.keys(output.results); const hasMultipleModes = modes.length > 1; @@ -164,12 +222,8 @@ function formatOutput(output: Output): string { const seen = new Set(); for (const mode of modes) { - // Rows sort across the whole mode, where the JSON keeps its dep-type sections to sort within. const rows = Object.values(output.results[mode]).flatMap(typeDeps => Object.entries(typeDeps)); for (const [name, data] of rows.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) { - // Key on the visible columns (incl. versions) so the same dep at - // different versions across dep-sections/workspace members keeps a row - // each; only truly identical rows collapse. const id = `${mode}|${name}|${data.old}|${data.new}`; if (seen.has(id)) continue; seen.add(id); @@ -184,7 +238,7 @@ function formatOutput(output: Output): string { } } - return textTable(arr, ansiLen); + return textTable(arr, str => stripVTControlCharacters(str).length); } try { diff --git a/modes/actions.test.ts b/modes/actions.test.ts index 6fc5fc8..94941ec 100644 --- a/modes/actions.test.ts +++ b/modes/actions.test.ts @@ -4,17 +4,21 @@ import { getForgeApiBaseUrl, formatActionVersion, isWorkflowFile, - updateWorkflowFile, + updateWorkflowFile as updateWorkflowContent, fetchActionTagDate, resolveWorkflowFiles, parseUsesLine, } from "./actions.ts"; -import {type ModeContext, fetchTimeout, hashRe, isVersionLikeRef} from "./shared.ts"; +import {type ModeContext, commitHashRe, fetchTimeout, isVersionLikeRef} from "./shared.ts"; + +const workflowPrefix = "runs:\n steps:\n"; +const updateWorkflowFile = (content: string, updates: Parameters[1]) => + updateWorkflowContent(workflowPrefix + content, updates).slice(workflowPrefix.length); -// parseActionRef test.each([ ["standard ref", "actions/checkout@v4", {host: null, owner: "actions", repo: "checkout", ref: "v4", name: "actions/checkout", isHash: false}], - ["hash ref", "actions/checkout@abc1234567890", {host: null, owner: "actions", repo: "checkout", ref: "abc1234567890", name: "actions/checkout", isHash: true}], + ["hash ref", "actions/checkout@abc1234", {host: null, owner: "actions", repo: "checkout", ref: "abc1234", name: "actions/checkout", isHash: true}], + ["hex tag", "actions/checkout@deadbeef", {host: null, owner: "actions", repo: "checkout", ref: "deadbeef", name: "actions/checkout", isHash: false}], ["sub-path", "actions/cache/restore@v4", {host: null, owner: "actions", repo: "cache", ref: "v4", name: "actions/cache/restore", isHash: false}], ["URL with host", "https://gitea.example.com/owner/repo@v1", {host: "gitea.example.com", owner: "owner", repo: "repo", ref: "v1", name: "gitea.example.com/owner/repo", isHash: false}], ["docker prefix", "docker://node:18", null], @@ -25,7 +29,6 @@ test.each([ expect(parseActionRef(uses)).toEqual(expected); }); -// getForgeApiBaseUrl test.each([ ["falls back to the configured forge without a host", null, "https://gitea.example.com/api/v1", "https://gitea.example.com/api/v1"], ["lets a github host win over the configured forge", "github.com", "https://gitea.example.com/api/v1", "https://api.github.com"], @@ -41,11 +44,10 @@ test.each([ expect(formatActionVersion(newVersion, oldRef)).toBe(expected); }); -// isWorkflowFile test.each([ [".github/workflows/ci.yml", true], [".github/workflows/deploy.yaml", true], - [".github\\workflows\\ci.yml", true], // windows backslashes + [".github\\workflows\\ci.yml", true], [".github/actions/my-action/action.yml", true], [".github/actions/my-action/action.yaml", true], [".github/actions/group/sub/action.yml", true], @@ -56,13 +58,12 @@ test.each([ [".forgejo/actions/my-action/action.yml", true], ["ci.yml", false], [".github/ci.yml", false], - [".github/actions/my-action/other.yml", false], // only action.yml counts as a composite action - ["actions/my-action/action.yml", false], // and only inside a forge dir + [".github/actions/my-action/other.yml", false], + ["actions/my-action/action.yml", false], ])("isWorkflowFile %s", (path, expected) => { expect(isWorkflowFile(path)).toBe(expected); }); -// updateWorkflowFile test.each([ ["a plain ref", {name: "actions/checkout", oldRef: "v3", newRef: "v4"}, " uses: actions/checkout@v3\n", " uses: actions/checkout@v4\n"], @@ -77,21 +78,24 @@ test.each([ expect(updateWorkflowFile(content, [replacement])).toBe(expected); }); -test("updateWorkflowFile multiple replacements", () => { +test("updateWorkflowFile handles multiple, comment-qualified and CRLF replacements", () => { const content = " uses: actions/checkout@v3\n uses: actions/setup-node@v3\n"; const result = updateWorkflowFile(content, [ {name: "actions/checkout", oldRef: "v3", newRef: "v4"}, {name: "actions/setup-node", oldRef: "v3", newRef: "v4"}, ]); expect(result).toBe(" uses: actions/checkout@v4\n uses: actions/setup-node@v4\n"); -}); - -test("updateWorkflowFile moves the version comment along with the sha", () => { - const content = " uses: actions/checkout@11bd719 # v4.2.2\n uses: actions/checkout@11bd719\n"; - const result = updateWorkflowFile(content, [ + expect(updateWorkflowFile(" uses: actions/checkout@11bd719 # v4.2.2\n uses: actions/checkout@11bd719\n", [ {name: "actions/checkout", oldRef: "11bd719", newRef: "3d3c42e", newComment: "v7.0.1"}, - ]); - expect(result).toBe(" uses: actions/checkout@3d3c42e # v7.0.1\n uses: actions/checkout@3d3c42e\n"); + ])).toBe(" uses: actions/checkout@3d3c42e # v7.0.1\n uses: actions/checkout@3d3c42e\n"); + const oldRef = "11bd719"; + expect(updateWorkflowFile(` uses: actions/checkout@${oldRef} # main\n uses: actions/checkout@${oldRef} # release\n`, [ + {name: "actions/checkout", oldRef, oldComment: "main", newRef: "aaaaaaa"}, + {name: "actions/checkout", oldRef, oldComment: "release", newRef: "bbbbbbb"}, + ])).toBe(" uses: actions/checkout@aaaaaaa # main\n uses: actions/checkout@bbbbbbb # release\n"); + expect(updateWorkflowFile(" uses: actions/checkout@11bd719 # v4.2.2\r\n uses: actions/checkout@11bd719\r\n", [ + {name: "actions/checkout", oldRef: "11bd719", newRef: "3d3c42e", newComment: "v7.0.1"}, + ])).toBe(" uses: actions/checkout@3d3c42e # v7.0.1\r\n uses: actions/checkout@3d3c42e\r\n"); }); test.each([ @@ -101,12 +105,13 @@ test.each([ ["pin prefix", "actions/checkout@11bd719 # pin v4.2.2", "actions/checkout@3d3c42e # v7.0.1"], ["renovate prefix", "actions/checkout@11bd719 # renovate: tag=v4.2.2", "actions/checkout@3d3c42e # v7.0.1"], ["ratchet prefix", "actions/checkout@11bd719 # ratchet:actions/checkout@v4.2.2", "actions/checkout@3d3c42e # v7.0.1"], + ["subpath ratchet prefix", "actions/cache/restore@11bd719 # ratchet:actions/cache/restore@v4.2.2", "actions/cache/restore@3d3c42e # v7.0.1"], ["no space before hash", "actions/checkout@11bd719 #v4.2.2", "actions/checkout@3d3c42e # v7.0.1"], ["text after the version", "actions/checkout@11bd719 # v4.2.2 (keep me)", "actions/checkout@3d3c42e # v7.0.1 (keep me)"], ["comment naming no version", "actions/checkout@11bd719 # ratchet:exclude", "actions/checkout@3d3c42e # ratchet:exclude"], ])("updateWorkflowFile rewrites the comment of a %s", (_name, oldLine, newLine) => { const result = updateWorkflowFile(` - uses: ${oldLine}\n`, [ - {name: "actions/checkout", oldRef: "11bd719", newRef: "3d3c42e", newComment: "v7.0.1"}, + {name: oldLine.startsWith("actions/cache/") ? "actions/cache/restore" : "actions/checkout", oldRef: "11bd719", newRef: "3d3c42e", newComment: "v7.0.1"}, ]); expect(result).toBe(` - uses: ${newLine}\n`); }); @@ -119,12 +124,23 @@ test.each([ expect(updateWorkflowFile(`${line}\n`, [{name: "actions/checkout", oldRef: "v3", newRef: "v4"}])).toBe(`${line}\n`); }); -test("updateWorkflowFile keeps crlf line endings", () => { - const content = " uses: actions/checkout@11bd719 # v4.2.2\r\n uses: actions/checkout@11bd719\r\n"; - const result = updateWorkflowFile(content, [ - {name: "actions/checkout", oldRef: "11bd719", newRef: "3d3c42e", newComment: "v7.0.1"}, - ]); - expect(result).toBe(" uses: actions/checkout@3d3c42e # v7.0.1\r\n uses: actions/checkout@3d3c42e\r\n"); +test("updateWorkflowFile skips uses lines inside YAML block scalars", () => { + const content = ` - run: | + cat > generated.yml < generated.yml < { @@ -137,14 +153,10 @@ test("parseUsesLine splits a quoted sha pin from its prefixed comment", () => { pinnedVersion: "v4.2.2", pinnedEnd: 12, }); -}); - -test("parseUsesLine ignores lines the reader ignores", () => { expect(parseUsesLine(" - run: echo uses: actions/checkout@v3")).toBeNull(); expect(parseUsesLine(" # uses: actions/checkout@v3")).toBeNull(); }); -// isVersionLikeRef test("isVersionLikeRef separates versions from branches and other tag schemes", () => { expect(isVersionLikeRef("v4")).toBe(true); expect(isVersionLikeRef("4.1.2")).toBe(true); @@ -154,14 +166,12 @@ test("isVersionLikeRef separates versions from branches and other tag schemes", expect(isVersionLikeRef("main")).toBe(false); }); -// hashRe -test("hashRe accepts short shas but not all-numeric tags", () => { - expect(hashRe.test("3d3c42")).toBe(true); - expect(hashRe.test("11bd71901bbe5b1630ceea73d27597364c9af683")).toBe(true); - expect(hashRe.test("20240115")).toBe(false); +test("commitHashRe accepts short shas but not all-numeric tags", () => { + expect(commitHashRe.test("3d3c42")).toBe(true); + expect(commitHashRe.test("11bd71901bbe5b1630ceea73d27597364c9af683")).toBe(true); + expect(commitHashRe.test("20240115")).toBe(false); }); -// fetchActionTagDate test.each([ ["returns committer date", "https://api.github.com", () => Promise.resolve({ok: true, json: () => Promise.resolve({committer: {date: "2025-01-01T00:00:00Z"}, author: {date: "2024-12-01T00:00:00Z"}})}), @@ -183,7 +193,6 @@ test.each([ expect(await fetchActionTagDate(apiUrl, "actions", "checkout", "abc123", ctx)).toBe(expected); }); -// Passing a forge failure off as an unknown date would let a cooldown read it as "held back". test.each([ ["a server fault", () => Promise.resolve({ok: false, status: 500, statusText: "Server Error"}), /Received 500/], ["a network failure", () => Promise.reject(new Error("network error")), /network error/], @@ -192,7 +201,6 @@ test.each([ await expect(fetchActionTagDate("https://api.github.com", "actions", "checkout", "abc123", ctx)).rejects.toThrow(expected); }); -// resolveWorkflowFiles test.each([ ["yaml files", "fixtures/docker-actions/.github", ["workflows/ci.yaml"]], ["composite actions", "fixtures/actions-composite/.github", diff --git a/modes/actions.ts b/modes/actions.ts index 3f47278..20d1033 100644 --- a/modes/actions.ts +++ b/modes/actions.ts @@ -1,7 +1,9 @@ import {resolve, join} from "node:path"; import {readdirSync} from "node:fs"; import {parse} from "../utils/semver.ts"; -import {type ModeContext, ForgeError, stripv, hashRe, fetchForge, formatVersionPrecision, githubApiUrl, parseCommitDate} from "./shared.ts"; +import { + type ModeContext, commitHashRe, ForgeError, stripv, fetchForge, formatVersionPrecision, githubApiUrl, parseCommitDate, +} from "./shared.ts"; import {getCache, setCache} from "../utils/fetchCache.ts"; import {forgeDirs, longestFirstAlternation} from "../utils/utils.ts"; @@ -27,20 +29,15 @@ export function parseActionRef(uses: string): ActionRef | null { const segments = pathPart.split("/"); if (segments.length < 2) return null; const name = host ? `${host}/${pathPart}` : pathPart; - return {host, owner: segments[0], repo: segments[1], ref, name, isHash: hashRe.test(ref)}; + return {host, owner: segments[0], repo: segments[1], ref, name, isHash: commitHashRe.test(ref)}; } -// A host spelled out in the ref wins over the configured forge, so `https://gitea.com/o/r@v1` -// resolves against gitea.com even when the run defaults to GitHub. A bare `o/r@v1` has no host -// to go on and follows the default. export function getForgeApiBaseUrl(host: string | null, forgeApiUrl: string): string { if (!host) return forgeApiUrl; return host === "github.com" ? githubApiUrl : `https://${host}/api/v1`; } -// "" is a commit with no date, which holds a cooldown candidate back, undefined is a failed request. export async function fetchActionTagDate(apiUrl: string, owner: string, repo: string, commitSha: string, ctx: ModeContext): Promise { - // Commit data is immutable — cache the resolved date forever keyed by URL. const url = `${apiUrl}/repos/${owner}/${repo}/git/commits/${commitSha}`; if (!ctx.noCache) { const cached = await getCache(url); @@ -48,13 +45,12 @@ export async function fetchActionTagDate(apiUrl: string, owner: string, repo: st } try { const res = await fetchForge(url, ctx); - if (res.status === 404) return ""; // the commit is gone, so no date will ever exist + if (res.status === 404) return ""; if (!res.ok) return undefined; const date = parseCommitDate(await res.json()); if (date && !ctx.noCache) setCache(url, "immutable", date); return date; } catch (err) { - // A classified forge failure is the dependency's result, a malformed body is worth degrading over. if (err instanceof ForgeError) throw err; return undefined; } @@ -65,34 +61,26 @@ export function formatActionVersion(newFullVersion: string, oldRef: string): str return formatVersionPrecision(newParsed?.version ?? stripv(newFullVersion), oldRef); } -// Reader and writer share this, so the writer can never reach a `uses:` the reader did not extract, -// like a commented-out step or one quoted inside a `run:` script. -const usesLineRe = /^(\s*(?:-\s*)?uses:\s*)([^\n]*)$/; +const yamlPairRe = /^(\s*)(?:-\s*)?(?:"([^"]+)"|'([^']+)'|([^\s:#][^:#]*)):\s*([^\r\n]*)\r?$/; -// The version a trailing comment names, behind the `renovate:`, `pin `, `tag=` and `ratchet:` -// prefixes a pinned sha's comment carries. Mirrors renovate's pinTokenRe. -const pinTokenRe = /^\s*(?:(?:renovate\s*:\s*)?(?:pin\s+|tag\s*=\s*)?|ratchet:[\w-]+\/[.\w-]+)@?((?:[\w-]*[-/])?v?\d+(?:\.\d+(?:\.\d+)?)?(?:-[a-zA-Z0-9.]+)?)/; +const pinTokenRe = /^\s*(?:(?:renovate\s*:\s*)?(?:pin\s+|tag\s*=\s*)?|ratchet:[\w-]+\/[.\w-]+(?:\/[.\w-]+)*)@?((?:[\w-]*[-/])?v?\d+(?:\.\d+(?:\.\d+)?)?(?:-[a-zA-Z0-9.]+)?)/; export type UsesLine = { - prefix: string, // indentation, the list dash and `uses:` with its trailing space - quote: string, // the quote around the value, empty when it is unquoted - value: string, // the `[scheme://]owner/repo[/path]@ref` text, unquoted - gap: string, // whatever sits between the value and the comment - comment: string, // the comment including its `#`, empty when the line has none - pinnedVersion: string, // the version the comment names, empty when it names none - pinnedEnd: number, // offset into `comment` just past the token that named it + prefix: string, + quote: string, + value: string, + gap: string, + comment: string, + pinnedVersion: string, + pinnedEnd: number, }; export function parseUsesLine(line: string): UsesLine | null { - const match = usesLineRe.exec(line); + const match = /^(\s*(?:-\s*)?uses:\s*)(?:(["'])(.*?)\2|((?!["'])[^\s#]+))([^\n]*)$/.exec(line); if (!match) return null; - const [, prefix, remainder] = match; - const quote = remainder[0] === "'" || remainder[0] === '"' ? remainder[0] : ""; - const quoteEnd = quote ? remainder.indexOf(quote, 1) : 0; - if (quoteEnd === -1) return null; - const value = quote ? remainder.slice(1, quoteEnd) : /^[^\s#]*/.exec(remainder)![0]; + const [, prefix, quote = "", quotedValue, plainValue, rest] = match; + const value = quotedValue ?? plainValue; if (!value) return null; - const rest = remainder.slice(quote ? quoteEnd + 1 : value.length); const hash = rest.indexOf("#"); const comment = hash === -1 ? "" : rest.slice(hash); const pin = comment ? pinTokenRe.exec(comment.slice(1)) : null; @@ -105,19 +93,40 @@ export function parseUsesLine(line: string): UsesLine | null { }; } +export type ActionUpdate = {name: string, oldRef: string, newRef: string, oldComment?: string, newComment?: string}; + const schemeRe = /^https?:\/\//; -export function updateWorkflowFile(content: string, actionDeps: Array<{name: string, oldRef: string, newRef: string, newComment?: string}>): string { - const depByUses = new Map(actionDeps.map(dep => [`${dep.name}@${dep.oldRef}`, dep])); +export function updateWorkflowFile(content: string, actionDeps: Array): string { + const depByUses = new Map(actionDeps.map(dep => [`${dep.name}@${dep.oldRef}${dep.oldComment ? `#${dep.oldComment}` : ""}`, dep])); + const yamlPath: Array<{indent: number, key: string}> = []; + let blockIndent = -1; return content.split("\n").map(line => { + if (blockIndent !== -1) { + if (!line.trim() || line.length - line.trimStart().length > blockIndent) return line; + blockIndent = -1; + } + const pair = yamlPairRe.exec(line); + if (!pair) return line; + const indent = pair[1].length; + while (yamlPath.length && yamlPath.at(-1)!.indent >= indent) yamlPath.pop(); + const key = (pair[2] ?? pair[3] ?? pair[4]).trim(); + const isUses = key === "uses" && ( + yamlPath[0]?.key === "jobs" && yamlPath.length === 3 && yamlPath[2].key === "steps" || + yamlPath[0]?.key === "runs" && yamlPath.length === 2 && yamlPath[1].key === "steps" + ); + const pairValue = pair[5].replace(/(?:^|\s)#.*$/, "").trim(); + if (!pairValue) yamlPath.push({indent, key}); + if (/^[>|](?:[+-]?\d?|\d[+-]?)$/.test(pairValue)) { blockIndent = indent; return line; } + if (!isUses) return line; const parsed = parseUsesLine(line); if (!parsed) return line; const {prefix, quote, value, gap, comment, pinnedVersion, pinnedEnd} = parsed; const scheme = schemeRe.exec(value)?.[0] ?? ""; - const dep = depByUses.get(value.slice(scheme.length)); + const oldComment = pinnedVersion || /^#\s*(\S+)\s*$/.exec(comment)?.[1] || ""; + const dep = depByUses.get(`${value.slice(scheme.length)}${oldComment ? `#${oldComment}` : ""}`) ?? + depByUses.get(value.slice(scheme.length)); if (!dep) return line; - // A sha pin's trailing comment names the version and would otherwise keep naming the old one. - // Renovate rewrites it to the version alone, dropping any `tag=`/`pin`/`ratchet:` prefix. const newComment = dep.newComment && pinnedVersion ? `# ${dep.newComment}${comment.slice(pinnedEnd)}` : comment; return `${prefix}${quote}${scheme}${dep.name}@${dep.newRef}${quote}${gap}${newComment}`; }).join("\n"); @@ -144,4 +153,3 @@ export function resolveWorkflowFiles(forgeDir: string): Array { } catch {} return Array.from(found); } - diff --git a/modes/cargo.test.ts b/modes/cargo.test.ts index ab6eac6..e2bbb9f 100644 --- a/modes/cargo.test.ts +++ b/modes/cargo.test.ts @@ -10,7 +10,7 @@ function sparseCtx(body: string, urls: Array = []): ModeContext { noCache: true, doFetch: (url: string) => { urls.push(url); - return Promise.resolve({ok: true, text: () => Promise.resolve(body)}); + return Promise.resolve(new Response(body)); }, } as unknown as ModeContext; } @@ -20,19 +20,20 @@ test.each([ `dependencies${fieldSep}serde`, {old: "1.0.0", new: "1.0.1"}, `[dependencies]\nserde = "1.0.1"\n`], [`single-quoted name = 'version'`, `[dependencies]\nserde = '1.0.0'\n`, `dependencies${fieldSep}serde`, {old: "1.0.0", new: "2.0.0"}, `[dependencies]\nserde = '2.0.0'\n`], + [`dependency without touching a crate named version`, `[dependencies]\nfoo = "1.0.0"\nversion = "1.0.0"\n`, + `dependencies${fieldSep}foo`, {old: "1.0.0", new: "2.0.0"}, `[dependencies]\nfoo = "2.0.0"\nversion = "1.0.0"\n`], [`inline table name = {version, features}`, `[dependencies]\nserde = { version = "1.0.0", features = ["derive"] }\n`, `dependencies${fieldSep}serde`, {old: "1.0.0", new: "1.1.0"}, `[dependencies]\nserde = { version = "1.1.0", features = ["derive"] }\n`], [`extended table [dependencies.name]`, `[dependencies.serde]\nversion = "1.0.0"\nfeatures = ["derive"]\n`, `dependencies${fieldSep}serde`, {old: "1.0.0", new: "1.2.0"}, `[dependencies.serde]\nversion = "1.2.0"\nfeatures = ["derive"]\n`], - // an extended table names no bare `[dependencies]` header, which must not widen the scope to the file + [`extended table skips comments and multiline strings`, `[dependencies.serde]\n# version = "1.0.0" was old\nnote = """\nversion = "1.0.0"\n"""\nversion = "1.0.0"\n`, + `dependencies${fieldSep}serde`, {old: "1.0.0", new: "1.1.0"}, `[dependencies.serde]\n# version = "1.0.0" was old\nnote = """\nversion = "1.0.0"\n"""\nversion = "1.1.0"\n`], [`extended table beside a same-named dev entry`, `[dependencies.serde]\nversion = "1.0.0"\n\n[dev-dependencies]\nserde = "1.0.0"\n`, `dependencies${fieldSep}serde`, {old: "1.0.0", new: "1.0.1"}, `[dependencies.serde]\nversion = "1.0.1"\n\n[dev-dependencies]\nserde = "1.0.0"\n`], - // TOML permits indentation and a trailing comment on a header, and a scope that misses one - // widens back to the file, so the dev entry below would be rewritten with it [`indented header beside a same-named dev entry`, ` [dependencies] # pinned\n serde = "1.0.0"\n\n [dev-dependencies]\n serde = "1.0.0"\n`, `dependencies${fieldSep}serde`, {old: "1.0.0", new: "1.0.1"}, ` [dependencies] # pinned\n serde = "1.0.1"\n\n [dev-dependencies]\n serde = "1.0.0"\n`], - // a bracketed line inside a multi-line string is text: taking it for a header loses the rewrite, - // and letting the extended-table pass reach it rewrites the package metadata instead + [`multiline delimiter in a comment`, `# """\n[dependencies]\nserde = "1.0.0"\n[dev-dependencies]\nserde = "1.0.0"\n`, + `dependencies${fieldSep}serde`, {old: "1.0.0", new: "1.1.0"}, `# """\n[dependencies]\nserde = "1.1.0"\n[dev-dependencies]\nserde = "1.0.0"\n`], [`header the description only quotes`, `[package]\ndescription = """\n[dependencies.serde]\nversion = "1.0.0"\n"""\n\n[dependencies]\nserde = "1.0.0"\n`, `dependencies${fieldSep}serde`, {old: "1.0.0", new: "1.0.1"}, `[package]\ndescription = """\n[dependencies.serde]\nversion = "1.0.0"\n"""\n\n[dependencies]\nserde = "1.0.1"\n`], [`oldOrig instead of old`, `[dependencies]\nserde = "1.0.0"\n`, @@ -47,35 +48,43 @@ test.each([ expect(updateCargoToml(input, {[key]: dep as any})).toBe(expected); }); -test("preserves surrounding content", () => { +test("updateCargoToml fails when its dependency table cannot be located", () => { + expect(() => updateCargoToml(`serde = "1.0.0"\n`, { + [`dependencies${fieldSep}serde`]: {old: "1.0.0", new: "1.1.0"} as any, + })).toThrow("Unable to locate Cargo table"); +}); + +test("updateCargoToml rewrites multiple dependencies within one table", () => { const input = [ `[package]`, `name = "my-crate"`, `version = "0.1.0"`, ``, `[dependencies]`, - `serde = "1.0.0"`, - `tokio = { version = "1.28.0", features = ["full"] }`, - `serde = { version = "1.0.0.1", features = ["derive", "rc"] }`, - ...Array.from({length: 10}, () => `tokio = { version = "1.0", default-features = false, features = ["net", "time"] }`), + `"serde" = "1.0.0"`, + `'tokio' = { version = "1.28.0", features = ["full"] }`, ``, `[dev-dependencies]`, `rand = "0.8.5"`, `serde = "1.0.0"`, ``, + `[dependencies."serde_json"]`, + `version = "1.0.0"`, + ``, ].join("\n"); const deps = { [`dependencies${fieldSep}serde`]: {old: "1.0.0", new: "1.0.1"} as any, [`dependencies${fieldSep}tokio`]: {old: "1.28.0", new: "1.30.0"} as any, + [`dependencies${fieldSep}serde_json`]: {old: "1.0.0", new: "1.0.2"} as any, }; const result = updateCargoToml(input, deps); - expect(result).toContain(`serde = "1.0.1"`); + expect(result).toContain(`"serde" = "1.0.1"`); expect(result).toContain(`version = "1.30.0", features = ["full"]`); expect(result).toContain(`name = "my-crate"`); expect(result).toContain(`[dev-dependencies]\nrand = "0.8.5"\nserde = "1.0.0"`); + expect(result).toContain(`[dependencies."serde_json"]\nversion = "1.0.2"`); }); -// fetchCratesIoInfo test("fetchCratesIoInfo happy path", async () => { const urls: Array = []; const body = sparse( @@ -106,7 +115,7 @@ test("fetchCratesIoInfo distills a large index body to the fields it reads", asy const body = sparse(...records); expect(body.length).toBeGreaterThan(16384); const [data] = await fetchCratesIoInfo("bulky", sparseCtx(body)); - expect(Object.keys(data.versions).length).toBe(199); // the yanked one is still dropped + expect(Object.keys(data.versions).length).toBe(199); expect(data["dist-tags"].latest).toBe("1.198.0"); expect(data.time["1.198.0"]).toBe("2025-01-01T00:00:00Z"); }); @@ -122,7 +131,7 @@ test("fetchCratesIoInfo shards the index path by name length", async () => { test("fetchCratesIoInfo latest is the highest release, not the newest published", async () => { const body = sparse( {vers: "2.0.0", yanked: false, pubtime: "2025-01-01T00:00:00Z"}, - {vers: "1.9.1", yanked: false, pubtime: "2025-06-01T00:00:00Z"}, // backport published later + {vers: "1.9.1", yanked: false, pubtime: "2025-06-01T00:00:00Z"}, {vers: "3.0.0-rc.1", yanked: false, pubtime: "2025-07-01T00:00:00Z"}, ); const [data] = await fetchCratesIoInfo("backported", sparseCtx(body)); @@ -135,16 +144,6 @@ test("fetchCratesIoInfo falls back to a prerelease when nothing is released", as expect(data["dist-tags"].latest).toBe("0.1.0-alpha.2"); }); -test("fetchCratesIoInfo filters yanked versions", async () => { - const body = sparse( - {vers: "1.0.0", yanked: false, pubtime: "2024-01-01T00:00:00Z"}, - {vers: "2.0.0", yanked: true, pubtime: "2025-02-01T00:00:00Z"}, - ); - const [data] = await fetchCratesIoInfo("serde-yanked", sparseCtx(body)); - expect(Object.keys(data.versions)).toEqual(["1.0.0"]); - expect(data["dist-tags"].latest).toBe("1.0.0"); -}); - test("fetchCratesIoInfo fetch failure throws", async () => { const ctx = { cratesIoUrl: "https://crates.io", @@ -166,30 +165,9 @@ test("fetchCratesIoInfo empty versions", async () => { expect(data["dist-tags"].latest).toBe(""); }); -test("quoted dependency keys are rewritten", () => { - const input = [ - `[dependencies]`, - `"serde" = "1.0.0"`, - `'rand' = { version = "0.8.0" }`, - ``, - `[dependencies."serde_json"]`, - `version = "1.0.0"`, - ``, - ].join("\n"); - const deps = { - [`dependencies${fieldSep}serde`]: {old: "1.0.0", new: "1.0.1"} as any, - [`dependencies${fieldSep}rand`]: {old: "0.8.0", new: "0.9.0"} as any, - [`dependencies${fieldSep}serde_json`]: {old: "1.0.0", new: "1.0.2"} as any, - }; - const result = updateCargoToml(input, deps); - expect(result).toContain(`"serde" = "1.0.1"`); - expect(result).toContain(`'rand' = { version = "0.9.0" }`); - expect(result).toContain(`[dependencies."serde_json"]\nversion = "1.0.2"`); -}); - test("target sections", () => { const input = [ - `[target.'cfg(unix)'.dependencies]`, + `[target.'cfg(feature = "foo.bar")'.dependencies]`, `libc = "0.2.0"`, ``, `[target.x86_64-pc-windows-msvc.dependencies.winapi]`, @@ -198,75 +176,59 @@ test("target sections", () => { `[target.'cfg(windows)'.build-dependencies.cc]`, `version = "1.0.0"`, ``, + `[dev-dependencies]`, + `libc = "0.2.0"`, + ``, ].join("\n"); const deps = { - [`target.cfg(unix).dependencies${fieldSep}libc`]: {old: "0.2.0", new: "0.2.1"} as any, - [`target.x86_64-pc-windows-msvc.dependencies${fieldSep}winapi`]: {old: "0.3.0", new: "0.3.9"} as any, - [`target.cfg(windows).build-dependencies${fieldSep}cc`]: {old: "1.0.0", new: "1.1.0"} as any, + [`${JSON.stringify(["target", `cfg(feature = "foo.bar")`, "dependencies"])}|crates/a${fieldSep}libc`]: + {old: "0.2.0", new: "0.2.1"} as any, + [`${JSON.stringify(["target", "x86_64-pc-windows-msvc", "dependencies"])}${fieldSep}winapi`]: + {old: "0.3.0", new: "0.3.9"} as any, + [`${JSON.stringify(["target", "cfg(windows)", "build-dependencies"])}${fieldSep}cc`]: + {old: "1.0.0", new: "1.1.0"} as any, }; const result = updateCargoToml(input, deps); expect(result).toContain(`libc = "0.2.1"`); expect(result).toContain(`[target.x86_64-pc-windows-msvc.dependencies.winapi]\nversion = "0.3.9"`); expect(result).toContain(`[target.'cfg(windows)'.build-dependencies.cc]\nversion = "1.1.0"`); + expect(result).toContain(`[dev-dependencies]\nlibc = "0.2.0"`); }); -// parseCargoLock -test("parseCargoLock returns name→versions map", () => { +test("parseCargoLock collects valid package versions", () => { const lock = ` [[package]] name = "serde" version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -`; - const map = parseCargoLock(lock); - expect(map.get("serde")).toEqual(["1.0.200"]); - expect(map.get("rand")).toEqual(["0.8.5"]); - expect(map.size).toBe(2); -}); - -test("parseCargoLock collects all versions when package appears multiple times", () => { - const lock = ` [[package]] name = "serde" -version = "1.0.100" - -[[package]] -name = "serde" -version = "1.0.200" -`; - expect(parseCargoLock(lock).get("serde")).toEqual(["1.0.100", "1.0.200"]); -}); - -test("parseCargoLock ignores entries without valid semver", () => { - const lock = ` -[[package]] -name = "my-crate" -version = "0.1.0" +version = "1.0.201" [[package]] name = "bad" version = "not-a-version" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" `; const map = parseCargoLock(lock); - expect(map.get("my-crate")).toEqual(["0.1.0"]); + expect(map.get("serde")).toEqual(["1.0.200", "1.0.201"]); + expect(map.get("rand")).toEqual(["0.8.5"]); + expect(map.size).toBe(2); expect(map.has("bad")).toBe(false); -}); - -test("parseCargoLock returns empty map for empty input", () => { expect(parseCargoLock("").size).toBe(0); }); -// findLockedVersion const locked = new Map([ ["serde", ["1.0.200"]], ["twoLines", ["0.8.5", "0.9.0"]], ["patched", ["1.0.100", "1.0.200"]], ["wide", ["1.0.5", "1.5.0", "2.0.0"]], + ["list", ["0.1.6"]], ]); test.each([ @@ -275,17 +237,17 @@ test.each([ ["twoLines", "0.9", "0.9.0"], ["twoLines", "^0.8", "0.8.5"], ["twoLines", "^0.9", "0.9.0"], - ["patched", "1.0", "1.0.200"], // the highest match, not the first + ["patched", "1.0", "1.0.200"], ["wide", ">= 1.0.0, < 2.0.0", "1.5.0"], ["wide", "1.0.*", "1.0.5"], ["wide", "1.*", "1.5.0"], + ["list", "0.1.0, 0.1.4, 0.1.6", "0.1.6"], ["unknown", "1.0", undefined], ])("findLockedVersion %s %s", (name, range, expected) => { expect(findLockedVersion(locked, name, range)).toBe(expected); }); test.each([ - // bare ranges keep the part count they were authored with ["1", "2.0.0", "2"], ["1.0", "1.1.0", "1.1"], ["1.0", "2.0.0", "2.0"], @@ -302,12 +264,12 @@ test.each([ ["1.0.*", "1.1.0", "1.1.*"], ["0.8.*", "0.9.0", "0.9.*"], ["1.x", "2.1.0", "2.x"], - ["1.0.*", "1.1.0-rc.1", "1.1.*"], + ["1.0.*", "1.1.0-rc.1", "1.1.0-rc.1"], + [" = 1.0.0", "1.1.0", " = 1.1.0"], [">= 0.1.21, < 0.2.0", "0.1.24", ">= 0.1.24, < 0.2.0"], [">= 0.1.21, <= 0.2.0", "0.1.24", ">= 0.1.24, <= 0.2.0"], [">= 0.0.1, < 0.1", "0.2.1", ">= 0.2.1, < 0.3"], [">=1.0.0,<2.0.0", "1.5.0", ">=1.5.0,<2.0.0"], - // an upper bound the new version already clears stays as authored ["<2.0.0", "1.5.0", "<2.0.0"], ["<1.3.4", "1.5.0", "<1.5.1"], ])("updateCargoRange %s to %s", (range, version, expected) => { @@ -317,5 +279,8 @@ test.each([ test("cargoToNpmRange swaps comma separators for whitespace", () => { expect(cargoToNpmRange(">= 1.0.0, < 2.0.0")).toBe(">= 1.0.0 < 2.0.0"); expect(cargoToNpmRange(">=1.0.0,<2.0.0")).toBe(">=1.0.0 <2.0.0"); + expect(cargoToNpmRange(" 1.0.0")).toBe("^1.0.0"); + expect(cargoToNpmRange("0.1.0, 0.1.4, 0.1.6")).toBe("^0.1.0 ^0.1.4 ^0.1.6"); + expect(cargoToNpmRange("1.*, 2.*")).toBe("1.* 2.*"); expect(cargoToNpmRange("^1.0")).toBe("^1.0"); }); diff --git a/modes/cargo.ts b/modes/cargo.ts index 19fdf1f..935291f 100644 --- a/modes/cargo.ts +++ b/modes/cargo.ts @@ -7,7 +7,6 @@ type SparseIndexRecord = {vers?: string; yanked?: boolean; pubtime?: string}; const cratesIoByCtx = new WeakMap>>>(); -// The index is sharded by lowercased name length: `1/a`, `2/ab`, `3/a/abc`, `se/rd/serde`. function indexSuffix(name: string): string { const lower = name.toLowerCase(); if (lower.length <= 2) return `${lower.length}/${lower}`; @@ -15,17 +14,11 @@ function indexSuffix(name: string): string { return `${lower.slice(0, 2)}/${lower.slice(2, 4)}/${lower}`; } -// The HTTP API caps a page at 100 versions and carries no etag, the sparse index answers with the -// whole crate in one revalidatable request. Only crates.io splits the two, so an override is an -// index root. function indexUrl(cratesIoUrl: string, name: string): string { const base = normalizeUrl(cratesIoUrl); return `${base === "https://crates.io" ? "https://index.crates.io" : base}/${indexSuffix(name)}`; } -// Index records carry the crate's whole dependency list, features and checksum, none of which is -// read. A line that is not a record is kept verbatim, so a body of nothing but those still reads -// as the malformed response it is. function reduceSparseIndex(body: string): string { return body.split("\n").map(line => { if (!line) return line; @@ -41,14 +34,11 @@ function reduceSparseIndex(body: string): string { export async function fetchCratesIoInfo(name: string, ctx: ModeContext): Promise { const url = indexUrl(ctx.cratesIoUrl, name); - // dedup in-flight/completed requests per run; disk-cache staleness is gated by ctx.noCache inside fetchWithEtag const data = await dedupe(cratesIoByCtx, ctx, url, async () => { const result = await fetchWithEtag(url, ctx, getFetchOpts(), reduceSparseIndex); if (!("body" in result)) throwFetchError(result.res, url, name, ctx.cratesIoUrl); const versions: Record> = {}; const time: Record = {}; - // crates.io has no maintainer-set tag, so its "max stable version" stands in: the highest - // release, prereleases counting only for a crate that has published nothing else. let latest = ""; let latestPre = ""; let parsedLines = 0; @@ -62,7 +52,6 @@ export async function fetchCratesIoInfo(name: string, ctx: ModeContext): Promise } parsedLines++; if (!record?.vers || record.yanked) continue; - // Build metadata (`0.14.7+wasi-0.2.4`) is no part of a range, and renovate's crate datasource drops it too. const version = record.vers.split("+")[0]; versions[version] = {}; if (record.pubtime) time[version] = record.pubtime; @@ -94,48 +83,46 @@ export function parseCargoLock(lockStr: string): Map { return map; } -// Cargo treats bare version strings as caret ranges (e.g. "1.0" = "^1.0"). const startsWithDigitRe = /^\d/; -// A wildcard requirement is not one of those: `1.0.*` caps at 1.0.x where `^1.0` would not. -// Cargo normalizes `1.x` and `1.*.*` to `1.*`, so both spellings have to be recognized. const wildcardRe = /^(\d+(?:\.\d+)*)((?:\.[*xX])+)$/; -const commaRe = /\s*,\s*/g; -const commaSplitRe = /(\s*,\s*)/; +const trimRe = /^(\s*)(.*?)(\s*)$/s; -// npm range syntax has no comma, so a comparator list travels whitespace-separated and gets its -// authored separators back in updateCargoRange. -export const cargoToNpmRange = (range: string): string => range.includes(",") ? range.replace(commaRe, " ") : range; - -const toNpmRange = (range: string): string => - cargoToNpmRange(startsWithDigitRe.test(range) && !wildcardRe.test(range) ? `^${range}` : range); +export const cargoToNpmRange = (range: string): string => range.split(/\s*,\s*/) + .map(part => { + const value = part.trim(); + return startsWithDigitRe.test(value) && !wildcardRe.test(value) ? `^${value}` : value; + }) + .join(" "); function updateComparator(comparator: string, newVersion: string): string { - // An upper bound the new version already clears stays as authored, as renovate leaves a matching range alone. - if (comparator.startsWith("<") && satisfies(newVersion, comparator)) return comparator; + const [, leading, value, trailing] = trimRe.exec(comparator)!; + if (value.startsWith("<") && satisfies(newVersion, value)) return comparator; - const wildcard = wildcardRe.exec(comparator); + const wildcard = wildcardRe.exec(value); + let updated: string; if (wildcard) { - const [, digits, stars] = wildcard; - return `${newVersion.split(/[-+]/)[0].split(".").slice(0, digits.split(".").length).join(".")}${stars}`; - } - if (startsWithDigitRe.test(comparator)) { - return updateVersionRange(normalizeRange(`^${comparator}`), newVersion, `^${comparator}`).replace(/^\^/, ""); + if (parse(newVersion)?.prerelease.length) { + updated = newVersion; + } else { + const [, digits, stars] = wildcard; + updated = `${newVersion.split(/[-+]/)[0].split(".").slice(0, digits.split(".").length).join(".")}${stars}`; + } + } else if (startsWithDigitRe.test(value)) { + updated = updateVersionRange(normalizeRange(`^${value}`), newVersion, `^${value}`).replace(/^\^/, ""); + } else { + updated = updateVersionRange(normalizeRange(value), newVersion, value); } - return updateVersionRange(normalizeRange(comparator), newVersion, comparator); + return `${leading}${updated}${trailing}`; } export function updateCargoRange(oldOrig: string, newVersion: string): string { - if (!oldOrig.includes(",")) return updateComparator(oldOrig, newVersion); - // Splitting on a capturing separator keeps the authored spacing for the rejoin. - const parts = oldOrig.split(commaSplitRe); - for (let i = 0; i < parts.length; i += 2) parts[i] = updateComparator(parts[i], newVersion); - return parts.join(""); + return oldOrig.split(/(\s*,\s*)/).map((part, idx) => idx % 2 ? part : updateComparator(part, newVersion)).join(""); } export function findLockedVersion(allVersions: Map, name: string, range: string): string | undefined { const versions = allVersions.get(name); if (!versions) return undefined; - const npmRange = toNpmRange(range); + const npmRange = cargoToNpmRange(range); let best: string | undefined; for (const version of versions) { if (satisfies(version, npmRange) && (!best || gt(version, best))) { @@ -145,39 +132,47 @@ export function findLockedVersion(allVersions: Map, name: stri return best; } -// A TOML key may be written bare, "quoted" or 'quoted' and the parser hands back the bare name, so -// a key matched back in the source has to accept all three spellings. const tomlKey = (key: string) => `(?:${esc(key)}|"${esc(key)}"|'${esc(key)}')`; +const jsonStringArrayRe = /^\[(?:"(?:\\.|[^"\\])*"(?:,"(?:\\.|[^"\\])*")*)?\]/; -// `[x]` or `[[x]]`, with the indentation and trailing comment TOML permits around it. const tableHeaderRe = /^[ \t]*\[(\[?)[ \t]*([^[\]]+?)[ \t]*\]\1[ \t]*(?:#.*)?[ \t\r]*$/; -// An odd count opens a multi-line string, inside which a bracketed line is text, not a header. -function multilineDelim(line: string): string { - for (const delim of [`"""`, `'''`]) { - if (line.split(delim).length % 2 === 0) return delim; +function multilineDelim(line: string, delimiter: string): string { + for (let index = 0; index < line.length; index++) { + const char = line[index]; + if (delimiter.length === 3) { + let backslashes = 0; + while (line[index - backslashes - 1] === `\\`) backslashes++; + if (line.startsWith(delimiter, index) && !(delimiter === `"""` && backslashes % 2)) { + index += 2; + delimiter = ""; + } + } else if (delimiter) { + if (delimiter === `"` && char === `\\`) index++; + else if (char === delimiter) delimiter = ""; + } else if (char === "#") { + break; + } else if ((char === `"` || char === `'`) && line.startsWith(char.repeat(3), index)) { + delimiter = char.repeat(3); index += 2; + } else if (char === `"` || char === `'`) { delimiter = char; } } - return ""; + return delimiter.length === 3 ? delimiter : ""; } -// The span each table occupies, its header line included. An array of tables holds no dependency -// but still ends the table before it, so it takes part with a path nothing matches. -function tableSpans(str: string): Array<{path: string, start: number, end: number}> { - const spans: Array<{path: string, start: number, end: number}> = []; - let delim = ""; +type CargoRewrite = {simpleRe: RegExp, inlineRe: RegExp, versionRe?: RegExp, newValue: string}; +type CargoTable = {path: string, start: number, end: number, rewrites: Array}; + +function tableSpans(str: string): Array { + const spans: Array = []; + let delimiter = ""; let pos = 0; for (const line of str.split("\n")) { - if (delim) { - if (line.includes(delim)) delim = ""; + const header = delimiter ? null : tableHeaderRe.exec(line); + if (header) { + if (spans.length) spans.at(-1)!.end = pos; + spans.push({path: header[1] ? "" : header[2], start: pos, end: str.length, rewrites: []}); } else { - const header = tableHeaderRe.exec(line); - if (header) { - const previous = spans.at(-1); - if (previous) previous.end = pos; - spans.push({path: header[1] ? "" : header[2], start: pos, end: str.length}); - } else { - delim = multilineDelim(line); - } + delimiter = multilineDelim(line, delimiter); } pos += line.length + 1; } @@ -185,35 +180,45 @@ function tableSpans(str: string): Array<{path: string, start: number, end: numbe } export function updateCargoToml(pkgStr: string, deps: Deps): string { - let newPkgStr = pkgStr; + const spans = tableSpans(pkgStr); for (const [key, dep] of Object.entries(deps)) { const [typeKey, name] = key.split(fieldSep); const oldValue = dep.oldOrig || dep.old; const newValue = dep.new; const nameEsc = tomlKey(name); const oldEsc = esc(oldValue); - // Built from the dep's own type so `[target.'cfg(unix)'.dependencies]` and any other configured - // section work. Workspace members carry a `|path` suffix on the type. - const sectionEsc = typeKey.split("|")[0].split(".").map(tomlKey).join("\\."); - - // The forms that name no table of their own stay inside the table the dependency was read from, - // as the same name at the same version may well sit in another one too. Its own `[section.name]` - // wins, or the bare `[section]` above it would claim the rewrite, and a table neither pattern - // finds leaves the whole file as the scope rather than losing the rewrite. + const typePath: Array = typeKey.startsWith("[") ? JSON.parse(jsonStringArrayRe.exec(typeKey)![0]) : + typeKey.split("|", 1)[0].split("."); + const sectionEsc = typePath.map(tomlKey).join("\\."); const ownRe = new RegExp(`^${sectionEsc}\\.${nameEsc}$`); const sectionRe = new RegExp(`^${sectionEsc}$`); - const rewrite = (scope: string) => scope - // Simple form: name = "version" or name = 'version' - .replace(new RegExp(`^(\\s*${nameEsc}\\s*=\\s*["'])${oldEsc}(["'].*)$`, "gm"), `$1${newValue}$2`) - // Inline table: name = { ..., version = "x.y.z", ... } (version need not be the first key) - .replace(new RegExp(`^(\\s*${nameEsc}\\s*=\\s*\\{(?:"[^"\\n]*"|'[^'\\n]*'|[^"'}\\n])*?\\bversion\\s*=\\s*["'])${oldEsc}(["'])`, "gm"), `$1${newValue}$2`) - // Extended table: [section.name] with version = "x.y.z", which the scope above is that table - .replace(new RegExp(`(\\[${sectionEsc}\\.${nameEsc}\\](?:(?!\\n\\[)[\\s\\S])*?version\\s*=\\s*["'])${oldEsc}(["'])`, "g"), `$1${newValue}$2`); - const spans = tableSpans(newPkgStr); - const span = spans.find(entry => ownRe.test(entry.path)) ?? spans.find(entry => sectionRe.test(entry.path)); - newPkgStr = span ? - newPkgStr.slice(0, span.start) + rewrite(newPkgStr.slice(span.start, span.end)) + newPkgStr.slice(span.end) : - rewrite(newPkgStr); + const ownSpan = spans.find(entry => ownRe.test(entry.path)); + const span = ownSpan ?? spans.find(entry => sectionRe.test(entry.path)); + if (!span) throw new Error(`Unable to locate Cargo table for ${typeKey}.${name}`); + span.rewrites.push({ + simpleRe: new RegExp(`^(\\s*${nameEsc}\\s*=\\s*["'])${oldEsc}(["'].*)$`), + inlineRe: new RegExp(`^(\\s*${nameEsc}\\s*=\\s*\\{(?:"[^"\\n]*"|'[^'\\n]*'|[^"'}\\n])*?\\bversion\\s*=\\s*["'])${oldEsc}(["'])`), + ...(ownSpan && {versionRe: new RegExp(`^(\\s*version\\s*=\\s*["'])${oldEsc}(["'].*)$`)}), + newValue, + }); + } + let result = pkgStr; + for (const span of spans.reverse()) { + if (!span.rewrites.length) continue; + let delimiter = ""; + const scope = pkgStr.slice(span.start, span.end).replace(/^.*$/gm, originalLine => { + let line = originalLine; + if (!delimiter) { + for (const rewrite of span.rewrites) { + line = line.replace(rewrite.simpleRe, `$1${rewrite.newValue}$2`) + .replace(rewrite.inlineRe, `$1${rewrite.newValue}$2`); + if (rewrite.versionRe) line = line.replace(rewrite.versionRe, `$1${rewrite.newValue}$2`); + } + } + delimiter = multilineDelim(line, delimiter); + return line; + }); + result = result.slice(0, span.start) + scope + result.slice(span.end); } - return newPkgStr; + return result; } diff --git a/modes/docker.test.ts b/modes/docker.test.ts index e3ba2b7..14a6edb 100644 --- a/modes/docker.test.ts +++ b/modes/docker.test.ts @@ -1,39 +1,30 @@ import { - parseDockerImageRef, - parseDockerTag, - formatDockerVersion, - isComposeFile, - dockerExactFileNames, - isDockerfile, - isDockerFileName, - getDockerInfoUrl, - extractDockerRefs, - findDockerVersion, - updateDockerfile, - updateComposeFile, - updateWorkflowDockerImages, - getExtractionRegex, - dockerfileFromRe, - composeImageRe, - fetchDockerHubTags, - fetchDockerInfo, - dockerImageNames, - filterStableTags, + composeImageRe, dockerExactFileNames, dockerfileFromRe, dockerImageNames, extractDockerRefs, fetchDockerHubTags, + fetchDockerInfo, fetchDockerTagDigest, filterStableTags, findDockerVersion, formatDockerVersion, getDockerInfoUrl, + getExtractionRegex, isComposeFile, isDockerfile, isDockerFileName, parseDockerImageRef, parseDockerTag, + updateComposeFile, updateDockerfile, updateWorkflowDockerImages, } from "./docker.ts"; import {type ModeContext, fetchTimeout, fieldSep} from "./shared.ts"; -// parseDockerImageRef +const allSemvers = new Set(["patch", "minor", "major"]); +const oldDigest = `sha256:${"a".repeat(64)}`; +const newDigest = `sha256:${"b".repeat(64)}`; + test.each([ ["simple library image", "node:18", {registry: null, namespace: "library", repo: "node", tag: "18", fullImage: "node"}], ["namespaced image", "myorg/myapp:1.0.0", {registry: null, namespace: "myorg", repo: "myapp", tag: "1.0.0", fullImage: "myorg/myapp"}], ["a registry", "ghcr.io/owner/repo:v1.2.3", {registry: "ghcr.io", namespace: "owner", repo: "repo", tag: "v1.2.3", fullImage: "ghcr.io/owner/repo"}], ["a docker:// prefix", "docker://node:18", {registry: null, namespace: "library", repo: "node", tag: "18", fullImage: "node"}], - // docker.io and index.docker.io are Docker Hub, so neither counts as a registry ["docker.io", "docker.io/library/node:18", {registry: null, namespace: "library", repo: "node", tag: "18", fullImage: "docker.io/library/node"}], ["index.docker.io", "index.docker.io/myorg/myapp:1.0.0", {registry: null, namespace: "myorg", repo: "myapp", tag: "1.0.0", fullImage: "index.docker.io/myorg/myapp"}], + ["registry-1.docker.io", "registry-1.docker.io/node:18", {registry: null, namespace: "library", repo: "node", tag: "18", fullImage: "registry-1.docker.io/node"}], + ["a deep Hub path", "org/team/image:1.2.3", {registry: null, namespace: "org/team", repo: "image", tag: "1.2.3", fullImage: "org/team/image"}], + ["localhost registry", "localhost/owner/image:1.2.3", {registry: "localhost", namespace: "owner", repo: "image", tag: "1.2.3", fullImage: "localhost/owner/image"}], ["a tag with suffix", "node:18-alpine", {registry: null, namespace: "library", repo: "node", tag: "18-alpine", fullImage: "node"}], ["full semver with suffix", "node:18.19.1-bookworm", {registry: null, namespace: "library", repo: "node", tag: "18.19.1-bookworm", fullImage: "node"}], - ["a digest", "node@sha256:abc123", null], + ["a digest", "node@sha256:abc123", {registry: null, namespace: "library", repo: "node", tag: "latest", fullImage: "node", digest: "sha256:abc123", digestOnly: true}], + ["a tag and digest", "node:18@sha256:abc123", {registry: null, namespace: "library", repo: "node", tag: "18", fullImage: "node", digest: "sha256:abc123"}], + ["a non-version tag and digest", "node:latest@sha256:abc123", {registry: null, namespace: "library", repo: "node", tag: "latest", fullImage: "node", digest: "sha256:abc123"}], ["no tag", "node", null], ["a non-semver tag", "node:latest", null], ["a non-semver word tag", "node:bullseye", null], @@ -44,19 +35,20 @@ test.each([ test("dockerImageNames", () => { const hub = ["mysql", "library/mysql", "docker.io/mysql", "docker.io/library/mysql"]; expect(dockerImageNames("mysql")).toEqual(hub); - expect(dockerImageNames("docker.io/mysql")).toEqual(["docker.io/mysql", ...hub.filter(n => n !== "docker.io/mysql")]); + expect(dockerImageNames("docker.io/mysql")).toEqual([ + "docker.io/mysql", ...hub.filter(name => name !== "docker.io/mysql"), + ]); expect(dockerImageNames("index.docker.io/library/mysql")).toEqual(["index.docker.io/library/mysql", ...hub]); expect(dockerImageNames("grafana/grafana")).toEqual(["grafana/grafana", "docker.io/grafana/grafana"]); expect(dockerImageNames("ghcr.io/foo/bar")).toEqual(["ghcr.io/foo/bar"]); }); -// parseDockerTag test.each([ ["18", {version: "18", prerelease: "", suffix: ""}], ["18.19.1", {version: "18.19.1", prerelease: "", suffix: ""}], ["18-alpine", {version: "18", prerelease: "", suffix: "-alpine"}], ["v1.2.3", {version: "v1.2.3", prerelease: "", suffix: ""}], - // a hyphen starts the variant suffix, so `-rc` is a channel while `rc3` is a prerelease of 1.27 + ["1.2.3.4-alpine", {version: "1.2.3.4", prerelease: "", suffix: "-alpine"}], ["1.27-rc", {version: "1.27", prerelease: "", suffix: "-rc"}], ["1.27rc3", {version: "1.27", prerelease: "rc3", suffix: ""}], ["1.27rc3-alpine", {version: "1.27", prerelease: "rc3", suffix: "-alpine"}], @@ -93,12 +85,10 @@ test.each([ }); test("dockerExactFileNames stay within isDockerFileName", () => { - // findUpSync needs literal names, but api.ts routes them by predicate afterwards expect(dockerExactFileNames.every(isDockerFileName)).toBe(true); expect(dockerExactFileNames).toContain("compose.yaml"); }); -// getDockerInfoUrl test.each([ ["library image", {registry: null, namespace: "library", repo: "node", tag: "18", fullImage: "node"}, "https://hub.docker.com/_/node"], ["user image", {registry: null, namespace: "myorg", repo: "myapp", tag: "1.0", fullImage: "myorg/myapp"}, "https://hub.docker.com/r/myorg/myapp"], @@ -107,51 +97,49 @@ test.each([ expect(getDockerInfoUrl(ref)).toBe(expected); }); -// extractDockerRefs -test("extractDockerRefs with Dockerfile content", () => { - const content = "FROM node:18\nFROM --platform=linux/amd64 nginx:1.25.3\nFROM ubuntu:latest\n"; - const results = extractDockerRefs(content, dockerfileFromRe); - expect(results).toHaveLength(2); - expect(results[0].ref.repo).toBe("node"); - expect(results[0].ref.tag).toBe("18"); - expect(results[1].ref.repo).toBe("nginx"); - expect(results[1].ref.tag).toBe("1.25.3"); -}); - -test("extractDockerRefs with compose content", () => { - const content = "services:\n web:\n image: node:20.11.1\n db:\n image: postgres:16.2\n"; - const results = extractDockerRefs(content, composeImageRe); - expect(results).toHaveLength(2); - expect(results[0].match).toBe("node:20.11.1"); - expect(results[1].match).toBe("postgres:16.2"); +test("extractDockerRefs", () => { + const dockerfile = [ + "ARG NODE_VERSION=18", + `FROM node:\${NODE_VERSION}`, + "FROM --platform=$BUILDPLATFORM \\", + " nginx:1.25.3@sha256:abc123", + "FROM ubuntu:latest", + "", + ].join("\n"); + const dockerfileRefs = extractDockerRefs(dockerfile, dockerfileFromRe); + expect(dockerfileRefs).toHaveLength(2); + expect(dockerfileRefs[0].ref.repo).toBe("node"); + expect(dockerfileRefs[0].ref.tag).toBe("18"); + expect(dockerfileRefs[1].ref.repo).toBe("nginx"); + expect(dockerfileRefs[1].ref.tag).toBe("1.25.3"); + expect(dockerfileRefs[1].ref.digest).toBe("sha256:abc123"); + const compose = "services:\n web:\n image: node:20.11.1\n db:\n image: postgres:16.2\n build: .\n"; + const composeRefs = extractDockerRefs(compose, composeImageRe); + expect(composeRefs).toHaveLength(1); + expect(composeRefs[0].match).toBe("node:20.11.1"); }); -// findDockerVersion -test("findDockerVersion finds upgrade with same suffix", () => { +test("findDockerVersion basic selection", () => { const tagMap: Record = { "18": "2024-01-01", "20": "2024-06-01", "20-alpine": "2024-06-01", "18-alpine": "2024-01-01", }; - const result = findDockerVersion(tagMap, "18", new Set(["patch", "minor", "major"])); + const result = findDockerVersion(tagMap, "18", allSemvers); expect(result).toEqual({newTag: "20", date: "2024-06-01"}); -}); - -test("findDockerVersion returns null when no upgrade", () => { - const tagMap: Record = {"18": "2024-01-01"}; - expect(findDockerVersion(tagMap, "18", new Set(["patch", "minor", "major"]))).toBeNull(); + expect(findDockerVersion({"18": "2024-01-01"}, "18", allSemvers)).toBeNull(); + expect(findDockerVersion({"20": "2024-01-01"}, "latest", allSemvers)).toBeNull(); }); test("findDockerVersion filters by suffix", () => { - const semvers = new Set(["patch", "minor", "major"]); const tagMap: Record = { "18-alpine": "2024-01-01", "20": "2024-06-01", "20-alpine": "2024-06-01", }; - expect(findDockerVersion(tagMap, "18-alpine", semvers)).toEqual({newTag: "20-alpine", date: "2024-06-01"}); - expect(findDockerVersion({"18": "2024-01-01", "20-alpine": "2024-06-01"}, "18", semvers)).toBeNull(); + expect(findDockerVersion(tagMap, "18-alpine", allSemvers)).toEqual({newTag: "20-alpine", date: "2024-06-01"}); + expect(findDockerVersion({"18": "2024-01-01", "20-alpine": "2024-06-01"}, "18", allSemvers)).toBeNull(); const suffixed: Record = { "1.2.3-alpine3.19": "2024-01-01", "1.3.0-alpine": "2024-06-01", @@ -159,29 +147,24 @@ test("findDockerVersion filters by suffix", () => { "1.3.0-alpine3.19": "2024-06-02", "1.3.0-nanoserver-1809": "2024-06-03", }; - expect(findDockerVersion(suffixed, "1.2.3-alpine3.19", semvers)).toEqual({newTag: "1.3.0-alpine3.19", date: "2024-06-02"}); - expect(findDockerVersion(suffixed, "1.2.3-nanoserver-1809", semvers)).toEqual({newTag: "1.3.0-nanoserver-1809", date: "2024-06-03"}); -}); - -test("findDockerVersion returns null for invalid tag", () => { - expect(findDockerVersion({"20": "2024-01-01"}, "latest", new Set(["patch", "minor", "major"]))).toBeNull(); + expect(findDockerVersion(suffixed, "1.2.3-alpine3.19", allSemvers)).toEqual({newTag: "1.3.0-alpine3.19", date: "2024-06-02"}); + expect(findDockerVersion(suffixed, "1.2.3-nanoserver-1809", allSemvers)).toEqual({newTag: "1.3.0-nanoserver-1809", date: "2024-06-03"}); }); test("findDockerVersion keeps the authored precision", () => { - const semvers = new Set(["patch", "minor", "major"]); const tagMap: Record = { "18": "2024-01-01", "20": "2024-06-01", "20.11": "2024-06-10", "20.11.1": "2024-06-15", }; - expect(findDockerVersion(tagMap, "18", semvers)).toEqual({newTag: "20", date: "2024-06-01"}); - expect(findDockerVersion(tagMap, "18.19", semvers)).toEqual({newTag: "20.11", date: "2024-06-10"}); - // a floating "18" must not be pinned to "20.11.1" when no same-precision tag was fetched - expect(findDockerVersion({"18": "2024-01-01", "20.11.1": "2024-06-15"}, "18", semvers)).toBeNull(); - // a coerced version that does not spell back to the real tag still writes the real tag - expect(findDockerVersion({"24.04": "2024-04-01", "26.04": "2026-04-01"}, "24.04", semvers)) + expect(findDockerVersion(tagMap, "18", allSemvers)).toEqual({newTag: "20", date: "2024-06-01"}); + expect(findDockerVersion(tagMap, "18.19", allSemvers)).toEqual({newTag: "20.11", date: "2024-06-10"}); + expect(findDockerVersion({"18": "2024-01-01", "20.11.1": "2024-06-15"}, "18", allSemvers)).toBeNull(); + expect(findDockerVersion({"24.04": "2024-04-01", "26.04": "2026-04-01"}, "24.04", allSemvers)) .toEqual({newTag: "26.04", date: "2026-04-01"}); + expect(findDockerVersion({"1.2.3.4-alpine": "2024-01-01", "1.2.3.5-alpine": "2024-06-01"}, "1.2.3.4-alpine", allSemvers)) + .toEqual({newTag: "1.2.3.5-alpine", date: "2024-06-01"}); }); test("findDockerVersion ignores tags from another versioning scheme", () => { @@ -191,41 +174,30 @@ test("findDockerVersion ignores tags from another versioning scheme", () => { "3.24.1": "2026-06-16", "20260127": "2026-01-28", }; - const semvers = new Set(["patch", "minor", "major"]); - expect(findDockerVersion(tagMap, "3.24", semvers)).toBeNull(); // fewer fields - expect(findDockerVersion(tagMap, "3", semvers)).toBeNull(); // same fields, date magnitude - // same scheme as the authored tag, so date snapshots still upgrade among themselves - expect(findDockerVersion(tagMap, "20251224", semvers)).toEqual({newTag: "20260127", date: "2026-01-28"}); - // a real major bump that grows a digit is not a scheme change - expect(findDockerVersion({"9": "2020-01-01", "10": "2020-06-01"}, "9", semvers)).toEqual({newTag: "10", date: "2020-06-01"}); + expect(findDockerVersion(tagMap, "3.24", allSemvers)).toBeNull(); + expect(findDockerVersion(tagMap, "3", allSemvers)).toBeNull(); + expect(findDockerVersion(tagMap, "20251224", allSemvers)).toEqual({newTag: "20260127", date: "2026-01-28"}); + expect(findDockerVersion({"9": "2020-01-01", "10": "2020-06-01"}, "9", allSemvers)).toEqual({newTag: "10", date: "2020-06-01"}); }); test("findDockerVersion cooldown needs a timestamp", () => { - const semvers = new Set(["patch", "minor", "major"]); const now = Date.parse("2024-07-01"); const tagMap: Record = {"18": "2024-01-01", "20": "2024-06-25", "19": ""}; - expect(findDockerVersion(tagMap, "18", semvers, 30, now)).toBeNull(); - expect(findDockerVersion(tagMap, "18", semvers)).toEqual({newTag: "20", date: "2024-06-25"}); + expect(findDockerVersion(tagMap, "18", allSemvers, 30, now)).toBeNull(); + expect(findDockerVersion(tagMap, "18", allSemvers)).toEqual({newTag: "20", date: "2024-06-25"}); }); -test("findDockerVersion respects pinnedRange and blocks out-of-range upgrade", () => { - const tagMap: Record = { +test("findDockerVersion respects pinnedRange", () => { + expect(findDockerVersion({ "8.0": "2024-01-01", "8.0.41": "2024-06-01", "9.7": "2024-12-01", - }; - const result = findDockerVersion(tagMap, "8.0", new Set(["patch", "minor", "major"]), undefined, undefined, "8.0"); - expect(result).toBeNull(); -}); - -test("findDockerVersion respects pinnedRange and allows in-range upgrade", () => { - const tagMap: Record = { + }, "8.0", allSemvers, undefined, undefined, "8.0")).toBeNull(); + expect(findDockerVersion({ "8.0.0": "2024-01-01", "8.0.41": "2024-06-01", "9.7": "2024-12-01", - }; - const result = findDockerVersion(tagMap, "8.0.0", new Set(["patch", "minor", "major"]), undefined, undefined, "8.0"); - expect(result).toEqual({newTag: "8.0.41", date: "2024-06-01"}); + }, "8.0.0", allSemvers, undefined, undefined, "8.0")).toEqual({newTag: "8.0.41", date: "2024-06-01"}); }); test.each([ @@ -258,17 +230,46 @@ test.each([ expect(update(content, {[`docker${fieldSep}${image}`]: dep})).toBe(expected); }); -test("updateDockerfile leaves digest-pinned and suffixed occurrences alone", () => { - // the extractor skips digest-pinned refs, so rewriting the tag here would leave the file - // claiming a version the untouched digest contradicts - const digest = `@sha256:${"a".repeat(64)}`; - const content = `FROM node:18 AS build\nFROM node:18${digest}\nFROM node:18+build\n`; +test("updateDockerfile rewrites tag and digest atomically", () => { + const content = `FROM node:18 AS build\nFROM node:18@${oldDigest}\nFROM node:18+build\n`; + const deps = {[`docker${fieldSep}node`]: {old: "18", new: "20", oldDigest, newDigest}}; + expect(updateDockerfile(content, deps)).toBe(`FROM node:18 AS build\nFROM node:20@${newDigest}\nFROM node:18+build\n`); + expect(updateDockerfile(content, {[`docker${fieldSep}node`]: {old: "18", new: "20"}})) + .toBe(`FROM node:20 AS build\nFROM node:18@${oldDigest}\nFROM node:18+build\n`); +}); + +test("Docker image writers rewrite digest references atomically", () => { + const tagged = {[`docker${fieldSep}node`]: {old: "18", new: "20", oldDigest, newDigest}}; + expect(updateComposeFile(`services:\n app:\n image: node:18@${oldDigest}\n`, tagged)) + .toBe(`services:\n app:\n image: node:20@${newDigest}\n`); + expect(updateWorkflowDockerImages(`steps:\n - uses: docker://node:18@${oldDigest}\n`, tagged)) + .toBe(`steps:\n - uses: docker://node:20@${newDigest}\n`); + const digestOnly = {[`docker${fieldSep}node`]: {old: "latest", new: "latest", oldDigest, newDigest, digestOnly: true}}; + expect(updateWorkflowDockerImages(`steps:\n - uses: docker://node@${oldDigest}\n`, digestOnly)) + .toBe(`steps:\n - uses: docker://node@${newDigest}\n`); +}); + +test("updateDockerfile rewrites the ARG owning a multiline FROM version", () => { + const version = "$" + "{VERSION}"; + const content = `ARG VERSION=18\nFROM --platform=$BUILDPLATFORM \\\n node:${version}\n`; + const deps = {[`docker${fieldSep}node`]: {old: "18", new: "20"}}; + expect(updateDockerfile(content, deps)).toBe(`ARG VERSION=20\nFROM --platform=$BUILDPLATFORM \\\n node:${version}\n`); +}); + +test("updateDockerfile rewrites an ARG and digest atomically", () => { + const version = "$" + "{VERSION}"; + const content = `ARG VERSION=18\nFROM node:${version}@${oldDigest}\n`; + const deps = {[`docker${fieldSep}node`]: {old: "18", new: "20", oldDigest, newDigest}}; + expect(updateDockerfile(content, deps)).toBe(`ARG VERSION=20\nFROM node:${version}@${newDigest}\n`); +}); + +test("updateComposeFile leaves locally built service images alone", () => { + const content = "services:\n built:\n image: node:18\n build: .\n pulled:\n image: node:18\n"; const deps = {[`docker${fieldSep}node`]: {old: "18", new: "20"}}; - expect(updateDockerfile(content, deps)).toBe(`FROM node:20 AS build\nFROM node:18${digest}\nFROM node:18+build\n`); + expect(updateComposeFile(content, deps)).toBe("services:\n built:\n image: node:18\n build: .\n pulled:\n image: node:20\n"); }); test("updateDockerfile rewrites one image at several tags without cascading", () => { - // one dep's new tag is another dep's old tag, which a per-dep pass would rewrite twice const content = "FROM node:18 AS build\nFROM node:18-alpine\nFROM node:20\n"; const deps = { [`docker${fieldSep}node${fieldSep}18`]: {old: "18", new: "20"}, @@ -278,7 +279,6 @@ test("updateDockerfile rewrites one image at several tags without cascading", () expect(updateDockerfile(content, deps)).toBe("FROM node:20 AS build\nFROM node:20-alpine\nFROM node:22\n"); }); -// getExtractionRegex test.each([ ["Dockerfile", dockerfileFromRe], ["Dockerfile.dev", dockerfileFromRe], @@ -290,8 +290,7 @@ test.each([ const hubCtx = (doFetch: (url: string) => Promise, extra: Record = {}): ModeContext => ({dockerApiUrl: "https://hub.docker.com", fetchTimeout, doFetch, ...extra} as unknown as ModeContext); -const hubBody = (body: any) => () => Promise.resolve({ok: true, json: () => Promise.resolve(body)}); -// Unrouted pages are the end of the listing, which is how the walk terminates. +const hubBody = (body: any) => () => Promise.resolve(Response.json(body)); const hubPages = (pages: Record, seen: Array = []) => (url: string) => { const page = /page=\d+/.exec(url)![0]; seen.push(page); @@ -309,26 +308,36 @@ test.each([ test("fetchDockerHubTags walks every page", async () => { const ctx = hubCtx(hubPages({ - "page=1": {count: 250, results: [{name: "18", tag_last_pushed: "2024-01-01"}]}, - "page=2": {count: 250, results: [{name: "20", tag_last_pushed: "2024-06-01"}]}, - "page=3": {count: 250, results: [{name: "22", tag_last_pushed: "2025-01-01"}]}, + "page=1": {count: 1, next: "?page=2", results: [{name: "18", tag_last_pushed: "2024-01-01"}]}, + "page=2": {count: 1, next: "?page=3", results: [{name: "20", tag_last_pushed: "2024-06-01"}]}, + "page=3": {count: 2500, results: [{name: "22", tag_last_pushed: "2025-01-01"}]}, })); expect(await fetchDockerHubTags("library", "node", ctx)).toEqual({"18": "2024-01-01", "20": "2024-06-01", "22": "2025-01-01"}); }); test("fetchDockerHubTags walks past pages older than the authored tag", async () => { - // `ordering=last_updated` is a push order, so a backport leaves `20` pages behind the authored `18` const fetched: Array = []; const ctx = hubCtx(hubPages({ - "page=1": {count: 400, results: [{name: "18", tag_last_pushed: "2026-01-01"}]}, - "page=2": {count: 400, results: [{name: "17", tag_last_pushed: "2025-06-01"}]}, - "page=3": {count: 400, results: [{name: "16", tag_last_pushed: "2025-01-01"}]}, - "page=4": {count: 400, results: [{name: "20", tag_last_pushed: "2024-06-01"}]}, + "page=1": {count: 4000, results: [{name: "18", tag_last_pushed: "2026-01-01"}]}, + "page=2": {count: 4000, results: [{name: "17", tag_last_pushed: "2025-06-01"}]}, + "page=3": {count: 4000, results: [{name: "16", tag_last_pushed: "2025-01-01"}]}, + "page=4": {count: 4000, results: [{name: "20", tag_last_pushed: "2024-06-01"}]}, }, fetched), {concurrency: 1}); const tags = await fetchDockerHubTags("library", "node", ctx); expect(fetched).toEqual(["page=1", "page=2", "page=3", "page=4"]); - expect(findDockerVersion(tags, "18", new Set(["patch", "minor", "major"]))).toEqual({newTag: "20", date: "2024-06-01"}); + expect(findDockerVersion(tags, "18", allSemvers)).toEqual({newTag: "20", date: "2024-06-01"}); +}); + +test("fetchDockerHubTags caps count and next pagination at 20 pages", async () => { + const fetched: Array = []; + const ctx = hubCtx((url: string) => { + const page = Number(new URL(url).searchParams.get("page")); + fetched.push(page); + return hubBody({count: 1000000, next: `?page=${page + 1}`, results: [{name: String(page)}]})(); + }, {noCache: true}); + await fetchDockerHubTags("library", "bounded", ctx); + expect(fetched).toEqual(Array.from({length: 20}, (_, index) => index + 1)); }); test("fetchDockerHubTags reports registry failures instead of no update", async () => { @@ -343,6 +352,16 @@ test("fetchDockerHubTags reports registry failures instead of no update", async .rejects.toThrow("ECONNREFUSED"); }); +test("fetchDockerTagDigest returns the registry digest and reports failures", async () => { + await expect(fetchDockerTagDigest("library", "node", "20", hubCtx(hubBody({digest: newDigest})))) + .resolves.toBe(newDigest); + await expect(fetchDockerTagDigest("library", "node", "20", hubCtx(() => Promise.resolve({ + ok: false, status: 429, statusText: "Too Many Requests", + })))).rejects.toThrow("Received 429 Too Many Requests"); + await expect(fetchDockerTagDigest("library", "node", "20", hubCtx(hubBody({})))) + .rejects.toThrow("Malformed Docker Hub tag response"); +}); + test("fetchDockerInfo library image", async () => { const ctx = hubCtx(hubBody({count: 1, results: [{name: "18", tag_last_pushed: "2024-01-01"}]})); const [data] = await fetchDockerInfo("node", ctx); @@ -350,11 +369,6 @@ test("fetchDockerInfo library image", async () => { expect(data.tags).toEqual({"18": "2024-01-01"}); }); -test("fetchDockerInfo namespaced image", async () => { - const [data] = await fetchDockerInfo("myorg/myapp", hubCtx(hubBody({count: 0, results: []}))); - expect(data.name).toBe("myorg/myapp"); -}); - test("filterStableTags drops the ubuntu development series", () => { const tags: Record = { "22.04": "2026-08-04", "24.04": "2026-08-04", "26.04": "2026-08-04", @@ -362,7 +376,6 @@ test("filterStableTags drops the ubuntu development series", () => { latest: "2026-08-04", devel: "2026-07-16", }; const now = Date.UTC(2026, 7, 4); - // only released even-year LTS numbers survive, 26.10 is devel and 28.04 has not shipped yet expect(Object.keys(filterStableTags("ubuntu", tags, now))).toEqual(["22.04", "24.04", "26.04", "latest", "devel"]); expect(filterStableTags("node", tags, now)).toBe(tags); }); diff --git a/modes/docker.ts b/modes/docker.ts index 6efead7..7792891 100644 --- a/modes/docker.ts +++ b/modes/docker.ts @@ -1,6 +1,9 @@ -import {coerce, diff, gt, parse, satisfies} from "../utils/semver.ts"; +import {parse, satisfies, semverVersioning} from "../utils/semver.ts"; import {longestFirstAlternation} from "../utils/utils.ts"; -import {type Deps, type ModeContext, type PackageInfo, dedupe, fieldSep, fetchWithEtag, effectiveConcurrency, getLimiter, isSameVersionScheme, passesCooldown, prereleaseOpts, reduceJson, stripv, throwFetchError, formatVersionPrecision, maxTagPages} from "./shared.ts"; +import { + type Deps, type ModeContext, type PackageInfo, dedupe, fieldSep, fetchWithEtag, isSameVersionScheme, + passesCooldown, prereleaseOpts, reduceJson, stripv, throwFetchError, formatVersionPrecision, +} from "./shared.ts"; export type DockerImageRef = { registry: string | null, @@ -8,40 +11,62 @@ export type DockerImageRef = { repo: string, tag: string, fullImage: string, + digest?: string, + digestOnly?: boolean, }; +type DockerTag = {version: string, prerelease: string, suffix: string}; -// Match semver or semver-prefix tags, with an optional prerelease glued to the version and an -// optional suffix like -alpine. A hyphen starts the suffix, so `1.27-rc` is the `-rc` variant while -// `1.27rc3` is a prerelease of 1.27, the same split renovate's docker versioning makes. -// Examples: "18", "18.19", "v1.2.3", "18-alpine", "1.27rc3", "1.27rc3-alpine" -const dockerTagRe = /^(v?\d+(?:\.\d+){0,2})([a-z][a-z0-9]*)?(-.+)?$/i; +const dockerTagRe = /^(v?\d+(?:\.\d+)*)([a-z][a-z0-9]*)?(-.+)?$/i; -// Extraction regexes -// Dockerfile instructions are case-insensitive -export const dockerfileFromRe = /^\s*FROM\s+(?:--platform=\S+\s+)?(\S+)/gim; -export const composeImageRe = /^\s*image:\s*['"]?([^\s'"#]+)['"]?/gm; -// Matches shorthand `container: image:tag` (not object form with `{`) -export const workflowContainerRe = /^\s*container:\s*['"]?([^\s'"#{}]+:[^\s'"#{}:]+)['"]?\s*$/gm; -// Matches `uses: docker://image:tag` -export const workflowDockerUsesRe = /^\s*(?:-\s*)?uses:\s*['"]?docker:\/\/([^'"#\s]+)['"]?/gm; +export const dockerfileFromRe = /^[ \t]*FROM\b[^\r\n]*(?:(?<=\\)[ \t]*\r?\n[^\r\n]*)*/gim; +export const composeImageRe = /^[ \t]*image:\s*['"]?([^\s'"#]+)['"]?/gm; +const dockerArgRe = /^[ \t]*ARG\s+(\w+)(?:[ =](\S*))?/i; +const dockerFromInstructionRe = /^[ \t]*FROM\s+(?:--platform=\S+\s+)?(\S+)/i; +const unfoldDockerInstruction = (instruction: string) => instruction.replace(/\\[ \t]*\r?\n[ \t]*/g, " "); -// docker.io and index.docker.io are Docker Hub itself, not a third-party registry. -const hubRegistryRe = /^(?:index\.)?docker\.io$/; +function resolveDockerVariables(value: string, getValue: (name: string) => string | undefined): string { + return value.replace(/\$\{(\w+)\}|\$(\w+)/g, (variable, braced, bare) => { + const resolved = getValue(braced || bare); + return resolved === undefined ? variable : resolved; + }); +} + +type DockerArg = {value: string, resolved: string, start: number}; + +function *dockerfileFromInstructions(content: string, recursive = false): Generator<{ + instruction: RegExpMatchArray, args: Map, from: RegExpExecArray, resolved: string, +}> { + const args = new Map(); + let sawFrom = false; + for (const instruction of content.matchAll(/^[ \t]*(?:ARG|FROM)\b[^\r\n]*(?:(?<=\\)[ \t]*\r?\n[^\r\n]*)*/gim)) { + const unfolded = unfoldDockerInstruction(instruction[0]); + const arg = dockerArgRe.exec(unfolded); + if (arg) { + if (!sawFrom) { + const value = arg[2]?.replace(/^(['"])(.*)\1$/, "$2") ?? ""; + const relativeStart = instruction[0].lastIndexOf(value); + args.set(arg[1], {value, resolved: resolveDockerVariables(value, name => args.get(name)?.resolved), + start: relativeStart < 0 ? -1 : instruction.index + relativeStart}); + } + continue; + } + sawFrom = true; + const from = dockerFromInstructionRe.exec(unfolded); + if (from) yield {instruction, args, from, + resolved: resolveDockerVariables(from[1], name => args.get(name)?.[recursive ? "resolved" : "value"])}; + } +} + +const hubRegistryRe = /^(?:(?:index|registry-1)\.)?docker\.io$/; function parseImageParts(imagePart: string): {registry: string | null, namespace: string, repo: string} { const parts = imagePart.split("/"); if (parts.length > 1 && hubRegistryRe.test(parts[0])) parts.shift(); - if (parts.length === 1) { - return {registry: null, namespace: "library", repo: parts[0]}; - } else if (parts.length === 2 && !parts[0].includes(".") && !parts[0].includes(":")) { - return {registry: null, namespace: parts[0], repo: parts[1]}; - } else { - return {registry: parts[0], namespace: parts.slice(1, -1).join("/"), repo: parts[parts.length - 1]}; - } + const registry = parts.length > 1 && (parts[0] === "localhost" || parts[0].includes(".") || parts[0].includes(":")) ? + parts.shift()! : null; + return {registry, namespace: parts.length === 1 ? "library" : parts.slice(0, -1).join("/"), repo: parts.at(-1)!}; } -// Hub images are addressable with or without the `docker.io/` registry and `library/` -// namespace, so a user-supplied name in any of those spellings matches the image. export function dockerImageNames(image: string): Array { const {registry, namespace, repo} = parseImageParts(image); if (registry) return [image]; @@ -52,23 +77,23 @@ export function dockerImageNames(image: string): Array { export function parseDockerImageRef(ref: string): DockerImageRef | null { ref = ref.replace(/^docker:\/\//, ""); - if (ref.includes("@")) return null; // digest-pinned, skip + const [taggedRef, digest, ...extra] = ref.split("@"); + if (extra.length || digest && !/^[a-z][a-z0-9+._-]*:[0-9a-f]+$/i.test(digest)) return null; - const colonIndex = ref.lastIndexOf(":"); - if (colonIndex === -1 || ref.lastIndexOf("/") > colonIndex) { - return null; // no tag specified, skip - } + const colonIndex = taggedRef.lastIndexOf(":"); + const hasTag = colonIndex !== -1 && taggedRef.lastIndexOf("/") < colonIndex; + if (!hasTag && !digest) return null; - const imagePart = ref.substring(0, colonIndex); - const tag = ref.substring(colonIndex + 1); + const imagePart = hasTag ? taggedRef.substring(0, colonIndex) : taggedRef; + const tag = hasTag ? taggedRef.substring(colonIndex + 1) : "latest"; - if (!tag || !dockerTagRe.test(tag)) return null; // non-semver tag + if (hasTag && !digest && (!tag || !dockerTagRe.test(tag))) return null; const {registry, namespace, repo} = parseImageParts(imagePart); - return {registry, namespace, repo, tag, fullImage: imagePart}; + return {registry, namespace, repo, tag, fullImage: imagePart, ...(digest && {digest}), ...(!hasTag && {digestOnly: true})}; } -export function parseDockerTag(tag: string): {version: string, prerelease: string, suffix: string} | null { +export function parseDockerTag(tag: string): DockerTag | null { const match = dockerTagRe.exec(tag); if (!match) return null; return {version: match[1], prerelease: match[2] || "", suffix: match[3] || ""}; @@ -82,89 +107,106 @@ export function formatDockerVersion(newSemver: string, oldTag: string, prereleas export function extractDockerRefs(content: string, regex: RegExp): Array<{ref: DockerImageRef, match: string}> { const results: Array<{ref: DockerImageRef, match: string}> = []; + if (regex === dockerfileFromRe) { + for (const {from, resolved} of dockerfileFromInstructions(content, true)) { + const ref = parseDockerImageRef(resolved); + if (ref) results.push({ref, match: from[1]}); + } + return results; + } + const locallyBuilt = regex === composeImageRe ? locallyBuiltImages(content) : null; for (const m of content.matchAll(regex)) { + if (locallyBuilt?.has(m.index + m[0].indexOf("image:"))) continue; const ref = parseDockerImageRef(m[1]); if (ref) results.push({ref, match: m[1]}); } return results; } -// A Dockerfile and a Makefile can reference the same image from independent fetch tasks, which -// would double the requests and race the cache writes. Keyed by ctx so each run starts fresh. -const hubTagsByCtx = new WeakMap>>>(); - -export function fetchDockerHubTags(namespace: string, repo: string, ctx: ModeContext): Promise> { - return dedupe(hubTagsByCtx, ctx, `${namespace}/${repo}`, () => - fetchDockerHubTagsUncached(namespace, repo, ctx)); +function locallyBuiltImages(content: string): Set { + const result = new Set(); + const scopes = new Map}>(); + for (const line of content.matchAll(/^.*$/gm)) { + if (!line[0].trim()) continue; + const indent = /^[ \t]*/.exec(line[0])![0].length; + for (const level of scopes.keys()) { + if (level > indent) scopes.delete(level); + } + const scope = scopes.get(indent) ?? {built: false, images: []}; + scopes.set(indent, scope); + if (/^[ \t]*build\s*:/.test(line[0])) { + scope.built = true; + for (const offset of scope.images) result.add(offset); + } else if (/^[ \t]*image\s*:/.test(line[0])) { + const offset = line.index + indent; + if (scope.built) result.add(offset); + else scope.images.push(offset); + } + } + return result; } -// "Nothing to offer" rather than "the registry is unwell": an unknown repo, and the 401/403 an -// anonymous read of a private one gets, which renovate also swallows. +const hubTagsByCtx = new WeakMap>>>(); const noTagsStatus = new Set([401, 403, 404]); +const maxDockerTagPages = 20; -const tagDate = (result: Record): string => result.tag_last_pushed || result.last_updated || ""; - -async function fetchDockerHubTagsUncached(namespace: string, repo: string, ctx: ModeContext): Promise> { - const tags: Record = {}; - const baseUrl = `${ctx.dockerApiUrl}/v2/repositories/${namespace}/${repo}/tags`; - const pageUrl = (page: number) => `${baseUrl}?page_size=100&ordering=last_updated&page=${page}`; - const pageOpts = {headers: {"accept-encoding": "gzip, deflate, br"}}; - - // Hub tag pages carry per-architecture image lists; only name and push date are read. - const reduceTagsPage = (data: Record) => ({ - count: data.count, - results: (data.results || []).map((r: Record) => ({ - name: r.name, tag_last_pushed: r.tag_last_pushed, last_updated: r.last_updated, - })), +export function fetchDockerHubTags(namespace: string, repo: string, ctx: ModeContext): Promise> { + return dedupe(hubTagsByCtx, ctx, `${namespace}/${repo}`, async () => { + const tags: Record = {}; + const baseUrl = `${ctx.dockerApiUrl}/v2/repositories/${namespace}/${repo}/tags`; + const pageUrl = (page: number) => `${baseUrl}?page_size=1000&ordering=last_updated&page=${page}`; + const fetchPage = async (url: string): Promise => { + const result = await fetchWithEtag(url, ctx, {headers: {"accept-encoding": "gzip, deflate, br"}}, reduceJson(data => ({ + count: data.count, + next: data.next, + results: (data.results || []).map((tag: Record) => ({ + name: tag.name, tag_last_pushed: tag.tag_last_pushed, last_updated: tag.last_updated, + })), + }))); + if ("body" in result) { + const page = JSON.parse(result.body); + for (const tag of page?.results ?? []) tags[tag.name] = tag.tag_last_pushed || tag.last_updated || ""; + return page; + } + if (!noTagsStatus.has(result.res?.status as number)) throwFetchError(result.res, url, `${namespace}/${repo}`, ctx.dockerApiUrl); + return null; + }; + + const firstPage = await fetchPage(pageUrl(1)); + if (!firstPage) return tags; + const seen = new Set(); + let page = firstPage; + for (let pageNumber = 2; pageNumber <= maxDockerTagPages && + (page.next || pageNumber <= Math.ceil((firstPage.count || 0) / 1000)); pageNumber++) { + const nextUrl = page.next ? new URL(page.next, baseUrl).href : pageUrl(pageNumber); + if (new URL(nextUrl).origin !== new URL(baseUrl).origin || seen.has(nextUrl)) break; + seen.add(nextUrl); + const result = await fetchPage(nextUrl); + if (!result) break; + page = result; + } + return tags; }); - - const fetchPage = async (page: number): Promise => { - const url = pageUrl(page); - const result = await fetchWithEtag(url, ctx, pageOpts, reduceJson(reduceTagsPage)); - if ("body" in result) return JSON.parse(result.body); - // Everything else is a host problem, and a rate-limited or broken registry read as up to date - // hides the updates the run exists to find. Renovate raises ExternalHostError for those. - if (!noTagsStatus.has(result.res?.status as number)) throwFetchError(result.res, url, `${namespace}/${repo}`, ctx.dockerApiUrl); - return null; - }; - - const addPage = (page: any) => { - for (const result of page?.results || []) tags[result.name] = tagDate(result); - }; - - const limit = getLimiter(ctx); - const firstPage = await limit(() => fetchPage(1)); - if (!firstPage) return tags; - addPage(firstPage); - // Every page is walked: `ordering=last_updated` is a push order, so a backport or an unevenly - // rebuilt tag puts a higher version behind an older page and no date bounds the walk. Hub reports - // the total up front, so the rest go out a wave at a time, doubling up to the socket budget. - const totalPages = Math.min(Math.ceil((firstPage.count || 0) / 100), maxTagPages); - const maxWave = effectiveConcurrency(ctx); - for (let next = 2, wave = 1; next <= totalPages; next += wave, wave = Math.min(wave * 2, maxWave)) { - const pages = await Promise.all( - Array.from({length: Math.min(wave, totalPages - next + 1)}, (_, idx) => limit(() => fetchPage(next + idx))), - ); - for (const page of pages) addPage(page); - } - return tags; } -// Resolve the manifest digest for a single tag (used to keep `image:tag@sha256:…` pins in sync). -export async function fetchDockerTagDigest(namespace: string, repo: string, tag: string, ctx: ModeContext): Promise { +export async function fetchDockerTagDigest( + namespace: string, + repo: string, + tag: string, + ctx: ModeContext, +): Promise { const url = `${ctx.dockerApiUrl}/v2/repositories/${namespace}/${repo}/tags/${tag}`; - try { - const result = await fetchWithEtag(url, ctx, {headers: {"accept-encoding": "gzip, deflate, br"}}, reduceJson(data => ({digest: data.digest}))); - if (!("body" in result)) return null; - const digest = JSON.parse(result.body).digest; - return typeof digest === "string" ? digest : null; - } catch { return null; } + const result = await fetchWithEtag(url, ctx, {headers: {"accept-encoding": "gzip, deflate, br"}}, + reduceJson(data => ({digest: data.digest}))); + if ("body" in result) { + const data = JSON.parse(result.body); + if (typeof data?.digest !== "string") throw new Error(`Malformed Docker Hub tag response: ${namespace}/${repo}:${tag}`); + return data.digest; + } + if (!noTagsStatus.has(result.res?.status as number)) throwFetchError(result.res, url, `${namespace}/${repo}:${tag}`, ctx.dockerApiUrl); + return null; } -// Ubuntu numbers a release after the year and month it ships in, so only an even-year `.04` is ever -// an LTS, and Hub publishes the development series under its future number months before it ships. -// Renovate reads both facts out of bundled distro-info data. The number alone dates the release, at -// the start of the following month because a release lands in the second half of its own. const ubuntuLtsRe = /^\d?[02468]\.04$/; function isStableUbuntuVersion(version: string, now: number): boolean { @@ -173,9 +215,6 @@ function isStableUbuntuVersion(version: string, now: number): boolean { return now >= Date.UTC(2000 + Number(year), Number(month)); } -// Images renovate gives a distro versioning to in its dockerfile manager. Keyed by repo so any -// namespace matches, as renovate's `depName === 'ubuntu' || depName.endsWith('/ubuntu')` does. -// Debian needs no entry: Hub only ever numbers a released Debian. const imageStability: Record boolean> = { ubuntu: isStableUbuntuVersion, }; @@ -200,10 +239,33 @@ export async function fetchDockerInfo(name: string, ctx: ModeContext): Promise

prerelease ? `${coerced}-${prerelease}` : coerced; +function coerceDockerVersion(version: string): string | null { + const parts = stripv(version).split(".").slice(0, 3); + if (!parts.length || parts.some(part => !/^\d+$/.test(part))) return null; + return [...parts.map(part => String(Number(part))), ...new Array(3 - parts.length).fill("0")].join("."); +} + +function compareExtendedDockerTags(left: DockerTag, right: DockerTag): number { + const leftParts = stripv(left.version).split(".").map(Number); + const rightParts = stripv(right.version).split(".").map(Number); + for (let index = 0; index < leftParts.length; index++) { + if (leftParts[index] !== rightParts[index]) return leftParts[index] - rightParts[index]; + } + if (!left.prerelease && right.prerelease) return 1; + if (left.prerelease && !right.prerelease) return -1; + return left.prerelease.localeCompare(right.prerelease); +} + +function extendedDockerLevel(left: DockerTag, right: DockerTag): string | null { + const leftParts = stripv(left.version).split(".").map(Number); + const rightParts = stripv(right.version).split(".").map(Number); + const changed = leftParts.findIndex((part, index) => part !== rightParts[index]); + if (changed === -1) return left.prerelease === right.prerelease ? null : "patch"; + return changed === 0 ? "major" : changed === 1 ? "minor" : "patch"; +} + export function findDockerVersion( tagMap: Record, oldTag: string, @@ -217,39 +279,42 @@ export function findDockerVersion( const oldParsed = parseDockerTag(oldTag); if (!oldParsed) return null; - const oldCoerced = coerce(stripv(oldParsed.version))?.version; + const oldCoerced = coerceDockerVersion(oldParsed.version); if (!oldCoerced) return null; const oldFields = stripv(oldParsed.version).split(".").length; - // Same prerelease policy as every other mode: only --prerelease, or a tag that already names one, - // puts prereleases in play, and --release takes them back out. const oldSemver = dockerSemver(oldCoerced, oldParsed.prerelease); const {effectiveSemvers, skipsPrerelease} = prereleaseOpts(oldSemver, usePre, useRel, semvers); - - let bestVersion = oldSemver; + const extended = oldFields > 3; + let bestVersion = parse(oldSemver)!; + let bestParsed = oldParsed; let bestTag = ""; let bestDate = ""; for (const [tagName, lastUpdated] of Object.entries(tagMap)) { const parsed = parseDockerTag(tagName); - if (!parsed || parsed.suffix !== oldParsed.suffix) continue; - // Only tags of the authored precision are candidates, as renovate's docker isCompatible - // requires an equal release length: a floating `1.2` must not become a pinned `1.3.6`. - if (stripv(parsed.version).split(".").length !== oldFields) continue; - if (!isSameVersionScheme(parsed.version, oldParsed.version)) continue; + if (!parsed || parsed.suffix !== oldParsed.suffix || stripv(parsed.version).split(".").length !== oldFields || + !isSameVersionScheme(parsed.version, oldParsed.version)) continue; + if (!passesCooldown(lastUpdated, cooldownDays, now)) continue; - const coerced = coerce(stripv(parsed.version))?.version; - if (!coerced) continue; - // The tag's own text already says whether it is a prerelease, so only those pay for a parse. - if (parsed.prerelease && skipsPrerelease(parse(dockerSemver(coerced, parsed.prerelease)))) continue; + if (extended) { + if (parsed.prerelease && (!usePre && !oldParsed.prerelease || useRel)) continue; + if (compareExtendedDockerTags(parsed, bestParsed) <= 0) continue; + const level = extendedDockerLevel(oldParsed, parsed); + if (!level || !semvers.has(level)) continue; + bestParsed = parsed; + bestTag = tagName; + bestDate = lastUpdated; + continue; + } + const coerced = coerceDockerVersion(parsed.version); + if (!coerced) continue; + const candidate = parse(dockerSemver(coerced, parsed.prerelease))!; + if (parsed.prerelease && skipsPrerelease(candidate)) continue; if (pinnedRange && !satisfies(coerced, pinnedRange)) continue; - if (!passesCooldown(lastUpdated, cooldownDays, now)) continue; - - const candidate = dockerSemver(coerced, parsed.prerelease); - if (candidate === bestVersion) { - // duplicate tags coerce to the same version — keep the most recently pushed one + if (candidate.version === bestVersion.version) { if (bestTag && Date.parse(lastUpdated) > Date.parse(bestDate)) { bestTag = tagName; bestDate = lastUpdated; @@ -257,73 +322,109 @@ export function findDockerVersion( continue; } - const d = diff(bestVersion, candidate); + const d = semverVersioning.diff(bestVersion, candidate); if (!d || !effectiveSemvers.has(d)) continue; - if (gt(candidate, bestVersion)) { + if (semverVersioning.compare(candidate, bestVersion) > 0) { bestVersion = candidate; bestTag = tagName; bestDate = lastUpdated; } } - if (!bestTag || bestVersion === oldSemver) return null; - // The formatted tag is synthesized from a coerced version, so keep the real Hub tag when the - // registry does not publish that spelling, as `26.04` coerces and formats back to `26.4`. - // Neither half of a dockerSemver holds a `-`, so the winner splits back apart without parsing. - const [bestRelease, bestPre = ""] = bestVersion.split("-"); + if (extended) return bestTag ? {newTag: bestTag, date: bestDate} : null; + if (!bestTag || bestVersion.version === oldSemver) return null; + const [bestRelease, bestPre = ""] = bestVersion.version.split("-"); const formatted = formatDockerVersion(bestRelease, oldTag, bestPre); const newTag = formatted in tagMap ? formatted : bestTag; if (newTag === oldTag) return null; return {newTag, date: bestDate}; } -// Ends a tag match. Excludes `@` and `+` on top of tag characters so a digest-pinned or -// build-suffixed occurrence, which the extractor skips, is never rewritten to a bare tag -// the digest then contradicts. const tagEnd = "(?![\\w.@+-])"; -// One pass per pattern over an alternation of every authored `image:tag`, longest first so -// `node:18-alpine` wins over `node:18`. Keys carry the authored case, as a tag is case-sensitive -// and only the Dockerfile instruction keyword needs a case-insensitive match. -function replaceImageRefs(content: string, deps: Deps, patterns: Array<(refs: string) => RegExp>): string { +function imageReplacements(deps: Deps): Map { const byRef = new Map(); for (const [key, dep] of Object.entries(deps)) { const name = key.split(fieldSep)[1]; - byRef.set(`${name}:${dep.oldOrig || dep.old}`, `${name}:${dep.new}`); + if (!dep.oldDigest) byRef.set(`${name}:${dep.oldOrig || dep.old}`, `${name}:${dep.new}`); + else if (dep.newDigest) byRef.set(dep.digestOnly ? `${name}@${dep.oldDigest}` : + `${name}:${dep.oldOrig || dep.old}@${dep.oldDigest}`, dep.digestOnly ? `${name}@${dep.newDigest}` : + `${name}:${dep.new}@${dep.newDigest}`); } + return byRef; +} + +function replaceImageRefs( + content: string, + byRef: Map, + prefixes: Array, + canReplace: (offset: number) => boolean = () => true, +): string { if (!byRef.size) return content; const refs = longestFirstAlternation(byRef.keys()); let newContent = content; - for (const makeRegex of patterns) { - // A ref that case-insensitively matched some other dep's spelling is left alone. - newContent = newContent.replace(makeRegex(refs), (_, prefix, ref) => `${prefix}${byRef.get(ref) ?? ref}`); + for (const prefix of prefixes) { + newContent = newContent.replace(new RegExp(`(${prefix})(${refs})${tagEnd}`, "g"), (match, start, ref, offset) => + canReplace(offset) ? `${start}${byRef.get(ref) ?? ref}` : match); } return newContent; } export function updateDockerfile(content: string, deps: Deps): string { - return replaceImageRefs(content, deps, [ - refs => new RegExp(`(FROM\\s+(?:--platform=\\S+\\s+)?)(${refs})${tagEnd}`, "gi"), - ]); + const separator = "(?:[ \\t]+|\\\\[ \\t]*\\r?\\n[ \\t]*)"; + const replacements = imageReplacements(deps); + const refs = longestFirstAlternation(replacements.keys()); + const updated = replacements.size ? content.replace( + new RegExp(`(FROM${separator}+(?:--platform=\\S+${separator}+)?)(${refs})${tagEnd}`, "gi"), + (_match, prefix, ref) => `${prefix}${replacements.get(ref) ?? ref}`, + ) : content; + const edits = new Map(); + for (const {instruction, args, from, resolved} of dockerfileFromInstructions(updated)) { + const replacement = replacements.get(resolved); + if (!replacement) continue; + const oldDigest = resolved.slice(resolved.lastIndexOf("@") + 1); + const newDigest = replacement.slice(replacement.lastIndexOf("@") + 1); + const replacesDigest = resolved.includes("@") && replacement.includes("@") && oldDigest !== newDigest; + if (replacesDigest) { + const relativeDigest = instruction[0].lastIndexOf(oldDigest); + if (relativeDigest !== -1) edits.set(instruction.index! + relativeDigest, { + end: instruction.index! + relativeDigest + oldDigest.length, value: newDigest, + }); + } + for (const variable of from[1].matchAll(/\$(?:\{(\w+)\}|(\w+))/g)) { + const argValue = args.get(variable[1] || variable[2]); + const prefix = resolveDockerVariables(from[1].slice(0, variable.index), name => args.get(name)?.value); + let suffix = resolveDockerVariables(from[1].slice(variable.index + variable[0].length), name => args.get(name)?.value); + if (replacesDigest) suffix = suffix.replace(oldDigest, newDigest); + if (!argValue || argValue.start < 0 || !replacement.startsWith(prefix) || !replacement.endsWith(suffix)) continue; + edits.set(argValue.start, { + end: argValue.start + argValue.value.length, + value: replacement.slice(prefix.length, suffix ? -suffix.length : undefined), + }); + } + } + let result = updated; + for (const [start, edit] of [...edits].sort(([left], [right]) => right - left)) { + result = `${result.slice(0, start)}${edit.value}${result.slice(edit.end)}`; + } + return result; } export function updateComposeFile(content: string, deps: Deps): string { - return replaceImageRefs(content, deps, [ - refs => new RegExp(`(image:\\s*['"]?)(${refs})${tagEnd}`, "g"), - ]); + const locallyBuilt = locallyBuiltImages(content); + return replaceImageRefs(content, imageReplacements(deps), [String.raw`image:\s*['"]?`], + offset => !locallyBuilt.has(offset)); } export function updateWorkflowDockerImages(content: string, deps: Deps): string { - return replaceImageRefs(content, deps, [ - refs => new RegExp(`((?:container|image):\\s*['"]?)(${refs})${tagEnd}`, "g"), - refs => new RegExp(`(uses:\\s*['"]?docker://)(${refs})${tagEnd}`, "g"), + return replaceImageRefs(content, imageReplacements(deps), [ + String.raw`(?:container|image):\s*['"]?`, + String.raw`uses:\s*['"]?docker://`, ]); } -// Exact filenames for auto-discovery via findUpSync, which cannot glob. Deliberately -// narrower than isDockerFileName, which every entry must still satisfy. export const dockerExactFileNames = [ "Dockerfile", "compose.yml", @@ -333,7 +434,6 @@ export const dockerExactFileNames = [ ]; export function isComposeFile(filename: string): boolean { - // `compose` is the canonical Compose Spec name; `docker-` also covers swarm stack files return /^(?:docker-|compose).*\.ya?ml$/.test(filename); } diff --git a/modes/go.test.ts b/modes/go.test.ts index a010697..abf2704 100644 --- a/modes/go.test.ts +++ b/modes/go.test.ts @@ -1,4 +1,6 @@ import {resolve} from "node:path"; +import {mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; import { type GoProxyEntry, parseGoProxy, @@ -12,12 +14,13 @@ import { goModulePathForVersion, parseGoMod, parseGoWork, + resolveGoWorkModule, shortenGoModule, shortenGoVersion, getGoInfoUrl, updateGoMod, fetchGoProxyInfo, - probeMajorVersions, + rewriteGoImportPaths, rewriteGoImports, } from "./go.ts"; import {type ModeContext, fieldSep, isGoPseudoVersion} from "./shared.ts"; @@ -53,7 +56,7 @@ test("resolveGoProxyChain", async () => { expect(resolveGoProxyChain()).toEqual([{url: "https://a", fallback: ","}, {url: "https://b", fallback: ","}]); expect(resolveGoProxyChain("http://127.0.0.1:1/")).toEqual([{url: "http://127.0.0.1:1", fallback: ","}]); }); - await withGoProxyEnv(",,", () => expect(resolveGoProxyChain()).toEqual([{url: "https://proxy.golang.org", fallback: ","}])); + await withGoProxyEnv(",,", () => expect(() => resolveGoProxyChain()).toThrow(/contains no entries/)); await withGoProxyEnv(undefined, () => expect(resolveGoProxyChain()[0].url).toBe("https://proxy.golang.org")); await withGoProxyEnv("direct", () => expect(resolveGoProxyChain()[0].url).toBe("direct")); await withGoProxyEnv("off,https://backup.proxy", () => expect(resolveGoProxyChain()[0].url).toBe("off")); @@ -62,7 +65,6 @@ test("resolveGoProxyChain", async () => { test("pickGoListVersion", () => { expect(pickGoListVersion("v1.0.0\nv1.2.0\nv1.1.0\n")).toEqual({Version: "v1.2.0", Time: ""}); expect(pickGoListVersion("v1.0.0 2019-10-16T16:15:28Z\n")).toEqual({Version: "v1.0.0", Time: "2019-10-16T16:15:28Z"}); - // a release outranks any prerelease, pseudo-versions included expect(pickGoListVersion("v1.3.0-rc.1\nv1.2.0\n")).toEqual({Version: "v1.2.0", Time: ""}); expect(pickGoListVersion("v0.0.0-20221128193559-754e69321358\nv0.1.0")).toEqual({Version: "v0.1.0", Time: ""}); expect(pickGoListVersion("v1.3.0-rc.1\nv1.3.0-rc.2\n")).toEqual({Version: "v1.3.0-rc.2", Time: ""}); @@ -95,7 +97,6 @@ test("isGoNoProxy", () => { expect(isGoNoProxy("github.com/private/sub", ["github.com/private"])).toBe(true); expect(isGoNoProxy("github.com/public", ["github.com/private"])).toBe(false); expect(isGoNoProxy("anything", [])).toBe(false); - // go matches these with path.Match, so globs stay inside one path element expect(isGoNoProxy("github.com/mycorp/secret", ["github.com/mycorp/*"])).toBe(true); expect(isGoNoProxy("github.com/mycorp/secret/sub", ["github.com/mycorp/*"])).toBe(true); expect(isGoNoProxy("git.corp.example.com/a/b", ["*.corp.example.com"])).toBe(true); @@ -108,32 +109,25 @@ test("encodeGoModulePath", () => { expect(encodeGoModulePath("github.com/Azure/azure-sdk")).toBe("github.com/!azure/azure-sdk"); }); -test("extractGoMajor", () => { +test("Go module path transforms", () => { expect(extractGoMajor("github.com/foo/bar")).toBe(1); expect(extractGoMajor("github.com/foo/bar/v2")).toBe(2); expect(extractGoMajor("github.com/foo/bar/v15")).toBe(15); expect(extractGoMajor("gopkg.in/yaml.v2")).toBe(2); -}); - -test("buildGoModulePath", () => { expect(buildGoModulePath("github.com/foo/bar/v2", 3)).toBe("github.com/foo/bar/v3"); expect(buildGoModulePath("github.com/foo/bar/v2", 1)).toBe("github.com/foo/bar"); expect(buildGoModulePath("github.com/foo/bar", 2)).toBe("github.com/foo/bar/v2"); expect(buildGoModulePath("github.com/foo/bar", 1)).toBe("github.com/foo/bar"); - // gopkg.in encodes the major on the last element and has no unsuffixed form expect(buildGoModulePath("gopkg.in/yaml.v2", 3)).toBe("gopkg.in/yaml.v3"); expect(buildGoModulePath("gopkg.in/yaml.v2", 1)).toBe("gopkg.in/yaml.v1"); -}); - -test("goModulePathForVersion", () => { expect(goModulePathForVersion("github.com/foo/bar/v2", "3.0.0")).toBe("github.com/foo/bar/v3"); expect(goModulePathForVersion("github.com/foo/bar", "2.1.0")).toBe("github.com/foo/bar/v2"); expect(goModulePathForVersion("github.com/foo/bar/v2", "2.5.0")).toBe("github.com/foo/bar/v2"); expect(goModulePathForVersion("github.com/foo/bar", "1.4.0")).toBe("github.com/foo/bar"); expect(goModulePathForVersion("github.com/foo/bar", "3.0.0+incompatible")).toBe("github.com/foo/bar"); - expect(goModulePathForVersion("github.com/foo/bar/v2", "garbage")).toBe("github.com/foo/bar/v2"); // non-numeric major → unchanged + expect(goModulePathForVersion("github.com/foo/bar/v2", "garbage")).toBe("github.com/foo/bar/v2"); expect(goModulePathForVersion("gopkg.in/yaml.v2", "3.0.1")).toBe("gopkg.in/yaml.v3"); - expect(goModulePathForVersion("github.com/foo/bar/v2", "1.5.0")).toBe("github.com/foo/bar"); // major downgrade drops the suffix + expect(goModulePathForVersion("github.com/foo/bar/v2", "1.5.0")).toBe("github.com/foo/bar"); }); test("isGoPseudoVersion", () => { @@ -144,23 +138,21 @@ test("isGoPseudoVersion", () => { test.each([ ["sorts requires, indirects, replaces and tools", - ["module example.com/mymod", "", "go 1.21", "", "require (", "\tgithub.com/foo/bar v1.2.3", + ["module example.com/mymod", "", "require (", "\tgithub.com/foo/bar v1.2.3", "\tgithub.com/baz/qux v0.5.0 // indirect", ")", "", - "replace github.com/old/mod => github.com/new/mod v1.0.0", "", "tool github.com/foo/bar/cmd/tool"], + "replace github.com/old/mod => github.com/new/mod v1.0.0", "", "exclude (", "\tgithub.com/foo/bar v1.3.0", + "\tgithub.com/foo/bar v1.4.0", ")", "", "tool (", "\tgithub.com/foo/bar/cmd/tool // build tool", ")"], {deps: {}, indirect: {"github.com/baz/qux": "v0.5.0"}, replace: {"github.com/new/mod": "v1.0.0"}, - tool: {"github.com/foo/bar": "v1.2.3"}}], + tool: {"github.com/foo/bar": "v1.2.3"}, exclude: {"github.com/foo/bar": ["v1.3.0", "v1.4.0"]}}], ["a single-line require", ["module example.com/mod", "", "require foo v1.0.0"], {deps: {"foo": "v1.0.0"}, indirect: {}, replace: {}, tool: {}}], ["replace block syntax", ["module example.com/mod", "", "require (", "\tgithub.com/orig/mod v1.0.0", ")", "", "replace (", "\tgithub.com/orig/mod => github.com/fork/mod v2.0.0", ")"], {deps: {}, indirect: {}, replace: {"github.com/fork/mod": "v2.0.0"}, tool: {}}], - // the local checkout is what builds, so the require version is inert; leaving it in deps - // meant an update bumped it and stripped the replace, silently un-forking the dependency ["a local replace, which takes its require out of play", ["module example.com/mod", "", "require github.com/foo/bar v1.2.3", "", "replace github.com/foo/bar => ../local/bar"], {deps: {}, indirect: {}, replace: {}, tool: {}}], - // the replace only redirects v1.0.0, so the required v1.2.3 is live ["a version-specific replace, which leaves its require updatable", ["module example.com/mod", "", "require github.com/foo/bar v1.2.3", "", "replace github.com/foo/bar v1.0.0 => github.com/fork/bar v1.0.1"], @@ -176,18 +168,12 @@ test.each([ expect(parseGoMod(lines.join("\n"))).toEqual(expected); }); -test("shortenGoModule", () => { +test("Go display transforms", () => { expect(shortenGoModule("github.com/foo/bar/v2")).toBe("github.com/foo/bar"); expect(shortenGoModule("github.com/foo/bar/v10")).toBe("github.com/foo/bar"); expect(shortenGoModule("github.com/foo/bar")).toBe("github.com/foo/bar"); -}); - -test("shortenGoVersion", () => { expect(shortenGoVersion("v0.0.0-20221128193559-754e69321358")).toBe("v0.0.0-2022112"); expect(shortenGoVersion("v1.2.3")).toBe("v1.2.3"); -}); - -test("getGoInfoUrl", () => { expect(getGoInfoUrl("github.com/foo/bar")).toBe("https://github.com/foo/bar"); expect(getGoInfoUrl("github.com/foo/bar/v2")).toBe("https://github.com/foo/bar"); expect(getGoInfoUrl("github.com/foo/bar/pkg/sub")).toBe("https://github.com/foo/bar/tree/HEAD/pkg/sub"); @@ -204,6 +190,11 @@ test.each([ goMod("require (", "\tgithub.com/foo/bar v1.0.0 // indirect", ")"), {[`indirect${fieldSep}github.com/foo/bar`]: {old: "1.0.0", new: "1.2.0"}}, goMod("require (", "\tgithub.com/foo/bar v1.2.0 // indirect", ")"), {}], + ["an indirect dep major bump", + goMod("require github.com/foo/bar v1.0.0 // indirect"), + {[`indirect${fieldSep}github.com/foo/bar`]: {old: "1.0.0", new: "2.0.0"}}, + goMod("require github.com/foo/bar/v2 v2.0.0 // indirect"), + {"github.com/foo/bar": "github.com/foo/bar/v2"}], ["a replace dep bump", goMod("require (", "\tgithub.com/orig/mod v1.0.0", ")", "", "replace github.com/orig/mod => github.com/new/mod v1.0.0"), {[`replace${fieldSep}github.com/new/mod`]: {old: "1.0.0", new: "1.5.0"}}, @@ -221,12 +212,10 @@ test.each([ goMod(`replace "github.com/old/mod" => "github.com/new/mod" v1.0.0`), {[`replace${fieldSep}github.com/new/mod`]: {old: "1.0.0", new: "1.5.0"}}, goMod(`replace "github.com/old/mod" => "github.com/new/mod" v1.5.0`), {}], - // a replace target's version has to match its path's major or go refuses to parse the file ["both sides of a self-replace across a major", goMod("replace (", "\tgithub.com/grpc-ecosystem/grpc-gateway => github.com/grpc-ecosystem/grpc-gateway v1.16.0", ")", ""), {[`replace${fieldSep}github.com/grpc-ecosystem/grpc-gateway`]: {old: "1.16.0", new: "2.28.0"}}, goMod("replace (", "\tgithub.com/grpc-ecosystem/grpc-gateway/v2 => github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0", ")", ""), {}], - // go applies the replacement only to the path the require names, so a stale require does nothing ["the require a self-replace across a major applies to", goMod("require github.com/grpc-ecosystem/grpc-gateway v1.16.0", "", "replace github.com/grpc-ecosystem/grpc-gateway => github.com/grpc-ecosystem/grpc-gateway v1.16.0"), @@ -243,44 +232,27 @@ test.each([ {[`deps${fieldSep}github.com/foo/bar/v2`]: {old: "2.1.0", new: "3.0.0"}}, goMod("require (", "\tgithub.com/foo/bar/v3 v3.0.0", ")"), {"github.com/foo/bar/v2": "github.com/foo/bar/v3"}], - // without `oldOrig` the shortened `old` partial-matches the full version and corrupts the tail ["a pseudo-version to a release, anchored on oldOrig", goMod("require github.com/foo/bar v0.0.0-20221128193559-754e69321358 // indirect"), {[`indirect${fieldSep}github.com/foo/bar`]: {old: "v0.0.0-2022112", oldOrig: "0.0.0-20221128193559-754e69321358", new: "1.2.3"}}, goMod("require github.com/foo/bar v1.2.3 // indirect"), {}], ["a tool's major, in the require and the tool block alike", - goMod("require (", "\tgithub.com/foo/bar/v2 v2.1.0", ")", "", "tool (", "\tgithub.com/foo/bar/v2/cmd/mytool", ")"), + goMod("require (", "\tgithub.com/foo/bar/v2 v2.1.0", ")", "", "tool (", "\tgithub.com/foo/bar/v2/cmd/mytool // tool", ")"), {[`tool${fieldSep}github.com/foo/bar/v2`]: {old: "2.1.0", new: "3.0.0"}}, - goMod("require (", "\tgithub.com/foo/bar/v3 v3.0.0", ")", "", "tool (", "\tgithub.com/foo/bar/v3/cmd/mytool", ")"), + goMod("require (", "\tgithub.com/foo/bar/v3 v3.0.0", ")", "", "tool (", "\tgithub.com/foo/bar/v3/cmd/mytool // tool", ")"), {"github.com/foo/bar/v2": "github.com/foo/bar/v3"}], + ["only the matching require directive", + goMod("require foo v1.0.0", "exclude foo v1.0.0", "replace other => foo v1.0.0"), + {[`deps${fieldSep}foo`]: {old: "1.0.0", new: "1.1.0"}}, + goMod("require foo v1.1.0", "exclude foo v1.0.0", "replace other => foo v1.0.0"), {}], ])("updateGoMod rewrites %s", (_name, content, deps, expected, expectedRewrites) => { const [result, rewrites] = updateGoMod(content, deps); expect(result).toBe(expected); expect(rewrites).toEqual(expectedRewrites); }); -// probeMajorVersions -const probeOf = (major: number, pre = false) => ({Version: `v${major}.0.0${pre ? "-rc.1" : ""}`, Time: "", path: `mod/v${major}`}); -const makeProbe = (existing: Array, prerelease: Array) => - (major: number) => Promise.resolve(existing.includes(major) ? probeOf(major, prerelease.includes(major)) : null); - -test.each([ - ["returns null when firstProbe is null", null, [99], null], - ["returns firstProbe when no higher major exists", probeOf(2), [], probeOf(2)], - ["finds the highest major", probeOf(2), [2, 3, 4, 5], probeOf(5)], - ["finds the highest major across a large gap", probeOf(2), Array.from({length: 19}, (_, idx) => idx + 2), probeOf(20)], - // v2 exists but v3 does not — exponential search hits v3 first and stops - ["stops at the first gap in the exponential search", probeOf(2), [2, 4], probeOf(2)], - // a prerelease-only top major would hide the released one below it, and stands in only alone - ["skips a prerelease-only highest major", probeOf(2), [2, 3, 4], probeOf(3), [4]], - ["keeps a prerelease-only major when no probed one has a release", probeOf(2, true), [2], probeOf(2, true), [2]], -])("probeMajorVersions %s", async (_name, firstProbe, existing, expected, prerelease = []) => { - expect(await probeMajorVersions(1, firstProbe, makeProbe(existing, prerelease))).toEqual(expected); -}); - const goProxyBase = "https://proxy"; -// A route value is either a response body (200) or a bare status, anything unrouted 404s. function makeGoCtx(routes: Record, seen: Array = [], goProxyChain: Array = [{url: goProxyBase, fallback: ","}]): ModeContext { return { fetchTimeout: 100, @@ -313,17 +285,50 @@ test("fetchGoProxyInfo falls back to @v/list when the proxy omits @latest", asyn [`${goProxyBase}/${modPath}/@v/v1.2.0.info`]: JSON.stringify({Version: "v1.2.0", Time: "2024-01-01T00:00:00Z"}), }, seen)); expect(data).toMatchObject({name: modPath, old: "1.0.0", new: "1.2.0", Time: "2024-01-01T00:00:00Z"}); - // the major probe cannot trust a 404 from an endpoint this proxy does not serve expect(seen).toContain(`${goProxyBase}/${modPath}/v2/@v/list`); }); -test("fetchGoProxyInfo keeps a single request per module when @latest is served", async () => { +test("fetchGoProxyInfo stops major probing at the first absent major", async () => { const seen: Array = []; const [data] = await infoFor(makeGoCtx({ [`${goProxyBase}/${modPath}/@latest`]: JSON.stringify({Version: "v1.2.0", Time: "2024-01-01T00:00:00Z"}), }, seen)); expect(data.new).toBe("1.2.0"); - expect(seen).toEqual([`${goProxyBase}/${modPath}/@latest`, `${goProxyBase}/${modPath}/v2/@latest`]); + expect(seen).toHaveLength(2); + expect(seen).toContain(`${goProxyBase}/${modPath}/@latest`); + expect(seen).toContain(`${goProxyBase}/${modPath}/v2/@latest`); + expect(seen.some(url => url.endsWith("/@v/list"))).toBe(false); +}); + +test.each([ + ["indirect", "1.0.0"], + ["deps", "v0.0.0-20221128193559-754e69321358"], +])("fetchGoProxyInfo probes major versions for %s dependencies", async (type, currentVersion) => { + const seen: Array = []; + const ctx = makeGoCtx({ + [`${goProxyBase}/${modPath}/@latest`]: JSON.stringify({Version: "v1.2.0", Time: ""}), + [`${goProxyBase}/${modPath}/v2/@latest`]: JSON.stringify({Version: "v2.0.0", Time: ""}), + }, seen); + const [data] = await fetchGoProxyInfo(modPath, type, currentVersion, ".", ctx, []); + expect(data).toMatchObject({new: "2.0.0", newPath: `${modPath}/v2`}); +}); + +test("fetchGoProxyInfo rejects an excluded latest version from root and workspace member manifests", async () => { + const projectDir = mkdtempSync(resolve(tmpdir(), "updates-go-")); + try { + mkdirSync(resolve(projectDir, "app")); + for (const [type, memberPath] of [["deps", ""], ["deps|./app", "app"]]) { + writeFileSync(resolve(projectDir, memberPath, "go.mod"), goMod(`exclude ${modPath} v1.3.0`)); + const [data] = await fetchGoProxyInfo(modPath, type, "1.0.0", projectDir, makeGoCtx({ + [`${goProxyBase}/${modPath}/@latest`]: JSON.stringify({Version: "v1.3.0", Time: ""}), + [`${goProxyBase}/${modPath}/@v/list`]: "v1.1.0\nv1.2.0\nv1.3.0\n", + [`${goProxyBase}/${modPath}/@v/v1.2.0.info`]: JSON.stringify({Version: "v1.2.0", Time: ""}), + }), []); + expect(data.new).toBe("1.2.0"); + } + } finally { + rmSync(projectDir, {recursive: true}); + } }); test("fetchGoProxyInfo raises once no proxy in the chain has the module", async () => { @@ -357,20 +362,22 @@ test("fetchGoProxyInfo falls through a `|` list on a proxy failure", async () => expect(data.new).toBe("1.2.0"); }); -// `direct` routes to a VCS lookup, and neither token may reach a proxy. Stubbing execFile keeps a -// real `go` out of it: that one resolves over the network and has to be killed mid-run. test.each([ ["off", ".", /disabled by GOPROXY=off/], ["direct", resolve("."), /go list -m github.com\/foo\/bar@latest failed: no such host/], ])("fetchGoProxyInfo fails without contacting a proxy for GOPROXY=%s", async (value, cwd, message) => { const seen: Array = []; - const execFile = () => Promise.reject(Object.assign(new Error("Command failed"), {stderr: "no such host"})); + let subprocessGoProxy = ""; + const execFile = (_file: string, _args: Array, options: Record) => { + subprocessGoProxy = options.env?.GOPROXY ?? ""; + return Promise.reject(Object.assign(new Error("Command failed"), {stderr: "no such host"})); + }; const ctx = {...makeGoCtx({}, seen, parseGoProxy(value)), execFile}; await expect(infoFor(ctx, cwd)).rejects.toThrow(message); expect(seen).toEqual([]); + if (value === "direct") expect(subprocessGoProxy).toBe("direct"); }); -// parseGoWork test.each([ ["block use", ["go 1.24", "", "use (", "\t./app", "\t./lib", ")"], @@ -387,9 +394,6 @@ test.each([ ["use with inline comment", ["go 1.24", "", "use (", "\t./app // main application", "\t./lib", ")"], {use: ["./app", "./lib"], replace: {}}], - ["with toolchain ignored", - ["go 1.24", "toolchain go1.24.2", "", "use ./app"], - {use: ["./app"], replace: {}}], ["replace block syntax", ["go 1.24", "", "use ./app", "", "replace (", "\tgithub.com/old/a => github.com/new/a v1.0.0", "\tgithub.com/old/b v1.2.0 => github.com/new/b v2.0.0", ")"], @@ -398,18 +402,54 @@ test.each([ expect(parseGoWork(lines.join("\n"))).toEqual(expected); }); -// rewriteGoImports -test("rewriteGoImports empty map does nothing", () => { - rewriteGoImports(resolve("fixtures/go"), {}, () => { throw new Error("unexpected write"); }); +test("resolveGoWorkModule contains members after resolving symlinks", () => { + const parent = mkdtempSync(resolve(tmpdir(), "updates-go-work-")); + try { + const root = resolve(parent, "project"); + const member = resolve(root, "member"); + const outside = resolve(parent, "trusted"); + mkdirSync(member, {recursive: true}); + mkdirSync(outside); + writeFileSync(resolve(member, "go.mod"), "module example.com/member\n"); + writeFileSync(resolve(outside, "go.mod"), "module example.com/trusted\n"); + symlinkSync(outside, resolve(root, "linked")); + expect(resolveGoWorkModule(root, "member")).toBe(realpathSync(resolve(member, "go.mod"))); + expect(resolveGoWorkModule(root, "../trusted")).toBeNull(); + expect(resolveGoWorkModule(root, "linked")).toBeNull(); + } finally { + rmSync(parent, {recursive: true}); + } }); -test("rewriteGoImports no .go files does nothing", () => { +test("rewriteGoImports rewrites matching imports and skips empty work", () => { + rewriteGoImports(resolve("fixtures/go"), {}, () => { throw new Error("unexpected write"); }); rewriteGoImports(resolve("fixtures/cargo"), {"github.com/old": "github.com/new"}, () => { throw new Error("unexpected write"); }); -}); - -test("rewriteGoImports rewrites matching imports", () => { let written = ""; rewriteGoImports(resolve("fixtures/go"), {"github.com/google/uuid": "github.com/google/uuid/v2"}, (_, content) => { written = content; }); expect(written).toContain(`"github.com/google/uuid/v2"`); expect(written).not.toContain(`"github.com/google/uuid"`); }); + +test("rewriteGoImportPaths only rewrites import declarations", () => { + const content = `package main + +// import "github.com/old/comment" +import ( + alias "github.com/old/sub" + _ \`github.com/old/raw\` + // "github.com/old/comment" +) +import "github.com/old" + +var ordinary = "github.com/old/string" +`; + expect(rewriteGoImportPaths(content, {"github.com/old": "github.com/new/v2"})).toBe(content + .replace('"github.com/old/sub"', '"github.com/new/v2/sub"') + .replace("`github.com/old/raw`", "`github.com/new/v2/raw`") + .replace('import "github.com/old"', 'import "github.com/new/v2"')); +}); + +test("rewriteGoImportPaths handles a 10 MB unterminated block comment", () => { + const content = `/*${"a".repeat(10 * 1024 * 1024)}`; + expect(rewriteGoImportPaths(content, {"github.com/old": "github.com/new/v2"})).toBe(content); +}); diff --git a/modes/go.ts b/modes/go.ts index d7a8d0c..b9911da 100644 --- a/modes/go.ts +++ b/modes/go.ts @@ -1,72 +1,69 @@ import {env} from "node:process"; -import {join, dirname} from "node:path"; -import {readFileSync, globSync} from "node:fs"; +import {dirname, isAbsolute, join, relative, resolve, sep} from "node:path"; +import {globSync, readFileSync, realpathSync} from "node:fs"; import { type Deps, type GoProxyEntry, type ModeContext, type PackageInfo, dedupe, fieldSep, stripv, getSubDir, normalizeUrl, - fetchWithRetry, defaultApiUrls, isGoPseudoVersion, isVersionPrerelease, + fetchWithRetry, defaultApiUrls, isVersionPrerelease, throwFetchError, } from "./shared.ts"; import {gt, valid} from "../utils/semver.ts"; -import {esc, getOrSet, longestFirstAlternation, tryOrNull} from "../utils/utils.ts"; +import {esc, getOrSet, pushTo, tryOrNull} from "../utils/utils.ts"; export type {GoProxyEntry}; -// go turns a bare host into an https URL, so `GOPROXY=proxy.corp/mod` reaches the same endpoint. function goProxyEntryUrl(url: string): string { const absolute = url.includes(":/") || url.startsWith("/"); return normalizeUrl(!absolute && /[.:/]/.test(url) ? `https://${url}` : url); } -// GOPROXY is an ordered list: `,` moves on only when the module is absent there, `|` on any error. -// `off` (fail, look nothing up) and `direct` (VCS only) both end the list, exactly as in go. +export async function fetchFromGoProxyChain( + chain: Array, fetchEntry: (url: string) => Promise, +): Promise { + for (const {url, fallback} of chain) { + try { + const result = await fetchEntry(url); + if (result !== null) return result; + } catch (error) { + if (fallback === ",") throw error; + } + } + return null; +} + export function parseGoProxy(value: string): Array { const entries: Array = []; - let rest = value; - while (rest) { - const sepIdx = rest.search(/[,|]/); - const url = (sepIdx === -1 ? rest : rest.slice(0, sepIdx)).trim(); - const fallback = sepIdx !== -1 && rest[sepIdx] === "|" ? "|" : ","; - rest = sepIdx === -1 ? "" : rest.slice(sepIdx + 1); + for (const match of value.matchAll(/([^,|]*)([,|]?)/g)) { + const url = match[1].trim(); if (!url) continue; + const fallback = match[2] === "|" ? "|" : ","; if (url === "off" || url === "direct") { entries.push({url, fallback}); break; } entries.push({url: goProxyEntryUrl(url), fallback}); } return entries; } -// An endpoint override stands in for the whole list, without one GOPROXY spells it out. `off` and -// `direct` are tokens, not URLs, so no origin can be derived from them. export function resolveGoProxyChain(override?: string): Array { if (typeof override === "string") return [{url: normalizeUrl(override), fallback: ","}]; - const list = parseGoProxy(env.GOPROXY || `${defaultApiUrls.goproxy},direct`); - return list.length ? list : [{url: defaultApiUrls.goproxy, fallback: ","}]; + const value = env.GOPROXY || `${defaultApiUrls.goproxy},direct`; + const list = parseGoProxy(value); + if (!list.length) throw new Error("GOPROXY list is not the empty string, but contains no entries"); + return list; } export function parseGoNoProxy(): Array { const value = env.GONOPROXY || env.GOPRIVATE || ""; - return value.split(",").map(s => s.trim().replace(/\/+$/, "")).filter(Boolean); + return value.split(",").map(entry => entry.trim().replace(/\/+$/, "")).filter(Boolean); } -// Go matches these with path.Match, so `*` and `?` stay within a path element and `[…]` -// is a class. A match on any prefix element covers the whole subtree. const goPatternCache = new Map(); function goPatternToRegex(pattern: string): RegExp { return getOrSet(goPatternCache, pattern, () => { let body = ""; - for (let idx = 0; idx < pattern.length; idx++) { - const char = pattern[idx]; - if (char === "*") { - body += "[^/]*"; - } else if (char === "?") { - body += "[^/]"; - } else if (char === "[" && pattern.includes("]", idx + 1)) { - const end = pattern.indexOf("]", idx + 1); - body += `[${pattern.slice(idx + 1, end).replace(/\\/g, "\\\\")}]`; - idx = end; - } else { - body += esc(char); // an unterminated `[` lands here too, as a literal - } + for (const match of pattern.matchAll(/\[([^\]]*)\]|./gs)) { + const [token, characterClass] = match; + body += characterClass !== undefined ? `[${characterClass.replace(/\\/g, "\\\\")}]` : + token === "*" ? "[^/]*" : token === "?" ? "[^/]" : esc(token); } return new RegExp(`^${body}(?:/.*)?$`); }); @@ -81,7 +78,6 @@ export function encodeGoModulePath(modulePath: string): string { } const goMajorSuffixRe = /\/v(\d+)$/; -// gopkg.in encodes the major as `.vN` on the last element instead of a `/vN` element. const gopkgMajorSuffixRe = /^gopkg\.in\/.*?\.v(\d+)$/; export function extractGoMajor(name: string): number { @@ -91,15 +87,12 @@ export function extractGoMajor(name: string): number { export function buildGoModulePath(name: string, major: number): string { if (name.startsWith("gopkg.in/")) { - // gopkg.in has no unsuffixed form, v1 is `.v1` return `${name.replace(/\.v\d+$/, "")}.v${major}`; } const base = name.replace(goMajorSuffixRe, ""); return major <= 1 ? base : `${base}/v${major}`; } -// Module path adjusted for a target version's major suffix: .../v2 -> .../v3 on a -// major bump, unchanged for same-major and +incompatible versions. export function goModulePathForVersion(modulePath: string, version: string): string { if (version.includes("+incompatible")) return modulePath; const newMajor = Number.parseInt(stripv(version).split(".")[0]); @@ -107,209 +100,193 @@ export function goModulePathForVersion(modulePath: string, version: string): str return buildGoModulePath(modulePath, newMajor); } -type ReplaceMatch = {origModule: string, origVersion: string, targetModule: string, targetVersion: string}; +type GoDirectiveKind = "require" | "replace" | "exclude" | "tool" | "use"; +type GoDirective = {kind: GoDirectiveKind, value: string, lineNumber: number}; -// Line-scanning regexes, hoisted out of the per-line loops in parseGoMod/parseGoWork. -const requireBlockRe = /^require\s*\(/; -const replaceBlockRe = /^replace\s*\(/; -const toolBlockRe = /^tool\s*\(/; -const useBlockRe = /^use\s*\(/; -const replaceLineRe = /^replace\s+/; +const directiveRe = /^(require|replace|exclude|tool|use)(?:\s*\(\s*(?:\/\/.*)?$|\s+(.+)$)/; const requireEntryRe = /^(\S+)\s+(v\S+)/; -const requireLineRe = /^require\s+(\S+)\s+(v\S+)/; -const toolLineRe = /^tool\s+(\S+)/; -const useLineRe = /^use\s+(\S+)/; -const firstWordRe = /^(\S+)/; const replaceInBlockRe = /^(\S+)(?:\s+(v\S+))?\s+=>\s+(\S+)(?:\s+(v\S+))?/; -const replaceDirectiveRe = /^replace\s+(\S+)(?:\s+(v\S+))?\s+=>\s+(\S+)(?:\s+(v\S+))?/; +type ParsedReplace = {origModule: string, origVersion: string, targetModule: string, targetVersion: string}; -const quotedRe = /^"(.*)"$/; // a module path may be quoted, and the quotes are not part of it -const trimQuotes = (str: string): string => quotedRe.exec(str)?.[1] ?? str; +const trimQuotes = (str: string): string => str.replace(/^"(.*)"$/, "$1"); -// Local paths carry no version, so they have to parse too — the caller needs to know the -// module is replaced even when there is nothing to update on the right-hand side. function isLocalReplaceTarget(target: string): boolean { return target.startsWith("./") || target.startsWith("/") || target.startsWith("../"); } -function parseReplaceDirective(trimmed: string, inBlock: boolean): ReplaceMatch | null { - const match = (inBlock ? replaceInBlockRe : replaceDirectiveRe).exec(trimmed); +function parseReplaceDirective(value: string): ParsedReplace | null { + const match = replaceInBlockRe.exec(value); if (!match) return null; const [, origModule, origVersion, targetModule, targetVersion] = match; return {origModule: trimQuotes(origModule), origVersion: origVersion ?? "", targetModule: trimQuotes(targetModule), targetVersion: targetVersion ?? ""}; } -function shouldSkipMajorProbe(name: string, type: string, currentVersion: string): boolean { - return type === "indirect" || name.startsWith("golang.org/x/") || isGoPseudoVersion(currentVersion); +function* scanGoDirectives(lines: Array): Generator { + let block: GoDirectiveKind | null = null; + + for (const [lineNumber, line] of lines.entries()) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("//")) continue; + if (block) { + if (/^\)\s*(?:\/\/.*)?$/.test(trimmed)) { block = null; continue; } + yield {kind: block, value: trimmed, lineNumber}; + continue; + } + const match = directiveRe.exec(trimmed); + if (!match) continue; + if (match[2] === undefined) block = match[1] as GoDirectiveKind; + else yield {kind: match[1] as GoDirectiveKind, value: match[2], lineNumber}; + } +} + +function shouldSkipMajorProbe(name: string): boolean { + return name.startsWith("golang.org/x/"); } type ProbeResult = {Version: string, Time: string, path: string}; -export async function probeMajorVersions( +async function probeMajorVersions( currentMajor: number, firstProbe: ProbeResult | null, probeFn: (major: number) => Promise, ): Promise { - if (!firstProbe) return null; let highest = firstProbe; - // A major whose `@latest` is a prerelease or pseudo-version carries nothing to upgrade to, and - // taking it would hide every released major below it, so it only stands in when none has one. - let highestRelease = isVersionPrerelease(firstProbe.Version) ? null : firstProbe; - - // Stop at first gap — Go majors are conventionally contiguous. - const cap = currentMajor + 101; - let from = currentMajor + 2; - let batchSize = 7; - while (from <= cap) { - const to = Math.min(from + batchSize - 1, cap); - const results = await Promise.all(Array.from({length: to - from + 1}, (_, idx) => probeFn(from + idx))); - const gapIdx = results.indexOf(null); - const hits = gapIdx === -1 ? results : results.slice(0, gapIdx); - if (hits.length) { - highest = hits.at(-1)!; - highestRelease = hits.findLast(hit => !isVersionPrerelease(hit!.Version)) ?? highestRelease; - } - if (gapIdx !== -1) break; - from = to + 1; - batchSize *= 2; + let highestRelease = firstProbe && !isVersionPrerelease(firstProbe.Version) ? firstProbe : null; + + for (let major = currentMajor + 2; highest; major++) { + const result = await probeFn(major); + if (!result) break; + highest = result; + if (!isVersionPrerelease(result.Version)) highestRelease = result; } return highestRelease ?? highest; } function buildGoPackageInfo( - name: string, currentVersion: string, - probe: ProbeResult | null, - latestVersion: string, latestTime: string, + name: string, currentVersion: string, probe: ProbeResult | null, latest: ProbeResult, ): PackageInfo { - const highestVersion = probe?.Version ?? latestVersion; - const highestTime = probe?.Time ?? latestTime; - const highestPath = probe?.path ?? name; + const highest = probe ?? latest; return [{ - name, - old: currentVersion, - new: stripv(highestVersion), - Time: highestTime, - ...(highestPath !== name && {newPath: highestPath}), - sameMajorNew: stripv(latestVersion), - sameMajorTime: latestTime, + name, old: currentVersion, new: stripv(highest.Version), Time: highest.Time, + ...(highest.path !== name && {newPath: highest.path}), + sameMajorNew: stripv(latest.Version), sameMajorTime: latest.Time, }, null]; } -export function parseGoMod(content: string): {deps: Record, indirect: Record, replace: Record, tool: Record} { +export function parseGoMod(content: string) { const deps: Record = {}; const indirect: Record = {}; const replace: Record = {}; const tool: Record = {}; + const exclude: Record> = {}; const replacedModules = new Set(); const toolPaths: string[] = []; - const lines = content.split(/\r?\n/); - let inRequire = false; - let inReplace = false; - let inTool = false; - for (const line of lines) { - const trimmed = line.trim(); - if (requireBlockRe.test(trimmed)) { inRequire = true; continue; } - if (replaceBlockRe.test(trimmed)) { inReplace = true; continue; } - if (toolBlockRe.test(trimmed)) { inTool = true; continue; } - if (trimmed === ")") { inRequire = false; inReplace = false; inTool = false; continue; } - if (trimmed.startsWith("//")) continue; // full-line comments are not dependencies - - if (inTool) { - if (trimmed) toolPaths.push(trimQuotes(trimmed)); + for (const directive of scanGoDirectives(content.split(/\r?\n/))) { + if (directive.kind === "tool") { + toolPaths.push(trimQuotes(directive.value.split(/\s/, 1)[0])); continue; } - - const toolMatch = toolLineRe.exec(trimmed); - if (toolMatch) { toolPaths.push(trimQuotes(toolMatch[1])); continue; } - - const isIndirect = trimmed.includes("// indirect"); - - if (inReplace || replaceLineRe.test(trimmed)) { - const parsed = parseReplaceDirective(trimmed, inReplace); + if (directive.kind === "exclude") { + const match = requireEntryRe.exec(directive.value); + if (match) (exclude[trimQuotes(match[1])] ??= []).push(match[2]); + continue; + } + if (directive.kind === "replace") { + const parsed = parseReplaceDirective(directive.value); if (parsed) { if (parsed.targetVersion && !isLocalReplaceTarget(parsed.targetModule)) { replace[parsed.targetModule] = parsed.targetVersion; } - // A replace pinned to one version leaves the require version live and updatable; - // an unversioned or local one takes over, making the require version inert. if (!parsed.origVersion) replacedModules.add(parsed.origModule); } continue; } - - const match = (inRequire ? requireEntryRe : requireLineRe).exec(trimmed); - if (match) { - (isIndirect ? indirect : deps)[trimQuotes(match[1])] = match[2]; - } + if (directive.kind !== "require") continue; + const match = requireEntryRe.exec(directive.value); + if (match) (directive.value.includes("// indirect") ? indirect : deps)[trimQuotes(match[1])] = match[2]; } - // Exclude replaced modules from deps - for (const mod of replacedModules) { - delete deps[mod]; - delete indirect[mod]; - } + for (const mod of replacedModules) { delete deps[mod]; delete indirect[mod]; } - // Match tool paths to their modules in require and move them to tool - if (toolPaths.length) { - const allModules = [...Object.keys(indirect), ...Object.keys(deps)]; - for (const toolPath of toolPaths) { - let bestMatch = ""; - for (const mod of allModules) { - if ((toolPath === mod || toolPath.startsWith(`${mod}/`)) && mod.length > bestMatch.length) { - bestMatch = mod; - } - } - const source = indirect[bestMatch] ? indirect : deps[bestMatch] ? deps : null; - if (source) { - tool[bestMatch] = source[bestMatch]; - delete source[bestMatch]; + const allModules = [...Object.keys(indirect), ...Object.keys(deps)]; + for (const toolPath of toolPaths) { + let bestMatch = ""; + for (const mod of allModules) { + if ((toolPath === mod || toolPath.startsWith(`${mod}/`)) && mod.length > bestMatch.length) { + bestMatch = mod; } } + const source = indirect[bestMatch] ? indirect : deps; + if (source[bestMatch]) { + tool[bestMatch] = source[bestMatch]; + delete source[bestMatch]; + } } - return {deps, indirect, replace, tool}; + return {deps, indirect, replace, tool, ...(Object.keys(exclude).length && {exclude})}; } -async function fetchGoVcsInfo(name: string, type: string, currentVersion: string, goCwd: string, ctx: ModeContext): Promise { +type GoExcludes = Map>; + +function getGoExcludes(goCwd: string, type: string): GoExcludes { + const suffix = type.includes("|") ? type.slice(type.indexOf("|") + 1) : ""; + // multi-root workspaces prefix the member with `:` + const memberPath = suffix.includes(":") ? suffix.slice(suffix.indexOf(":") + 1) : suffix; + for (const manifest of memberPath ? [join(goCwd, memberPath, "go.mod"), join(goCwd, "go.mod")] : [join(goCwd, "go.mod")]) { + try { + return new Map(Object.entries(parseGoMod(readFileSync(manifest, "utf8")).exclude ?? {}).map( + ([name, versions]) => [name, new Set(versions)], + )); + } catch {} + } + return new Map(); +} + +async function fetchGoVcsInfo( + name: string, currentVersion: string, goCwd: string, ctx: ModeContext, excludes: GoExcludes, +): Promise { const currentMajor = extractGoMajor(name); - // A missing `go`, an unreachable host, an auth prompt and a nonexistent module all leave - // `go list` with the same exit, and nothing follows `direct`, so any failure is the dep's error. - const goListQuery = async (modulePath: string, timeout: number): Promise => { + const goListQuery = async (modulePath: string, timeout: number, version?: string): Promise}> => { try { - const {stdout} = await ctx.execFile("go", ["list", "-m", "-json", `${modulePath}@latest`], {timeout, cwd: goCwd, env}); - const data = JSON.parse(stdout) as {Version: string, Time?: string}; - return {Version: data.Version, Time: data.Time || "", path: modulePath}; + const query = version ?? "latest"; + const args = ["list", "-m", ...(!version ? ["-versions"] : []), "-json", `${modulePath}@${query}`]; + const {stdout} = await ctx.execFile("go", args, {timeout, cwd: goCwd, env: {...env, GOPROXY: "direct"}}); + const data = JSON.parse(stdout) as {Version: string, Time?: string, Versions?: Array}; + return {Version: data.Version, Time: data.Time || "", path: modulePath, Versions: data.Versions}; } catch (err: any) { - // go names the reason on stderr, where execFile's own message only repeats the command const reason = String(err?.stderr ?? "").trim().split("\n")[0] || err?.message || String(err); - throw new Error(`go list -m ${modulePath}@latest failed: ${reason}`); + throw new Error(`go list -m ${modulePath}@${version ?? "latest"} failed: ${reason}`); } }; - // A probe only answers "does this major exist", so a failing one costs no more than a missing one. - const probeQuery = (modulePath: string) => tryOrNull(goListQuery(modulePath, ctx.goProbeTimeout)); + const latestQuery = async (modulePath: string, timeout: number) => { + const latest = await goListQuery(modulePath, timeout); + const excluded = excludes.get(modulePath); + if (!excluded?.has(latest.Version)) return latest; + const available = pickGoListVersion((latest.Versions ?? []).join("\n"), goPathMajor(modulePath), excluded); + if (!available) throw new Error(`No non-excluded versions found for ${modulePath}`); + return goListQuery(modulePath, timeout, available.Version); + }; + const probeQuery = (modulePath: string) => tryOrNull(latestQuery(modulePath, ctx.goProbeTimeout)); - // Fetch @latest and first major probe in parallel - const skip = shouldSkipMajorProbe(name, type, currentVersion); const [latest, firstProbe] = await Promise.all([ - goListQuery(name, ctx.fetchTimeout), - skip ? null : probeQuery(buildGoModulePath(name, currentMajor + 1)), + latestQuery(name, ctx.fetchTimeout), + shouldSkipMajorProbe(name) ? null : probeQuery(buildGoModulePath(name, currentMajor + 1)), ]); - const probeResult = await probeMajorVersions(currentMajor, firstProbe, major => - probeQuery(buildGoModulePath(name, major)), + return buildGoPackageInfo( + name, currentVersion, + await probeMajorVersions(currentMajor, firstProbe, major => probeQuery(buildGoModulePath(name, major))), + latest, ); - return buildGoPackageInfo(name, currentVersion, probeResult, latest.Version, latest.Time); } export const goProxyHeaders = {"accept-encoding": "gzip, deflate, br"}; type ProxyFetch = (url: string) => Promise; -type GoModuleFetch = (doFetch: ProxyFetch, base: string, path: string) => Promise; -// 404 and 410 are the protocol's "not here", anything else is a proxy failure, never "up to date". const isGoProxyMiss = (status: number): boolean => status === 404 || status === 410; async function readGoProxyInfo(res: Response, url: string, path: string): Promise { @@ -320,8 +297,7 @@ async function readGoProxyInfo(res: Response, url: string, path: string): Promis throw new Error(`Invalid response from ${url}`); } -// `@latest` is optional in the GOPROXY protocol, so a miss only says this endpoint has nothing. -const fetchGoLatest: GoModuleFetch = async (doFetch, base, path) => { +const fetchGoLatest = async (doFetch: ProxyFetch, base: string, path: string): Promise => { const url = `${base}/${encodeGoModulePath(path)}/@latest`; const res = await doFetch(url); if (res.ok) return readGoProxyInfo(res, url, path); @@ -331,40 +307,30 @@ const fetchGoLatest: GoModuleFetch = async (doFetch, base, path) => { const goLatestByCtx = new WeakMap>>(); -// One `@latest` per endpoint and module path for the whole run, as the make mode's probe makes the -// same request as the lookup that follows. A rejected one is evicted so the lookup still retries. export function fetchGoLatestOnce(ctx: ModeContext, doFetch: ProxyFetch, base: string, path: string): Promise { return dedupe(goLatestByCtx, ctx, `${base}/${path}`, () => fetchGoLatest(doFetch, base, path)); } -// The order `@latest` reports: a release outranks any prerelease, pseudo-versions among them. -function isHigherGoVersion(candidate: string, best: string): boolean { - const candidateIsPre = isVersionPrerelease(candidate); - if (candidateIsPre !== isVersionPrerelease(best)) return !candidateIsPre; - return gt(candidate, best); -} - -// `@v/list` is `version [timestamp]` per line, unordered. `major` bounds the answer: some proxies -// list every major under each path, and a version the path cannot carry writes a go.mod go refuses. -export function pickGoListVersion(body: string, major = 0): {Version: string, Time: string} | null { +export function pickGoListVersion(body: string, major = 0, excluded = new Set()): {Version: string, Time: string} | null { let best: {Version: string, Time: string} | null = null; for (const line of body.split("\n")) { const [version, time] = line.trim().split(/\s+/); - if (!version || !valid(version)) continue; + if (!version || !valid(version) || excluded.has(version)) continue; if (major && Number.parseInt(stripv(version)) !== major) continue; - if (best && !isHigherGoVersion(version, best.Version)) continue; + if (best) { + const prerelease = isVersionPrerelease(version); + if (prerelease === isVersionPrerelease(best.Version) ? !gt(version, best.Version) : prerelease) continue; + } best = {Version: version, Time: time ?? ""}; } return best; } -// 0 for a path without a major suffix: v0, v1 and `+incompatible` v2+ all live there. function goPathMajor(path: string): number { return goMajorSuffixRe.test(path) || gopkgMajorSuffixRe.test(path) ? extractGoMajor(path) : 0; } -// `//@v/list`, what go falls back to when a proxy omits `@latest`. -const fetchGoList: GoModuleFetch = async (doFetch, base, path) => { +async function fetchGoList(doFetch: ProxyFetch, base: string, path: string, excluded = new Set()): Promise { const encoded = encodeGoModulePath(path); const url = `${base}/${encoded}/@v/list`; const res = await doFetch(url); @@ -372,132 +338,196 @@ const fetchGoList: GoModuleFetch = async (doFetch, base, path) => { if (isGoProxyMiss(res.status)) return null; throwFetchError(res, url, path, base); } - const best = pickGoListVersion(await res.text(), goPathMajor(path)); - if (!best) return null; // a known module with no versions yet + const best = pickGoListVersion(await res.text(), goPathMajor(path), excluded); + if (!best) return null; if (best.Time) return {...best, path}; - // No timestamp in the list, so stat the one version picked as go does. A failure costs only the date. const infoUrl = `${base}/${encoded}/@v/${encodeGoModulePath(best.Version)}.info`; try { const infoRes = await doFetch(infoUrl); if (infoRes.ok) return readGoProxyInfo(infoRes, infoUrl, path); } catch {} return {...best, path}; -}; +} -// null when this proxy does not know the module, so the caller can move down the chain. -async function fetchGoProxyModule(base: string, name: string, type: string, currentVersion: string, ctx: ModeContext): Promise { +async function fetchGoProxyModule( + base: string, name: string, currentVersion: string, ctx: ModeContext, excludes: GoExcludes, +): Promise { const currentMajor = extractGoMajor(name); - const goLatest: GoModuleFetch = (doFetch, proxy, path) => fetchGoLatestOnce(ctx, doFetch, proxy, path); const primaryFetch: ProxyFetch = url => fetchWithRetry(ctx, url, {headers: goProxyHeaders}); const probeFetch: ProxyFetch = url => ctx.doFetch(url, {signal: AbortSignal.timeout(ctx.goProbeTimeout), headers: goProxyHeaders}); - const probeWith = (fetchModule: GoModuleFetch) => async (path: string) => { + const primaryLatestPromise = fetchGoLatestOnce(ctx, primaryFetch, base, name); + const primaryPromise = (async () => { + const primaryLatest = await primaryLatestPromise; + return primaryLatest && !excludes.get(name)?.has(primaryLatest.Version) ? primaryLatest : + fetchGoList(primaryFetch, base, name, excludes.get(name)); + })(); + const probe = async (path: string) => { try { - return await fetchModule(probeFetch, base, path); + const [latest, primaryLatest] = await Promise.all([ + fetchGoLatestOnce(ctx, probeFetch, base, path), primaryLatestPromise, + ]); + const excluded = excludes.get(path); + if (latest && !excluded?.has(latest.Version)) return latest; + if (!latest && primaryLatest) return null; + return fetchGoList(probeFetch, base, path, excluded); } catch { return null; } }; - // Fetch @latest and probe for next major version in parallel - const skip = shouldSkipMajorProbe(name, type, currentVersion); - const nextMajorPath = buildGoModulePath(name, currentMajor + 1); - const [latest, latestProbe] = await Promise.all([ - goLatest(primaryFetch, base, name), - skip ? null : probeWith(goLatest)(nextMajorPath), + const [primary, firstProbe] = await Promise.all([ + primaryPromise, + shouldSkipMajorProbe(name) ? null : probe(buildGoModulePath(name, currentMajor + 1)), ]); - const primary = latest ?? await fetchGoList(primaryFetch, base, name); if (!primary) return null; - // A proxy serves `@latest` for every major of a module or for none, and `latestProbe` settled - // which, so further majors go straight to that endpoint rather than missing on the other first. - const probe = probeWith(latest ? goLatest : fetchGoList); - const firstProbe = skip || latest ? latestProbe : await probeWith(fetchGoList)(nextMajorPath); - const probeResult = await probeMajorVersions(currentMajor, firstProbe, major => probe(buildGoModulePath(name, major))); - - return buildGoPackageInfo(name, currentVersion, probeResult, primary.Version, primary.Time); + return buildGoPackageInfo( + name, currentVersion, + await probeMajorVersions(currentMajor, firstProbe, major => probe(buildGoModulePath(name, major))), + primary, + ); } export async function fetchGoProxyInfo(name: string, type: string, currentVersion: string, goCwd: string, ctx: ModeContext, goNoProxy: Array): Promise { - if (isGoNoProxy(name, goNoProxy)) return fetchGoVcsInfo(name, type, currentVersion, goCwd, ctx); + const excludes = getGoExcludes(goCwd, type); + if (isGoNoProxy(name, goNoProxy)) return fetchGoVcsInfo(name, currentVersion, goCwd, ctx, excludes); - for (const {url, fallback} of ctx.goProxyChain) { - // go fails the lookup outright, and a lookup that could not run is not an up-to-date dependency. + const info = await fetchFromGoProxyChain(ctx.goProxyChain, async url => { if (url === "off") throw new Error("Module lookup disabled by GOPROXY=off"); - if (url === "direct") return fetchGoVcsInfo(name, type, currentVersion, goCwd, ctx); - try { - const info = await fetchGoProxyModule(url, name, type, currentVersion, ctx); - if (info) return info; - } catch (err) { - if (fallback === ",") throw err; // only `|` moves past a proxy that is broken rather than empty - } - } + if (url === "direct") return fetchGoVcsInfo(name, currentVersion, goCwd, ctx, excludes); + return fetchGoProxyModule(url, name, currentVersion, ctx, excludes); + }); + if (info) return info; throw new Error(`Unable to find ${name} on any GOPROXY entry`); } -// Module paths may be quoted in go.mod and the quotes have to survive a rewrite. `group` is this -// capture's number in the whole pattern, so the backreference demands a matching quote. -const quotedPath = (name: string, group: number) => `("?)${esc(name)}\\${group}`; +const quotedPath = (name: string) => `("?)${esc(name)}\\2`; export function updateGoMod(pkgStr: string, deps: Deps): [string, Record] { - let newPkgStr = pkgStr; const majorVersionRewrites: Record = {}; - for (const [key, {old, oldOrig}] of Object.entries(deps)) { + const entries = Object.entries(deps); + if (!entries.length) return [pkgStr, majorVersionRewrites]; + const newline = pkgStr.includes("\r\n") ? "\r\n" : "\n"; + const lines = pkgStr.split(newline); + const rewriteLines = (lineNumbers: Array | undefined, pattern: RegExp, replacement: string): boolean => { + let rewritten = false; + for (const lineNumber of lineNumbers ?? []) { + const line = lines[lineNumber]; + lines[lineNumber] = line.replace(pattern, replacement); + rewritten ||= lines[lineNumber] !== line; + } + return rewritten; + }; + const requireLines = new Map>(); + const replaceDirectives = new Map>(); + const toolLines = new Map>(); + for (const directive of scanGoDirectives(lines)) { + if (directive.kind === "require") { + const match = requireEntryRe.exec(directive.value); + if (match) pushTo(requireLines, trimQuotes(match[1]), directive.lineNumber); + } else if (directive.kind === "replace") { + const parsed = parseReplaceDirective(directive.value); + if (parsed) pushTo(replaceDirectives, parsed.targetModule, {...parsed, lineNumber: directive.lineNumber}); + } else if (directive.kind === "tool") { + let name = trimQuotes(directive.value.split(/\s/, 1)[0]); + while (name) { + pushTo(toolLines, name, directive.lineNumber); + const slash = name.lastIndexOf("/"); + if (slash === -1) break; + name = name.slice(0, slash); + } + } + } + for (const [key, {old, oldOrig, new: newValue}] of entries) { const [depType, name] = key.split(fieldSep); const oldValue = oldOrig || old; - const newValue = deps[key].new; const newPath = goModulePathForVersion(name, newValue); - if (depType === "replace") { - // go rejects a replace whose version does not match the path's major, so a major bump moves - // the target onto a new path. Only a self-replace carries its left-hand side along. - if (newPath !== name) { - const beforeSelfReplace = newPkgStr; - newPkgStr = newPkgStr.replace( - new RegExp(`(^\\s*(?:replace\\s+)?)${quotedPath(name, 2)}(\\s+=>\\s+)${quotedPath(name, 4)}(\\s+)v${esc(oldValue)}`, "gm"), - `$1$2${newPath}$2$3$4${newPath}$4$5v${newValue}`, + let selfReplace = false; + for (const parsed of replaceDirectives.get(name) ?? []) { + if (stripv(parsed.targetVersion) !== oldValue) continue; + const {lineNumber} = parsed; + lines[lineNumber] = lines[lineNumber].replace( + new RegExp(`(=>\\s+)${quotedPath(name)}(\\s+)v${esc(oldValue)}(?=\\s*(?://.*)?$)`), + `$1$2${newPath}$2$3v${newValue}`, ); - // A self-replace is not in `deps`, so its require line is reachable only here, and go - // applies the replacement solely to the path the require names. The lookahead spares a - // `name vOLD => other vNEW` line, which carries a dep of its own. - if (newPkgStr !== beforeSelfReplace) { - const beforeRequire = newPkgStr; - newPkgStr = newPkgStr.replace( - new RegExp(`(^\\s*(?:require\\s+)?)${quotedPath(name, 2)}(\\s+)v\\S+(?=\\s*(?://.*)?$)`, "gm"), - `$1$2${newPath}$2$3v${newValue}`, + if (newPath !== name && parsed.origModule === name && !parsed.origVersion) { + lines[lineNumber] = lines[lineNumber].replace( + new RegExp(`(^\\s*(?:replace\\s+)?)${quotedPath(name)}(?=\\s+=>)`), + `$1$2${newPath}$2`, ); - if (newPkgStr !== beforeRequire) majorVersionRewrites[name] = newPath; + selfReplace = true; } } - // Update version in replace line: => targetModule vOLD -> => targetModule vNEW - newPkgStr = newPkgStr.replace(new RegExp(`(=>\\s+)${quotedPath(name, 2)}(\\s+)v${esc(oldValue)}`, "g"), `$1$2${newPath}$2$3v${newValue}`); + if (selfReplace && rewriteLines( + requireLines.get(name), new RegExp(`(^\\s*(?:require\\s+)?)${quotedPath(name)}(\\s+)v\\S+(?=\\s*(?://.*)?$)`), + `$1$2${newPath}$2$3v${newValue}`, + )) majorVersionRewrites[name] = newPath; continue; } - // An indirect dep only ever bumps its version: no path rewrite and no replace removal. - if (newPath !== name && depType !== "indirect") { - newPkgStr = newPkgStr.replace(new RegExp(`${quotedPath(name, 1)} +v${esc(oldValue)}`, "g"), `$1${newPath}$1 v${newValue}`); - // Rewrite tool paths referencing the old module path - if (depType === "tool") { - newPkgStr = newPkgStr.replace(new RegExp(`(^\\s+|^tool\\s+)("?)${esc(name)}((?:/[^"\\s]+)?)\\2\\s*$`, "gm"), `$1$2${newPath}$3$2`); - } - majorVersionRewrites[name] = newPath; - } else { - newPkgStr = newPkgStr.replace(new RegExp(`(${quotedPath(name, 2)}) +v${esc(oldValue)}`, "g"), `$1 v${newValue}`); + if (rewriteLines( + requireLines.get(name), new RegExp(`(^\\s*(?:require\\s+)?)${quotedPath(name)}(\\s+)v${esc(oldValue)}(?=\\s*(?://.*)?$)`), + `$1$2${newPath}$2$3v${newValue}`, + ) && newPath !== name) majorVersionRewrites[name] = newPath; + if (depType === "tool" && newPath !== name) { + rewriteLines( + toolLines.get(name), new RegExp(`(^\\s*(?:tool\\s+)?)("?)${esc(name)}((?:/[^"\\s]+)?)\\2(?=\\s*(?://.*)?$)`), + `$1$2${newPath}$3$2`, + ); + } + } + return [lines.join(newline), majorVersionRewrites]; +} + +const goTokenRe = /\s+|\/\/[^\n]*(?:\n|$)|\/\*[\s\S]*?(?:\*\/|$)|[A-Za-z_][A-Za-z0-9_]*|"(?:\\[\s\S]|[^"\\])*(?:"|$)|`[^`]*(?:`|$)|'(?:\\[\s\S]|[^'\\])*(?:'|$)|./g; + +export function rewriteGoImportPaths(content: string, rewrites: Record): string { + const entries = Object.entries(rewrites).sort(([left], [right]) => right.length - left.length); + if (!entries.length) return content; + const replacements: Array<{start: number, end: number, value: string}> = []; + const addImport = (value: string, start: number): boolean => { + if (!`"'\``.includes(value[0])) return false; + if (value[0] === "'") return true; + const path = value.slice(1, -1); + const match = entries.find(([oldPath]) => path === oldPath || path.startsWith(`${oldPath}/`)); + if (match) replacements.push({start: start + 1, end: start + value.length - 1, value: `${match[1]}${path.slice(match[0].length)}`}); + return true; + }; + + let importBlock = false; + let importTokens = 0; + for (const match of content.matchAll(goTokenRe)) { + const value = match[0]; + if (/^\s/.test(value) || value.startsWith("//") || value.startsWith("/*")) continue; + if (importBlock) { + if (value === ")") importBlock = false; + else addImport(value, match.index); + continue; + } + if (value === "import") { + importTokens = 2; + } else if (importTokens && value === "(") { + importBlock = true; + importTokens = 0; + } else if (importTokens && (addImport(value, match.index) || --importTokens === 0)) { + importTokens = 0; } } - return [newPkgStr, majorVersionRewrites]; + + let result = content; + for (const replacement of replacements.reverse()) { + result = `${result.slice(0, replacement.start)}${replacement.value}${result.slice(replacement.end)}`; + } + return result; } export function rewriteGoImports(projectDir: string, majorVersionRewrites: Record, write: (file: string, content: string) => void): void { - const entries = Object.entries(majorVersionRewrites); - if (!entries.length) return; - const lookup = new Map(entries); - const combinedRe = new RegExp(`"(${longestFirstAlternation(lookup.keys())})(/|")`, "g"); - const goFiles = globSync("**/*.go", {cwd: projectDir}); - for (const relPath of goFiles) { + if (!Object.keys(majorVersionRewrites).length) return; + for (const relPath of globSync("**/*.go", {cwd: projectDir})) { const filePath = join(projectDir, relPath); const content = readFileSync(filePath, "utf8"); - const replaced = content.replace(combinedRe, (_, oldPath, sep) => `"${lookup.get(oldPath)}${sep}`); + const replaced = rewriteGoImportPaths(content, majorVersionRewrites); if (replaced !== content) write(filePath, replaced); } } @@ -505,27 +535,11 @@ export function rewriteGoImports(projectDir: string, majorVersionRewrites: Recor export function parseGoWork(content: string): {use: string[], replace: Record} { const use: string[] = []; const replace: Record = {}; - const lines = content.split(/\r?\n/); - let inUse = false; - let inReplace = false; - - for (const line of lines) { - const trimmed = line.trim(); - if (useBlockRe.test(trimmed)) { inUse = true; continue; } - if (replaceBlockRe.test(trimmed)) { inReplace = true; continue; } - if (trimmed === ")") { inUse = false; inReplace = false; continue; } - - if (inUse) { - const useEntry = firstWordRe.exec(trimmed); - if (useEntry && !trimmed.startsWith("//")) use.push(useEntry[1]); - continue; - } - - const useMatch = useLineRe.exec(trimmed); - if (useMatch) { use.push(useMatch[1]); continue; } - - if (inReplace || replaceLineRe.test(trimmed)) { - const parsed = parseReplaceDirective(trimmed, inReplace); + for (const directive of scanGoDirectives(content.split(/\r?\n/))) { + if (directive.kind === "use") { + use.push(directive.value.split(/\s/, 1)[0]); + } else if (directive.kind === "replace") { + const parsed = parseReplaceDirective(directive.value); if (parsed?.targetVersion && !isLocalReplaceTarget(parsed.targetModule)) { replace[parsed.targetModule] = parsed.targetVersion; } @@ -535,10 +549,22 @@ export function parseGoWork(content: string): {use: string[], replace: Record 3) { const [, user, repo, ...other] = pathParts; url.pathname = `/${user}/${repo}/${getSubDir(str)}/${other.join("/")}`; @@ -552,7 +578,6 @@ export function shortenGoModule(module: string): string { return goMajorSuffixRe.test(module) ? dirname(module) : module; } -// turn "v0.0.0-20221128193559-754e69321358" into "v0.0.0-2022112" export function shortenGoVersion(version: string): string { return version.replace(/(\d{7})\d{7}-[0-9a-f]{12}$/, "$1"); } diff --git a/modes/make.test.ts b/modes/make.test.ts index 2c7dd52..e6206e0 100644 --- a/modes/make.test.ts +++ b/modes/make.test.ts @@ -3,16 +3,10 @@ import { parseMakeGoInstalls, parseMakeImageValue, parseMakeDockerImages, - moduleRootFromMajor, resolveGoModuleRoot, - fetchMakeInfo, - fetchMakeDockerInfo, updateMakefile, } from "./make.ts"; -import {type ModeContext, fetchTimeout, goProbeTimeout} from "./shared.ts"; - -const allSemvers = new Set(["patch", "minor", "major"]); -const defaultOpts = {semvers: allSemvers, useGreatest: false, usePre: false, useRel: false, allowDowngrade: false as const}; +import {type ExecFile, type GoProxyEntry, type ModeContext, fetchTimeout} from "./shared.ts"; const sample = `GOLANGCI_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 AIR_PACKAGE := github.com/air-verse/air@v1.65.1 @@ -21,9 +15,14 @@ GOVULNCHECK_PACKAGE := golang.org/x/vuln/cmd/govulncheck@v1.2.0 # COMMENTED_PACKAGE := github.com/foo/bar@v9.9.9 MISSPELL_PACKAGE ?= github.com/golangci/misspell/cmd/misspell@v0.8.0 # inline note NOT_GO := some-local-tool@v1.0.0 -SOURCE_FILES := $(wildcard *.go)`; +SOURCE_FILES := $(wildcard *.go) +PSEUDO := golang.org/x/tools/cmd/goimports@v0.0.0-20200103221440-774c71fcf114 +PRE := github.com/foo/bar@v1.2.3-rc.1 +INCOMPAT := github.com/foo/baz@v2.0.0+incompatible +TABS := github.com/foo/qux@v1.0.0 +TOOLS += \\ + 'github.com/foo/quoted@v1.2.3' github.com/foo/aggregate@v2.0.0`; -// isMakeFileName test("isMakeFileName matches make filenames", () => { expect(isMakeFileName("Makefile")).toBe(true); expect(isMakeFileName("makefile")).toBe(true); @@ -33,7 +32,6 @@ test("isMakeFileName matches make filenames", () => { expect(isMakeFileName("Dockerfile")).toBe(false); }); -// parseMakeGoInstalls test("parseMakeGoInstalls extracts go install specs across assignment operators", () => { expect(parseMakeGoInstalls(sample)).toEqual([ {installPath: "github.com/golangci/golangci-lint/v2/cmd/golangci-lint", version: "v2.12.2"}, @@ -41,116 +39,74 @@ test("parseMakeGoInstalls extracts go install specs across assignment operators" {installPath: "github.com/go-delve/delve/cmd/dlv", version: "v1"}, {installPath: "golang.org/x/vuln/cmd/govulncheck", version: "v1.2.0"}, {installPath: "github.com/golangci/misspell/cmd/misspell", version: "v0.8.0"}, - ]); -}); - -test("parseMakeGoInstalls skips commented lines and non-go values", () => { - const paths = parseMakeGoInstalls(sample).map(i => i.installPath); - expect(paths).not.toContain("github.com/foo/bar"); // full-line comment - expect(paths).not.toContain("some-local-tool"); // no dotted host - expect(parseMakeGoInstalls(sample).some(i => i.version === "v9.9.9")).toBe(false); -}); - -test("parseMakeGoInstalls accepts pseudo-versions, prereleases and +incompatible", () => { - const content = [ - "PSEUDO := golang.org/x/tools/cmd/goimports@v0.0.0-20200103221440-774c71fcf114", - "PRE := github.com/foo/bar@v1.2.3-rc.1", - "INCOMPAT := github.com/foo/baz@v2.0.0+incompatible", - "TABS\t:=\tgithub.com/foo/qux@v1.0.0", - ].join("\n"); - expect(parseMakeGoInstalls(content)).toEqual([ {installPath: "golang.org/x/tools/cmd/goimports", version: "v0.0.0-20200103221440-774c71fcf114"}, {installPath: "github.com/foo/bar", version: "v1.2.3-rc.1"}, {installPath: "github.com/foo/baz", version: "v2.0.0+incompatible"}, {installPath: "github.com/foo/qux", version: "v1.0.0"}, + {installPath: "github.com/foo/quoted", version: "v1.2.3"}, + {installPath: "github.com/foo/aggregate", version: "v2.0.0"}, ]); }); -// moduleRootFromMajor -test("moduleRootFromMajor returns the path up to a /vN segment", () => { - expect(moduleRootFromMajor("github.com/golangci/golangci-lint/v2/cmd/golangci-lint")).toBe("github.com/golangci/golangci-lint/v2"); - expect(moduleRootFromMajor("git.kcservices.at/libs/go-golangci-config/v13")).toBe("git.kcservices.at/libs/go-golangci-config/v13"); - expect(moduleRootFromMajor("github.com/air-verse/air")).toBeNull(); -}); - -// updateMakefile -test("updateMakefile rewrites version while preserving operator, spacing and comments", () => { +test("updateMakefile rewrites versions and install paths while preserving comments", () => { const updated = updateMakefile(sample, [ {oldSpec: "github.com/air-verse/air@v1.65.1", newSpec: "github.com/air-verse/air@v1.65.3"}, {oldSpec: "github.com/golangci/misspell/cmd/misspell@v0.8.0", newSpec: "github.com/golangci/misspell/cmd/misspell@v0.9.0"}, + { + oldSpec: "github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2", + newSpec: "github.com/golangci/golangci-lint/v3/cmd/golangci-lint@v3.0.0", + }, + {oldSpec: "github.com/foo/bar@v9.9.9", newSpec: "github.com/foo/bar@v10.0.0"}, ]); expect(updated).toContain("AIR_PACKAGE := github.com/air-verse/air@v1.65.3"); expect(updated).toContain("MISSPELL_PACKAGE ?= github.com/golangci/misspell/cmd/misspell@v0.9.0 # inline note"); expect(updated).toContain("# COMMENTED_PACKAGE := github.com/foo/bar@v9.9.9"); -}); - -test("updateMakefile rewrites the install path on a major bump", () => { - const updated = updateMakefile(sample, [{ - oldSpec: "github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2", - newSpec: "github.com/golangci/golangci-lint/v3/cmd/golangci-lint@v3.0.0", - }]); - expect(updated).toContain("GOLANGCI_PACKAGE ?= github.com/golangci/golangci-lint/v3/cmd/golangci-lint@v3.0.0"); -}); - -test("updateMakefile preserves CRLF line endings", () => { - const crlf = "AIR := github.com/air-verse/air@v1.0.0\r\nFOO := bar\r\n"; - const updated = updateMakefile(crlf, [{ - oldSpec: "github.com/air-verse/air@v1.0.0", newSpec: "github.com/air-verse/air@v1.1.0", - }]); - expect(updated).toBe("AIR := github.com/air-verse/air@v1.1.0\r\nFOO := bar\r\n"); -}); - -test("updateMakefile leaves a commented-out install untouched", () => { - const updated = updateMakefile(sample, [{ - oldSpec: "github.com/foo/bar@v9.9.9", newSpec: "github.com/foo/bar@v10.0.0", - }]); - expect(updated).toContain("# COMMENTED_PACKAGE := github.com/foo/bar@v9.9.9"); expect(updated).not.toContain("v10.0.0"); + expect(updated).toContain("GOLANGCI_PACKAGE ?= github.com/golangci/golangci-lint/v3/cmd/golangci-lint@v3.0.0"); }); -test("updateMakefile rewrites a spec directly followed by a # comment", () => { - const updated = updateMakefile("IMG := koalaman/app:1.0.0#pinned\n", [{oldSpec: "koalaman/app:1.0.0", newSpec: "koalaman/app:1.1.0"}]); - expect(updated).toBe("IMG := koalaman/app:1.1.0#pinned\n"); -}); - -test("updateMakefile rewrites every spec on a line", () => { - const content = "\tgo install github.com/air-verse/air@v1.60.0 github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.0 # tools\n"; - expect(updateMakefile(content, [ - {oldSpec: "github.com/air-verse/air@v1.60.0", newSpec: "github.com/air-verse/air@v1.62.0"}, - {oldSpec: "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.0", newSpec: "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.0"}, - ])).toBe("\tgo install github.com/air-verse/air@v1.62.0 github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.0 # tools\n"); -}); - -test("updateMakefile keeps two tags of the same image apart", () => { - const content = "OLD := koalaman/shellcheck:v0.11.0\nNEW := koalaman/shellcheck:v0.12.0\n"; - const updated = updateMakefile(content, [ +test.each([ + ["preserves CRLF", "AIR := github.com/air-verse/air@v1.0.0\r\nFOO := bar\r\n", + [{oldSpec: "github.com/air-verse/air@v1.0.0", newSpec: "github.com/air-verse/air@v1.1.0"}], + "AIR := github.com/air-verse/air@v1.1.0\r\nFOO := bar\r\n"], + ["rewrites quoted and comment-adjacent specs", "QUOTED := \"koalaman/app:1.0.0\"\nCOMMENTED := koalaman/app:1.0.0#pinned\n", + [{oldSpec: "koalaman/app:1.0.0", newSpec: "koalaman/app:1.1.0"}], + "QUOTED := \"koalaman/app:1.1.0\"\nCOMMENTED := koalaman/app:1.1.0#pinned\n"], + ["rewrites every spec on a line", + "\tgo install github.com/air-verse/air@v1.60.0 github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.0 # tools\n", + [ + {oldSpec: "github.com/air-verse/air@v1.60.0", newSpec: "github.com/air-verse/air@v1.62.0"}, + {oldSpec: "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.0", newSpec: "github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.0"}, + ], + "\tgo install github.com/air-verse/air@v1.62.0 github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.0 # tools\n"], + ["keeps two tags apart", "OLD := koalaman/shellcheck:v0.11.0\nNEW := koalaman/shellcheck:v0.12.0\n", [ {oldSpec: "koalaman/shellcheck:v0.11.0", newSpec: "koalaman/shellcheck:v0.12.0"}, {oldSpec: "koalaman/shellcheck:v0.12.0", newSpec: "koalaman/shellcheck:v0.13.0"}, - ]); - expect(updated).toBe("OLD := koalaman/shellcheck:v0.12.0\nNEW := koalaman/shellcheck:v0.13.0\n"); -}); - -test("updateMakefile leaves a bare image inside a registry-prefixed one alone", () => { - const content = "PREFIXED := docker.io/koalaman/shellcheck:v0.11.0\n"; - expect(updateMakefile(content, [{oldSpec: "koalaman/shellcheck:v0.11.0", newSpec: "koalaman/shellcheck:v0.12.0"}])).toBe(content); -}); - -test("updateMakefile rewrites a docker image tag and digest in place", () => { - const content = "SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.11.0@sha256:aaa # renovate: datasource=docker\n"; - const updated = updateMakefile(content, [{ - oldSpec: "docker.io/koalaman/shellcheck:v0.11.0@sha256:aaa", - newSpec: "docker.io/koalaman/shellcheck:v0.12.0@sha256:bbb", - }]); - expect(updated).toBe("SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.12.0@sha256:bbb # renovate: datasource=docker\n"); -}); - -// resolveGoModuleRoot -const execFileFails = () => Promise.reject(new Error("no such file or directory")); -const rootCtx = (doFetch: (url: string) => Promise, goProxyUrl = "https://proxy") => - ({goProxyUrl, fetchTimeout, doFetch, execFile: execFileFails}) as unknown as ModeContext; + ], "OLD := koalaman/shellcheck:v0.12.0\nNEW := koalaman/shellcheck:v0.13.0\n"], + ["does not match inside a registry prefix", "PREFIXED := docker.io/koalaman/shellcheck:v0.11.0\n", + [{oldSpec: "koalaman/shellcheck:v0.11.0", newSpec: "koalaman/shellcheck:v0.12.0"}], + "PREFIXED := docker.io/koalaman/shellcheck:v0.11.0\n"], + ["rewrites tag and digest", "SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.11.0@sha256:aaa # renovate: datasource=docker\n", + [{oldSpec: "docker.io/koalaman/shellcheck:v0.11.0@sha256:aaa", newSpec: "docker.io/koalaman/shellcheck:v0.12.0@sha256:bbb"}], + "SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.12.0@sha256:bbb # renovate: datasource=docker\n"], +])("updateMakefile %s", (_name, content, rewrites, expected) => { + expect(updateMakefile(content, rewrites)).toBe(expected); +}); + +const execFileFails: ExecFile = () => Promise.reject(new Error("no such file or directory")); +const rootCtx = ( + doFetch: (url: string) => Promise, goProxyUrl = "https://proxy", execFile: ExecFile = execFileFails, + goProxyChain: Array = [{url: goProxyUrl, fallback: ","}], +) => ({goProxyUrl, goProxyChain, fetchTimeout, doFetch, execFile}) as unknown as ModeContext; const rootHit = (path: string) => (url: string) => Promise.resolve({ ok: url.endsWith(`${path}/@latest`), status: 404, json: () => Promise.resolve({Version: "v1.1.4"}), } as any); +const goListMiss = (candidate: string) => Promise.resolve({stdout: JSON.stringify({ + Path: candidate, + Version: "latest", + Error: {Err: `module ${candidate}: no matching versions for query "latest"`}, + Origin: {VCS: "git", URL: "https://example.com/repo"}, +}), stderr: ""}); test("resolveGoModuleRoot takes the /vN heuristic, else the longest prefix that resolves", async () => { let fetched = false; @@ -160,15 +116,52 @@ test("resolveGoModuleRoot takes the /vN heuristic, else the longest prefix that expect(await resolveGoModuleRoot("golang.org/x/vuln/cmd/govulncheck", ".", ctx, [])).toBe("golang.org/x/vuln"); }); +test("resolveGoModuleRoot applies GOPROXY comma and pipe error fallbacks", async () => { + const resolveWith = (fallback: GoProxyEntry["fallback"], seen: Array) => { + const chain: Array = [{url: "https://broken", fallback}, {url: "https://backup", fallback: ","}]; + return resolveGoModuleRoot("example.com/mod/cmd/tool", ".", rootCtx(url => { + seen.push(url); + const ok = url === "https://backup/example.com/mod/@latest"; + return Promise.resolve({ + ok, + status: ok ? 200 : url.startsWith("https://broken/") ? 500 : 404, + statusText: "failed", + json: () => Promise.resolve({Version: "v1.2.0"}), + } as any); + }, chain[0].url, execFileFails, chain), []); + }; + const pipeSeen: Array = []; + await expect(resolveWith("|", pipeSeen)).resolves.toBe("example.com/mod"); + expect(pipeSeen).toContain("https://backup/example.com/mod/@latest"); + const commaSeen: Array = []; + await expect(resolveWith(",", commaSeen)).rejects.toThrow("500"); + expect(commaSeen.some(url => url.startsWith("https://backup/"))).toBe(false); +}); + +test("resolveGoModuleRoot falls through when a proxy omits @latest", async () => { + const chain: Array = [ + {url: "https://without-latest", fallback: "|"}, + {url: "https://backup", fallback: ","}, + ]; + const seen: Array = []; + const ctx = rootCtx(url => { + seen.push(url); + const ok = url === "https://backup/example.com/mod/@latest"; + return Promise.resolve({ + ok, status: ok ? 200 : 404, json: () => Promise.resolve({Version: "v1.2.0"}), + } as any); + }, chain[0].url, execFileFails, chain); + expect(await resolveGoModuleRoot("example.com/mod/cmd/tool", ".", ctx, [])).toBe("example.com/mod"); + expect(seen).toContain("https://without-latest/example.com/mod/@latest"); + expect(seen).toContain("https://backup/example.com/mod/@latest"); +}); + test("resolveGoModuleRoot returns null when nothing resolves and throws when a probe fails", async () => { - // 404 is the probe's legitimate "not the module root"; a 429, a 5xx or a network failure - // answers nothing, and dropping the tool on one reads as "up to date". const path = "golang.org/x/vuln/cmd/govulncheck"; expect(await resolveGoModuleRoot(path, ".", rootCtx(rootHit("nothing")), [])).toBeNull(); const rateLimited = rootCtx(() => Promise.resolve({ok: false, status: 429, statusText: "Too Many Requests"} as any)); await expect(resolveGoModuleRoot(path, ".", rateLimited, [])).rejects.toThrow("429"); await expect(resolveGoModuleRoot(path, ".", rootCtx(() => Promise.reject(new Error("network"))), [])).rejects.toThrow("network"); - // A cold proxy path is slow, not an answer, so the retry decides the root. let calls = 0; const slow = rootCtx(url => ++calls === 1 ? Promise.reject(Object.assign(new Error("timeout"), {transient: true})) : @@ -176,55 +169,44 @@ test("resolveGoModuleRoot returns null when nothing resolves and throws when a p expect(await resolveGoModuleRoot(path, ".", slow, [])).toBe(path); }); +test("resolveGoModuleRoot uses VCS origin metadata through a direct fallback", async () => { + const moduleRoot = "golang.org/x/vuln"; + const seen: Array = []; + const execFile = (_file: string, args: Array, opts: Record) => { + expect(args.slice(0, 4)).toEqual(["list", "-m", "-e", "-json"]); + expect(opts.env.GOPROXY).toBe("direct"); + const candidate = args.at(-1)!.replace(/@latest$/, ""); + return candidate === moduleRoot ? Promise.resolve({stdout: JSON.stringify({Path: candidate, Version: "v1.2.0"}), stderr: ""}) : goListMiss(candidate); + }; + const chain: Array = [{url: "https://empty", fallback: ","}, {url: "direct", fallback: ","}]; + const ctx = rootCtx(url => { + seen.push(url); + return Promise.resolve({ok: false, status: 404} as any); + }, chain[0].url, execFile, chain); + expect(await resolveGoModuleRoot(`${moduleRoot}/cmd/govulncheck`, ".", ctx, [])).toBe(moduleRoot); + expect(seen).toContain(`https://empty/${moduleRoot}/@latest`); +}); + test("resolveGoModuleRoot never builds a proxy URL for off, direct or a GONOPROXY match", async () => { let fetched = false; - const ctx = (goProxyUrl: string) => rootCtx(() => { fetched = true; return Promise.resolve({ok: true} as any); }, goProxyUrl); + const execFile = (_file: string, args: Array) => goListMiss(args.at(-1)!.replace(/@latest$/, "")); + const ctx = (goProxyUrl: string) => rootCtx(() => { fetched = true; return Promise.resolve({ok: true} as any); }, goProxyUrl, execFile); expect(await resolveGoModuleRoot("golang.org/x/vuln/cmd/govulncheck", ".", ctx("off"), [])).toBeNull(); expect(await resolveGoModuleRoot("golang.org/x/vuln/cmd/govulncheck", ".", ctx("direct"), [])).toBeNull(); expect(await resolveGoModuleRoot("git.corp.example/x/cmd/tool", ".", ctx("https://proxy"), ["git.corp.example"])).toBeNull(); expect(fetched).toBe(false); }); -// fetchMakeInfo - -const goProxyCtx = (resolves: string, Version = "", Time = ""): ModeContext => ({ - fetchTimeout, goProbeTimeout, goProxyUrl: "https://proxy", noCache: true, - goProxyChain: [{url: "https://proxy", fallback: ","}], - doFetch: (url: string) => Promise.resolve({ok: url.includes(resolves), status: 404, json: () => Promise.resolve({Version, Time})} as any), -} as unknown as ModeContext); - -test("fetchMakeInfo resolves the latest version and preserves the install path", async () => { - const ctx = goProxyCtx("golangci-lint/v2/@latest", "v2.15.0", "2026-05-01T00:00:00Z"); - expect(await fetchMakeInfo("github.com/golangci/golangci-lint/v2/cmd/golangci-lint", "v2.12.2", ".", ctx, [], defaultOpts)).toEqual({ - newInstallPath: "github.com/golangci/golangci-lint/v2/cmd/golangci-lint", - newVersion: "v2.15.0", - date: "2026-05-01T00:00:00Z", - info: "https://github.com/golangci/golangci-lint", - }); -}); - -test("fetchMakeInfo upgrades a pseudo-version to a newer release", async () => { - const ctx = goProxyCtx("pseudoupd/@latest", "v1.5.0", "2026-02-01T00:00:00Z"); - expect(await fetchMakeInfo("github.com/example/pseudoupd", "v0.0.0-20221128193559-754e69321358", ".", ctx, [], defaultOpts)).toEqual({ - newInstallPath: "github.com/example/pseudoupd", - newVersion: "v1.5.0", - date: "2026-02-01T00:00:00Z", - info: "https://github.com/example/pseudoupd", - }); -}); - -test.each([ - ["the module cannot be resolved", "golang.org/x/vuln/cmd/govulncheck", "v1.2.0", "nothing/@latest", "", ""], - ["a pseudo-version would be downgraded to a lower release", "github.com/example/pseudopkg", - "v0.4.2-0.20230802210424-5b0b94c5c0d3", "pseudopkg/@latest", "v0.4.1", "2026-01-01T00:00:00Z"], - ["a partial version stays the same after precision formatting", "github.com/example/dlv", "v1", - "example/dlv/@latest", "v1.25.2", "2026-03-01T00:00:00Z"], -])("fetchMakeInfo returns null when %s", async (_name, installPath, version, resolves, latest, time) => { - const ctx = goProxyCtx(resolves, latest, time); - expect(await fetchMakeInfo(installPath, version, ".", ctx, [], defaultOpts)).toBeNull(); +test("resolveGoModuleRoot surfaces direct and GONOPROXY lookup failures", async () => { + const failure = (reason: string) => Promise.resolve({stdout: JSON.stringify({ + Path: "git.corp.example/x/cmd/tool", Version: "latest", Error: {Err: reason}, + }), stderr: ""}); + const direct = rootCtx(() => Promise.resolve({ok: true} as any), "direct", () => failure("dial tcp: lookup failed")); + await expect(resolveGoModuleRoot("git.corp.example/x/cmd/tool", ".", direct, [])).rejects.toThrow("lookup failed"); + const noProxy = rootCtx(() => Promise.resolve({ok: true} as any), "https://proxy", () => failure("authentication required")); + await expect(resolveGoModuleRoot("git.corp.example/x/cmd/tool", ".", noProxy, ["git.corp.example"])).rejects.toThrow("authentication required"); }); -// parseMakeImageValue / parseMakeDockerImages const digestA = `sha256:${"a".repeat(64)}`; const digestB = `sha256:${"b".repeat(64)}`; @@ -234,12 +216,9 @@ test("parseMakeImageValue parses a Hub image with registry prefix and digest", ( ref: {registry: null, namespace: "koalaman", repo: "shellcheck", tag: "v0.11.0", fullImage: "koalaman/shellcheck"}, digest: digestA, }); -}); - -test("parseMakeImageValue skips library images, host:port and non-Hub registries", () => { - expect(parseMakeImageValue("mysql:3306")).toBeNull(); // host:port, library namespace - expect(parseMakeImageValue("golang:1.21")).toBeNull(); // bare library image - expect(parseMakeImageValue("ghcr.io/foo/bar:1.2.3")).toBeNull(); // non-Hub registry + expect(parseMakeImageValue("mysql:3306")).toBeNull(); + expect(parseMakeImageValue("golang:1.21")).toBeNull(); + expect(parseMakeImageValue("ghcr.io/foo/bar:1.2.3")).toBeNull(); expect(parseMakeImageValue("plain-no-tag")).toBeNull(); }); @@ -247,87 +226,15 @@ test("parseMakeDockerImages extracts only namespaced Hub images, skipping commen const content = [ `SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.11.0@${digestA} # renovate: datasource=docker`, "PLAIN := koalaman/shellcheck:0.9.0", + "IMAGES += \\", + " \"koalaman/shellcheck:0.10.0\" koalaman/shellcheck:0.11.0", "MYSQL_HOST ?= mysql:3306", `# DISABLED := koalaman/shellcheck:0.1.0@${digestB}`, ].join("\n"); expect(parseMakeDockerImages(content).map(i => ({image: i.writtenImage, tag: i.ref.tag, digest: i.digest}))).toEqual([ {image: "docker.io/koalaman/shellcheck", tag: "v0.11.0", digest: digestA}, {image: "koalaman/shellcheck", tag: "0.9.0", digest: null}, + {image: "koalaman/shellcheck", tag: "0.10.0", digest: null}, + {image: "koalaman/shellcheck", tag: "0.11.0", digest: null}, ]); }); - -// fetchMakeDockerInfo -function dockerHubCtx(): ModeContext { - return { - dockerApiUrl: "https://hub.docker.com", fetchTimeout, noCache: true, - doFetch: (url: string) => { - if (url.includes("/tags/v0.12.0")) return Promise.resolve({ok: true, json: () => Promise.resolve({digest: digestB})} as any); - if (url.includes("/tags")) return Promise.resolve({ok: true, json: () => Promise.resolve({count: 2, results: [ - {name: "v0.11.0", tag_last_pushed: "2025-01-01T00:00:00Z"}, - {name: "v0.12.0", tag_last_pushed: "2025-06-01T00:00:00Z"}, - ]})} as any); - return Promise.resolve({ok: false} as any); - }, - } as unknown as ModeContext; -} - -test("fetchMakeDockerInfo bumps the tag and re-resolves the digest", async () => { - const image = parseMakeImageValue(`docker.io/koalaman/shellcheck:v0.11.0@${digestA}`)!; - expect(await fetchMakeDockerInfo(image, dockerHubCtx(), defaultOpts)).toEqual({ - newTag: "v0.12.0", - newDigest: digestB, - date: "2025-06-01T00:00:00Z", - info: "https://hub.docker.com/r/koalaman/shellcheck", - }); -}); - -test("fetchMakeDockerInfo bumps the tag only when no digest is pinned", async () => { - const image = parseMakeImageValue("koalaman/shellcheck:v0.11.0")!; - expect(await fetchMakeDockerInfo(image, dockerHubCtx(), defaultOpts)).toEqual({ - newTag: "v0.12.0", - newDigest: null, - date: "2025-06-01T00:00:00Z", - info: "https://hub.docker.com/r/koalaman/shellcheck", - }); -}); - -test("fetchMakeDockerInfo returns null when the new tag's digest cannot be resolved", async () => { - const ctx = { - dockerApiUrl: "https://hub.docker.com", fetchTimeout, noCache: true, - doFetch: (url: string) => { - if (url.includes("/tags/v0.12.0")) return Promise.resolve({ok: false} as any); // digest lookup fails - if (url.includes("/tags")) return Promise.resolve({ok: true, json: () => Promise.resolve({count: 2, results: [ - {name: "v0.11.0", tag_last_pushed: "2025-01-01T00:00:00Z"}, - {name: "v0.12.0", tag_last_pushed: "2025-06-01T00:00:00Z"}, - ]})} as any); - return Promise.resolve({ok: false} as any); - }, - } as unknown as ModeContext; - const image = parseMakeImageValue(`docker.io/koalaman/shellcheck:v0.11.0@${digestA}`)!; - expect(await fetchMakeDockerInfo(image, ctx, defaultOpts)).toBeNull(); -}); - -test("fetchMakeDockerInfo returns null when the registry publishes no tag at the authored precision", async () => { - // Pinning a floating `v0.12` to a 3-part tag swaps the deployment policy the author chose, - // which renovate's docker isCompatible refuses too. - const ctx = { - dockerApiUrl: "https://hub.docker.com", fetchTimeout, noCache: true, - doFetch: (url: string) => { - if (url.includes("/tags")) return Promise.resolve({ok: true, json: () => Promise.resolve({count: 2, results: [ - {name: "v0.12.0", tag_last_pushed: "2025-01-01T00:00:00Z"}, - {name: "v0.13.0", tag_last_pushed: "2025-06-01T00:00:00Z"}, - ]})} as any); - return Promise.resolve({ok: false} as any); - }, - } as unknown as ModeContext; - const image = parseMakeImageValue(`docker.io/koalaman/shellcheck:v0.12@${digestA}`)!; - expect(await fetchMakeDockerInfo(image, ctx, defaultOpts)).toBeNull(); -}); - -test("fetchMakeDockerInfo returns null when no newer tag exists", async () => { - const ctx = { - dockerApiUrl: "https://hub.docker.com", fetchTimeout, noCache: true, - doFetch: () => Promise.resolve({ok: true, json: () => Promise.resolve({count: 1, results: [{name: "v0.11.0", tag_last_pushed: "2025-01-01T00:00:00Z"}]})} as any), - } as unknown as ModeContext; - expect(await fetchMakeDockerInfo(parseMakeImageValue("koalaman/shellcheck:v0.11.0")!, ctx, defaultOpts)).toBeNull(); -}); diff --git a/modes/make.ts b/modes/make.ts index f819733..bf488d2 100644 --- a/modes/make.ts +++ b/modes/make.ts @@ -1,11 +1,8 @@ import {env} from "node:process"; -import {type ModeContext, stripv, formatVersionPrecision, findNewVersion, fetchWithRetry} from "./shared.ts"; -import {longestFirstAlternation, tryOrNull} from "../utils/utils.ts"; -import {goModulePathForVersion, fetchGoLatestOnce, fetchGoProxyInfo, getGoInfoUrl, isGoNoProxy, goProxyHeaders} from "./go.ts"; -import { - type DockerImageRef, - parseDockerImageRef, fetchDockerInfo, findDockerVersion, getDockerInfoUrl, fetchDockerTagDigest, -} from "./docker.ts"; +import {type ModeContext, fetchWithRetry} from "./shared.ts"; +import {longestFirstAlternation} from "../utils/utils.ts"; +import {fetchFromGoProxyChain, fetchGoLatestOnce, isGoNoProxy, goProxyHeaders} from "./go.ts"; +import {type DockerImageRef, parseDockerImageRef} from "./docker.ts"; export const makeExactFileNames = ["Makefile", "makefile", "GNUmakefile"]; @@ -15,187 +12,147 @@ export function isMakeFileName(filename: string): boolean { export type MakeInstall = {installPath: string, version: string}; -// Variable assignment holding a single `go install` spec, e.g. -// AIR_PACKAGE ?= github.com/air-verse/air@v1.65.1 -// Captures: 1=install path, 2=version. Assignment operators: = := ::= ?= += -const makeAssignPrefix = String.raw`^\s*[A-Za-z_][\w.]*\s*(?:::=|:=|\?=|\+=|=)\s*`; -const makeAssignRe = new RegExp(`${makeAssignPrefix}${String.raw`(\S+)@(v\d\S*)\s*$`}`); -// Module path must start with a host segment containing a dot (github.com, golang.org, …) +const makeAssignRe = /^\s*[A-Za-z_][\w.]*\s*(?:::=|:=|\?=|\+=|=)\s*(.*)$/; +const makeGoInstallRe = /^([^@\s]+)@(v\d\S*)$/; const goHostRe = /^[^/\s]+\.[^/\s]+\//; -// Strip Make comments transparently: everything from the first `#`. -function stripComment(line: string): string { - const idx = line.indexOf("#"); - return idx === -1 ? line : line.slice(0, idx); +function* makeAssignmentValues(content: string): Generator { + let logicalLine = ""; + for (const rawLine of [...content.split(/\r?\n/), ""]) { + const commentIndex = rawLine.indexOf("#"); + const line = commentIndex === -1 ? rawLine : rawLine.slice(0, commentIndex); + let backslashes = 0; + while (line[line.length - backslashes - 1] === "\\") backslashes++; + if (backslashes % 2) { + logicalLine += `${line.slice(0, -1)} `; + continue; + } + const assignment = makeAssignRe.exec(logicalLine + line); + logicalLine = ""; + if (!assignment) continue; + let quote = ""; + let value = ""; + for (const char of assignment[1]) { + if (quote) { + if (char === quote) quote = ""; + else value += char; + } else if (char === "\"" || char === "'") { + quote = char; + } else if (/\s/.test(char)) { + if (value) yield value; + value = ""; + } else { + value += char; + } + } + if (value && !quote) yield value; + } } export function parseMakeGoInstalls(content: string): Array { const installs: Array = []; - for (const rawLine of content.split(/\r?\n/)) { - const line = stripComment(rawLine); - const match = makeAssignRe.exec(line); - if (!match) continue; + for (const value of makeAssignmentValues(content)) { + const match = makeGoInstallRe.exec(value); + if (!match || !goHostRe.test(match[1])) continue; const [, installPath, version] = match; - if (!goHostRe.test(installPath)) continue; installs.push({installPath, version}); } return installs; } export type MakeDockerImage = { - writtenImage: string, // image part exactly as authored, may include a `docker.io/` prefix - ref: DockerImageRef, // normalized for Docker Hub resolution (registry stripped); ref.tag holds the tag - digest: string | null, // `sha256:…` pin if present + writtenImage: string, + ref: DockerImageRef, + digest: string | null, }; -// Variable assignment holding a single container image, e.g. -// SHELLCHECK_IMAGE ?= docker.io/koalaman/shellcheck:v0.11.0@sha256:61862… -const makeImageRe = new RegExp(`${makeAssignPrefix}${String.raw`(\S+)\s*$`}`); const makeImageDigestRe = /@(sha256:[0-9a-f]{64})$/; -// Reassemble a `[registry/]namespace/repo:tag[@sha256:…]` spec exactly as authored. export function formatMakeImageSpec(writtenImage: string, tag: string, digest: string | null): string { return `${writtenImage}:${tag}${digest ? `@${digest}` : ""}`; } export function parseMakeImageValue(value: string): MakeDockerImage | null { - let digest: string | null = null; - let imageWithTag = value; const digestMatch = makeImageDigestRe.exec(value); - if (digestMatch) { - digest = digestMatch[1]; - imageWithTag = value.slice(0, digestMatch.index); - } - // `docker.io/` is Docker Hub; strip it for resolution but keep it in writtenImage. + const digest = digestMatch?.[1] ?? null; + const imageWithTag = digestMatch ? value.slice(0, digestMatch.index) : value; const ref = parseDockerImageRef(imageWithTag.replace(/^docker\.io\//, "")); - // Require a Hub namespace: skips bare library images and `host:port` vars (mysql:3306). if (!ref || ref.registry || ref.namespace === "library") return null; - const writtenImage = imageWithTag.slice(0, imageWithTag.lastIndexOf(":")); - return {writtenImage, ref, digest}; + return {writtenImage: imageWithTag.slice(0, imageWithTag.lastIndexOf(":")), ref, digest}; } export function parseMakeDockerImages(content: string): Array { - const images: Array = []; - for (const rawLine of content.split(/\r?\n/)) { - const match = makeImageRe.exec(stripComment(rawLine)); - if (!match) continue; - const image = parseMakeImageValue(match[1]); - if (image) images.push(image); - } - return images; + return Array.from(makeAssignmentValues(content)).flatMap(value => parseMakeImageValue(value) ?? []); } -// Module root is some prefix of the install path. A `/vN` segment (v2+ only, per -// Go's module-path convention) marks the boundary without a lookup; otherwise probe -// prefixes longest-first and take the longest that resolves as a module. v0/v1 carry -// no path suffix, so a literal /v0 or /v1 segment is an ordinary directory. const midMajorRe = /\/v(?:[2-9]|[1-9]\d+)(?=\/|$)/; -export function moduleRootFromMajor(installPath: string): string | null { - const match = midMajorRe.exec(installPath); - return match ? installPath.slice(0, match.index + match[0].length) : null; -} - -// Does this path resolve as a module? `direct` and a GONOPROXY match go through the VCS, as -// fetchGoProxyInfo routes them: neither literal is a URL, so interpolating one builds a nonsense -// address whose failure would silently drop every tool in the file. -async function probeGoModuleRoot(candidate: string, goCwd: string, ctx: ModeContext, useVcs: boolean): Promise { - if (useVcs) { - // `go list` exits non-zero for a path that is no module and for an unreachable host alike. - return await tryOrNull(ctx.execFile("go", ["list", "-m", "-json", `${candidate}@latest`], {timeout: ctx.fetchTimeout, cwd: goCwd, env})) !== null; - } - // The root decides which module the tool tracks, so this is the lookup itself, sharing its request - // through fetchGoLatestOnce and its retries. null is a 404/410, a throw a 429 or 5xx on the tool. - const probeFetch = (url: string) => fetchWithRetry(ctx, url, {headers: goProxyHeaders}); - return Boolean(await fetchGoLatestOnce(ctx, probeFetch, ctx.goProxyUrl, candidate)); +async function probeGoModuleRoot(candidate: string, goCwd: string, ctx: ModeContext, chain: ModeContext["goProxyChain"]): Promise { + return await fetchFromGoProxyChain(chain, async url => { + if (url === "off") return false; + if (url !== "direct") { + return await fetchGoLatestOnce( + ctx, requestUrl => fetchWithRetry(ctx, requestUrl, {headers: goProxyHeaders}), url, candidate, + ) ? true : null; + } + let stdout: string; + try { + ({stdout} = await ctx.execFile("go", ["list", "-m", "-e", "-json", `${candidate}@latest`], { + timeout: ctx.fetchTimeout, cwd: goCwd, env: {...env, GOPROXY: "direct"}, + })); + } catch (err: any) { + const reason = String(err?.stderr ?? "").trim().split("\n")[0] || err?.message || String(err); + throw new Error(`go list -m ${candidate}@latest failed: ${reason}`); + } + let result: {Version?: string, Error?: {Err?: string}, Origin?: unknown}; + try { + result = JSON.parse(stdout); + } catch { + throw new Error(`go list -m ${candidate}@latest returned malformed JSON`); + } + if (result.Error) { + const reason = result.Error.Err; + if (result.Origin && reason?.endsWith('no matching versions for query "latest"')) return null; + throw new Error(`go list -m ${candidate}@latest failed: ${reason || "unknown error"}`); + } + if (typeof result.Version !== "string") throw new Error(`go list -m ${candidate}@latest returned malformed JSON`); + return true; + }) ?? false; } export async function resolveGoModuleRoot(installPath: string, goCwd: string, ctx: ModeContext, goNoProxy: Array): Promise { - const heuristic = moduleRootFromMajor(installPath); - if (heuristic) return heuristic; - const useVcs = ctx.goProxyUrl === "direct" || isGoNoProxy(installPath, goNoProxy); - if (!useVcs && ctx.goProxyUrl === "off") return null; // go looks nothing up, so neither does the probe + const major = midMajorRe.exec(installPath); + if (major) return installPath.slice(0, major.index + major[0].length); + const chain = isGoNoProxy(installPath, goNoProxy) ? [{url: "direct", fallback: ","} as const] : + ctx.goProxyChain; + if (chain[0].url === "off") return null; const parts = installPath.split("/"); const candidates = Array.from({length: parts.length - 1}, (_, idx) => parts.slice(0, parts.length - idx).join("/")); - if (useVcs) { // one at a time: each of these probes is a `go list -m` subprocess inside an outer fan + if (chain[0].url === "direct") { for (const candidate of candidates) { - if (await probeGoModuleRoot(candidate, goCwd, ctx, true)) return candidate; + if (await probeGoModuleRoot(candidate, goCwd, ctx, chain)) return candidate; } return null; } - // All at once, and a failure holds its place: it may hide the root a shorter hit would replace. - const probes = await Promise.allSettled(candidates.map(candidate => probeGoModuleRoot(candidate, goCwd, ctx, false))); - for (const [idx, probe] of probes.entries()) { - if (probe.status === "rejected") throw probe.reason; - if (probe.value) return candidates[idx]; - } - return null; -} - -export type MakeUpdate = { - newInstallPath: string, - newVersion: string, - date: string, - info: string, -}; - -export type MakeVersionOpts = { - semvers: Set, - useGreatest: boolean, - usePre: boolean, - useRel: boolean, - allowDowngrade: boolean, - pinnedRange?: string, - pinNoDowngrade?: boolean, - cooldownDays?: number, - now?: number, -}; - -export async function fetchMakeInfo(installPath: string, version: string, goCwd: string, ctx: ModeContext, goNoProxy: Array, opts: MakeVersionOpts): Promise { - const modulePath = await resolveGoModuleRoot(installPath, goCwd, ctx, goNoProxy); - if (!modulePath) return null; - - const [data] = await fetchGoProxyInfo(modulePath, "tool", stripv(version), goCwd, ctx, goNoProxy); - - // Route through the same selection as the go mode so downgrades, pseudo-versions, - // prereleases, pins and cooldowns are handled identically. - const newVersion = findNewVersion(data, {...opts, mode: "go", range: stripv(version)}); - if (!newVersion) return null; - - const newModulePath = data.newPath ?? goModulePathForVersion(modulePath, newVersion); - const newInstallPath = `${newModulePath}${installPath.slice(modulePath.length)}`; - const newVersionFormatted = formatVersionPrecision(newVersion, version); - if (newInstallPath === installPath && newVersionFormatted === version) return null; - return {newInstallPath, newVersion: newVersionFormatted, date: data.Time ?? "", info: getGoInfoUrl(newModulePath)}; -} - -export type MakeDockerUpdate = {newTag: string, newDigest: string | null, date: string, info: string}; - -export async function fetchMakeDockerInfo(image: MakeDockerImage, ctx: ModeContext, opts: MakeVersionOpts): Promise { - const {namespace, repo, fullImage, tag} = image.ref; - const [data] = await fetchDockerInfo(fullImage, ctx); // throws for non-Hub registries - const result = findDockerVersion(data.tags, tag, opts.semvers, opts.cooldownDays, opts.now, opts.pinnedRange, opts.usePre, opts.useRel); - if (!result) return null; - - let newDigest: string | null = null; - if (image.digest) { - // Resolve the digest for the tag actually being written, and skip rather than write a - // stale one — a tag paired with another tag's digest silently pulls the wrong image. - newDigest = await fetchDockerTagDigest(namespace, repo, result.newTag, ctx); - if (!newDigest) return null; + const directIndex = chain.findIndex(({url}) => url === "direct"); + const proxyChain = directIndex === -1 ? chain : chain.slice(0, directIndex); + const probes = await Promise.allSettled(candidates.map(candidate => probeGoModuleRoot(candidate, goCwd, ctx, proxyChain))); + const failed = probes.find(probe => probe.status === "rejected"); + if (failed) throw failed.reason; + const firstHit = probes.findIndex(probe => probe.status === "fulfilled" && probe.value); + if (directIndex !== -1) { + for (const candidate of candidates.slice(0, firstHit === -1 ? undefined : firstHit)) { + if (await probeGoModuleRoot(candidate, goCwd, ctx, chain.slice(directIndex))) return candidate; + } } - return {newTag: result.newTag, newDigest, date: result.date, info: getDockerInfoUrl(image.ref)}; + return firstHit === -1 ? null : candidates[firstHit]; } export type MakeRewrite = {oldSpec: string, newSpec: string}; -// The outer pass hands the inner one each line's code portion alone, so an occurrence inside a -// comment is never rewritten, while the leading boundary keeps a bare `ns/img:tag` out of a -// `docker.io/ns/img:tag` elsewhere in the file. export function updateMakefile(content: string, rewrites: Array): string { const bySpec = new Map(rewrites.map(({oldSpec, newSpec}) => [oldSpec, newSpec])); if (!bySpec.size) return content; - const specs = longestFirstAlternation(bySpec.keys()); - const specRe = new RegExp(`(? code.replace(specRe, spec => bySpec.get(spec)!)); } diff --git a/modes/npm.test.ts b/modes/npm.test.ts index 491e338..aace8d9 100644 --- a/modes/npm.test.ts +++ b/modes/npm.test.ts @@ -1,42 +1,43 @@ -import {mkdtempSync, writeFileSync} from "node:fs"; +import {mkdirSync, mkdtempSync, rmSync, writeFileSync} from "node:fs"; import {join} from "node:path"; import {tmpdir} from "node:os"; +import {env, platform} from "node:process"; import { - isJsr, isLocalDep, isCatalogRef, parseJsrDependency, parseNpmAlias, updateVersionRange, normalizeRange, resolutionsBasePackage, - updatePackageJson, fetchJsrInfo, getLatestCommit, getTags, checkUrlDep, fetchNpmInfo, + checkUrlDep, fetchJsrInfo, fetchNpmInfo, getLatestCommit, getTags, isCatalogRef, isJsr, isLocalDep, normalizeRange, + parseJsrDependency, parseNpmAlias, resolutionsBasePackage, updatePackageJson, updateVersionRange, } from "./npm.ts"; import {type ModeContext, fetchTimeout, fieldSep} from "./shared.ts"; -test("isJsr", () => { - expect(isJsr("npm:@jsr/std__semver@1.0.5")).toBe(true); - expect(isJsr("jsr:@std/semver@1.0.5")).toBe(true); - expect(isJsr("jsr:1.0.5")).toBe(true); - expect(isJsr("^1.0.0")).toBe(false); - expect(isJsr("npm:something")).toBe(false); - expect(isJsr("")).toBe(false); -}); - -test("isLocalDep", () => { - expect(isLocalDep("link:../foo")).toBe(true); - expect(isLocalDep("file:./bar")).toBe(true); - expect(isLocalDep("^1.0.0")).toBe(false); - expect(isLocalDep("")).toBe(false); +test("dependency reference classifiers", () => { + for (const [value, expected] of [["npm:@jsr/std__semver@1.0.5", true], ["jsr:@std/semver@1.0.5", true], + ["jsr:1.0.5", true], ["^1.0.0", false], ["npm:something", false], ["", false]] as const) { + expect(isJsr(value)).toBe(expected); + } + for (const [value, expected] of [["link:../foo", true], ["file:./bar", true], ["^1.0.0", false], ["", false]] as const) { + expect(isLocalDep(value)).toBe(expected); + } + for (const [value, expected] of [["catalog:", true], ["catalog:tools", true], ["^1.0.0", false]] as const) { + expect(isCatalogRef(value)).toBe(expected); + } }); test("parseNpmAlias", () => { expect(parseNpmAlias("npm:left-pad@^1.2.0")).toEqual({name: "left-pad", range: "^1.2.0"}); expect(parseNpmAlias("npm:@hapi/hapi@18.3.0")).toEqual({name: "@hapi/hapi", range: "18.3.0"}); - expect(parseNpmAlias("npm:left-pad@latest")).toBeNull(); // a dist-tag is no range to move + for (const [range, updated] of [ + ["~>1.2.3", "~>2.0.0"], + ["*.*.*", "*.*.*"], + ["1.2.3 - 2.3.x", "1.2.3 - 3.0.x"], + ]) { + const alias = parseNpmAlias(`npm:left-pad@${range}`)!; + expect(alias).toEqual({name: "left-pad", range}); + expect(updateVersionRange(alias.range, range === "1.2.3 - 2.3.x" ? "3.0.0" : "2.0.0", alias.range)).toBe(updated); + } + expect(parseNpmAlias("npm:left-pad@latest")).toBeNull(); expect(parseNpmAlias("npm:left-pad")).toBeNull(); expect(parseNpmAlias("^1.2.0")).toBeNull(); }); -test("isCatalogRef", () => { - expect(isCatalogRef("catalog:")).toBe(true); - expect(isCatalogRef("catalog:tools")).toBe(true); - expect(isCatalogRef("^1.0.0")).toBe(false); -}); - test("parseJsrDependency", () => { expect(parseJsrDependency("npm:@jsr/std__semver@1.0.5")).toEqual({scope: "std", name: "semver", version: "1.0.5"}); expect(parseJsrDependency("jsr:@std/semver@1.0.5")).toEqual({scope: "std", name: "semver", version: "1.0.5"}); @@ -57,62 +58,51 @@ test("updateVersionRange", () => { expect(updateVersionRange("^5.9.0", "6.1.0", "^5.9")).toBe("^6.1"); expect(updateVersionRange("^1.2.3", "1.3.0", undefined)).toBe("^1.3.0"); expect(updateVersionRange("^1.0.0-alpha.1", "1.0.0-beta.2", undefined)).toBe("^1.0.0-beta.2"); - // partial range bumped to a prerelease: keep the full version (can't shrink past major.minor.patch) expect(updateVersionRange("^5.0.0", "6.0.0-beta.1", "^5")).toBe("^6.0.0-beta.1"); expect(updateVersionRange("~1.2.0", "1.3.0-rc.1", "~1.2")).toBe("~1.3.0-rc.1"); expect(updateVersionRange(">=5.0.0", "6.0.0-beta.1", ">=5")).toBe(">=6.0.0-beta.1"); - // a strict bound must admit the new version, never land on it expect(updateVersionRange("<2.0.0", "2.5.0", undefined)).toBe("<3.0.0"); expect(updateVersionRange("<2.1.3", "2.5.0", undefined)).toBe("<2.5.1"); expect(updateVersionRange("< 2.0", "2.5.0", undefined)).toBe("< 2.6"); expect(updateVersionRange("<2", "2.5.0", undefined)).toBe("<3"); - // a strict lower bound already admits it, so it stays as authored expect(updateVersionRange(">1.9.0", "2.5.0", undefined)).toBe(">1.9.0"); expect(updateVersionRange("1.x", "2.0.1", "1.x")).toBe("2.x"); expect(updateVersionRange("1.0.x", "1.1.0", "1.0.x")).toBe("1.1.x"); expect(updateVersionRange("1.*", "2.1.0", "1.*")).toBe("2.*"); expect(updateVersionRange("18.0.0", "19.1.0", "18.0")).toBe("19.1"); - // build metadata describes the version it was authored with, corepack rejects a stale hash expect(updateVersionRange("9.0.0+sha512.0f5b", "11.20.0", "9.0.0+sha512.0f5b")).toBe("11.20.0"); -}); - -test("updateVersionRange widens peer and compound ranges", () => { expect(updateVersionRange("^18.0.0", "19.0.0", "^18.0.0", "peerDependencies")).toBe("^18.0.0 || ^19.0.0"); expect(updateVersionRange("^17.0.0 || ^18.0.0", "19.0.0", "^17.0.0 || ^18.0.0", "peerDependencies")).toBe("^17.0.0 || ^18.0.0 || ^19.0.0"); expect(updateVersionRange("^4.0.0", "5.9.2", "^4", "peerDependencies")).toBe("^4 || ^5"); expect(updateVersionRange("^18.0.0", "18.3.1", "^18.0.0", "peerDependencies")).toBe("^18.0.0"); expect(updateVersionRange("<2.0.0", "2.0.1", "<2.0.0", "peerDependencies")).toBe("<3.0.0"); - // a multi-comparator range widens in every dep type, a replace would drop all but the last expect(updateVersionRange(">=1.0.0 <2.0.0", "2.5.0", ">=1.0.0 <2.0.0", "dependencies")).toBe(">=1.0.0 <3.0.0"); expect(updateVersionRange("^1.0.0 || ^2.0.0", "3.0.1", "^1.0.0 || ^2.0.0", "dependencies")).toBe("^1.0.0 || ^2.0.0 || ^3.0.1"); expect(updateVersionRange("1.x >2.0.0", "2.1.0", "1.x >2.0.0", "dependencies")).toBe("1.x >2.0.0"); -}); - -test("updateVersionRange never writes a range the new version fails", () => { const orChain = updateVersionRange("^0.4.0||^1.0.0", "2.0.0", "^0.4.0||^1.0.0", "peerDependencies"); expect(orChain).toBe("^0.4.0||^1.0.0 || ^2.0.0"); expect(updateVersionRange(orChain, "2.0.0", orChain, "peerDependencies")).toBe(orChain); - // `=5.0.0 <7.0.0-0", "7.0.0", ">=5.0.0 <7.0.0-0", "dependencies")).toBe(">=5.0.0 <8.0.0-0"); + expect(updateVersionRange(">=5.0.0 <7.0.0-0", "7.0.0", ">=5.0.0 <7.0.0-0", "dependencies")).toBe(">=5.0.0 <7.0.1"); + expect(updateVersionRange(">=2.0.0 <2.1.0-0", "2.1.0", undefined, "dependencies")).toBe(">=2.0.0 <2.1.1"); + expect(updateVersionRange(">=2.0.0 <2.1.3-0", "2.1.3", undefined, "dependencies")).toBe(">=2.0.0 <2.1.4"); + expect(updateVersionRange("1.0.0", "2.0.0-rc.1", ">1.0.0", "peerDependencies")).toBe(">1.0.0"); expect(updateVersionRange("^1.0.0 <1.5.0", "2.0.0", "^1.0.0 <1.5.0", "dependencies")).toBe("^1.0.0 <1.5.0"); expect(updateVersionRange("~1.0.0 <1.5.0", "2.0.0", "~1.0.0 <1.5.0", "dependencies")).toBe("~1.0.0 <1.5.0"); - expect(updateVersionRange(">=1.0.0 <1.5.0", "2.0.0", ">=1.0.0 <1.5.0", "dependencies")).toBe(">=1.0.0 <2.0.1"); + expect(updateVersionRange(">=1.0.0 <1.5.0", "2.0.0", ">=1.0.0 <1.5.0", "dependencies")).toBe(">=1.0.0 <2.1.0"); expect(updateVersionRange("1.2.3 - 2.3.4", "1.0.1", "1.2.3 - 2.3.4", "dependencies")).toBe("1.2.3 - 2.3.4"); expect(updateVersionRange("1.2.3 - 2.3.4", "3.0.0", "1.2.3 - 2.3.4", "dependencies")).toBe("1.2.3 - 3.0.0"); -}); - -test("updateVersionRange keeps an authored v prefix", () => { expect(updateVersionRange("^v1.0.0", "2.0.0", "^v1.0.0")).toBe("^v2.0.0"); expect(updateVersionRange("~v1.2.0", "1.3.0-rc.1", "~v1.2.0")).toBe("~v1.3.0-rc.1"); }); -test("resolutionsBasePackage", () => { +test("package selector normalization", () => { expect(resolutionsBasePackage("@babel/core")).toBe("@babel/core"); expect(resolutionsBasePackage("config/glob")).toBe("glob"); expect(resolutionsBasePackage("**/@angular/cli")).toBe("@angular/cli"); @@ -120,9 +110,6 @@ test("resolutionsBasePackage", () => { expect(resolutionsBasePackage("foo/bar@1.0.0")).toBe("bar"); expect(resolutionsBasePackage("@verdaccio/core/ajv@8.17.1")).toBe("ajv"); expect(resolutionsBasePackage("foo/@babel/core@7.0.0")).toBe("@babel/core"); -}); - -test("normalizeRange", () => { expect(normalizeRange("^5")).toBe("^5.0.0"); expect(normalizeRange("^5.9")).toBe("^5.9.0"); expect(normalizeRange("^5.9.3")).toBe("^5.9.3"); @@ -147,17 +134,14 @@ test("updatePackageJson", () => { [pmKey]: {old: "8.0.0", new: "9.0.0"}, }); expect(result2).toContain(`"packageManager": "pnpm@9.0.0"`); -}); - -test("updatePackageJson only rewrites the dep's own section", () => { const sections = ["dependencies", "peerDependencies", "overrides", "scripts", "resolutions", "invented"]; - const pkg = JSON.stringify({ + const sectionPkg = JSON.stringify({ ...Object.fromEntries(sections.map(section => [section, {"react": "^18.0.0"}])), pnpm: {overrides: {"react": "^18.0.0"}}, packageManager: "pnpm@9.0.0+sha512.0f5b", }, null, 2); - const result = updatePackageJson(pkg, { + const result = updatePackageJson(sectionPkg, { [`peerDependencies${fieldSep}react`]: {old: "^18.0.0", oldOrig: "^18.0.0", new: "^18.0.0 || ^19.0.0"}, [`packageManager${fieldSep}pnpm`]: {old: "9.0.0+sha512.0f5b", oldOrig: "9.0.0+sha512.0f5b", new: "11.20.0"}, }); @@ -180,8 +164,6 @@ test("updatePackageJson only rewrites the dep's own section", () => { overrides: {"react": "^19.0.0"}, }); - // url deps are re-inserted after the regular ones, so a section's cursor can already be past - // the pair a later dep needs. const outOfOrder = JSON.stringify({ dependencies: {"foo": "github:u/r#v1.0.0", "bar": "^1.0.0"}, optionalDependencies: {"foo": "github:u/r#v1.0.0"}, @@ -200,31 +182,23 @@ test("updatePackageJson only rewrites the dep's own section", () => { const modeCtx = (props: Record): ModeContext => ({fetchTimeout, ...props} as unknown as ModeContext); const forgeCtx = (props: Record) => modeCtx({forgeApiUrl: "https://api.github.com", ...props}); const textRes = (body: unknown) => Promise.resolve({ok: true, text: () => Promise.resolve(JSON.stringify(body)), headers: new Headers()}); +const jsonRes = (body: unknown) => Promise.resolve({ok: true, json: () => Promise.resolve(body), headers: new Headers()}); -// fetchJsrInfo -test("fetchJsrInfo happy path", async () => { +test("fetchJsrInfo", async () => { const jsrData = {latest: "1.0.0", versions: {"1.0.0": {createdAt: "2025-01-01T00:00:00Z"}, "0.9.0": {createdAt: "2024-06-01T00:00:00Z"}}}; - const ctx = modeCtx({jsrApiUrl: "https://jsr.io", doFetch: () => Promise.resolve({ok: true, json: () => Promise.resolve(jsrData)})}); + const ctx = modeCtx({jsrApiUrl: "https://jsr.io", doFetch: () => textRes(jsrData)}); const [data, registry] = await fetchJsrInfo("@std/semver", ctx); expect(registry).toBe("https://jsr.io"); expect(data.name).toBe("@std/semver"); expect(data["dist-tags"].latest).toBe("1.0.0"); expect(Object.keys(data.versions)).toEqual(["1.0.0", "0.9.0"]); expect(data.time["1.0.0"]).toBe("2025-01-01T00:00:00Z"); -}); - -test("fetchJsrInfo invalid package name throws", async () => { - const ctx = {} as unknown as ModeContext; - await expect(fetchJsrInfo("noscopepkg", ctx)).rejects.toThrow("Invalid JSR package name"); -}); - -test("fetchJsrInfo fetch failure throws", async () => { - const ctx = modeCtx({jsrApiUrl: "https://jsr.io", + await expect(fetchJsrInfo("noscopepkg", {} as ModeContext)).rejects.toThrow("Invalid JSR package name"); + const failureCtx = modeCtx({jsrApiUrl: "https://jsr.io", doFetch: () => Promise.resolve({ok: false, status: 404, statusText: "Not Found"})}); - await expect(fetchJsrInfo("@std/semver", ctx)).rejects.toThrow("404"); + await expect(fetchJsrInfo("@std/semver", failureCtx)).rejects.toThrow("404"); }); -// fetchNpmInfo test("fetchNpmInfo resolutions key keeps scope", async () => { let fetchedUrl = ""; const ctx = modeCtx({noCache: true, doFetch: (url: string) => { @@ -232,33 +206,67 @@ test("fetchNpmInfo resolutions key keeps scope", async () => { return textRes({}); }}); await fetchNpmInfo("@babel/core", "resolutions", {}, {}, ctx); - // the scope must survive: fetch @babel/core, never the unscoped `core` expect(fetchedUrl.endsWith("/@babel%2fcore")).toBe(true); - // corepack publishes yarn 2 and up as @yarnpkg/cli, yarn 1 alone stays on `yarn` await fetchNpmInfo("yarn", "packageManager", {}, {}, ctx, undefined, "4.9.2"); expect(fetchedUrl.endsWith("/@yarnpkg%2fcli")).toBe(true); await fetchNpmInfo("yarn", "packageManager", {}, {}, ctx, undefined, "1.22.22"); expect(fetchedUrl.endsWith("/yarn")).toBe(true); - // an `overrides` key is a selector like a `resolutions` one, never a name to request verbatim await fetchNpmInfo("noty@3", "overrides", {}, {}, ctx); expect(fetchedUrl.endsWith("/noty")).toBe(true); }); -test("fetchNpmInfo reads .npmrc from the manifest dir and honors an uncredentialed scoped registry", async () => { - const dir = mkdtempSync(join(tmpdir(), "updates-npmrc-")); - writeFileSync(join(dir, ".npmrc"), "registry=https://default.test\n@myorg:registry=https://private.test\n"); +test.each([ + ["npmrc scoped registry", { + ".npmrc": "registry=https://default.test\n@myorg:registry=https://private.test\n", + }, ["https://private.test/@myorg%2fpkg", "https://default.test/lodash"]], + ["pnpm workspace registries", { + "pnpm-workspace.yaml": "registry: https://pnpm.test\nregistries:\n '@myorg': https://scope.pnpm.test\n", + }, ["https://scope.pnpm.test/@myorg%2fpkg", "https://pnpm.test/lodash"]], +])("fetchNpmInfo honors %s", async (_name, files, expected) => { + const dir = mkdtempSync(join(tmpdir(), "updates-registry-")); const urls: Array = []; const ctx = modeCtx({noCache: true, doFetch: (url: string) => { urls.push(url); return textRes({}); }}); - await fetchNpmInfo("@myorg/pkg", "dependencies", {}, {}, ctx, dir); - await fetchNpmInfo("lodash", "dependencies", {}, {}, ctx, dir); - expect(urls).toEqual(["https://private.test/@myorg%2fpkg", "https://default.test/lodash"]); + try { + for (const [filename, content] of Object.entries(files)) writeFileSync(join(dir, filename), content); + await fetchNpmInfo("@myorg/pkg", "dependencies", {}, {}, ctx, dir); + await fetchNpmInfo("lodash", "dependencies", {}, {}, ctx, dir); + expect(urls).toEqual(expected); + } finally { + rmSync(dir, {recursive: true}); + } +}); + +test("fetchNpmInfo never sends unscoped _auth to a repository registry", async () => { + const dir = mkdtempSync(join(tmpdir(), "updates-auth-")); + const home = join(dir, "home"); + const project = join(dir, "project"); + const homeVar = platform === "win32" ? "USERPROFILE" : "HOME"; + const originalHome = env[homeVar]; + const authorizations: Array = []; + const ctx = modeCtx({noCache: true, doFetch: (_url: string, opts: RequestInit) => { + authorizations.push(new Headers(opts.headers).get("authorization")); + return textRes({}); + }}); + try { + mkdirSync(home); + mkdirSync(project); + writeFileSync(join(home, ".npmrc"), "_auth=dXNlcjpzZWNyZXQ=\n"); + writeFileSync(join(project, ".npmrc"), "registry=https://attacker.example\n"); + env[homeVar] = home; + await fetchNpmInfo("untrusted", "dependencies", {}, {}, ctx, project); + await fetchNpmInfo("trusted", "dependencies", {}, {registry: "https://registry.npmjs.org"}, ctx, project); + expect(authorizations).toEqual([null, "Basic dXNlcjpzZWNyZXQ="]); + } finally { + if (originalHome === undefined) delete env[homeVar]; + else env[homeVar] = originalHome; + rmSync(dir, {recursive: true}); + } }); test("fetchNpmInfo requests the full doc only when dates are needed, never reusing the abbreviated one", async () => { - // the abbreviated doc omits the `time` map, which would make cooldown a silent no-op const accepts: Array = []; const ctx = modeCtx({noCache: true, doFetch: (_url: string, opts: any) => { accepts.push(opts?.headers?.accept); @@ -273,49 +281,58 @@ test("fetchNpmInfo requests the full doc only when dates are needed, never reusi expect(accepts.slice(2)).toEqual(["application/vnd.npm.install-v1+json", undefined]); }); -// getLatestCommit -test("getLatestCommit happy path", async () => { +test("getLatestCommit", async () => { const ctx = forgeCtx({noCache: true, doFetch: () => textRes([{sha: "abc1234567890", commit: {committer: {date: "2025-01-01"}}}])}); const result = await getLatestCommit("user", "repo", ctx); expect(result.hash).toBe("abc1234567890"); expect(result.commit.committer.date).toBe("2025-01-01"); + for (const doFetch of [() => textRes([]), () => Promise.resolve({ok: false})]) { + expect(await getLatestCommit("user", "repo", forgeCtx({doFetch}))).toEqual({hash: "", commit: {}}); + } + await expect(getLatestCommit("user", "repo", forgeCtx({doFetch: () => Promise.reject(new Error("network error"))}))) + .rejects.toThrow(/network error/); }); -test.each([ - ["a repository with no commits", () => textRes([])], - ["a repository that is gone", () => Promise.resolve({ok: false})], -])("getLatestCommit returns empty for %s", async (_name, doFetch) => { - expect(await getLatestCommit("user", "repo", forgeCtx({doFetch}))).toEqual({hash: "", commit: {}}); -}); - -test("getLatestCommit throws on a fetch failure", async () => { - const ctx = forgeCtx({doFetch: () => Promise.reject(new Error("network error"))}); - await expect(getLatestCommit("user", "repo", ctx)).rejects.toThrow(/network error/); -}); - -// getTags test("getTags returns tag names, or none when the fetch fails", async () => { const tagsData = [{name: "v1.0.0", commit: {sha: "abc"}}, {name: "v2.0.0", commit: {sha: "def"}}]; - const ctx = forgeCtx({doFetch: () => Promise.resolve({ok: true, json: () => Promise.resolve(tagsData), headers: new Headers()})}); + const fetched: Array = []; + const ctx = forgeCtx({noCache: true, doFetch: (url: string) => { + fetched.push(url); + if (url.includes("/releases?")) return Promise.resolve({ok: false, status: 500, statusText: "Internal Server Error"}); + return jsonRes(tagsData); + }}); expect(await getTags("user", "repo", "v1.0.0", ctx)).toEqual(["v1.0.0", "v2.0.0"]); + expect(fetched.every(url => url.includes("/tags?"))).toBe(true); expect(await getTags("user", "repo", "v1.0.0", forgeCtx({doFetch: () => Promise.resolve({ok: false})}))).toEqual([]); }); -// checkUrlDep -test("checkUrlDep unparseable URL returns null", async () => { +test("checkUrlDep parses refs and refreshes hashes", async () => { const ctx = forgeCtx({doFetch: () => Promise.resolve({ok: false})}); expect(await checkUrlDep("key", {old: "not-a-url", new: ""} as any, ctx)).toBeNull(); -}); - -test("checkUrlDep hash-based with update", async () => { - const ctx = forgeCtx({noCache: true, doFetch: () => textRes([{sha: "def5678901234", commit: {committer: {date: "2025-03-01"}}}])}); - const result = await checkUrlDep("key", {old: "https://github.com/user/repo/abc1234", new: ""}, ctx); + let fetches = 0; + const hashCtx = forgeCtx({noCache: true, doFetch: () => { + fetches++; + return textRes([{sha: "def5678901234", commit: {committer: {date: "2025-03-01"}}}]); + }}); + const result = await checkUrlDep("key", {old: "github:user/repo#1234567", new: ""}, hashCtx); expect(result).not.toBeNull(); + expect(result!.newRange).toBe("github:user/repo#def5678"); expect(result!.newRef).toBe("def5678"); expect(result!.newDate).toBe("2025-03-01"); + expect(await checkUrlDep("key", {old: "github:user/repo#abc123", new: ""}, hashCtx)).toBeNull(); + expect(fetches).toBe(1); + expect(await checkUrlDep("key", {old: "git+https://github.com/user/repo.git#abc1234", new: ""} as any, + forgeCtx({noCache: true, doFetch: () => textRes([{sha: "abc1234567890", commit: {}}])}))).toBeNull(); }); -test("checkUrlDep hash-based no change returns null", async () => { - const ctx = forgeCtx({noCache: true, doFetch: () => textRes([{sha: "abc1234567890", commit: {}}])}); - expect(await checkUrlDep("key", {old: "https://github.com/user/repo/abc1234", new: ""} as any, ctx)).toBeNull(); +test.each([ + ["github:user/repo#v1.2.3", "github:user/repo#v2.0.0"], + ["git+https://github.com/user/repo.git#v1.2.3-beta.1", "git+https://github.com/user/repo.git#v2.0.0"], + ["git+ssh://git@github.com/user/repo.git#v1.2.3", "git+ssh://git@github.com/user/repo.git#v2.0.0"], + ["git@github.com:user/repo.git#v1.2.3", "git@github.com:user/repo.git#v2.0.0"], + ["github:user/repo#semver:^1", "github:user/repo#semver:^2"], +])("checkUrlDep updates %s", async (old, expected) => { + const tags = [{name: "v1.2.3", commit: {sha: "abc"}}, {name: "v2.0.0", commit: {sha: "def"}}]; + const ctx = forgeCtx({noCache: true, doFetch: (url: string) => jsonRes(url.includes("/releases?") ? [] : tags)}); + expect((await checkUrlDep("key", {old, new: ""}, ctx))?.newRange).toBe(expected); }); diff --git a/modes/npm.ts b/modes/npm.ts index f71a69e..b50db61 100644 --- a/modes/npm.ts +++ b/modes/npm.ts @@ -1,87 +1,31 @@ import {env} from "node:process"; -import {parse, satisfies, validRange} from "../utils/semver.ts"; +import {parse, satisfies, valid, validRange} from "../utils/semver.ts"; import rc from "../utils/rc.ts"; import {getOrSet, tryOrNull} from "../utils/utils.ts"; +import {resolveNativeNpmRegistry} from "../utils/workspace.ts"; import { type Config, type CheckResult, type Dep, type Deps, type ModeContext, type PackageInfo, type PackageRepository, normalizeUrl, getFetchOpts, fieldSep, fetchForgeEtag, selectTag, fetchWithEtag, fetchImmutable, dedupe, - coerceToVersion, hashRe, fetchActionTags, throwFetchError, fetchWithRetry, defaultApiUrls, parseCommitDate, reduceJson, + coerceToVersion, hashRe, fetchForgeTags, throwFetchError, fetchWithRetry, defaultApiUrls, parseCommitDate, + reduceJson, } from "./shared.ts"; -export type Npmrc = { - registry: string, - ca?: string, - cafile?: string, - cert?: string, - certfile?: string, - key?: string, - keyfile?: string, - [other: string]: any, -}; - -export type AuthAndRegistry = { - auth: { - token: string, - type: string, - username?: string | undefined, - password?: string | undefined, - } | undefined, - registry: string, -}; +type Npmrc = Record & {registry: string}; +type AuthAndRegistry = {auth: {token: string, type: string} | undefined, registry: string}; -// regexes for url dependencies. does only github and only hash or exact semver -// https://regex101.com/r/gCZzfK/2 -const stripRe = /^.*?:\/\/(.*?@)?(github\.com[:/])/i; -const partsRe = /^([^/]+)\/([^/]+)\/(?:.*\/)?([0-9a-f]+|v?[0-9]+\.[0-9]+\.[0-9]+)$/i; const npmVersionRe = /[0-9]+(\.[0-9]+)?(\.[0-9]+)?/g; -// matches each path segment incl. its scope, e.g. `foo/@scope/pkg` -> [`foo`, `@scope/pkg`] -const segmentRe = /(@[^/]+\/)?([^/]+)/g; -// the published name inside one segment, dropping yarn's `@range` discriminator: `qs@~6.14.1` -> `qs` -const segmentNameRe = /^(?:@[^/]+\/)?[^@]+/; -// The dep types whose keys are selectors rather than plain package names. export const selectorTypes = new Set(["resolutions", "overrides"]); -// resolves a `resolutions`/`overrides` key to the published package, keeping the scope on the -// final segment, e.g. `@babel/core` -> `@babel/core`, `foo/@scope/pkg` -> `@scope/pkg` export function resolutionsBasePackage(name: string): string { - const segments = name.match(segmentRe); - const last = segments ? segments[segments.length - 1] : name; - return segmentNameRe.exec(last)?.[0] ?? last; + return /(?:^|\/)((?:@[^/]+\/)?[^/@]+)(?:@[^/]*)?$/.exec(name)?.[1] ?? name; } const defaultRegistry = defaultApiUrls.registry; const npmrcCache = new Map(); const authCache = new Map(); -// npm resolves `.npmrc` beside or above the manifest, so `dir` is the manifest's directory and two -// manifests can carry different ones. Omitting it falls back to the cwd. -export function getNpmrc(dir?: string): Npmrc { - return getOrSet(npmrcCache, dir ?? "", () => rc("npm", {registry: defaultRegistry}, dir) as Npmrc); -} - -function replaceEnvVar(token: string): string { - return token.replace(/^\$\{?([^}]*)\}?$/, (_, envVar) => env[envVar] || ""); -} - -function getAuthInfoForUrl(regUrl: string, config: Npmrc): AuthAndRegistry["auth"] { - const get = (key: string) => config[`${regUrl}:${key}`] || config[`${regUrl}/:${key}`]; - - const bearerToken = get("_authToken"); - if (bearerToken) return {token: replaceEnvVar(bearerToken), type: "Bearer"}; - - const username = get("username"); - const password = get("_password"); - if (username && password) { - const pass = Buffer.from(replaceEnvVar(password), "base64").toString("utf8"); - return {token: Buffer.from(`${username}:${pass}`).toString("base64"), type: "Basic", username, password: pass}; - } - - const legacyToken = get("_auth"); - if (legacyToken) return {token: replaceEnvVar(legacyToken), type: "Basic"}; - - return undefined; -} +const replaceEnvVar = (token: string): string => token.replace(/^\$\{?([^}]*)\}?$/, (_, envVar) => env[envVar] || ""); function getRegistryAuthToken(registryUrl: string, config: Npmrc): AuthAndRegistry["auth"] { const parsed = new URL(registryUrl.startsWith("//") ? `http:${registryUrl}` : registryUrl); @@ -90,67 +34,54 @@ function getRegistryAuthToken(registryUrl: string, config: Npmrc): AuthAndRegist while (pathname !== "/" && parsed.pathname !== pathname) { pathname = parsed.pathname || "/"; const regUrl = `//${parsed.host}${pathname.replace(/\/$/, "")}`; - const authInfo = getAuthInfoForUrl(regUrl, config); - if (authInfo) return authInfo; - const normalized = pathname.endsWith("/") ? pathname : `${pathname}/`; - parsed.pathname = new URL("..", new URL(normalized, "http://x")).pathname; + const get = (key: string) => config[`${regUrl}:${key}`] || config[`${regUrl}/:${key}`]; + const bearerToken = get("_authToken"); + if (bearerToken) return {token: replaceEnvVar(bearerToken), type: "Bearer"}; + const username = get("username"); + const password = get("_password"); + if (username && password) { + const pass = Buffer.from(replaceEnvVar(password), "base64").toString("utf8"); + return {token: Buffer.from(`${username}:${pass}`).toString("base64"), type: "Basic"}; + } + const legacyToken = get("_auth"); + if (legacyToken) return {token: replaceEnvVar(legacyToken), type: "Basic"}; + parsed.pathname = new URL("..", new URL(pathname.endsWith("/") ? pathname : `${pathname}/`, "http://x")).pathname; } - // Global legacy fallback - const globalAuth = config["_auth"]; - if (globalAuth) return {token: replaceEnvVar(globalAuth), type: "Basic"}; + if (registryUrl === defaultRegistry && config["_auth"]) return {token: replaceEnvVar(config["_auth"]), type: "Basic"}; return undefined; } -// Never the default registry as a fallback, that would override an explicit --registry. -function scopedRegistry(scope: string, npmrcConfig: Npmrc): string { - const url: string = npmrcConfig[`${scope}:registry`] || ""; - return !url || url.endsWith("/") ? url : `${url}/`; -} - -function getAuthAndRegistry(name: string, registry: string, dir: string | undefined): AuthAndRegistry { - const npmrcConfig = getNpmrc(dir); +function resolveNpmRegistry(name: string, config: Config, args: Record, dir: string | undefined): AuthAndRegistry { + const npmrcConfig = getOrSet(npmrcCache, dir ?? "", () => rc("npm", {registry: defaultRegistry}, dir) as Npmrc); + const registry = normalizeUrl((typeof args.registry === "string" ? args.registry : false) || + config.registry || npmrcConfig.registry || defaultRegistry); const scope = name.startsWith("@") ? name.split("/")[0] : ""; - return getOrSet(authCache, `${dir ?? ""}${fieldSep}${scope}:${registry}`, () => { - // A scope's own registry wins whether or not it carries credentials, as renovate registers the - // scope→registry rule unconditionally: falling back would leak a private name to the wrong host. - let result: AuthAndRegistry | undefined; - const scoped = scope && scopedRegistry(scope, npmrcConfig); + const nativeRegistry = dir ? resolveNativeNpmRegistry(name, dir) : null; + return getOrSet(authCache, `${dir ?? ""}${fieldSep}${scope}:${registry}:${nativeRegistry ?? ""}`, () => { + let resolvedRegistry = nativeRegistry ? normalizeUrl(nativeRegistry) : registry; + const scoped = !nativeRegistry && scope && npmrcConfig[`${scope}:registry`]; if (scoped) { try { const url = normalizeUrl(scoped); - if (url !== registry) result = {auth: getRegistryAuthToken(url, npmrcConfig), registry: url}; + if (url !== registry) resolvedRegistry = url; } catch {} } - return result ?? {auth: getRegistryAuthToken(registry, npmrcConfig), registry}; + return {auth: getRegistryAuthToken(resolvedRegistry, npmrcConfig), registry: resolvedRegistry}; }); } -function resolveNpmRegistry(name: string, config: Config, args: Record, dir: string | undefined): AuthAndRegistry & {originalRegistry: string} { - const originalRegistry = normalizeUrl((typeof args.registry === "string" ? args.registry : false) || - config.registry || getNpmrc(dir).registry || defaultRegistry, - ); - return {...getAuthAndRegistry(name, originalRegistry, dir), originalRegistry}; -} - -function npmPackageUrl(registry: string, name: string, version?: string): string { +const npmPackageUrl = (registry: string, name: string, version?: string): string => { const base = `${registry}/${name.replace(/\//g, "%2f")}`; return version ? `${base}/${version}` : base; -} +}; -// Per run, like go's goLatestByCtx and docker's hubTagsByCtx: a second updates() call in one -// process must re-request rather than answer from the finished run's map. const npmDataByCtx = new WeakMap>>>(); const npmVersionInfoByCtx = new WeakMap>>(); const npmFullDataByCtx = new WeakMap | null>>>(); -// Bumped when the reducer drops or adds a field: an entry an older shape wrote still revalidates -// to a 304, so only a key it was never stored under refetches it. -const docShape = "v2"; -// Keyed by url and doc flavor too, as the abbreviated doc omits fields a later dated call needs. -const docCacheKey = (url: string, needsDates: boolean) => `${url}\0${docShape}${needsDates ? "-dates" : ""}`; +const docCacheKey = (url: string, needsDates: boolean) => `${url}\0v2${needsDates ? "-dates" : ""}`; -// The doc shape is kept, so legacy cache entries with full bodies stay readable. function reduceNpmDoc(data: Record): Record { const versions: Record = {}; for (const version of Object.keys(data.versions ?? {})) versions[version] = data.versions[version]?.deprecated ? {deprecated: true} : {}; @@ -158,20 +89,16 @@ function reduceNpmDoc(data: Record): Record { } export async function fetchNpmInfo(name: string, type: string, config: Config, args: Record, ctx: ModeContext, dir?: string, version = ""): Promise { - // corepack publishes yarn 2 and up as `@yarnpkg/cli`, only yarn 1 lives under `yarn`. renovate's rule. const packageName = selectorTypes.has(type) ? resolutionsBasePackage(name) : type === "packageManager" && name === "yarn" && (parse(version)?.major ?? 0) > 1 ? "@yarnpkg/cli" : name; - // The published name, not the manifest key, is what a scoped `.npmrc` registry and its token key on. const {auth, registry} = resolveNpmRegistry(packageName, config, args, dir); const url = npmPackageUrl(registry, packageName); const cacheKey = docCacheKey(url, Boolean(args.needsDates)); const data = await dedupe(npmDataByCtx, ctx, cacheKey, async () => { const opts = getFetchOpts(auth?.type, auth?.token); - // The abbreviated doc is a fraction of the size but omits the `time` map that cooldown reads. - if (!args.needsDates) { - opts.headers = {...opts.headers as Record, "accept": "application/vnd.npm.install-v1+json"}; - } + if (!args.needsDates) opts.headers = {...opts.headers as Record, + "accept": "application/vnd.npm.install-v1+json"}; const result = await fetchWithEtag(url, ctx, opts, reduceJson(reduceNpmDoc), cacheKey); if (!("body" in result)) throwFetchError(result.res, url, name, registry); return JSON.parse(result.body); @@ -188,8 +115,6 @@ export async function fetchNpmVersionInfo(name: string, version: string, config: return dedupe(npmVersionInfoByCtx, ctx, url, async (): Promise => { try { const fetchOpts = getFetchOpts(auth?.type, auth?.token); - // Per-version npm metadata is immutable — cache forever. - // Undefined fields drop out at JSON.stringify time. const result = await fetchImmutable(url, ctx, fetchOpts, reduceJson(data => ({ repository: data.repository, homepage: data.homepage, @@ -198,16 +123,11 @@ export async function fetchNpmVersionInfo(name: string, version: string, config: if (!("body" in result)) return {}; const data = JSON.parse(result.body); let date = ""; - const tmp: string | undefined = data?._npmOperationalInternal?.tmp; - if (tmp) { - const match = /(\d{13})/.exec(tmp); - if (match) date = new Date(Number(match[1])).toISOString(); - } + const match = /(\d{13})/.exec(data?._npmOperationalInternal?.tmp ?? ""); + if (match) date = new Date(Number(match[1])).toISOString(); const fullUrl = npmPackageUrl(registry, name); - // With a cooldown active the package doc was already fetched in full, under the dated key. if (!date && args.needsDates) date = (await npmDataByCtx.get(ctx)?.get(docCacheKey(fullUrl, true)))?.time?.[version] || ""; if (!date) { - // _npmOperationalInternal is absent on some registries, fetch full metadata const fullData = await tryOrNull(dedupe(npmFullDataByCtx, ctx, fullUrl, async () => { const res = await fetchWithRetry(ctx, fullUrl, fetchOpts); return res?.ok ? await res.json() : null; @@ -229,37 +149,23 @@ export function isLocalDep(value: string): boolean { return value.startsWith("link:") || value.startsWith("file:"); } -// A pnpm catalog reference: `catalog:` for the default catalog, `catalog:` for a named one. export function isCatalogRef(value: string): boolean { return value.startsWith("catalog:"); } -// The `npm:@jsr/…` flavour is a jsr specifier and belongs to isJsr, which callers test first. const npmAliasRe = /^npm:((?:@[^/@]+\/)?[^@/][^@]*)@(.+)$/; -// Null when the value is no alias this tool can resolve, e.g. one naming a dist-tag, not a range. export function parseNpmAlias(value: string): {name: string, range: string} | null { const match = npmAliasRe.exec(value); return match && validRange(match[2]) ? {name: match[1], range: match[2]} : null; } -// Both spellings carrying their own scope, name and version, anchored so they need no prefix check. -const jsrRefRes = [ - /^npm:@jsr\/([^_]+)__([^@]+)@(.+)$/, // npm:@jsr/std__semver@1.0.5 - /^jsr:@([^/]+)\/([^@]+)@(.+)$/, // jsr:@std/semver@1.0.5 -]; +const jsrRefRe = /^(?:npm:@jsr\/([^_]+)__([^@]+)|jsr:@([^/]+)\/([^@]+))@(.+)$/; const jsrScopedNameRe = /^@([^/]+)\/(.+)$/; -// - "npm:@jsr/std__semver@1.0.5" -> { scope: "std", name: "semver", version: "1.0.5" } -// - "jsr:@std/semver@1.0.5" -> { scope: "std", name: "semver", version: "1.0.5" } -// - "jsr:1.0.5" (when package name is known) -> { scope: null, name: null, version: "1.0.5" } export function parseJsrDependency(value: string, packageName?: string): {scope: string | null, name: string | null, version: string} { - for (const re of jsrRefRes) { - const match = re.exec(value); - if (match) return {scope: match[1], name: match[2], version: match[3]}; - } - // A bare `jsr:1.0.5` takes scope and name from the dependency key instead. `jsr:@` is - // excluded so a scoped ref without a version is not read as one. + const ref = jsrRefRe.exec(value); + if (ref) return {scope: ref[1] || ref[3], name: ref[2] || ref[4], version: ref[5]}; if (value.startsWith("jsr:") && !value.startsWith("jsr:@")) { const match = jsrScopedNameRe.exec(packageName ?? ""); if (match) return {scope: match[1], name: match[2], version: value.substring(4)}; @@ -268,9 +174,7 @@ export function parseJsrDependency(value: string, packageName?: string): {scope: } export async function fetchJsrInfo(packageName: string, ctx: ModeContext): Promise { - if (!jsrScopedNameRe.test(packageName)) { - throw new Error(`Invalid JSR package name: ${packageName}`); - } + if (!jsrScopedNameRe.test(packageName)) throw new Error(`Invalid JSR package name: ${packageName}`); const url = `${ctx.jsrApiUrl}/${packageName}/meta.json`; const result = await fetchWithEtag(url, ctx, { @@ -282,7 +186,6 @@ export async function fetchJsrInfo(packageName: string, ctx: ModeContext): Promi if (!("body" in result)) throwFetchError(result.res, url, packageName, "JSR"); const data = JSON.parse(result.body); - // Transform JSR format to match npm-like format for compatibility const versions: Record = {}; const time: Record = {}; for (const [version, metadata] of Object.entries((data.versions ?? {}) as Record)) { @@ -292,107 +195,78 @@ export async function fetchJsrInfo(packageName: string, ctx: ModeContext): Promi return [{name: packageName, "dist-tags": {latest: data.latest}, versions, time}, ctx.jsrApiUrl]; } -const jsonSpace = new Set([0x20, 0x09, 0x0a, 0x0d]); - -// The text span of every top-level pair, key included. A textual `content.indexOf('"overrides"')` -// would find a nested `pnpm.overrides` written above the top-level one, hence the structural scan. -function topLevelSpans(content: string): Map { - const spans = new Map(); - let depth = 0; - let key: string | null = null; - let start = 0; - for (let index = 0; index < content.length; index++) { - const code = content.charCodeAt(index); - if (code === 0x22) { // '"' - const from = index; - while (content.charCodeAt(++index) !== 0x22 && index < content.length) if (content.charCodeAt(index) === 0x5c) index++; // '\' - if (depth === 1 && key === null) { - key = content.slice(from + 1, index); - start = from; +export function updatePackageJson(pkgStr: string, deps: Deps): string { + try { JSON.parse(pkgStr); } catch { return pkgStr; } + const spans = new Map(); + let position = 0; + const whitespace = () => { while (/\s/.test(pkgStr[position] ?? "")) position++; }; + const string = () => { + const start = position++; + while (position < pkgStr.length) { + if (pkgStr[position] === "\\") position += 2; + else if (pkgStr[position++] === '"') break; + } + return {start, end: position, value: JSON.parse(pkgStr.slice(start, position)) as string}; + }; + const value = (path: Array) => { + whitespace(); + if (pkgStr[position] === '"') { + spans.set(JSON.stringify(path), string()); + } else if (pkgStr[position] === "{") { + position++; + whitespace(); + while (pkgStr[position] !== "}" && position < pkgStr.length) { + const key = string().value; + whitespace(); + if (pkgStr[position++] !== ":") return; + value([...path, key]); + whitespace(); + if (pkgStr[position] === ",") { position++; whitespace(); } else break; + } + if (pkgStr[position] === "}") position++; + } else if (pkgStr[position] === "[") { + position++; + let index = 0; + whitespace(); + while (pkgStr[position] !== "]" && position < pkgStr.length) { + value([...path, index++]); + whitespace(); + if (pkgStr[position] === ",") { position++; whitespace(); } else break; } - } else if (code === 0x7b || code === 0x5b) { // '{' '[' - depth++; - } else if (code === 0x7d || code === 0x5d) { // '}' ']' - if (--depth === 0 && key !== null) spans.set(key, {start, end: index}); - if (depth === 0) key = null; - } else if (code === 0x2c && depth === 1 && key !== null) { // ',' - spans.set(key, {start, end: index}); - key = null; + if (pkgStr[position] === "]") position++; + } else { + while (position < pkgStr.length && !/[,}\]]/.test(pkgStr[position])) position++; } + }; + value([]); + const edits: Array<{start: number, end: number, value: string}> = []; + for (const [key, dep] of Object.entries(deps)) { + const [depType, name, identity] = key.split(fieldSep); + let oldValue = dep.oldOrig || dep.old; + let span = spans.get(JSON.stringify(identity ? JSON.parse(identity) : [depType, name])); + let newValue = dep.new; + if (!span) { + span = spans.get(JSON.stringify([depType])); + oldValue = `${name}@${oldValue}`; + newValue = `${name}@${newValue}`; + } + if (span?.value === oldValue) edits.push({...span, value: JSON.stringify(newValue)}); } - return spans; -} - -// Matched as a `"key": "value"` pair, so two names sharing a value in one section are not confused. -function pairValueIndex(content: string, from: number, keyJson: string, valueJson: string): number { - for (let index = content.indexOf(keyJson, from); index !== -1; index = content.indexOf(keyJson, index + 1)) { - let pos = index + keyJson.length; - while (jsonSpace.has(content.charCodeAt(pos))) pos++; - if (content.charCodeAt(pos) !== 0x3a) continue; // ':' - pos++; - while (jsonSpace.has(content.charCodeAt(pos))) pos++; - if (content.startsWith(valueJson, pos)) return pos; - } - return -1; -} - -// A dep whose pair is nowhere in its own top-level span is left alone. Deps arrive in document -// order, so a per-section cursor keeps the sweep linear. -export function updatePackageJson(pkgStr: string, deps: Deps): string { - let doc: Record; - try { - doc = JSON.parse(pkgStr); - } catch { - return pkgStr; - } - const spans = topLevelSpans(pkgStr); - const cursors = new Map(); - const edits: Array<{index: number, length: number, text: string}> = []; - for (const [depKey, {old, oldOrig, new: newVal}] of Object.entries(deps)) { - const [depType, name] = depKey.split(fieldSep); - const section = doc[depType]; - // `packageManager` is the one dep type whose section is the value itself. - const inline = typeof section === "string"; - const oldValue = inline ? `${name}@${oldOrig || old}` : oldOrig || old; - const span = spans.get(depType); - if (!span || (inline ? section : section?.[name]) !== oldValue) continue; - const keyJson = JSON.stringify(inline ? depType : name); - const valueJson = JSON.stringify(oldValue); - let index = pairValueIndex(pkgStr, cursors.get(depType) ?? span.start, keyJson, valueJson); - // url deps are re-inserted after the regular ones, so a cursor-relative hit can land later. - if (index === -1 || index >= span.end) index = pairValueIndex(pkgStr, span.start, keyJson, valueJson); - if (index === -1 || index >= span.end) continue; - cursors.set(depType, index + valueJson.length); - edits.push({index, length: valueJson.length, text: JSON.stringify(inline ? `${name}@${newVal}` : newVal)}); - } - if (!edits.length) return pkgStr; - - edits.sort((a, b) => a.index - b.index); - const parts: Array = []; - let pos = 0; - for (const {index, length, text} of edits) { - if (index < pos) continue; // two deps landed on one span, so this one was placed wrong - parts.push(pkgStr.slice(pos, index), text); - pos = index + length; + for (const edit of edits.sort((left, right) => right.start - left.start)) { + pkgStr = `${pkgStr.slice(0, edit.start)}${edit.value}${pkgStr.slice(edit.end)}`; } - parts.push(pkgStr.slice(pos)); - return parts.join(""); + return pkgStr; } -const operators = String.raw`[<>]=?|[\^~=]`; -// operator (plus any space), an optional `v`, release parts with x-ranges, an optional prerelease +const operators = String.raw`[<>]=?|~>|[\^~=]`; const comparatorRe = new RegExp(String.raw`^(${operators})?(\s*)(v?)((?:\d+|[xX*])(?:\.(?:\d+|[xX*]))*)(-[0-9A-Za-z.-]+)?$`); const operatorRe = new RegExp(`^(?:${operators})$`); const xPartRe = /^[xX*]$/; -// build metadata belongs to the version it was authored with, never to its successor const buildMetaRe = /\+[0-9A-Za-z.-]+/g; -// a range is complex when it holds more than one comparator, `||` and hyphen ranges included const complexRangeRe = /\|\||[\dxX*]\s+\S/; -// Keeps the space an operator may have before its version, leaving a hyphen range's `-` as a marker. function comparators(range: string): Array { const out: Array = []; - // `||` needs no surrounding space, so whitespace alone leaves `^1.0.0||^2.0.0` as one comparator. for (const token of range.trim().replaceAll("||", " || ").split(/\s+/)) { if (out.length && operatorRe.test(out[out.length - 1])) out[out.length - 1] += ` ${token}`; else out.push(token); @@ -400,49 +274,33 @@ function comparators(range: string): Array { return out; } -// Keeps the operator, the authored precision and any x-range placeholder: `^5.9` -> `^6.1`, `1.x` -> `2.x`. function replaceComparator(comparator: string, newVersion: string): string { const match = comparatorRe.exec(comparator); if (!match) return comparator; const [, operator = "", space, vPrefix, digits, pre = ""] = match; const parts = digits.split("."); - // An exclusive upper bound rewritten onto the new version excludes the very version being - // installed, so `<2.0.0` has to clear it rather than land on it, and `` already admits it; a `<` the branch above could not read cannot move without landing under it. if (operator === ">" || operator === "<") return comparator; - // A prerelease can't be spelled in fewer than 3 numeric parts, so it replaces the whole range. const newParts = newVersion.split("-")[0].split("."); if (newParts.join(".") !== newVersion) return `${operator}${space}${vPrefix}${newVersion}`; return `${operator}${space}${vPrefix}${parts.map((part, i) => xPartRe.test(part) ? part : newParts[i] ?? "0").join(".")}`; } -// Renovate widens rather than replaces when a range must keep admitting what it already admits: -// peer ranges always, and any multi-comparator range, whose earlier comparators a replace would -// silently drop. lib/modules/manager/npm/range.ts -function widens(depType: string | undefined, range: string): boolean { - return depType === "peerDependencies" || complexRangeRe.test(range); -} - -// lib/modules/versioning/npm/range.ts, rangeStrategy=widen: an upper bound moves out to admit -// the new version, everything else gains an or-branch for it. function widenRange(range: string, newVersion: string): string { - if (satisfies(newVersion, range)) return range; // already admitted, nothing to widen + if (satisfies(newVersion, range)) return range; const parts = comparators(range); const last = parts[parts.length - 1]; if (!last.startsWith("<") && parts[parts.length - 2] !== "-") { - // A complex range ending in a lower bound has no widening renovate will spell out. const branch = replaceComparator(last, newVersion); if (parts.length > 1 && last.startsWith(">") || branch === last) return range; return `${range} || ${branch}`; @@ -452,13 +310,9 @@ function widenRange(range: string, newVersion: string): string { } export function updateVersionRange(oldRange: string, newVersion: string, oldOrig: string | undefined, depType?: string): string { - // corepack refuses a `packageManager` whose integrity hash no longer matches its version, so - // `9.0.0+sha512.…` has to become a plain `11.20.0` rather than keep 9.0.0's hash. const authored = (oldOrig || oldRange).replace(buildMetaRe, ""); - const updated = widens(depType, authored) ? widenRange(authored, newVersion) : replaceComparator(authored, newVersion); - // A range that excludes the version it was rewritten for installs something other than what the - // run reports, so it is left as authored and the caller drops the dependency. `^1.0.0 <1.5.0` - // widened onto 2.0.0 is one: only the `<` moves, and the caret still caps below the new major. + const updated = depType === "peerDependencies" || complexRangeRe.test(authored) ? + widenRange(authored, newVersion) : replaceComparator(authored, newVersion); return satisfies(newVersion, updated) ? updated : authored; } @@ -469,16 +323,10 @@ export function normalizeRange(range: string): string { return range.replace(npmVersionRe, coerceToVersion(versionMatches[0])); } -type CommitInfo = { - hash: string, - commit: Record, -}; +type CommitInfo = {hash: string, commit: Record}; -// A failed lookup throws, as getTags does: swallowing it read a rate-limited or broken forge as a -// dependency with no newer commit. A repository with no commits at all is the one genuine empty. export async function getLatestCommit(user: string, repo: string, ctx: ModeContext): Promise { const url = `${ctx.forgeApiUrl}/repos/${user}/${repo}/commits`; - // Only the newest commit's date-bearing fields are read; drop the rest before caching. const body = await fetchForgeEtag(url, ctx, async res => { const [latest] = JSON.parse(await res.text()); return JSON.stringify(latest ? [{sha: latest.sha, commit: {committer: latest.commit?.committer, author: latest.commit?.author}}] : []); @@ -488,19 +336,42 @@ export async function getLatestCommit(user: string, repo: string, ctx: ModeConte } export async function getTags(user: string, repo: string, oldRef: string, ctx: ModeContext): Promise> { - const entries = await fetchActionTags(ctx.forgeApiUrl, user, repo, ctx, [oldRef]); - return entries.map(e => e.name); + const entries = await fetchForgeTags(ctx.forgeApiUrl, user, repo, ctx, [oldRef]); + return entries.map(entry => entry.name); +} + +type GitHubSpec = {user: string, repo: string, ref: string, selector: string | null}; + +function parseGitHubSpec(value: string): GitHubSpec | null { + const hash = value.lastIndexOf("#"); + if (hash < 1 || hash === value.length - 1) return null; + let source = value.slice(0, hash).replace(/^git\+/i, ""); + const fragment = value.slice(hash + 1); + if (/^github:/i.test(source)) source = source.slice(7); + else if (/^git@github\.com:/i.test(source)) source = source.replace(/^git@github\.com:/i, ""); + else if (/^(?:https?|git|ssh):/i.test(source)) { + const match = /^(?:https?|git|ssh):\/\/(?:[^/@]+@)?github\.com[/:](.+)$/i.exec(source); + if (!match) return null; + source = match[1]; + } else if (source.includes(":")) { + return null; + } + source = source.replace(/\.git$/i, ""); + const parts = source.split("/"); + if (parts.length !== 2 || !/^[a-z\d](?:-?[a-z\d]){0,38}$/i.test(parts[0]) || !/^[a-z\d._-]{1,100}$/i.test(parts[1])) return null; + const selector = fragment.startsWith("semver:") && validRange(fragment.slice(7)) ? fragment.slice(7) : null; + if (!selector && !hashRe.test(fragment) && !valid(fragment)) return null; + return {user: parts[0], repo: parts[1], ref: selector ?? fragment, selector}; } export async function checkUrlDep(key: string, dep: Dep, ctx: ModeContext): Promise { - const stripped = dep.old.replace(stripRe, ""); - const [, user, repo, oldRef] = partsRe.exec(stripped) || []; - if (!user || !repo || !oldRef) return null; + const parsed = parseGitHubSpec(dep.old); + if (!parsed) return null; + const {user, repo, ref: oldRef, selector} = parsed; - // replace the trailing ref occurrence, not an earlier coincidental match in the URL const replaceRef = (ref: string) => { - const idx = dep.old.lastIndexOf(oldRef); - return dep.old.slice(0, idx) + ref + dep.old.slice(idx + oldRef.length); + const index = dep.old.lastIndexOf("#") + 1; + return `${dep.old.slice(0, index)}${selector ? `semver:${ref}` : ref}`; }; if (hashRe.test(oldRef)) { @@ -514,12 +385,12 @@ export async function checkUrlDep(key: string, dep: Dep, ctx: ModeContext): Prom } } else { const tags = await getTags(user, repo, oldRef, ctx); - const newTag = selectTag(tags, oldRef); + const newTag = selectTag(tags, selector ? coerceToVersion(selector) : oldRef); if (newTag) { - return {key, newRange: replaceRef(newTag), user, repo, oldRef, newRef: newTag}; + const newRef = selector ? updateVersionRange(selector, newTag.replace(/^v/, ""), selector) : newTag; + if (newRef !== oldRef) return {key, newRange: replaceRef(newRef), user, repo, oldRef, newRef}; } } return null; } - diff --git a/modes/pypi.test.ts b/modes/pypi.test.ts index f26b245..5cef25e 100644 --- a/modes/pypi.test.ts +++ b/modes/pypi.test.ts @@ -1,39 +1,7 @@ -import {updatePyprojectToml, fetchPypiInfo} from "./pypi.ts"; +import {updatePyprojectToml, fetchPypiInfo, pypiSatisfies} from "./pypi.ts"; import {type ModeContext, fetchTimeout, fieldSep} from "./shared.ts"; import {parseUvDependencies} from "../utils/utils.ts"; -test("replaces >= operator", () => { - const input = `dependencies = [\n "requests >=2.28.0",\n]\n`; - const deps = { - [`dependencies${fieldSep}requests`]: {old: "2.28.0", new: "2.31.0"} as any, - }; - expect(updatePyprojectToml(input, deps)).toBe(`dependencies = [\n "requests >=2.31.0",\n]\n`); -}); - -test("replaces == operator", () => { - const input = `dependencies = [\n "flask ==2.3.0",\n]\n`; - const deps = { - [`dependencies${fieldSep}flask`]: {old: "2.3.0", new: "2.4.0"} as any, - }; - expect(updatePyprojectToml(input, deps)).toBe(`dependencies = [\n "flask ==2.4.0",\n]\n`); -}); - -test("replaces ~= operator", () => { - const input = `dependencies = [\n "django ~=4.2.0",\n]\n`; - const deps = { - [`dependencies${fieldSep}django`]: {old: "4.2.0", new: "4.3.0"} as any, - }; - expect(updatePyprojectToml(input, deps)).toBe(`dependencies = [\n "django ~=4.3.0",\n]\n`); -}); - -test("package with extras", () => { - const input = `dependencies = [\n "transformers[torch] >=4.39.3",\n]\n`; - const deps = { - [`dependencies${fieldSep}transformers`]: {old: "4.39.3", new: "4.40.0"} as any, - }; - expect(updatePyprojectToml(input, deps)).toBe(`dependencies = [\n "transformers[torch] >=4.40.0",\n]\n`); -}); - test("preserves surrounding content", () => { const input = [ `[project]`, @@ -41,48 +9,81 @@ test("preserves surrounding content", () => { `version = "1.0.0"`, `dependencies = [`, ` "requests >=2.28.0",`, + ` "requests-oauthlib >=2.28.0",`, ` "flask >=2.3.0",`, ` "click >=8.1.0",`, `]`, ``, ].join("\n"); const deps = { - [`dependencies${fieldSep}flask`]: {old: "2.3.0", new: "2.4.0"} as any, + [`project.dependencies${fieldSep}flask`]: {old: "2.3.0", new: "2.4.0"} as any, + [`project.dependencies${fieldSep}click`]: {old: "8.1.0", new: "8.2.0"} as any, + [`project.dependencies${fieldSep}requests`]: {old: "2.28.0", new: "2.31.0"} as any, }; const result = updatePyprojectToml(input, deps); expect(result).toContain(`"flask >=2.4.0"`); expect(result).toContain(`name = "my-project"`); - expect(result).toContain(`"requests >=2.28.0"`); - expect(result).toContain(`"click >=8.1.0"`); + expect(result).toContain(`"requests >=2.31.0"`); + expect(result).toContain(`"requests-oauthlib >=2.28.0"`); + expect(result).toContain(`"click >=8.2.0"`); }); -test("uses oldOrig when present", () => { - const input = `dependencies = [\n "requests >=2.28.0",\n]\n`; +test("rewrites only the dependency's originating group", () => { + const input = [ + `[project]`, + `dependencies = ["pkg>=1.0"]`, + ``, + `[project.optional-dependencies]`, + `extra = ["pkg>=1.0"]`, + ``, + `[dependency-groups]`, + `"test.unit" = ["pkg>=1.0"]`, + ``, + ].join("\n"); const deps = { - [`dependencies${fieldSep}requests`]: {old: "2.28.0", oldOrig: "2.28.0", new: "2.31.0"} as any, + [`project.optional-dependencies.extra${fieldSep}pkg`]: {old: "1.0", new: "2.0"} as any, }; - expect(updatePyprojectToml(input, deps)).toBe(`dependencies = [\n "requests >=2.31.0",\n]\n`); + expect(updatePyprojectToml(input, deps)).toBe(input.replace(`extra = ["pkg>=1.0"]`, `extra = ["pkg>=2.0"]`)); }); -// fetchPypiInfo test("fetchPypiInfo happy path", async () => { const mockData = {info: {version: "2.31.0"}, releases: {"2.31.0": [{}]}}; + let url = ""; + const ctx = { + pypiApiUrl: "https://pypi.org", + fetchTimeout, + doFetch: (input: string) => { + url = input; + return Promise.resolve({ok: true, text: () => Promise.resolve(JSON.stringify(mockData)), headers: new Headers()}); + }, + } as unknown as ModeContext; + const result = await fetchPypiInfo("Foo_Bar.baz", ctx); + expect(result).toEqual([{...mockData, name: "Foo_Bar.baz"}, null]); + expect(url).toBe("https://pypi.org/pypi/foo-bar-baz/json"); +}); + +test("fetchPypiInfo shares a normalized request in flight", async () => { + let requests = 0; const ctx = { pypiApiUrl: "https://pypi.org", fetchTimeout, - doFetch: () => Promise.resolve({ok: true, json: () => Promise.resolve(mockData)}), + noCache: true, + doFetch: async () => { + requests++; + await new Promise(resolve => setImmediate(resolve)); + return {ok: true, text: () => Promise.resolve(JSON.stringify({info: {}, releases: {}})), headers: new Headers()}; + }, } as unknown as ModeContext; - const result = await fetchPypiInfo("requests", ctx); - expect(result).toEqual([{...mockData, name: "requests"}, null]); + await Promise.all([fetchPypiInfo("Foo_Bar", ctx), fetchPypiInfo("foo-bar", ctx)]); + expect(requests).toBe(1); }); -test("fetchPypiInfo keeps yanked flags through the size reducer", async () => { - // Only docs above the 16 KB threshold get reduced, and pypi yanks per file, not per release. - const files = (yankLast: boolean) => Array.from({length: 40}, (_, idx) => ({ +test("fetchPypiInfo preserves yank and upload metadata through the size reducer", async () => { + const files = (allYanked: boolean) => Array.from({length: 40}, (_, idx) => ({ filename: `pkg-${idx}.whl`, url: `https://files.pythonhosted.org/packages/${"0".repeat(200)}/pkg-${idx}.whl`, - upload_time_iso_8601: "2025-01-01T00:00:00.000000Z", - yanked: yankLast && idx === 39, + upload_time_iso_8601: idx === 39 ? "2024-12-01T00:00:00.000000Z" : "2025-01-01T00:00:00.000000Z", + yanked: allYanked || idx === 39, })); const mockData = {info: {name: "pkg", version: "1.0.1"}, releases: {"1.0.0": files(false), "1.0.1": files(true)}}; const ctx = { @@ -92,35 +93,35 @@ test("fetchPypiInfo keeps yanked flags through the size reducer", async () => { doFetch: () => Promise.resolve({ok: true, text: () => Promise.resolve(JSON.stringify(mockData)), headers: new Headers()}), } as unknown as ModeContext; const [data] = await fetchPypiInfo("reduced-pkg", ctx); - expect(data.releases["1.0.0"]).toEqual([{upload_time_iso_8601: "2025-01-01T00:00:00.000000Z"}]); - expect(data.releases["1.0.1"]).toEqual([{upload_time_iso_8601: "2025-01-01T00:00:00.000000Z", yanked: true}]); + expect(data.releases["1.0.0"]).toHaveLength(40); + expect(data.releases["1.0.0"][0]).toEqual({ + upload_time_iso_8601: "2025-01-01T00:00:00.000000Z", + }); + expect(data.releases["1.0.0"][39]).toEqual({ + upload_time_iso_8601: "2024-12-01T00:00:00.000000Z", + yanked: true, + }); + expect(data.releases["1.0.1"]).toHaveLength(40); + expect(data.releases["1.0.1"].every((file: any) => file.yanked)).toBe(true); }); -test.each([ - ["fetch failure", () => Promise.resolve({ok: false, status: 404, statusText: "Not Found"}), "404"], - ["null response", () => Promise.resolve(undefined), "Unable to fetch"], -])("fetchPypiInfo %s throws", async (_name, doFetch, message) => { - const ctx = {pypiApiUrl: "https://pypi.org", fetchTimeout, doFetch} as unknown as ModeContext; - await expect(fetchPypiInfo("nonexistent", ctx)).rejects.toThrow(message); +test("fetchPypiInfo failure throws", async () => { + const ctx = {pypiApiUrl: "https://pypi.org", fetchTimeout, + doFetch: () => Promise.resolve({ok: false, status: 404, statusText: "Not Found"})} as unknown as ModeContext; + await expect(fetchPypiInfo("nonexistent", ctx)).rejects.toThrow("404"); }); -test("operator without space", () => { - const input = `dependencies = [\n "requests>=2.28.0",\n]\n`; - const deps = { - [`dependencies${fieldSep}requests`]: {old: "2.28.0", new: "2.31.0"} as any, - }; - expect(updatePyprojectToml(input, deps)).toBe(`dependencies = [\n "requests>=2.31.0",\n]\n`); +test("pypiSatisfies handles allowedVersions forms", () => { + expect(pypiSatisfies("2.1+corp", ">=2,<3")).toBe(true); + expect(pypiSatisfies("2.1", "")).toBe(true); + expect(pypiSatisfies("2.1+corp", "2.1")).toBe(true); + expect(pypiSatisfies("2.2", "2.1")).toBe(false); + expect(pypiSatisfies("2.1", "[extra]>=2")).toBe(false); + expect(pypiSatisfies("2.1", `>=2; python_version >= "3.12"`)).toBe(false); + expect(pypiSatisfies("not-a-version", ">=2")).toBe(false); + expect(pypiSatisfies("2.1", "not-a-range")).toBe(false); }); -test("rewrites single-quoted dependency preserving single quotes", () => { - const input = `dependencies = [\n 'requests >=2.28.0',\n]\n`; - const deps = { - [`dependencies${fieldSep}requests`]: {old: "2.28.0", new: "2.31.0"} as any, - }; - expect(updatePyprojectToml(input, deps)).toBe(`dependencies = [\n 'requests >=2.31.0',\n]\n`); -}); - -// A requirement whose marker uses double quotes has to live in a single-quoted TOML string. const quoted = (spec: string) => spec.includes(`"`) ? `'${spec}'` : `"${spec}"`; test.each([ @@ -131,39 +132,28 @@ test.each([ ["violated cap raised at its own precision", "sphinx>=7.0.0,<8", "7.0.0", "8.2.0", "sphinx>=8.2.0,<9"], ["violated two-part cap", "sphinx >=7.0.0, <8.0", "7.0.0", "8.2.0", "sphinx >=8.2.0, <8.3"], ["violated three-part cap", "protobuf>=3.20.2,<5.0.0", "3.20.2", "5.29.0", "protobuf>=5.29.0,<5.30.0"], - // the lower bound's own precision says nothing about the cap's, which is where the bump lands ["violated cap over a one-part lower bound", "sphinx>=7,<7.4.0", "7", "7.5.1", "sphinx>=7.5.1,<7.5.2"], ["violated inclusive cap", "urllib3>=1.26.0,<=2.0", "1.26.0", "2.2.3", "urllib3>=2.2.3,<=2.2.3"], ["exclusion the new version misses", "packaging>=20.9,!=22.0", "20.9", "21.3", "packaging>=21.3,!=22.0"], ["exclusion the new version hits", "packaging>=20.9,!=22.0", "20.9", "22.0", "packaging>=20.9,!=22.0"], ["wildcard exclusion the new version hits", "numpy>=1.20,!=1.25.*", "1.20", "1.25.2", "numpy>=1.20,!=1.25.*"], - // PEP 440 excludes the bound's own release, so `<8` does not admit `8.0.0b1` and the cap moves. ["cap violated by a prerelease of its own release", "sphinx>=7.0.0,<8", "7.0.0", "8.0.0b1", "sphinx>=8.0.0b1,<9"], ["cap violated by a dev release of its own release", "sphinx>=7.0.0,<8.0.0", "7.0.0", "8.0.0.dev1", "sphinx>=8.0.0.dev1,<8.1.0"], ["compatible release trimmed to the authored precision", "django~=4.2", "4.2", "4.3.1", "django~=4.3"], ["compatible release padded to the authored precision", "django~=4.2.0", "4.2.0", "5.0", "django~=5.0.0"], + ["ordered local version", "pkg>=1.0", "1.0", "2.0+corp", "pkg>=2.0"], + ["equality local version", "pkg==1.0", "1.0", "2.0+corp", "pkg==2.0+corp"], + ["public exclusion hit by a local version", "pkg>=1.0,!=2.0", "1.0", "2.0+corp", "pkg>=1.0,!=2.0"], + ["local exclusion missed by another local version", "pkg>=1.0,!=2.0+other", "1.0", "2.0+corp", "pkg>=2.0,!=2.0+other"], + ["epoch-compatible release precision", "pkg~=1!1.4", "1!1.4", "1!1.5.1", "pkg~=1!1.5"], + ["compatible release suffixes", "pkg~=1!1.4", "1!1.4", "1!1.5.1rc2.post3.dev4+corp", "pkg~=1!1.5rc2.post3.dev4"], + ["compatible release from another epoch", "pkg>=1!1.4,~=1!1.4", "1!1.4", "2!1.5", "pkg>=1!1.4,~=1!1.4"], ])("updatePyprojectToml handles a %s", (_name, spec, old, newVersion, expected) => { const input = `dependencies = [\n ${quoted(spec)},\n]\n`; const deps = {[`dependencies${fieldSep}${/^[\w.-]+/.exec(spec)![0]}`]: {old, new: newVersion} as any}; expect(updatePyprojectToml(input, deps)).toBe(`dependencies = [\n ${quoted(expected)},\n]\n`); }); -test("only rewrites the package whose name the requirement starts with", () => { - const input = [ - `dependencies = [`, - ` "requests >=2.28.0",`, - ` "requests-oauthlib >=2.28.0",`, - `]`, - ``, - ].join("\n"); - const deps = { - [`dependencies${fieldSep}requests`]: {old: "2.28.0", new: "2.31.0"} as any, - }; - expect(updatePyprojectToml(input, deps)).toContain(`"requests >=2.31.0"`); - expect(updatePyprojectToml(input, deps)).toContain(`"requests-oauthlib >=2.28.0"`); -}); - -// When the two disagree, a dependency is reported as outdated and then silently left unwritten. test.each([ "requests >=2.28.0", "flask <3,>=2.2", diff --git a/modes/pypi.ts b/modes/pypi.ts index de0b20d..de7c3b3 100644 --- a/modes/pypi.ts +++ b/modes/pypi.ts @@ -1,79 +1,96 @@ -import {type Deps, type ModeContext, type PackageInfo, fieldSep, fetchWithEtag, reduceJson, throwFetchError} from "./shared.ts"; +import { + type Deps, type ModeContext, type PackageInfo, fieldSep, fetchWithEtag, reduceJson, throwFetchError, +} from "./shared.ts"; import {type Pep440, comparePep440, parsePep440} from "../utils/semver.ts"; -import {type Pep508Specifier, anchorSpecifier, esc, parsePep508, serializePep508} from "../utils/utils.ts"; +import {type Pep508Specifier, anchorSpecifier, esc, getOrSet, parsePep508, serializePep508} from "../utils/utils.ts"; -function reducePypiDoc(data: Record): Record { - const releases: Record> = {}; - for (const [version, files] of Object.entries(data.releases ?? {})) { - const list = (files as Array>) ?? []; - releases[version] = list.slice(0, 1).map(file => ({ +type PypiFile = {upload_time_iso_8601?: string, yanked?: boolean}; + +function reducePypiReleases(data: Record): Record> { + const releases: Record> = {}; + for (const [version, files] of Object.entries(data ?? {})) { + releases[version] = ((files as Array>) ?? []).map(file => ({ ...(file.upload_time_iso_8601 && {upload_time_iso_8601: file.upload_time_iso_8601}), - ...(list.some(entry => entry?.yanked) && {yanked: true}), + ...(file.yanked && {yanked: true}), })); } - const {name, version, project_urls} = data.info ?? {}; - return {info: {name, version, project_urls}, releases}; + return releases; } export async function fetchPypiInfo(name: string, ctx: ModeContext): Promise { - const url = `${ctx.pypiApiUrl}/pypi/${name}/json`; + const url = `${ctx.pypiApiUrl}/pypi/${name.toLowerCase().replace(/[-_.]+/g, "-")}/json`; const result = await fetchWithEtag(url, ctx, { headers: {"accept-encoding": "gzip, deflate, br"}, - }, reduceJson(reducePypiDoc)); - // A pypi document names itself under `info`, so the resolved name is restored where every - // other mode puts it. Post-parse, as small bodies never reach `reducePypiDoc`. - if ("body" in result) return [{...JSON.parse(result.body), name}, null]; + }, reduceJson(data => { + const {name: reducedName, version, project_urls} = data.info ?? {}; + return {info: {name: reducedName, version, project_urls}, releases: reducePypiReleases(data.releases)}; + })); + if ("body" in result) { + const data = JSON.parse(result.body); + return [{...data, releases: reducePypiReleases(data.releases), name}, null]; + } throwFetchError(result.res, url, name, ctx.pypiApiUrl); } -// Only as much of PEP 440 as proving a rewrite safe needs, with comparisons in plain version order. function specifierAllows(version: Pep440, {op, version: text}: Pep508Specifier): boolean { - if (op === "===") return text === version.version; // arbitrary equality compares the string + if (op === "===") return text === version.version; if (text.endsWith(".*")) { const prefix = parsePep440(text.slice(0, -2)); - if (!prefix || (op !== "==" && op !== "!=")) return false; + if (!prefix || op !== "==" && op !== "!=") return false; const matches = prefix.epoch === version.epoch && prefix.release.every((part, idx) => (version.release[idx] ?? 0) === part); return op === "==" ? matches : !matches; } const parsed = parsePep440(text); if (!parsed) return false; - const cmp = comparePep440(version, parsed); - if (op === "==") return cmp === 0; - if (op === "!=") return cmp !== 0; + if (parsed.local && op !== "==" && op !== "!=") return false; + const publicVersion = version.local ? {...version, local: null} : version; + const cmp = comparePep440(publicVersion, parsed); + const equalityCmp = parsed.local ? comparePep440(version, parsed) : cmp; + if (op === "==") return equalityCmp === 0; + if (op === "!=") return equalityCmp !== 0; if (op === ">=") return cmp >= 0; - // PEP 440: an exclusive comparison excludes the bound's own release, so `<2.0` rejects - // `2.0rc1` and `>2.0` rejects `2.0.post1`, unless the bound already says pre or post itself. const sameRelease = version.epoch === parsed.epoch && Array.from({length: Math.max(version.release.length, parsed.release.length)}) .every((_, idx) => (version.release[idx] ?? 0) === (parsed.release[idx] ?? 0)); if (op === ">") return cmp > 0 && !(sameRelease && version.post !== null && parsed.post === null); if (op === "<=") return cmp <= 0; if (op === "<") return cmp < 0 && !(sameRelease && (version.pre || version.dev !== null) && !parsed.pre && parsed.dev === null); - // `~=X.Y.Z` is `>=X.Y.Z` with only the last segment free to move. - return cmp >= 0 && parsed.release.length > 1 && + return cmp >= 0 && parsed.release.length > 1 && parsed.epoch === version.epoch && parsed.release.slice(0, -1).every((part, idx) => (version.release[idx] ?? 0) === part); } -// Renovate's getRangePrecision: a cap the new version reaches moves up by one at the segment where -// it first rises above the lower bound, one further down when the segment below that is a zero -// (`>=3.20.2,<5.0.0` is minor-wide), with the segments under it taken from the new version or zeroed. +function orderedVersion(version: Pep440, release = version.release): string { + if (!version.local && release === version.release) return version.version; + const epoch = version.epoch || /^[vV]?\d+!/.test(version.version) ? `${version.epoch}!` : ""; + const pre = version.pre ? `${version.pre[0]}${version.pre[1]}` : ""; + const post = version.post === null ? "" : `.post${version.post}`; + const dev = version.dev === null ? "" : `.dev${version.dev}`; + return `${epoch}${release.join(".")}${pre}${post}${dev}`; +} + function raisedUpperBound(cap: Pep440, lower: Pep440, version: Pep440): string { let precision = cap.release.findIndex((part, idx) => part > lower.release[idx]); if (precision === 0 && cap.release[1] === 0) precision = 1; else if (precision === -1) precision = cap.release.length - 1; - return cap.release.map((_, idx) => idx > precision ? 0 : (version.release[idx] ?? 0) + Number(idx === precision)).join("."); + const release = cap.release.map((_, idx) => idx > precision ? 0 : (version.release[idx] ?? 0) + Number(idx === precision)); + return `${version.epoch ? `${version.epoch}!` : ""}${release.join(".")}`; } -// Renovate's updateRangeValue for `~=`: its precision is the constraint the author stated, so the -// new version is trimmed or zero-padded to it rather than replacing it. -function fitCompatibleRelease(base: Pep440, version: Pep440): string { - if (base.release.length === version.release.length) return version.version; - return Array.from({length: base.release.length}, (_, idx) => version.release[idx] ?? 0).join("."); +export function pypiSatisfies(version: string, range: string): boolean { + const parsed = parsePep440(version); + if (!parsed) return false; + const trimmed = range.trim(); + if (!trimmed) return true; + if (parsePep440(trimmed)) { + return specifierAllows(parsed, {lead: "", op: "==", sep: "", version: trimmed, trail: ""}); + } + const requirement = parsePep508(`x${trimmed}`); + const specifiers = requirement?.specifiers; + if (!requirement || requirement.extras || requirement.marker || !specifiers?.length) return false; + return specifiers.every(specifier => specifierAllows(parsed, specifier)); } -// Bump the specifier the reported version was read from, leaving every other one satisfiable, or -// null to leave the requirement as authored. Selection calls this too, so a decline is never reported. export function updateRequirement(text: string, oldValue: string, newValue: string): string | null { const parsed = parsePep508(text); const specifiers = parsed?.specifiers; @@ -84,29 +101,117 @@ export function updateRequirement(text: string, oldValue: string, newValue: stri if (anchor?.version !== oldValue) return null; for (const specifier of specifiers) { if (specifier === anchor) { - specifier.version = specifier.op === "~=" ? fitCompatibleRelease(oldParsed, newParsed) : newValue; - continue; + specifier.version = specifier.op === "~=" ? oldParsed.release.length === newParsed.release.length ? orderedVersion(newParsed) : + orderedVersion(newParsed, Array.from({length: oldParsed.release.length}, (_, idx) => newParsed.release[idx] ?? 0)) : + specifier.op === "==" || specifier.op === "===" ? newValue : orderedVersion(newParsed); + } else if (!specifierAllows(newParsed, specifier)) { + const cap = parsePep440(specifier.version); + if (specifier.op === "<" && cap) specifier.version = raisedUpperBound(cap, oldParsed, newParsed); + else if (specifier.op === "<=") specifier.version = orderedVersion(newParsed); } - if (specifierAllows(newParsed, specifier)) continue; - // A cap the new lower bound passes is raised rather than left unsatisfiable. An exclusion stays - // as authored, as renovate takes one to be there for a reason, and the guard below then bails. - const cap = parsePep440(specifier.version); - if (specifier.op === "<" && cap) specifier.version = raisedUpperBound(cap, oldParsed, newParsed); - else if (specifier.op === "<=") specifier.version = newValue; + if (!specifierAllows(newParsed, specifier)) return null; } - if (specifiers.some(specifier => !specifierAllows(newParsed, specifier))) return null; return serializePep508(parsed, specifiers); } +function splitTomlPath(text: string): Array { + const parts: Array = []; + const partRe = /\s*(?:"((?:\\.|[^"\\])*)"|'([^']*)'|([\w-]+))\s*(?:\.|$)/gy; + try { + while (partRe.lastIndex < text.length) { + const match = partRe.exec(text); + if (!match) return []; + parts.push(match[1] === undefined ? match[2] ?? match[3] : JSON.parse(`"${match[1]}"`)); + } + } catch { + return []; + } + return parts; +} + +function assignmentIndex(line: string): number { + let quote = ""; + let escaped = false; + for (let idx = 0; idx < line.length; idx++) { + const char = line[idx]; + if (quote) { + if (quote === `"` && char === `\\` && !escaped) escaped = true; + else { + if (char === quote && !escaped) quote = ""; + escaped = false; + } + } else if (char === `"` || char === `'`) quote = char; + else if (char === "=") return idx; + else if (char === "#") return -1; + } + return -1; +} + +function arrayEnd(text: string, start: number): number { + const open = text.indexOf("[", start); + if (open === -1 || text.slice(start, open).trim()) return -1; + let depth = 0; + let quote = ""; + let escaped = false; + let comment = false; + for (let idx = open; idx < text.length; idx++) { + const char = text[idx]; + if (comment) { + if (char === "\n") comment = false; + } else if (quote) { + if (quote === `"` && char === `\\` && !escaped) escaped = true; + else { + if (char === quote && !escaped) quote = ""; + escaped = false; + } + } else if (char === `"` || char === `'`) quote = char; + else if (char === "#") comment = true; + else if (char === "[") depth++; + else if (char === "]" && --depth === 0) return idx + 1; + } + return -1; +} + +function dependencyArrays(text: string, depTypes: Set): Map { + const spans = new Map(); + let section: Array = []; + let offset = 0; + for (const line of text.split(/(?<=\n)/)) { + const table = /^\[([^\]]+)\](?:\s*#.*)?$/.exec(line.trim()); + if (table) section = splitTomlPath(table[1]); + else { + const eq = assignmentIndex(line); + const depType = eq === -1 ? "" : [...section, ...splitTomlPath(line.slice(0, eq))].join("."); + if (depTypes.has(depType) && !spans.has(depType)) { + const start = offset + eq + 1; + const end = arrayEnd(text, start); + if (end !== -1) spans.set(depType, [start, end]); + } + } + offset += line.length; + } + return spans; +} + export function updatePyprojectToml(pkgStr: string, deps: Deps): string { + const depsByType = new Map>(); + for (const [key, dep] of Object.entries(deps)) { + const [depType, name] = key.split(fieldSep); + getOrSet(depsByType, depType, () => new Map()).set(name, dep); + } + const spans = dependencyArrays(pkgStr, new Set(depsByType.keys())); let newPkgStr = pkgStr; - for (const [key, {old, oldOrig, new: newValue}] of Object.entries(deps)) { - const name = key.split(fieldSep)[1]; - const oldValue = oldOrig || old; - // The whole quoted PEP 508 requirement, ending at its own quote: a marker may hold the other one. - const re = new RegExp(`(['"])( *${esc(name)}(?![\\w.-]).*?)(?=\\1)`, "g"); - newPkgStr = newPkgStr.replace(re, (_, quote, requirement) => - `${quote}${updateRequirement(requirement, oldValue, newValue) ?? requirement}`); + for (const [depType, span] of Array.from(spans).sort((left, right) => right[1][0] - left[1][0])) { + const byName = depsByType.get(depType)!; + const names = Array.from(byName.keys()).sort((left, right) => right.length - left.length).map(esc).join("|"); + const value = newPkgStr.slice(...span).replace( + new RegExp(`(['"])( *(${names})(?![\\w.-]).*?)(?=\\1)`, "g"), + (_, quote, requirement, name) => { + const {old, oldOrig, new: newValue} = byName.get(name)!; + return `${quote}${updateRequirement(requirement, oldOrig || old, newValue) ?? requirement}`; + }, + ); + newPkgStr = `${newPkgStr.slice(0, span[0])}${value}${newPkgStr.slice(span[1])}`; } return newPkgStr; } diff --git a/modes/shared.test.ts b/modes/shared.test.ts index 64062cb..a5c7948 100644 --- a/modes/shared.test.ts +++ b/modes/shared.test.ts @@ -1,30 +1,9 @@ import {Buffer} from "node:buffer"; import { - findNewVersion, - stripv, - normalizeUrl, - getFetchOpts, - isVersionPrerelease, - coerceToVersion, - selectTag, - resolvePackageJsonUrl, - parseTags, - throwFetchError, - formatVersionPrecision, - getSubDir, - findVersion, - getInfoUrl, - packageVersion, - getForgeTokens, - parseExtraheaders, - fetchForge, - fetchActionTags, - fetchWithEtag, - fetchWithRetry, - fetchImmutable, - fetchTimeout, - getLimiter, - ForgeError, + coerceToVersion, fetchActionTags, fetchForge, fetchImmutable, fetchTimeout, fetchWithEtag, fetchWithRetry, + findNewVersion, findVersion, ForgeError, formatVersionPrecision, getFetchOpts, getForgeTokens, getInfoUrl, + getLimiter, getSubDir, hashRe, isVersionPrerelease, normalizeUrl, packageVersion, parseExtraheaders, parseTags, + resolvePackageJsonUrl, selectTag, stripv, throwFetchError, type ModeContext, } from "./shared.ts"; import {esc} from "../utils/utils.ts"; @@ -33,10 +12,8 @@ import {flushCacheWrites} from "../utils/fetchCache.ts"; const defaultOpts = {allowDowngrade: false as any}; -// npm-mode findNewVersion reads `data` without mutating it, so rows may share fixtures. const npmOpts = {mode: "npm", useGreatest: false, usePre: false, useRel: false, semvers: new Set(["patch", "minor", "major"]), ...defaultOpts}; -// Abbreviated npm metadata: has versions and dist-tags but no time const tsAbbrev = {name: "typescript", "dist-tags": {latest: "6.0.2"}, versions: {"5.9.2": {}, "5.9.3": {}, "6.0.0": {}, "6.0.1": {}, "6.0.2": {}}}; const tsFull = {...tsAbbrev, time: { "5.9.2": "2025-01-01T00:00:00Z", @@ -53,66 +30,43 @@ test.each([ ["pin selects greatest within range when no time data", {name: "typescript", "dist-tags": {latest: "6.0.2"}, versions: {"5.9.2": {}, "5.9.3": {}, "5.9.4": {}, "5.9.5": {}, "6.0.2": {}}}, {range: "6.0.2", pinnedRange: "^5.9.3"}, "5.9.5"], - // offers the upgrade within the pinned range (18.2.0 -> 18.3.1) rather than the 19.0.0 latest ["pin with no downgrade returns null without allow-downgrade", {name: "react", "dist-tags": {latest: "19.0.0"}, versions: {"18.2.0": {}, "18.3.0": {}, "18.3.1": {}, "19.0.0": {}}}, {range: "18.2.0", pinnedRange: "^18.0.0"}, "18.3.1"], - // renovate's allowedVersions is a ceiling on newer releases, never a reason to roll back ["renovate-derived pin rolls back without the marker", cropper, {range: "^2.0.0", pinnedRange: "^1"}, "1.6.2"], ["renovate-derived pin filters but never downgrades", cropper, {range: "^2.0.0", pinnedRange: "^1", pinNoDowngrade: true}, null], ])("%s", (_name, data, opts, expected) => { expect(findNewVersion(data, {...npmOpts, ...opts})).toBe(expected); }); -test("stripv removes leading v", () => { +test("shared value helpers", () => { expect(stripv("v1.0.0")).toBe("1.0.0"); expect(stripv("1.0.0")).toBe("1.0.0"); - expect(stripv("v0.1.0")).toBe("0.1.0"); -}); - -test("esc escapes regex special chars", () => { for (const str of ["foo.bar", "a[b]", "no-special", "plain", "a+b*c?", "(x)|{y}^$"]) { expect(new RegExp(`^${esc(str)}$`).test(str)).toBe(true); } - // special chars must match literally, not act as metacharacters expect(new RegExp(`^${esc("a.b")}$`).test("axb")).toBe(false); -}); - -test("normalizeUrl strips trailing slash", () => { expect(normalizeUrl("https://example.com/")).toBe("https://example.com"); expect(normalizeUrl("https://example.com")).toBe("https://example.com"); - expect(normalizeUrl("https://example.com/path/")).toBe("https://example.com/path"); -}); - -test("getFetchOpts sends an auth header only with a token", () => { const headers = getFetchOpts().headers as Record; expect(headers["user-agent"]).toBe(`updates/${packageVersion}`); expect(headers["accept-encoding"]).toBe("gzip, deflate, br"); expect(headers["Authorization"]).toBeUndefined(); expect((getFetchOpts("Bearer", "mytoken123").headers as Record)["Authorization"]).toBe("Bearer mytoken123"); -}); - -test("isVersionPrerelease detects prereleases", () => { expect(isVersionPrerelease("1.0.0-alpha")).toBe(true); expect(isVersionPrerelease("1.0.0-beta.1")).toBe(true); expect(isVersionPrerelease("1.0.0")).toBe(false); expect(isVersionPrerelease("invalid")).toBe(false); - // pep440 spells them without a hyphen, which the semver rules read as stable expect(isVersionPrerelease("2.0.0b1")).toBe(false); expect(isVersionPrerelease("2.0.0b1", pep440Versioning)).toBe(true); expect(isVersionPrerelease("1.1.0.dev1", pep440Versioning)).toBe(true); expect(isVersionPrerelease("2026.3.post1", pep440Versioning)).toBe(false); -}); - -test("coerceToVersion extracts a version, or nothing", () => { expect(coerceToVersion("^1.2.3")).toBe("1.2.3"); expect(coerceToVersion("5")).toBe("5.0.0"); expect(coerceToVersion("~2.1.0")).toBe("2.1.0"); expect(coerceToVersion("")).toBe(""); }); -// GitHub's /tags has no guaranteed order, and its reverse-chronological default defeats a -// lexicographic one by mixing a shorter v9 with a longer v10. test.each([ [["v1.0.0", "v1.1.0", "v2.0.0"], "v1.0.0", "v2.0.0"], [["v1.0.0", "v3.0.0", "v2.0.0"], "v1.0.0", "v3.0.0"], @@ -146,6 +100,11 @@ test("parseTags transforms tag data, commit or not", () => { ]); }); +test("hashRe only recognizes npm commit lengths", () => { + expect(["deadbee", "1234567", "a".repeat(40)].every(value => hashRe.test(value))).toBe(true); + expect(["abc123", "deadbeef", "a".repeat(39), "a".repeat(41), "b".repeat(64)].some(value => hashRe.test(value))).toBe(false); +}); + test("throwFetchError names the status, or the package when there is none", () => { const res = {status: 404, statusText: "Not Found"} as Response; expect(() => throwFetchError(res, "https://example.com", "pkg", "npm")).toThrow("Received 404 Not Found from https://example.com"); @@ -171,9 +130,6 @@ test.each([ const findVersionOpts = {range: "1.0.0", semvers: new Set(["major", "minor", "patch"]), useGreatest: false, usePre: false, useRel: false}; -// A step down is only offered with --allow-downgrade, and only onto the tag, that being where the -// maintainer stepped back to. With no tag published the one worthwhile step down is off a -// prerelease train onto the release below it, and a release has nowhere to go at all. test.each([ ["a pre to a higher release", "1.0.0-alpha", ["2.0.0"], {}, "2.0.0"], ["a pre to a lower release", "2.0.0-alpha", ["1.0.0"], {}, null], @@ -195,7 +151,6 @@ test.each([ ["nothing outside the semver filter", ["1.0.1", "2.0.0"], {}, {semvers: new Set(["patch"])}, "1.0.1"], ["nothing outside pinnedRange", ["1.1.0", "2.0.0"], {}, {pinnedRange: "^1.0.0"}, "1.1.0"], ["no prerelease without --pre", ["1.1.0", "1.2.0-alpha"], {}, {}, "1.1.0"], - // 1.1.0 is 15 days old and eligible, 1.2.0 and 1.3.0 are 3 and 1 days old, so too new ["the newest version past its cooldown", ["1.0.0", "1.1.0", "1.2.0", "1.3.0"], cooldownTimes, cooldownNow, "1.1.0"], ["nothing while every candidate is inside the cooldown", ["1.1.0", "1.2.0"], {"1.1.0": "2026-04-23T00:00:00Z", "1.2.0": "2026-04-24T00:00:00Z"}, cooldownNow, null], @@ -207,7 +162,7 @@ test.each([ expect(findVersion(data, versions, {...findVersionOpts, ...opts})).toBe(expected); }); -test("findVersion picks the highest prerelease regardless of order", () => { +test("findVersion handles prerelease ordering and filtering", () => { const opts = { range: "1.0.0", semvers: new Set(["major", "minor", "patch"]), @@ -218,18 +173,24 @@ test("findVersion picks the highest prerelease regardless of order", () => { expect(findVersion({}, ["2.0.0-rc.2", "2.0.0-rc.1"], opts)).toBe("2.0.0-rc.2"); expect(findVersion({}, ["2.0.0-rc.1", "2.0.0-rc.2"], opts)).toBe("2.0.0-rc.2"); expect(findVersion({}, ["1.0.0-beta.10", "1.0.0-beta.5", "1.0.0-beta.3"], {...opts, range: "1.0.0-beta.1"})).toBe("1.0.0-beta.10"); - // a prerelease below the authored release is a downgrade, not an upgrade expect(findVersion({}, ["1.0.0-beta.10", "1.0.0-beta.5"], opts)).toBe(null); - // a release must win over a same-main prerelease expect(findVersion({}, ["2.0.0-rc.1", "2.0.0"], opts)).toBe("2.0.0"); expect(findVersion({}, ["2.0.0", "2.0.0-rc.1"], opts)).toBe("2.0.0"); + expect(findVersion({}, ["1.2.3+corp.1"], {...opts, range: "1.2.2", usePre: false})).toBe("1.2.3+corp.1"); + const data = {versions: {"2.0.0-rc.1": {}, "2.0.0-rc.2": {}}, + time: {"2.0.0-rc.1": "2025-01-01T00:00:00Z", "2.0.0-rc.2": "2025-01-02T00:00:00Z"}}; + const versions = ["2.0.0-rc.1", "2.0.0-rc.2"]; + const rangeOpts = {range: "^2.0.0-rc.1", useGreatest: false, usePre: false, useRel: false} as const; + expect(findVersion(data, versions, {...rangeOpts, semvers: new Set(["patch"])})).toBe("2.0.0-rc.2"); + expect(findVersion(data, versions, {...rangeOpts, semvers: new Set(["patch"]), cooldownDays: 3650, + now: Date.parse("2025-01-03T00:00:00Z")})).toBe(null); }); test("findVersion selects by version even when publish dates disagree", () => { const data = { versions: {"1.1.0": {}, "1.2.0": {}, "1.3.0": {}}, time: { - "1.1.0": "2025-03-01T00:00:00Z", // a backport published after the higher versions + "1.1.0": "2025-03-01T00:00:00Z", "1.2.0": "2025-01-01T00:00:00Z", "1.3.0": "2025-02-01T00:00:00Z", }, @@ -244,16 +205,6 @@ test("findVersion selects by version even when publish dates disagree", () => { expect(findVersion(data, ["1.1.0", "1.2.0", "1.3.0"], {...opts, range: "1.2.0"})).toBe("1.3.0"); }); -test("findVersion never reports an unpublished release for a prerelease range", () => { - // every candidate filtered out must leave the authored version untouched, not the - // release it is a prerelease of - const data = {versions: {"2.0.0-rc.1": {}, "2.0.0-rc.2": {}}, time: {"2.0.0-rc.1": "2025-01-01T00:00:00Z", "2.0.0-rc.2": "2025-01-02T00:00:00Z"}}; - const versions = ["2.0.0-rc.1", "2.0.0-rc.2"]; - const opts = {range: "^2.0.0-rc.1", useGreatest: false, usePre: false, useRel: false} as const; - expect(findVersion(data, versions, {...opts, semvers: new Set(["patch"])})).toBe("2.0.0-rc.2"); - expect(findVersion(data, versions, {...opts, semvers: new Set(["patch"]), cooldownDays: 3650, now: Date.parse("2025-01-03T00:00:00Z")})).toBe(null); -}); - test.each([ ["a string repository URL", {repository: "https://github.com/user/repo"}, null, "pkg", "https://github.com/user/repo"], ["an object repository with a directory", {repository: {type: "git", url: "https://github.com/user/repo", directory: "packages/foo"}}, @@ -287,7 +238,6 @@ const preLatest = (latest: string) => ({ test.each([ ["wildcard range returns null", twoVersions, {range: "*"}, null], - // Ranked against the last branch, so the authored version is 2.0.0 and 1.1.0 no upgrade ["or-chain resolves against its newest branch", threeVersions, {range: "^1.0.0 || ^2.0.0", semvers: new Set(["minor"])}, null], ["compound range resolves", threeVersions, {range: ">=1.0.0 <2.0.0"}, "2.0.0"], ["useGreatest returns version directly", threeVersions, {range: "1.0.0", useGreatest: true}, "2.0.0"], @@ -306,14 +256,9 @@ test.each([ time: {"1.0.0": "2025-01-01", "1.0.1": "2025-02-01", "2.0.0": "2025-03-01"}}, {range: "1.0.0", semvers: new Set(["patch"])}, "1.0.1"], ["useRel with prerelease latest", preLatest("2.0.0-rc.1"), {range: "1.0.0", useRel: true}, "1.1.0"], - // --release turns the prereleases --prerelease opted into back off, leaving plain latest mode, - // so the 2.0.0 the maintainer never tagged stays behind the ceiling ["--release turns --prerelease back to releases", {name: "pkg", "dist-tags": {latest: "1.1.0"}, versions: {"1.0.0": {}, "1.1.0": {}, "2.0.0-rc.1": {}, "2.0.0": {}}}, {range: "1.0.0", usePre: true, useRel: true}, "1.1.0"], ["latestTag is prerelease, latest mode", preLatest("2.0.0-beta.1"), {range: "1.0.0"}, "1.1.0"], - // Abbreviated metadata (no time field) so findVersion picks the greatest in-range candidate. - // latest dist-tag (1.9.9) is below the installed 2.0.0, so the downgrade guard must not - // discard the valid 2.0.1 upgrade. ["falls back to in-range upgrade when latest dist-tag is a lower release", {name: "pkg", "dist-tags": {latest: "1.9.9"}, versions: {"1.9.9": {}, "2.0.0": {}, "2.0.1": {}}}, {range: "2.0.0"}, "2.0.1"], @@ -328,21 +273,15 @@ test.each([ ["deprecated versions stay in reach of a version that is itself deprecated", {name: "pkg", "dist-tags": {latest: "2.0.0"}, versions: {"1.0.0": {deprecated: true}, "1.1.0": {}, "2.0.0": {deprecated: true}}}, {range: "1.0.0"}, "2.0.0"], - // a deprecated tag is still the ceiling, so the 3.0.0 the maintainer never tagged stays out of - // reach, and the releases below it are no downgrade target either ["a deprecated latest does not promote an off-tag release", {name: "pkg", "dist-tags": {latest: "2.0.0"}, versions: {"1.0.0": {}, "1.1.0": {}, "2.0.0": {deprecated: true}, "3.0.0": {}}}, {range: "1.0.0"}, "1.1.0"], ["a deprecated latest is no downgrade target", {name: "pkg", "dist-tags": {latest: "2.0.0"}, versions: {"1.0.0": {}, "1.1.0": {}, "2.0.0": {deprecated: true}, "3.0.0": {}}}, {range: "3.0.0", allowDowngrade: true}, null], - // semver orders alphanumeric prerelease identifiers lexically, so rc99 outranks rc331 while the - // maintainer only ever tagged rc331. respectLatest keeps the untagged one out of reach, and lets - // a train that already runs past the tag carry on regardless. ["a prerelease past the latest tag is out of reach", preRcs, {range: "0.6.0-rc330"}, "0.6.0-rc331"], ["a prerelease train past the latest tag still moves", preRcs, {range: "0.6.0-rc98"}, "0.6.0-rc99"], ["--prerelease takes the one past the latest tag", preRcs, {range: "0.6.0-rc330", usePre: true}, "0.6.0-rc99"], - // `^10` coerces to a version the package never published, so the exemption above cannot fire ["a wholly deprecated package keeps its newest release for a range naming no published version", {name: "pkg", "dist-tags": {latest: "10.1.0"}, versions: {"10.0.1": {deprecated: true}, "10.1.0": {deprecated: true}}}, {range: "^10"}, "10.1.0"], @@ -377,6 +316,20 @@ test.each([ expect(findNewVersion(data, {...pypiOpts, ...opts} as any)).toBe(expected); }); +test("findNewVersion filters PyPI files by yank and earliest upload", () => { + const data = {info: {version: "1.3.0"}, releases: { + "1.0.0": [{upload_time_iso_8601: "2025-01-01T00:00:00Z"}], + "1.1.0": [ + {upload_time_iso_8601: "2026-04-24T00:00:00Z", yanked: true}, + {upload_time_iso_8601: "2026-04-01T00:00:00Z"}, + ], + "1.2.0": [{upload_time_iso_8601: "2026-04-24T00:00:00Z"}], + "1.3.0": [{upload_time_iso_8601: "2026-04-01T00:00:00Z", yanked: true}], + }}; + expect(findNewVersion(data, {...pypiOpts, range: "1.0.0", cooldownDays: 5, + now: Date.parse("2026-04-25T00:00:00Z")})).toBe("1.1.0"); +}); + test("findNewVersion does not follow an unstable train across a major", () => { const data = { name: "react", @@ -392,17 +345,14 @@ test("findNewVersion does not follow an unstable train across a major", () => { test("findNewVersion tolerates a packument missing versions or naming an absent latest", () => { expect(findNewVersion({name: "pkg", "dist-tags": {latest: "2.0.0"}}, {...npmOpts, range: "1.0.0"})).toBe(null); - // a latest dist-tag the registry does not carry would write a version npm cannot resolve expect(findNewVersion({name: "pkg", "dist-tags": {latest: "9.9.9"}, versions: {"1.0.0": {}, "1.1.0": {}}}, {...npmOpts, range: "1.0.0"})).toBe("1.1.0"); }); -// go mode reads the resolved versions off `data` rather than a packument const goOpts = {mode: "go", useGreatest: false, usePre: false, useRel: false, semvers: new Set(["patch", "minor", "major"]), ...defaultOpts}; const goData = {name: "github.com/foo/bar", old: "1.0.0", new: "3.0.0", Time: "2025-03-01"}; const goSameMajor = (sameMajorNew: string) => ({...goData, sameMajorNew, sameMajorTime: "2025-02-01"}); -// coercing a prerelease pin away would compare 0.4.2-0.2023… against 0.4.2 as equal and stall const pseudo = "0.4.2-0.20230802210424-5b0b94c5c0d3"; test.each([ @@ -410,6 +360,8 @@ test.each([ ["the same-major fallback when major is filtered out", goSameMajor("1.5.0"), {range: "1.0.0", semvers: new Set(["patch", "minor"])}, "1.5.0"], ["a pseudo-version pin moved to its release", {...goData, old: pseudo, new: "0.4.2"}, {range: pseudo}, "0.4.2"], + ["a newer pseudo-version candidate", {...goData, old: "0.4.2", new: "0.4.3-0.20260821120000-6c1a2b3c4d5e"}, + {range: "0.4.2", semvers: new Set(["patch"])}, "0.4.3-0.20260821120000-6c1a2b3c4d5e"], ["a prerelease pin moved to its release", {...goData, old: "1.5.0-rc.1", new: "1.5.0"}, {range: "1.5.0-rc.1"}, "1.5.0"], ["nothing when pinnedRange excludes the cross-major target", goData, {range: "1.0.0", semvers: new Set(["major"]), pinnedRange: "<2.0.0"}, null], @@ -421,29 +373,21 @@ test.each([ expect(findNewVersion(data, {...goOpts, ...opts})).toBe(expected); }); -// UPDATES_FORGE_TOKENS is one process-wide slot, and the two tests below hold a value of their -// own across awaits, so neither may run while the other does, under either runner's concurrency. const sequential = test.sequential ?? (test as any).serial ?? test; sequential("getForgeTokens", async () => { - // empty host (unparseable url) -> no token expect(await getForgeTokens("", "https://api.github.com")).toEqual([]); - // foreign forge host without a configured token -> no github fallback - // (github-host delegation is covered with teeth by the fetchForge test below) expect(await getForgeTokens("gitea.example.com", "https://api.github.com")).toEqual([]); const forHost = (host: string) => getForgeTokens(host, "https://api.github.com"); const saved = process.env.UPDATES_FORGE_TOKENS; process.env.UPDATES_FORGE_TOKENS = "localhost:3500:ported,git.example.com:bare"; try { - // a port-qualified entry must not be split at the first colon expect(await forHost("localhost:3500")).toEqual(["ported"]); expect(await forHost("git.example.com")).toEqual(["bare"]); - // another port on a configured host is a different endpoint, and must not inherit its token expect(await forHost("localhost:9999")).toEqual([]); expect(await forHost("git.example.com:8080")).toEqual([]); - // nor may the bare host claim a ported entry, which would hand back `3500:ported` expect(await forHost("localhost")).toEqual([]); } finally { if (saved === undefined) delete process.env.UPDATES_FORGE_TOKENS; @@ -459,7 +403,6 @@ test("parseExtraheaders reads a CI token per host", () => { "http.https://other.example.com/.extraheader AUTHORIZATION: bearer not-basic", ].join("\n")); expect(tokens.get("github.com")).toEqual("gh-tok"); - // a ported instance is its own endpoint, and only `basic` carries the base64 credential expect(tokens.get("gitea.example.com:8443")).toEqual("gitea-tok"); expect(tokens.has("gitea.example.com")).toEqual(false); expect(tokens.has("other.example.com")).toEqual(false); @@ -468,16 +411,10 @@ test("parseExtraheaders reads a CI token per host", () => { const modeCtx = (props: Record): ModeContext => ({fetchTimeout, ...props} as unknown as ModeContext); test("fetchForge only sends github credentials to github hosts", async () => { - // Inject a github token deterministically. `getGithubTokens` reads env per call, so plain - // mutation works under both vitest and bun. The forge host is unique to this test because - // workingTokenCache is module-level: on a CI runner an earlier fetch caches the extraheader - // credential under api.github.com and would short-circuit the injected token. const tokenEnv = ["UPDATES_GITHUB_API_TOKEN", "GITHUB_API_TOKEN", "GH_TOKEN", "GITHUB_TOKEN", "HOMEBREW_GITHUB_API_TOKEN"]; const saved = Object.fromEntries(tokenEnv.map(name => [name, process.env[name]])); for (const name of tokenEnv) delete process.env[name]; process.env.GH_TOKEN = "ghp_regression_secret"; - // Restore in `finally` so a failed assertion can't leak env into concurrent - // sibling tests (isolate: false). try { const authByHost: Record = {}; const ctx = modeCtx({forgeApiUrl: "https://forge.regression.test", @@ -491,7 +428,6 @@ test("fetchForge only sends github credentials to github hosts", async () => { expect(authByHost["forge.regression.test"]).toBe("Bearer ghp_regression_secret"); expect(authByHost["attacker.example"]).toBeUndefined(); - // GitHub's own API hostname still resolves the github credentials expect(await getForgeTokens("api.github.com", "https://api.github.com")).toContain("ghp_regression_secret"); } finally { for (const [name, value] of Object.entries(saved)) { @@ -501,8 +437,29 @@ test("fetchForge only sends github credentials to github hosts", async () => { } }); -// One tag per page, with page 1 announcing `lastPage` through the link header. +sequential("fetchForge does not reuse a cached token removed from the environment", async () => { + const saved = process.env.UPDATES_FORGE_TOKENS; + const authorizations: Array = []; + const ctx = modeCtx({forgeApiUrl: "https://api.github.com", doFetch: (_url: string, opts: RequestInit) => { + authorizations.push((opts.headers as Record)?.Authorization); + return Promise.resolve({ok: true, status: 200, headers: new Headers()}); + }}); + try { + process.env.UPDATES_FORGE_TOKENS = "rotated-token.test:old"; + await fetchForge("https://rotated-token.test/repos/o/r/tags", ctx); + process.env.UPDATES_FORGE_TOKENS = "rotated-token.test:new"; + await expect(fetchForge("https://rotated-token.test/repos/o/r/tags", ctx)).resolves.toMatchObject({status: 200}); + expect(authorizations).toEqual(["Bearer old", "Bearer new"]); + } finally { + if (saved === undefined) delete process.env.UPDATES_FORGE_TOKENS; + else process.env.UPDATES_FORGE_TOKENS = saved; + } +}); + const tagPage = (url: string, lastPage: number) => { + if (new URL(url).pathname.endsWith("/releases")) { + return {ok: true, json: () => Promise.resolve([]), headers: new Headers()}; + } const page = Number(new URL(url).searchParams.get("page")); return { ok: true, @@ -513,14 +470,38 @@ const tagPage = (url: string, lastPage: number) => { test("fetchActionTags single page no link header", async () => { const tagsData = [{name: "v1.0.0", commit: {sha: "abc"}}, {name: "v2.0.0", commit: {sha: "def"}}]; - const ctx = modeCtx({doFetch: () => Promise.resolve({ok: true, json: () => Promise.resolve(tagsData), headers: new Headers()})}); + const ctx = modeCtx({doFetch: (url: string) => Promise.resolve({ok: true, + json: () => Promise.resolve(new URL(url).pathname.endsWith("/releases") ? [ + {tag_name: "v1.0.0", prerelease: false, draft: false}, + {tag_name: "v2.0.0", prerelease: true, draft: false}, + ] : tagsData), headers: new Headers()})}); const result = await fetchActionTags("https://api.github.com", "actions", "checkout", ctx); - expect(result).toEqual([{name: "v1.0.0", commitSha: "abc"}, {name: "v2.0.0", commitSha: "def"}]); + expect(result).toEqual([ + {name: "v1.0.0", commitSha: "abc", isStable: true}, + {name: "v2.0.0", commitSha: "def", isStable: false}, + ]); + + const malformed = modeCtx({noCache: true, doFetch: (url: string) => Promise.resolve({ok: true, + json: () => Promise.resolve(new URL(url).pathname.endsWith("/releases") ? [ + {tag_name: null, prerelease: false}, + ] : tagsData), headers: new Headers()})}); + await expect(fetchActionTags("https://api.github.com", "actions", "checkout", malformed)) + .rejects.toThrow("Invalid Forge release entry"); +}); + +test("fetchActionTags skips release metadata when stability is unused", async () => { + const urls: Array = []; + const ctx = modeCtx({noCache: true, doFetch: (url: string) => { + urls.push(url); + return Promise.resolve({ok: true, json: () => Promise.resolve([]), headers: new Headers()}); + }}); + await fetchActionTags("https://api.github.com", "actions", "checkout", ctx, [], false); + expect(urls).toHaveLength(1); + expect(urls[0]).toContain("/tags?"); }); test("fetchActionTags walks until the authored ref turns up, and no further", async () => { const lastPage = 40; - // [tags read, pages fetched] const walk = async (refs: Array) => { let fetched = 0; const ctx = modeCtx({noCache: true, doFetch: (url: string) => { @@ -529,10 +510,9 @@ test("fetchActionTags walks until the authored ref turns up, and no further", as }}); return [(await fetchActionTags("https://api.github.com", "actions", "checkout", ctx, refs)).length, fetched]; }; - expect(await walk([])).toEqual([lastPage, lastPage]); // no ref to look for, so the whole list - expect(await walk(["v1.0.0"])).toEqual([1, 1]); - // waves of 1, 2, 4 and 8 reach page 11, so the walk reads 16 pages to resolve a sha on it - expect(await walk(["sha11"])).toEqual([16, 16]); + expect(await walk([])).toEqual([lastPage, lastPage + 1]); + expect(await walk(["v1.0.0"])).toEqual([1, 2]); + expect(await walk(["sha11"])).toEqual([16, 17]); }); test("every request shares the run's one socket budget", async () => { @@ -542,12 +522,10 @@ test("every request shares the run's one socket budget", async () => { const ctx = modeCtx({noCache: true, concurrency: 3, doFetch: async (url: string) => { inFlight++; peak = Math.max(peak, inFlight); - await new Promise(resolve => setImmediate(resolve)); // every admitted request is in flight by now + await new Promise(resolve => setImmediate(resolve)); inFlight--; return tagPage(url, lastPage); }}); - // Each fan already runs inside the fan over dependencies, so a budget of its own would multiply. - // The last group is the docker walk's shape, a slot taken above one it must pass straight through. const limit = getLimiter(ctx); const [tags] = await Promise.all([ fetchActionTags("https://api.github.com", "o", "r", ctx), @@ -558,13 +536,22 @@ test("every request shares the run's one socket budget", async () => { expect(peak).toBe(3); }); +test.each([429, 503, 403])("fetchWithRetry retries returned %s responses twice", async status => { + let calls = 0; + const ctx = modeCtx({noCache: true, doFetch: () => { + calls++; + return Promise.resolve({ok: false, status, headers: new Headers(status === 403 ? [["retry-after", "0"]] : [])}); + }}); + expect((await fetchWithRetry(ctx, `https://retry-${status}.test`)).status).toBe(status); + expect(calls).toBe(3); +}); + sequential("fetchForge classifies rate limits and server faults, fetchActionTags lets them through", async () => { const reset = Math.floor(Date.parse("2026-05-01T00:00:00Z") / 1000); - // keyed by hostname label so each case gets a host of its own, as workingTokenCache is module-level const responses: Record> = { limited: {status: 403, headers: new Headers([["x-ratelimit-remaining", "0"], ["x-ratelimit-reset", String(reset)]])}, secondary: {status: 403, headers: new Headers(), json: () => Promise.resolve({message: "You have exceeded a secondary rate limit"})}, - retryafter: {status: 429, headers: new Headers([["retry-after", "60"]])}, + retryafter: {status: 429, headers: new Headers([["retry-after", "0"]])}, down: {status: 502, statusText: "Bad Gateway", headers: new Headers()}, forbidden: {status: 403, headers: new Headers(), json: () => Promise.resolve({message: "Resource not accessible by integration"})}, tokened: {status: 403, headers: new Headers([["x-ratelimit-remaining", "0"]])}, @@ -612,8 +599,15 @@ test("fetchActionTags reports an unreachable forge instead of an empty tag list" .rejects.toMatchObject({name: "ForgeError", kind: "network"}); }); -// Tests use timestamped URLs so each invocation hashes to a unique cache file; -// real-cache side effects are isolated. +test.each([ + ["invalid JSON", {json: () => Promise.reject(new SyntaxError("bad JSON")), headers: new Headers()}], + ["malformed pagination", {json: () => Promise.resolve([]), + headers: new Headers([["link", "; rel=\"last\""]])}], +])("fetchActionTags reports %s", async (_name, response) => { + const ctx = modeCtx({noCache: true, doFetch: () => Promise.resolve({ok: true, ...response})}); + await expect(fetchActionTags("https://api.github.com", "actions", "checkout", ctx)).rejects.toThrow(); +}); + const ifNoneMatch = (opts: RequestInit) => (opts.headers as Record | undefined)?.["if-none-match"]; test("fetchWithEtag returns body on 200 and sends If-None-Match on second call", async () => { @@ -635,6 +629,10 @@ test("fetchWithEtag returns body on 200 and sends If-None-Match on second call", const r2 = await fetchWithEtag(url, ctx); expect("body" in r2).toBe(true); expect(lastIfNoneMatch).toBe(`W/"1"`); + const failed = await fetchWithEtag("https://example.test/404", modeCtx({noCache: true, + doFetch: () => Promise.resolve({ok: false, status: 404, statusText: "Not Found", headers: new Headers()})})); + expect("body" in failed).toBe(false); + expect(failed.res?.status).toBe(404); }); test("fetchWithEtag returns cached body on 304", async () => { @@ -665,19 +663,10 @@ test("fetchWithEtag keeps flavors of one url in separate cache entries", async ( await fetchWithEtag(url, ctx); await flushCacheWrites(); - // a registry that etags per url alone would revalidate the abbreviated body into this call await fetchWithEtag(url, ctx, {}, undefined, `${url}\0dates`); expect(seenIfNoneMatch).toBeUndefined(); }); -test("fetchWithEtag returns {res} on non-ok", async () => { - const ctx = modeCtx({noCache: true, - doFetch: () => Promise.resolve({ok: false, status: 404, statusText: "Not Found", headers: new Headers()})}); - const r = await fetchWithEtag("https://example.test/404", ctx); - expect("body" in r).toBe(false); - expect(r.res?.status).toBe(404); -}); - test("fetchImmutable serves cached body without fetching on second call", async () => { const url = `https://example.test/immutable-${Date.now()}`; let calls = 0; @@ -711,4 +700,3 @@ test.each([["fetchWithEtag", fetchWithEtag], ["fetchImmutable", fetchImmutable]] expect(seenIfNoneMatch).toBeUndefined(); expect(calls).toBe(2); }); - diff --git a/modes/shared.ts b/modes/shared.ts index faf30f9..055edb6 100644 --- a/modes/shared.ts +++ b/modes/shared.ts @@ -1,7 +1,10 @@ import {AsyncLocalStorage} from "node:async_hooks"; import {Buffer} from "node:buffer"; import {env} from "node:process"; -import {type Versioning, coerce, diff, gt, satisfies, semverVersioning, pep440Versioning, valid} from "../utils/semver.ts"; +import {setTimeout as delay} from "node:timers/promises"; +import { + type Versioning, coerce, diff, gt, satisfies, semverVersioning, pep440Versioning, valid, +} from "../utils/semver.ts"; import {getCache, setCache} from "../utils/fetchCache.ts"; import {commaSeparatedToArray, getOrSet} from "../utils/utils.ts"; import pkg from "../package.json" with {type: "json"}; @@ -9,62 +12,27 @@ import pkg from "../package.json" with {type: "json"}; export type {Config} from "../config.ts"; export type Dep = { - old: string, - new: string, - oldPrint?: string, - newPrint?: string, - oldOrig?: string, - info?: string, - age?: string, - date?: string, -}; - -export type Deps = { - [name: string]: Dep, -}; - -export type DepsByMode = { - [mode: string]: Deps, + old: string, new: string, oldPrint?: string, newPrint?: string, oldOrig?: string, info?: string, age?: string, + date?: string, oldDigest?: string, newDigest?: string, digestOnly?: boolean, }; +export type Deps = {[name: string]: Dep}; +export type DepsByMode = {[mode: string]: Deps}; export type Output = { - results: { - [mode: string]: { - [type: string]: Deps, - } - }, + results: {[mode: string]: {[type: string]: Deps}}, message?: string, }; -export type CooldownOpts = { - cooldownDays?: number, - now?: number, - // A version whose date is unknown never passes an active cooldown, so only return published dates. - getVersionDate?: (version: string) => string | undefined, -}; +export type CooldownOpts = {cooldownDays?: number, now?: number, + getVersionDate?: (version: string) => string | undefined}; export type FindVersionOpts = { - range: string, - semvers: Set, - useGreatest: boolean, - usePre: boolean, - useRel: boolean, - // The tag published for the package, which selection may not climb past outside greatest mode. - latest?: string, - pinnedRange?: string, - pinNoDowngrade?: boolean, - // Already resolved against the --allow-downgrade patterns by the caller, for every name the - // dependency answers to, so selection only reads the verdict. - allowDowngrade?: boolean, - versioning?: Versioning, + range: string, semvers: Set, useGreatest: boolean, usePre: boolean, useRel: boolean, latest?: string, + pinnedRange?: string, pinNoDowngrade?: boolean, allowDowngrade?: boolean, versioning?: Versioning, } & CooldownOpts; -export type FindNewVersionOpts = Omit & { - mode: string, -}; +export type FindNewVersionOpts = Omit & {mode: string}; -// An active cooldown requires a timestamp, as renovate's minimumReleaseAgeBehaviour default does: -// a mirror omitting the newest release's date would let through the release it exists to hold back. export function passesCooldown(date: string | undefined, cooldownDays: number | undefined, now: number | undefined): boolean { if (!cooldownDays || !now) return true; const ms = date ? Date.parse(date) : NaN; @@ -72,56 +40,30 @@ export function passesCooldown(date: string | undefined, cooldownDays: number | return (now - ms) / (24 * 3600 * 1000) >= cooldownDays; } -// [data, registry]; registry is null for modes that have no per-package registry. export type PackageInfo = [Record, string | null]; -export type PackageRepository = string | { - type: string, - url: string, - directory: string, -}; +export type PackageRepository = string | {type: string, url: string, directory: string}; -// Lives here rather than in the go mode so ModeContext can name it without a cycle. export type GoProxyEntry = {url: string, fallback: "," | "|"}; export type ModeContext = { - fetchTimeout: number, - goProbeTimeout: number, - /** The run's socket budget, `--sockets` or `maxSockets` */ - concurrency: number, - forgeApiUrl: string, - pypiApiUrl: string, - jsrApiUrl: string, - /** The single endpoint a caller that can only address one uses, `goProxyChain`'s first entry */ - goProxyUrl: string, - goProxyChain: Array, - cratesIoUrl: string, - dockerApiUrl: string, - doFetch: typeof doFetch, - execFile: ExecFile, // subprocess seam, as doFetch is for the network - noCache: boolean, + fetchTimeout: number, goProbeTimeout: number, concurrency: number, forgeApiUrl: string, pypiApiUrl: string, + jsrApiUrl: string, goProxyUrl: string, goProxyChain: Array, cratesIoUrl: string, dockerApiUrl: string, + doFetch: typeof doFetch, execFile: ExecFile, noCache: boolean, }; -export type ExecFile = ( - file: string, args: Array, opts: Record, -) => Promise<{stdout: string, stderr: string}>; +export type ExecFile = (file: string, args: Array, opts: Record) => +Promise<{stdout: string, stderr: string}>; export const packageVersion = pkg.version; export const fieldSep = "\0"; export const fetchTimeout = 5000; export const goProbeTimeout = 2500; export const maxSockets = 25; -// Tag lists are walked to the end: a truncated walk loses the tag a pinned sha resolves to, and on -// Docker Hub hides a same-precision tag past the window. Renovate likewise stops only at this cap. export const maxTagPages = 100; -// GitHub serves its API from a hostname of its own, unlike Gitea and Forgejo which serve -// /api/v1 from the forge host itself. Also the default forge, hence forgeapi below. export const githubApiUrl = "https://api.github.com"; -// Default endpoint per API override flag, no trailing slash. The single source for the URLs -// requests are built from and for the origins prewarming opens sockets to, so the two cannot -// name different hosts. export const defaultApiUrls = { registry: "https://registry.npmjs.org", jsrapi: "https://jsr.io", @@ -132,6 +74,7 @@ export const defaultApiUrls = { goproxy: "https://proxy.golang.org", } as const; export const fetchRetries = 2; +const maxRetryAfter = 60000; export const stripv = (str: string): string => str[0] === "v" ? str.substring(1) : str; export const normalizeUrl = (url: string) => url.endsWith("/") ? url.slice(0, -1) : url; @@ -146,7 +89,6 @@ export function getFetchOpts(authType?: string, authToken?: string): RequestInit }; } -// Retryable failures; deterministic ones (bad URL, NXDOMAIN, TLS) are not. const transientErrorCodes = new Set([ "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EAI_AGAIN", "EPIPE", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT", @@ -154,7 +96,6 @@ const transientErrorCodes = new Set([ export function isTransientFetchError(err: any): boolean { if (err?.name === "TimeoutError" || err?.name === "AbortError") return true; - // fetch wraps socket/DNS errors as a TypeError with the real error in `cause`. const code = err?.code ?? err?.cause?.code; return typeof code === "string" && transientErrorCodes.has(code); } @@ -169,151 +110,128 @@ export async function doFetch(url: string, opts?: RequestInit): Promise { const limit = getLimiter(ctx); for (let attempt = 0; ; attempt++) { try { - return await limit(() => ctx.doFetch(url, {...opts, signal: AbortSignal.timeout(ctx.fetchTimeout)})); + const res = await limit(() => ctx.doFetch(url, {...opts, signal: AbortSignal.timeout(ctx.fetchTimeout)})); + const value = res?.headers?.get?.("retry-after")?.trim(); + const date = value && !/^\d+$/.test(value) ? Date.parse(value) : NaN; + const retryAfter = !value ? null : /^\d+$/.test(value) ? Number(value) * 1000 : + Number.isNaN(date) ? null : Math.max(date - Date.now(), 0); + const retryDelay = res && res.status >= 500 && res.status < 600 ? + (retryAfter !== null && retryAfter <= maxRetryAfter ? retryAfter : 0) : res && + (res.status === 429 || res.status === 403 && retryAfter !== null) && + (retryAfter === null || retryAfter <= maxRetryAfter) ? retryAfter ?? 0 : null; + if (retryDelay === null || attempt >= fetchRetries) return res; + if (res.body) await res.body.cancel(); + if (retryDelay) await delay(retryDelay); } catch (err: any) { if (attempt >= fetchRetries || !err?.transient) throw err; } } } -// Read a Response body as text, falling back to JSON re-stringification for -// lightweight mocks in tests. -async function readBody(res: Response): Promise { - if (typeof res.text === "function") return res.text(); - return JSON.stringify(await res.json()); -} - -// Shrink a body to the subset of fields a mode actually reads before it is -// cached or returned. Must keep the original document shape so legacy cache -// entries holding full bodies stay readable. Bodies below the threshold are -// kept as-is: reducing them costs more than the re-parse it saves. export type BodyReducer = (body: string) => string; const reduceThreshold = 16384; export const reduceJson = (reduce: (data: any) => any): BodyReducer => body => JSON.stringify(reduce(JSON.parse(body))); -function reduceBody(body: string, reduce: BodyReducer | undefined): string { - if (!reduce || body.length < reduceThreshold) return body; - try { - return reduce(body); - } catch { - return body; // non-JSON or unexpected shape: cache as-is +type FetchResult = {body: string, res?: Response} | {res: Response | undefined}; +const fetchesByCtx = new WeakMap>>(); + +function fetchCached( + url: string, ctx: ModeContext, opts: RequestInit, reduce: BodyReducer | undefined, cacheKey: string, immutable: boolean, +): Promise { + const requestKey = JSON.stringify([url, cacheKey, immutable, opts.method ?? "GET", + Array.from(new Headers(opts.headers).entries()).sort(), opts.body ?? null, reduce?.toString()]); + const requests = getOrSet(fetchesByCtx, ctx, () => new Map>()); + let request = requests.get(requestKey); + if (!request) { + request = (async () => { + try { + const cached = ctx.noCache ? null : await getCache(cacheKey); + if (immutable && cached) return {body: cached.body}; + const baseHeaders = opts.headers as Record | undefined; + const headers = cached ? {...baseHeaders, "if-none-match": cached.etag} : baseHeaders; + const res = await fetchWithRetry(ctx, url, {...opts, headers}); + if (res.status === 304 && cached) return {body: cached.body, res}; + if (!res.ok) return {res}; + let body = await res.text(); + if (reduce && body.length >= reduceThreshold) { + try { body = reduce(body); } catch {} + } + const etag = immutable ? "immutable" : res.headers.get("etag"); + if (etag && !ctx.noCache) setCache(cacheKey, etag, body); + return {body, res}; + } finally { + requests.delete(requestKey); + } + })(); + requests.set(requestKey, request); } + return request; } -// Read and reduce a response body, persisting it under `cacheTag` when caching -// is on. Never-revalidated URLs pass the literal "immutable" tag. -async function readAndCache( - cacheKey: string, res: Response, ctx: ModeContext, reduce: BodyReducer | undefined, cacheTag: string | null | undefined, -): Promise<{body: string, res: Response}> { - const body = reduceBody(await readBody(res), reduce); - if (cacheTag && !ctx.noCache) setCache(cacheKey, cacheTag, body); - return {body, res}; +export function fetchWithEtag( + url: string, ctx: ModeContext, opts: RequestInit = {}, reduce?: BodyReducer, cacheKey: string = url, +): Promise { + return fetchCached(url, ctx, opts, reduce, cacheKey, false); } -// Fetch with ETag revalidation against the persistent disk cache. The timeout -// signal is created after the cache read, so slow disks do not eat the network -// budget. Returns {body} on success, or {res} on error. -// `cacheKey` separates responses that share a url but vary by request header, which a registry -// that etags per url alone would revalidate into one another. -export async function fetchWithEtag( - url: string, ctx: ModeContext, opts: RequestInit = {}, reduce?: BodyReducer, cacheKey: string = url, -): Promise<{body: string, res?: Response} | {res: Response | undefined}> { - const cached = ctx.noCache ? null : await getCache(cacheKey); - const baseHeaders = opts.headers as Record | undefined; - const headers = cached ? {...baseHeaders, "if-none-match": cached.etag} : baseHeaders; - const res = await fetchWithRetry(ctx, url, {...opts, headers}); - if (!res) return {res: undefined}; - if (res.status === 304 && cached) return {body: cached.body, res}; - if (!res.ok) return {res}; - return readAndCache(cacheKey, res, ctx, reduce, res.headers?.get?.("etag")); -} - -// Persistent cache for immutable URLs (e.g. per-version metadata, commit -// dates). No revalidation — once cached, reused forever. -export async function fetchImmutable( +export function fetchImmutable( url: string, ctx: ModeContext, opts: RequestInit = {}, reduce?: BodyReducer, -): Promise<{body: string, res?: Response} | {res: Response | undefined}> { - if (!ctx.noCache) { - const cached = await getCache(url); - if (cached) return {body: cached.body}; - } - const res = await fetchWithRetry(ctx, url, opts); - if (!res) return {res: undefined}; - if (!res.ok) return {res}; - return readAndCache(url, res, ctx, reduce, "immutable"); -} - -// Share one in-flight/completed promise per key so concurrent lookups for the -// same resource issue a single request. A rejected promise is evicted so the -// next caller retries rather than inheriting the failure forever. Keyed per run, -// so a second updates() call in one process re-requests rather than answering -// from the finished run's map. -export function dedupe(byCtx: WeakMap>>, ctx: ModeContext, key: string, fn: () => Promise): Promise { +): Promise { + return fetchCached(url, ctx, opts, reduce, url, true); +} + +export async function dedupe(byCtx: WeakMap>>, ctx: ModeContext, key: string, fn: () => Promise): Promise { const cache = getOrSet(byCtx, ctx, () => new Map>()); - let promise = cache.get(key); - if (!promise) { - cache.set(key, promise = (async () => { - try { - return await fn(); - } catch (err) { - cache.delete(key); - throw err; - } - })()); + const promise = cache.get(key); + if (promise) return promise; + const request = fn(); + cache.set(key, request); + try { + return await request; + } catch (err) { + cache.delete(key); + throw err; } - return promise; } export type Limiter = (fn: () => Promise) => Promise; -// Set for the duration of a slot. Acquiring the same budget twice for one request deadlocks at -// saturation, so a limiter reached from inside a slot passes straight through. const inSlot = new AsyncLocalStorage(); -function createLimiter(concurrency: number): Limiter { - let active = 0; - let head = 0; - let waiting: Array<() => void> = []; - return async (fn: () => Promise): Promise => { - if (inSlot.getStore()) return fn(); - if (active < concurrency) active++; - else await new Promise(resolve => { waiting.push(resolve); }); - try { - return await inSlot.run(true, fn); - } finally { - // A cursor, not `shift`, so releasing a slot is O(1) with a large `--sockets` queue. - if (head < waiting.length) { - waiting[head++](); - if (head === waiting.length) { - waiting = []; - head = 0; - } - } else { - active--; - } - } - }; -} - -export const effectiveConcurrency = (ctx: ModeContext): number => Math.max(ctx.concurrency || maxSockets, 1); +const effectiveConcurrency = (ctx: ModeContext): number => Math.max(ctx.concurrency || maxSockets, 1); const limiterByCtx = new WeakMap(); export function getLimiter(ctx: ModeContext): Limiter { let limiter = limiterByCtx.get(ctx); - if (!limiter) limiterByCtx.set(ctx, limiter = createLimiter(effectiveConcurrency(ctx))); + if (!limiter) { + const concurrency = effectiveConcurrency(ctx); + let active = 0; + let head = 0; + let waiting: Array<() => void> = []; + limiter = async (fn: () => Promise): Promise => { + if (inSlot.getStore()) return fn(); + if (active < concurrency) active++; + else await new Promise(resolve => { waiting.push(resolve); }); + try { + return await inSlot.run(true, fn); + } finally { + if (head < waiting.length) { + waiting[head++](); + if (head === waiting.length) { waiting = []; head = 0; } + } else active--; + } + }; + limiterByCtx.set(ctx, limiter); + } return limiter; } @@ -322,44 +240,22 @@ export function isVersionPrerelease(version: string, versioning: Versioning = se return Boolean(parsed && versioning.isPrerelease(parsed)); } -// Build a prerelease-augmented copy of a semvers set without mutating the -// input — getVersionOpts() caches its sets per-package, so mutating in place -// silently leaks state across packages. -function cachedVariants(cache: WeakMap, Set>, semvers: Set, add: (out: Set) => void): Set { - const cached = cache.get(semvers); - if (cached) return cached; - const out = new Set(semvers); - add(out); - cache.set(semvers, out); - return out; -} - const allPrereleaseCache = new WeakMap, Set>(); const sameReleasePrereleaseCache = new WeakMap, Set>(); -// Prerelease candidates are in play when --prerelease is set or the authored version already is -// one, and classifying against an uncoerced prerelease yields `pre*` diffs the raw set lacks. -// An authored prerelease alone only reaches prereleases of its own release, as renovate keeps an -// unstable candidate only when major, minor and patch match, so a `17.0.0-rc.0` pin must not -// follow an unreleased 18.x canary train. --prerelease opts into every one, as ignoreUnstable=false -// does, and --release turns them all away again, as ncu's `--pre 0` overrides its own auto-enable. -// Every mode filters through the returned `skipsPrerelease`, as renovate keeps its unstable filter -// in the shared lookup rather than per datasource. export function prereleaseOpts(range: string, usePre: boolean, useRel: boolean, semvers: Set, versioning: Versioning = semverVersioning) { const anyPrerelease = usePre || versioning.isRangePrerelease(range); let effectiveSemvers = semvers; - if (usePre) { - effectiveSemvers = cachedVariants(allPrereleaseCache, semvers, out => { - out.add("prerelease"); - if (semvers.has("patch")) out.add("prepatch"); - if (semvers.has("minor")) out.add("preminor"); - if (semvers.has("major")) out.add("premajor"); - }); - } else if (anyPrerelease) { - effectiveSemvers = cachedVariants(sameReleasePrereleaseCache, semvers, out => out.add("prerelease")); + if (anyPrerelease) { + const cache = usePre ? allPrereleaseCache : sameReleasePrereleaseCache; + effectiveSemvers = cache.get(semvers) ?? new Set(semvers).add("prerelease"); + if (usePre) { + if (semvers.has("patch")) effectiveSemvers.add("prepatch"); + if (semvers.has("minor")) effectiveSemvers.add("preminor"); + if (semvers.has("major")) effectiveSemvers.add("premajor"); + } + cache.set(semvers, effectiveSemvers); } - // An unparseable candidate is no prerelease, matching isVersionPrerelease, so the go mode can - // hand its raw strings straight in. const skipsPrerelease = (parsed: any) => (!anyPrerelease || useRel) && Boolean(parsed) && versioning.isPrerelease(parsed); return {effectiveSemvers, skipsPrerelease}; } @@ -369,59 +265,37 @@ export function coerceToVersion(rangeOrVersion: string): string { } export function findVersion(data: any, versions: Array, {range, semvers, useGreatest, usePre, useRel, latest, pinnedRange, pinNoDowngrade, allowDowngrade, cooldownDays, now, getVersionDate, versioning = semverVersioning}: FindVersionOpts): string | null { - // Rank and classify against the authored version with its prerelease intact. Coercing - // drops it, which would sort a prerelease pin above its own release. const oldParsed = versioning.parseRange(range); if (!oldParsed) return null; const {effectiveSemvers, skipsPrerelease} = prereleaseOpts(range, usePre, useRel, semvers, versioning); - // renovate's respectLatest: the tag is a ceiling, so a release the maintainer published without - // blessing it stays out of reach however it sorts. A version already past the tag may still climb - // further, as renovate exempts it. --greatest asks for the greatest version there is, which is to - // say no ceiling at all, and so does --prerelease unless --release takes its prereleases away. - // A pin whose range the tag falls outside is a target of its own, so it lifts the ceiling too. const latestParsed = latest ? versioning.parse(latest) : null; const ceiling = useGreatest || (usePre && !useRel) || (latestParsed && pinnedRange && !versioning.satisfiesRange(latestParsed, pinnedRange)) ? null : latestParsed; const pastCeiling = Boolean(ceiling && versioning.compare(oldParsed, ceiling) > 0); - // Two things open a step down, onto different targets: an authored pin the version already - // violates moves into the pin's range, while --allow-downgrade follows the tag the maintainer - // stepped back to. A renovate-derived pin opens neither, being a ceiling rather than a target. - // --allow-downgrade lands on the tag itself, so a tag that is gone or deprecated offers nowhere - // to land. With no tag published there is nothing to land on but the release below a prerelease - // train, which is the only step down worth taking blind. const intoPin = Boolean(pinnedRange) && !pinNoDowngrade && !versioning.satisfiesRange(oldParsed, pinnedRange!); const ontoTag = Boolean(allowDowngrade) && (Boolean(ceiling) || versioning.isPrerelease(oldParsed)); const time = data?.time; const cooldownActive = Boolean(cooldownDays && now); - // Highest candidate that passes every check, as renovate takes the first walking high to low. - // A publish date never outranks a version, so a backport released later cannot win. - let newVersionParsed: {version: string} | null = null; + let newVersionParsed: {raw?: string, version: string} | null = null; for (const version of versions) { const parsed = versioning.parse(version); if (!parsed || skipsPrerelease(parsed)) continue; - // Candidates only ever move forward, matching renovate's release filter. Cheaper than - // the range check below, so it runs first and rejects most of them. const stepDown = versioning.compare(parsed, oldParsed) <= 0; if (stepDown && !intoPin && !ontoTag) continue; if (newVersionParsed && versioning.compare(parsed, newVersionParsed) <= 0) continue; - // A pin moves into its own range, wherever the tag sits; anything else lands on the tag. if (stepDown && !intoPin && ceiling && versioning.compare(parsed, ceiling) !== 0) continue; if (!stepDown && ceiling && !pastCeiling && versioning.compare(parsed, ceiling) > 0) continue; if (pinnedRange && !versioning.satisfiesRange(parsed, pinnedRange)) continue; if (cooldownActive && !passesCooldown(getVersionDate ? getVersionDate(version) : time?.[version], cooldownDays, now)) continue; - // Always classified against the authored version, never against a candidate - // picked earlier, so a chain of small steps cannot add up past the semvers gate. - // A stable candidate is classified by its own level, as the `pre` prefix a diff carries when - // the authored prerelease is the higher of the pair says nothing about it. const d = versioning.diff(oldParsed, parsed); const level = d && !versioning.isPrerelease(parsed) ? d.replace(/^pre/, "") : d; if (!level || !effectiveSemvers.has(level)) continue; @@ -429,20 +303,17 @@ export function findVersion(data: any, versions: Array, {range, semvers, newVersionParsed = parsed; } - return newVersionParsed?.version ?? null; + return newVersionParsed?.raw ?? newVersionParsed?.version ?? null; } -// TODO: maybe include pseudo-versions with --greatest export function isGoPseudoVersion(version: string): boolean { return /\d{14}-[0-9a-f]{12}$/.test(version); } export function findNewVersion(data: any, {mode, range: authoredRange, useGreatest, usePre, useRel, semvers, pinnedRange, pinNoDowngrade, cooldownDays, now, allowDowngrade}: FindNewVersionOpts): string | null { - if (authoredRange === "*") return null; // ignore wildcard + if (authoredRange === "*") return null; const versioning: Versioning = mode === "pypi" ? pep440Versioning : semverVersioning; - // Selection runs against an or-chain's last branch, as renovate reads a range's last comparator: - // `^17.0.0 || ^18.0.0` is an 18. The full range still decides how the update is written back. const range = authoredRange.includes("||") ? authoredRange.split("||").pop()!.trim() : authoredRange; let versions: Array = []; let latestTag = ""; @@ -450,19 +321,21 @@ export function findNewVersion(data: any, {mode, range: authoredRange, useGreate if (mode === "pypi") { const releases = data?.releases; if (!releases) return null; - versions = Object.keys(releases).filter(version => !releases[version]?.some((file: any) => file?.yanked)); - getVersionDate = (version: string) => releases[version]?.[0]?.upload_time_iso_8601; + versions = Object.keys(releases).filter(version => + Array.isArray(releases[version]) && releases[version].some((file: any) => file && !file.yanked)); + getVersionDate = (version: string) => releases[version].reduce( + (earliest: {date?: string, time: number}, file: any) => { + const date = file?.upload_time_iso_8601; + const time = typeof date === "string" ? Date.parse(date) : NaN; + return !Number.isNaN(time) && time < earliest.time ? {date, time} : earliest; + }, {time: Infinity}, + ).date; latestTag = data.info?.version ?? ""; } else if (mode === "npm" || mode === "cargo") { if (!data?.versions) return null; versions = Object.keys(data.versions); latestTag = data["dist-tags"]?.latest ?? ""; - // renovate's ignoreDeprecated: a range resolving to a live version never moves onto a deprecated one, - // npm-only because crates.io version records are `{}` and its yanked releases are dropped at fetch if (mode === "npm" && !data.versions[coerceToVersion(range)]?.deprecated) { - // A wholly deprecated package has nothing else to offer, and its range need not name a - // published version for the exemption above to have been the one that applies. The tag stays - // findVersion's ceiling either way, so a deprecated one still holds back what it published. const live = versions.filter(version => !data.versions[version]?.deprecated); versions = live.length ? live : versions; } @@ -470,30 +343,24 @@ export function findNewVersion(data: any, {mode, range: authoredRange, useGreate const oldVersion = coerceToVersion(range); if (!oldVersion) return null; const {effectiveSemvers, skipsPrerelease} = prereleaseOpts(range, usePre, useRel, semvers, versioning); - // Use full original version for prerelease detection (range is shortened for Go) const originalOldVersion = data.old || range; const oldParsed = versioning.parseRange(originalOldVersion); - // A step down off the authored version is only offered with --allow-downgrade, as findVersion does. const mayStepDown = Boolean(allowDowngrade) || (Boolean(pinnedRange) && !pinNoDowngrade && Boolean(oldParsed) && !versioning.satisfiesRange(oldParsed, pinnedRange!)); - // A candidate is taken only if it is a real, allowed upgrade under the active semver, step-down, - // cooldown and pin constraints. No ceiling among them: the proxy's `@latest` is the only - // candidate source, so it already is its own. const accepts = (candidate: string, time: string | undefined): boolean => { const coerced = coerceToVersion(candidate); const parsed = versioning.parse(candidate); - if (!coerced || isGoPseudoVersion(candidate) || skipsPrerelease(parsed)) return false; - // Classify against the authored version first: coercing strips the prerelease, so - // a `-rc.1` or pseudo-version pin would compare equal to its own release and stall. + const pseudo = isGoPseudoVersion(candidate); + if (!coerced || !pseudo && skipsPrerelease(parsed)) return false; const d = diff(originalOldVersion, candidate) ?? diff(oldVersion, coerced); - if (!d || !effectiveSemvers.has(d)) return false; + const level = pseudo ? d?.replace(/^pre/, "") : d; + if (!level || !effectiveSemvers.has(level)) return false; if (!mayStepDown && parsed && oldParsed && versioning.compare(parsed, oldParsed) < 0) return false; if (!passesCooldown(time, cooldownDays, now)) return false; return !pinnedRange || satisfies(coerced, pinnedRange); }; - // Cross-major upgrade, else fall back to same-major. if (accepts(data.new, data.Time)) return data.new; if (accepts(data.sameMajorNew, data.sameMajorTime)) { data.Time = data.sameMajorTime; @@ -502,13 +369,10 @@ export function findNewVersion(data: any, {mode, range: authoredRange, useGreate } return null; } - // The tag is findVersion's ceiling and the floor a step down lands on, so it has nothing left to - // decide here. Modes without a tag pass none and are simply uncapped. return findVersion(data, versions, {range, semvers, useGreatest, usePre, useRel, latest: latestTag, pinnedRange, pinNoDowngrade, allowDowngrade, cooldownDays, now, getVersionDate, versioning}); } -// A forge host includes its port: two instances on one hostname are different endpoints. export function urlHost(url: string): string { try { return new URL(url).host; @@ -517,45 +381,29 @@ export function urlHost(url: string): string { } } -// Entries are `host:token` and the host may carry a port, so the last colon separates the two. -// Splitting at the first one read `localhost:3500:tok` as host `localhost` with token -// `3500:tok`, so a token may not itself contain a colon. function pairToken(host: string): string | null { - for (const entry of commaSeparatedToArray(env.UPDATES_FORGE_TOKENS ?? "")) { + const entry = commaSeparatedToArray(env.UPDATES_FORGE_TOKENS ?? "").find(entry => { const sep = entry.lastIndexOf(":"); - if (sep > 0 && entry.slice(0, sep) === host) return entry.slice(sep + 1); - } - return null; + return sep > 0 && entry.slice(0, sep) === host; + }); + return entry ? entry.slice(entry.lastIndexOf(":") + 1) : null; } -let execFilePromise: ReturnType | undefined; -async function loadExecFile() { - const [{execFile}, {promisify}] = await Promise.all([ - import("node:child_process"), - import("node:util"), - ]); - return promisify(execFile); -} -export function getExecFile() { - return execFilePromise ??= loadExecFile(); +let execFilePromise: Promise | undefined; +export function getExecFile(): Promise { + if (!execFilePromise) execFilePromise = (async () => { + const [{execFile}, {promisify}] = await Promise.all([import("node:child_process"), import("node:util")]); + return promisify(execFile) as ExecFile; + })(); + return execFilePromise; } const githubTokenEnvNames = ["UPDATES_GITHUB_API_TOKEN", "GITHUB_API_TOKEN", "GH_TOKEN", "GITHUB_TOKEN", "HOMEBREW_GITHUB_API_TOKEN"]; -function envGithubTokens(): string[] { - return Array.from(new Set( - githubTokenEnvNames.map(name => env[name]).filter((value): value is string => Boolean(value)), - )); -} - -// Env is read per call rather than snapshotted at import, so the token set at -// request time always wins. Only the `gh auth token` probe is memoized: a sync -// execFileSync in a concurrent `fetchForge` flow blocks the event loop long -// enough on Windows that parallel fetches can hit their AbortSignal timeout. It -// is skipped entirely when an env token is already set. let githubTokensPromise: Promise | undefined; export function getGithubTokens(): Promise { - const tokens = envGithubTokens(); + const tokens = Array.from(new Set(githubTokenEnvNames + .map(name => env[name]).filter((value): value is string => Boolean(value)))); if (tokens.length) return Promise.resolve(tokens); return githubTokensPromise ??= (async () => { try { @@ -571,8 +419,6 @@ export function getGithubTokens(): Promise { const reExtraheader = /^http\.(\S+)\/\.extraheader AUTHORIZATION:\s*basic\s+(\S+)$/i; -// actions/checkout and its Gitea and Forgejo forks leave the job token in git config as -// `http./.extraheader`, base64 of `x-access-token:`, keyed by GITHUB_SERVER_URL. export function parseExtraheaders(config: string): Map { const tokens = new Map(); for (const line of config.split(/\r?\n/)) { @@ -581,13 +427,11 @@ export function parseExtraheaders(config: string): Map { const host = urlHost(match[1]); const decoded = Buffer.from(match[2], "base64").toString("utf8"); const token = decoded.slice(decoded.indexOf(":") + 1); - if (host && token && !tokens.has(host)) tokens.set(host, token); // first wins, as git resolves it + if (host && token && !tokens.has(host)) tokens.set(host, token); } return tokens; } -// Read once, one subprocess serves every host in a run. `--local` would miss it, the -// credentials file arrives via includeIf. let extraheaderTokensPromise: Promise> | undefined; function getExtraheaderTokens(): Promise> { return extraheaderTokensPromise ??= (async () => { @@ -603,17 +447,12 @@ function getExtraheaderTokens(): Promise> { const workingTokenCache = new Map(); -// GitHub credentials (env tokens and `gh auth token`) are only ever sent to -// GitHub itself or to the configured default forge endpoint. A host taken from -// a workflow `uses:` ref must never receive them — it gets a token only when -// one is explicitly configured for it via UPDATES_FORGE_TOKENS. export async function getForgeTokens(host: string, forgeApiUrl: string): Promise { if (!host) return []; const hostToken = pairToken(host); if (hostToken) return [hostToken]; - // credentials are keyed by forge host, and GitHub alone serves its API from another hostname const forgeHost = host === "api.github.com" ? "github.com" : host; const isGithubHost = forgeHost === "github.com" || host === urlHost(forgeApiUrl); @@ -625,8 +464,6 @@ export async function getForgeTokens(host: string, forgeApiUrl: string): Promise return Array.from(new Set(header ? [...tokens, header] : tokens)); } -// A forge failure the run must report rather than read as "no update". Renovate draws the same -// line with PLATFORM_RATE_LIMIT_EXCEEDED and ExternalHostError. export type ForgeErrorKind = "rateLimit" | "server" | "network"; export class ForgeError extends Error { @@ -634,7 +471,7 @@ export class ForgeError extends Error { readonly kind: ForgeErrorKind; readonly host: string; readonly status: number; - readonly reset: number; // `x-ratelimit-reset` in epoch seconds, 0 when the forge sent none + readonly reset: number; constructor(kind: ForgeErrorKind, host: string, message: string, {status = 0, reset = 0, cause}: {status?: number, reset?: number, cause?: unknown} = {}) { super(message, {cause}); @@ -645,18 +482,15 @@ export class ForgeError extends Error { } } -// A rate limit is a 403 or 429 the headers or the body identify as one. A plain 403 is a -// credential problem and still falls through to the next token. async function rateLimitReset(res: Response): Promise { if (res.status !== 403 && res.status !== 429) return null; - const reset = () => Number(res.headers?.get?.("x-ratelimit-reset")) || 0; - if (res.headers?.get?.("x-ratelimit-remaining") === "0" || res.headers?.get?.("retry-after")) return reset(); + const reset = Number(res.headers?.get?.("x-ratelimit-reset")) || 0; + if (res.headers?.get?.("x-ratelimit-remaining") === "0" || res.headers?.get?.("retry-after")) return reset; try { - // Cloned so a caller that reads the body of a non-rate-limited 403 still can. const {message} = await (typeof res.clone === "function" ? res.clone() : res).json(); if (typeof message !== "string") return null; if (message.includes("rate limit exceeded") || message.includes("abuse detection mechanism") || - message.startsWith("You have exceeded a secondary rate limit")) return reset(); + message.startsWith("You have exceeded a secondary rate limit")) return reset; } catch {} return null; } @@ -674,32 +508,24 @@ async function checkForgeResponse(res: Response, url: string, host: string, hasT export async function fetchForge(url: string, ctx: ModeContext, extraHeaders?: Record): Promise { const host = urlHost(url); - - // Resolve tokens before starting the AbortSignal timer so the lazy - // `gh auth token` probe does not consume the fetch's timeout budget. const tokens = await getForgeTokens(host, ctx.forgeApiUrl); - - const optsFor = (token?: string): RequestInit => { + const attempt = async (token?: string) => { const opts = getFetchOpts("Bearer", token); - if (extraHeaders) opts.headers = {...opts.headers as Record, ...extraHeaders}; - return opts; + opts.headers = {...opts.headers as Record, ...extraHeaders}; + return checkForgeResponse(await fetchWithRetry(ctx, url, opts), url, host, Boolean(tokens.length)); }; - const attempt = async (token?: string) => - checkForgeResponse(await fetchWithRetry(ctx, url, optsFor(token)), url, host, tokens.length > 0); - try { if (!tokens.length) return await attempt(); const cached = workingTokenCache.get(host); - if (cached) return await attempt(cached); - - for (const token of tokens) { + for (const token of cached && tokens.includes(cached) ? [cached, ...tokens.filter(token => token !== cached)] : tokens) { const response = await attempt(token); if (response.status !== 401 && response.status !== 403) { workingTokenCache.set(host, token); return response; } + if (token === cached) workingTokenCache.delete(host); } return await attempt(); } catch (err: any) { @@ -708,9 +534,6 @@ export async function fetchForge(url: string, ctx: ModeContext, extraHeaders?: R } } -// Picks the highest valid semver tag. GitHub does not guarantee a particular -// ordering for the /tags endpoint, so relying on array position (`tags.at(-1)`) -// silently picks the wrong tag. export function selectTag(tags: Array, oldRef: string): string | null { const oldRefBare = stripv(oldRef); if (!valid(oldRefBare)) return null; @@ -725,49 +548,52 @@ export function selectTag(tags: Array, oldRef: string): string | null { bestBare = tagBare; } } - if (bestTag && gt(bestBare, oldRefBare)) return bestTag; - return null; + return bestTag && gt(bestBare, oldRefBare) ? bestTag : null; } export function resolvePackageJsonUrl(url: string): string { const cleaned = url.replace("git@", "").replace(/.+?\/\//, "https://").replace(/\.git$/, ""); - if (/^[a-z]+:[a-z0-9-]+\/[a-z0-9-]+$/.test(cleaned)) { // foo:user/repo + if (/^[a-z]+:[a-z0-9-]+\/[a-z0-9-]+$/.test(cleaned)) { return cleaned.replace(/^(.+?):/, (_, p1) => `https://${p1}.com/`); - } else if (/^[a-z0-9-]+\/[a-z0-9-]+$/.test(cleaned)) { // user/repo - return `https://github.com/${cleaned}`; - } else { - return cleaned; } + return /^[a-z0-9-]+\/[a-z0-9-]+$/.test(cleaned) ? `https://github.com/${cleaned}` : cleaned; } -// Requires a hex letter so an all-numeric tag like `20240115` is read as a version rather -// than a commit, and accepts 6 characters, which git and renovate both treat as a short sha. -export const hashRe = /^(?=.*[a-f])[0-9a-f]{6,}$/i; +const commitHashPattern = "(?:[0-9a-f]{6,7}|[0-9a-f]{40}|[0-9a-f]{64})"; +export const commitHashRe = new RegExp(`^${commitHashPattern}$`, "i"); +export const hashRe = /^(?:[0-9a-f]{7}|[0-9a-f]{40})$/i; -// A ref that names a version, as opposed to a branch (`release/v1`) or another tag scheme -// (`codeql-bundle-v2.20.3`). Those must keep their text, never be replaced by a version tag. export function isVersionLikeRef(ref: string): boolean { return /^v?\d+(?:\.\d+)*(?:[-+][\w.-]+)?$/.test(ref); } -export type TagEntry = { - name: string, - commitSha: string, -}; +export type TagEntry = {name: string, commitSha: string, isStable?: boolean}; -// GitHub puts the dates at the top level of a commit, Gitea nests them under `commit`. export function parseCommitDate(data: any): string { const commit = data?.commit ?? data; return commit?.committer?.date || commit?.author?.date || ""; } export function parseTags(data: Array): Array { - return data.map((tag: any) => ({name: tag.name, commitSha: tag.commit?.sha || ""})); -} + if (!Array.isArray(data)) throw new TypeError("Invalid Forge tags response"); + return data.map((tag: any) => { + if (typeof tag?.name !== "string" || tag.commit?.sha !== undefined && typeof tag.commit.sha !== "string") { + throw new TypeError("Invalid Forge tag entry"); + } + return {name: tag.name, commitSha: tag.commit?.sha || ""}; + }); +} + +const parseTagPage = (data: any, cached: boolean): Array => { + if (!cached) return parseTags(data); + if (!Array.isArray(data)) throw new TypeError("Invalid cached Forge tags response"); + return data.map(tag => { + if (typeof tag?.name !== "string" || typeof tag.commitSha !== "string" || + tag.isStable !== undefined && typeof tag.isStable !== "boolean") throw new TypeError("Invalid cached Forge tag entry"); + return {...tag}; + }); +}; -// Fetch a forge URL with ETag revalidation, returning the cached body verbatim on 304 and -// otherwise the string `reduce` distills the response into, which is what gets cached. -// `reduce` takes the Response so each caller picks its own read method. export async function fetchForgeEtag(url: string, ctx: ModeContext, reduce: (res: Response) => Promise): Promise { const cached = ctx.noCache ? null : await getCache(url); const res = await fetchForge(url, ctx, cached ? {"if-none-match": cached.etag} : undefined); @@ -779,62 +605,102 @@ export async function fetchForgeEtag(url: string, ctx: ModeContext, reduce: (res return body; } -// GitHub strips the Link header on 304 responses, so cache it alongside the body. -async function fetchTagsPage(url: string, ctx: ModeContext): Promise<{tags: Array, link: string} | null> { +type Release = {name: string, isStable: boolean}; + +function parseReleases(data: any, cached = false): Array { + if (!Array.isArray(data)) throw new TypeError(`Invalid ${cached ? "cached " : ""}Forge releases response`); + return data.map(release => { + const name = cached ? release?.name : release?.tag_name; + const isStable = cached ? release?.isStable : !release?.prerelease && release?.draft !== true; + if (typeof name !== "string" || (cached ? typeof isStable !== "boolean" : + typeof release?.prerelease !== "boolean" || release.draft !== undefined && typeof release.draft !== "boolean")) { + throw new TypeError(`Invalid ${cached ? "cached " : ""}Forge release entry`); + } + return {name, isStable}; + }); +} + +type ForgePage = {entries: Array, link: string}; + +async function fetchForgePage( + url: string, ctx: ModeContext, key: "tags" | "releases", parse: (data: any, cached: boolean) => Array, +): Promise | null> { const body = await fetchForgeEtag(url, ctx, async res => JSON.stringify({ - link: res.headers.get("link") || "", tags: parseTags(await res.json()), + link: res.headers.get("link") || "", + [key]: parse(await res.json(), false), })); if (!body) return null; - try { - const parsed = JSON.parse(body); - return {tags: parsed.tags || [], link: parsed.link || ""}; - } catch { return null; } + const parsed = JSON.parse(body); + if (typeof parsed?.link !== "string") throw new TypeError(`Invalid cached Forge ${key} response`); + return {entries: parse(parsed[key], true), link: parsed.link}; +} + +function lastPageFromLink(link: string): number { + const last = /<([^>]+)>;\s*rel="last"/.exec(link); + if (!last) return 0; + const page = Number(new URL(last[1]).searchParams.get("page")); + if (!Number.isSafeInteger(page) || page < 1) throw new TypeError("Invalid Forge pagination URL"); + return Math.min(page, maxTagPages); +} + +async function fetchReleaseStability(owner: string, repo: string, ctx: ModeContext): Promise> { + const releasesUrl = (page: number) => `${githubApiUrl}/repos/${owner}/${repo}/releases?per_page=100&page=${page}`; + const page1 = await fetchForgePage(releasesUrl(1), ctx, "releases", parseReleases); + if (!page1) return new Map(); + const pages = await Promise.all(Array.from({length: Math.max(lastPageFromLink(page1.link) - 1, 0)}, + (_, idx) => fetchForgePage(releasesUrl(idx + 2), ctx, "releases", parseReleases))); + const stability = new Map(); + for (const page of [page1, ...pages]) { + for (const release of page?.entries ?? []) stability.set(release.name, release.isStable); + } + return stability; } -// `oldRefs` are the refs the caller has to resolve, a sha pin's commit or a tag's own name. GitHub -// serves tags newest-first, so the walk stops once every one has been seen. Naming none reads all. -export async function fetchActionTags(apiUrl: string, owner: string, repo: string, ctx: ModeContext, oldRefs: Array = []): Promise> { +export async function fetchForgeTags( + apiUrl: string, owner: string, repo: string, ctx: ModeContext, oldRefs: Array = [], +): Promise> { const tagsUrl = (page: number) => `${apiUrl}/repos/${owner}/${repo}/tags?per_page=100&page=${page}`; const tags: Array = []; const unresolved = new Set(oldRefs.filter(Boolean)); const bounded = unresolved.size > 0; - const take = (page: {tags: Array} | null): boolean => { - for (const entry of page?.tags ?? []) { + const take = (page: ForgePage | null): boolean => { + for (const entry of page?.entries ?? []) { for (const ref of unresolved) if (ref === entry.name || entry.commitSha.startsWith(ref)) unresolved.delete(ref); tags.push(entry); } return bounded && !unresolved.size; }; - try { - const page1 = await fetchTagsPage(tagsUrl(1), ctx); - if (!page1) return tags; - const last = /<([^>]+)>;\s*rel="last"/.exec(page1.link); - const lastPage = last ? Math.min(Number(new URL(last[1]).searchParams.get("page")), maxTagPages) : 0; - // Each wave is one round trip and doubles up to the socket budget, the limiter caps the flight. - const maxWave = effectiveConcurrency(ctx); - for (let next = 2, wave = 1, done = take(page1); next <= lastPage && !done; next += wave, wave = Math.min(wave * 2, maxWave)) { - const pages = await Promise.all( - Array.from({length: Math.min(wave, lastPage - next + 1)}, (_, idx) => fetchTagsPage(tagsUrl(next + idx), ctx)), - ); - for (const page of pages) done = take(page); + const page1 = await fetchForgePage(tagsUrl(1), ctx, "tags", parseTagPage); + if (!page1) return tags; + const lastPage = lastPageFromLink(page1.link); + const maxWave = effectiveConcurrency(ctx); + for (let next = 2, wave = 1, done = take(page1); next <= lastPage && !done; next += wave, wave = Math.min(wave * 2, maxWave)) { + const pages = await Promise.all( + Array.from({length: Math.min(wave, lastPage - next + 1)}, (_, idx) => + fetchForgePage(tagsUrl(next + idx), ctx, "tags", parseTagPage)), + ); + for (const page of pages) done = take(page); + } + return tags; +} + +export async function fetchActionTags( + apiUrl: string, owner: string, repo: string, ctx: ModeContext, oldRefs: Array = [], includeStability = true, +): Promise> { + const tags = await fetchForgeTags(apiUrl, owner, repo, ctx, oldRefs); + if (apiUrl === githubApiUrl && includeStability) { + try { + const stability = await fetchReleaseStability(owner, repo, ctx); + for (const tag of tags) if (stability.has(tag.name)) tag.isStable = stability.get(tag.name); + } catch (err) { + if (!(err instanceof ForgeError)) throw err; } - return tags; - } catch (err) { - // A classified failure is the dependency's result, unlike a malformed page worth degrading over. - if (err instanceof ForgeError) throw err; - return []; } + return tags; } -export type CheckResult = { - key: string, - newRange: string, - user: string, - repo: string, - oldRef: string, - newRef: string, - newDate?: string, -}; +export type CheckResult = {key: string, newRange: string, user: string, repo: string, oldRef: string, newRef: string, + newDate?: string}; export function throwFetchError(res: Response | undefined, url: string, name: string, source: string): never { if (res?.status && res.statusText) { @@ -843,45 +709,30 @@ export function throwFetchError(res: Response | undefined, url: string, name: st throw new Error(`Unable to fetch ${name} from ${source}`); } -// Renovate caps date-like versions at this in its doNotUpgradeFromAlpineStableToEdge preset. const dateVersionMin = 20000000; -const isDateVersion = (fields: Array) => Number(fields[0]) >= dateVersionMin; - -// Whether a candidate is versioned the same way as the authored version. Alpine publishes -// `20260127` snapshot tags next to its `3.24` releases, and those coerce so high they win -// every comparison, so both the field count and the magnitude have to line up. export function isSameVersionScheme(candidate: string, oldVersion: string): boolean { const candidateFields = stripv(candidate).split("."); const oldFields = stripv(oldVersion).split("."); - // More fields stay allowed so a short authored version can still upgrade off a registry - // that only publishes full versions; fewer means another scheme. if (candidateFields.length < oldFields.length) return false; - // A YYYYMMDD snapshot outranks every real release, so only ever reach one from another. - return !isDateVersion(candidateFields) || isDateVersion(oldFields); + return Number(candidateFields[0]) < dateVersionMin || Number(oldFields[0]) >= dateVersionMin; } export function formatVersionPrecision(newVersion: string, oldVersion: string, suffix = ""): string { const bare = stripv(newVersion); const numParts = stripv(oldVersion).split(".").length; const newParts = bare.split("."); - // A shorter authored version keeps its precision, padding missing fields with 0. const formatted = numParts >= 3 ? bare : Array.from({length: numParts}, (_, idx) => newParts[idx] || "0").join("."); return `${oldVersion.startsWith("v") ? "v" : ""}${formatted}${suffix}`; } export function getSubDir(url: string): string { - if (url.startsWith("https://bitbucket.org")) { - return "src/HEAD"; - } else { - return "tree/HEAD"; - } + return url.startsWith("https://bitbucket.org") ? "src/HEAD" : "tree/HEAD"; } -// pypi project_urls keys holding a repository link, in preference order const pypiRepoKeys = ["repository", "Repository", "repo", "Repo", "source", "Source", "source code", "Source code", "Source Code", "homepage", "Homepage"]; export function getInfoUrl({repository, homepage, info}: {repository?: PackageRepository, homepage?: string, info?: Record}, registry: string | null, name: string): string { - if (info) { // pypi + if (info) { const urls = info.project_urls; for (const key of pypiRepoKeys) { if (!urls?.[key]) continue; @@ -894,7 +745,8 @@ export function getInfoUrl({repository, homepage, info}: {repository?: PackageRe let infoUrl = ""; if (registry === "https://npm.pkg.github.com") { return `https://github.com/${name.replace(/^@/, "")}`; - } else if (repository) { + } + if (repository) { const url = typeof repository === "string" ? repository : repository.url; infoUrl = resolvePackageJsonUrl(url); if (infoUrl && typeof repository !== "string" && repository.directory) { @@ -904,4 +756,3 @@ export function getInfoUrl({repository, homepage, info}: {repository?: PackageRe return infoUrl || homepage || ""; } - diff --git a/utils/dns.test.ts b/utils/dns.test.ts new file mode 100644 index 0000000..cb40f94 --- /dev/null +++ b/utils/dns.test.ts @@ -0,0 +1,46 @@ +import dns from "node:dns"; +import {enableDnsCache} from "./dns.ts"; + +const systemLookup = dns.lookup; + +afterEach(() => { + dns.lookup = systemLookup; + vi.restoreAllMocks(); +}); + +test("DNS caching preserves lookup semantics and can be disabled", async () => { + const calls: Array<{hostname: string, options: unknown}> = []; + const original = ((hostname: string, options: unknown, callback: (...args: Array) => void) => { + calls.push({hostname, options}); + queueMicrotask(() => callback(null, "192.0.2.1", 4)); + }) as typeof dns.lookup; + dns.lookup = original; + const now = vi.spyOn(Date, "now").mockReturnValue(1000); + const disable = enableDnsCache(); + const options = {family: 4, hints: dns.ADDRCONFIG, order: "ipv4first" as const}; + const lookup = (lookupOptions: object) => new Promise>(resolve => { + dns.lookup("example.com", lookupOptions, (...args: Array) => resolve(args)); + }); + + expect(await lookup(options)).toEqual([null, "192.0.2.1", 4]); + let synchronous = true; + const cached = new Promise(resolve => { + dns.lookup("example.com", options, () => { + expect(synchronous).toBe(false); + resolve(); + }); + }); + synchronous = false; + await cached; + expect(calls).toEqual([{hostname: "example.com", options}]); + + const allOptions = {all: true, hints: dns.V4MAPPED, verbatim: false}; + await lookup(allOptions); + expect(calls[1]).toEqual({hostname: "example.com", options: allOptions}); + + now.mockReturnValue(61001); + await lookup(options); + expect(calls).toHaveLength(3); + disable(); + expect(dns.lookup).toBe(original); +}); diff --git a/utils/dns.ts b/utils/dns.ts index c09ceb9..b9e2110 100644 --- a/utils/dns.ts +++ b/utils/dns.ts @@ -1,60 +1,82 @@ -// DNS cache to avoid ENOTFOUND errors from parallel lookups -// TODO: Use undici once https://github.com/nodejs/node/issues/43187 is resolved import dns from "node:dns"; -// Hand a lookup result to one waiter, honoring a requested address family when the -// result has a match and falling back to the first address otherwise. -function deliver(callback: (...args: any[]) => void, options: any, addresses: {address: string, family: number}[]) { - if (options.all) { - callback(null, addresses); - } else { - const addr = addresses.find(({family}) => family === options.family) ?? addresses[0]; - callback(null, addr.address, addr.family); +const maxEntries = 512; +const ttl = 60000; +let active: {lookup: typeof dns.lookup, original: typeof dns.lookup, users: number} | null = null; + +export function enableDnsCache(): () => void { + if (active) { + if (dns.lookup === active.lookup) { + active.users++; + return disable(active); + } + active = null; } -} -export function enableDnsCache() { - const dnsCache = new Map(); - const dnsInflight = new Map void}>>(); + const dnsCache = new Map}>(); + const dnsInflight = new Map void>>(); const origLookup = dns.lookup as any; - dns.lookup = function(hostname: string, ...rest: any[]) { - let options: any = {}; - let callback: (...args: any[]) => void; - if (typeof rest[0] === "function") { - callback = rest[0]; - } else { - options = typeof rest[0] === "number" ? {family: rest[0]} : (rest[0] || {}); - callback = rest[1]; - } + const lookup = function(hostname: string, ...rest: any[]) { + const hasOptions = typeof rest[0] !== "function"; + const lookupOptions = hasOptions ? rest[0] : undefined; + const options = typeof lookupOptions === "number" ? {family: lookupOptions} : lookupOptions || {}; + const callback: (...args: any[]) => void = hasOptions ? rest[1] : rest[0]; + if (typeof callback !== "function") return origLookup.call(dns, hostname, ...rest); + const key = JSON.stringify([ + hostname, options.family ?? 0, options.hints ?? 0, Boolean(options.all), + options.order ?? "", options.verbatim ?? "", + ]); - const cached = dnsCache.get(hostname); + const cached = dnsCache.get(key); if (cached) { - deliver(callback, options, cached); - return; + if (cached.expires > Date.now()) { + queueMicrotask(() => callback(null, ...cached.result)); + return; + } + dnsCache.delete(key); } - if (dnsInflight.has(hostname)) { - dnsInflight.get(hostname)!.push({options, callback}); + const inflight = dnsInflight.get(key); + if (inflight) { + inflight.push(callback); return; } - dnsInflight.set(hostname, [{options, callback}]); - origLookup.call(dns, hostname, {all: true}, (err: any, addresses: any) => { - const waiters = dnsInflight.get(hostname)!; - dnsInflight.delete(hostname); - if (!err && addresses?.length) { - dnsCache.set(hostname, addresses); - } - // A success with no addresses would crash the non-`all` branch on addr.address; treat it as a failure. - const lookupErr = err || (addresses?.length ? null : Object.assign(new Error(`getaddrinfo ENOTFOUND ${hostname}`), {code: "ENOTFOUND", hostname})); - for (const {options: opts, callback: cb} of waiters) { - if (lookupErr) { - cb(lookupErr); - } else { - deliver(cb, opts, addresses); + dnsInflight.set(key, [callback]); + const complete = (err: any, ...result: Array) => { + queueMicrotask(() => { + const waiters = dnsInflight.get(key)!; + dnsInflight.delete(key); + if (!err) { + if (dnsCache.size >= maxEntries) dnsCache.delete(dnsCache.keys().next().value!); + dnsCache.set(key, {expires: Date.now() + ttl, result}); } - } - }); - } as any; + for (const waiter of waiters) waiter(err, ...result); + }); + }; + try { + if (hasOptions) origLookup.call(dns, hostname, lookupOptions, complete); + else origLookup.call(dns, hostname, complete); + } catch (err) { + dnsInflight.delete(key); + throw err; + } + } as typeof dns.lookup; + + dns.lookup = lookup; + active = {lookup, original: origLookup, users: 1}; + return disable(active); +} + +function disable(state: NonNullable): () => void { + let disabled = false; + return () => { + if (disabled) return; + disabled = true; + state.users--; + if (state.users) return; + if (dns.lookup === state.lookup) dns.lookup = state.original; + if (active === state) active = null; + }; } diff --git a/utils/fetchCache.test.ts b/utils/fetchCache.test.ts index ac02805..13e5bea 100644 --- a/utils/fetchCache.test.ts +++ b/utils/fetchCache.test.ts @@ -1,26 +1,70 @@ -import {getCache, setCache, flushCacheWrites} from "./fetchCache.ts"; +import {createHash} from "node:crypto"; +import {mkdir, mkdtemp, readdir, rm, utimes, writeFile} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {basename, join} from "node:path"; +import {flushCacheWrites, getCache, setCache} from "./fetchCache.ts"; -test("setCache and getCache round-trip", async () => { +let cacheRoot: string; + +beforeAll(async () => { + cacheRoot = await mkdtemp(join(tmpdir(), "updates-fetch-cache-")); +}); + +afterAll(async () => { + await rm(cacheRoot, {recursive: true, force: true}); +}); + +async function makeCacheDir(name: string): Promise { + const dir = join(cacheRoot, name); + await mkdir(dir); + return dir; +} + +test("setCache and getCache round-trip preserves newlines", async () => { + const cacheDir = await makeCacheDir("round-trip"); const url = "https://test.example.com/fetchCache-round-trip-test"; - setCache(url, "W/\"abc123\"", '{"versions":{"1.0.0":{}}}'); - await flushCacheWrites(); - const result = await getCache(url); - expect(result).toEqual({etag: "W/\"abc123\"", body: '{"versions":{"1.0.0":{}}}'}); + const body = '{"a":1}\n{"b":2}\n{"c":3}'; + setCache(url, "W/\"abc123\"", body, cacheDir); + await flushCacheWrites(cacheDir); + expect(await getCache(url, cacheDir)).toEqual({etag: "W/\"abc123\"", body}); + expect(await readdir(cacheDir)).toContain(`${createHash("sha256").update(url).digest("hex")}.cache`); }); test("getCache returns null for unknown URL", async () => { - expect(await getCache("https://test.example.com/nonexistent-url-12345")).toBeNull(); + expect(await getCache("https://test.example.com/nonexistent-url-12345", await makeCacheDir("unknown"))).toBeNull(); }); test("getCache returns null when the key can not be derived", async () => { - expect(await getCache(undefined as unknown as string)).toBeNull(); + expect(await getCache(undefined as unknown as string, await makeCacheDir("invalid-key"))).toBeNull(); }); -test("setCache and getCache preserve body with newlines", async () => { - const url = "https://test.example.com/fetchCache-newline-test"; - const body = '{"a":1}\n{"b":2}\n{"c":3}'; - setCache(url, "etag-val", body); - await flushCacheWrites(); - const result = await getCache(url); - expect(result).toEqual({etag: "etag-val", body}); +test("expired cache entries are removed", async () => { + const cacheDir = await makeCacheDir("expired"); + const url = "https://test.example.com/fetchCache-expired-test"; + setCache(url, "etag-val", "body", cacheDir); + await flushCacheWrites(cacheDir); + const file = join(cacheDir, `${createHash("sha256").update(url).digest("hex")}.cache`); + await utimes(file, new Date(0), new Date(0)); + expect(await getCache(url, cacheDir)).toBeNull(); + expect(await readdir(cacheDir)).not.toContain(basename(file)); +}); + +test("cache eviction retains a recently used entry and bounds files across runs", async () => { + const cacheDir = await makeCacheDir("eviction"); + const usedUrl = "https://test.example.com/fetchCache-used-before-eviction"; + const usedFile = `${createHash("sha256").update(usedUrl).digest("hex")}.cache`; + const now = Date.now(); + await Promise.all(Array.from({length: 4096}, async (_value, index) => { + const file = join(cacheDir, index === 0 ? usedFile : `${index}.cache`); + await writeFile(file, "etag\nbody"); + const time = new Date(now - (4097 - index) * 1000); + await utimes(file, time, time); + })); + expect(await getCache(usedUrl, cacheDir)).toEqual({etag: "etag", body: "body"}); + setCache("https://test.example.com/fetchCache-newest", "etag", "body", cacheDir); + await flushCacheWrites(cacheDir); + const files = (await readdir(cacheDir)).filter(name => name.endsWith(".cache")); + expect(files).toHaveLength(4096); + expect(files).toContain(usedFile); + expect(files).not.toContain("1.cache"); }); diff --git a/utils/fetchCache.ts b/utils/fetchCache.ts index b5249ac..336361c 100644 --- a/utils/fetchCache.ts +++ b/utils/fetchCache.ts @@ -1,13 +1,11 @@ import {createHash} from "node:crypto"; import {readFile} from "node:fs"; -import {writeFile, mkdir, rename} from "node:fs/promises"; +import {writeFile, mkdir, readdir, rename, stat, unlink, utimes} from "node:fs/promises"; import {join} from "node:path"; import {env, platform, pid} from "node:process"; import {homedir} from "node:os"; -import {getOrSet} from "./utils.ts"; +import {tryOrNull} from "./utils.ts"; -// The callback API, not fs/promises: a run reads one cache entry per url and a FileHandle costs -// an extra fstat plus its finalizer registration on every one. function readFileUtf8(path: string): Promise { return new Promise((resolve, reject) => { readFile(path, "utf8", (err, content) => err ? reject(err) : resolve(content)); @@ -21,23 +19,31 @@ const cacheDir = join( "updates", ); -let dirCreated: Promise | null = null; +const createdDirs = new Map>(); -// Memoized — the same URL is hashed twice (read then write) per cold-cache -// fetch, and many URLs are visited each run. -const cacheKeyMemo = new Map(); function cacheKey(url: string): string { - return getOrSet(cacheKeyMemo, url, () => createHash("sha256").update(url).digest("hex").substring(0, 16)); + return createHash("sha256").update(url).digest("hex"); } -export async function getCache(url: string): Promise<{etag: string, body: string} | null> { +const maxAge = 7 * 24 * 60 * 60 * 1000; +const maxEntries = 4096; + +export async function getCache(url: string, dir: string = cacheDir): Promise<{etag: string, body: string} | null> { try { - const content = await readFileUtf8(join(cacheDir, `${cacheKey(url)}.cache`)); + const path = join(dir, `${cacheKey(url)}.cache`); + if (Date.now() - (await stat(path)).mtimeMs > maxAge) { + await unlink(path); + return null; + } + const content = await readFileUtf8(path); const idx = content.indexOf("\n"); if (idx === -1) return null; const etag = content.substring(0, idx); const body = content.substring(idx + 1); - return etag && body ? {etag, body} : null; + if (!etag || !body) return null; + const now = new Date(); + await tryOrNull(utimes(path, now, now)); + return {etag, body}; } catch { return null; } @@ -46,32 +52,43 @@ export async function getCache(url: string): Promise<{etag: string, body: string const pendingWrites = new Set>(); let tmpCounter = 0; -// Writes are intentionally not awaited by callers so a response can be -// consumed without waiting on disk. flushCacheWrites() awaits completion -// before the process may exit. The temp-file + rename dance keeps writes -// atomic: an interrupted process can never leave a torn entry that would -// poison later runs (the etag would revalidate but the body fail to parse). -export function setCache(url: string, etag: string, body: string): void { +export function setCache(url: string, etag: string, body: string, dir: string = cacheDir): void { const write = (async () => { try { - await (dirCreated ??= mkdir(cacheDir, {recursive: true})); + let created = createdDirs.get(dir); + if (!created) { + created = mkdir(dir, {recursive: true}); + createdDirs.set(dir, created); + } + await created; } catch { - dirCreated = null; // a transient mkdir failure must not poison every later write this run + createdDirs.delete(dir); return; } + let tmpFile: string | undefined; try { - const file = join(cacheDir, `${cacheKey(url)}.cache`); - const tmpFile = `${file}.${pid}-${tmpCounter++}.tmp`; + const file = join(dir, `${cacheKey(url)}.cache`); + tmpFile = `${file}.${pid}-${tmpCounter++}.tmp`; await writeFile(tmpFile, `${etag}\n${body}`); await rename(tmpFile, file); - } catch {} + } catch { + if (tmpFile) await tryOrNull(unlink(tmpFile)); + } })(); pendingWrites.add(write); (async () => { await write; pendingWrites.delete(write); })(); } -export async function flushCacheWrites(): Promise { - // Loop: concurrent updates() calls share this set and may enqueue while a - // flush is in progress. +export async function flushCacheWrites(dir: string = cacheDir): Promise { while (pendingWrites.size) await Promise.all(pendingWrites); + try { + const files = (await readdir(dir)).filter(name => name.endsWith(".cache")); + const entries = (await Promise.all(files.map(async name => { + const path = join(dir, name); + try { return {path, mtime: (await stat(path)).mtimeMs}; } catch { return null; } + }))).filter(entry => entry !== null).sort((a, b) => b.mtime - a.mtime); + const fresh = entries.filter(entry => Date.now() - entry.mtime <= maxAge); + await Promise.all([...entries.filter(entry => Date.now() - entry.mtime > maxAge), ...fresh.slice(maxEntries)] + .map(entry => tryOrNull(unlink(entry.path)))); + } catch {} } diff --git a/utils/json5.test.ts b/utils/json5.test.ts index e116385..1dfdac5 100644 --- a/utils/json5.test.ts +++ b/utils/json5.test.ts @@ -10,12 +10,13 @@ test("line comments", () => { // top "a": 1 // trailing }`)).toEqual({a: 1}); - // the newline closing a line comment still separates the tokens around it expect(() => parseJsonish("[1// c\n2]")).toThrow(); }); test("block comments", () => { expect(parseJsonish(`{ /* x */ "a": /* y */ 1 }`)).toEqual({a: 1}); + expect(() => parseJsonish("[1/* c */2]")).toThrow(); + expect(() => parseJsonish("1/* c */0")).toThrow(); }); test("trailing commas", () => { diff --git a/utils/json5.ts b/utils/json5.ts index 090dc5b..45f70ef 100644 --- a/utils/json5.ts +++ b/utils/json5.ts @@ -1,10 +1,3 @@ -/** - * Minimal JSON5/JSONC-tolerant parser. Strips line/block comments and trailing - * commas, converts single-quoted strings and unquoted identifier keys to their - * JSON equivalents, then defers to JSON.parse. Covers JSONC fully and the common - * subset of JSON5 used in the wild; does not support exotic escapes (\x, \0), - * hex/Infinity/NaN literals, or line continuations inside single-quoted strings. - */ const identStart = /[A-Za-z_$]/; const identPart = /[A-Za-z0-9_$]/; @@ -12,10 +5,7 @@ export function parseJsonish(text: string): unknown { let out = ""; let i = 0; const n = text.length; - let pendingComma = -1; // index in `out` of a comma that becomes trailing if the next token closes a container - // Skip the comment starting at index j, returning the index just past it. Trailing - // whitespace is left in place so it keeps separating the tokens around the comment. function skipComment(j: number): number { if (text[j + 1] === "/") { j += 2; @@ -27,7 +17,6 @@ export function parseJsonish(text: string): unknown { return j + 2; } - // Skip whitespace and comments starting at index j, returning the next significant index. function skipTrivia(j: number): number { while (j < n) { if (/\s/.test(text[j])) { j++; continue; } @@ -41,7 +30,6 @@ export function parseJsonish(text: string): unknown { const ch = text[i]; if (ch === '"') { - pendingComma = -1; out += ch; i++; while (i < n) { @@ -58,23 +46,21 @@ export function parseJsonish(text: string): unknown { continue; } - // Single-quoted string: re-emit as a double-quoted JSON string. if (ch === "'") { - pendingComma = -1; out += '"'; i++; while (i < n) { const c = text[i]; if (c === "\\") { const next = text[i + 1]; - if (next === "'") { out += "'"; i += 2; continue; } // \' -> ' - if (next === "\n") { i += 2; continue; } // line continuation, drop + if (next === "'") { out += "'"; i += 2; continue; } + if (next === "\n") { i += 2; continue; } out += c; if (i + 1 < n) { out += next; i += 2; } else { i++; } continue; } - if (c === '"') { out += '\\"'; i++; continue; } // escape embedded double quote - if (c === "'") { out += '"'; i++; break; } // closing quote + if (c === '"') { out += '\\"'; i++; continue; } + if (c === "'") { out += '"'; i++; break; } out += c; i++; } @@ -83,35 +69,34 @@ export function parseJsonish(text: string): unknown { if (ch === "/" && (text[i + 1] === "/" || text[i + 1] === "*")) { i = skipComment(i); + out += " "; continue; } - // Strip trailing commas in a single pass so commas inside string values are left untouched. if (ch === ",") { - pendingComma = out.length; + const next = text[skipTrivia(i + 1)]; + if (next === "}" || next === "]") { + i++; + continue; + } out += ch; i++; continue; } if (ch === "}" || ch === "]") { - if (pendingComma >= 0) out = out.slice(0, pendingComma) + out.slice(pendingComma + 1); - pendingComma = -1; out += ch; i++; continue; } - // Unquoted identifier: a key if followed by ':', otherwise a literal (true/false/null). if (identStart.test(ch)) { - pendingComma = -1; let ident = ""; while (i < n && identPart.test(text[i])) { ident += text[i]; i++; } out += text[skipTrivia(i)] === ":" ? JSON.stringify(ident) : ident; continue; } - if (!/\s/.test(ch)) pendingComma = -1; out += ch; i++; } diff --git a/utils/prewarm.test.ts b/utils/prewarm.test.ts index 3d3e98e..1729b4b 100644 --- a/utils/prewarm.test.ts +++ b/utils/prewarm.test.ts @@ -6,6 +6,19 @@ import {prewarmOrigins} from "./prewarm.ts"; import {forgeDirs, modeByFileName} from "./utils.ts"; const created: Array = []; +const npmOrigins = ["https://registry.npmjs.org/"]; +const sampleContent = (path: string, content: string) => { + if (content && content !== "{}") return content; + if (path.endsWith("package.json")) return JSON.stringify({dependencies: {react: "18.0.0"}}); + if (path.endsWith("pnpm-workspace.yaml")) return "catalog:\n react: 18.0.0\n"; + if (path.endsWith("pyproject.toml")) return 'dependencies = [\n "requests>=2",\n]\n'; + if (path.endsWith("Cargo.toml")) return '[dependencies]\nserde = "1"\n'; + if (path.endsWith("go.mod") || path.endsWith("go.work")) return "require example.com/pkg v1.0.0\n"; + if (path.includes("Dockerfile") || /compose|docker-stack/.test(path)) return "FROM node:22\n"; + if (/Makefile|makefile|GNUmakefile|\.mk$/.test(path)) return "go install example.com/tool@v1.0.0\ndocker image node:22\n"; + if (/\.ya?ml$/.test(path)) return "uses: actions/checkout@v4\ncontainer: node:22\n"; + return content; +}; function makeDir(files: Record = {}): string { const dir = mkdtempSync(join(tmpdir(), "updates-prewarm-")); @@ -13,12 +26,11 @@ function makeDir(files: Record = {}): string { for (const [path, content] of Object.entries(files)) { const full = join(dir, path); mkdirSync(join(full, ".."), {recursive: true}); - writeFileSync(full, content); + writeFileSync(full, sampleContent(path, content)); } return dir; } -// an ambient GOPROXY must not leak into the default cases const origGoProxy = process.env.GOPROXY; beforeAll(() => { delete process.env.GOPROXY; }); @@ -28,55 +40,30 @@ afterAll(() => { else process.env.GOPROXY = origGoProxy; }); -test("empty dir returns no origins", () => { - expect(prewarmOrigins(makeDir(), {})).toEqual([]); -}); - -// A mode gaining a manifest, or a new mode entirely, must reach apisByMode or it silently -// never prewarms. modeByFileName covers the exact-name modes, the rest match by predicate. test.each([...Object.keys(modeByFileName), "Dockerfile", "Makefile", "tools.mk"])("%s is prewarmed", (filename) => { expect(prewarmOrigins(makeDir({[filename]: ""}), {})).not.toEqual([]); }); -test("package.json triggers npm + jsr + github", () => { +test("package.json prewarms only the registry its dependency uses", () => { + expect(prewarmOrigins(makeDir(), {})).toEqual([]); const origins = prewarmOrigins(makeDir({"package.json": "{}"}), {}); - expect(origins).toEqual(expect.arrayContaining([ - "https://registry.npmjs.org/", - "https://jsr.io/", - "https://api.github.com/", - ])); - expect(origins).toHaveLength(3); + expect(origins).toEqual(npmOrigins); + expect(prewarmOrigins(makeDir({"pnpm-workspace.yaml": ""}), {})).toEqual(expect.arrayContaining(npmOrigins)); expect(prewarmOrigins(makeDir({"package.json": "{}"}), {modes: "docker"})).toEqual([]); - // a workflow's docker images are read with docker alone enabled, so the forge dir still warms the hub expect(prewarmOrigins(makeDir({".github/workflows/ci.yml": ""}), {modes: "docker"})).toEqual(["https://hub.docker.com/"]); + expect(prewarmOrigins(makeDir({".github/workflows/ci.yml": "steps:\n - run: |\n uses: docker://node:18\n"}), {})).toEqual([]); }); -test("pnpm-workspace.yaml triggers same set as package.json", () => { - const origins = prewarmOrigins(makeDir({"pnpm-workspace.yaml": ""}), {}); - expect(origins).toEqual(expect.arrayContaining([ - "https://registry.npmjs.org/", - "https://jsr.io/", - "https://api.github.com/", - ])); -}); - -test("pyproject.toml triggers pypi", () => { - expect(prewarmOrigins(makeDir({"pyproject.toml": ""}), {})).toEqual(["https://pypi.org/"]); -}); - -test("Cargo.toml triggers crates.io", () => { - expect(prewarmOrigins(makeDir({"Cargo.toml": ""}), {})).toEqual(["https://crates.io/"]); -}); - -test("go.mod triggers proxy.golang.org", () => { - expect(prewarmOrigins(makeDir({"go.mod": ""}), {})).toEqual(["https://proxy.golang.org/"]); +test.each([ + ["pyproject.toml", "https://pypi.org/"], + ["Cargo.toml", "https://crates.io/"], + ["go.mod", "https://proxy.golang.org/"], + ["go.work", "https://proxy.golang.org/"], + ["Dockerfile", "https://hub.docker.com/"], +])("%s triggers its registry", (filename, origin) => { + expect(prewarmOrigins(makeDir({[filename]: ""}), {})).toEqual([origin]); }); -test("go.work triggers proxy.golang.org", () => { - expect(prewarmOrigins(makeDir({"go.work": ""}), {})).toEqual(["https://proxy.golang.org/"]); -}); - -// warming proxy.golang.org while every lookup goes elsewhere opens a socket the run never uses test("GOPROXY decides the go origin", () => { const dir = makeDir({"go.mod": ""}); process.env.GOPROXY = "https://internal.proxy,https://proxy.golang.org,direct"; @@ -90,18 +77,11 @@ test("GOPROXY decides the go origin", () => { delete process.env.GOPROXY; }); -test("Dockerfile triggers hub.docker.com", () => { - expect(prewarmOrigins(makeDir({"Dockerfile": ""}), {})).toEqual(["https://hub.docker.com/"]); -}); - -// Matched by pattern rather than an exact name, so they used to be discovered without ever -// prewarming the origin they then contacted. test.each(["docker-compose.yml", "compose.yaml", "compose.prod.yaml", "docker-stack.yml", "Dockerfile.dev"])( "%s triggers hub.docker.com", (filename) => { expect(prewarmOrigins(makeDir({[filename]: ""}), {})).toEqual(["https://hub.docker.com/"]); }); -// make resolves `go install` tool versions and docker image tags, so it needs both test.each(["Makefile", "makefile", "GNUmakefile", "tools.mk"])("%s triggers proxy.golang.org + hub.docker.com", (filename) => { expect(prewarmOrigins(makeDir({[filename]: ""}), {})).toEqual(expect.arrayContaining([ "https://proxy.golang.org/", @@ -113,6 +93,7 @@ test.each(forgeDirs)("%s/workflows dir triggers github + hub.docker.com", (forge const dir = mkdtempSync(join(tmpdir(), "updates-prewarm-")); created.push(dir); mkdirSync(join(dir, forgeDir, "workflows"), {recursive: true}); + writeFileSync(join(dir, forgeDir, "workflows", "ci.yml"), sampleContent("ci.yml", "")); expect(prewarmOrigins(dir, {})).toEqual(expect.arrayContaining([ "https://api.github.com/", "https://hub.docker.com/", @@ -120,7 +101,9 @@ test.each(forgeDirs)("%s/workflows dir triggers github + hub.docker.com", (forge }); test("API override args redirect origins", () => { - const origins = prewarmOrigins(makeDir({"package.json": "{}"}), { + const origins = prewarmOrigins(makeDir({"package.json": JSON.stringify({dependencies: { + registry: "1.0.0", jsr: "jsr:@std/path@1.0.0", forge: "github:user/repo", + }})}), { registry: "http://127.0.0.1:1234/", jsrapi: "http://127.0.0.1:2345", forgeapi: "http://127.0.0.1:3456/sub/path", @@ -133,13 +116,9 @@ test("API override args redirect origins", () => { expect(origins).toHaveLength(3); }); -test("registry from .npmrc in start dir is used", () => { +test("registry args override .npmrc", () => { const dir = makeDir({"package.json": "{}", ".npmrc": "registry=http://127.0.0.1:1234/\nsave-exact=false"}); expect(prewarmOrigins(dir, {})).toContain("http://127.0.0.1:1234/"); -}); - -test("registry arg wins over .npmrc registry", () => { - const dir = makeDir({"package.json": "{}", ".npmrc": "registry=http://127.0.0.1:1234/"}); expect(prewarmOrigins(dir, {registry: "http://127.0.0.1:5678/"})).toContain("http://127.0.0.1:5678/"); }); @@ -155,20 +134,22 @@ test("per-ecosystem overrides", () => { test("multi-mode project: package.json + Cargo.toml dedupes correctly", () => { const origins = prewarmOrigins(makeDir({"package.json": "{}", "Cargo.toml": ""}), {}); - expect(origins).toEqual(expect.arrayContaining([ - "https://registry.npmjs.org/", - "https://jsr.io/", - "https://api.github.com/", - "https://crates.io/", - ])); - expect(origins).toHaveLength(4); + expect(origins).toEqual(expect.arrayContaining([...npmOrigins, "https://crates.io/"])); + expect(origins).toHaveLength(2); +}); + +test("local npm dependencies do not prewarm a registry", () => { + expect(prewarmOrigins(makeDir({"package.json": JSON.stringify({dependencies: { + file: "file:../file", link: "link:../link", + }})}), {})).toEqual([]); }); test("github overlap is deduplicated when both package.json and .github/workflows present", () => { const dir = mkdtempSync(join(tmpdir(), "updates-prewarm-")); created.push(dir); - writeFileSync(join(dir, "package.json"), "{}"); + writeFileSync(join(dir, "package.json"), JSON.stringify({dependencies: {repo: "github:user/repo"}})); mkdirSync(join(dir, ".github", "workflows"), {recursive: true}); + writeFileSync(join(dir, ".github", "workflows", "ci.yml"), "uses: actions/checkout@v4\n"); const origins = prewarmOrigins(dir, {}); expect(origins.filter(origin => origin === "https://api.github.com/")).toHaveLength(1); }); diff --git a/utils/prewarm.ts b/utils/prewarm.ts index 970b0ad..866e981 100644 --- a/utils/prewarm.ts +++ b/utils/prewarm.ts @@ -1,86 +1,113 @@ -import {readFileSync, readdirSync} from "node:fs"; -import {join} from "node:path"; -import {isDockerFileName} from "../modes/docker.ts"; -import {isMakeFileName} from "../modes/make.ts"; -import {resolveGoProxyChain} from "../modes/go.ts"; -import {defaultApiUrls} from "../modes/shared.ts"; -import {forgeDirs, modeByFileName} from "./utils.ts"; -import {parseIni} from "./rc.ts"; -import {parseMixedArg, type Arg} from "../config.ts"; +import {readFileSync, readdirSync, statSync} from "node:fs"; +import {basename, join, resolve} from "node:path"; -function npmrcRegistry(dir: string): string | undefined { - try { - return parseIni(readFileSync(join(dir, ".npmrc"), "utf8")).registry; - } catch { - return undefined; - } -} +const defaults = { + registry: "https://registry.npmjs.org", + jsrapi: "https://jsr.io", + forgeapi: "https://api.github.com", + pypiapi: "https://pypi.org", + cargoapi: "https://crates.io", + dockerapi: "https://hub.docker.com", + goproxy: "https://proxy.golang.org", +} as const; -// The origin of the override when set (so tests and custom registries warm the host actually -// contacted), of the default otherwise, null when unparsable. -function resolveOrigin(override: unknown, defaultUrl: string): string | null { - try { - return `${new URL(typeof override === "string" && override ? override : defaultUrl).origin}/`; - } catch { - return null; - } -} - -// Which APIs each mode contacts, named by their override flag so the URLs live in -// defaultApiUrls alone. Keyed by mode rather than by filename, so giving a mode another -// manifest or another API has one place to update — prewarm.test.ts fails on a missing mode. -const apisByMode: Record> = { - npm: ["registry", "jsrapi", "forgeapi"], - pypi: ["pypiapi"], - cargo: ["cargoapi"], - go: ["goproxy"], - docker: ["dockerapi"], - actions: ["forgeapi", "dockerapi"], // workflows carry action refs and docker images - make: ["goproxy", "dockerapi"], // Makefiles carry `go install` tools and docker images -}; - -// The mode that claims a file, mirroring resolveFiles so prewarming cannot warm a different -// set of origins than the run goes on to contact. -function modeForFile(filename: string): string | undefined { - if (modeByFileName[filename]) return modeByFileName[filename]; - if (isDockerFileName(filename)) return "docker"; - if (isMakeFileName(filename)) return "make"; - return undefined; -} +const dependencyFields = ["dependencies", "devDependencies", "optionalDependencies", "peerDependencies", "resolutions"]; +const modeByName = (filename: string) => filename === "package.json" || filename === "pnpm-workspace.yaml" ? "npm" : + filename === "pyproject.toml" ? "pypi" : filename === "Cargo.toml" ? "cargo" : + filename === "go.mod" || filename === "go.work" ? "go" : + /^Dockerfile(?:\..+)?$/.test(filename) || /^(?:docker-|compose).*\.ya?ml$/.test(filename) ? "docker" : + ["Makefile", "makefile", "GNUmakefile"].includes(filename) || filename.endsWith(".mk") ? "make" : + /\.ya?ml$/.test(filename) ? "actions" : ""; -// Detect which registry origins should have a TLS keep-alive socket pre-warmed -// based on files present in `dir`, honoring the API override flags in `args`. -// Registry overrides from the config file are not seen here: it loads later. export function prewarmOrigins(dir: string, args: Record): string[] { - const modes = new Set(); - // `-M` only: a config-file `modes` has not loaded yet, as with the registry overrides below. - const cliModes = parseMixedArg(args.modes as Arg); - const enabled = (mode: string) => !(cliModes instanceof Set) || cliModes.has(mode); - try { - for (const entry of readdirSync(dir, {withFileTypes: true})) { - if (entry.isFile()) { - const mode = modeForFile(entry.name); - if (mode && enabled(mode)) modes.add(mode); - } else if (entry.isDirectory() && forgeDirs.some(forgeDir => forgeDir === entry.name)) { - // Bare forge dir, matching resolveFiles' auto-discovery: workflows also live - // outside `workflows/` as `/**/action.yml`. A workflow's docker images are - // read with docker alone enabled, so that mode claims the dir when actions is off. - if (enabled("actions")) modes.add("actions"); - else if (enabled("docker")) modes.add("docker"); + const enabledModes = Array.isArray(args.modes) ? new Set(args.modes) : typeof args.modes === "string" ? + new Set(args.modes.split(",")) : null; + const resources = new Set(); + const candidates = new Set(); + const paths = Array.isArray(args.files) && args.files.length ? args.files.filter(path => typeof path === "string") : [dir]; + for (const input of paths) { + const path = resolve(input); + try { + if (statSync(path).isFile()) { + candidates.add(path); + continue; } - } - } catch {} + for (const entry of readdirSync(path, {withFileTypes: true})) { + if (entry.isFile()) candidates.add(join(path, entry.name)); + else if ([".github", ".gitea", ".forgejo"].includes(entry.name)) { + try { + for (const workflow of readdirSync(join(path, entry.name, "workflows"), {withFileTypes: true})) { + if (workflow.isFile() && /\.ya?ml$/.test(workflow.name)) candidates.add(join(path, entry.name, "workflows", workflow.name)); + } + } catch {} + } + } + } catch {} + } + + for (const path of candidates) { + const filename = basename(path); + const mode = modeByName(filename); + if (!mode || enabledModes && !enabledModes.has(mode) && !(mode === "actions" && enabledModes.has("docker"))) continue; + try { + const content = readFileSync(path, "utf8"); + if (filename === "package.json") { + let data: Record; + try { data = JSON.parse(content); } catch { continue; } + const specs = dependencyFields.flatMap(field => Object.values(data[field] ?? {})); + if (typeof data.packageManager === "string") specs.push(data.packageManager.split("@", 1)[0]); + for (const spec of specs) { + if (typeof spec !== "string" || /^(?:file|link|workspace):/.test(spec)) continue; + if (/^(?:jsr:|npm:@jsr\/)/.test(spec)) resources.add("jsrapi"); + else if (/^(?:git(?:\+https?|\+ssh)?:|https?:\/\/[^/]*(?:github|gitea|forgejo)|github:|gitea:|forgejo:)/.test(spec)) { + resources.add("forgeapi"); + } else resources.add("registry"); + } + } else if (filename === "pnpm-workspace.yaml") { + if (/\b(?:jsr:|npm:@jsr\/)/.test(content)) resources.add("jsrapi"); + if (/\b(?:catalogs?|overrides):|:\s*["']?[~^<>=]*\d/.test(content)) resources.add("registry"); + } else if (filename === "pyproject.toml") { + if (/^[ \t]*["'][A-Za-z0-9][\w.-]*(?:\[[^\]]+\])?\s*(?:[<>=!~]|@)/m.test(content)) resources.add("pypiapi"); + } else if (filename === "Cargo.toml") { + if (/^\s*\[(?:target\.[^\]]+\.)?(?:dev-|build-)?dependencies\]/m.test(content)) resources.add("cargoapi"); + } else if (filename === "go.mod" || filename === "go.work") { + if (/^\s*(?:require\s+)?\S+\s+v\d/m.test(content)) resources.add("goproxy"); + } else if (/^Dockerfile(?:\..+)?$/.test(filename) || /^(?:docker-|compose).*\.ya?ml$/.test(filename)) { + if (/^\s*(?:FROM\s+(?:--\S+\s+)*|image\s*:\s*)[^\s#]+[:@]/im.test(content)) resources.add("dockerapi"); + } else if (["Makefile", "makefile", "GNUmakefile"].includes(filename) || filename.endsWith(".mk")) { + if (/\bgo\s+install\s+\S+@v\d/.test(content)) resources.add("goproxy"); + if (/\b(?:docker|image)\b[^\n]*[\w./-]+:[\w.-]+/i.test(content)) resources.add("dockerapi"); + } else { + let blockIndent = -1; + for (const line of content.split(/\r?\n/)) { + const indent = line.search(/\S|$/); + if (blockIndent !== -1 && (!line.trim() || indent > blockIndent)) continue; + blockIndent = /:\s*[|>](?:[1-9][+-]?|[+-][1-9]?)?\s*(?:#.*)?$/.test(line) ? indent : -1; + if ((!enabledModes || enabledModes.has("actions")) && + /^\s*(?:-\s*)?uses\s*:\s*["']?(?!\.\/|docker:\/\/)[^\s#]+@/.test(line)) resources.add("forgeapi"); + if ((!enabledModes || enabledModes.has("docker")) && + /^\s*(?:(?:container|image)\s*:\s*["']?[^\s#]+[:@]|(?:-\s*)?uses\s*:\s*["']?docker:\/\/)/.test(line)) { + resources.add("dockerapi"); + } + } + } + } catch {} + } const origins = new Set(); - for (const mode of modes) { - for (const api of apisByMode[mode] ?? []) { - // the npm registry is the only one that can also come from a file - const override = api === "registry" && typeof args.registry !== "string" ? npmrcRegistry(dir) : args[api]; - // GOPROXY, not the default, is where go lookups go, and its `off` and `direct` parse as no - // URL, so they warm nothing. - const origin = resolveOrigin(override, api === "goproxy" ? resolveGoProxyChain()[0].url : defaultApiUrls[api]); - if (origin) origins.add(origin); + for (const resource of resources) { + let value = args[resource]; + if (resource === "registry" && typeof value !== "string") { + try { value = /^\s*registry\s*=\s*(\S+)\s*$/m.exec(readFileSync(join(dir, ".npmrc"), "utf8"))?.[1]; } catch {} + } else if (resource === "goproxy" && typeof value !== "string") { + value = process.env.GOPROXY; + } + let origin = typeof value === "string" && value ? value : defaults[resource as keyof typeof defaults]; + if (resource === "goproxy") { + origin = origin.split(/[|,]/, 1)[0].trim(); + if (origin === "off" || origin === "direct") continue; } + try { origins.add(`${new URL(origin).origin}/`); } catch {} } return Array.from(origins); } diff --git a/utils/rc.test.ts b/utils/rc.test.ts index a2cfe02..43786b4 100644 --- a/utils/rc.test.ts +++ b/utils/rc.test.ts @@ -3,112 +3,47 @@ import {join} from "node:path"; import {tmpdir} from "node:os"; import rc, {parseIni, parseEnvVars} from "./rc.ts"; -// --- parseIni --- - -test("basic key=value", () => { - expect(parseIni("key=value")).toEqual({key: "value"}); -}); - -test("multiple lines", () => { - expect(parseIni("a=1\nb=2\nc=3")).toEqual({a: "1", b: "2", c: "3"}); -}); - -test("whitespace around keys and values", () => { - expect(parseIni(" key = value ")).toEqual({key: "value"}); -}); - -test("comments with # and ;", () => { - expect(parseIni("# comment\n; comment\nkey=value")).toEqual({key: "value"}); -}); - -test("empty lines are skipped", () => { - expect(parseIni("\n\nkey=value\n\n")).toEqual({key: "value"}); -}); - -test("lines without = are skipped", () => { - expect(parseIni("noequals\nkey=value")).toEqual({key: "value"}); -}); - -test("values with = in them", () => { - expect(parseIni("key=a=b=c")).toEqual({key: "a=b=c"}); -}); - -test("double-quoted values have quotes stripped", () => { - expect(parseIni("key=\"value\"")).toEqual({key: "value"}); -}); - -test("single-quoted values have quotes stripped", () => { - expect(parseIni("key='value'")).toEqual({key: "value"}); -}); - -test("mismatched quotes are preserved", () => { - expect(parseIni("key=\"value'")).toEqual({key: "\"value'"}); -}); - -test("single quote character is preserved", () => { - expect(parseIni("key=\"")).toEqual({key: "\""}); -}); - -test("npmrc registry-scoped auth tokens with quotes", () => { - const content = [ - "@scope:registry=https://npm.test", - "//registry.npmjs.org/:_authToken=\"npm_token123\"", - "//npm.test/:_authToken=\"private_token456\"", - ].join("\n"); - const result = parseIni(content); - expect(result["@scope:registry"]).toBe("https://npm.test"); - expect(result["//registry.npmjs.org/:_authToken"]).toBe("npm_token123"); - expect(result["//npm.test/:_authToken"]).toBe("private_token456"); -}); - -test("windows line endings", () => { - expect(parseIni("a=1\r\nb=2\r\n")).toEqual({a: "1", b: "2"}); -}); - -test("JSON content", () => { - expect(parseIni("{\"key\": \"value\"}")).toEqual({key: "value"}); -}); - -test("JSON and INI produce same result for simple object", () => { - const obj = {hello: "true"}; - const json = parseIni(JSON.stringify(obj)); - const ini = parseIni("hello=true"); - expect(json).toEqual(ini); -}); - -test("empty string", () => { - expect(parseIni("")).toEqual({}); -}); - -test("only comments", () => { - expect(parseIni("# comment\n; another")).toEqual({}); -}); - -test("npmrc basic auth with quoted password", () => { - const content = [ - "//npm.test/:username=user", - "//npm.test/:_password=\"cGFzcw==\"", - ].join("\n"); - const result = parseIni(content); - expect(result["//npm.test/:username"]).toBe("user"); - expect(result["//npm.test/:_password"]).toBe("cGFzcw=="); -}); - -test("npmrc legacy _auth", () => { - const result = parseIni("_auth=\"dXNlcjpwYXNz\""); - expect(result["_auth"]).toBe("dXNlcjpwYXNz"); -}); - -test("project config is found upwards from the given dir, not the cwd", () => { +test("parseIni", () => { + const cases: Array<[string, string, Record]> = [ + ["basic", "key=value", {key: "value"}], + ["multiple lines", "a=1\nb=2\nc=3", {a: "1", b: "2", c: "3"}], + ["whitespace", " key = value ", {key: "value"}], + ["comments", "# comment\n; comment\nkey=value", {key: "value"}], + ["empty lines", "\n\nkey=value\n\n", {key: "value"}], + ["line without equals", "noequals\nkey=value", {key: "value"}], + ["equals in value", "key=a=b=c", {key: "a=b=c"}], + ["double quotes", "key=\"value\"", {key: "value"}], + ["single quotes", "key='value'", {key: "value"}], + ["mismatched quotes", "key=\"value'", {key: "\"value'"}], + ["single quote character", "key=\"", {key: "\""}], + ["Windows lines", "a=1\r\nb=2\r\n", {a: "1", b: "2"}], + ["JSON", "{\"key\": \"value\"}", {key: "value"}], + ["empty", "", {}], + ["only comments", "# comment\n; another", {}], + ["scoped auth", [ + "@scope:registry=https://npm.test", + "//registry.npmjs.org/:_authToken=\"npm_token123\"", + "//npm.test/:_authToken=\"private_token456\"", + ].join("\n"), { + "@scope:registry": "https://npm.test", + "//registry.npmjs.org/:_authToken": "npm_token123", + "//npm.test/:_authToken": "private_token456", + }], + ["basic auth", "//npm.test/:username=user\n//npm.test/:_password=\"cGFzcw==\"", { + "//npm.test/:username": "user", "//npm.test/:_password": "cGFzcw==", + }], + ["legacy auth", "_auth=\"dXNlcjpwYXNz\"", {_auth: "dXNlcjpwYXNz"}], + ]; + for (const [, input, expected] of cases) expect(parseIni(input)).toEqual(expected); +}); + +test("project config is found from the supplied directory", () => { const dir = mkdtempSync(join(tmpdir(), "updates-rc-")); writeFileSync(join(dir, ".npmrc"), "registry=https://from-manifest-dir.test"); expect(rc("npm", {registry: "https://default.test"}, dir).registry).toBe("https://from-manifest-dir.test"); expect(rc("npm", {registry: "https://default.test"}).registry).not.toBe("https://from-manifest-dir.test"); }); -// --- parseEnvVars --- - -// Each case uses its own prefix so the vars cannot collide across tests. function withEnv(vars: Record, fn: () => void) { const originals = Object.keys(vars).map(key => [key, process.env[key]] as const); Object.assign(process.env, vars); @@ -122,47 +57,14 @@ function withEnv(vars: Record, fn: () => void) { } } -test("basic env var", () => { - withEnv({testrc_option: "42"}, () => { - expect(parseEnvVars("testrc_")).toEqual({option: "42"}); - }); -}); - -test("nested env vars with __", () => { +test("parseEnvVars", () => { + withEnv({testrc_option: "42"}, () => expect(parseEnvVars("testrc_")).toEqual({option: "42"})); withEnv({testrc2_someOpt__a: "42", testrc2_someOpt__z: "99"}, () => { - const result = parseEnvVars("testrc2_"); - expect(result.someOpt.a).toBe("42"); - expect(result.someOpt.z).toBe("99"); - }); -}); - -test("deeply nested env vars", () => { - withEnv({testrc3_a__b__c: "deep"}, () => { - expect(parseEnvVars("testrc3_").a.b.c).toBe("deep"); - }); -}); - -test("case-insensitive prefix matching", () => { - withEnv({TESTRC4_upperCase: "187"}, () => { - expect(parseEnvVars("testrc4_").upperCase).toBe("187"); - }); -}); - -test("scalar value not overridden by deeper key", () => { - withEnv({testrc5_opt__a: "42", testrc5_opt__a__b: "186"}, () => { - // Once opt.a is set as scalar, opt.a.b cannot override it - expect(parseEnvVars("testrc5_").opt.a).toBe("42"); - }); -}); - -test("trailing __ segments are filtered", () => { - withEnv({testrc6_w__w__: "18629"}, () => { - expect(parseEnvVars("testrc6_").w.w).toBe("18629"); - }); -}); - -test("leading __ segments are filtered", () => { - withEnv({testrc7___z__i__: "9999"}, () => { - expect(parseEnvVars("testrc7_").z.i).toBe("9999"); + expect(parseEnvVars("testrc2_")).toEqual({someOpt: {a: "42", z: "99"}}); }); + withEnv({testrc3_a__b__c: "deep"}, () => expect(parseEnvVars("testrc3_").a.b.c).toBe("deep")); + withEnv({TESTRC4_upperCase: "187"}, () => expect(parseEnvVars("testrc4_").upperCase).toBe("187")); + withEnv({testrc5_opt__a: "42", testrc5_opt__a__b: "186"}, () => expect(parseEnvVars("testrc5_").opt.a).toBe("42")); + withEnv({testrc6_w__w__: "18629"}, () => expect(parseEnvVars("testrc6_").w.w).toBe("18629")); + withEnv({testrc7___z__i__: "9999"}, () => expect(parseEnvVars("testrc7_").z.i).toBe("9999")); }); diff --git a/utils/rc.ts b/utils/rc.ts index 2019ef2..d6f3643 100644 --- a/utils/rc.ts +++ b/utils/rc.ts @@ -66,8 +66,6 @@ export function parseEnvVars(prefix: string): Record { return result; } -// `startDir` is where the project-level file is searched for upwards. A caller holding a manifest -// path must pass its directory, as npm resolves `.npmrc` as a sibling or parent of the package file. export default function rc(name: string, defaults: Record = {}, startDir: string = cwd()): Record { const win = platform === "win32"; const home = win ? env.USERPROFILE : env.HOME; diff --git a/utils/renovate.test.ts b/utils/renovate.test.ts index 596204a..39d161b 100644 --- a/utils/renovate.test.ts +++ b/utils/renovate.test.ts @@ -1,23 +1,25 @@ import {test, expect, afterAll} from "vitest"; -import {mkdtempSync, rmSync, mkdirSync, writeFileSync, copyFileSync} from "node:fs"; +import {mkdtempSync, rmSync, mkdirSync, writeFileSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; -import {fileURLToPath} from "node:url"; -import {loadRenovateConfig, makePresetFetcher, type PresetFetcher} from "./renovate.ts"; - -const fixturesDir = fileURLToPath(new URL("../fixtures/renovate/", import.meta.url)); - -// Adapt a synchronous URL→body resolver into a PresetFetcher, keeping mocks terse. -const fetcher = (fn: (url: string) => string | null): PresetFetcher => (url) => Promise.resolve(fn(url)); - -const noFetch = fetcher(() => null); - -const emptyPresets = fetcher(() => "{}"); - -// Both runners run a file's tests concurrently, so the globalThis.fetch swap below has to opt out. -const sequential = test.sequential ?? (test as any).serial ?? test; +import {loadRenovateConfig} from "./renovate.ts"; +import {esc, patternToRegex} from "./utils.ts"; const created: Array = []; +const exact = (name: string) => new RegExp(`^${esc(name)}$`); + +type ExpectedImport = Record & {$disabled?: Array, $enabled?: Array}; + +function expectImport(actual: Record, {$disabled = [], $enabled = [], ...expected}: ExpectedImport): void { + const enabled = (name: string) => { + if (actual.exclude?.some((pattern: string | RegExp) => patternToRegex(pattern).test(name))) return false; + return !actual.include?.length || actual.include.some((pattern: string | RegExp) => patternToRegex(pattern).test(name)); + }; + for (const name of $disabled) expect(enabled(name), `${name} should be disabled`).toBe(false); + for (const name of $enabled) expect(enabled(name), `${name} should be enabled`).toBe(true); + const {include: _include, exclude: _exclude, ...rest} = actual; + expect(rest).toEqual(expected); +} function makeDir(): string { const d = mkdtempSync(join(tmpdir(), "updates-renovate-")); @@ -32,66 +34,92 @@ afterAll(() => { test.each([ ["no config at all", null, null, {}], ["minimumReleaseAge, which needs --cooldown to opt in", "renovate.json", {minimumReleaseAge: "3 days"}, {}], - ["ignoreDeps", "renovate.json", {ignoreDeps: ["foo", "bar"]}, {exclude: ["foo", "bar"]}], + ["ignoreDeps", "renovate.json", {ignoreDeps: ["foo", "bar"]}, {$disabled: ["foo", "bar"], $enabled: ["Foo", "foo*"]}], ["a disabled packageRule", "renovate.json", - {packageRules: [{matchPackageNames: ["foo", "bar"], enabled: false}]}, {exclude: ["foo", "bar"]}], - // renovate disables everything except @types, which is an allow-list, not a no-op exclude + {packageRules: [{matchPackageNames: ["foo", "bar"], enabled: false}]}, {$disabled: ["foo", "bar"], $enabled: ["baz"]}], ["a disabled rule whose matchers are all negated", "renovate.json", - {packageRules: [{matchPackageNames: ["!/^@types/"], enabled: false}]}, {include: [/^@types/]}], + {packageRules: [{matchPackageNames: ["!/^@types/"], enabled: false}]}, {$disabled: ["react"], $enabled: ["@types/node"]}], ["allowedVersions", "renovate.json", {packageRules: [{matchPackageNames: ["react"], allowedVersions: "^18.0.0"}]}, - {pin: {react: "^18.0.0"}, pinNoDowngrade: true}], - ["an invalid allowedVersions range", "renovate.json", - {packageRules: [{matchPackageNames: ["foo"], allowedVersions: "not-a-range"}]}, {}], + {pin: {react: "^18.0.0"}, pinNoDowngrade: true, + renovateVersionRules: [{matchPackageNames: ["react"], allowedVersions: "^18.0.0"}]}], + ["later glob and regex allowedVersions rules replace literal pins", "renovate.json", {packageRules: [ + {matchPackageNames: ["react"], allowedVersions: "^18"}, + {matchPackageNames: ["*"], allowedVersions: "^19"}, + {matchPackageNames: ["vue"], allowedVersions: "^2"}, + {matchPackageNames: ["/^vue$/"], allowedVersions: "^3"}, + ]}, { + pin: {react: "^19", vue: "^3"}, + pinNoDowngrade: true, + renovateVersionRules: [ + {matchPackageNames: ["react"], allowedVersions: "^18"}, + {matchPackageNames: ["*"], allowedVersions: "^19"}, + {matchPackageNames: ["vue"], allowedVersions: "^2"}, + {matchPackageNames: [/^vue$/], allowedVersions: "^3"}, + ], + }], ["a deny-all followed by an allow-list", "renovate.json", {packageRules: [ {matchPackageNames: ["react"], enabled: false}, {matchPackageNames: ["*"], enabled: false}, - {matchPackageNames: ["react", "react-dom"], enabled: true}, // clears the earlier exclude too - ]}, {include: ["react", "react-dom"]}], + {matchPackageNames: ["react", "react-dom"], enabled: true}, + ]}, {$disabled: ["vue"], $enabled: ["react", "react-dom"]}], ["a deny-all with no matcher and nothing re-enabled", "renovate.json", - {packageRules: [{enabled: false}]}, {exclude: ["*"]}], + {packageRules: [{enabled: false}]}, {$disabled: ["foo"]}], ["a later enabled rule, which clears an earlier exclude", "renovate.json", { ignoreDeps: ["ignored"], packageRules: [ {matchPackageNames: ["foo", "bar"], enabled: false}, {matchPackageNames: ["foo"], enabled: true}, - {matchPackageNames: ["ignored"], enabled: true}, // ignoreDeps is not a packageRule, so it stays + {matchPackageNames: ["ignored"], enabled: true}, ], - }, {exclude: ["ignored", "bar"]}], + }, {$disabled: ["ignored", "bar"], $enabled: ["foo"]}], ["a later enabled rule, which clears every copy of an earlier exclude", "renovate.json", {packageRules: [ {matchPackageNames: ["foo"], enabled: false}, {matchPackageNames: ["foo"], enabled: false}, {matchPackageNames: ["foo"], enabled: true}, - ]}, {}], + ]}, {$enabled: ["foo"]}], ["none of the packageRules with non-name matchers", "renovate.json", {packageRules: [ {matchPackageNames: ["foo"], matchUpdateTypes: ["major"], enabled: false}, {matchManagers: ["npm"], enabled: false}, {matchPackageNames: ["webpack"], updateTypes: ["major"], enabled: false}, {matchPackageNames: ["rollup"], excludeDepNames: ["rollup"], enabled: false}, {matchPackageNames: ["vite"], depTypeList: ["devDependencies"], allowedVersions: "^1"}, - ]}, {}], + ]}, {$enabled: ["foo", "webpack", "rollup", "vite"]}], ["legacy package matchers", "renovate.json", {packageRules: [ + {packageName: "singular", enabled: false}, + {packagePattern: "^pattern", enabled: false}, {packageNames: ["foo"], enabled: false}, {packagePatterns: ["^bar"], enabled: false}, {matchPackagePrefixes: ["@baz/"], enabled: false}, - {matchPackageNames: ["qux"], excludePackageNames: ["qux"], enabled: false}, // and-not, skipped - {matchPackageNames: ["@qux/{/,}**"], enabled: false}, // the prefix form configMigration emits - ]}, {exclude: ["foo", /^bar/, "@baz/*", "@qux/*"]}], - // renovate needs a positive and every negation to match, which exclude cannot express - ["no packageRule mixing positive and negated matchers", "renovate.json", - {packageRules: [{matchPackageNames: ["@babel/*", "!@babel/core"], enabled: false}]}, {}], - ["a wider exclude a later rule cannot punch a hole in", "renovate.json", {packageRules: [ + {matchPackageNames: ["qux"], excludePackageNames: ["qux"], enabled: false}, + {matchPackageNames: ["@qux/{/,}**"], enabled: false}, + ]}, {$disabled: ["singular", "patterned", "foo", "barrel", "@baz/pkg", "@qux/pkg"], $enabled: ["qux"]}], + ["a packageRule mixing positive and negated matchers", "renovate.json", + {packageRules: [{matchPackageNames: ["@babel/*", "!@babel/core"], enabled: false}]}, + {$disabled: ["@babel/parser"], $enabled: ["@babel/core", "react"]}], + ["a wider exclude can be re-enabled by a later rule", "renovate.json", {packageRules: [ {matchPackageNames: ["@babel/*"], enabled: false}, {matchPackageNames: ["@babel/core"], enabled: true}, - ]}, {exclude: ["@babel/*"]}], + ]}, {$disabled: ["@babel/parser"], $enabled: ["@babel/core"]}], ["top-level enabled false, which disables everything", "renovate.json", - {enabled: false, ignoreDeps: ["foo"]}, {exclude: ["*"]}], - ["renovate.jsonc, comments and all", "renovate.jsonc", `{\n // ignore foo\n "ignoreDeps": ["foo"]\n}`, - {exclude: ["foo"]}], + {enabled: false, ignoreDeps: ["foo"]}, {$disabled: ["foo", "bar"]}], + ["only a literal renovate.json", "renovate.jsonc", {ignoreDeps: ["foo"]}, {$enabled: ["foo"]}], ])("loadRenovateConfig reads %s", async (_name, file, config, expected) => { const dir = makeDir(); if (file) writeFileSync(join(dir, file), typeof config === "string" ? config : JSON.stringify(config)); - expect(await loadRenovateConfig(dir)).toEqual(expected); + expectImport(await loadRenovateConfig(dir), expected); +}); + +test("regex allowedVersions forms are preserved for release filtering", async () => { + const dir = makeDir(); + writeFileSync(join(dir, "renovate.json"), JSON.stringify({packageRules: [ + {matchPackageNames: ["foo*", "!foobar"], allowedVersions: "/^1\\./"}, + {matchPackageNames: ["bar"], allowedVersions: "!/beta/i"}, + ]})); + expect(await loadRenovateConfig(dir)).toEqual({renovateVersionRules: [ + {matchPackageNames: ["foo*"], excludePackageNames: ["foobar"], allowedVersions: "/^1\\./"}, + {matchPackageNames: ["bar"], allowedVersions: "!/beta/i"}, + ]}); }); test.each([["3 days", 3], ["1 week", 7], ["12 hours", 0.5]])("minimumReleaseAge %s → cooldown", async (age, cooldown) => { @@ -107,254 +135,26 @@ test("a packageRule minimumReleaseAge with no matcher applies to every dependenc {matchPackageNames: ["esbuild"], minimumReleaseAge: "1 day"}, ]})); expect(await loadRenovateConfig(dir, {cooldown: true})) - .toEqual({overrides: [{cooldown: 7}, {include: ["esbuild"], cooldown: 1}]}); + .toEqual({renovateVersionRules: [{cooldownDays: 7}, {matchPackageNames: ["esbuild"], cooldownDays: 1}]}); }); test("a subdirectory inherits the config of a parent directory", async () => { const dir = makeDir(); writeFileSync(join(dir, "renovate.json"), JSON.stringify({ignoreDeps: ["foo"]})); mkdirSync(join(dir, "pkg")); - expect(await loadRenovateConfig(join(dir, "pkg"))).toEqual({exclude: ["foo"]}); -}); - -test("renovate.json5 comments, trailing commas, unquoted keys and single quotes", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json5"), `{ - // pin react - extends: ['github>sxzz/renovate-config'], - automerge: true, - packageRules: [ - {matchPackageNames: ['react'], allowedVersions: '^18.0.0',}, - ], - }`); - expect(await loadRenovateConfig(dir, {}, emptyPresets)).toEqual({pin: {react: "^18.0.0"}, pinNoDowngrade: true}); -}); - -test("extends github preset is fetched and merged", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json5"), `{ - extends: ['github>sxzz/renovate-config'], - ignoreDeps: ['local-dep'], - }`); - const fetched: Array = []; - const fetchText = fetcher((url) => { - fetched.push(url); - if (url.endsWith("/default.json")) { - return JSON.stringify({ - extends: ["config:recommended"], // built-in, skipped without network - ignoreDeps: ["node"], - packageRules: [{matchPackageNames: ["react"], allowedVersions: "^18"}], - }); - } - return null; - }); - expect(await loadRenovateConfig(dir, {}, fetchText)).toEqual({ - exclude: ["node", "local-dep"], - pin: {react: "^18"}, - pinNoDowngrade: true, - }); - expect(fetched[0]).toBe("https://raw.githubusercontent.com/sxzz/renovate-config/HEAD/default.json"); -}); - -// Presets keyed by their `org/` path, so a row only spells the graph it needs. -test.each([ - ["recursively", ["github>org/a"], - {a: {extends: ["github>org/b"], ignoreDeps: ["a"]}, b: {ignoreDeps: ["b"]}}, ["b", "a"]], - ["without looping on a cycle", ["github>org/a"], - {a: {extends: ["github>org/b"], ignoreDeps: ["a"]}, b: {extends: ["github>org/a"], ignoreDeps: ["b"]}}, ["b", "a"]], - // c is reached via both a and b (path-scoped seen), so it contributes on each path - ["on each path of a diamond", ["github>org/a", "github>org/b"], - {a: {extends: ["github>org/c"], ignoreDeps: ["a"]}, b: {extends: ["github>org/c"], ignoreDeps: ["b"]}, - c: {ignoreDeps: ["c"]}}, ["c", "a", "c", "b"]], -])("extends resolves %s", async (_name, extendsList, presets: Record, exclude) => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({extends: extendsList})); - const fetchText = fetcher((url) => { - const key = /\/org\/(\w+)\//.exec(url)?.[1]; - return key && presets[key] ? JSON.stringify(presets[key]) : null; - }); - expect(await loadRenovateConfig(dir, {}, fetchText)).toEqual({exclude}); -}); - -test("named preset is a file in the repo, subpath fetches the file", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({ - extends: ["github>org/a:group", "github>org/a:file/key", "gitlab>org/b//path/file", "github>org/c:default"], - })); - const urls: Array = []; - const fetchText = fetcher((url) => { - urls.push(url); - // `:group` is group.json, not a `presets` map inside the repo's default.json - if (url.endsWith("/org/a/HEAD/group.json")) return JSON.stringify({ignoreDeps: ["g"]}); - if (url.endsWith("/org/a/HEAD/file.json")) return JSON.stringify({key: {ignoreDeps: ["k"]}}); - if (url.endsWith("/org/b/-/raw/HEAD/path/file.json")) return JSON.stringify({ignoreDeps: ["f"]}); - if (url.endsWith("/org/c/HEAD/default.json")) return JSON.stringify({ignoreDeps: ["d"]}); - return null; - }); - expect(await loadRenovateConfig(dir, {}, fetchText)).toEqual({exclude: ["g", "k", "f", "d"]}); - expect(urls).toContain("https://raw.githubusercontent.com/org/a/HEAD/group.json"); - expect(urls).not.toContain("https://raw.githubusercontent.com/org/a/HEAD/default.json"); -}); - -test("named preset with an explicit extension is fetched verbatim", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({extends: ["github>org/a:group.jsonc"]})); - const fetchText = fetcher((url) => { - if (url.endsWith("/org/a/HEAD/group.jsonc")) return `{"ignoreDeps": ["g"]} // comment`; - return null; - }); - expect(await loadRenovateConfig(dir, {}, fetchText)).toEqual({exclude: ["g"]}); -}); - -// bun 1.3.14 deadlocks when these rejections are asserted from concurrent tests, so run them in order. -sequential.each([ - ["a named preset the file does not carry", ["github>org/a:file/foo"], - fetcher((url) => url.endsWith("/file.json") ? JSON.stringify({other: {ignoreDeps: ["nope"]}}) : null), - "Unable to resolve renovate preset github>org/a:file/foo: no preset foo in file"], - ["a preset that resolves to nothing", ["github>org/a"], noFetch, - "Unable to resolve renovate preset github>org/a: not found"], - ["an unreachable preset host", ["github>org/a"], () => Promise.reject(new Error("connect ECONNREFUSED")), - "Unable to resolve renovate preset github>org/a: connect ECONNREFUSED"], - ["an unparseable preset", ["github>org/a"], fetcher(() => "{bad json"), - "Unable to resolve renovate preset github>org/a: invalid JSON in https://raw.githubusercontent.com/org/a/HEAD/default.json"], -])("%s is fatal", async (_name, extendsList, fetchText, message) => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({extends: extendsList, ignoreDeps: ["own"]})); - await expect(loadRenovateConfig(dir, {}, fetchText)).rejects.toThrow(message); + expect(await loadRenovateConfig(join(dir, "pkg"))).toEqual({exclude: [exact("foo")]}); }); test.each([ - ["built-in and unresolvable presets", ["config:recommended", ":pinVersions", "local>org/a", "bitbucket>org/b"]], - ["inherited-key forges", ["__proto__>org/a", "constructor>org/b"]], -])("%s are skipped without fetching", async (_name, extendsList) => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({extends: extendsList, ignoreDeps: ["own"]})); - let called = false; - const fetchText = fetcher(() => { called = true; return null; }); - expect(await loadRenovateConfig(dir, {}, fetchText)).toEqual({exclude: ["own"]}); - expect(called).toBe(false); -}); - -test("gitea and forgejo presets resolve against their default endpoints", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({ - extends: ["gitea>org/a", "forgejo>org/b"], - })); - // Only the exact default-endpoint URLs return content, so a pass proves the URLs. - const fetchText = fetcher((url) => { - if (url === "https://gitea.com/api/v1/repos/org/a/raw/default.json?ref=HEAD") return JSON.stringify({ignoreDeps: ["gt"]}); - if (url === "https://code.forgejo.org/api/v1/repos/org/b/raw/default.json?ref=HEAD") return JSON.stringify({ignoreDeps: ["fj"]}); - return null; - }); - expect(await loadRenovateConfig(dir, {}, fetchText)).toEqual({exclude: ["gt", "fj"]}); -}); - -test("http preset is fetched directly as a single file", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({ - extends: ["https://git.example.com/org/repo/raw/branch/main/renovate.json"], - ignoreDeps: ["own"], - })); - const urls: Array = []; - const fetchText = fetcher((url) => { - urls.push(url); - return JSON.stringify({ignoreDeps: ["remote"]}); - }); - expect(await loadRenovateConfig(dir, {}, fetchText)).toEqual({exclude: ["remote", "own"]}); - expect(urls).toEqual(["https://git.example.com/org/repo/raw/branch/main/renovate.json"]); -}); - -test("extends accepts a bare string", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({extends: "github>org/a", ignoreDeps: ["own"]})); - const fetchText = fetcher((url) => url.endsWith("/default.json") ? JSON.stringify({ignoreDeps: ["remote"]}) : null); - expect(await loadRenovateConfig(dir, {}, fetchText)).toEqual({exclude: ["remote", "own"]}); -}); - -// Swap globalThis.fetch directly (not vi.stubGlobal, which bun's test runner lacks). -async function withFetch(impl: typeof fetch, fn: () => Promise): Promise { - const original = globalThis.fetch; - globalThis.fetch = impl; - try { - await fn(); - } finally { - globalThis.fetch = original; - } -} - -sequential.each([ - ["a failed body read", "x", () => Promise.resolve({ - ok: true, status: 200, headers: new Headers(), text: () => Promise.reject(new Error("reset")), - }), "https://example.com/x: reset"], - ["a non-ok response", "y", () => Promise.resolve(new Response(null, {status: 503})), "https://example.com/y: HTTP 503"], - ["an unreachable host", "w", () => Promise.reject(new Error("fetch failed")), "https://example.com/w: fetch failed"], -])("makePresetFetcher throws on %s", async (_name, path, impl, message) => { - const fetchText = makePresetFetcher({noCache: true}); - await withFetch(impl as unknown as typeof fetch, async () => { - await expect(fetchText(`https://example.com/${path}`)).rejects.toThrow(message); - }); -}); - -sequential("makePresetFetcher retries a transient failure rather than failing the run", async () => { - const fetchText = makePresetFetcher({noCache: true}); - let calls = 0; - const impl = () => ++calls === 1 ? - Promise.reject(Object.assign(new Error("socket"), {code: "ECONNRESET"})) : - Promise.resolve(new Response("{}", {status: 200})); - await withFetch(impl, async () => { - expect(await fetchText("https://example.com/r")).toBe("{}"); - }); -}); - -sequential("makePresetFetcher returns null on 404, so another candidate file can be tried", async () => { - const fetchText = makePresetFetcher({noCache: true}); - await withFetch(() => Promise.resolve(new Response(null, {status: 404})), async () => { - expect(await fetchText("https://example.com/z")).toBe(null); - }); -}); - -test.each([".github", ".gitea", ".forgejo", ".gitlab"])("forge dir config in %s", async (forge) => { - const dir = makeDir(); - mkdirSync(join(dir, forge)); - writeFileSync(join(dir, forge, "renovate.json"), JSON.stringify({minimumReleaseAge: "2 days"})); - expect(await loadRenovateConfig(dir, {cooldown: true})).toEqual({cooldown: 2}); -}); - -test("package.json renovate field", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "package.json"), JSON.stringify({ - name: "x", - renovate: {minimumReleaseAge: "5 days", ignoreDeps: ["foo"]}, - })); - expect(await loadRenovateConfig(dir, {cooldown: true})).toEqual({cooldown: 5, exclude: ["foo"]}); -}); - -test("renovate.json wins over forge config", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), JSON.stringify({minimumReleaseAge: "1 day"})); - mkdirSync(join(dir, ".github")); - writeFileSync(join(dir, ".github", "renovate.json"), JSON.stringify({minimumReleaseAge: "9 days"})); - expect(await loadRenovateConfig(dir, {cooldown: true})).toEqual({cooldown: 1}); -}); - -test("real-world config", async () => { - const dir = makeDir(); - copyFileSync(join(fixturesDir, "real-world.json5"), join(dir, "renovate.json5")); - expect(await loadRenovateConfig(dir, {cooldown: true})).toEqual({ - cooldown: 5, - exclude: [/^@types\//], - pin: { - "@mcaptcha/vanilla-glue": "^0.1", - "cropperjs": "^1", - "tailwindcss": "^3", - }, - pinNoDowngrade: true, - overrides: [{include: ["esbuild"], cooldown: 1}], - }); -}); - -test("malformed config throws", async () => { - const dir = makeDir(); - writeFileSync(join(dir, "renovate.json"), `{bad json`); - await expect(loadRenovateConfig(dir)).rejects.toThrow(/Unable to parse renovate config/); + ["invalid allowedVersions", {packageRules: [{matchPackageNames: ["foo"], allowedVersions: "not-a-range"}]}, + "Invalid renovate allowedVersions: not-a-range"], + ["top-level extends", {extends: ["config:recommended"]}, "extends"], + ["nested extends", {packageRules: [{matchPackageNames: ["foo"], extends: [":disableRenovate"]}]}, + "extends"], + ["malformed JSON", "{bad json", "Unable to parse renovate config"], +])("%s is rejected", async (_name, config, error) => { + const dir = makeDir(); + writeFileSync(join(dir, "renovate.json"), typeof config === "string" ? config : JSON.stringify(config)); + await expect(loadRenovateConfig(dir)).rejects.toThrow(error === "extends" ? + `Renovate extends is unsupported in ${join(dir, "renovate.json")}` : error); }); diff --git a/utils/renovate.ts b/utils/renovate.ts index 15a5ea5..70e0486 100644 --- a/utils/renovate.ts +++ b/utils/renovate.ts @@ -1,25 +1,8 @@ import {join} from "node:path"; import {readFile} from "node:fs/promises"; -import {parseJsonish} from "./json5.ts"; import {validRange} from "./semver.ts"; -import {walkUp, memoizeAsync, forgeDirs, getOrSet} from "./utils.ts"; -import {getCache, setCache} from "./fetchCache.ts"; -import {fetchRetries, isTransientFetchError} from "../modes/shared.ts"; -import type {Config, Override} from "../config.ts"; - -// Renovate also reads .gitlab, which has no workflow files and so is absent from the actions list. -const renovateDirs = [...forgeDirs, ".gitlab"]; - -const configFileNames = [ - "renovate.json", - "renovate.jsonc", - "renovate.json5", - ...renovateDirs.flatMap(dir => [`${dir}/renovate.json`, `${dir}/renovate.jsonc`, `${dir}/renovate.json5`]), - ".renovaterc", - ".renovaterc.json", - ".renovaterc.jsonc", - ".renovaterc.json5", -]; +import {walkUp, patternToRegex, esc} from "./utils.ts"; +import type {Config} from "../config.ts"; const durationUnits: Record = { y: 365, year: 365, years: 365, @@ -31,23 +14,17 @@ const durationUnits: Record = { s: 1 / 86400, second: 1 / 86400, seconds: 1 / 86400, }; -/** Parse a renovate duration string ("3 days", "1 week", "12 hours") into days. */ function parseRenovateDuration(str: string): number | undefined { - let total = 0; - let matched = false; - const re = /(\d+(?:\.\d+)?)\s*([a-z]+)/gi; - let m: RegExpExecArray | null; - while ((m = re.exec(str)) !== null) { - const mult = durationUnits[m[2].toLowerCase()]; - if (mult === undefined) return undefined; - total += Number(m[1]) * mult; - matched = true; + let total: number | undefined; + for (const match of str.matchAll(/(\d+(?:\.\d+)?)\s*([a-z]+)/gi)) { + const multiplier = durationUnits[match[2].toLowerCase()]; + if (multiplier === undefined) return undefined; + total = (total ?? 0) + Number(match[1]) * multiplier; } - return matched ? total : undefined; + return total; } type RenovateConfig = { - extends?: Array | string; enabled?: boolean; minimumReleaseAge?: string; ignoreDeps?: Array; @@ -62,169 +39,170 @@ type RenovatePackageRule = { [key: string]: unknown; }; -type Matcher = string | RegExp; - -// Deprecated spellings renovate still migrates into matchPackageNames. A prefix is minimatch -// `foo{/,}**`, which is updates' `foo*`, and renovate's own configMigration emits that form into -// matchPackageNames. -const packageNameKeys: Record string> = { - matchPackageNames: name => name.replace(/\{\/,\}\*\*$/, "*"), - packageNames: name => name, - matchPackagePatterns: pattern => pattern === "*" ? "*" : `/${pattern}/`, - packagePatterns: pattern => pattern === "*" ? "*" : `/${pattern}/`, - matchPackagePrefixes: prefix => `${prefix}*`, - excludePackageNames: name => `!${name}`, - excludePackagePatterns: pattern => `!/${pattern}/`, - excludePackagePrefixes: prefix => `!${prefix}*`, +export type Matcher = string | RegExp; + +type MatcherTarget = "package" | "dep"; + +const packageNameKeys: Record string}> = { + packageName: {target: "package", convert: name => name}, + packagePattern: {target: "package", convert: pattern => pattern === "*" ? "*" : `/${pattern}/`}, + matchPackageNames: {target: "package", convert: name => name}, + packageNames: {target: "package", convert: name => name}, + matchPackagePatterns: {target: "package", convert: pattern => pattern === "*" ? "*" : `/${pattern}/`}, + packagePatterns: {target: "package", convert: pattern => pattern === "*" ? "*" : `/${pattern}/`}, + matchPackagePrefixes: {target: "package", convert: prefix => `${prefix}{/,}**`}, + excludePackageNames: {target: "package", convert: name => `!${name}`}, + excludedPackageNames: {target: "package", convert: name => `!${name}`}, + excludePackagePatterns: {target: "package", convert: pattern => `!/${pattern}/`}, + excludePackagePrefixes: {target: "package", convert: prefix => `!${prefix}{/,}**`}, + matchDepNames: {target: "dep", convert: name => name}, + matchDepPatterns: {target: "dep", convert: pattern => `/${pattern}/`}, + matchDepPrefixes: {target: "dep", convert: prefix => `${prefix}{/,}**`}, + excludeDepNames: {target: "dep", convert: name => `!${name}`}, + excludeDepPatterns: {target: "dep", convert: pattern => `!/${pattern}/`}, + excludeDepPrefixes: {target: "dep", convert: prefix => `!${prefix}{/,}**`}, }; -// Matchers that migrate to something other than matchPackageNames and lack a match/exclude prefix. -const legacyMatcherKeys = ["updateTypes", "managers", "datasources", "depTypeList", "paths", "languages", "baseBranchList", "sourceUrlPrefixes"]; +const legacyMatcherKeys = new Set([ + "updateTypes", "managers", "datasources", "depTypeList", "paths", "languages", "baseBranchList", + "sourceUrlPrefixes", "matchFiles", "matchPaths", +]); -// Renovate's `*` matches everything (lib/util/string-match.ts), as does minimatch `**`. -const catchAllNames = new Set(["*", "**"]); - -// The name patterns a packageRule matches on, `[]` when it has no matcher at all (renovate then -// applies it to every dependency), or undefined when a matcher cannot be mapped, either because it -// matches on something else or because its value is not a list of names. -function ruleNames(rule: RenovatePackageRule): Array | undefined { - const names: Array = []; +function compileRule(rule: RenovatePackageRule): {matchers: RenovateVersionRule, literals: Array} | undefined { + const names = {Package: [] as Array, Dep: [] as Array}; for (const [key, value] of Object.entries(rule)) { - // Object.hasOwn, not `in`: `in` matches inherited keys like __proto__/constructor. - const toName = Object.hasOwn(packageNameKeys, key) ? packageNameKeys[key] : undefined; - if (!toName) { - if (key.startsWith("match") || key.startsWith("exclude") || legacyMatcherKeys.includes(key)) return undefined; + const matcher = Object.hasOwn(packageNameKeys, key) ? packageNameKeys[key] : undefined; + if (!matcher) { + if (key.startsWith("match") || key.startsWith("exclude") || legacyMatcherKeys.has(key)) return undefined; continue; } - const list = typeof value === "string" ? [value] : value; // renovate allowString + const list = typeof value === "string" || key === "packageName" || key === "packagePattern" ? [value] : value; if (!Array.isArray(list)) return undefined; for (const entry of list) { if (typeof entry !== "string" || !entry) return undefined; - names.push(toName(entry)); + names[matcher.target === "package" ? "Package" : "Dep"].push(matcher.convert(entry)); + } + } + const matchers: RenovateVersionRule = {}; + const literals: Array = []; + for (const [target, values] of Object.entries(names) as Array<["Package" | "Dep", Array]>) { + const include: Array = []; + const exclude: Array = []; + for (const name of values) { + const list = name.startsWith("!") ? exclude : include; + const value = list === exclude ? name.slice(1) : name; + list.push(renovateRegex(value) ?? value); + if (list === include && !/[*?[\]{}!()|+]/.test(value) && !renovateRegex(value)) literals.push(value); } + if (include.length) matchers[`match${target}Names`] = include; + if (exclude.length) matchers[`exclude${target}Names`] = exclude; } - return names; + return {matchers, literals}; } -// Renovate matchers negate with a leading `!`, and a rule applies when one positive and every -// negative matches. -function splitNames(names: Array): {positive: Array, negated: Array} { - const groups = Object.groupBy(names, name => name.startsWith("!") && name.length > 1 ? "negated" : "positive"); - return {positive: groups.positive ?? [], negated: groups.negated?.map(name => name.substring(1)) ?? []}; +function renovateRegex(value: string): RegExp | undefined { + const match = /^!?\/(.*)\/(i?)$/.exec(value); + if (!match) return undefined; + try { + return new RegExp(match[1], match[2]); + } catch { + return undefined; + } } -// Renovate compares matchers by value, so two identical /pattern/flags are the same matcher. -function sameMatcher(a: Matcher, b: Matcher): boolean { - return a instanceof RegExp && b instanceof RegExp ? a.source === b.source && a.flags === b.flags : a === b; -} +export type RenovateImportOptions = {cooldown?: boolean}; -// Candidates are read concurrently rather than one await at a time — the common -// case is a directory holding none of them, and `find` still honors the priority -// order of configFileNames. -async function readFirstExisting(rootDir: string): Promise<{path: string, text: string} | undefined> { - const reads = await Promise.all(configFileNames.map(async name => { - const path = join(rootDir, ...name.split("/")); - try { - return {path, text: await readFile(path, "utf8")}; - } catch { - return null; - } - })); - const found = reads.find(read => read !== null); - if (found) return found; +export type RenovateVersionRule = { + matchPackageNames?: Array; + excludePackageNames?: Array; + matchDepNames?: Array; + excludeDepNames?: Array; + allowedVersions?: string; + cooldownDays?: number; +}; - try { - const pkgPath = join(rootDir, "package.json"); - const pkg = JSON.parse(await readFile(pkgPath, "utf8")); - if (pkg && typeof pkg === "object" && pkg.renovate && typeof pkg.renovate === "object") { - return {path: pkgPath, text: JSON.stringify(pkg.renovate)}; - } - } catch {} - return undefined; +const matchesRuleList = (value: string, include?: Array, exclude?: Array) => + (!include?.length || include.some(pattern => patternToRegex(pattern).test(value))) && + (!exclude?.length || exclude.every(pattern => !patternToRegex(pattern).test(value))); + +export function matchesRenovateRule(rule: RenovateVersionRule, packageName: string, depName: string): boolean { + return matchesRuleList(packageName, rule.matchPackageNames, rule.excludePackageNames) && + matchesRuleList(depName, rule.matchDepNames, rule.excludeDepNames); } -/** Renovate uses /pattern/ or /pattern/flags for regex matchers. */ -function toMatcher(name: string): string | RegExp { - const m = /^\/(.+)\/([a-z]*)$/.exec(name); - if (!m) return name; - try { - return new RegExp(m[1], m[2]); - } catch { - return name; +const nameMatcherTests = new WeakMap boolean>(); + +class RenovateNameMatcher extends RegExp { + constructor(source: string, predicate: (packageName: string, depName: string) => boolean) { + super(source); + nameMatcherTests.set(this, predicate); + } + + testNames(packageName: string, depName: string): boolean { + return nameMatcherTests.get(this)!(packageName, depName); } -} -// Renovate matchPackageNames entries may be minimatch globs (e.g. "@babel/*"), -// whose characters are never valid in a package identifier across ecosystems. -function isGlob(name: string): boolean { - return /[*?[\]{}!()|+]/.test(name); + override test(name: string): boolean { + return this.testNames(name, name); + } } -export type RenovateImportOptions = { - /** Import minimumReleaseAge as cooldown. Off by default. */ - cooldown?: boolean; -}; +export function testRenovateMatcher(matcher: RegExp, value: string, packageName: string, depName: string): boolean { + return matcher instanceof RenovateNameMatcher ? matcher.testNames(packageName, depName) : matcher.test(value); +} -// Apply the packageRules in order, as renovate does: a matching rule disables or re-enables what it -// matches and the last match wins. `allowed` becomes non-null once a rule disables everything, -// turning the remainder into an allow-list. A rule needing an and-not is skipped, not approximated. -function applyRules(rules: Array): {allowed: Array | null, disabled: Array, pin: Record, cooldowns: Array} { - let allowed: Array | null = null; - let disabled: Array = []; +function applyRules(rules: Array, inheritCooldown: boolean): { + disabled?: RegExp, pin: Record, versionRules: Array +} { + const enabledRules: Array<{enabled: boolean, matchers: RenovateVersionRule}> = []; const pin: Record = {}; - const cooldowns: Array = []; + const pinCandidates = new Set(); + const versionRules: Array = []; for (const rule of rules) { if (!rule || typeof rule !== "object") continue; - const names = ruleNames(rule); - if (!names) continue; - const {positive, negated} = splitNames(names); - - // a rule mixing matchers with negations needs an and-not, which exclude cannot express - if (rule.enabled === false && !(positive.length && negated.length)) { - if (!names.length || positive.some(name => catchAllNames.has(name))) { - allowed = []; // disables everything, until a later rule re-enables names - } else if (!positive.length) { - if (allowed === null) allowed = negated.map(toMatcher); // all-negated leaves an allow-list - } else { - for (const name of positive) disabled.push(toMatcher(name)); - } - } else if (rule.enabled === true && (allowed !== null || disabled.length)) { - if (!names.length) { - allowed = null; // re-enables every dependency - disabled = []; - } else if (positive.length && !negated.length) { // a re-enable by negation is not expressible - for (const name of positive) { - const matcher = toMatcher(name); - // an exclude the name only partly overlaps cannot be punched a hole in - disabled = disabled.filter(entry => !sameMatcher(entry, matcher)); - allowed?.push(matcher); - } - } - } + const compiled = compileRule(rule); + if (!compiled) continue; + const {matchers, literals} = compiled; + if (typeof rule.enabled === "boolean") enabledRules.push({enabled: rule.enabled, matchers}); + const versionRule: RenovateVersionRule = {...matchers}; - // renovate's per-rule minimumReleaseAge beats the top-level one for what it matches, last rule - // winning, which is exactly an updates override - if (typeof rule.minimumReleaseAge === "string" && !negated.length) { + if (inheritCooldown && typeof rule.minimumReleaseAge === "string") { const days = parseRenovateDuration(rule.minimumReleaseAge); - if (days !== undefined) cooldowns.push({include: names.length ? positive.map(toMatcher) : undefined, cooldown: days}); + if (days !== undefined) versionRule.cooldownDays = days; } - if (typeof rule.allowedVersions === "string" && validRange(rule.allowedVersions)) { - // Renovate applies allowedVersions as a ceiling on releases already newer than the current - // one, so it never rolls a dependency back, unlike a pin the authored version violates - // (findVersion in modes/shared.ts). Only literal names can pin. - for (const name of positive) { - if (!isGlob(name) && typeof toMatcher(name) === "string") pin[name] = rule.allowedVersions; + if (typeof rule.allowedVersions === "string") { + const allowedRange = validRange(rule.allowedVersions); + if (!allowedRange && !renovateRegex(rule.allowedVersions)) { + throw new Error(`Invalid renovate allowedVersions: ${rule.allowedVersions}`); } + versionRule.allowedVersions = rule.allowedVersions; + if (allowedRange) for (const name of literals) pinCandidates.add(name); } + if (versionRule.allowedVersions || versionRule.cooldownDays !== undefined) versionRules.push(versionRule); } - return {allowed, disabled, pin, cooldowns}; + for (const name of pinCandidates) { + const allowedVersions = versionRules.findLast(rule => + rule.allowedVersions !== undefined && matchesRenovateRule(rule, name, name))?.allowedVersions; + if (allowedVersions && validRange(allowedVersions)) pin[name] = allowedVersions; + } + + let disabled: RegExp | undefined; + if (enabledRules.some(rule => !rule.enabled)) { + disabled = new RenovateNameMatcher("renovate-package-rules", (packageName, depName) => { + let enabled = true; + for (const rule of enabledRules) { + if (matchesRenovateRule(rule.matchers, packageName, depName)) enabled = rule.enabled; + } + return !enabled; + }); + } + return {disabled, pin, versionRules}; } function normalize(raw: RenovateConfig, opts: RenovateImportOptions): Partial { - // renovate skips a repository whose config disables it (lib/workers/repository/configured.ts) if (raw.enabled === false) return {exclude: ["*"]}; const out: Partial = {}; @@ -234,259 +212,47 @@ function normalize(raw: RenovateConfig, opts: RenovateImportOptions): Partial 0) out.cooldown = days; } - // no `enabled: true` rule clears ignoreDeps, so it stays out of applyRules and merges at the end - const ignored: Array = Array.isArray(raw.ignoreDeps) ? - raw.ignoreDeps.filter(dep => typeof dep === "string" && Boolean(dep)) : []; - const {allowed, disabled, pin, cooldowns} = applyRules(Array.isArray(raw.packageRules) ? raw.packageRules : []); + const ignored: Array = Array.isArray(raw.ignoreDeps) ? raw.ignoreDeps + .filter(dep => typeof dep === "string" && Boolean(dep)) + .map(dep => new RenovateNameMatcher(`^${esc(dep)}$`, (_packageName, depName) => depName === dep)) : []; + const {disabled, pin, versionRules} = applyRules(Array.isArray(raw.packageRules) ? raw.packageRules : [], Boolean(opts.cooldown)); - const exclude = [...ignored, ...disabled]; - if (allowed?.length) out.include = allowed; - if (allowed && !allowed.length) exclude.push("*"); // everything disabled, nothing re-enabled + const exclude = disabled ? [...ignored, disabled] : ignored; if (exclude.length) out.exclude = exclude; if (Object.keys(pin).length) { out.pin = pin; out.pinNoDowngrade = true; } - if (opts.cooldown && cooldowns.length) out.overrides = cooldowns; + if (versionRules.length) (out as Partial & {renovateVersionRules: Array}).renovateVersionRules = versionRules; return out; } -// Fetch a preset file as text, or null if it does not exist. Throws when the host cannot be reached, -// which renovate also treats as fatal rather than as an empty preset. Injectable for tests. -export type PresetFetcher = (url: string) => Promise; - -// Forge presets resolve against a fixed public endpoint (as Renovate does): the -// host is never part of the `forge>` string. gitea>/forgejo> point at gitea.com / -// code.forgejo.org, matching Renovate's default endpoints. A self-hosted instance -// is reached instead via an `http` preset (a full raw URL). local> needs the -// running platform and built-in presets (config:, :x, helpers:, …) have no URL; -// both are skipped. -const forgeRawUrl: Record string> = { - github: (slug, ref, file) => `https://raw.githubusercontent.com/${slug}/${ref}/${file}`, - gitlab: (slug, ref, file) => `https://gitlab.com/${slug}/-/raw/${ref}/${file}`, - gitea: (slug, ref, file) => `https://gitea.com/api/v1/repos/${slug}/raw/${file}?ref=${ref}`, - forgejo: (slug, ref, file) => `https://code.forgejo.org/api/v1/repos/${slug}/raw/${file}?ref=${ref}`, -}; - -const maxPresetDepth = 10; - -type PresetLocation = - {kind: "forge", forge: string, slug: string, ref: string, name?: string, subpath?: string} | - {kind: "http", url: string}; - -/** - * Parse a Renovate preset reference into a fetchable location, or null if it is - * a built-in or otherwise unresolvable preset. Handles a full `http(s)://` URL, - * `forge>owner/repo`, `:preset` names, `//path` subpaths, `#ref` refs and - * `(params)` (params ignored). - */ -function parsePreset(preset: string): PresetLocation | null { - if (/^https?:\/\//i.test(preset)) return {kind: "http", url: preset}; // custom-FQDN raw preset file - const gt = preset.indexOf(">"); - if (gt === -1) return null; // built-in preset, no URL - const forge = preset.slice(0, gt); - // Object.hasOwn, not `in`: `in` matches inherited keys like __proto__/constructor. - if (!Object.hasOwn(forgeRawUrl, forge)) return null; // local>, unknown forge - let rest = preset.slice(gt + 1).replace(/\([^)]*\)\s*$/, ""); // drop params, unsupported - let ref = "HEAD"; - const hash = rest.indexOf("#"); - if (hash !== -1) { ref = rest.slice(hash + 1) || "HEAD"; rest = rest.slice(0, hash); } - let subpath: string | undefined; - let name: string | undefined; - const dslash = rest.indexOf("//"); - if (dslash !== -1) { - subpath = rest.slice(dslash + 2); - rest = rest.slice(0, dslash); - } else { - const colon = rest.indexOf(":"); - if (colon !== -1) { name = rest.slice(colon + 1); rest = rest.slice(0, colon); } - } - const slug = rest.replace(/\/+$/, ""); - if (!slug.includes("/")) return null; - return {kind: "forge", forge, slug, ref, name, subpath}; -} - -// Repo config files Renovate probes, in order, to locate a preset's source. -const presetConfigFiles = ["default.json", "default.json5", "renovate.json", "renovate.json5", ".renovaterc.json", ".renovaterc"]; - -// Renovate takes .json, .json5 and .jsonc as explicit extensions and appends .json otherwise. -// .json5 is probed too, as the repo config file list already does. -function presetFiles(file: string): Array { - return /\.json[5c]?$/.test(file) ? [file] : [`${file}.json`, `${file}.json5`]; -} - -// A missing file is null and left to the caller, an unparseable one is fatal as renovate's PRESET_INVALID_JSON. -function parsePresetBody(body: string | null, url: string): RenovateConfig | null { - if (body === null) return null; - let parsed: unknown; - try { - parsed = parseJsonish(body); - } catch { - throw new Error(`invalid JSON in ${url}`); - } - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error(`invalid preset in ${url}`); - return parsed as RenovateConfig; +function containsExtends(value: unknown): boolean { + return Boolean(value && typeof value === "object" && + (Object.hasOwn(value, "extends") || Object.values(value).some(containsExtends))); } -async function fetchPresetConfig(loc: PresetLocation, fetchText: PresetFetcher): Promise { - // An `http` preset is a full URL to a single file; the whole document is the preset. - if (loc.kind === "http") return parsePresetBody(await fetchText(loc.url), loc.url); - - const build = forgeRawUrl[loc.forge]; - const firstExisting = async (files: Array) => { - for (const file of files) { - const url = build(loc.slug, loc.ref, file); - const parsed = parsePresetBody(await fetchText(url), url); - if (parsed) return parsed; - } - return null; - }; - - // An explicit `//path` points straight at a file. - if (loc.subpath) return firstExisting(presetFiles(loc.subpath)); - if (!loc.name) return firstExisting(presetConfigFiles); - - // A named preset is `file[/key[/subkey]]` inside the repo, so `:npm` is npm.json and `:file/key` - // is that key of file.json. There is no `presets` map for forge presets, that is npm-preset only. - const [file, ...keys] = loc.name.split("/"); - let preset = await firstExisting(presetFiles(file)); - for (const key of keys) { - const sub = preset && Object.hasOwn(preset, key) ? preset[key] : undefined; - if (!sub || typeof sub !== "object") throw new Error(`no preset ${key} in ${file}`); - preset = sub as RenovateConfig; - } - return preset; -} - -// Concatenate arrays (packageRules, ignoreDeps), let scalars from `over` win. -function mergeRenovate(base: RenovateConfig, over: RenovateConfig): RenovateConfig { - const out: RenovateConfig = {...base}; - for (const [key, value] of Object.entries(over)) { - const prev = out[key]; - out[key] = Array.isArray(value) && Array.isArray(prev) ? [...prev, ...value] : value; - } - return out; -} - -/** - * Recursively resolve `extends` presets and merge them ahead of the config's own fields (which - * take precedence), mirroring Renovate. A preset that cannot be resolved is fatal, like - * Renovate's CONFIG_VALIDATION, so an unreachable or private preset never silently drops the - * restrictions it carries. `seen` is the current resolution path (cloned per branch) so cycles - * are caught while diamonds still resolve on each path, matching Renovate's path-scoped recursion. - */ -async function resolveExtends( - cfg: RenovateConfig, fetchText: PresetFetcher, seen: Set, depth: number, -): Promise { - let merged: RenovateConfig = {}; - // renovate declares extends as allowString and massages it into an array (lib/config/massage.ts) - const presets = typeof cfg.extends === "string" ? [cfg.extends] : Array.isArray(cfg.extends) ? cfg.extends : []; - for (const preset of presets) { - if (typeof preset !== "string" || depth >= maxPresetDepth || seen.has(preset)) continue; - const loc = parsePreset(preset); - if (!loc) continue; - let raw: RenovateConfig | null; +export async function loadRenovateConfig( + rootDir: string, opts: RenovateImportOptions = {}, +): Promise> { + const found = await walkUp(rootDir, async dir => { + const path = join(dir, "renovate.json"); + let text: string; try { - raw = await fetchPresetConfig(loc, fetchText); - } catch (err: any) { - throw new Error(`Unable to resolve renovate preset ${preset}: ${err.message}`); - } - if (!raw) throw new Error(`Unable to resolve renovate preset ${preset}: not found`); - merged = mergeRenovate(merged, await resolveExtends(raw, fetchText, new Set(seen).add(preset), depth + 1)); - } - const {extends: _extends, ...own} = cfg; - return mergeRenovate(merged, own); -} - -/** Options controlling the production preset fetcher, mirroring the CLI's cache/timeout flags. */ -export type PresetFetchOptions = {noCache?: boolean, timeout?: number}; - -// Fallback only for direct API callers; the CLI always passes the resolved -// config.timeout (default 5000). Kept bounded so a hanging preset host can never -// stall startup the way a fixed 30s per candidate file did. -const defaultPresetTimeout = 10000; - -// Build request headers (shared user-agent/encoding via getFetchOpts) plus a host -// token for private presets, matching Renovate's hostRules model. Reuses updates' -// token resolution (UPDATES_FORGE_TOKENS per host, plus GitHub env/`gh` tokens), -// imported lazily so config loads without presets pull in nothing. -async function presetHeaders(url: string, etag?: string): Promise> { - const {getForgeTokens, getFetchOpts, githubApiUrl, urlHost} = await import("../modes/shared.ts"); - const host = urlHost(url); - const [token] = host ? await getForgeTokens(host, githubApiUrl) : []; - const headers = {...getFetchOpts("Bearer", token).headers} as Record; - if (etag) headers["if-none-match"] = etag; - return headers; -} - -/** - * Build the production preset fetcher: ETag-revalidated, honoring `noCache` and `timeout`, - * sending a host token when one is configured. Only a 404 is a missing file; a cached copy - * answers anything else that goes wrong, and without one the failure is thrown so a preset - * outage cannot pass for an empty preset. - */ -export function makePresetFetcher({noCache = false, timeout = defaultPresetTimeout}: PresetFetchOptions = {}): PresetFetcher { - return async (url) => { - const cached = noCache ? null : await getCache(url); - const headers = await presetHeaders(url, cached?.etag); - let res: Response; - // This runs before any dependency, so an unretried blip would cost the whole run. - for (let attempt = 0; ; attempt++) { - try { - res = await fetch(url, {headers, signal: AbortSignal.timeout(timeout)}); - break; - } catch (err: any) { - if (attempt < fetchRetries && isTransientFetchError(err)) continue; - if (cached) return cached.body; - throw new Error(`${url}: ${err.message}`); // offline / connect failure / timeout - } - } - if (res.status === 304 && cached) return cached.body; - if (res.status === 404) return null; // file is absent, the caller may have another candidate - if (!res.ok) { - if (cached) return cached.body; - throw new Error(`${url}: HTTP ${res.status}`); // server error / rate-limit / private repo + text = await readFile(path, "utf8"); + } catch { + return null; } - let body: string; + let parsed: unknown; try { - body = await res.text(); + parsed = JSON.parse(text); } catch (err: any) { - if (cached) return cached.body; - throw new Error(`${url}: ${err.message}`); // mid-stream abort/reset + throw new Error(`Unable to parse renovate config ${path}: ${err.message}`); } - const etag = res.headers.get("etag"); - if (etag && !noCache) setCache(url, etag, body); - return body; - }; -} - -type RenovateRaw = {parsed: RenovateConfig, path: string}; - -const findRenovateUp = memoizeAsync((startDir: string) => walkUp(startDir, async (dir): Promise => { - const found = await readFirstExisting(dir); - if (!found) return null; - let raw: unknown; - try { - raw = parseJsonish(found.text); - } catch (err: any) { - throw new Error(`Unable to parse renovate config ${found.path}: ${err.message}`); - } - if (!raw || typeof raw !== "object") return null; - return {parsed: raw as RenovateConfig, path: found.path}; -})); - -// Both keyed by config-file path, as loadRenovateConfig runs once per manifest directory and a -// monorepo shares one config across them, where resolving `extends` is network I/O. Callers spread -// the result rather than mutate it. -const resolvedExtendsCache = new Map>(); -const normalizedCache = new Map>>(); - -export async function loadRenovateConfig( - rootDir: string, opts: RenovateImportOptions = {}, fetchText: PresetFetcher = makePresetFetcher(), -): Promise> { - const found = await findRenovateUp(rootDir); + return parsed && typeof parsed === "object" ? {parsed: parsed as RenovateConfig, path} : null; + }); if (!found) return {}; - const resolved = getOrSet(resolvedExtendsCache, found.path, () => resolveExtends(found.parsed, fetchText, new Set(), 0)); - return getOrSet(normalizedCache, `${opts.cooldown ? 1 : 0}${found.path}`, async () => normalize(await resolved, opts)); + if (containsExtends(found.parsed)) throw new Error(`Renovate extends is unsupported in ${found.path}`); + return normalize(found.parsed, opts); } diff --git a/utils/semver.test.ts b/utils/semver.test.ts index 0b21598..c6d4a3c 100644 --- a/utils/semver.test.ts +++ b/utils/semver.test.ts @@ -1,323 +1,148 @@ -import {valid, parse, coerce, diff, gt, satisfies, validRange, parsePep440, comparePep440, diffPep440, pep440Versioning, semverVersioning} from "./semver.ts"; - -test("valid", () => { - expect(valid("1.0.0")).toBe("1.0.0"); - expect(valid("v1.0.0")).toBe("1.0.0"); - expect(valid("1.2.3")).toBe("1.2.3"); - expect(valid("1.2.3-alpha.1")).toBe("1.2.3-alpha.1"); - expect(valid("1.0.0-beta")).toBe("1.0.0-beta"); - expect(valid("1.0.0+build")).toBe("1.0.0"); - expect(valid("1.0.0-alpha+build")).toBe("1.0.0-alpha"); - expect(valid(" 1.0.0 ")).toBe("1.0.0"); - expect(valid("abc")).toBeNull(); - expect(valid("")).toBeNull(); - expect(valid("1.0")).toBeNull(); - expect(valid("1")).toBeNull(); - expect(valid("1.0.0.0")).toBeNull(); -}); - -test("parse", () => { - const result = parse("1.2.3"); - expect(result).toEqual({major: 1, minor: 2, patch: 3, prerelease: [], version: "1.2.3"}); - - const prerelease = parse("1.2.3-alpha.1"); - expect(prerelease).toEqual({major: 1, minor: 2, patch: 3, prerelease: ["alpha", 1], version: "1.2.3-alpha.1"}); - - const numericPre = parse("1.0.0-0.3.7"); - expect(numericPre!.prerelease).toEqual([0, 3, 7]); - - const mixedPre = parse("1.0.0-beta.11"); - expect(mixedPre!.prerelease).toEqual(["beta", 11]); +import { + valid, parse, coerce, diff, gt, satisfies, validRange, parsePep440, comparePep440, diffPep440, + pep440Versioning, semverVersioning, +} from "./semver.ts"; + +test("valid and parse", () => { + for (const [input, expected] of [ + ["1.0.0", "1.0.0"], ["v1.0.0", "1.0.0"], ["1.2.3", "1.2.3"], + ["1.2.3-alpha.1", "1.2.3-alpha.1"], ["1.0.0-beta", "1.0.0-beta"], + ["1.0.0+build", "1.0.0"], ["1.0.0-alpha+build", "1.0.0-alpha"], [" 1.0.0 ", "1.0.0"], + ]) expect(valid(input)).toBe(expected); + for (const input of ["abc", "", "1.0", "1", "1.0.0.0", "01.2.3", "1.2.3-alpha.01", "1.2.3-alpha_1", "9007199254740993.0.0"]) { + expect(valid(input)).toBeNull(); + } + expect(parse("1.2.3")).toEqual({major: 1, minor: 2, patch: 3, prerelease: [], build: [], raw: "1.2.3", version: "1.2.3"}); + expect(parse("1.2.3-alpha.1")).toEqual({ + major: 1, minor: 2, patch: 3, prerelease: ["alpha", 1], build: [], raw: "1.2.3-alpha.1", version: "1.2.3-alpha.1", + }); + expect(parse("1.0.0-0.3.7")!.prerelease).toEqual([0, 3, 7]); + expect(parse("1.0.0-beta.11")!.prerelease).toEqual(["beta", 11]); expect(parse("v2.0.0")!.version).toBe("2.0.0"); + expect(parse("1.2.3+corp.1")).toMatchObject({build: ["corp", "1"], raw: "1.2.3+corp.1", version: "1.2.3"}); expect(parse("invalid")).toBeNull(); expect(parse("")).toBeNull(); }); -test("coerce", () => { - expect(coerce("1.2.3")).toEqual({version: "1.2.3"}); - expect(coerce("v1.2.3")).toEqual({version: "1.2.3"}); - expect(coerce("v1.2")).toEqual({version: "1.2.0"}); - expect(coerce("v1")).toEqual({version: "1.0.0"}); - expect(coerce("42.6.7")).toEqual({version: "42.6.7"}); - expect(coerce("foo1.2.3bar")).toEqual({version: "1.2.3"}); - expect(coerce("version3.2")).toEqual({version: "3.2.0"}); - expect(coerce("v10")).toEqual({version: "10.0.0"}); +test("coerce, diff, and ordering", () => { + for (const [input, expected] of [ + ["1.2.3", "1.2.3"], ["v1.2.3", "1.2.3"], ["v1.2", "1.2.0"], ["v1", "1.0.0"], + ["42.6.7", "42.6.7"], ["foo1.2.3bar", "1.2.3"], ["version3.2", "3.2.0"], ["v10", "10.0.0"], + ]) expect(coerce(input)).toEqual({version: expected}); expect(coerce("no version here")).toBeNull(); expect(coerce("...")).toBeNull(); -}); - -test("diff", () => { - expect(diff("1.0.0", "2.0.0")).toBe("major"); - expect(diff("1.0.0", "1.1.0")).toBe("minor"); - expect(diff("1.0.0", "1.0.1")).toBe("patch"); - expect(diff("1.0.0", "1.0.0")).toBeNull(); - expect(diff("abc", "1.0.0")).toBeNull(); - expect(diff("1.0.0", "abc")).toBeNull(); - expect(diff("1.0.0-alpha.1", "1.0.0-alpha.2")).toBe("prerelease"); - expect(diff("1.0.0", "2.0.0-alpha")).toBe("premajor"); - expect(diff("1.0.0", "1.1.0-alpha")).toBe("preminor"); - expect(diff("1.0.0", "1.0.1-alpha")).toBe("prepatch"); - // low has prerelease, high doesn't: minor=0,patch=0 means "major" - expect(diff("1.0.0-alpha", "1.0.0")).toBe("major"); - expect(diff("0.0.0-alpha", "1.0.0")).toBe("major"); - expect(diff("1.1.0-alpha", "1.1.0")).toBe("minor"); - // argument order shouldn't matter for the type - expect(diff("2.0.0", "1.0.0")).toBe("major"); - expect(diff("1.1.0", "1.0.0")).toBe("minor"); -}); - -test("gt", () => { - expect(gt("2.0.0", "1.0.0")).toBe(true); - expect(gt("1.0.0", "2.0.0")).toBe(false); - expect(gt("1.0.0", "1.0.0")).toBe(false); - expect(gt("1.1.0", "1.0.0")).toBe(true); - expect(gt("1.0.1", "1.0.0")).toBe(true); - // release beats prerelease - expect(gt("1.0.0", "1.0.0-alpha")).toBe(true); - expect(gt("1.0.0-alpha", "1.0.0")).toBe(false); - // prerelease ordering - expect(gt("1.0.0-alpha.2", "1.0.0-alpha.1")).toBe(true); - expect(gt("1.0.0-beta", "1.0.0-alpha")).toBe(true); - // numbers sort before strings - expect(gt("1.0.0-alpha", "1.0.0-1")).toBe(true); - expect(gt("1.0.0-1", "1.0.0-alpha")).toBe(false); - // invalid input - expect(gt("abc", "1.0.0")).toBe(false); - expect(gt("1.0.0", "abc")).toBe(false); -}); - -test("satisfies caret ranges", () => { - expect(satisfies("1.5.0", "^1.2.3")).toBe(true); - expect(satisfies("1.2.3", "^1.2.3")).toBe(true); - expect(satisfies("1.9.9", "^1.2.3")).toBe(true); - expect(satisfies("2.0.0", "^1.2.3")).toBe(false); - expect(satisfies("1.2.2", "^1.2.3")).toBe(false); - // ^0.x behavior - expect(satisfies("0.2.5", "^0.2.3")).toBe(true); - expect(satisfies("0.3.0", "^0.2.3")).toBe(false); - // ^0.0.x behavior - expect(satisfies("0.0.3", "^0.0.3")).toBe(true); - expect(satisfies("0.0.4", "^0.0.3")).toBe(false); - // ^0 (no minor/patch) - expect(satisfies("0.5.0", "^0")).toBe(true); - expect(satisfies("0.0.0", "^0")).toBe(true); - expect(satisfies("1.0.0", "^0")).toBe(false); - // prerelease in caret: must share same major.minor.patch with comparator - expect(satisfies("1.2.3-beta.1", "^1.2.3-alpha.0")).toBe(true); - expect(satisfies("1.2.4-beta.1", "^1.2.3-alpha.0")).toBe(false); - expect(satisfies("1.2.5", "^1.2.3-alpha.0")).toBe(true); -}); - -test("satisfies tilde ranges", () => { - expect(satisfies("1.2.5", "~1.2.3")).toBe(true); - expect(satisfies("1.2.3", "~1.2.3")).toBe(true); - expect(satisfies("1.3.0", "~1.2.3")).toBe(false); - expect(satisfies("1.2.2", "~1.2.3")).toBe(false); - // ~1 (no minor) - expect(satisfies("1.5.0", "~1")).toBe(true); - expect(satisfies("1.0.0", "~1")).toBe(true); - expect(satisfies("2.0.0", "~1")).toBe(false); -}); - -test("satisfies hyphen ranges", () => { - expect(satisfies("1.5.0", "1.0.0 - 2.0.0")).toBe(true); - expect(satisfies("1.0.0", "1.0.0 - 2.0.0")).toBe(true); - expect(satisfies("2.0.0", "1.0.0 - 2.0.0")).toBe(true); - expect(satisfies("3.0.0", "1.0.0 - 2.0.0")).toBe(false); - expect(satisfies("0.9.9", "1.0.0 - 2.0.0")).toBe(false); -}); - -test("satisfies x-ranges", () => { - expect(satisfies("1.5.0", "1.x")).toBe(true); - expect(satisfies("1.0.0", "1.x")).toBe(true); - expect(satisfies("2.0.0", "1.x")).toBe(false); - expect(satisfies("1.5.0", "1.x.x")).toBe(true); - expect(satisfies("2.0.0", "1.x.x")).toBe(false); - // star matches everything - expect(satisfies("999.0.0", "*")).toBe(true); - expect(satisfies("0.0.0", "*")).toBe(true); -}); - -test("satisfies comparison operators", () => { - expect(satisfies("2.0.0", ">=1.5.0")).toBe(true); - expect(satisfies("1.5.0", ">=1.5.0")).toBe(true); - expect(satisfies("1.4.9", ">=1.5.0")).toBe(false); - expect(satisfies("1.0.0", ">1.0.0")).toBe(false); - expect(satisfies("1.0.1", ">1.0.0")).toBe(true); - expect(satisfies("1.0.0", "<2.0.0")).toBe(true); - expect(satisfies("2.0.0", "<2.0.0")).toBe(false); - expect(satisfies("2.0.0", "<=2.0.0")).toBe(true); - // spaces in operator - expect(satisfies("3.1.0", ">= 3.1")).toBe(true); -}); - -test("satisfies exact version", () => { - expect(satisfies("1.0.0", "1.0.0")).toBe(true); - expect(satisfies("1.0.1", "1.0.0")).toBe(false); -}); - -test("satisfies OR groups", () => { - expect(satisfies("2.0.0", "^1.0.0 || ^2.0.0")).toBe(true); - expect(satisfies("1.5.0", "^1.0.0 || ^2.0.0")).toBe(true); - expect(satisfies("3.0.0", "^1.0.0 || ^2.0.0")).toBe(false); -}); - -test("satisfies AND groups", () => { - expect(satisfies("1.0.0", ">=1.0.0 <2.0.0")).toBe(true); - expect(satisfies("1.9.9", ">=1.0.0 <2.0.0")).toBe(true); - expect(satisfies("2.0.0", ">=1.0.0 <2.0.0")).toBe(false); - expect(satisfies("0.9.9", ">=1.0.0 <2.0.0")).toBe(false); -}); -test("satisfies prerelease versions", () => { - expect(satisfies("1.0.0-alpha.2", ">=1.0.0-alpha.1")).toBe(true); - expect(satisfies("1.0.0-alpha.1", ">=1.0.0-alpha.1")).toBe(true); - // prerelease on different major.minor.patch tuple excluded - expect(satisfies("2.0.0-alpha.1", ">=1.0.0")).toBe(false); -}); - -test("satisfies bare partials", () => { - expect(satisfies("1.2.5", "1.2")).toBe(true); - expect(satisfies("1.2.0", "1.2")).toBe(true); - expect(satisfies("1.3.0", "1.2")).toBe(false); -}); - -test("satisfies invalid input", () => { - expect(satisfies("abc", "^1.0.0")).toBe(false); - expect(satisfies("1.0.0", "not valid!!")).toBe(false); -}); - -test("validRange", () => { - expect(validRange("^1.0.0")).toBe("^1.0.0"); - expect(validRange(">=1.0.0 <2.0.0")).toBe(">=1.0.0 <2.0.0"); - expect(validRange("1.0.0 - 2.0.0")).toBe("1.0.0 - 2.0.0"); - expect(validRange("*")).toBe("*"); + for (const [left, right, expected] of [ + ["1.0.0", "2.0.0", "major"], ["1.0.0", "1.1.0", "minor"], ["1.0.0", "1.0.1", "patch"], + ["1.0.0", "1.0.0", null], ["abc", "1.0.0", null], ["1.0.0", "abc", null], + ["1.0.0-alpha.1", "1.0.0-alpha.2", "prerelease"], ["1.0.0", "2.0.0-alpha", "premajor"], + ["1.0.0", "1.1.0-alpha", "preminor"], ["1.0.0", "1.0.1-alpha", "prepatch"], + ["1.0.0-alpha", "1.0.0", "major"], ["0.0.0-alpha", "1.0.0", "major"], + ["1.1.0-alpha", "1.1.0", "minor"], ["2.0.0", "1.0.0", "major"], ["1.1.0", "1.0.0", "minor"], + ] as Array<[string, string, string | null]>) expect(diff(left, right), `${left}, ${right}`).toBe(expected); + + for (const [left, right, expected] of [ + ["2.0.0", "1.0.0", true], ["1.0.0", "2.0.0", false], ["1.0.0", "1.0.0", false], + ["1.1.0", "1.0.0", true], ["1.0.1", "1.0.0", true], ["1.0.0", "1.0.0-alpha", true], + ["1.0.0-alpha", "1.0.0", false], ["1.0.0-alpha.2", "1.0.0-alpha.1", true], + ["1.0.0-beta", "1.0.0-alpha", true], ["1.0.0-alpha", "1.0.0-1", true], + ["1.0.0-1", "1.0.0-alpha", false], ["abc", "1.0.0", false], ["1.0.0", "abc", false], + ] as Array<[string, string, boolean]>) expect(gt(left, right), `${left}, ${right}`).toBe(expected); +}); + +test("ranges", () => { + const cases: Array<[string, string, boolean]> = [ + ["1.5.0", "^1.2.3", true], ["1.2.3", "^1.2.3", true], ["1.9.9", "^1.2.3", true], + ["2.0.0", "^1.2.3", false], ["1.2.2", "^1.2.3", false], ["0.2.5", "^0.2.3", true], + ["0.3.0", "^0.2.3", false], ["0.0.3", "^0.0.3", true], ["0.0.4", "^0.0.3", false], + ["0.5.0", "^0", true], ["0.0.0", "^0", true], ["1.0.0", "^0", false], + ["1.2.3-beta.1", "^1.2.3-alpha.0", true], ["1.2.4-beta.1", "^1.2.3-alpha.0", false], + ["1.2.5", "^1.2.3-alpha.0", true], ["1.2.5", "~1.2.3", true], ["1.2.3", "~1.2.3", true], + ["1.3.0", "~1.2.3", false], ["1.2.2", "~1.2.3", false], ["1.5.0", "~1", true], + ["1.0.0", "~1", true], ["2.0.0", "~1", false], ["1.5.0", "1.0.0 - 2.0.0", true], + ["1.0.0", "1.0.0 - 2.0.0", true], ["2.0.0", "1.0.0 - 2.0.0", true], + ["3.0.0", "1.0.0 - 2.0.0", false], ["0.9.9", "1.0.0 - 2.0.0", false], + ["1.5.0", "1.x", true], ["1.0.0", "1.x", true], ["2.0.0", "1.x", false], + ["1.5.0", "1.x.x", true], ["2.0.0", "1.x.x", false], ["999.0.0", "*", true], ["0.0.0", "*", true], + ["1.2.3-alpha", "*", false], ["1.2.3-alpha", "x", false], ["1.2.3-alpha", "*.*.*", false], + ["1.2.3-alpha", "", false], ["1.2.3-alpha", "|| >2.0.0", false], + ["1.2.3-alpha", ">2.0.0 ||", false], ["1.2.3-alpha", "|| >=1.2.3-alpha", true], + ["2.0.0", ">=1.5.0", true], ["1.5.0", ">=1.5.0", true], ["1.4.9", ">=1.5.0", false], + ["1.0.0", ">1.0.0", false], ["1.0.1", ">1.0.0", true], ["1.0.0", "<2.0.0", true], + ["2.0.0", "<2.0.0", false], ["2.0.0", "<=2.0.0", true], ["3.1.0", ">= 3.1", true], + ["1.0.0", "1.0.0", true], ["1.0.1", "1.0.0", false], ["2.0.0", "^1.0.0 || ^2.0.0", true], + ["1.5.0", "^1.0.0 || ^2.0.0", true], ["3.0.0", "^1.0.0 || ^2.0.0", false], + ["1.0.0", ">=1.0.0 <2.0.0", true], ["1.9.9", ">=1.0.0 <2.0.0", true], + ["2.0.0", ">=1.0.0 <2.0.0", false], ["0.9.9", ">=1.0.0 <2.0.0", false], + ["1.0.0-alpha.2", ">=1.0.0-alpha.1", true], ["1.0.0-alpha.1", ">=1.0.0-alpha.1", true], + ["2.0.0-alpha.1", ">=1.0.0", false], ["1.2.5", "1.2", true], ["1.2.0", "1.2", true], + ["1.3.0", "1.2", false], ["abc", "^1.0.0", false], ["1.0.0", "not valid!!", false], + ["1.2.0", "1.2.x", true], ["1.2.5", "1.2.x", true], ["1.3.0", "1.2.x", false], + ["1.1.9", "1.2.x", false], ["1.5.0", "1", true], ["1.0.0", "1", true], + ["1.99.99", "1", true], ["2.0.0", "1", false], ["0.9.9", "1", false], ["5.0.0", "1", false], + ["2.3.9", "1.2.3 - 2.3", true], ["2.4.0", "1.2.3 - 2.3", false], + ["2.99.99", "1.2.3 - 2", true], ["3.0.0", "1.2.3 - 2", false], + ["1.0.0", "^1.x", true], ["1.9.9", "^1.x", true], ["2.0.0", "^1.x", false], + ["1.5.0", "^1.x.x", true], ["2.0.0", "^1.x.x", false], ["1.2.0", "^1.2.x", true], + ["1.3.0", "^1.2.x", true], ["1.9.9", "^1.2.x", true], ["2.0.0", "^1.2.x", false], + ["1.1.9", "^1.2.x", false], ["0.9.9", "^0.x", true], ["1.0.0", "^0.x", false], + ["1.9.9", "~1.x", true], ["2.0.0", "~1.x", false], ["1.2.0", "~1.2.x", true], + ["1.2.9", "~1.2.x", true], ["1.3.0", "~1.2.x", false], ["1.5.0", ">=1.2.x", true], + ["1.2.0", ">=1.2.x", true], ["1.1.0", ">=1.2.x", false], ["1.5.0", ">=1.2.x <2.0.0", true], + ["2.0.0", ">=1.2.x <2.0.0", false], ["1.1.0", ">=1.2.x <2.0.0", false], + ["0.9.9", ">=1.x", false], ["3.0.0", ">=1.x", true], ["1.2.9", "<=1.2.x", true], + ["1.3.0", "<=1.2.x", false], ["1.9.9", ">1.x", false], ["2.0.0", ">1.x", true], + ["1.2.4", "~>1.2.3", true], ["2.3.9", "1.2.3 - 2.3.x", true], + ]; + for (const [version, range, expected] of cases) expect(satisfies(version, range)).toBe(expected); + + for (const range of [ + "^1.0.0", ">=1.0.0 <2.0.0", "1.0.0 - 2.0.0", "*", "~>1.2.3", "1.2.3+build", + "^1.2.3+build", "*.*.*", "1.2.3 - 2.3.x", "1.2.x", "^1.x", "^1.x.x", "^1.2.x", "^0.x", + "~1.x", "~1.2.x", ">=1.2.x", ">=1.2.x <2.0.0", + ]) expect(validRange(range)).toBe(range); + expect(validRange("1.*.3")).toBeNull(); expect(validRange("not valid!!")).toBeNull(); -}); - -test("satisfies 1.2.x pattern", () => { - expect(satisfies("1.2.0", "1.2.x")).toBe(true); - expect(satisfies("1.2.5", "1.2.x")).toBe(true); - expect(satisfies("1.3.0", "1.2.x")).toBe(false); - expect(satisfies("1.1.9", "1.2.x")).toBe(false); - expect(validRange("1.2.x")).toBe("1.2.x"); -}); - -test("satisfies bare single number partial", () => { - expect(satisfies("1.5.0", "1")).toBe(true); - expect(satisfies("1.0.0", "1")).toBe(true); - expect(satisfies("1.99.99", "1")).toBe(true); - expect(satisfies("2.0.0", "1")).toBe(false); - expect(satisfies("0.9.9", "1")).toBe(false); - expect(satisfies("5.0.0", "1")).toBe(false); -}); - -test("validRange non-string input", () => { expect(validRange(undefined as any)).toBeNull(); expect(validRange(null as any)).toBeNull(); }); -test("satisfies partial hyphen ranges", () => { - // 1.2.3 - 2.3 := >=1.2.3 <2.4.0-0 - expect(satisfies("2.3.9", "1.2.3 - 2.3")).toBe(true); - expect(satisfies("2.4.0", "1.2.3 - 2.3")).toBe(false); - // 1.2.3 - 2 := >=1.2.3 <3.0.0-0 - expect(satisfies("2.99.99", "1.2.3 - 2")).toBe(true); - expect(satisfies("3.0.0", "1.2.3 - 2")).toBe(false); -}); - -test("satisfies caret with trailing wildcard", () => { - // ^1.x := >=1.0.0 <2.0.0-0 - expect(validRange("^1.x")).toBe("^1.x"); - expect(satisfies("1.0.0", "^1.x")).toBe(true); - expect(satisfies("1.9.9", "^1.x")).toBe(true); - expect(satisfies("2.0.0", "^1.x")).toBe(false); - // ^1.x.x := >=1.0.0 <2.0.0-0 - expect(validRange("^1.x.x")).toBe("^1.x.x"); - expect(satisfies("1.5.0", "^1.x.x")).toBe(true); - expect(satisfies("2.0.0", "^1.x.x")).toBe(false); - // ^1.2.x := >=1.2.0 <2.0.0-0 (caret keeps the major-level upper bound) - expect(validRange("^1.2.x")).toBe("^1.2.x"); - expect(satisfies("1.2.0", "^1.2.x")).toBe(true); - expect(satisfies("1.3.0", "^1.2.x")).toBe(true); - expect(satisfies("1.9.9", "^1.2.x")).toBe(true); - expect(satisfies("2.0.0", "^1.2.x")).toBe(false); - expect(satisfies("1.1.9", "^1.2.x")).toBe(false); - // ^0.x := >=0.0.0 <1.0.0-0 - expect(validRange("^0.x")).toBe("^0.x"); - expect(satisfies("0.9.9", "^0.x")).toBe(true); - expect(satisfies("1.0.0", "^0.x")).toBe(false); -}); - -test("satisfies tilde with trailing wildcard", () => { - // ~1.x := >=1.0.0 <2.0.0-0 - expect(validRange("~1.x")).toBe("~1.x"); - expect(satisfies("1.9.9", "~1.x")).toBe(true); - expect(satisfies("2.0.0", "~1.x")).toBe(false); - // ~1.2.x := >=1.2.0 <1.3.0-0 (tilde keeps the minor-level upper bound) - expect(validRange("~1.2.x")).toBe("~1.2.x"); - expect(satisfies("1.2.0", "~1.2.x")).toBe(true); - expect(satisfies("1.2.9", "~1.2.x")).toBe(true); - expect(satisfies("1.3.0", "~1.2.x")).toBe(false); -}); - -test("parsePep440 normalizes every release form", () => { - expect(parsePep440("26.3")).toMatchObject({epoch: 0, release: [26, 3], pre: null, post: null, dev: null, local: null}); - expect(parsePep440("1!1.0")).toMatchObject({epoch: 1, release: [1, 0]}); - expect(parsePep440("2.32.0.20250602")).toMatchObject({release: [2, 32, 0, 20250602]}); - expect(parsePep440("17.04.0")).toMatchObject({release: [17, 4, 0]}); // zero-padded segments are numbers - expect(parsePep440("2.9.0.post0")).toMatchObject({post: 0}); - expect(parsePep440("1.0-1")).toMatchObject({release: [1, 0], post: 1}); // implicit post syntax - expect(parsePep440("1.1.0.dev1")).toMatchObject({dev: 1}); - expect(parsePep440("0.0.1a19")).toMatchObject({pre: ["a", 19]}); - expect(parsePep440("1.0.0+ubuntu.1")).toMatchObject({local: ["ubuntu", 1]}); - expect(parsePep440("v1.2.3")).toMatchObject({release: [1, 2, 3]}); +test("PEP 440 parsing and ordering", () => { + for (const [version, expected] of [ + ["26.3", {epoch: 0, release: [26, 3], pre: null, post: null, dev: null, local: null}], + ["1!1.0", {epoch: 1, release: [1, 0]}], ["2.32.0.20250602", {release: [2, 32, 0, 20250602]}], + ["17.04.0", {release: [17, 4, 0]}], ["2.9.0.post0", {post: 0}], + ["1.0-1", {release: [1, 0], post: 1}], ["1.1.0.dev1", {dev: 1}], ["0.0.1a19", {pre: ["a", 19]}], + ["1.0.0+ubuntu.1", {local: ["ubuntu", 1]}], ["v1.2.3", {release: [1, 2, 3]}], + ] as Array<[string, Record]>) expect(parsePep440(version)).toMatchObject(expected); expect(parsePep440("not_a_version")).toBeNull(); for (const [input, letter] of [["1.0alpha", "a"], ["1.0.beta2", "b"], ["1.0c1", "rc"], ["1.0-pre", "rc"], ["1.0_preview3", "rc"], ["1.0RC4", "rc"]] as const) { expect(parsePep440(input)!.pre![0]).toBe(letter); } expect(parsePep440("1.0alpha")!.pre![1]).toBe(0); -}); - -test("comparePep440 orders epoch, release, pre, post, dev and local", () => { - const cmp = (a: string, b: string) => Math.sign(comparePep440(parsePep440(a)!, parsePep440(b)!)); - expect(cmp("1.0", "1.0.0")).toBe(0); // trailing zeros are insignificant - expect(cmp("1!1.0", "2.0")).toBe(1); // an epoch outranks the release segment - expect(cmp("26.3", "26.2")).toBe(1); - expect(cmp("2.32.4.20250611", "2.32.0.20250602")).toBe(1); - expect(cmp("1.0", "1.0rc1")).toBe(1); - expect(cmp("1.0rc1", "1.0b1")).toBe(1); - expect(cmp("1.0b1", "1.0a1")).toBe(1); - expect(cmp("1.0a10", "1.0a9")).toBe(1); - expect(cmp("1.0.post1", "1.0")).toBe(1); - expect(cmp("1.0", "1.0.dev1")).toBe(1); - expect(cmp("1.0a1", "1.0.dev1")).toBe(1); // a bare dev release precedes every pre-release - expect(cmp("1.0.post1.dev1", "1.0")).toBe(1); // but a post-dev release still follows the release - expect(cmp("1.0+local", "1.0")).toBe(1); - expect(cmp("1.0+1", "1.0+abc")).toBe(1); // numeric local segments sort after alphanumeric ones -}); -test("diffPep440 buckets by release level with a pre prefix for unstable candidates", () => { - const d = (a: string, b: string) => diffPep440(parsePep440(a)!, parsePep440(b)!); - expect(d("1.0", "1.0.0")).toBe(null); - expect(d("1.0", "6.0")).toBe("major"); - expect(d("3.4.0.20240423", "3.5.0.20250801")).toBe("minor"); - expect(d("2.32.0.20240622", "2.32.4.20250611")).toBe("patch"); - // renovate buckets everything below the minor as a patch, so a fourth segment has no level of its own - expect(d("3.4.0.20240103", "3.4.0.20240423")).toBe("patch"); - expect(d("0.0.1a15", "0.0.1a19")).toBe("prerelease"); - expect(d("1.0", "2.0b1")).toBe("premajor"); - expect(d("1.0", "1.1.0.dev1")).toBe("preminor"); - expect(d("1.0rc1", "1.0")).toBe("patch"); -}); - -test("semverVersioning reads the prerelease off a range's comparator", () => { + const compareVersions = (left: string, right: string) => Math.sign(comparePep440(parsePep440(left)!, parsePep440(right)!)); + for (const [left, right, expected] of [ + ["1.0", "1.0.0", 0], ["1!1.0", "2.0", 1], ["26.3", "26.2", 1], + ["2.32.4.20250611", "2.32.0.20250602", 1], ["1.0", "1.0rc1", 1], ["1.0rc1", "1.0b1", 1], + ["1.0b1", "1.0a1", 1], ["1.0a10", "1.0a9", 1], ["1.0.post1", "1.0", 1], + ["1.0", "1.0.dev1", 1], ["1.0a1", "1.0.dev1", 1], ["1.0.post1.dev1", "1.0", 1], + ["1.0+local", "1.0", 1], ["1.0+1", "1.0+abc", 1], + ] as Array<[string, string, number]>) expect(compareVersions(left, right)).toBe(expected); + + const versionDiff = (left: string, right: string) => diffPep440(parsePep440(left)!, parsePep440(right)!); + for (const [left, right, expected] of [ + ["1.0", "1.0.0", null], ["1.0", "6.0", "major"], ["3.4.0.20240423", "3.5.0.20250801", "minor"], + ["2.32.0.20240622", "2.32.4.20250611", "patch"], ["3.4.0.20240103", "3.4.0.20240423", "patch"], + ["0.0.1a15", "0.0.1a19", "prerelease"], ["1.0", "2.0b1", "premajor"], + ["1.0", "1.1.0.dev1", "preminor"], ["1.0rc1", "1.0", "patch"], + ] as Array<[string, string, string | null]>) expect(versionDiff(left, right)).toBe(expected); +}); + +test("versioning adapters", () => { expect(["^1.0.0-alpha", ">=2.0.0-rc.1"].every(range => semverVersioning.isRangePrerelease(range))).toBe(true); expect(["^1.0.0", "~2.0.0"].some(range => semverVersioning.isRangePrerelease(range))).toBe(false); -}); - -test("pep440Versioning classifies prereleases the semver rules miss", () => { for (const version of ["2.0.0b1", "1.0rc1", "0.0.1a19", "1.1.0.dev1"]) { expect(pep440Versioning.isRangePrerelease(version)).toBe(true); expect(pep440Versioning.isPrerelease(pep440Versioning.parse(version)!)).toBe(true); @@ -326,29 +151,6 @@ test("pep440Versioning classifies prereleases the semver rules miss", () => { expect(pep440Versioning.isRangePrerelease(version)).toBe(false); } expect(pep440Versioning.parseRange(">=2.28.0")!.release).toEqual([2, 28, 0]); - // --pin takes a semver range, matched against the first three release segments expect(pep440Versioning.satisfiesRange(pep440Versioning.parse("6.0")!, "^6.0.0")).toBe(true); expect(pep440Versioning.satisfiesRange(pep440Versioning.parse("2.32.4.20250611")!, "^2.33.0")).toBe(false); }); - -test("satisfies operator-prefixed x-ranges", () => { - // >=1.2.x := >=1.2.0 - expect(validRange(">=1.2.x")).toBe(">=1.2.x"); - expect(satisfies("1.5.0", ">=1.2.x")).toBe(true); - expect(satisfies("1.2.0", ">=1.2.x")).toBe(true); - expect(satisfies("1.1.0", ">=1.2.x")).toBe(false); - // >=1.2.x <2.0.0 := >=1.2.0 <2.0.0 - expect(validRange(">=1.2.x <2.0.0")).toBe(">=1.2.x <2.0.0"); - expect(satisfies("1.5.0", ">=1.2.x <2.0.0")).toBe(true); - expect(satisfies("2.0.0", ">=1.2.x <2.0.0")).toBe(false); - expect(satisfies("1.1.0", ">=1.2.x <2.0.0")).toBe(false); - // >=1.x := >=1.0.0 - expect(satisfies("0.9.9", ">=1.x")).toBe(false); - expect(satisfies("3.0.0", ">=1.x")).toBe(true); - // <=1.2.x := <1.3.0-0 (any 1.2.z passes) - expect(satisfies("1.2.9", "<=1.2.x")).toBe(true); - expect(satisfies("1.3.0", "<=1.2.x")).toBe(false); - // >1.x := >=2.0.0 (greater than the whole 1.x line) - expect(satisfies("1.9.9", ">1.x")).toBe(false); - expect(satisfies("2.0.0", ">1.x")).toBe(true); -}); diff --git a/utils/semver.ts b/utils/semver.ts index 85c4b74..a8ea2e5 100644 --- a/utils/semver.ts +++ b/utils/semver.ts @@ -5,10 +5,15 @@ export type SemVer = { minor: number; patch: number; prerelease: ReadonlyArray; + build: ReadonlyArray; + raw: string; version: string; }; -const semverRe = /^v?(\d+)\.(\d+)\.(\d+)(?:-([a-zA-Z0-9_-]+(?:\.[a-zA-Z0-9_-]+)*))?(?:\+[a-zA-Z0-9._-]+)?$/; +const numericIdentifier = "0|[1-9]\\d*"; +const numericIdentifierRe = /^(?:0|[1-9]\d*)$/; +const prereleaseIdentifier = "0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*"; +const semverRe = new RegExp(`^v?(${numericIdentifier})\\.(${numericIdentifier})\\.(${numericIdentifier})(?:-((?:${prereleaseIdentifier})(?:\\.(?:${prereleaseIdentifier}))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$`); const parseCache = new Map(); @@ -20,20 +25,26 @@ function parseVersion(v: string): SemVer | null { const major = Number(m[1]); const minor = Number(m[2]); const patch = Number(m[3]); + if (!Number.isSafeInteger(major) || !Number.isSafeInteger(minor) || !Number.isSafeInteger(patch)) return null; const prerelease: Array = m[4] ? - m[4].split(".").map(p => /^\d+$/.test(p) ? Number(p) : p) : + m[4].split(".").map(part => /^\d+$/.test(part) && Number(part) < Number.MAX_SAFE_INTEGER ? Number(part) : part) : []; + const build = m[5]?.split(".") ?? []; const version = `${major}.${minor}.${patch}${prerelease.length ? `-${prerelease.join(".")}` : ""}`; - return {major, minor, patch, prerelease, version}; + return {major, minor, patch, prerelease, build, raw: v, version}; }); } function compareIdentifiers(a: string | number, b: string | number): number { - const aNum = typeof a === "number"; - const bNum = typeof b === "number"; - if (aNum && bNum) return a - b; - if (aNum) return -1; // numbers sort before strings - if (bNum) return 1; + const aNumeric = typeof a === "number" || /^\d+$/.test(a); + const bNumeric = typeof b === "number" || /^\d+$/.test(b); + if (aNumeric && bNumeric) { + const aString = String(a); + const bString = String(b); + return aString.length - bString.length || (aString < bString ? -1 : aString > bString ? 1 : 0); + } + if (aNumeric) return -1; + if (bNumeric) return 1; return a < b ? -1 : a > b ? 1 : 0; } @@ -41,18 +52,14 @@ function compareMain(a: SemVer, b: SemVer): number { return (a.major - b.major) || (a.minor - b.minor) || (a.patch - b.patch); } -// Compare pre-parsed versions, letting hot loops skip the parse-cache lookups. function compareParsed(a: SemVer, b: SemVer): number { const main = compareMain(a, b); if (main !== 0) return main; const aHasPre = a.prerelease.length > 0; const bHasPre = b.prerelease.length > 0; - // no prerelease on either => equal if (!aHasPre && !bHasPre) return 0; - // prerelease has lower precedence than release if (aHasPre && !bHasPre) return -1; if (!aHasPre && bHasPre) return 1; - // both have prerelease const len = Math.max(a.prerelease.length, b.prerelease.length); for (let i = 0; i < len; i++) { if (a.prerelease[i] === undefined) return -1; @@ -89,7 +96,6 @@ export function diff(v1: string, v2: string): string | null { return diffParsed(a, b); } -// Lets hot loops pass already-parsed inputs and skip the parseVersion cache lookup. function diffParsed(a: SemVer, b: SemVer): string | null { if (a.version === b.version) return null; @@ -99,7 +105,6 @@ function diffParsed(a: SemVer, b: SemVer): string | null { const highHasPre = highVersion.prerelease.length > 0; const lowHasPre = lowVersion.prerelease.length > 0; - // Special case: going from prerelease to release if (lowHasPre && !highHasPre) { if (!lowVersion.patch && !lowVersion.minor) return "major"; if (compareMain(lowVersion, highVersion) === 0) { @@ -125,23 +130,52 @@ export function gt(v1: string, v2: string): boolean { return (compare(v1, v2) ?? -1) > 0; } -// --- Range parsing --- - type Comparator = { - op: string; // >=, <=, >, <, = (empty means =) + op: string; semver: SemVer; }; -function parseComparator(comp: string): Comparator | null { - const m = /^(>=|<=|>|<|=)?\s*v?(\d+)(?:\.(\d+))?(?:\.(\d+))?((?:-[a-zA-Z0-9_-]+(?:\.[a-zA-Z0-9_-]+)*)?)$/.exec(comp.trim()); - if (!m) return null; - const major = m[2]; - const minor = m[3] ?? "0"; - const patch = m[4] ?? "0"; - const pre = m[5] || ""; - const sv = parseVersion(`${major}.${minor}.${patch}${pre}`); - if (!sv) return null; - return {op: m[1] || "=", semver: sv}; +type PartialVersion = { + major: number | null; + minor: number | null; + patch: number | null; + suffix: string; +}; + +function parsePartial(value: string): PartialVersion | null { + const match = /^v?([^.+-]+(?:\.[^.+-]+){0,2})(-[0-9a-zA-Z.-]+)?(\+[0-9a-zA-Z.-]+)?$/.exec(value); + if (!match) return null; + const parts = match[1].split("."); + const parsed: Array = []; + let wildcard = false; + for (const part of parts) { + if (/^[xX*]$/.test(part)) { + wildcard = true; + parsed.push(null); + } else { + if (wildcard || !numericIdentifierRe.test(part)) return null; + const number = Number(part); + if (!Number.isSafeInteger(number)) return null; + parsed.push(number); + } + } + while (parsed.length < 3) parsed.push(null); + if ((match[2] || match[3]) && parsed.some(part => part === null)) return null; + if (match[2] || match[3]) { + const complete = `${parsed.join(".")}${match[2] ?? ""}${match[3] ?? ""}`; + if (!parseVersion(complete)) return null; + } + return {major: parsed[0], minor: parsed[1], patch: parsed[2], suffix: `${match[2] ?? ""}${match[3] ?? ""}`}; +} + +function comparator(op: string, major: number, minor: number, patch: number, suffix = ""): Comparator | null { + const semver = parseVersion(`${major}.${minor}.${patch}${suffix}`); + return semver ? {op, semver} : null; +} + +function comparators(...values: Array): Array | null { + const result = values.filter((value): value is Comparator => value !== null); + return result.length === values.length ? result : null; } function testComparator(v: SemVer, comp: Comparator): boolean { @@ -152,194 +186,93 @@ function testComparator(v: SemVer, comp: Comparator): boolean { case ">": return cmp > 0; case "<": return cmp < 0; case "=": return cmp === 0; - default: return cmp === 0; + default: return false; } } -// Returns upper bound with -0 appended for exclusive upper bounds -function upperBound(major: number, minor: number, patch: number): string { - return `${major}.${minor}.${patch}-0`; -} - -// ~1.2.3 := >=1.2.3 <1.3.0-0 ^1.2.3 := >=1.2.3 <2.0.0-0 -// ~1.2 := >=1.2.0 <1.3.0-0 ^0.2.3 := >=0.2.3 <0.3.0-0 -// ~1 := >=1.0.0 <2.0.0-0 ^0.0.3 := >=0.0.3 <0.0.4-0 -// ^0.0 := >=0.0.0 <0.1.0-0 -// ^0 := >=0.0.0 <1.0.0-0 -// Trailing wildcard segments (e.g. ~1.x, ^1.2.x) are consumed and treated as omitted. -function expandTildeCaret(range: string): string { - return range.replace(/([~^])\s*v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.[xX*])*((?:-[a-zA-Z0-9._-]+)?)/g, (_, op, rawMajor, rawMinor, rawPatch, rawPre) => { - const major = Number(rawMajor); - const minor = rawMinor !== undefined ? Number(rawMinor) : 0; - const patch = rawPatch !== undefined ? Number(rawPatch) : 0; - // The prerelease is kept only when the segment it belongs to was given: ~ needs a minor, ^ a patch. - const pre = (op === "~" ? rawMinor : rawPatch) !== undefined ? rawPre : ""; - let upper: string; - if (rawMinor === undefined) upper = upperBound(major + 1, 0, 0); - else if (op === "~") upper = upperBound(major, minor + 1, 0); - else if (major !== 0) upper = upperBound(major + 1, 0, 0); - else if (rawPatch !== undefined && minor === 0) upper = upperBound(0, 0, patch + 1); - else upper = upperBound(0, minor + 1, 0); - return `>=${major}.${minor}.${patch}${pre} <${upper}`; - }); -} - -function expandHyphen(range: string): string { - // A - B := >=A <=B - // 1.2.3 - 2.3.4 := >=1.2.3 <=2.3.4 - // 1.2 - 2.3.4 := >=1.2.0 <=2.3.4 - // 1.2.3 - 2.3 := >=1.2.3 <2.4.0-0 - // 1.2.3 - 2 := >=1.2.3 <3.0.0-0 - return range.replace(/v?(\d+)(?:\.(\d+))?(?:\.(\d+))?((?:-[a-zA-Z0-9._-]+)?)\s+-\s+v?(\d+)(?:\.(\d+))?(?:\.(\d+))?((?:-[a-zA-Z0-9._-]+)?)/g, - (_, aM, am, ap, aPre, bM, bm, bp, bPre) => { - const fromM = Number(aM); - const fromm = am !== undefined ? Number(am) : 0; - const fromp = ap !== undefined ? Number(ap) : 0; - const fromPre = aPre || ""; - const toM = Number(bM); - - let upper: string; - if (bp !== undefined) { - const tom = Number(bm); - const top = Number(bp); - const toPre = bPre || ""; - upper = `<=${toM}.${tom}.${top}${toPre}`; - } else if (bm !== undefined) { - const tom = Number(bm); - upper = `<${upperBound(toM, tom + 1, 0)}`; - } else { - upper = `<${upperBound(toM + 1, 0, 0)}`; - } - return `>=${fromM}.${fromm}.${fromp}${fromPre} ${upper}`; - }); +function upperComparator(major: number, minor: number, patch: number): Comparator | null { + return comparator("<", major, minor, patch, "-0"); } -// Expands an x-range component (e.g. 1.2.x or 1.x), honoring a leading comparison operator the way -// node-semver does: a bare/`=` x-range becomes a `>=lo =1.2.x := >=1.2.0, >1.2.x := >=1.3.0, <=1.2.x := <1.3.0-0). -function expandXRangeComparator(op: string | undefined, major: number, minor: number, wildMinor: boolean): string { - if (!op || op === "=") { - return wildMinor ? - `>=${major}.0.0 <${upperBound(major + 1, 0, 0)}` : - `>=${major}.${minor}.0 <${upperBound(major, minor + 1, 0)}`; +function partialBounds(partial: PartialVersion, op: string): Array | null { + if (partial.major === null) return partial.minor === null && partial.patch === null ? [] : null; + const major = partial.major; + const minor = partial.minor ?? 0; + const patch = partial.patch ?? 0; + if (op === "^" || op === "~") { + const lower = comparator(">=", major, minor, patch, partial.suffix); + if (partial.minor === null) return comparators(lower, upperComparator(major + 1, 0, 0)); + if (op === "~") return comparators(lower, upperComparator(major, minor + 1, 0)); + if (major !== 0) return comparators(lower, upperComparator(major + 1, 0, 0)); + if (partial.patch !== null && minor === 0) return comparators(lower, upperComparator(0, 0, patch + 1)); + return comparators(lower, upperComparator(0, minor + 1, 0)); } - if (op === ">") { - // >1 := >=2.0.0, >1.2 := >=1.3.0 - return wildMinor ? `>=${major + 1}.0.0` : `>=${major}.${minor + 1}.0`; - } - if (op === "<=") { - // <=1.x := <2.0.0-0, <=1.2.x := <1.3.0-0 (any matching patch should pass) - return wildMinor ? `<${upperBound(major + 1, 0, 0)}` : `<${upperBound(major, minor + 1, 0)}`; - } - if (op === "<") { - return wildMinor ? `<${major}.0.0-0` : `<${major}.${minor}.0-0`; + if (partial.minor !== null && partial.patch !== null) return comparators(comparator(op || "=", major, minor, patch, partial.suffix)); + if (!op || op === "=") { + return partial.minor === null ? + comparators(comparator(">=", major, 0, 0), upperComparator(major + 1, 0, 0)) : + comparators(comparator(">=", major, minor, 0), upperComparator(major, minor + 1, 0)); } - // >= - return wildMinor ? `>=${major}.0.0` : `>=${major}.${minor}.0`; + if (op === ">") return partial.minor === null ? comparators(comparator(">=", major + 1, 0, 0)) : comparators(comparator(">=", major, minor + 1, 0)); + if (op === "<=") return partial.minor === null ? comparators(upperComparator(major + 1, 0, 0)) : comparators(upperComparator(major, minor + 1, 0)); + if (op === "<") return partial.minor === null ? comparators(upperComparator(major, 0, 0)) : comparators(upperComparator(major, minor, 0)); + return comparators(comparator(">=", major, minor, 0)); } -function expandXRanges(range: string): string { - // *, x, X -> >=0.0.0 - // 1.x, 1.*, 1 -> >=1.0.0 <2.0.0-0 - // 1.2.x, 1.2.*, 1.2 -> >=1.2.0 <1.3.0-0 - - // Handle standalone wildcard - if (/^\s*[*xX]\s*$/.test(range)) { - return ">=0.0.0"; +function parseHyphen(fromValue: string, toValue: string): Array | null { + const from = parsePartial(fromValue); + const to = parsePartial(toValue); + if (!from || !to) return null; + const result: Array = []; + if (from.major !== null) { + const lower = comparator(">=", from.major, from.minor ?? 0, from.patch ?? 0, from.suffix); + if (!lower) return null; + result.push(lower); } + if (to.major === null) return result; + const upper = to.minor === null ? upperComparator(to.major + 1, 0, 0) : + to.patch === null ? upperComparator(to.major, to.minor + 1, 0) : + comparator("<=", to.major, to.minor, to.patch, to.suffix); + if (!upper) return null; + result.push(upper); + return result; +} - // Handle patterns like 1.2.x, 1.2.* (before the 2-part rule below, which would otherwise mis-match these) - // wildMinor=false: minor is fixed (1.2.x), so the implied range spans one minor. true: minor is wild (1.x). - range = range.replace(/(>=|<=|>|<|=)?\s*v?(\d+)\.(\d+)\.[xX*]/g, (_, op, major, minor) => - expandXRangeComparator(op, Number(major), Number(minor), false)); - - // Handle patterns like 1.x, 1.*, 1.X, 1.x.x etc. - range = range.replace(/(>=|<=|>|<|=)?\s*v?(\d+)\.[xX*](?:\.[xX*])?/g, (_, op, major) => - expandXRangeComparator(op, Number(major), 0, true)); - - // Handle bare partials "1.2" and "1", honoring a leading comparison operator the same way - // the x-range passes above do (e.g. >1.2 := >=1.3.0, <=1 := <2.0.0-0). A bare/`=` partial - // collapses to the `>=lo =|<=|>|<|=)?\s*v?(\d+)\.(\d+)(?=\s|$)/g, (_, prefix, op, major, minor) => - `${prefix}${expandXRangeComparator(op, Number(major), Number(minor), false)}`); - - range = range.replace(/(^|[\s|])(>=|<=|>|<|=)?\s*v?(\d+)(?=\s|$)/g, (_, prefix, op, major) => - `${prefix}${expandXRangeComparator(op, Number(major), 0, true)}`); - - return range; +function parseComparatorSet(group: string): Array | null { + const hyphen = /^(\S+)\s+-\s+(\S+)$/.exec(group); + if (hyphen) return parseHyphen(hyphen[1], hyphen[2]); + const normalized = group.replace(/~\s*>\s*/g, "~").replace(/(>=|<=|>|<|=|~|\^)\s+/g, "$1"); + const result: Array = []; + for (const token of normalized.split(/\s+/).filter(Boolean)) { + const match = /^(>=|<=|>|<|=|~|\^)?(.+)$/.exec(token); + const partial = match && parsePartial(match[2]); + const bounds = partial ? partialBounds(partial, match?.[1] ?? "") : null; + if (!bounds) return null; + result.push(...bounds); + } + return result; } const rangeCache = new Map> | null>(); function parseRange(range: string): Array> | null { return getOrSet(rangeCache, range, () => { - const orGroups = range.split("||").map(g => g.trim()); - const result: Array> = []; - - for (let group of orGroups) { - if (!group) { - // Empty group in || means match anything - result.push([]); - continue; - } - - // Expand in order: hyphen -> caret/tilde -> x-range - group = expandHyphen(group); - group = expandTildeCaret(group); - group = expandXRanges(group); - - // Merge operators with their following version (handle spaces like ">= 3.1"). - // Must run before the normalize pass below, else ">= 1.0.0" gets an "=" inserted. - group = group.replace(/(>=|<=|>|<|=)\s+/g, "$1"); - - // Normalize = prefix for exact versions - group = group.replace(/(^|[\s])v?(\d+\.\d+\.\d+(?:-[a-zA-Z0-9_-]+(?:\.[a-zA-Z0-9_-]+)*)?)\b/g, - (_, prefix, version) => `${prefix}=${version}`); - - const comparators: Array = []; - for (const part of group.split(/\s+/).filter(Boolean)) { - const comp = parseComparator(part); - if (!comp) return null; - comparators.push(comp); - } - - if (comparators.length === 0) return null; - result.push(comparators); - } - - return result.length ? result : null; + const groups = range.split("||").map(group => parseComparatorSet(group.trim())); + return groups.some(group => group === null) ? null : groups as Array>; }); } function testWithPrerelease(version: SemVer, comparators: Array): boolean { - // All comparators in the AND group must pass if (comparators.some(comp => !testComparator(version, comp))) return false; - - // Prerelease filtering: if version has prerelease tags, - // at least one comparator must share the same [major, minor, patch] - // and also have a prerelease tag - if (version.prerelease.length > 0) { - return comparators.some(comp => - comp.semver.prerelease.length > 0 && - comp.semver.major === version.major && - comp.semver.minor === version.minor && - comp.semver.patch === version.patch); - } - - return true; + return !version.prerelease.length || comparators.some(comp => comp.semver.prerelease.length > 0 && + comp.semver.major === version.major && comp.semver.minor === version.minor && comp.semver.patch === version.patch); } export function satisfies(version: string, range: string): boolean { const v = parseVersion(version); if (!v) return false; const parsed = parseRange(range); - if (!parsed) return false; - - for (const group of parsed) { - if (group.length === 0) return true; // empty group matches all - if (testWithPrerelease(v, group)) return true; - } - return false; + return Boolean(parsed?.some(group => testWithPrerelease(v, group))); } export function validRange(range: string): string | null { @@ -357,9 +290,6 @@ export type Pep440 = { version: string; }; -// https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions -// 1 epoch, 2 release, 3-4 pre letter/number, 5 implicit post number, 6-7 post letter/number, -// 8-9 dev marker/number, 10 local. const pep440Pattern = "v?(?:(\\d+)!)?(\\d+(?:\\.\\d+)*)(?:[-_.]?(a|b|c|rc|alpha|beta|pre|preview)[-_.]?(\\d+)?)?(?:-(\\d+)|[-_.]?(post|rev|r)[-_.]?(\\d+)?)?(?:[-_.]?(dev)[-_.]?(\\d+)?)?(?:\\+([a-z0-9]+(?:[-_.][a-z0-9]+)*))?"; const pep440Re = new RegExp(`^${pep440Pattern}$`, "i"); const pep440SearchRe = new RegExp(pep440Pattern, "i"); @@ -385,14 +315,12 @@ export function parsePep440(v: string): Pep440 | null { }); } -// A pypi range is authored as a bare version, but a comparator may still be glued to it. function parsePep440Range(range: string): Pep440 | null { return parsePep440(range) ?? parsePep440(pep440SearchRe.exec(range)?.[0] ?? ""); } const isPep440Prerelease = (v: Pep440): boolean => Boolean(v.pre || v.dev); -// Alphanumeric local segments sort before numeric ones, shorter before longer. function compareLocal(a: Array | null, b: Array | null): number { if (!a || !b) return a ? 1 : b ? -1 : 0; const len = Math.min(a.length, b.length); @@ -407,13 +335,11 @@ function compareLocal(a: Array | null, b: Array = { parse: (version: string) => T | null; - // Pulls the authored version out of a range, prerelease included. parseRange: (range: string) => T | null; compare: (a: T, b: T) => number; diff: (a: T, b: T) => string | null; @@ -467,22 +389,17 @@ export const semverVersioning: Versioning = { compare: compareParsed, diff: diffParsed, isPrerelease: parsed => parsed.prerelease.length > 0, - // can not use coerce here because it ignores prerelease tags isRangePrerelease: range => /[0-9]+\.[0-9]+\.[0-9]+-.+/.test(range), satisfiesRange: (parsed, range) => satisfies(parsed.version, range), }; -// Actions are tagged with floating majors and minors (`v3`, `v3.19`) as often as with full -// versions, and plain semver rejects both. Ported from renovate's github-actions versioning. const actionsParseCache = new Map(); function parseActionsVersion(v: string): SemVer | null { return getOrSet(actionsParseCache, v, () => { const stripped = v.trim().replace(/^v/i, ""); - // `major.minor-prerelease` (`2.2-rc.1`) normalizes onto `major.minor.0-prerelease` const parsed = parse(stripped) ?? parse(stripped.replace(/^(\d+\.\d+)(-.+)$/, "$1.0$2")); if (parsed) return parsed; - // without the guard, coerce reads a foreign tag scheme like `codeql-bundle-v2.20.3` as a version if (!/^\d/.test(stripped)) return null; return parse(coerce(stripped)?.version ?? ""); }); @@ -505,6 +422,5 @@ export const pep440Versioning: Versioning = { const parsed = parsePep440Range(range); return Boolean(parsed && isPep440Prerelease(parsed)); }, - // --pin takes a semver range, so match it against the first three release segments. satisfiesRange: ({release}, range) => satisfies(`${release[0] ?? 0}.${release[1] ?? 0}.${release[2] ?? 0}`, range), }; diff --git a/utils/toml.test.ts b/utils/toml.test.ts index 0d946f5..87d0b0a 100644 --- a/utils/toml.test.ts +++ b/utils/toml.test.ts @@ -1,185 +1,62 @@ import {readFileSync} from "node:fs"; import {parseToml} from "./toml.ts"; -test("basic string value", () => { - expect(parseToml(`key = "value"`)).toEqual({key: "value"}); -}); - -test("integer value", () => { - expect(parseToml(`port = 8080`)).toEqual({port: 8080}); -}); - -test("float value", () => { - expect(parseToml(`num = 1.5`)).toEqual({num: 1.5}); -}); - -test("boolean values", () => { - expect(parseToml(`enabled = true\ndebug = false`)).toEqual({enabled: true, debug: false}); -}); - -test("table header", () => { - expect(parseToml(`[tool]\nname = "x"`)).toEqual({tool: {name: "x"}}); -}); - -test("nested table header", () => { - expect(parseToml(`[tool.poetry]\nname = "x"`)).toEqual({tool: {poetry: {name: "x"}}}); - expect(parseToml(`[__proto__.poetry]\nname = "x"`)).toEqual({["__proto__"]: {poetry: {name: "x"}}}); +test("TOML syntax", () => { + const cases: Array<[string, string, unknown]> = [ + ["basic string", `key = "value"`, {key: "value"}], + ["integer", `port = 8080`, {port: 8080}], + ["float", `num = 1.5`, {num: 1.5}], + ["booleans", `enabled = true\ndebug = false`, {enabled: true, debug: false}], + ["table", `[tool]\nname = "x"`, {tool: {name: "x"}}], + ["nested table", `[tool.poetry]\nname = "x"`, {tool: {poetry: {name: "x"}}}], + ["prototype key", `[__proto__.poetry]\nname = "x"`, {["__proto__"]: {poetry: {name: "x"}}}], + ["dotted key", `a.b = "val"`, {a: {b: "val"}}], + ["dotted key in table", `[section]\na.b = "val"`, {section: {a: {b: "val"}}}], + ["quoted dotted key", `"dotted.key" = "val"`, {"dotted.key": "val"}], + ["literal string", `key = 'hello'`, {key: "hello"}], + ["inline multiline basic string", `key = """hello"""`, {key: "hello"}], + ["inline multiline literal string", `key = '''hello'''`, {key: "hello"}], + ["multiline basic string", `a = """\nhello\nworld\n"""\nb = "x"`, {a: "hello\nworld\n", b: "x"}], + ["key text in multiline string", `[dependencies]\nhelp = """\nfoo = bar\n"""\nserde = "1.0"`, + {dependencies: {help: "foo = bar\n", serde: "1.0"}}], + ["multiline literal string", `a = '''\nx = 1\n'''\n[dependencies]\nserde = "1"`, + {a: "x = 1\n", dependencies: {serde: "1"}}], + ["basic escapes", `a = "line1\\nline2"\nb = "col1\\tcol2"`, {a: "line1\nline2", b: "col1\tcol2"}], + ["short unicode escape", `ch = "\\u0041"`, {ch: "A"}], + ["long unicode escape", `ch = "\\U00000041"`, {ch: "A"}], + ["quote and slash escapes", `a = "he said \\"hi\\""\nb = "c:\\\\path"`, {a: `he said "hi"`, b: "c:\\path"}], + ["inline array", `tags = ["a", "b", "c"]`, {tags: ["a", "b", "c"]}], + ["multiline array", `deps = [\n "foo",\n "bar",\n]`, {deps: ["foo", "bar"]}], + ["multiline array with brackets in strings", `deps = [\n "apispec[marshmallow]==6.10.0",\n "foo",\n]`, + {deps: ["apispec[marshmallow]==6.10.0", "foo"]}], + ["multiline nested arrays", `a = [\n [1, 2],\n [3, 4],\n]`, {a: [[1, 2], [3, 4]]}], + ["multiline inline tables", `a = [\n { f = ["x"] },\n { f = ["y"] },\n]`, {a: [{f: ["x"]}, {f: ["y"]}]}], + ["inline array with brackets in strings", `deps = ["apispec[marshmallow]==6.10.0", "foo"]`, + {deps: ["apispec[marshmallow]==6.10.0", "foo"]}], + ["hash in string", `name = "url#fragment"`, {name: "url#fragment"}], + ["trailing comment", `name = "x" # trailing`, {name: "x"}], + ["array of tables", `[[tool.pytest]]\nx = 1\n[[tool.pytest]]\nx = 2`, {tool: {pytest: [{x: 1}, {x: 2}]}}], + ["array of tables after table", `[package]\nname = "pkg"\n[[bin]]\nname = "bin"\n[dependencies]\nserde = "1"`, + {package: {name: "pkg"}, bin: [{name: "bin"}], dependencies: {serde: "1"}}], + ["nested arrays", `a = [[1,2],[3,4]]`, {a: [[1, 2], [3, 4]]}], + ["array of inline tables", `a = [{x=1},{x=2}]`, {a: [{x: 1}, {x: 2}]}], + ["inline table", `point = {x = 1, y = 2}`, {point: {x: 1, y: 2}}], + ["comments", `key = "value" # comment\n# full line comment\nother = 1`, {key: "value", other: 1}], + ["blank lines", `\n\n \nkey = "value"\n\n`, {key: "value"}], + ["multiple tables", `[a]\nx = 1\n[b]\ny = 2`, {a: {x: 1}, b: {y: 2}}], + ["mixed array", `vals = [1, "two", true]`, {vals: [1, "two", true]}], + ["multiline basic escapes", `key = """hello\\nworld"""`, {key: "hello\nworld"}], + ["multiline literal content", `key = '''raw\\nstring'''`, {key: "raw\\nstring"}], + ["inline table in table", `[section]\npoint = {x = 1, y = 2}`, {section: {point: {x: 1, y: 2}}}], + ["backspace and form feed", `a = "\\b"\nb = "\\f"`, {a: "\b", b: "\f"}], + ["carriage return", `key = "\\r"`, {key: "\r"}], + ]; + for (const [, input, expected] of cases) expect(parseToml(input)).toEqual(expected); expect(({} as Record).poetry).toBeUndefined(); }); -test("dotted keys", () => { - expect(parseToml(`a.b = "val"`)).toEqual({a: {b: "val"}}); -}); - -test("dotted keys within table", () => { - expect(parseToml(`[section]\na.b = "val"`)).toEqual({section: {a: {b: "val"}}}); -}); - -test("quoted key preserves dots", () => { - expect(parseToml(`"dotted.key" = "val"`)).toEqual({"dotted.key": "val"}); -}); - -test("literal string", () => { - expect(parseToml(`key = 'hello'`)).toEqual({key: "hello"}); -}); - -test("multi-line basic string", () => { - expect(parseToml(`key = """hello"""`)).toEqual({key: "hello"}); -}); - -test("multi-line literal string", () => { - expect(parseToml(`key = '''hello'''`)).toEqual({key: "hello"}); -}); - -test("multi-line basic string spanning lines keeps body and leaks no keys", () => { - expect(parseToml(`a = """\nhello\nworld\n"""\nb = "x"`)).toEqual({a: "hello\nworld\n", b: "x"}); - // a "key = value" line inside the string must not become a phantom dependency - expect(parseToml(`[dependencies]\nhelp = """\nfoo = bar\n"""\nserde = "1.0"`)) - .toEqual({dependencies: {help: "foo = bar\n", serde: "1.0"}}); -}); - -test("multi-line literal string spanning lines", () => { - expect(parseToml(`a = '''\nx = 1\n'''\n[dependencies]\nserde = "1"`)) - .toEqual({a: "x = 1\n", dependencies: {serde: "1"}}); -}); - -test("escape sequences in basic strings", () => { - expect(parseToml(`a = "line1\\nline2"\nb = "col1\\tcol2"`)).toEqual({a: "line1\nline2", b: "col1\tcol2"}); -}); - -test("unicode escape \\uXXXX", () => { - expect(parseToml(`ch = "\\u0041"`)).toEqual({ch: "A"}); -}); - -test("unicode escape \\UXXXXXXXX", () => { - expect(parseToml(`ch = "\\U00000041"`)).toEqual({ch: "A"}); -}); - -test("escaped quote and backslash", () => { - expect(parseToml(`a = "he said \\"hi\\""\nb = "c:\\\\path"`)).toEqual({a: `he said "hi"`, b: "c:\\path"}); -}); - -test("inline array", () => { - expect(parseToml(`tags = ["a", "b", "c"]`)).toEqual({tags: ["a", "b", "c"]}); -}); - -test("multi-line array", () => { - const input = `deps = [\n "foo",\n "bar",\n]`; - expect(parseToml(input)).toEqual({deps: ["foo", "bar"]}); -}); - -test("multi-line array with brackets inside strings", () => { - const input = `deps = [\n "apispec[marshmallow]==6.10.0",\n "foo",\n]`; - expect(parseToml(input)).toEqual({deps: ["apispec[marshmallow]==6.10.0", "foo"]}); -}); - -test("multi-line array of nested arrays", () => { - expect(parseToml(`a = [\n [1, 2],\n [3, 4],\n]`)).toEqual({a: [[1, 2], [3, 4]]}); -}); - -test("multi-line array of inline tables with inner arrays", () => { - expect(parseToml(`a = [\n { f = ["x"] },\n { f = ["y"] },\n]`)).toEqual({a: [{f: ["x"]}, {f: ["y"]}]}); -}); - -test("inline array with brackets inside strings", () => { - expect(parseToml(`deps = ["apispec[marshmallow]==6.10.0", "foo"]`)) - .toEqual({deps: ["apispec[marshmallow]==6.10.0", "foo"]}); -}); - -test("hash inside string is not a comment", () => { - expect(parseToml(`name = "url#fragment"`)).toEqual({name: "url#fragment"}); -}); - -test("trailing comment after value", () => { - expect(parseToml(`name = "x" # trailing`)).toEqual({name: "x"}); -}); - -test("array of tables", () => { - const input = `[[tool.pytest]]\nx = 1\n[[tool.pytest]]\nx = 2`; - expect(parseToml(input)).toEqual({tool: {pytest: [{x: 1}, {x: 2}]}}); -}); - -test("array of tables does not leak into prior table", () => { - const input = `[package]\nname = "pkg"\n[[bin]]\nname = "bin"\n[dependencies]\nserde = "1"`; - expect(parseToml(input)).toEqual({ - package: {name: "pkg"}, - bin: [{name: "bin"}], - dependencies: {serde: "1"}, - }); -}); - -test("nested arrays", () => { - expect(parseToml(`a = [[1,2],[3,4]]`)).toEqual({a: [[1, 2], [3, 4]]}); -}); - -test("array of inline tables", () => { - expect(parseToml(`a = [{x=1},{x=2}]`)).toEqual({a: [{x: 1}, {x: 2}]}); -}); - -test("inline table", () => { - expect(parseToml(`point = {x = 1, y = 2}`)).toEqual({point: {x: 1, y: 2}}); -}); - -test("comments are stripped", () => { - expect(parseToml(`key = "value" # comment\n# full line comment\nother = 1`)).toEqual({key: "value", other: 1}); -}); - -test("empty and blank lines are ignored", () => { - expect(parseToml(`\n\n \nkey = "value"\n\n`)).toEqual({key: "value"}); -}); - -test("multiple tables", () => { - const input = `[a]\nx = 1\n[b]\ny = 2`; - expect(parseToml(input)).toEqual({a: {x: 1}, b: {y: 2}}); -}); - -test("mixed types in array", () => { - expect(parseToml(`vals = [1, "two", true]`)).toEqual({vals: [1, "two", true]}); -}); - -test("multi-line basic string with escapes", () => { - expect(parseToml(`key = """hello\\nworld"""`)).toEqual({key: "hello\nworld"}); -}); - -test("multi-line literal string preserves content", () => { - expect(parseToml(`key = '''raw\\nstring'''`)).toEqual({key: "raw\\nstring"}); -}); - -test("nested inline table within table", () => { - expect(parseToml(`[section]\npoint = {x = 1, y = 2}`)).toEqual({section: {point: {x: 1, y: 2}}}); -}); - -test("backspace and form feed escape sequences", () => { - expect(parseToml(`a = "\\b"\nb = "\\f"`)).toEqual({a: "\b", b: "\f"}); -}); - -test("carriage return escape sequence", () => { - expect(parseToml(`key = "\\r"`)).toEqual({key: "\r"}); -}); - -test("real-world pyproject.toml", () => { - const content = readFileSync("fixtures/uv/pyproject.toml", "utf8"); - const result = parseToml(content); +test("real pyproject.toml", () => { + const result = parseToml(readFileSync("fixtures/uv/pyproject.toml", "utf8")); expect(result.project).toEqual({ name: "uvproject", version: "0.0.0", diff --git a/utils/toml.ts b/utils/toml.ts index 3663e40..b6a7182 100644 --- a/utils/toml.ts +++ b/utils/toml.ts @@ -1,14 +1,9 @@ -// Minimal TOML parser for pyproject.toml files. -// Supports: tables, dotted keys, basic strings, literal strings, -// arrays of strings, booleans, integers, floats, inline tables. - type TomlValue = string | number | boolean | Array | TomlObject; type TomlObject = {[key: string]: TomlValue}; const arrayTableRe = /^\[\[([^\]]+)\]\]$/; const tableRe = /^\[([^[\]]+)\]$/; -// A `__proto__` table is a plain key of the document, never a write into Object.prototype. const emptyTable = (): TomlObject => Object.create(null); export function parseToml(input: string): TomlObject { @@ -21,7 +16,6 @@ export function parseToml(input: string): TomlObject { const line = stripComment(raw).trim(); if (!line) continue; - // Array of tables: [[name]] const arrayTableMatch = arrayTableRe.exec(line); if (arrayTableMatch) { let target: TomlObject = root; @@ -43,25 +37,22 @@ export function parseToml(input: string): TomlObject { continue; } - // Table header const tableMatch = tableRe.exec(line); if (tableMatch) { current = descend(root, splitDottedKey(tableMatch[1])); continue; } - // Key = value - const eqIdx = indexOfUnquoted(line, "="); + const eqIdx = unquotedIndex(line, "="); if (eqIdx < 0) continue; const rawKey = line.slice(0, eqIdx).trim(); const rawVal = line.slice(eqIdx + 1).trim(); const keys = splitDottedKey(rawKey); const target = descend(current, keys.slice(0, -1)); const finalKey = keys[keys.length - 1]; - const mlDelim = multilineStringDelim(rawVal); + const mlDelim = ['"""', "'''"].find(delimiter => + rawVal.startsWith(delimiter) && !rawVal.includes(delimiter, 3)) ?? ""; - // Multi-line array or inline table: gather lines until the outer "]"/"}" (depth-aware), then - // parse the full text with parseValue so nested arrays and inline tables stay intact. if ((rawVal.startsWith("[") || rawVal.startsWith("{")) && !inlineTableClosed(rawVal)) { let body = rawVal; let j = i + 1; @@ -72,8 +63,6 @@ export function parseToml(input: string): TomlObject { i = j; target[finalKey] = parseValue(body); } else if (mlDelim) { - // Multi-line basic/literal string: gather raw lines up to the closing delimiter, then - // re-wrap and hand to parseValue so escaping/literal handling stays in one place. let body = rawVal.slice(3); let j = i + 1; for (; j < lines.length; j++) { @@ -94,17 +83,10 @@ export function parseToml(input: string): TomlObject { return root; } -// Returns the opening delimiter if raw starts a multi-line string that does not close on the same line, else "". -function multilineStringDelim(raw: string): string { - const delim = raw.slice(0, 3); - if (delim !== '"""' && delim !== "'''") return ""; - return raw.includes(delim, 3) ? "" : delim; -} - function parseValue(raw: string): TomlValue { if (raw.startsWith("[")) { const items: Array = []; - const closeIdx = lastIndexOfUnquoted(raw, "]"); + const closeIdx = raw.lastIndexOf("]"); for (const part of splitTopLevel(raw.slice(1, closeIdx < 0 ? raw.length : closeIdx))) { const clean = part.trim(); if (clean) items.push(parseValue(clean)); @@ -114,11 +96,9 @@ function parseValue(raw: string): TomlValue { if (raw.startsWith("{")) { return parseInlineTable(raw); } - // Multi-line basic string if (raw.startsWith('"""')) { return unescapeString(raw.slice(3, raw.lastIndexOf('"""'))); } - // Multi-line literal string if (raw.startsWith("'''")) { return raw.slice(3, raw.lastIndexOf("'''")); } @@ -128,10 +108,6 @@ function parseValue(raw: string): TomlValue { if (raw.startsWith("'")) { return raw.slice(1, raw.lastIndexOf("'")); } - return inferScalar(raw); -} - -function inferScalar(raw: string): TomlValue { if (raw === "true") return true; if (raw === "false") return false; if (/^[+-]?\d+(\.\d+)?$/.test(raw)) return Number(raw); @@ -151,10 +127,11 @@ function parseInlineTable(raw: string): TomlObject { return obj; } -// True once the brackets/braces in `s` balance out — i.e. the inline table that opened with "{" has closed. -function inlineTableClosed(s: string): boolean { +function scanValue(s: string, split: boolean): Array | null { + const parts: Array = []; let depth = 0; let inStr: string | null = null; + let start = 0; for (let k = 0; k < s.length; k++) { const ch = s[k]; if (inStr) { @@ -166,35 +143,23 @@ function inlineTableClosed(s: string): boolean { depth++; } else if (ch === "}" || ch === "]") { depth--; - if (depth === 0) return true; + if (!split && depth === 0) return parts; + } else if (split && ch === "," && depth === 0) { + parts.push(s.slice(start, k)); + start = k + 1; } } - return false; + if (!split) return null; + if (start < s.length) parts.push(s.slice(start)); + return parts; +} + +function inlineTableClosed(s: string): boolean { + return scanValue(s, false) !== null; } function splitTopLevel(s: string): Array { - const parts: Array = []; - let depth = 0; - let inStr: string | null = null; - let start = 0; - for (let i = 0; i < s.length; i++) { - const ch = s[i]; - if (inStr) { - if (ch === "\\" && inStr === '"') { i++; continue; } - if (ch === inStr) inStr = null; - } else if (ch === '"' || ch === "'") { - inStr = ch; - } else if (ch === "[" || ch === "{") { - depth++; - } else if (ch === "]" || ch === "}") { - depth--; - } else if (ch === "," && depth === 0) { - parts.push(s.slice(start, i)); - start = i + 1; - } - } - if (start < s.length) parts.push(s.slice(start)); - return parts; + return scanValue(s, true)!; } function splitDottedKey(key: string): Array { @@ -219,11 +184,11 @@ function splitDottedKey(key: string): Array { } function stripComment(line: string): string { - const idx = indexOfUnquoted(line, "#"); - return idx < 0 ? line : line.slice(0, idx); + const index = unquotedIndex(line, "#"); + return index < 0 ? line : line.slice(0, index); } -function* unquotedIndices(s: string, target: string): Generator { +function unquotedIndex(s: string, target: string): number { let inStr: string | null = null; for (let i = 0; i < s.length; i++) { const ch = s[i]; @@ -233,19 +198,10 @@ function* unquotedIndices(s: string, target: string): Generator { } else if (ch === '"' || ch === "'") { inStr = ch; } else if (ch === target) { - yield i; + return i; } } -} - -function indexOfUnquoted(s: string, target: string): number { - return unquotedIndices(s, target).next().value ?? -1; -} - -function lastIndexOfUnquoted(s: string, target: string): number { - let last = -1; - for (const i of unquotedIndices(s, target)) last = i; - return last; + return -1; } function descend(target: TomlObject, keys: Array): TomlObject { diff --git a/utils/utils.test.ts b/utils/utils.test.ts index 6ffb639..16cb170 100644 --- a/utils/utils.test.ts +++ b/utils/utils.test.ts @@ -1,38 +1,30 @@ import { highlightDiff, parseUvDependencies, parseDuration, matchesAny, commaSeparatedToArray, - timestamp, textTable, pMap, expandDepTypes, uvTypes, cargoTypes, cargoTargetTypes, + timestamp, textTable, pMap, expandDepTypes, uvTypes, cargoTypes, cargoTargetTypes, patternToRegex, + npmTypes, } from "./utils.ts"; const c = (s: string) => `[${s}]`; test("highlightDiff", () => { - // equal strings return unchanged expect(highlightDiff("1.0.0", "1.0.0", c)).toBe("1.0.0"); - // major version diff expect(highlightDiff("1.0.0", "2.0.0", c)).toBe("[1.0.0]"); expect(highlightDiff("2.0.0", "1.0.0", c)).toBe("[2.0.0]"); - // minor version diff expect(highlightDiff("1.0.0", "1.2.0", c)).toBe("1.[0.0]"); expect(highlightDiff("1.2.0", "1.0.0", c)).toBe("1.[2.0]"); - // patch version diff expect(highlightDiff("1.0.0", "1.0.3", c)).toBe("1.0.[0]"); expect(highlightDiff("1.0.3", "1.0.0", c)).toBe("1.0.[3]"); - // multi-digit numbers stay intact expect(highlightDiff("10.0.0", "12.0.0", c)).toBe("[10.0.0]"); expect(highlightDiff("12.0.0", "10.0.0", c)).toBe("[12.0.0]"); expect(highlightDiff("1.10.0", "1.12.0", c)).toBe("1.[10.0]"); - // v prefix preserved expect(highlightDiff("v5", "v6", c)).toBe("v[5]"); expect(highlightDiff("v10", "v12", c)).toBe("v[10]"); expect(highlightDiff("v10.0", "v12.0", c)).toBe("v[10.0]"); - // range prefixes preserved expect(highlightDiff("^4", "^5", c)).toBe("^[4]"); expect(highlightDiff("^1.0.0", "^2.0.0", c)).toBe("^[1.0.0]"); expect(highlightDiff("~1.0.0", "~1.5.0", c)).toBe("~1.[0.0]"); expect(highlightDiff(">=2.0.0", ">=2.6.5", c)).toBe(">=2.[0.0]"); - // prerelease expect(highlightDiff("4.0.0-alpha.2", "4.0.0-beta.11", c)).toBe("4.0.0-[alpha.2]"); - // hashes (no common prefix) expect(highlightDiff("537ccb7", "6941e05", c)).toBe("[537ccb7]"); }); @@ -52,16 +44,15 @@ test("parseUvDependencies", () => { "importlib-metadata (==8.0.0)", "wheel (>=0.40.0); python_version < \"3.8\"", "urllib3===1.26.0", - // no lower bound to bump, no version at all, or nothing bumpable in place "certifi!=2024.2.2", "idna<4", - "flask>2.3.0,<3", // `>` cannot be bumped without changing which versions are allowed + "flask>2.3.0,<3", "click==1.4.*", "requests", "anyio[trio]", "typing-extensions; python_version < \"3.8\"", "torchvision @ https://example.com/torchvision-0.17.2.whl", - {"include-group": "lint"}, // PEP 735 + {"include-group": "lint"}, ]); expect(parsed[0].spec).toBe("tqdm >=4.66.2,<5"); expect(parsed.map(({name, version}) => ({name, version}))).toEqual([ @@ -86,7 +77,6 @@ test("expandDepTypes", () => { const pyproject = { project: { "dependencies": ["requests>=2.0.0"], - // a group name may legally contain a dot, which a re-split of the joined path would lose "optional-dependencies": {"cli": ["click>=8.0.0"], "extra.one": ["sphinx>=7.0.0"]}, }, "dependency-groups": {"docs": ["mkdocs>=1.6.0"], "test.unit": [{"include-group": "docs"}]}, @@ -104,32 +94,56 @@ test("expandDepTypes", () => { dependencies: {serde: "1.0"}, target: { "cfg(unix)": {dependencies: {nix: "0.29"}}, + "cfg(feature = \"foo.bar\")": {dependencies: {feature: "1"}}, "x86_64-pc-windows-msvc": {"dependencies": {winapi: "0.3"}, "build-dependencies": {cc: "1.0"}}, }, }; expect(expandDepTypes([...cargoTypes, ...cargoTargetTypes], cargo).map(([type]) => type)).toEqual([ "dependencies", - "target.cfg(unix).dependencies", - "target.x86_64-pc-windows-msvc.dependencies", - "target.x86_64-pc-windows-msvc.build-dependencies", + `["target","cfg(unix)","dependencies"]`, + `["target","cfg(feature = \\"foo.bar\\")","dependencies"]`, + `["target","x86_64-pc-windows-msvc","dependencies"]`, + `["target","x86_64-pc-windows-msvc","build-dependencies"]`, ]); }); +test("default npm dependency types", () => { + expect(npmTypes).toContain("overrides"); + expect(npmTypes).toContain("pnpm.overrides"); +}); + +test.each([ + ["foo*", ["FOO", "foo.bar", "foo/bar"], [true, true, false]], + ["foo/**/bar", ["foo/bar", "foo/a/bar", ".foo/bar"], [true, true, false]], + ["file?.[jt]s", ["file1.js", "fileA.ts", "file10.js"], [true, true, false]], + ["{foo,bar}/@(one|two)", ["foo/one", "bar/two", "baz/one"], [true, true, false]], + ["v{1..3}", ["v1", "v3", "v4"], [true, true, false]], + ["?(foo)", ["", "foo", "foofoo"], [true, true, false]], + ["*(foo)", ["", "foo", "foofoo", "foofoofoo"], [true, true, true, true]], + ["+(foo)", ["", "foo", "foofoo"], [false, true, true]], + ["!(foo)", ["foo", "bar"], [false, true]], + ["/Foo/", ["Foo", "foo"], [true, false]], + ["!/foo/i", ["FOO", "bar"], [false, true]], +])("patternToRegex supports %s", (pattern, values, expected) => { + expect(values.map(value => patternToRegex(pattern).test(value))).toEqual(expected); +}); + +test("patternToRegex avoids extglob backtracking", () => { + const start = performance.now(); + expect(patternToRegex("+(a|aa)").test(`${"a".repeat(40)}b`)).toBe(false); + expect(performance.now() - start).toBeLessThan(100); +}); + test("matchesAny", () => { - expect(matchesAny("foo", new Set([/foo/]))).toBe(true); - expect(matchesAny("bar", new Set([/foo/]))).toBe(false); - expect(matchesAny("foobar", new Set([/^foo/]))).toBe(true); - expect(matchesAny("foo", new Set([/bar/, /foo/]))).toBe(true); - expect(matchesAny("foo", false)).toBe(false); - expect(matchesAny("foo", true)).toBe(true); - expect(matchesAny("foo", new Set())).toBe(false); + expect([ + matchesAny("foo", new Set([/foo/])), matchesAny("bar", new Set([/foo/])), matchesAny("foobar", new Set([/^foo/])), + matchesAny("foo", new Set([/bar/, /foo/])), matchesAny("foo", false), matchesAny("foo", true), matchesAny("foo", new Set()), + ]).toEqual([true, false, true, true, false, true, false]); }); test("commaSeparatedToArray", () => { - expect(commaSeparatedToArray("a,b,c")).toEqual(["a", "b", "c"]); - expect(commaSeparatedToArray("a")).toEqual(["a"]); - expect(commaSeparatedToArray("")).toEqual([]); - expect(commaSeparatedToArray("a,,b")).toEqual(["a", "b"]); + expect(["a,b,c", "a", "", "a,,b"].map(commaSeparatedToArray)) + .toEqual([["a", "b", "c"], ["a"], [], ["a", "b"]]); }); test("timestamp", () => { @@ -138,36 +152,20 @@ test("timestamp", () => { }); test("textTable", () => { - const len = (s: string) => s.length; - expect(textTable([["a", "bb"], ["ccc", "d"]], len)).toBe("a bb\nccc d"); - expect(textTable([["x"]], len)).toBe("x"); + expect(textTable([["a", "bb"], ["ccc", "d"]], value => value.length)).toBe("a bb\nccc d"); + expect(textTable([["x"]], value => value.length)).toBe("x"); }); test("parseDuration", () => { - expect(parseDuration("7")).toBe(7); - expect(parseDuration("2y")).toBe(730); - expect(parseDuration("3m")).toBe(90); - expect(parseDuration("1w")).toBe(7); - expect(parseDuration("2d")).toBe(2); - expect(parseDuration("12h")).toBe(0.5); - expect(parseDuration("6h")).toBe(0.25); - expect(parseDuration("86400s")).toBe(1); + expect(["7", "2y", "3m", "1w", "2d", "12h", "6h", "86400s"].map(parseDuration)) + .toEqual([7, 730, 90, 7, 2, 0.5, 0.25, 1]); expect(parseDuration("10s")).toBeCloseTo(10 / 86400); expect(() => parseDuration("abc")).toThrow("Invalid cooldown value"); expect(() => parseDuration("12x")).toThrow("Invalid cooldown value"); }); -test("pMap basic", async () => { - const result = await pMap([1, 2, 3], (n) => Promise.resolve(n * 2)); - expect(result).toEqual([2, 4, 6]); -}); - -test("pMap limited concurrency", async () => { - const result = await pMap([10, 20, 30], (n) => Promise.resolve(n + 1), {concurrency: 2}); - expect(result).toEqual([11, 21, 31]); -}); - -test("pMap empty iterable", async () => { - const result = await pMap([], (n: number) => Promise.resolve(n)); - expect(result).toEqual([]); +test("pMap", async () => { + expect(await pMap([1, 2, 3], n => Promise.resolve(n * 2))).toEqual([2, 4, 6]); + expect(await pMap([10, 20, 30], n => Promise.resolve(n + 1), {concurrency: 2})).toEqual([11, 21, 31]); + expect(await pMap([], (n: number) => Promise.resolve(n))).toEqual([]); }); diff --git a/utils/utils.ts b/utils/utils.ts index 4a6560c..34cff67 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -4,14 +4,12 @@ export function highlightDiff(a: string, b: string, colorFn: (str: string) => st if (a === b) return a; let i = 0; while (i < a.length && i < b.length && a[i] === b[i]) i++; - // Back up to a version part boundary to avoid splitting numbers if (i > 0 && a[i - 1] !== "." && a[i - 1] !== "-") { let j = i - 1; while (j >= 0 && a[j] !== "." && a[j] !== "-") j--; if (j >= 0) { i = j + 1; } else { - // No separator found, preserve non-digit prefix (v, ^, >=, ~) let d = 0; while (d < i) { const code = a.charCodeAt(d); @@ -25,16 +23,10 @@ export function highlightDiff(a: string, b: string, colorFn: (str: string) => st return diff ? a.substring(0, i) + colorFn(diff) : a; } -// Name, optional extras and everything up to the marker, spacing captured for re-serializing. const pep508Re = /^(\s*)([A-Za-z0-9][A-Za-z0-9._-]*)(\s*)((?:\[[^\]]*\])?)(\s*)(.*)$/; -const pep508ParenRe = /^(\s*\()([^)]*)(\)\s*)$/; // `packaging (==20.0.0)` -// The version is anything non-blank, so a cap or exclusion is recognized even as a wildcard. +const pep508ParenRe = /^(\s*\()([^)]*)(\)\s*)$/; const pep440SpecifierRe = /^(\s*)(===|==|!=|~=|<=|>=|<|>)(\s*)(\S+)(\s*)$/; -// Only a lower bound states the version the project is on. `<`, `<=`, `!=` and `>` exclude versions, -// so bumping one would change what the spec allows. Renovate leaves `>` as authored too, which -// makes a `>`-only requirement unbumpable. const lowerBoundOps = new Set(["===", "==", ">=", "~="]); -// A wildcard (`==1.4.*`) or arbitrary equality on a non-version (`===foo`) has nothing to bump. const plainVersionRe = /^v?\d[0-9a-z.!+_-]*$/i; export type Pep508Specifier = {lead: string, op: string, sep: string, version: string, trail: string}; @@ -42,10 +34,8 @@ export type Pep508Specifier = {lead: string, op: string, sep: string, version: s export type Pep508 = { name: string; extras: string; - /** null when the set does not parse in full, so a writer never rewrites what it did not read. */ specifiers: Array | null; - marker: string; // the environment marker with its `;`, verbatim - // Verbatim spacing, name, extras and parens, so serializePep508 reproduces an untouched requirement. + marker: string; head: string; open: string; close: string; @@ -62,7 +52,6 @@ function parseSpecifiers(text: string): Array | null { return specifiers; } -/** Parse one PEP 508 requirement. https://peps.python.org/pep-0508/ */ export function parsePep508(text: string): Pep508 | null { const semi = text.indexOf(";"); const match = pep508Re.exec(semi === -1 ? text : text.slice(0, semi)); @@ -85,8 +74,6 @@ export function serializePep508({head, open, close, marker}: Pep508, specifiers: return `${head}${open}${set}${close}${marker}`; } -// The specifier a requirement's version is read from, and the only one a writer bumps: anchoring -// elsewhere would move a specifier the reported version never came from. export function anchorSpecifier(specifiers: Array): Pep508Specifier | undefined { return specifiers.find(({op, version}) => lowerBoundOps.has(op) && plainVersionRe.test(version)); } @@ -94,7 +81,7 @@ export function anchorSpecifier(specifiers: Array): Pep508Speci export function parseUvDependencies(specs: Array) { const ret: Array<{name: string, version: string, spec: string}> = []; for (const spec of specs) { - if (typeof spec !== "string") continue; // PEP 735 `{include-group = "..."}` and other tables + if (typeof spec !== "string") continue; const parsed = parsePep508(spec); if (!parsed?.specifiers) continue; const anchor = anchorSpecifier(parsed.specifiers); @@ -109,6 +96,8 @@ export const npmTypes = [ "peerDependencies", "optionalDependencies", "resolutions", + "overrides", + "pnpm.overrides", "packageManager", ]; @@ -118,12 +107,8 @@ export const nonPackageEngines = [ "bun", ]; -// Forge config directories holding workflow files, in discovery order. export const forgeDirs = [".github", ".gitea", ".forgejo"] as const; -// Manifest filenames that select a mode. Also drives which registry origins get -// a socket pre-warmed, so a new entry must be handled in prewarmOrigins too — -// utils/prewarm.test.ts fails if one is missed. export const modeByFileName: Record = { "pnpm-workspace.yaml": "npm", "package.json": "npm", @@ -153,29 +138,25 @@ export const cargoTypes = [ "workspace.dependencies", ]; -// Target names are arbitrary (`cfg(unix)`, `x86_64-pc-windows-msvc`), so these need a manifest. export const cargoTargetTypes = [ "target.*.dependencies", "target.*.dev-dependencies", "target.*.build-dependencies", ]; -// Resolve dep type paths against a parsed manifest, so `*` segments take group and target names -// from the document. Each path comes back with the table it resolved to, so a key that legally -// contains a dot (`[project.optional-dependencies."extra.one"]`) is never re-split and lost. export function expandDepTypes(types: Array, doc: Record): Array<[string, any]> { const ret: Array<[string, any]> = []; - const walk = (segments: Array, index: number, path: string, value: any) => { + const walk = (segments: Array, index: number, path: Array, value: any, structured: boolean) => { if (index === segments.length) { - if (value !== undefined) ret.push([path, value]); + if (value !== undefined) ret.push([structured ? JSON.stringify(path) : path.join("."), value]); return; } if (!value || typeof value !== "object") return; const segment = segments[index]; const keys = segment !== "*" ? [segment] : Array.isArray(value) ? [] : Object.keys(value); - for (const key of keys) walk(segments, index + 1, path ? `${path}.${key}` : key, value[key]); + for (const key of keys) walk(segments, index + 1, [...path, key], value[key], structured); }; - for (const type of types) walk(type.split("."), 0, "", doc); + for (const type of types) walk(type.split("."), 0, [], doc, type.startsWith("target.*.")); return ret; } @@ -192,11 +173,10 @@ export function commaSeparatedToArray(str: string): Array { export function timestamp(): string { const now = new Date(); - // Shifting by the offset makes the UTC fields of toISOString read as local time. return new Date(now.getTime() - now.getTimezoneOffset() * 60000).toISOString().replace("T", " ").slice(0, -1); } -export function textTable(rows: Array>, ansiLen: (str: string) => number, hsep = " "): string { +export function textTable(rows: Array>, ansiLen: (str: string) => number): string { const colSizes = new Array(rows[0].length).fill(0); const lens = new Array>(rows.length); for (let r = 0; r < rows.length; r++) { @@ -214,7 +194,7 @@ export function textTable(rows: Array>, ansiLen: (str: string) => const row = rows[r]; const lastCol = row.length - 1; for (let c = 0; c <= lastCol; c++) { - if (c > 0) ret += hsep; + if (c > 0) ret += " "; ret += row[c]; if (c !== lastCol) { const pad = colSizes[c] - lens[r][c]; @@ -228,7 +208,6 @@ export function textTable(rows: Array>, ansiLen: (str: string) => const durationUnits: Record = {y: 365, m: 30, w: 7, d: 1, h: 1 / 24, s: 1 / 86400}; -/** Parse a duration string (e.g. "7d", "2w", "1y") into days. Without unit, the value is treated as days. */ export function parseDuration(str: string): number { const match = /^(\d+(?:\.\d+)?)\s*([a-z])$/i.exec(str); if (match) { @@ -260,7 +239,6 @@ export async function pMap(iterable: Iterable, mapper: (item: T) => Pro return results; } -// Resolve a promise to its value, or null if it rejects. export async function tryOrNull(promise: Promise): Promise { try { return await promise; @@ -269,24 +247,221 @@ export async function tryOrNull(promise: Promise): Promise { } } -// RegExp.escape needs Node 24; fall back to a manual escape on Node 22. The -// feature check runs once, not on every call. export const esc: (str: string) => string = RegExp.escape ? (str) => RegExp.escape(str) : (str) => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -// Longest first, so a key that is a prefix of another cannot shadow it. One alternation also -// keeps a rewrite to a single pass, where per-key passes would re-match what a key just wrote. export function longestFirstAlternation(keys: Iterable): string { return Array.from(keys).sort((a, b) => b.length - a.length).map(esc).join("|"); } -// A string pattern is a case-insensitive glob, a RegExp is taken as authored. CLI `/regex/` strings -// are already RegExp objects by the time they arrive here. +const predicateTests = new WeakMap boolean>(); + +class PredicateRegExp extends RegExp { + constructor(source: string, predicate: (value: string) => boolean, flags?: string) { + super(source, flags); + predicateTests.set(this, predicate); + } + + override test(value: string): boolean { + return predicateTests.get(this)!(value); + } +} + +function splitGlobAlternatives(value: string): Array { + const alternatives: Array = []; + let depth = 0; + let start = 0; + for (let i = 0; i < value.length; i++) { + if (value[i] === "(" || value[i] === "{") depth++; + else if (value[i] === ")" || value[i] === "}") depth--; + else if ((value[i] === "|" || value[i] === ",") && depth === 0) { + alternatives.push(value.slice(start, i)); + start = i + 1; + } + } + alternatives.push(value.slice(start)); + return alternatives; +} + +function closingIndex(value: string, start: number, open: string, close: string): number { + let depth = 0; + for (let i = start; i < value.length; i++) { + if (value[i] === open) depth++; + else if (value[i] === close && --depth === 0) return i; + } + return -1; +} + +function braceAlternatives(value: string): Array { + const range = /^(-?\d+|[A-Za-z])\.\.(-?\d+|[A-Za-z])(?:\.\.(-?\d+))?$/.exec(value); + if (!range) return splitGlobAlternatives(value); + const numeric = /^-?\d+$/.test(range[1]) && /^-?\d+$/.test(range[2]); + let current = numeric ? Number(range[1]) : range[1].codePointAt(0)!; + const end = numeric ? Number(range[2]) : range[2].codePointAt(0)!; + const step = Math.abs(Number(range[3]) || 1) * (current <= end ? 1 : -1); + const alternatives: Array = []; + while ((step > 0 ? current <= end : current >= end) && alternatives.length < 1000) { + alternatives.push(numeric ? String(current) : String.fromCodePoint(current)); + current += step; + } + return alternatives; +} + +type GlobToken = + {kind: "char", matcher: RegExp} | + {kind: "span", slash: boolean} | + {kind: "globstarSlash"} | + {kind: "alternatives", alternatives: Array>, minimum: 0 | 1, repeat: boolean} | + {kind: "negative", alternatives: Array>}; + +function parseGlob(pattern: string): {source: string, tokens: Array} { + let source = ""; + const tokens: Array = []; + const addChar = (charSource: string) => { + source += charSource; + tokens.push({kind: "char", matcher: new RegExp(`^(?:${charSource})$`, "i")}); + }; + for (let i = 0; i < pattern.length; i++) { + const char = pattern[i]; + if (char === "\\" && i + 1 < pattern.length) { + addChar(esc(pattern[++i])); + } else if (char === "*" && pattern[i + 1] !== "(") { + if (pattern[i + 1] === "*") { + while (pattern[i + 1] === "*") i++; + if (pattern[i + 1] === "/") { + source += "(?:.*/)?"; + tokens.push({kind: "globstarSlash"}); + i++; + } else { + source += ".*"; + tokens.push({kind: "span", slash: true}); + } + } else { + source += "[^/]*"; + tokens.push({kind: "span", slash: false}); + } + } else if (char === "?" && pattern[i + 1] !== "(") { + addChar("[^/]"); + } else if (char === "[") { + const end = pattern.indexOf("]", i + 1); + if (end === -1) addChar("\\["); + else { + let content = pattern.slice(i + 1, end); + if (content.startsWith("!")) content = `^${content.slice(1)}`; + addChar(`[${content.replaceAll("\\", "\\\\")}]`); + i = end; + } + } else if (char === "{") { + const end = closingIndex(pattern, i, "{", "}"); + if (end === -1) addChar("\\{"); + else { + const alternatives = braceAlternatives(pattern.slice(i + 1, end)).map(parseGlob); + source += `(?:${alternatives.map(alternative => alternative.source).join("|")})`; + tokens.push({ + kind: "alternatives", + alternatives: alternatives.map(alternative => alternative.tokens), + minimum: 1, + repeat: false, + }); + i = end; + } + } else if ("@+?*!".includes(char) && pattern[i + 1] === "(") { + const end = closingIndex(pattern, i + 1, "(", ")"); + if (end === -1) addChar(esc(char)); + else { + const alternatives = splitGlobAlternatives(pattern.slice(i + 2, end)).map(parseGlob); + const alternativeSource = alternatives.map(alternative => alternative.source).join("|"); + source += char === "!" ? `(?!(?:${alternativeSource})(?:/|$))[^/]*` : + `(?:${alternativeSource})${char === "@" ? "" : char}`; + const alternativeTokens = alternatives.map(alternative => alternative.tokens); + tokens.push(char === "!" ? {kind: "negative", alternatives: alternativeTokens} : { + kind: "alternatives", alternatives: alternativeTokens, minimum: char === "?" || char === "*" ? 0 : 1, + repeat: char === "+" || char === "*", + }); + i = end; + } + } else { + addChar(esc(char)); + } + } + return {source, tokens}; +} + +function matchGlob(tokens: Array, value: string): boolean { + const memo = new WeakMap, Map>>(); + const addSpan = (positions: Set, start: number, slash: boolean) => { + for (let end = start; ; end++) { + positions.add(end); + if (end === value.length || !slash && value[end] === "/") break; + } + }; + const matchSequence = (sequence: Array, start: number): Set => { + let byStart = memo.get(sequence); + if (!byStart) { + byStart = new Map(); + memo.set(sequence, byStart); + } + const cached = byStart.get(start); + if (cached) return cached; + let positions = new Set([start]); + byStart.set(start, positions); + for (const token of sequence) { + const next = new Set(); + for (const position of positions) { + if (token.kind === "char") { + if (position < value.length && token.matcher.test(value[position])) next.add(position + 1); + } else if (token.kind === "span") { + addSpan(next, position, token.slash); + } else if (token.kind === "globstarSlash") { + next.add(position); + for (let end = position; end < value.length; end++) if (value[end] === "/") next.add(end + 1); + } else if (token.kind === "negative") { + const excluded = token.alternatives.some(alternative => + [...matchSequence(alternative, position)].some(end => end === value.length || value[end] === "/")); + if (!excluded) addSpan(next, position, false); + } else { + if (token.minimum === 0) next.add(position); + const first = new Set(token.alternatives.flatMap(alternative => [...matchSequence(alternative, position)])); + for (const end of first) next.add(end); + if (token.repeat) { + for (const end of next) { + for (const alternative of token.alternatives) { + for (const repeatedEnd of matchSequence(alternative, end)) next.add(repeatedEnd); + } + } + } + } + } + positions = next; + if (!positions.size) break; + } + byStart.set(start, positions); + return positions; + }; + return matchSequence(tokens, 0).has(value.length); +} + export function patternToRegex(pattern: string | RegExp): RegExp { - if (!(pattern instanceof RegExp)) return new RegExp(`^${esc(pattern).replaceAll("\\*", ".*")}$`, "i"); - // strip g/y: these matchers are only used with .test(), where a stateful lastIndex flakes - return /[gy]/.test(pattern.flags) ? new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, "")) : pattern; + if (pattern instanceof RegExp) { + return /[gy]/.test(pattern.flags) ? new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, "")) : pattern; + } + const match = /^(!?)\/(.*)\/(i?)$/.exec(pattern); + if (match) { + try { + const compiled = new RegExp(match[2], match[3]); + return match[1] ? new PredicateRegExp(pattern, value => !compiled.test(value)) : compiled; + } catch {} + } + let negateCount = 0; + while (pattern[negateCount] === "!" && pattern[negateCount + 1] !== "(") negateCount++; + const negated = negateCount % 2 === 1; + const glob = pattern.slice(negateCount); + const {source, tokens} = parseGlob(glob); + const compiled = new RegExp(`^${source}$`, "i"); + const predicate = (value: string) => matchGlob(tokens, value); + return negated ? new PredicateRegExp(pattern, value => !predicate(value)) : + new PredicateRegExp(compiled.source, predicate, compiled.flags); } export async function walkUp(startDir: string, probe: (dir: string) => Promise): Promise { @@ -300,7 +475,6 @@ export async function walkUp(startDir: string, probe: (dir: string) => Promis } } -// Append to a Map-of-arrays, creating the bucket on first use. export function pushTo(map: Map>, key: K, value: V): void { const list = map.get(key); if (list) { @@ -312,8 +486,6 @@ export function pushTo(map: Map>, key: K, value: V): void { type MapLike = {has: (key: K) => boolean, get: (key: K) => V | undefined, set: (key: K, value: V) => unknown}; -// Read through a memo, filling it on first use. `has` settles only the undefined case, so a -// cached null still counts. export function getOrSet(map: MapLike, key: K, make: () => V): V { const cached = map.get(key); if (cached !== undefined || map.has(key)) return cached!; diff --git a/utils/workspace.test.ts b/utils/workspace.test.ts index e44608d..c203011 100644 --- a/utils/workspace.test.ts +++ b/utils/workspace.test.ts @@ -1,114 +1,117 @@ -import {join} from "node:path"; -import {mkdtempSync, mkdirSync, writeFileSync} from "node:fs"; +import {join, relative} from "node:path"; +import {mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync} from "node:fs"; import {tmpdir} from "node:os"; -import {baseType, filterDepsForMember, resolveWorkspaceMembers, parsePnpmWorkspace, pnpmCatalogEntries, updatePnpmWorkspace} from "./workspace.ts"; +import { + baseType, filterDepsForMember, resolveWorkspaceMembers, parsePnpmWorkspace, pnpmCatalogEntries, updatePnpmWorkspace, + parsePnpmRegistryConfig, +} from "./workspace.ts"; import {fieldSep} from "../modes/shared.ts"; -const globalExpect = expect; +const created: Array = []; +const makeWorkspace = (files: Record = {}) => { + const dir = mkdtempSync(join(tmpdir(), "ws-test-")); + created.push(dir); + for (const [path, content] of Object.entries(files)) { + const full = join(dir, path); + mkdirSync(join(full, ".."), {recursive: true}); + writeFileSync(full, content); + } + return dir; +}; -test("baseType", ({expect = globalExpect}: any = {}) => { - expect(baseType("dependencies")).toBe("dependencies"); - expect(baseType("dependencies|./app")).toBe("dependencies"); - expect(baseType("dev-dependencies|./crate-a")).toBe("dev-dependencies"); - expect(baseType("workspace.dependencies")).toBe("workspace.dependencies"); - expect(baseType("deps|./lib")).toBe("deps"); -}); +afterAll(() => { for (const dir of created) rmSync(dir, {recursive: true, force: true}); }); -test("filterDepsForMember root", ({expect = globalExpect}: any = {}) => { - const allDeps = { - [`dependencies${fieldSep}serde`]: {old: "1.0", new: "1.1"}, - [`dependencies|./app${fieldSep}tokio`]: {old: "1.0", new: "1.35"}, - [`workspace.dependencies${fieldSep}serde_json`]: {old: "1.0", new: "1.1"}, - }; - const result = filterDepsForMember(allDeps, "."); - expect(Object.keys(result)).toHaveLength(2); - expect(result[`dependencies${fieldSep}serde`]).toBeDefined(); - expect(result[`workspace.dependencies${fieldSep}serde_json`]).toBeDefined(); +test("baseType", () => { + expect(["dependencies", "dependencies|./app", "dev-dependencies|./crate-a", "workspace.dependencies", "deps|./lib"] + .map(baseType)).toEqual(["dependencies", "dependencies", "dev-dependencies", "workspace.dependencies", "deps"]); }); -test("filterDepsForMember named member", ({expect = globalExpect}: any = {}) => { +test("filterDepsForMember", () => { const allDeps = { [`dependencies${fieldSep}serde`]: {old: "1.0", new: "1.1"}, [`dependencies|./app${fieldSep}tokio`]: {old: "1.0", new: "1.35"}, + [`workspace.dependencies${fieldSep}serde_json`]: {old: "1.0", new: "1.1"}, [`dev-dependencies|./app${fieldSep}rand`]: {old: "0.8", new: "0.9"}, }; - const result = filterDepsForMember(allDeps, "./app"); - expect(Object.keys(result)).toHaveLength(2); - expect(result[`dependencies${fieldSep}tokio`]).toBeDefined(); - expect(result[`dev-dependencies${fieldSep}rand`]).toBeDefined(); -}); - -test("resolveWorkspaceMembers literal paths", async ({expect = globalExpect}: any = {}) => { - const dir = mkdtempSync(join(tmpdir(), "ws-test-")); - mkdirSync(join(dir, "crate-a"), {recursive: true}); - mkdirSync(join(dir, "crate-b"), {recursive: true}); - writeFileSync(join(dir, "crate-a", "Cargo.toml"), "[package]\nname = \"a\""); - writeFileSync(join(dir, "crate-b", "Cargo.toml"), "[package]\nname = \"b\""); - - const members = await resolveWorkspaceMembers(["crate-a", "crate-b"], dir, "Cargo.toml"); - expect(members).toHaveLength(2); - expect(members[0].memberPath).toBe("./crate-a"); - expect(members[1].memberPath).toBe("./crate-b"); - expect(members[0].content).toContain("name = \"a\""); + expect(filterDepsForMember(allDeps, ".")).toEqual({ + [`dependencies${fieldSep}serde`]: allDeps[`dependencies${fieldSep}serde`], + [`workspace.dependencies${fieldSep}serde_json`]: allDeps[`workspace.dependencies${fieldSep}serde_json`], + }); + expect(filterDepsForMember(allDeps, "./app")).toEqual({ + [`dependencies${fieldSep}tokio`]: allDeps[`dependencies|./app${fieldSep}tokio`], + [`dev-dependencies${fieldSep}rand`]: allDeps[`dev-dependencies|./app${fieldSep}rand`], + }); }); -test("resolveWorkspaceMembers glob patterns", async ({expect = globalExpect}: any = {}) => { - const dir = mkdtempSync(join(tmpdir(), "ws-test-")); - mkdirSync(join(dir, "packages", "foo"), {recursive: true}); - mkdirSync(join(dir, "packages", "bar"), {recursive: true}); - writeFileSync(join(dir, "packages", "foo", "package.json"), "{\"name\": \"foo\"}"); - writeFileSync(join(dir, "packages", "bar", "package.json"), "{\"name\": \"bar\"}"); +test("resolveWorkspaceMembers resolves literals, globs and exclusions", async () => { + const literalDir = makeWorkspace({ + "crate-a/Cargo.toml": "[package]\nname = \"a\"", + "crate-b/Cargo.toml": "[package]\nname = \"b\"", + }); + const literalMembers = await resolveWorkspaceMembers(["crate-a", "crate-b"], literalDir, "Cargo.toml"); + expect(literalMembers.map(({memberPath}) => memberPath)).toEqual(["./crate-a", "./crate-b"]); + expect(literalMembers[0].content).toContain("name = \"a\""); - const members = await resolveWorkspaceMembers(["packages/*"], dir, "package.json"); - expect(members).toHaveLength(2); - const paths = members.map(m => m.memberPath).sort(); - expect(paths).toEqual(["./packages/bar", "./packages/foo"]); + const globDir = makeWorkspace({ + "packages/foo/package.json": "{\"name\": \"foo\"}", + "packages/bar/package.json": "{\"name\": \"bar\"}", + "packages/README.md": "workspace notes", + }); + const memberPaths = async (patterns: Array) => + (await resolveWorkspaceMembers(patterns, globDir, "package.json")).map(({memberPath}) => memberPath).sort(); + expect(await memberPaths(["packages/*"])).toEqual(["./packages/bar", "./packages/foo"]); + mkdirSync(join(globDir, "packages/internal")); + writeFileSync(join(globDir, "packages/internal/package.json"), "{\"name\": \"internal\"}"); + expect(await memberPaths(["packages/*", "!packages/internal"])).toEqual(["./packages/bar", "./packages/foo"]); }); -test("resolveWorkspaceMembers excludes negated patterns", async ({expect = globalExpect}: any = {}) => { - const dir = mkdtempSync(join(tmpdir(), "ws-test-")); - mkdirSync(join(dir, "packages", "foo"), {recursive: true}); - mkdirSync(join(dir, "packages", "bar"), {recursive: true}); - mkdirSync(join(dir, "packages", "internal"), {recursive: true}); - writeFileSync(join(dir, "packages", "foo", "package.json"), "{\"name\": \"foo\"}"); - writeFileSync(join(dir, "packages", "bar", "package.json"), "{\"name\": \"bar\"}"); - writeFileSync(join(dir, "packages", "internal", "package.json"), "{\"name\": \"internal\"}"); - - const members = await resolveWorkspaceMembers(["packages/*", "!packages/internal"], dir, "package.json"); - const paths = members.map(m => m.memberPath).sort(); - expect(paths).toEqual(["./packages/bar", "./packages/foo"]); +test("resolveWorkspaceMembers skips missing", async () => { + const dir = makeWorkspace(); + expect(await resolveWorkspaceMembers(["nonexistent"], dir, "Cargo.toml")).toEqual([]); + mkdirSync(join(dir, "member")); + await expect(resolveWorkspaceMembers(["member"], dir, ".")).rejects.toMatchObject({code: "EISDIR"}); }); -test("resolveWorkspaceMembers skips missing", async ({expect = globalExpect}: any = {}) => { - const dir = mkdtempSync(join(tmpdir(), "ws-test-")); - const members = await resolveWorkspaceMembers(["nonexistent"], dir, "Cargo.toml"); - expect(members).toHaveLength(0); +test("resolveWorkspaceMembers rejects traversal and escaping symlinks", async () => { + const dir = makeWorkspace(); + const outside = makeWorkspace({"package.json": "{\"name\": \"outside\"}"}); + symlinkSync(outside, join(dir, "linked"), "dir"); + mkdirSync(join(dir, "manifest-link")); + symlinkSync(join(outside, "package.json"), join(dir, "manifest-link", "package.json")); + expect(await resolveWorkspaceMembers([relative(dir, outside), "linked", "manifest-link"], dir, "package.json")).toEqual([]); }); -test("parsePnpmWorkspace", ({expect = globalExpect}: any = {}) => { +test("parsePnpmWorkspace", () => { expect(parsePnpmWorkspace("packages:\n - \"packages/*\"\n - 'apps/*'\n")).toEqual(["packages/*", "apps/*"]); expect(parsePnpmWorkspace("packages:\n - packages/*\n")).toEqual(["packages/*"]); + expect(parsePnpmWorkspace("packages: [packages/*, apps/*]\n")).toEqual(["packages/*", "apps/*"]); + expect(parsePnpmWorkspace('packages:\n - "packages/with space"\n')).toEqual(["packages/with space"]); expect(parsePnpmWorkspace("")).toEqual([]); expect(parsePnpmWorkspace("packages:\n # comment\n - libs/*\nnodeLinker: hoisted\n")).toEqual(["libs/*"]); }); -const catalogYaml = [ - "packages:", - " - \"packages/*\"", - "", - "catalog:", - " react: ^18.0.0", - " 'prismjs': \"^1.0.0\" # pinned", - "", - "catalogs:", - " tools:", - " typescript: ^4.9.5", - " legacy:", - " react: ^17.0.0", - "", -].join("\n"); +test("parse pnpm registry config", () => { + expect(parsePnpmRegistryConfig("registry: https://pnpm.test\nregistries: {'@foo': https://foo.pnpm.test, default: https://default.pnpm.test}\n")).toEqual({ + registry: "https://pnpm.test", + registries: {"@foo": "https://foo.pnpm.test", default: "https://default.pnpm.test"}, + }); +}); + +const catalogYaml = `packages: + - "packages/*" + +catalog: + react: ^18.0.0 + 'prismjs': "^1.0.0" # pinned -test("pnpmCatalogEntries", ({expect = globalExpect}: any = {}) => { +catalogs: + tools: + typescript: ^4.9.5 + legacy: + react: ^17.0.0 +`; + +test("pnpmCatalogEntries", () => { expect(Array.from(pnpmCatalogEntries(catalogYaml), ({type, name, value}) => [type, name, value])).toEqual([ ["catalog", "react", "^18.0.0"], ["catalog", "prismjs", "^1.0.0"], @@ -116,14 +119,20 @@ test("pnpmCatalogEntries", ({expect = globalExpect}: any = {}) => { ["catalogs.legacy", "react", "^17.0.0"], ]); expect(Array.from(pnpmCatalogEntries("packages:\n - \"packages/*\"\n"))).toEqual([]); + expect(Array.from(pnpmCatalogEntries("catalog: {react: ^18, vue: '~3'}\n"), ({type, name, value}) => [type, name, value])).toEqual([ + ["catalog", "react", "^18"], + ["catalog", "vue", "~3"], + ]); + expect(Array.from(pnpmCatalogEntries("catalogs: {web: {react: ^18}}\n"), ({type, name, value}) => [type, name, value])).toEqual([ + ["catalogs.web", "react", "^18"], + ]); }); -test("updatePnpmWorkspace", ({expect = globalExpect}: any = {}) => { +test("updatePnpmWorkspace", () => { const updated = updatePnpmWorkspace(catalogYaml, { [`catalog${fieldSep}react`]: {old: "^18.0.0", new: "^19.1.0"}, [`catalog${fieldSep}prismjs`]: {old: "^1.0.0", new: "^1.30.0"}, [`catalogs.tools${fieldSep}typescript`]: {old: "^4.9.5", new: "^5.9.2"}, - // stale, the file no longer holds this value [`catalogs.legacy${fieldSep}react`]: {old: "^16.0.0", new: "^19.1.0"}, }); expect(updated).toContain(" react: ^19.1.0\n"); @@ -131,4 +140,8 @@ test("updatePnpmWorkspace", ({expect = globalExpect}: any = {}) => { expect(updated).toContain(" typescript: ^5.9.2\n"); expect(updated).toContain(" react: ^17.0.0\n"); expect(updatePnpmWorkspace(catalogYaml, {})).toBe(catalogYaml); + expect(updatePnpmWorkspace("catalog: {react: ^18, vue: ~3}\n", { + [`catalog${fieldSep}react`]: {old: "^18", new: "^19.1.0"}, + [`catalog${fieldSep}vue`]: {old: "~3", new: "~4.2"}, + })).toBe("catalog: {react: ^19.1.0, vue: ~4.2}\n"); }); diff --git a/utils/workspace.ts b/utils/workspace.ts index 42fbe5f..57d3cec 100644 --- a/utils/workspace.ts +++ b/utils/workspace.ts @@ -1,8 +1,8 @@ -import {join, relative, resolve} from "node:path"; -import {globSync} from "node:fs"; -import {readFile} from "node:fs/promises"; +import {dirname, isAbsolute, join, relative, resolve, sep} from "node:path"; +import {globSync, readFileSync} from "node:fs"; +import {readFile, realpath} from "node:fs/promises"; import {type Deps, fieldSep} from "../modes/shared.ts"; -import {pMap} from "./utils.ts"; +import {getOrSet, pMap, pushTo} from "./utils.ts"; export type WorkspaceMember = { absPath: string, @@ -15,61 +15,72 @@ export function baseType(type: string): string { return idx === -1 ? type : type.slice(0, idx); } +const depsByMember = new WeakMap>>(); + export function filterDepsForMember(allDeps: Deps, memberPath: string): Deps { - const expectedSuffix = memberPath === "." ? "" : `|${memberPath}`; - const result: Deps = {}; - for (const [key, dep] of Object.entries(allDeps)) { - const [type, name] = key.split(fieldSep); - const base = baseType(type); - if (type === `${base}${expectedSuffix}`) { - result[`${base}${fieldSep}${name}`] = dep; + const byMember = getOrSet(depsByMember, allDeps, () => { + const result = new Map>(); + for (const [key, dep] of Object.entries(allDeps)) { + const [type, name] = key.split(fieldSep); + const separator = type.indexOf("|"); + const path = separator === -1 ? "." : type.slice(separator + 1); + pushTo(result, path, [`${baseType(type)}${fieldSep}${name}`, dep]); } - } - return result; + return result; + }); + return Object.fromEntries(byMember.get(memberPath) ?? []); } const globChars = /[*?{[]/; +function globDirectories(pattern: string, cwd: string): Array { + return globSync(pattern, {cwd, withFileTypes: true}) + .filter(entry => entry.isDirectory()) + .map(entry => resolve(entry.parentPath, entry.name)); +} + export async function resolveWorkspaceMembers(patterns: string[], workspaceDir: string, manifestFilename: string, concurrency = 32): Promise { - const includes = patterns.filter(pattern => !pattern.startsWith("!")); - const excludes = patterns.filter(pattern => pattern.startsWith("!")).map(pattern => pattern.slice(1)); - const excluded = new Set(excludes.flatMap(pattern => globSync(pattern, {cwd: workspaceDir})).map(dir => dir.replace(/\\/g, "/"))); + const workspaceRoot = await realpath(workspaceDir); + const excluded = new Set(patterns.filter(pattern => pattern.startsWith("!")) + .flatMap(pattern => globDirectories(pattern.slice(1), workspaceDir)) + .map(dir => relative(workspaceDir, dir).replace(/\\/g, "/"))); const seen = new Set(); - const candidates: Array<{absPath: string, memberPath: string}> = []; - for (const pattern of includes) { + const candidates: Array<{dir: string, memberPath: string}> = []; + for (const pattern of patterns) { + if (pattern.startsWith("!")) continue; const dirs = globChars.test(pattern) ? - globSync(pattern, {cwd: workspaceDir}).map(dir => resolve(join(workspaceDir, dir))) : + globDirectories(pattern, workspaceDir) : [resolve(join(workspaceDir, pattern))]; for (const dir of dirs) { const rel = relative(workspaceDir, dir).replace(/\\/g, "/"); if (excluded.has(rel)) continue; - const absPath = join(dir, manifestFilename); - if (seen.has(absPath)) continue; - seen.add(absPath); - candidates.push({absPath, memberPath: `./${rel}`}); + if (seen.has(dir)) continue; + seen.add(dir); + candidates.push({dir, memberPath: `./${rel}`}); } } - const reads = await pMap(candidates, async ({absPath, memberPath}) => { + const reads = await pMap(candidates, async ({dir, memberPath}) => { try { + const absPath = await realpath(join(dir, manifestFilename)); + const rel = relative(workspaceRoot, absPath); + if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null; return {absPath, content: await readFile(absPath, "utf8"), memberPath}; - } catch { - return null; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; } }, {concurrency}); return reads.filter((m): m is WorkspaceMember => m !== null); } export type PnpmCatalogEntry = { - /** `catalog` for the default catalog, `catalogs.` for a named one */ type: string, name: string, value: string, lineIndex: number, - /** index of `value` inside its line, quotes excluded, so a rewrite touches the value alone */ valueIndex: number, }; -// The value capture runs to the line's end, which is what makes its start index recoverable. const yamlPairRe = /^(\s*)(?:"([^"]*)"|'([^']*)'|([^\s#][^:#]*?))\s*:(?:\s+(.*))?$/; const yamlCommentRe = /\s#/; @@ -88,8 +99,127 @@ function parseYamlPair(line: string): {indent: number, key: string, value: strin return {indent: indent.length, key: doubleQuoted ?? singleQuoted ?? plain, value, valueIndex}; } -// Only block style is read, as `packages:` is. A member's `catalog:`/`catalog:` value only -// names a catalog, so the range lives here and is reported and rewritten here alone. +type FlowPair = {key: string, value: string, valueIndex: number}; + +type FlowPart = {colon: number, start: number, text: string}; + +function flowParts(content: string): Array | null { + const parts: Array = []; + let start = 1; + let colon = -1; + let depth = 0; + let quote = ""; + for (let index = 1; index < content.length - 1; index++) { + const char = content[index]; + if (quote) { + if (char === "\\" && quote === '"') index++; + else if (char === quote) { + if (quote === "'" && content[index + 1] === "'") index++; + else quote = ""; + } + } else if (char === '"' || char === "'") { + quote = char; + } else if (char === "{" || char === "[") { + depth++; + } else if (char === "}" || char === "]") { + if (depth === 0) return null; + depth--; + } else if (char === ":" && depth === 0 && colon === -1) { + colon = index - start; + } else if (char === "," && depth === 0) { + parts.push({colon, start, text: content.slice(start, index)}); + start = index + 1; + colon = -1; + } + } + if (quote || depth !== 0) return null; + parts.push({colon, start, text: content.slice(start, -1)}); + return parts; +} + +function yamlScalar(content: string): {value: string, valueIndex: number} | null { + const leading = content.length - content.trimStart().length; + const trimmed = content.trim(); + if (!trimmed) return null; + const quote = trimmed[0]; + if (quote === '"' || quote === "'") { + if (!trimmed.endsWith(quote)) return null; + return {value: trimmed.slice(1, -1), valueIndex: leading + 1}; + } + const commentIndex = trimmed.search(/\s#/); + return {value: (commentIndex === -1 ? trimmed : trimmed.slice(0, commentIndex)).trimEnd(), valueIndex: leading}; +} + +function flowPairs(content: string, contentIndex: number): FlowPair[] | null { + if (!content.startsWith("{") || !content.endsWith("}")) return null; + const result: FlowPair[] = []; + for (const part of flowParts(content) ?? []) { + if (part.colon === -1) return null; + const key = yamlScalar(part.text.slice(0, part.colon)); + const value = yamlScalar(part.text.slice(part.colon + 1)); + if (!key || !value) return null; + result.push({ + key: key.value, + value: value.value, + valueIndex: contentIndex + part.start + part.colon + 1 + value.valueIndex, + }); + } + return result; +} + +export type NpmRegistryConfig = { + registry?: string, + registries: Record, +}; + +export function parsePnpmRegistryConfig(content: string): NpmRegistryConfig { + let registry: string | undefined; + const registries: Record = {}; + let inRegistries = false; + for (const line of content.split(/\r?\n/)) { + const pair = parseYamlPair(line); + if (!pair) continue; + if (pair.indent === 0) { + inRegistries = pair.key === "registries"; + if (pair.key === "registry" && pair.value) registry = pair.value; + if (inRegistries) { + for (const entry of flowPairs(pair.value, pair.valueIndex) ?? []) registries[entry.key] = entry.value; + } + } else if (inRegistries && pair.value) { + registries[pair.key] = pair.value; + } + } + return { + registry, + registries: Object.fromEntries(Object.entries(registries).filter(([, url]) => !url.includes("${"))), + }; +} + +function readConfigUp(filename: string, startDir: string): string | null { + for (let dir = resolve(startDir); ; dir = dirname(dir)) { + try { + return readFileSync(join(dir, filename), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (dirname(dir) === dir) return null; + } +} + +const nativeRegistryCache = new Map(); + +export function resolveNativeNpmRegistry(name: string, startDir: string): string | null { + let config = nativeRegistryCache.get(startDir); + if (!config) { + config = parsePnpmRegistryConfig(readConfigUp("pnpm-workspace.yaml", startDir) ?? ""); + nativeRegistryCache.set(startDir, config); + } + for (const [scope, url] of Object.entries(config.registries)) { + if (scope !== "default" && name.startsWith(`${scope}/`)) return url; + } + return config.registries.default || config.registry || null; +} + export function* pnpmCatalogEntries(content: string): Generator { let section = ""; let catalogName = ""; @@ -102,12 +232,24 @@ export function* pnpmCatalogEntries(content: string): Generator b.lineIndex - a.lineIndex || b.valueIndex - a.valueIndex)) { const dep = deps[`${type}${fieldSep}${name}`]; if (!dep || (dep.oldOrig || dep.old) !== value) continue; const line = lines[lineIndex]; @@ -131,10 +273,16 @@ export function updatePnpmWorkspace(content: string, deps: Deps): string { export function parsePnpmWorkspace(content: string): string[] { const patterns: string[] = []; - const lines = content.split(/\r?\n/); let inPackages = false; - for (const line of lines) { - if (/^packages\s*:/.test(line)) { + for (const line of content.split(/\r?\n/)) { + const pair = parseYamlPair(line); + if (pair?.indent === 0 && pair.key === "packages") { + if (pair.value.startsWith("[") && pair.value.endsWith("]")) { + for (const part of flowParts(pair.value) ?? []) { + const scalar = yamlScalar(part.text); + if (scalar) patterns.push(scalar.value); + } + } inPackages = true; continue; } @@ -142,8 +290,8 @@ export function parsePnpmWorkspace(content: string): string[] { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith("#")) continue; if (!trimmed.startsWith("-")) break; - const match = /^\s*-\s+['"]?([^'"#\s]+)['"]?/.exec(line); - if (match) patterns.push(match[1]); + const scalar = yamlScalar(trimmed.slice(1)); + if (scalar) patterns.push(scalar.value); } } return patterns;