diff --git a/.ado/pipelines/azure-pipelines-build.yml b/.ado/pipelines/azure-pipelines-build.yml new file mode 100644 index 00000000000..ebdd56922a7 --- /dev/null +++ b/.ado/pipelines/azure-pipelines-build.yml @@ -0,0 +1,114 @@ +# Register this YAML in Azure DevOps with the exact pipeline definition name +# "FAST CD Build". .ado/pipelines/azure-pipelines-cd.yml references that +# definition name as a pipeline resource, watching specifically for the +# `BuildArtifacts` stage (see the comment above that stage for why +# `ValidateArtifacts` is a separate stage rather than a condition on the +# same stage). +trigger: + branches: + include: + - main +pr: none + +parameters: +- name: validationMode + displayName: Rebuild every publishable workspace for artifact validation + type: string + default: 'false' + values: + - 'false' + - 'true' + +stages: +- stage: PrepareRelease + displayName: Prepare release + jobs: + - job: SelectRelease + displayName: Select pending package releases + pool: + vmImage: ubuntu-latest + steps: + - checkout: self + # `pack-pending-releases.mjs --check-only` calls `git ls-remote origin` + # (via `gitTagExistsOnRemote`) to decide which workspaces are pending. + persistCredentials: true + + - task: UseNode@1 + displayName: Install Node.js + inputs: + version: "22.x" + + - task: Bash@3 + displayName: Resolve pending package releases + name: release + inputs: + targetType: inline + # FAST is multi-package: unlike a single workspace-wide release + # version, each publishable npm workspace gets its own + # `${name}_v${version}` tag, so "pending" is a per-package list + # rather than one selected version. `pack-pending-releases.mjs + # --check-only` walks the workspaces tree (no `npm ci` required) + # and reports whether any workspace's tag is missing from `origin`. + script: | + set -euo pipefail + node build/scripts/pack-pending-releases.mjs --check-only + env: + ALLOW_EXISTING_RELEASE: ${{ parameters.validationMode }} + +# Two stage NAMES (not one stage gated by a `validationMode` condition) are +# used deliberately. `FAST CD`'s `releaseBuild` pipeline resource trigger +# fires when a stage literally named `BuildArtifacts` completes on `main` — +# it has no way to also check which queue-time parameter values that run +# used. If validation runs reused the `BuildArtifacts` name, queuing this +# pipeline with `validationMode: true` (e.g. to test the artifact contract +# after changing this file) would auto-trigger a real `FAST CD` run, which +# would then fail loudly on the `validationMode` mismatch check in +# `read-release-manifest.mjs` — a confusing red run for what was only ever +# meant to be a local contract check. Giving the validation path its own +# stage name (`ValidateArtifacts`) means `FAST CD` is structurally unable to +# hear about it, with no reliance on that downstream check as the only +# safety net. (A skipped stage does not fire a pipeline resource trigger +# either — Azure Pipelines only triggers on `stages` that actually +# complete — so a `PrepareRelease` run with nothing pending, which skips +# `BuildArtifacts` entirely, is equally safe by that same default.) +- ${{ if eq(parameters.validationMode, 'false') }}: + - stage: BuildArtifacts + displayName: Build release artifacts + dependsOn: PrepareRelease + condition: | + and( + succeeded(), + eq(dependencies.PrepareRelease.outputs['SelectRelease.release.shouldBuild'], 'true') + ) + pool: + vmImage: ubuntu-latest + variables: + npm_config_cache: $(Pipeline.Workspace)/.npm + jobs: + - job: PackReleases + displayName: Build and pack pending releases + steps: + - template: templates/pack-release-steps.yml + parameters: + validationMode: ${{ parameters.validationMode }} + +- ${{ if eq(parameters.validationMode, 'true') }}: + - stage: ValidateArtifacts + displayName: Validate release artifacts (no CD trigger) + dependsOn: PrepareRelease + condition: | + and( + succeeded(), + eq(dependencies.PrepareRelease.outputs['SelectRelease.release.shouldBuild'], 'true') + ) + pool: + vmImage: ubuntu-latest + variables: + npm_config_cache: $(Pipeline.Workspace)/.npm + jobs: + - job: PackReleases + displayName: Build and pack every workspace for validation + steps: + - template: templates/pack-release-steps.yml + parameters: + validationMode: ${{ parameters.validationMode }} diff --git a/.ado/pipelines/azure-pipelines-cd.yml b/.ado/pipelines/azure-pipelines-cd.yml new file mode 100644 index 00000000000..1d654094aa8 --- /dev/null +++ b/.ado/pipelines/azure-pipelines-cd.yml @@ -0,0 +1,416 @@ +# Register this YAML in Azure DevOps with the exact pipeline definition name +# "FAST CD". +trigger: none +pr: none + +parameters: +- name: validationMode + displayName: Generate signed artifacts without external publication + type: string + default: 'false' + values: + - 'false' + - 'true' + +resources: + pipelines: + - pipeline: releaseBuild + source: FAST CD Build + # Manual Run Pipeline selection may intentionally override this default. + branch: refs/heads/main + trigger: + branches: + include: + - main + # Only a stage literally named `BuildArtifacts` fires this trigger, and + # only when that stage actually completes (a skipped stage — e.g. a + # `FAST CD Build` run with nothing pending — never fires it either). + # `azure-pipelines-build.yml`'s `validationMode: true` path runs under + # the separate `ValidateArtifacts` stage name specifically so those + # validation-only runs can never reach this trigger and pull a real + # `FAST CD` run into an unwanted (and confusingly red) execution. See + # the comment above that stage in `azure-pipelines-build.yml` for the + # full rationale. + stages: + - BuildArtifacts + repositories: + - repository: fastPipelines + type: git + name: open-source/FASTPipelineTemplates + ref: main + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + # The pipeline extends the 1ES PT, which injects SDL and compliance tasks. + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + parameters: + pool: + name: OneESPool + image: HostedPoolLinuxImage + os: linux + sdl: + sourceAnalysisPool: + name: OneESPool + image: HostedPoolWindowsImage + os: windows + settings: + networkIsolationPolicy: Permissive + stages: + - stage: SignArtifacts + displayName: Sign release artifacts + jobs: + - job: Sign + displayName: Sign release artifacts + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.SourcesDirectory)/publish_artifacts_npm' + artifactName: unsigned_npm_packages + displayName: Publish unsigned npm packages + - output: pipelineArtifact + targetPath: '$(Build.SourcesDirectory)/publish_artifacts_crates' + artifactName: unsigned_crate_packages + displayName: Publish unsigned crate packages + steps: + - checkout: self + clean: true + + - download: releaseBuild + artifact: release-metadata + displayName: Download release metadata + + - task: UseNode@1 + displayName: Install Node.js + inputs: + version: "22.x" + + - task: Bash@3 + name: release + displayName: Read release metadata + inputs: + targetType: inline + # FAST has no single workspace-wide release version, so the + # manifest (rather than a single `release-version.txt`) is the + # source of truth for which packages are pending. This emits + # one `NeedsRelease` / `ReleaseTag` / + # `ReleaseVersion` output per currently-publishable + # workspace, plus a shared `releaseCommit`. + script: | + set -euo pipefail + metadata_dir="${METADATA_INPUT_DIR:?METADATA_INPUT_DIR is required}" + validation_mode=$(<"$metadata_dir/validation-mode.txt") + + if [[ "$validation_mode" != "$EXPECTED_VALIDATION_MODE" ]]; then + echo "##vso[task.logissue type=error]Release metadata validation mode does not match this pipeline's validationMode parameter." + exit 1 + fi + + node build/scripts/read-release-manifest.mjs "$metadata_dir/release-manifest.json" + env: + METADATA_INPUT_DIR: $(Pipeline.Workspace)/releaseBuild/release-metadata + EXPECTED_VALIDATION_MODE: ${{ parameters.validationMode }} + + - download: releaseBuild + artifact: unsigned_npm_packages + displayName: Download unsigned npm packages + + - download: releaseBuild + artifact: unsigned_crate_packages + displayName: Download unsigned crate packages + + - task: CopyFiles@2 + displayName: Stage npm packages + inputs: + SourceFolder: $(Pipeline.Workspace)/releaseBuild/unsigned_npm_packages + Contents: '**' + TargetFolder: $(Build.SourcesDirectory)/publish_artifacts_npm + CleanTargetFolder: true + + - task: CopyFiles@2 + displayName: Stage crate packages + inputs: + SourceFolder: $(Pipeline.Workspace)/releaseBuild/unsigned_crate_packages + Contents: '**' + TargetFolder: $(Build.SourcesDirectory)/publish_artifacts_crates + CleanTargetFolder: true + + # FAST has no NuGet or native/standalone assets to ESRP-sign today, + # so this stage is a no-op for those asset types — the template + # still runs so npm/crate assets flow through the same + # SDL-compliant signing and publication path as any future + # signable asset type. + # + # Risk: this repository cannot access `FASTPipelineTemplates` to + # confirm `FAST.Sign.PipelineTemplate.yml`'s exact parameter + # contract (it is an internal Azure DevOps repository, not + # accessible from here). This zero-parameter invocation matches the + # only pattern this pipeline has ever used for it; if the template + # requires explicit inputs, this call site needs a follow-up. + - template: FAST.Sign.PipelineTemplate.yml@fastPipelines + + - ${{ if eq(parameters.validationMode, 'false') }}: + # A single `PublishRelease` stage (rather than a separate tagging + # stage that runs before publishing) with two ordered jobs: + # + # 1. `Publish` — the actual `npm publish` / `cargo publish`. + # 2. `PublishGitHub` (`dependsOn: Publish`) — creates the git tag + # and GitHub release per package, LAST. + # + # This ordering is deliberate. `pack-pending-releases.mjs` (run by the + # `FAST CD Build` pipeline) treats a workspace as "pending" purely + # based on whether its `${name}_v${version}` tag exists on `origin` — + # so whichever action creates that tag is also, structurally, the + # action that makes a package invisible to every future release-prep + # run. Creating the tag *before* the npm/crates publish step (as an + # earlier version of this pipeline did, via a dedicated `TagRelease` + # stage) meant a publish failure left the tag behind anyway: the next + # `FAST CD Build` run would see the tag, treat the package as already + # released, and never repack or retry it — permanently stranding the + # publish for that package with no automatic recovery path. Creating + # the tag only *after* `Publish` succeeds means a publish failure + # never leaves that tag behind, so the package stays "pending" and the + # very next `FAST CD Build` run retries it automatically. + # + # `GitHubRelease@1` (`tagSource: userSpecifiedTag`, `action: create`) + # creates the tag itself as part of creating the release when the tag + # doesn't already exist, so no separate pre-publish tagging step is + # needed at all. + # + # Residual risk: if `Publish` succeeds but `PublishGitHub` fails for a + # package (rather than the whole job failing outright, which + # `releaseTagCheck` below makes safe to just rerun), that package's + # tag still won't exist, so the next `FAST CD Build` run will try to + # republish it — and since it was already published, `npm publish` / + # `cargo publish` fails loudly (the existing, documented idempotency + # behavior for republishing an already-published version). That + # bounded, loud failure requires a maintainer to manually create the + # missing tag/GitHub release (the npm/crates side is already done); + # it is never silent and never leaves the package in an unrecoverable + # state. + - stage: PublishRelease + displayName: Publish npm, crates.io, and GitHub releases + dependsOn: SignArtifacts + variables: + releaseCommit: $[ stageDependencies.SignArtifacts.Sign.outputs['release.releaseCommit'] ] + # When adding a new publishable package, add its package-specific + # variables here and a matching GitHubRelease@1 task below. See + # .github/workflows/README.md > Continuous Deployment. + fastBuildNeedsRelease: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastBuildNeedsRelease'] ] + fastBuildReleaseTag: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastBuildReleaseTag'] ] + fastBuildReleaseVersion: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastBuildReleaseVersion'] ] + fastElementNeedsRelease: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastElementNeedsRelease'] ] + fastElementReleaseTag: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastElementReleaseTag'] ] + fastElementReleaseVersion: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastElementReleaseVersion'] ] + fastRouterNeedsRelease: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastRouterNeedsRelease'] ] + fastRouterReleaseTag: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastRouterReleaseTag'] ] + fastRouterReleaseVersion: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastRouterReleaseVersion'] ] + fastTestHarnessNeedsRelease: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastTestHarnessNeedsRelease'] ] + fastTestHarnessReleaseTag: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastTestHarnessReleaseTag'] ] + fastTestHarnessReleaseVersion: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastTestHarnessReleaseVersion'] ] + jobs: + - job: Publish + displayName: Publish npm and crates.io packages + steps: + - checkout: self + clean: true + + - task: DownloadPipelineArtifact@2 + displayName: Download npm packages + inputs: + buildType: current + artifactName: unsigned_npm_packages + targetPath: $(Build.SourcesDirectory)/publish_artifacts_npm + + - task: DownloadPipelineArtifact@2 + displayName: Download crate packages + inputs: + buildType: current + artifactName: unsigned_crate_packages + targetPath: $(Build.SourcesDirectory)/publish_artifacts_crates + + - task: Bash@3 + displayName: Remove empty artifact directories + inputs: + targetType: inline + # `pack-pending-releases.mjs` writes a `.no-crates-packed` + # placeholder file into `publish_artifacts_crates` when this + # release batch has no crate assets, purely so publishing and + # downloading that (otherwise empty) pipeline artifact along + # the way is robust. Strip any directory down to nothing here + # — after the placeholder (and any other non-package file) + # is discounted — so the release template sees a genuinely + # *absent* directory for whichever asset type this batch + # doesn't have, matching its documented "skips absent + # directories" contract precisely instead of an empty one. + script: | + set -euo pipefail + for dir in "$(Build.SourcesDirectory)/publish_artifacts_npm" "$(Build.SourcesDirectory)/publish_artifacts_crates"; do + if [ -d "$dir" ]; then + real_file=$(find "$dir" -mindepth 1 -type f ! -name '.no-crates-packed' -print -quit) + if [ -z "$real_file" ]; then + echo "Removing empty artifact directory: $dir" + rm -rf "$dir" + fi + fi + done + + - script: | + printf '\ntag=latest\n' >> .npmrc + displayName: "Configure npm publish dist-tag" + + # Both npm and crate assets are handed to a single invocation of + # the release template — rather than two parallel jobs each + # calling it separately with only one asset type downloaded — + # so one destination succeeding while the other fails can never + # leave a package half-published without the whole job failing as + # one unit. The template skips whichever of + # `publish_artifacts_npm` / `publish_artifacts_crates` is absent + # (removed above when empty). + - template: FAST.Release.PipelineTemplate.yml@fastPipelines + + - job: PublishGitHub + displayName: Publish GitHub Releases + dependsOn: Publish + condition: succeeded() + steps: + - checkout: self + clean: true + # `check-release-tags.mjs` calls `git ls-remote origin` to + # freshly check tag existence. + persistCredentials: true + + - download: releaseBuild + artifact: release-metadata + displayName: Download release metadata + + - task: DownloadPipelineArtifact@2 + displayName: Download npm packages + inputs: + buildType: current + artifactName: unsigned_npm_packages + targetPath: $(Build.SourcesDirectory)/publish_artifacts_npm + + - task: DownloadPipelineArtifact@2 + displayName: Download crate packages + inputs: + buildType: current + artifactName: unsigned_crate_packages + targetPath: $(Build.SourcesDirectory)/publish_artifacts_crates + + - task: UseNode@1 + displayName: Install Node.js + inputs: + version: "22.x" + + # Fresh, per-package check (independent of the `NeedsRelease` + # variables above, which only reflect what `SignArtifacts` saw + # before `Publish` ran) so rerunning this job after a partial + # failure is safe: `GitHubRelease@1` below is skipped for any + # package whose tag was already created by an earlier attempt of + # this same job, instead of failing trying to recreate a release + # that already exists. + - task: Bash@3 + name: releaseTagCheck + displayName: Check for already-created release tags + inputs: + targetType: inline + script: | + set -euo pipefail + node build/scripts/check-release-tags.mjs "$METADATA_INPUT_DIR/release-manifest.json" + env: + METADATA_INPUT_DIR: $(Pipeline.Workspace)/releaseBuild/release-metadata + + # `tagSource: userSpecifiedTag` + `action: create` creates the + # `tag` itself (at `target`) when it doesn't already exist yet, so + # this is the only place the release tag gets created — see the + # `PublishRelease` stage comment above for why that ordering + # matters, and the `releaseTagCheck` comment above for why reruns + # of this job are safe. + - task: GitHubRelease@1 + displayName: "Create @microsoft/fast-build GitHub Release" + condition: and(succeeded(), eq(variables['fastBuildNeedsRelease'], 'true'), eq(variables['releaseTagCheck.fastBuildReleaseTagExists'], 'false')) + inputs: + gitHubConnection: fast + repositoryName: microsoft/fast + action: create + target: $(releaseCommit) + tagSource: userSpecifiedTag + tag: $(fastBuildReleaseTag) + title: $(fastBuildReleaseTag) + releaseNotesSource: inline + releaseNotesInline: "Automated FAST release for @microsoft/fast-build@$(fastBuildReleaseVersion)." + addChangeLog: false + assets: | + $(Build.SourcesDirectory)/publish_artifacts_npm/microsoft-fast-build-$(fastBuildReleaseVersion).tgz + $(Build.SourcesDirectory)/publish_artifacts_crates/microsoft-fast-build-$(fastBuildReleaseVersion).crate + $(Build.SourcesDirectory)/publish_artifacts_crates/microsoft-fast-convert-$(fastBuildReleaseVersion).crate + assetUploadMode: replace + isDraft: false + isPreRelease: false + makeLatest: legacy + + - task: GitHubRelease@1 + displayName: "Create @microsoft/fast-element GitHub Release" + condition: and(succeeded(), eq(variables['fastElementNeedsRelease'], 'true'), eq(variables['releaseTagCheck.fastElementReleaseTagExists'], 'false')) + inputs: + gitHubConnection: fast + repositoryName: microsoft/fast + action: create + target: $(releaseCommit) + tagSource: userSpecifiedTag + tag: $(fastElementReleaseTag) + title: $(fastElementReleaseTag) + releaseNotesSource: inline + releaseNotesInline: "Automated FAST release for @microsoft/fast-element@$(fastElementReleaseVersion)." + addChangeLog: false + assets: | + $(Build.SourcesDirectory)/publish_artifacts_npm/microsoft-fast-element-$(fastElementReleaseVersion).tgz + assetUploadMode: replace + isDraft: false + isPreRelease: false + makeLatest: legacy + + - task: GitHubRelease@1 + displayName: "Create @microsoft/fast-router GitHub Release" + condition: and(succeeded(), eq(variables['fastRouterNeedsRelease'], 'true'), eq(variables['releaseTagCheck.fastRouterReleaseTagExists'], 'false')) + inputs: + gitHubConnection: fast + repositoryName: microsoft/fast + action: create + target: $(releaseCommit) + tagSource: userSpecifiedTag + tag: $(fastRouterReleaseTag) + title: $(fastRouterReleaseTag) + releaseNotesSource: inline + releaseNotesInline: "Automated FAST release for @microsoft/fast-router@$(fastRouterReleaseVersion)." + addChangeLog: false + assets: | + $(Build.SourcesDirectory)/publish_artifacts_npm/microsoft-fast-router-$(fastRouterReleaseVersion).tgz + assetUploadMode: replace + isDraft: false + isPreRelease: false + makeLatest: legacy + + - task: GitHubRelease@1 + displayName: "Create @microsoft/fast-test-harness GitHub Release" + condition: and(succeeded(), eq(variables['fastTestHarnessNeedsRelease'], 'true'), eq(variables['releaseTagCheck.fastTestHarnessReleaseTagExists'], 'false')) + inputs: + gitHubConnection: fast + repositoryName: microsoft/fast + action: create + target: $(releaseCommit) + tagSource: userSpecifiedTag + tag: $(fastTestHarnessReleaseTag) + title: $(fastTestHarnessReleaseTag) + releaseNotesSource: inline + releaseNotesInline: "Automated FAST release for @microsoft/fast-test-harness@$(fastTestHarnessReleaseVersion)." + addChangeLog: false + assets: | + $(Build.SourcesDirectory)/publish_artifacts_npm/microsoft-fast-test-harness-$(fastTestHarnessReleaseVersion).tgz + assetUploadMode: replace + isDraft: false + isPreRelease: false + makeLatest: legacy diff --git a/.ado/pipelines/templates/pack-release-steps.yml b/.ado/pipelines/templates/pack-release-steps.yml new file mode 100644 index 00000000000..59db62aeec2 --- /dev/null +++ b/.ado/pipelines/templates/pack-release-steps.yml @@ -0,0 +1,101 @@ +# Shared step list for building and packing pending releases, used by both +# the real `BuildArtifacts` stage and the `ValidateArtifacts` stage in +# `.ado/pipelines/azure-pipelines-build.yml`. Kept in one template so the two +# stages (which must have different names — see the comment above those +# stages for why) never drift out of sync with each other. +parameters: +- name: validationMode + type: string + default: 'false' + +steps: +- checkout: self + clean: true + # `pack-pending-releases.mjs` calls `git ls-remote origin` (via + # `gitTagExistsOnRemote`) to decide which workspaces are pending, which + # needs an authenticated remote when the repository is private/rate + # limited. + persistCredentials: true + +- task: UseNode@1 + displayName: Install Node.js + inputs: + version: "22.x" + +- task: Cache@2 + displayName: Cache npm + inputs: + key: 'npm | "$(Agent.OS)" | package-lock.json' + restoreKeys: | + npm | "$(Agent.OS)" + path: $(npm_config_cache) + +- script: | + npm ci + displayName: Install package dependencies + +- script: | + set -euo pipefail + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + export PATH="$HOME/.cargo/bin:$PATH" + echo "##vso[task.prependpath]$HOME/.cargo/bin" + rustup target add wasm32-unknown-unknown + cargo install wasm-pack + displayName: Install Rust and wasm-pack + +- script: | + npm run build + displayName: Build workspaces + +- task: Bash@3 + displayName: Pack pending release assets + inputs: + targetType: inline + # Packs the npm tarball (and any paired Rust crate archives) for + # every workspace whose release tag does not yet exist, and writes + # `publish_artifacts_meta/release-manifest.json` describing exactly + # what was packed. No GitHub release, git tag, or publish happens + # here — those are owned by the `FAST CD` pipeline. + script: | + set -euo pipefail + node build/scripts/pack-pending-releases.mjs + env: + ALLOW_EXISTING_RELEASE: ${{ parameters.validationMode }} + +- task: Bash@3 + displayName: Write release metadata + inputs: + targetType: inline + script: | + set -euo pipefail + mkdir -p "$METADATA_OUTPUT_DIR" + cp publish_artifacts_meta/release-manifest.json "$METADATA_OUTPUT_DIR/release-manifest.json" + printf '%s\n' "$VALIDATION_MODE" > "$METADATA_OUTPUT_DIR/validation-mode.txt" + env: + METADATA_OUTPUT_DIR: $(Build.ArtifactStagingDirectory)/release-metadata + VALIDATION_MODE: ${{ parameters.validationMode }} + +- task: PublishPipelineArtifact@1 + displayName: Upload unsigned npm packages + inputs: + targetPath: '$(Build.SourcesDirectory)/publish_artifacts_npm' + artifactName: unsigned_npm_packages + +- task: PublishPipelineArtifact@1 + displayName: Upload unsigned crate packages + inputs: + # `pack-pending-releases.mjs` writes a `.no-crates-packed` placeholder + # into this directory when no pending package has a paired Rust crate, + # so this publish step always has at least one file to upload — some + # agent/task combinations treat publishing a truly empty directory as a + # failure. The `PublishRelease` stage's `Publish` job strips that + # placeholder back out before invoking the release template, so an + # all-npm batch is still treated as having no crate assets to publish. + targetPath: '$(Build.SourcesDirectory)/publish_artifacts_crates' + artifactName: unsigned_crate_packages + +- task: PublishPipelineArtifact@1 + displayName: Upload release metadata + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/release-metadata' + artifactName: release-metadata diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 45e7710da60..8114095f08b 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -13,46 +13,72 @@ All CI workflows that run against pull requests are configured to skip draft PRs ## Continuous Deployment -Nightly publishing is split into two coordinated jobs so that npm credentials never leave the Azure environment. GitHub Releases are the source of truth, and `deployed/` git marker tags track which releases have already been published. +Release publishing is owned entirely by two Azure Pipelines under [`.ado/pipelines/`](../../.ado/pipelines/) — `azure-pipelines-build.yml` (registered as **`FAST CD Build`**) and `azure-pipelines-cd.yml` (registered as **`FAST CD`**) — so release credentials never leave the Azure environment. There is no GitHub Actions release workflow: GitHub Releases are created by Azure, not by CI running on `pull_request`/`push` GitHub Actions triggers. -- **`cd-github-releases.yml`** (GitHub Actions) runs nightly via cron (`0 8 * * *` UTC, ~12am PST) and on `workflow_dispatch`. It does **not** bump versions or push source changes to `main` — version bumps land on `main` through ordinary human-authored pull requests (for example, by running `npm run bump` locally and opening a PR). The cron is scheduled ~1 hour before the Azure CD pipeline (09:00 UTC) so any GitHub releases this job creates are picked up by that same night's publish run. The workflow has two jobs: - 1. **`detect`** — checks out `main` with `fetch-depth: 0` and runs [`build/scripts/create-github-releases.mjs --check-only`](../../build/scripts/create-github-releases.mjs). The script walks the workspaces tree (no `npm ci` required), computes `${name}_v${version}` for every non-private workspace, and emits `hasMissingReleases=true` if any of those git tags do not yet exist. - 2. **`release`** runs only when missing releases exist. Installs Node, the Rust toolchain (for `cargo package`), and the npm workspace dependencies, builds the repo, then runs the script in default mode. For every missing release the script: packs the npm tarball into `publish_artifacts_npm/`, packs any paired Rust crates into `publish_artifacts_crates/`, and creates the GitHub release with all assets attached via `gh release create --target `. `@microsoft/fast-build` is a bundled release: it uses one npm package, one tag, and one GitHub release containing both `microsoft-fast-build` and `microsoft-fast-convert` crate assets. The `gh` CLI creates the git tag atomically with the release, so "tag exists" and "release exists" are always the same fact — a failed release is safely retried on the next workflow run, with no orphan tag stranded behind. The script errors if a paired crate's version does not match the npm package's version — but this is purely a safety net: the [`postbump` hook in `beachball.config.js`](../../beachball.config.js) rewrites each crate's `Cargo.toml` (and the matching entry in `Cargo.lock`) automatically whenever `npm run bump` bumps the paired npm package, so they stay in sync from the same commit. -- **`azure-pipelines-cd.yml`** (Azure Pipelines) runs every night at **1am PST (`0 9 * * *` UTC)** with `always: true` so it still runs on no-op nights (it is checking external GitHub state, not repo commits). It is split into two stages so the heavy publish work is skipped on no-op nights: - 1. **`Check`** — runs [`build/scripts/download-github-releases.mjs --check-only`](../../build/scripts/download-github-releases.mjs). The script walks the current publishable workspaces, keeps only workspaces whose current `${name}_v${version}` release tag exists, filters out tags that already have a `deployed/` counterpart, and emits Azure Pipelines output variables for the overall deployment decision, npm dist-tag, and each package-specific release tag. No network calls to GitHub, npm, or crates.io are needed. - 2. **`Package`** — depends on `Check` and runs only when `needsDeployment == 'true'`. Conditional `DownloadGitHubRelease@0` tasks download undeployed release assets through the `fast` GitHub service connection, a shell step sorts them into `publish_artifacts_npm/` (`.tgz`) and `publish_artifacts_crates/` (`.crate`), configures npm to publish companion packages with the detected dist-tag, then `FAST.Release.PipelineTemplate.yml@fastPipelines` performs the actual `npm publish` / `cargo publish`. On success, the pipeline pushes a `deployed/` git marker tag for each release that was just published. The next nightly run will see those markers and skip the corresponding releases. +FAST is multi-package: unlike a single workspace-wide release version, each publishable npm workspace gets its own `${name}_v${version}` git tag (matching beachball's tag format), and "pending" is decided per package rather than for one selected version. A workspace is pending when its tag does not yet exist on `origin`. -Both scripts are thin Node.js wrappers around existing CLI tools and repository metadata — no extra npm dependencies and no custom GitHub API client. Idempotency is enforced entirely through git tags (`${name}_v${version}` on the GitHub side, `deployed/${name}_v${version}` on the Azure side), so neither side needs to talk to npm.org or crates.io to decide whether work is required. +- **`FAST CD Build`** (`azure-pipelines-build.yml`) triggers on every push to `main`. + 1. **`PrepareRelease`** runs [`build/scripts/pack-pending-releases.mjs --check-only`](../../build/scripts/pack-pending-releases.mjs), which walks the workspaces tree (no `npm ci` required), computes `${name}_v${version}` for every non-private workspace, and emits `shouldBuild=true` if any of those git tags do not yet exist on `origin`. + 2. **`BuildArtifacts`** (real releases, `validationMode: false`) or **`ValidateArtifacts`** (`validationMode: true`) runs only when `shouldBuild == 'true'`, using the shared [`templates/pack-release-steps.yml`](../../.ado/pipelines/templates/pack-release-steps.yml) step template. It installs Node, the Rust toolchain and `wasm-pack` (needed by `@microsoft/fast-build`'s WASM build step), installs npm workspace dependencies, builds the repo (`npm run build`), then runs the same script in its default mode. For every pending workspace the script packs the npm tarball into `publish_artifacts_npm/`, packs any paired Rust crates into `publish_artifacts_crates/`, and writes `publish_artifacts_meta/release-manifest.json` describing exactly what was packed (name, version, tag, and asset filenames). If a release batch has no crate assets at all, the script writes a `.no-crates-packed` placeholder file into `publish_artifacts_crates/` so publishing and downloading that otherwise-empty pipeline artifact stays robust. `@microsoft/fast-build` is a bundled release: one npm package, one tag, and both `microsoft-fast-build` and `microsoft-fast-convert` crate assets. The script errors if a paired crate's version does not match the npm package's version — a safety net, since the [`postbump` hook in `beachball.config.js`](../../beachball.config.js) keeps them in sync automatically whenever `npm run bump` runs. The stage publishes `unsigned_npm_packages`, `unsigned_crate_packages`, and `release-metadata` (the manifest plus the `validationMode` used) as pipeline artifacts. + + `BuildArtifacts` and `ValidateArtifacts` are two distinct stage *names* (chosen at compile time via `${{ if eq(parameters.validationMode, ...) }}`), not one stage gated by a runtime condition. `FAST CD`'s pipeline-resource trigger only fires when a stage literally named `BuildArtifacts` completes on `main`, so a `validationMode: true` run — which always executes under `ValidateArtifacts` instead — can never auto-trigger a real `FAST CD` run. A skipped stage (e.g. nothing pending) does not fire that trigger either, since Azure Pipelines only triggers on stages that actually complete. +- **`FAST CD`** (`azure-pipelines-cd.yml`) is an 1ES Official pipeline triggered automatically when `FAST CD Build`'s `BuildArtifacts` stage completes on `main` (it can also be queued manually). It extends `1ES.Official.PipelineTemplate.yml` and runs: + 1. **`SignArtifacts`** — downloads the build pipeline's artifacts, reads `release-manifest.json` via [`build/scripts/read-release-manifest.mjs`](../../build/scripts/read-release-manifest.mjs) (emitting one `NeedsRelease` / `ReleaseTag` / `ReleaseVersion` output per currently-publishable workspace, plus a shared `releaseCommit`), stages the npm/crate assets, and runs `FAST.Sign.PipelineTemplate.yml@fastPipelines` so every release asset flows through the same SDL-compliant path (FAST has no NuGet or native assets to ESRP-sign today, so this stage is a no-op for those asset types). + 2. **`PublishRelease`** (skipped when `validationMode: true`) — a single stage with two ordered jobs, deliberately publishing before tagging/releasing: + - **`Publish`** downloads both artifact folders, removes whichever one is empty (stripping the `.no-crates-packed` placeholder first), then hands both to a single invocation of `FAST.Release.PipelineTemplate.yml@fastPipelines`, which performs the actual `npm publish` / `cargo publish` for whichever asset types are present. Calling the release template once for both asset types (rather than in two parallel jobs, one per asset type, as an earlier version of this pipeline did) means one destination succeeding while the other fails can never leave a package half-published without the whole job failing as one unit. + - **`PublishGitHub`** (`dependsOn: Publish`, `condition: succeeded()`) runs strictly after `Publish` succeeds. It first runs [`build/scripts/check-release-tags.mjs`](../../build/scripts/check-release-tags.mjs), which freshly checks (via `git ls-remote origin`) whether each package's tag already exists — independent of the `NeedsRelease` variables computed earlier by `SignArtifacts`, since those only reflect the state before `Publish` ran. It then runs one `GitHubRelease@1` per package, conditioned on both that package's `NeedsRelease` variable *and* its tag not already existing (`releaseTagCheck.ReleaseTagExists == 'false'`), using `tagSource: userSpecifiedTag` so the task itself creates the tag as part of creating the release. + + This publish-then-tag ordering is the key fix for the release tag's dual role: `pack-pending-releases.mjs` treats a workspace as "pending" purely based on whether its `${name}_v${version}` tag exists on `origin`, so whichever step creates that tag also makes the package invisible to every future release-prep run. An earlier version of this pipeline created the tag *before* publishing (via a dedicated `TagRelease` stage); a publish failure then left the tag behind, permanently stranding that package with no automatic retry. Creating the tag only after `Publish` succeeds means a publish failure never leaves the tag behind, so the very next `FAST CD Build` run retries that package automatically — and the `releaseTagCheck` guard means rerunning a job that failed partway through `PublishGitHub` (Azure Pipelines reruns every task in a failed job, including ones that already succeeded) is also safe, since already-created releases are skipped rather than recreated. + + Residual risk: if `Publish` succeeds but `PublishGitHub` fails outright for a package (rather than a rerunnable partial failure), that package's tag still won't exist, so the next `FAST CD Build` run will try to republish it — which fails loudly since it's already published (existing, documented idempotency behavior; see below). That is a bounded, always manually-recoverable state (a maintainer creates the missing tag/GitHub release directly), never a silent or permanent one, and was judged the safest trade-off achievable within a single `PublishRelease` stage. + +Idempotency is enforced entirely through git tags (`${name}_v${version}`), so nothing needs to talk to npm.org or crates.io to decide whether work is required — republishing an already-published version simply fails loudly at the `npm publish` / `cargo publish` step, the same as a manual retry would. + +The queue-time `validationMode` parameter (both pipelines) defaults to `false`; setting it to `true` treats every publishable workspace as pending so its artifact contract can be rebuilt and validated through `SignArtifacts`, without creating tags or publishing anywhere. + +> **Note:** `FAST.Sign.PipelineTemplate.yml` and `FAST.Release.PipelineTemplate.yml` live in the internal `open-source/FASTPipelineTemplates` Azure DevOps repository, which is not accessible from GitHub tooling. Their exact parameter contracts (in particular how they handle an absent artifact directory) could not be independently verified while authoring this pipeline; the empty-directory removal step in the `Publish` job is a defense-in-depth measure taken because that contract could not be confirmed. ### Adding a publishable package -`cd-github-releases.yml` discovers publishable workspaces automatically from the root `package.json` `workspaces` list, but `azure-pipelines-cd.yml` must be updated because Azure Pipelines cannot create `DownloadGitHubRelease@0` tasks dynamically from the runtime detection output. +`pack-pending-releases.mjs` discovers publishable workspaces automatically from the root `package.json` `workspaces` list, but `.ado/pipelines/azure-pipelines-cd.yml` must be updated because Azure Pipelines cannot create `GitHubRelease@1` tasks dynamically from the runtime manifest. -The `npm run checkchange` command runs `build/scripts/check-publish-pipeline.mjs` to verify that every non-private workspace has matching Azure CD variables and a conditional `DownloadGitHubRelease@0` task. This guardrail runs in PR validation and fails when a new publishable package is added without updating the publish pipeline. +The `npm run checkchange` command runs `build/scripts/check-publish-pipeline.mjs` to verify that every non-private workspace has matching `PublishRelease` stage variables and a conditional `GitHubRelease@1` task. This guardrail runs in PR validation and fails when a new publishable package is added without updating the publish pipeline. When adding a new non-private workspace that should publish through CD: 1. Ensure the workspace is included in the root `package.json` `workspaces` list and has a `name` and `version`. 2. If the package has paired crate assets, place each crate at `crates//Cargo.toml`. By default, `` is the npm package name with the leading `@` removed and `/` replaced by `-`. `@microsoft/fast-build` is the special bundled release and pairs with both `crates/microsoft-fast-build/Cargo.toml` and `crates/microsoft-fast-convert/Cargo.toml`. -3. Add package-specific output variables to the `Package` stage in `azure-pipelines-cd.yml`. The output prefix is generated from the npm package name by converting `@microsoft/` to camel case. For example, `@microsoft/fast-foo` emits `fastFooNeedsDeployment` and `fastFooReleaseTag`. -4. Add a conditional `DownloadGitHubRelease@0` task for the package using the `fast` GitHub service connection, `defaultVersionType: 'specificTag'`, and the package's `$(ReleaseTag)` variable. -5. Confirm the artifact sorting step still covers the package assets. Packages should attach `.tgz` assets, and paired crates should also attach `.crate` assets. +3. Add package-specific output variables to the `PublishRelease` stage in `.ado/pipelines/azure-pipelines-cd.yml`. The output prefix is generated from the npm package name by converting `@microsoft/` to camel case. For example, `@microsoft/fast-foo` emits `fastFooNeedsRelease`, `fastFooReleaseTag`, and `fastFooReleaseVersion`. +4. Add a conditional `GitHubRelease@1` task for the package in the `PublishGitHub` job, using the `fast` GitHub service connection, `repositoryName: microsoft/fast`, `tagSource: userSpecifiedTag`, and the package's `$(ReleaseTag)` variable. The condition must include the `releaseTagCheck.ReleaseTagExists` clause (in addition to `NeedsRelease`) so rerunning the job after a partial failure is safe — see `check-release-tags.mjs`. +5. Confirm the task's `assets` globs use the exact versioned filename per asset (`$(ReleaseVersion)`, not a prefix wildcard) for the package's npm tarball and any paired crate archives, to avoid picking up a stale tarball left over from a previous packing attempt. Example Azure additions for `@microsoft/fast-foo`: ```yml variables: - fastFooNeedsDeployment: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastFooNeedsDeployment'] ] - fastFooReleaseTag: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastFooReleaseTag'] ] + fastFooNeedsRelease: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastFooNeedsRelease'] ] + fastFooReleaseTag: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastFooReleaseTag'] ] + fastFooReleaseVersion: $[ stageDependencies.SignArtifacts.Sign.outputs['release.fastFooReleaseVersion'] ] steps: -- task: DownloadGitHubRelease@0 - displayName: "Download @microsoft/fast-foo release assets" - condition: and(succeeded(), eq(variables['fastFooNeedsDeployment'], 'true')) +- task: GitHubRelease@1 + displayName: "Create @microsoft/fast-foo GitHub Release" + condition: and(succeeded(), eq(variables['fastFooNeedsRelease'], 'true'), eq(variables['releaseTagCheck.fastFooReleaseTagExists'], 'false')) inputs: - connection: fast - userRepository: microsoft/fast - defaultVersionType: 'specificTag' - version: '$(fastFooReleaseTag)' - downloadPath: '$(System.ArtifactsDirectory)' + gitHubConnection: fast + repositoryName: microsoft/fast + action: create + target: $(releaseCommit) + tagSource: userSpecifiedTag + tag: $(fastFooReleaseTag) + title: $(fastFooReleaseTag) + releaseNotesSource: inline + releaseNotesInline: "Automated FAST release for @microsoft/fast-foo@$(fastFooReleaseVersion)." + addChangeLog: false + assets: | + $(Build.SourcesDirectory)/publish_artifacts_npm/microsoft-fast-foo-$(fastFooReleaseVersion).tgz + assetUploadMode: replace + isDraft: false + isPreRelease: false + makeLatest: legacy ``` \ No newline at end of file diff --git a/.github/workflows/cd-github-releases.yml b/.github/workflows/cd-github-releases.yml deleted file mode 100644 index 9ee36df4b60..00000000000 --- a/.github/workflows/cd-github-releases.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Release packages to GitHub releases - -on: - workflow_dispatch: - schedule: - # 12am PST = 08:00 UTC (drifts to ~11pm PDT during US daylight time). - # Runs ~1 hour before the Azure CD pipeline (09:00 UTC) so any new - # GitHub releases this job creates are ready for that night's publish. - - cron: '0 8 * * *' - -permissions: { contents: write } - -jobs: - # Lightweight detection: walks the workspaces directory tree (no `npm ci`, - # no build, no `gh` API call) and checks whether each `${name}_v${version}` - # tag already exists in git. The downstream `release` job is skipped if - # every current version already has a tag. - detect: - runs-on: ubuntu-latest - outputs: - hasMissingReleases: ${{ steps.check.outputs.hasMissingReleases }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # need all tags for `git rev-parse refs/tags/`. - - - uses: actions/setup-node@v6 - with: - node-version: 22 - - - id: check - name: Detect publishable workspaces without a release tag - run: node build/scripts/create-github-releases.mjs --check-only - - release: - needs: detect - if: needs.detect.outputs.hasMissingReleases == 'true' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # need all tags so `git rev-parse` can detect existing releases. - - - uses: actions/setup-node@v6 - with: - node-version: 22 - - - name: Cache multiple paths - uses: actions/cache@v4 - env: - cache-name: cache-node-modules - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - - name: Install package dependencies - run: npm ci - - - name: Install Rust and wasm-pack - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - export PATH="$HOME/.cargo/bin:$PATH" - rustup target add wasm32-unknown-unknown - cargo install wasm-pack - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - - name: Build workspaces - run: npm run build - - - name: Pack and create GitHub releases - run: node build/scripts/create-github-releases.mjs - env: - GH_TOKEN: ${{ github.token }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e9c4e98c8d..b90d31195f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,7 +148,7 @@ Example of how to format a migration document: ### Publishing -Releases are produced from a dedicated **bump pull request** authored by a maintainer (not by CI). Once the bump PR lands on `main`, the [`cd-github-releases.yml`](.github/workflows/cd-github-releases.yml) workflow attaches the freshly-packed tarballs to a GitHub release per package, and the nightly Azure pipeline ([`azure-pipelines-cd.yml`](azure-pipelines-cd.yml)) downloads those assets and publishes them to npm and crates.io. The detailed CD design is documented in [`.github/workflows/README.md`](.github/workflows/README.md). +Releases are produced from a dedicated **bump pull request** authored by a maintainer (not by CI). Once the bump PR lands on `main`, two Azure Pipelines under [`.ado/pipelines/`](.ado/pipelines/) own the rest: `azure-pipelines-build.yml` (**`FAST CD Build`**) packs the freshly-bumped npm tarballs and paired Rust crates for every package whose release tag doesn't exist yet, and `azure-pipelines-cd.yml` (**`FAST CD`**) signs those artifacts, publishes to npm and crates.io, and then creates the GitHub release and git tag per package. The detailed CD design is documented in [`.github/workflows/README.md`](.github/workflows/README.md). This section covers the maintainer workflow for opening the bump PR. @@ -186,10 +186,10 @@ No commit, push, npm publish, or git tag is made by `npm run bump`. ```bash git status git diff -node build/scripts/create-github-releases.mjs --check-only +node build/scripts/pack-pending-releases.mjs --check-only ``` -The third command previews exactly which workspaces the post-merge CD will publish, by listing every workspace whose freshly-bumped `${name}_v${version}` tag is not present in the local git tag list. Run `git fetch --tags --prune origin` beforehand if you want the preview to reflect the current state on `origin` rather than your stale local refs — though for a fresh bump that hasn't been pushed yet, your local tag list is the source of truth anyway. +The third command previews exactly which workspaces the post-merge CD will publish, by listing every workspace whose freshly-bumped `${name}_v${version}` tag is not present on `origin` (it queries `origin` directly via `git ls-remote`, so it reflects the real remote state — no need to `git fetch --tags` first). A typical bump PR touches: @@ -210,14 +210,14 @@ gh pr create --fill --base main The bump PR goes through normal review. `npm run checkchange` will pass because the branch name matches `publish_` and the actor has admin on the repo (see [Manual version bumps](#manual-version-bumps)); the PR itself does **not** publish anything. :::note -Do not edit `package.json` or `Cargo.toml` versions by hand as part of a normal feature/fix PR. Let `npm run bump` and the postbump hook do it. [`create-github-releases.mjs`](build/scripts/create-github-releases.mjs) refuses to release a workspace whose npm version and paired crate version disagree. +Do not edit `package.json` or `Cargo.toml` versions by hand as part of a normal feature/fix PR. Let `npm run bump` and the postbump hook do it. [`pack-pending-releases.mjs`](build/scripts/pack-pending-releases.mjs) refuses to release a workspace whose npm version and paired crate version disagree. A narrow exception exists for the **manual version bump** flow described in [the next section](#manual-version-bumps) — hotfix overrides, paired Rust/npm sync recovery, or scripted version pins. Those edits are tolerated by `npm run checkchange` only on a `publish_` branch whose actor has the `admin` role on `microsoft/fast`. ::: #### 6. After merge -After merge, [`cd-github-releases.yml`](.github/workflows/cd-github-releases.yml) runs on its nightly cron (`0 8 * * *` UTC, ~12am PST) — or you can trigger it immediately via `gh workflow run cd-github-releases.yml` if you don't want to wait. Its `detect` job notices the new `${name}_v${version}` tags don't yet exist; the `release` job packs each `.tgz` (and any paired `.crate`) and atomically creates one GitHub release per bumped package. The next nightly run of [`azure-pipelines-cd.yml`](azure-pipelines-cd.yml) (scheduled ~1 hour later at 09:00 UTC) downloads those assets, hands off to `FAST.Release.PipelineTemplate` for the actual `npm publish` / `cargo publish`, and on success pushes `deployed/` marker tags so the publish is never repeated. +After merge, the [`FAST CD Build`](.ado/pipelines/azure-pipelines-build.yml) pipeline triggers automatically on the push to `main`. Its `PrepareRelease` stage notices the new `${name}_v${version}` tags don't yet exist on `origin`; `BuildArtifacts` builds the repo and packs each pending package's `.tgz` (and any paired `.crate`) into pipeline artifacts. Completion of that stage on `main` triggers [`FAST CD`](.ado/pipelines/azure-pipelines-cd.yml), which signs the artifacts, publishes to npm and crates.io via `FAST.Release.PipelineTemplate` first, and only then creates the git tag and GitHub release per bumped package. Publishing is deliberately ordered before tagging: the tag is what marks a package as "released" for future `FAST CD Build` runs, so a publish failure never leaves a tag behind, and the very next run automatically retries that package. Idempotency is enforced purely through the `${name}_v${version}` git tags — no separate marker tags are needed. See [`.github/workflows/README.md`](.github/workflows/README.md) for the full rationale. #### Hotfix or single-package bump diff --git a/azure-pipelines-cd.yml b/azure-pipelines-cd.yml deleted file mode 100644 index 3ce1dc18757..00000000000 --- a/azure-pipelines-cd.yml +++ /dev/null @@ -1,237 +0,0 @@ -trigger: none -pr: none - -schedules: -- cron: '0 9 * * *' # 09:00 UTC daily (~1am Pacific in standard time, ~2am during DST). - displayName: Daily npm/crates publish from GitHub Releases - branches: - include: - - main - always: true # run even when there are no new commits — we are checking external GitHub releases, not repo changes. - -# The `resources` specify the location and version of the 1ES PT. -resources: - repositories: - - repository: fastPipelines - type: git - name: open-source/FASTPipelineTemplates - ref: main - - repository: 1esPipelines - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release - -extends: - # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. - # For non-production pipelines, use "Unofficial" as defined below. - # For productions pipelines, use "Official". - template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines - parameters: - # Update the pool with your team's 1ES hosted pool. - pool: - name: OneESPool # Name of your hosted pool - image: HostedPoolLinuxImage # Name of the image in your pool. If not specified, first image of the pool is used - os: linux # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS - sdl: - sourceAnalysisPool: - name: OneESPool # Name of your hosted pool - image: HostedPoolWindowsImage # Name of the image in your pool. If not specified, first image of the pool is used - os: windows # OS of the image. Allowed values: windows, linux, macOS - settings: - networkIsolationPolicy: Permissive - stages: - - # ── Stage 1: Lightweight detection (runs every night, ~1-2 min) ── - # This stage is separate so that when no deployment is needed, the - # heavyweight Package stage (with all its 1ES SDL tasks) is skipped - # entirely — avoiding ~30 min of unnecessary agent provisioning and - # compliance scans on no-op nights. - - stage: Check - displayName: Detect undeployed GitHub releases - jobs: - - job: CheckVersion - steps: - - checkout: self - persistCredentials: "true" - fetchTags: true - - - task: UseNode@1 - inputs: - version: "22.x" - displayName: "Install Node.js" - - # The check enumerates current workspace release tags rather than every - # historical package tag so older beachball tags are not deployment - # candidates. The Package stage pushes a `deployed/` marker tag - # after a successful publish, and the next pipeline run sees that - # marker via `git tag -l`. - - script: | - node build/scripts/download-github-releases.mjs --check-only - displayName: "Detect releases whose tarballs have not been published" - name: deploymentCheck - env: - GITHUB_REPOSITORY: microsoft/fast - - # ── Stage 2: Download, publish, and mark deployed (skipped on no-op nights) ── - - stage: Package - displayName: Publish tarballs to npm and crates.io - dependsOn: Check - condition: eq(dependencies.Check.outputs['CheckVersion.deploymentCheck.needsDeployment'], 'true') - variables: - npm_config_tag: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.npmDistTag'] ] - undeployedTags: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.undeployedTags'] ] - # When adding a new publishable workspace, add its package-specific - # deployment outputs here and a matching DownloadGitHubRelease@0 task - # below. See .github/workflows/README.md > Continuous Deployment. - fastBuildNeedsDeployment: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastBuildNeedsDeployment'] ] - fastBuildReleaseTag: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastBuildReleaseTag'] ] - fastElementNeedsDeployment: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastElementNeedsDeployment'] ] - fastElementReleaseTag: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastElementReleaseTag'] ] - fastRouterNeedsDeployment: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastRouterNeedsDeployment'] ] - fastRouterReleaseTag: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastRouterReleaseTag'] ] - fastTestHarnessNeedsDeployment: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastTestHarnessNeedsDeployment'] ] - fastTestHarnessReleaseTag: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.fastTestHarnessReleaseTag'] ] - jobs: - - job: Deploy - steps: - - checkout: self - persistCredentials: "true" - fetchTags: true - - - script: | - git config --global user.email fastsvc@microsoft.com - git config --global user.name "Microsoft FAST Builds" - displayName: "Configure git for tag push" - - - task: UseNode@1 - inputs: - version: "22.x" - displayName: "Install Node.js" - - - task: DownloadGitHubRelease@0 - displayName: "Download @microsoft/fast-build release assets" - condition: and(succeeded(), eq(variables['fastBuildNeedsDeployment'], 'true')) - inputs: - connection: fast - userRepository: microsoft/fast - defaultVersionType: 'specificTag' - version: '$(fastBuildReleaseTag)' - downloadPath: '$(System.ArtifactsDirectory)' - - - task: DownloadGitHubRelease@0 - displayName: "Download @microsoft/fast-element release assets" - condition: and(succeeded(), eq(variables['fastElementNeedsDeployment'], 'true')) - inputs: - connection: fast - userRepository: microsoft/fast - defaultVersionType: 'specificTag' - version: '$(fastElementReleaseTag)' - downloadPath: '$(System.ArtifactsDirectory)' - - - task: DownloadGitHubRelease@0 - displayName: "Download @microsoft/fast-router release assets" - condition: and(succeeded(), eq(variables['fastRouterNeedsDeployment'], 'true')) - inputs: - connection: fast - userRepository: microsoft/fast - defaultVersionType: 'specificTag' - version: '$(fastRouterReleaseTag)' - downloadPath: '$(System.ArtifactsDirectory)' - - - task: DownloadGitHubRelease@0 - displayName: "Download @microsoft/fast-test-harness release assets" - condition: and(succeeded(), eq(variables['fastTestHarnessNeedsDeployment'], 'true')) - inputs: - connection: fast - userRepository: microsoft/fast - defaultVersionType: 'specificTag' - version: '$(fastTestHarnessReleaseTag)' - downloadPath: '$(System.ArtifactsDirectory)' - - - script: | - set -euo pipefail - rm -rf publish_artifacts_npm publish_artifacts_crates publish_artifacts_meta - mkdir -p publish_artifacts_meta - - if find "$(System.ArtifactsDirectory)" -name "*.tgz" -print -quit | grep -q .; then - mkdir -p publish_artifacts_npm - find "$(System.ArtifactsDirectory)" -name "*.tgz" -exec cp {} publish_artifacts_npm/ \; - fi - - if find "$(System.ArtifactsDirectory)" -name "*.crate" -print -quit | grep -q .; then - mkdir -p publish_artifacts_crates - find "$(System.ArtifactsDirectory)" -name "*.crate" -exec cp {} publish_artifacts_crates/ \; - fi - - printf '%s\n' "$(undeployedTags)" | tr ',' '\n' | sed '/^$/d' > publish_artifacts_meta/undeployed-tags.txt - - has_assets=false - if [ -d publish_artifacts_npm ] && find publish_artifacts_npm -maxdepth 1 -type f -print -quit | grep -q .; then - has_assets=true - fi - if [ -d publish_artifacts_crates ] && find publish_artifacts_crates -maxdepth 1 -type f -print -quit | grep -q .; then - has_assets=true - fi - if [ "$has_assets" != "true" ]; then - echo "No GitHub release assets were downloaded." - exit 1 - fi - - echo "npm artifacts:" - if [ -d publish_artifacts_npm ]; then - find publish_artifacts_npm -maxdepth 1 -type f -print | sort - else - echo "none" - fi - - echo "crate artifacts:" - if [ -d publish_artifacts_crates ]; then - find publish_artifacts_crates -maxdepth 1 -type f -print | sort - else - echo "none" - fi - - echo "deployment markers:" - cat publish_artifacts_meta/undeployed-tags.txt - displayName: "Separate release artifacts" - - - script: | - set -euo pipefail - if [ -z "$(npm_config_tag)" ]; then - echo "npm_config_tag was not provided by deployment detection." - exit 1 - fi - echo "Publishing npm artifacts with dist-tag: $(npm_config_tag)" - printf '\ntag=%s\n' "$(npm_config_tag)" >> .npmrc - printf '%s\n' "$(npm_config_tag)" > publish_artifacts_meta/npm-dist-tag.txt - displayName: "Configure npm publish dist-tag" - - - template: FAST.Release.PipelineTemplate.yml@fastPipelines # Template reference - - # Push a `deployed/` marker tag for each release that was just - # published so that the next nightly Check stage sees it and skips - # the release. Reads the list of tags prepared before publishing. - # Idempotent: a marker tag that already exists locally (i.e. fetched - # from origin via `fetchTags: true`) is left alone instead of failing. - - script: | - set -euo pipefail - META_FILE=publish_artifacts_meta/undeployed-tags.txt - if [ ! -s "$META_FILE" ]; then - echo "No tags to mark as deployed." - exit 0 - fi - while IFS= read -r tag; do - [ -z "$tag" ] && continue - DEPLOY_TAG="deployed/${tag}" - if git rev-parse --verify --quiet "refs/tags/${DEPLOY_TAG}" >/dev/null; then - echo "Already marked deployed: ${DEPLOY_TAG} (skipping)" - continue - fi - echo "Marking deployed: ${DEPLOY_TAG}" - # Point the marker at the release tag's commit (not the - # agent's current HEAD) so the marker stays accurate even - # if the pipeline is re-run from a later commit. - git tag "${DEPLOY_TAG}" "refs/tags/${tag}" - git push origin "${DEPLOY_TAG}" - done < "$META_FILE" - displayName: "Mark releases as deployed" diff --git a/build/scripts/check-publish-pipeline.mjs b/build/scripts/check-publish-pipeline.mjs index d5cee33b60c..1ac330c9737 100644 --- a/build/scripts/check-publish-pipeline.mjs +++ b/build/scripts/check-publish-pipeline.mjs @@ -2,73 +2,22 @@ /** * Guardrail for Azure CD coverage. * - * `cd-github-releases.yml` discovers publishable workspaces dynamically, but - * `azure-pipelines-cd.yml` must declare one `DownloadGitHubRelease@0` task per - * package because Azure Pipelines cannot create tasks from runtime output. This - * script keeps those surfaces in sync. + * `pack-pending-releases.mjs` discovers publishable workspaces dynamically, + * but `.ado/pipelines/azure-pipelines-cd.yml` must declare one static + * `GitHubRelease@1` task (plus matching `PublishRelease` stage variables) + * per package, because Azure Pipelines cannot create tasks from runtime + * manifest content. This script keeps those surfaces in sync. */ -import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + listPublishableWorkspaces, + repoRoot, + VersionDriftError, +} from "./lib/publishable-workspaces.mjs"; -const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); -const pipelinePath = join(repoRoot, "azure-pipelines-cd.yml"); - -function readJson(relativePath) { - return JSON.parse(readFileSync(join(repoRoot, relativePath), "utf8")); -} - -function npmNameToCrateName(npmName) { - return npmName.replace(/^@/, "").replace(/\//g, "-"); -} - -function npmNameToOutputPrefix(npmName) { - return npmNameToCrateName(npmName) - .replace(/^microsoft-/, "") - .replace(/-([a-z0-9])/g, (_, char) => char.toUpperCase()); -} - -function listWorkspaceLocations() { - const rootPkg = readJson("package.json"); - const locations = new Set(); - - for (const pattern of rootPkg.workspaces || []) { - if (pattern.endsWith("/*")) { - const parent = pattern.slice(0, -2); - const parentPath = join(repoRoot, parent); - if (!existsSync(parentPath)) continue; - for (const entry of readdirSync(parentPath, { withFileTypes: true })) { - if (entry.isDirectory()) { - locations.add(join(parent, entry.name)); - } - } - } else { - locations.add(pattern); - } - } - - return [...locations].sort(); -} - -function listPublishableWorkspaces() { - return listWorkspaceLocations() - .map(location => { - const pkgPath = join(location, "package.json"); - const absolutePkgPath = join(repoRoot, pkgPath); - if (!existsSync(absolutePkgPath)) return null; - - const pkg = readJson(pkgPath); - if (pkg.private === true || !pkg.name || !pkg.version) return null; - - return { - location, - name: pkg.name, - outputPrefix: npmNameToOutputPrefix(pkg.name), - }; - }) - .filter(Boolean); -} +const pipelinePath = join(repoRoot, ".ado", "pipelines", "azure-pipelines-cd.yml"); function getStepBlocks(pipeline, stepHeader) { const lines = pipeline.split(/\r?\n/); @@ -84,7 +33,9 @@ function getStepBlocks(pipeline, stepHeader) { const block = []; for (let j = i; j < lines.length; j++) { const current = lines[j]; - const nextStep = current.match(/^(\s*)- (checkout|script|task|template):/); + const nextStep = current.match( + /^(\s*)- (checkout|script|task|template|download):/, + ); if (j > i && nextStep && nextStep[1].length === indent) { break; } @@ -101,13 +52,13 @@ function validateUniquePrefixes(workspaces) { const failures = []; for (const workspace of workspaces) { - const previous = seen.get(workspace.outputPrefix); + const previous = seen.get(workspace.prefix); if (previous) { failures.push( - `${workspace.name} and ${previous.name} both map to Azure output prefix '${workspace.outputPrefix}'. Rename one package or update the prefix mapping.`, + `${workspace.name} and ${previous.name} both map to Azure output prefix '${workspace.prefix}'. Rename one package or update the prefix mapping.`, ); } else { - seen.set(workspace.outputPrefix, workspace); + seen.set(workspace.prefix, workspace); } } @@ -115,37 +66,63 @@ function validateUniquePrefixes(workspaces) { } const pipeline = readFileSync(pipelinePath, "utf8"); -const publishable = listPublishableWorkspaces(); -const downloadBlocks = getStepBlocks(pipeline, "- task: DownloadGitHubRelease@0"); + +let publishable; +try { + publishable = listPublishableWorkspaces(); +} catch (error) { + if (error instanceof VersionDriftError) { + console.error("[check-publish-pipeline] " + error.message); + process.exit(1); + } + throw error; +} + +const releaseBlocks = getStepBlocks(pipeline, "- task: GitHubRelease@1"); const failures = validateUniquePrefixes(publishable); -for (const { name, outputPrefix } of publishable) { - const needsVariable = `${outputPrefix}NeedsDeployment: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.${outputPrefix}NeedsDeployment'] ]`; - const tagVariable = `${outputPrefix}ReleaseTag: $[ stageDependencies.Check.CheckVersion.outputs['deploymentCheck.${outputPrefix}ReleaseTag'] ]`; - const condition = `condition: and(succeeded(), eq(variables['${outputPrefix}NeedsDeployment'], 'true'))`; - const version = `version: '$(${outputPrefix}ReleaseTag)'`; +for (const { name, prefix } of publishable) { + const needsVariable = `${prefix}NeedsRelease: $[ stageDependencies.SignArtifacts.Sign.outputs['release.${prefix}NeedsRelease'] ]`; + const tagVariable = `${prefix}ReleaseTag: $[ stageDependencies.SignArtifacts.Sign.outputs['release.${prefix}ReleaseTag'] ]`; + const versionVariable = `${prefix}ReleaseVersion: $[ stageDependencies.SignArtifacts.Sign.outputs['release.${prefix}ReleaseVersion'] ]`; + // The release-tag-exists clause is what makes rerunning a partially + // failed `PublishGitHub` job safe (see that job's comments in + // azure-pipelines-cd.yml): it must be present alongside the + // `NeedsRelease` check on every task, not just some of them. + const condition = `condition: and(succeeded(), eq(variables['${prefix}NeedsRelease'], 'true'), eq(variables['releaseTagCheck.${prefix}ReleaseTagExists'], 'false'))`; + const tag = `tag: $(${prefix}ReleaseTag)`; if (!pipeline.includes(needsVariable)) { - failures.push(`Missing Package stage variable for ${name}: ${needsVariable}`); + failures.push( + `Missing PublishRelease stage variable for ${name}: ${needsVariable}`, + ); } if (!pipeline.includes(tagVariable)) { - failures.push(`Missing Package stage variable for ${name}: ${tagVariable}`); + failures.push( + `Missing PublishRelease stage variable for ${name}: ${tagVariable}`, + ); + } + + if (!pipeline.includes(versionVariable)) { + failures.push( + `Missing PublishRelease stage variable for ${name}: ${versionVariable}`, + ); } - const hasDownloadTask = downloadBlocks.some( + const hasReleaseTask = releaseBlocks.some( block => - block.includes(`Download ${name} release assets`) && + block.includes(`Create ${name} GitHub Release`) && block.includes(condition) && - block.includes("connection: fast") && - block.includes("userRepository: microsoft/fast") && - block.includes("defaultVersionType: 'specificTag'") && - block.includes(version), + block.includes("gitHubConnection: fast") && + block.includes("repositoryName: microsoft/fast") && + block.includes("tagSource: userSpecifiedTag") && + block.includes(tag), ); - if (!hasDownloadTask) { + if (!hasReleaseTask) { failures.push( - `Missing DownloadGitHubRelease@0 task for ${name}. Add a task conditioned on '${outputPrefix}NeedsDeployment' and using '$(${outputPrefix}ReleaseTag)'.`, + `Missing GitHubRelease@1 task for ${name}. Add a task conditioned on '${prefix}NeedsRelease' and using '$(${prefix}ReleaseTag)'.`, ); } } @@ -153,7 +130,7 @@ for (const { name, outputPrefix } of publishable) { if (failures.length > 0) { console.error("[check-publish-pipeline] Azure CD publish coverage is incomplete."); console.error( - "Every non-private workspace must be represented in azure-pipelines-cd.yml. See .github/workflows/README.md > Adding a publishable package.", + "Every non-private workspace must be represented in .ado/pipelines/azure-pipelines-cd.yml. See .github/workflows/README.md > Adding a publishable package.", ); for (const failure of failures) { console.error(`- ${failure}`); diff --git a/build/scripts/check-release-tags.mjs b/build/scripts/check-release-tags.mjs new file mode 100644 index 00000000000..95ca4325dec --- /dev/null +++ b/build/scripts/check-release-tags.mjs @@ -0,0 +1,45 @@ +#!/usr/bin/env node +/** + * For every package recorded in `release-manifest.json`, freshly check (via + * `git ls-remote origin`) whether its `${name}_v${version}` release tag + * already exists on `origin` — independent of whatever + * `read-release-manifest.mjs` observed earlier in the `SignArtifacts` stage. + * + * `.ado/pipelines/azure-pipelines-cd.yml`'s `PublishGitHub` job runs + * `GitHubRelease@1` with `action: create` and `tagSource: userSpecifiedTag`, + * which creates the tag as part of creating the release (so no separate + * pre-publish tagging step exists — see that pipeline's `PublishRelease` + * stage comment for why). If that job partially fails (e.g. one package's + * `GitHubRelease@1` task fails after another package's already succeeded) + * and a maintainer reruns the failed job, Azure Pipelines reruns every task + * in the job, including the `GitHubRelease@1` tasks that already succeeded — + * which would otherwise fail trying to recreate a release (and tag) that + * already exists. This script's `${prefix}ReleaseTagExists` output lets each + * `GitHubRelease@1` task's `condition` skip packages whose tag is already + * there, so rerunning the job is safe. + * + * Usage: node build/scripts/check-release-tags.mjs + */ + +import { readFileSync } from "node:fs"; +import { gitTagExistsOnRemote } from "./lib/publishable-workspaces.mjs"; + +const manifestPath = process.argv[2]; +if (!manifestPath) { + console.error("Usage: check-release-tags.mjs "); + process.exit(1); +} + +function setAzureOutput(name, value) { + console.log(`##vso[task.setvariable variable=${name};isOutput=true]${value}`); +} + +const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + +for (const { name, tag, prefix } of manifest.packages || []) { + const exists = gitTagExistsOnRemote(tag); + console.log( + `${name}: ${tag} ${exists ? "already exists (release already published)" : "not yet created"}`, + ); + setAzureOutput(`${prefix}ReleaseTagExists`, exists ? "true" : "false"); +} diff --git a/build/scripts/create-github-releases.mjs b/build/scripts/create-github-releases.mjs deleted file mode 100644 index 669e5b04257..00000000000 --- a/build/scripts/create-github-releases.mjs +++ /dev/null @@ -1,322 +0,0 @@ -#!/usr/bin/env node -/** - * Create one GitHub release per non-private workspace whose - * `${name}_v${version}` tag does not yet exist in git. - * - * Invoked by `.github/workflows/cd-github-releases.yml`. The workflow does - * NOT bump versions or commit source changes — version bumps land on `main` - * through ordinary human-authored pull requests. This script: - * - * 1. Walks the root `package.json` `workspaces` globs to find every - * workspace's `package.json` (no `node_modules` required, so the - * `--check-only` mode can run before `npm ci`). - * 2. Skips workspaces whose package.json sets `private: true`. - * 3. For each remaining workspace, looks for paired Rust crates at - * `crates//Cargo.toml`. Most crate names are derived from - * the npm name by dropping the leading `@` and replacing `/` with `-`; - * `@microsoft/fast-build` bundles both `microsoft-fast-build` and - * `microsoft-fast-convert` into the same release. When a pair exists, - * the script errors if the two versions are not identical, forcing the - * version-bump PR to keep them in sync. - * 4. Computes `tag = ${name}_v${version}` (matching beachball's tag - * format) and skips the workspace if the git tag already exists - * (idempotent across re-runs). - * 5. Otherwise (when not `--check-only`) packs the npm tarball into - * `publish_artifacts_npm/` with `npm pack`, optionally packs the - * paired crates into `publish_artifacts_crates/` with - * `cargo package`, and creates the GitHub release with all - * assets attached (`gh release create --target `). The `gh` - * CLI creates the git tag atomically with the release, so the - * tag and the release exist if and only if each other does. - * - * Modes: - * - * - default: pack + tag + create any missing releases. - * - `--check-only`: only enumerate missing releases. Sets the - * `hasMissingReleases` GitHub Actions output (via `$GITHUB_OUTPUT`) - * to `"true"` or `"false"`. Performs no packing, no tagging, no - * release creation. Safe to run without `node_modules` populated. - * - * Set `FAST_RELEASE_SKIP_CRATES=true` to skip paired Rust crate validation and - * packaging. - * - * Authentication: the `gh` CLI reads `GH_TOKEN` from the environment. - */ - -import { execFileSync } from "node:child_process"; -import { - appendFileSync, - copyFileSync, - existsSync, - mkdirSync, - readdirSync, - readFileSync, -} from "node:fs"; -import { basename, join, resolve } from "node:path"; - -const NPM_DIR = "publish_artifacts_npm"; -const CRATES_DIR = "publish_artifacts_crates"; -const CHECK_ONLY = process.argv.includes("--check-only"); - -function run(file, args, opts = {}) { - return execFileSync(file, args, { encoding: "utf8", ...opts }); -} - -function gitTagExists(tag) { - try { - execFileSync("git", ["rev-parse", "--verify", `refs/tags/${tag}`], { - stdio: "ignore", - }); - return true; - } catch { - return false; - } -} - -function npmNameToCrateName(npmName) { - return npmName.replace(/^@/, "").replace(/\//g, "-"); -} - -const bundledCratesByPackage = new Map([ - ["@microsoft/fast-build", ["microsoft-fast-build", "microsoft-fast-convert"]], -]); - -function npmNameToCrateNames(npmName) { - return bundledCratesByPackage.get(npmName) ?? [npmNameToCrateName(npmName)]; -} - -function shouldSkipCrates() { - return process.env.FAST_RELEASE_SKIP_CRATES === "true"; -} - -function readCargoTomlVersion(cargoTomlPath) { - const content = readFileSync(cargoTomlPath, "utf8"); - let inPackage = false; - for (const rawLine of content.split("\n")) { - const line = rawLine.trim(); - if (line.startsWith("[")) { - inPackage = line === "[package]"; - continue; - } - if (!inPackage) continue; - const m = /^version\s*=\s*"([^"]+)"/.exec(line); - if (m) return m[1]; - } - return null; -} - -function listPairedCrates(pkgName, pkgVersion) { - if (shouldSkipCrates()) return []; - - const crates = []; - for (const crateName of npmNameToCrateNames(pkgName)) { - const cargoTomlPath = join("crates", crateName, "Cargo.toml"); - if (!existsSync(cargoTomlPath)) continue; - - const crateVersion = readCargoTomlVersion(cargoTomlPath); - if (crateVersion !== pkgVersion) { - throw new Error( - `Version mismatch for ${pkgName}: package.json is ${pkgVersion} ` + - `but ${cargoTomlPath} is ${crateVersion}. ` + - "Update one to match the other.", - ); - } - - crates.push({ crateName, cargoTomlPath }); - } - - return crates; -} - -function listPublishableWorkspaces() { - const rootPkg = JSON.parse(readFileSync("package.json", "utf8")); - const patterns = rootPkg.workspaces || []; - const locations = new Set(); - - for (const pattern of patterns) { - if (pattern.endsWith("/*")) { - const parent = pattern.slice(0, -2); - if (!existsSync(parent)) continue; - for (const entry of readdirSync(parent, { withFileTypes: true })) { - if (entry.isDirectory()) { - locations.add(join(parent, entry.name)); - } - } - } else { - locations.add(pattern); - } - } - - const workspaces = []; - for (const location of locations) { - const pkgPath = join(location, "package.json"); - if (!existsSync(pkgPath)) continue; - const pkg = JSON.parse(readFileSync(pkgPath, "utf8")); - if (pkg.private === true) continue; - if (!pkg.name || !pkg.version) continue; - - const crates = listPairedCrates(pkg.name, pkg.version); - - workspaces.push({ - location, - name: pkg.name, - version: pkg.version, - crates, - }); - } - - return workspaces; -} - -function setGitHubOutput(name, value) { - if (!process.env.GITHUB_OUTPUT) return; - appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`); -} - -const publishable = listPublishableWorkspaces(); -if (shouldSkipCrates()) { - console.log("Paired Rust crate assets are skipped for this release run."); -} - -if (publishable.length === 0) { - console.log("No publishable workspaces found."); - setGitHubOutput("hasMissingReleases", "false"); - process.exit(0); -} - -const missing = publishable.filter( - ({ name, version }) => !gitTagExists(`${name}_v${version}`), -); - -console.log(`Publishable workspaces: ${publishable.length}`); -console.log(`With existing git tag: ${publishable.length - missing.length}`); -console.log(`Missing git tag / release: ${missing.length}`); - -if (missing.length > 0) { - console.log("\nPackages that need a release:"); - for (const { name, version, crates } of missing) { - const suffix = - crates.length > 0 - ? ` (+ crates ${crates.map(crate => crate.crateName).join(", ")})` - : ""; - console.log(` - ${name}@${version}${suffix}`); - } -} - -setGitHubOutput("hasMissingReleases", missing.length > 0 ? "true" : "false"); - -if (CHECK_ONLY || missing.length === 0) { - process.exit(0); -} - -// `--check-only` only enumerates missing releases via local git state, -// but creating releases requires `gh release create`, which needs a -// token. Fail fast here so the workflow surfaces a clear error rather -// than a generic `gh` auth failure mid-loop. -if (!process.env.GH_TOKEN) { - console.error("GH_TOKEN must be set so the `gh` CLI can create GitHub releases."); - process.exit(1); -} - -mkdirSync(NPM_DIR, { recursive: true }); -mkdirSync(CRATES_DIR, { recursive: true }); - -let created = 0; -let hasErrors = false; - -for (const { name, version, location, crates } of missing) { - const tag = `${name}_v${version}`; - const assets = []; - - try { - console.log(`\nPacking ${name}@${version} from ${location}...`); - const packJson = run("npm", [ - "pack", - "--silent", - "--json", - `--workspace=${location}`, - `--pack-destination=${resolve(NPM_DIR)}`, - ]); - assets.push(join(NPM_DIR, JSON.parse(packJson)[0].filename)); - - for (const { crateName, cargoTomlPath } of crates) { - console.log(`Packaging crate ${crateName}@${version}...`); - run( - "cargo", - [ - "package", - "--no-verify", - "--allow-dirty", - "--manifest-path", - cargoTomlPath, - ], - { stdio: "inherit" }, - ); - const srcCrate = join( - "crates", - crateName, - "target", - "package", - `${crateName}-${version}.crate`, - ); - if (!existsSync(srcCrate)) { - throw new Error( - `Expected ${srcCrate} after cargo package, but it does not exist.`, - ); - } - const destCrate = join(CRATES_DIR, basename(srcCrate)); - copyFileSync(srcCrate, destCrate); - assets.push(destCrate); - } - - const notes = [ - `Nightly release for \`${name}@${version}\`.`, - "", - "Version bumps were landed via a regular pull request. The attached", - "assets will be downloaded and published to npm" + - (crates.length > 0 ? " and crates.io" : "") + - " by the nightly Azure release pipeline.", - ].join("\n"); - - // Let `gh release create` create the tag atomically with the - // release so that "tag exists" and "release exists" are always - // the same fact. If we created the tag separately and pushed - // it, and then `gh release create` failed, the next workflow - // run would think the release was already done (because the - // tag exists) and skip it forever. - const targetSha = ( - process.env.GITHUB_SHA || run("git", ["rev-parse", "HEAD"]) - ).trim(); - - console.log(`Creating release ${tag} at ${targetSha.slice(0, 7)}...`); - run( - "gh", - [ - "release", - "create", - tag, - ...assets, - "--target", - targetSha, - "--title", - `${name}@${version}`, - "--notes", - notes, - ], - { stdio: "inherit" }, - ); - - console.log(`Created release ${tag} with ${assets.length} asset(s)`); - created += 1; - } catch (error) { - hasErrors = true; - const message = error instanceof Error ? error.message : String(error); - console.error(`Failed to release ${name}@${version}: ${message}`); - } -} - -console.log(`\nReleases created: ${created}/${missing.length}`); - -if (hasErrors) { - process.exitCode = 1; -} diff --git a/build/scripts/download-github-releases.mjs b/build/scripts/download-github-releases.mjs deleted file mode 100644 index bb4eeda8642..00000000000 --- a/build/scripts/download-github-releases.mjs +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env node -/** - * Detect current publishable workspaces whose `${name}_v${version}` release tag - * has no `deployed/` counterpart. - * - * Invoked by `azure-pipelines-cd.yml` before the existing - * `FAST.Release.PipelineTemplate` runs. We use a `deployed/` git - * marker tag instead of `npm view` / `cargo search` because external - * calls from 1ES agents to npm/crates.io are unreliable. - * - * After a successful publish the Azure pipeline pushes the - * `deployed/` marker tag for each release that was published; the - * next run sees it via `git tag -l` and skips that release. - * - * The Azure pipeline uses this script during its Check stage, then downloads - * release assets through `DownloadGitHubRelease@0` using the repo's GitHub - * service connection. This avoids direct GitHub API calls from this script and - * keeps the flow aligned with the WebUI CD pipeline. - * - * Inputs: - * - * - `GITHUB_REPOSITORY` env var (`owner/repo`) — required. - * - `FAST_RELEASE_SKIP_CRATES=true` — skips paired Rust crate validation. - * - * The script reads workspace package manifests and paired Cargo manifests only: - * the source of truth for "what should be published" is the current workspace - * versions plus matching release tags. This keeps historical bare beachball tags - * from being treated as deployable releases while still working on a - * freshly-cloned 1ES agent with no `node_modules` or cargo registry. - * - * Most workspaces map to at most one crate by name convention. - * `@microsoft/fast-build` intentionally maps to both `microsoft-fast-build` - * and `microsoft-fast-convert`, but still uses one npm package release tag - * and one Azure download task. - */ - -import { execFileSync } from "node:child_process"; -import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { join } from "node:path"; - -const CHECK_ONLY = process.argv.includes("--check-only"); -if (!CHECK_ONLY) { - console.error( - "download-github-releases.mjs only supports --check-only; Azure Pipelines downloads assets with DownloadGitHubRelease@0.", - ); - process.exit(1); -} - -const repo = process.env.GITHUB_REPOSITORY; -if (!repo) { - console.error("GITHUB_REPOSITORY must be set to owner/repo"); - process.exit(1); -} - -function run(file, args, opts = {}) { - return execFileSync(file, args, { encoding: "utf8", ...opts }); -} - -function listGitTags() { - return run("git", ["tag", "--list"]) - .split("\n") - .map(t => t.trim()) - .filter(Boolean); -} - -function npmNameToCrateName(npmName) { - return npmName.replace(/^@/, "").replace(/\//g, "-"); -} - -const bundledCratesByPackage = new Map([ - ["@microsoft/fast-build", ["microsoft-fast-build", "microsoft-fast-convert"]], -]); - -function npmNameToCrateNames(npmName) { - return bundledCratesByPackage.get(npmName) ?? [npmNameToCrateName(npmName)]; -} - -function shouldSkipCrates() { - return process.env.FAST_RELEASE_SKIP_CRATES === "true"; -} - -function npmNameToOutputPrefix(npmName) { - return npmNameToCrateName(npmName) - .replace(/^microsoft-/, "") - .replace(/-([a-z0-9])/g, (_, char) => char.toUpperCase()); -} - -function readCargoTomlVersion(cargoTomlPath) { - const content = readFileSync(cargoTomlPath, "utf8"); - let inPackage = false; - for (const rawLine of content.split("\n")) { - const line = rawLine.trim(); - if (line.startsWith("[")) { - inPackage = line === "[package]"; - continue; - } - if (!inPackage) continue; - const m = /^version\s*=\s*"([^"]+)"/.exec(line); - if (m) return m[1]; - } - return null; -} - -function validatePairedCrates(pkgName, pkgVersion) { - if (shouldSkipCrates()) return; - - for (const crateName of npmNameToCrateNames(pkgName)) { - const cargoTomlPath = join("crates", crateName, "Cargo.toml"); - if (!existsSync(cargoTomlPath)) continue; - - const crateVersion = readCargoTomlVersion(cargoTomlPath); - if (crateVersion !== pkgVersion) { - throw new Error( - `Version mismatch for ${pkgName}: package.json is ${pkgVersion} ` + - `but ${cargoTomlPath} is ${crateVersion}. ` + - "Update one to match the other.", - ); - } - } -} - -function listPublishableWorkspaces() { - const rootPkg = JSON.parse(readFileSync("package.json", "utf8")); - const patterns = rootPkg.workspaces || []; - const locations = new Set(); - - for (const pattern of patterns) { - if (pattern.endsWith("/*")) { - const parent = pattern.slice(0, -2); - if (!existsSync(parent)) continue; - for (const entry of readdirSync(parent, { withFileTypes: true })) { - if (entry.isDirectory()) { - locations.add(join(parent, entry.name)); - } - } - } else { - locations.add(pattern); - } - } - - const workspaces = []; - for (const location of locations) { - const pkgPath = join(location, "package.json"); - if (!existsSync(pkgPath)) continue; - const pkg = JSON.parse(readFileSync(pkgPath, "utf8")); - if (pkg.private === true) continue; - if (!pkg.name || !pkg.version) continue; - - validatePairedCrates(pkg.name, pkg.version); - - workspaces.push({ - location, - name: pkg.name, - version: pkg.version, - tag: `${pkg.name}_v${pkg.version}`, - outputPrefix: npmNameToOutputPrefix(pkg.name), - }); - } - - return workspaces; -} - -function setAzureOutput(name, value) { - if (!process.env.TF_BUILD) return; - console.log(`##vso[task.setvariable variable=${name};isOutput=true]${value}`); -} - -const allTags = listGitTags(); -const tagSet = new Set(allTags); -const deployed = new Set( - allTags.filter(t => t.startsWith("deployed/")).map(t => t.slice("deployed/".length)), -); -const publishable = listPublishableWorkspaces(); -if (shouldSkipCrates()) { - console.log("Paired Rust crate assets are skipped for this deployment check."); -} -const releaseCandidates = publishable - .filter(({ tag }) => tagSet.has(tag)) - .sort((a, b) => a.tag.localeCompare(b.tag)); -const undeployed = releaseCandidates.filter(({ tag }) => !deployed.has(tag)); -const undeployedTagSet = new Set(undeployed.map(({ tag }) => tag)); -const npmDistTag = "latest"; - -console.log(`Publishable workspaces: ${publishable.length}`); -console.log(`Current release tags: ${releaseCandidates.length}`); -console.log(`Already deployed: ${releaseCandidates.length - undeployed.length}`); -console.log(`Undeployed: ${undeployed.length}`); -console.log(`npm dist-tag: ${npmDistTag}`); - -if (undeployed.length > 0) { - console.log("\nUndeployed tags:"); - for (const { tag } of undeployed) { - console.log(` - ${tag}`); - } -} - -setAzureOutput("needsDeployment", undeployed.length > 0 ? "true" : "false"); -setAzureOutput("undeployedTags", undeployed.map(({ tag }) => tag).join(",")); -setAzureOutput("npmDistTag", npmDistTag); -for (const workspace of publishable) { - setAzureOutput(`${workspace.outputPrefix}ReleaseTag`, workspace.tag); - setAzureOutput( - `${workspace.outputPrefix}NeedsDeployment`, - undeployedTagSet.has(workspace.tag) ? "true" : "false", - ); -} diff --git a/build/scripts/lib/publishable-workspaces.mjs b/build/scripts/lib/publishable-workspaces.mjs new file mode 100644 index 00000000000..5edcaeef7d3 --- /dev/null +++ b/build/scripts/lib/publishable-workspaces.mjs @@ -0,0 +1,204 @@ +/** + * Shared helpers for enumerating FAST's publishable npm workspaces and their + * paired Rust crates. + * + * Used by the Azure release pipeline scripts (`pack-pending-releases.mjs`, + * `read-release-manifest.mjs`, `check-release-tags.mjs`) and by + * `check-publish-pipeline.mjs`, so that "what is publishable" and "how does + * an npm name map to a crate name / Azure variable prefix" are defined in + * exactly one place. + */ + +import { execFileSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * Absolute path to the repository root, resolved from this file's own + * location (`build/scripts/lib/`) rather than `process.cwd()`. Every + * filesystem lookup in this module is anchored here so `listPublishableWorkspaces()` + * behaves identically no matter which directory the calling script (or a + * test) happens to be invoked from. + */ +export const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); + +/** + * Thrown when a publishable npm workspace's `package.json` version disagrees + * with a paired Rust crate's `Cargo.toml` version. Callers should catch this + * specifically and print `error.message` (no stack trace) rather than + * letting it surface as an uncaught exception, since it represents an + * actionable authoring mistake rather than a programming bug. + */ +export class VersionDriftError extends Error { + constructor(message) { + super(message); + this.name = "VersionDriftError"; + } +} + +/** Convert an npm package name into its paired Rust crate name, e.g. + * `@microsoft/fast-build` -> `microsoft-fast-build`. */ +export function npmNameToCrateName(npmName) { + return npmName.replace(/^@/, "").replace(/\//g, "-"); +} + +// `@microsoft/fast-build` bundles both `microsoft-fast-build` and +// `microsoft-fast-convert` into a single release. +const bundledCratesByPackage = new Map([ + ["@microsoft/fast-build", ["microsoft-fast-build", "microsoft-fast-convert"]], +]); + +export function npmNameToCrateNames(npmName) { + return bundledCratesByPackage.get(npmName) ?? [npmNameToCrateName(npmName)]; +} + +/** Convert an npm package name into a camelCase Azure Pipelines variable + * prefix, e.g. `@microsoft/fast-build` -> `fastBuild`. */ +export function npmNameToOutputPrefix(npmName) { + return npmNameToCrateName(npmName) + .replace(/^microsoft-/, "") + .replace(/-([a-z0-9])/g, (_, char) => char.toUpperCase()); +} + +export function shouldSkipCrates() { + return process.env.FAST_RELEASE_SKIP_CRATES === "true"; +} + +export function readCargoTomlVersion(cargoTomlPath) { + const content = readFileSync(cargoTomlPath, "utf8"); + let inPackage = false; + for (const rawLine of content.split("\n")) { + const line = rawLine.trim(); + if (line.startsWith("[")) { + inPackage = line === "[package]"; + continue; + } + if (!inPackage) continue; + const m = /^version\s*=\s*"([^"]+)"/.exec(line); + if (m) return m[1]; + } + return null; +} + +/** + * Resolve the paired Rust crates for `pkgName`, verifying each crate's + * `Cargo.toml` version matches `pkgVersion`. Every mismatch found is + * collected into `mismatches` instead of throwing immediately, so a caller + * enumerating many workspaces can report every version-drift problem in one + * pass rather than stopping at the first one. + */ +export function listPairedCrates(pkgName, pkgVersion, mismatches = []) { + if (shouldSkipCrates()) return []; + + const crates = []; + for (const crateName of npmNameToCrateNames(pkgName)) { + const cargoTomlPath = join(repoRoot, "crates", crateName, "Cargo.toml"); + if (!existsSync(cargoTomlPath)) continue; + + const crateVersion = readCargoTomlVersion(cargoTomlPath); + if (crateVersion !== pkgVersion) { + mismatches.push( + `${pkgName}: package.json is ${pkgVersion} but ${cargoTomlPath} is ${crateVersion}.`, + ); + continue; + } + + crates.push({ crateName, cargoTomlPath }); + } + + return crates; +} + +/** + * Walk the root `package.json` `workspaces` globs and return every + * non-private workspace with a `name` and `version`, including its paired + * Rust crates (if any). Requires no `npm ci` / `node_modules`. Resolved + * entirely against `repoRoot`, so this works no matter what `process.cwd()` + * happens to be. Results are sorted deterministically by package name so + * output ordering (manifest contents, Azure output variable emission order, + * log output) is stable across OSes/filesystems, which vary in the order + * `readdirSync` returns directory entries. + * + * Throws a single aggregated `VersionDriftError` if any workspace's paired + * crate version disagrees with its npm package version — every mismatch + * found across all workspaces is collected and reported together, rather + * than throwing (and hiding subsequent mismatches) at the first one found. + */ +export function listPublishableWorkspaces() { + const rootPkg = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")); + const patterns = rootPkg.workspaces || []; + const locations = new Set(); + + for (const pattern of patterns) { + if (pattern.endsWith("/*")) { + const parent = join(repoRoot, pattern.slice(0, -2)); + if (!existsSync(parent)) continue; + for (const entry of readdirSync(parent, { withFileTypes: true })) { + if (entry.isDirectory()) { + locations.add(join(parent, entry.name)); + } + } + } else { + locations.add(join(repoRoot, pattern)); + } + } + + const mismatches = []; + const workspaces = []; + for (const location of locations) { + const pkgPath = join(location, "package.json"); + if (!existsSync(pkgPath)) continue; + const pkg = JSON.parse(readFileSync(pkgPath, "utf8")); + if (pkg.private === true) continue; + if (!pkg.name || !pkg.version) continue; + + const crates = listPairedCrates(pkg.name, pkg.version, mismatches); + + workspaces.push({ + location, + name: pkg.name, + version: pkg.version, + tag: `${pkg.name}_v${pkg.version}`, + prefix: npmNameToOutputPrefix(pkg.name), + crates, + }); + } + + if (mismatches.length > 0) { + throw new VersionDriftError( + "Paired npm/crate version drift detected. Update one side to match " + + `the other for each of the following:\n${mismatches.map(m => ` - ${m}`).join("\n")}`, + ); + } + + workspaces.sort((a, b) => a.name.localeCompare(b.name)); + + return workspaces; +} + +function run(file, args, opts = {}) { + return execFileSync(file, args, { encoding: "utf8", cwd: repoRoot, ...opts }); +} + +/** + * Check whether `refs/tags/` (or its dereferenced annotated-tag + * counterpart) exists on `origin`, without requiring a full/unshallow local + * clone, so shallow 1ES agent checkouts work correctly. + */ +export function gitTagExistsOnRemote(tag) { + try { + const out = run("git", [ + "ls-remote", + "--exit-code", + "origin", + `refs/tags/${tag}`, + `refs/tags/${tag}^{}`, + ]); + return out.trim().length > 0; + } catch (error) { + // `git ls-remote --exit-code` exits with 2 when no refs match. + if (error?.status === 2) return false; + throw error; + } +} diff --git a/build/scripts/pack-pending-releases.mjs b/build/scripts/pack-pending-releases.mjs new file mode 100644 index 00000000000..e373c11534a --- /dev/null +++ b/build/scripts/pack-pending-releases.mjs @@ -0,0 +1,252 @@ +#!/usr/bin/env node +/** + * Enumerate FAST's publishable npm workspaces and, for every workspace whose + * `${name}_v${version}` tag does not yet exist on `origin`, pack its npm + * tarball (and any paired Rust crates) so the Azure `FAST CD Build` pipeline + * can hand the packed assets to the `FAST CD` pipeline for signing and + * publishing. + * + * This script does NOT create GitHub releases, git tags, or npm/crates.io + * publishes itself — those are owned by Azure Pipelines + * (`.ado/pipelines/azure-pipelines-build.yml` and + * `.ado/pipelines/azure-pipelines-cd.yml`) so that release credentials never + * leave the Azure environment. It does NOT bump versions or commit source + * changes either — version bumps land on `main` through ordinary + * human-authored pull requests (see CONTRIBUTING.md > Publishing). + * + * FAST is multi-package, so there is no single "the release version": each + * publishable workspace gets its own `${name}_v${version}` tag, so "pending" + * is evaluated per package, and a build can pack zero or more packages at + * once. + * + * 1. Walks the root `package.json` `workspaces` globs to find every + * workspace's `package.json` (no `node_modules` required, so + * `--check-only` can run before `npm ci`). + * 2. Skips workspaces whose package.json sets `private: true`. + * 3. For each remaining workspace, looks for paired Rust crates at + * `crates//Cargo.toml` (see + * `build/scripts/lib/publishable-workspaces.mjs` for the npm-name -> + * crate-name mapping, including the `@microsoft/fast-build` bundle). + * Errors if a paired crate's version does not match the npm package's + * version. + * 4. A workspace is "pending" when its `${name}_v${version}` tag does not + * yet exist on `origin` — or, when `ALLOW_EXISTING_RELEASE=true` + * (driven by the pipelines' `validationMode` parameter), every + * publishable workspace is treated as pending so its artifact contract + * can be rebuilt and validated without publishing. + * + * Modes: + * + * - `--check-only`: only enumerate pending workspaces and emit Azure + * Pipelines outputs (`##vso[task.setvariable ...]`) when running under + * Azure Pipelines (`$TF_BUILD` set). Performs no packing. Safe to run + * without `node_modules` populated. Also used for the local + * `CONTRIBUTING.md` "preview what CD will publish" step. + * - default: packs the npm tarball for every pending workspace into + * `publish_artifacts_npm/`, packs any paired Rust crates into + * `publish_artifacts_crates/`, and writes + * `publish_artifacts_meta/release-manifest.json` describing exactly + * what was packed (name, version, tag, npm tarball filename, crate + * filenames) for the downstream `read-release-manifest.mjs` and + * `check-release-tags.mjs` steps. + * + * Set `FAST_RELEASE_SKIP_CRATES=true` to skip paired Rust crate validation + * and packaging. + */ + +import { execFileSync } from "node:child_process"; +import { copyFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; +import { + gitTagExistsOnRemote, + listPublishableWorkspaces, + VersionDriftError, +} from "./lib/publishable-workspaces.mjs"; + +const NPM_DIR = "publish_artifacts_npm"; +const CRATES_DIR = "publish_artifacts_crates"; +const META_DIR = "publish_artifacts_meta"; +const MANIFEST_PATH = join(META_DIR, "release-manifest.json"); +const CHECK_ONLY = process.argv.includes("--check-only"); +const ALLOW_EXISTING_RELEASE = process.env.ALLOW_EXISTING_RELEASE === "true"; + +function run(file, args, opts = {}) { + return execFileSync(file, args, { encoding: "utf8", ...opts }); +} + +function setAzureOutput(name, value) { + if (!process.env.TF_BUILD) return; + console.log(`##vso[task.setvariable variable=${name};isOutput=true]${value}`); +} + +function isPending(workspace) { + if (ALLOW_EXISTING_RELEASE) return true; + return !gitTagExistsOnRemote(workspace.tag); +} + +function logError(message) { + if (process.env.TF_BUILD) { + console.error(`##vso[task.logissue type=error]${message}`); + } else { + console.error(message); + } +} + +let publishable; +try { + publishable = listPublishableWorkspaces(); +} catch (error) { + if (error instanceof VersionDriftError) { + logError(error.message); + process.exit(1); + } + throw error; +} +if (process.env.FAST_RELEASE_SKIP_CRATES === "true") { + console.log("Paired Rust crate assets are skipped for this release run."); +} +if (ALLOW_EXISTING_RELEASE) { + console.log( + "Validation mode: every publishable workspace is treated as pending, " + + "regardless of whether its release tag already exists.", + ); +} + +if (publishable.length === 0) { + console.log("No publishable workspaces found."); + setAzureOutput("shouldBuild", "false"); + process.exit(0); +} + +const pending = publishable.filter(isPending); + +console.log(`Publishable workspaces: ${publishable.length}`); +console.log(`Pending release: ${pending.length}`); + +if (pending.length > 0) { + console.log("\nPackages pending release:"); + for (const { name, version, tag, crates } of pending) { + const suffix = + crates.length > 0 + ? ` (+ crates ${crates.map(crate => crate.crateName).join(", ")})` + : ""; + console.log(` - ${name}@${version} [${tag}]${suffix}`); + } +} + +if (process.env.TF_BUILD) { + console.log( + `##vso[build.updatebuildnumber]release-prep-${process.env.BUILD_BUILDID || "local"}`, + ); +} +setAzureOutput("shouldBuild", pending.length > 0 ? "true" : "false"); + +if (CHECK_ONLY) { + process.exit(0); +} + +if (pending.length === 0) { + // This mode only runs once the earlier `--check-only` step already + // observed at least one pending workspace and gated the `BuildArtifacts` + // stage on it (`shouldBuild == 'true'`) — so reaching this point with + // zero pending workspaces means every previously-pending workspace's + // release tag appeared on `origin` in the window between that check + // and this pack step. That is almost always a concurrent release run + // (another `FAST CD Build`/`FAST CD` execution) winning the race, not a + // normal "nothing to do" outcome, so fail loudly here instead of + // silently exiting without writing `release-manifest.json` (which would + // otherwise surface later as a confusing "file not found" error when the + // pipeline tries to copy that manifest out of this job). + logError( + "No packages are pending release, but pack-pending-releases.mjs was invoked " + + "in packing mode after an earlier check found pending packages. This " + + "indicates a concurrent release run already tagged every previously-pending " + + "workspace between the check-only step and this pack step. Re-run 'FAST CD " + + "Build' if packages are still expected to be pending.", + ); + process.exit(1); +} + +mkdirSync(NPM_DIR, { recursive: true }); +mkdirSync(CRATES_DIR, { recursive: true }); +mkdirSync(META_DIR, { recursive: true }); + +const manifestPackages = []; +let hasErrors = false; + +for (const { name, version, tag, prefix, location, crates } of pending) { + try { + console.log(`\nPacking ${name}@${version} from ${location}...`); + const packJson = run("npm", [ + "pack", + "--silent", + "--json", + `--workspace=${location}`, + `--pack-destination=${resolve(NPM_DIR)}`, + ]); + const npmTarball = JSON.parse(packJson)[0].filename; + + const crateFiles = []; + for (const { crateName, cargoTomlPath } of crates) { + console.log(`Packaging crate ${crateName}@${version}...`); + run( + "cargo", + [ + "package", + "--no-verify", + "--allow-dirty", + "--manifest-path", + cargoTomlPath, + ], + { stdio: "inherit" }, + ); + const srcCrate = join( + dirname(cargoTomlPath), + "target", + "package", + `${crateName}-${version}.crate`, + ); + if (!existsSync(srcCrate)) { + throw new Error( + `Expected ${srcCrate} after cargo package, but it does not exist.`, + ); + } + const destCrate = join(CRATES_DIR, basename(srcCrate)); + copyFileSync(srcCrate, destCrate); + crateFiles.push(basename(srcCrate)); + } + + manifestPackages.push({ name, version, tag, prefix, npmTarball, crateFiles }); + console.log(`Packed ${name}@${version} (${1 + crateFiles.length} asset(s))`); + } catch (error) { + hasErrors = true; + const message = error instanceof Error ? error.message : String(error); + console.error(`Failed to pack ${name}@${version}: ${message}`); + } +} + +if (manifestPackages.every(pkg => pkg.crateFiles.length === 0)) { + // Guarantee `publish_artifacts_crates` always has at least one file so + // `PublishPipelineArtifact@1` (and any downstream `DownloadPipelineArtifact@2`) + // never has to handle a truly-empty directory. The `PublishRelease` + // stage's `Publish` job strips this placeholder back out before + // invoking the release template, so an all-npm batch still ends up + // treated as "no crate assets to publish". + writeFileSync(join(CRATES_DIR, ".no-crates-packed"), ""); +} + +const releaseCommit = ( + process.env.BUILD_SOURCEVERSION || run("git", ["rev-parse", "HEAD"]) +).trim(); + +writeFileSync( + MANIFEST_PATH, + `${JSON.stringify({ releaseCommit, packages: manifestPackages }, null, 4)}\n`, +); + +console.log(`\nPacked: ${manifestPackages.length}/${pending.length}`); +console.log(`Manifest written to ${MANIFEST_PATH}`); + +if (hasErrors) { + process.exitCode = 1; +} diff --git a/build/scripts/read-release-manifest.mjs b/build/scripts/read-release-manifest.mjs new file mode 100644 index 00000000000..7a4db8be923 --- /dev/null +++ b/build/scripts/read-release-manifest.mjs @@ -0,0 +1,80 @@ +#!/usr/bin/env node +/** + * Read `release-manifest.json` (written by `pack-pending-releases.mjs` in + * the `FAST CD Build` pipeline's `BuildArtifacts` stage) and cross-reference + * it against the workspaces that are publishable right now — from a fresh + * `checkout: self` in the `FAST CD` pipeline — to emit one set of Azure + * Pipelines output variables per currently-publishable workspace: + * + * - `NeedsRelease` - `"true"` when the workspace was packed by + * the build pipeline, `"false"` otherwise. + * - `ReleaseTag` - the workspace's `${name}_v${version}` tag. + * - `ReleaseVersion` - the workspace's version. + * + * `.ado/pipelines/azure-pipelines-cd.yml`'s `PublishRelease` stage declares + * one static `GitHubRelease@1` task per known publishable workspace + * (Azure Pipelines cannot create tasks dynamically from manifest content), + * each conditioned on that workspace's `NeedsRelease` variable. + * `check-publish-pipeline.mjs` verifies every current publishable workspace + * has matching coverage there. + * + * Usage: node build/scripts/read-release-manifest.mjs + */ + +import { readFileSync } from "node:fs"; +import { listPublishableWorkspaces } from "./lib/publishable-workspaces.mjs"; + +const manifestPath = process.argv[2]; +if (!manifestPath) { + console.error("Usage: read-release-manifest.mjs "); + process.exit(1); +} + +function setAzureOutput(name, value) { + console.log(`##vso[task.setvariable variable=${name};isOutput=true]${value}`); +} + +const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + +if (!/^[0-9a-f]{40}$/.test(manifest.releaseCommit || "")) { + console.error( + `##vso[task.logissue type=error]Invalid release commit in manifest: ${manifest.releaseCommit}`, + ); + process.exit(1); +} + +const packagesByName = new Map((manifest.packages || []).map(pkg => [pkg.name, pkg])); +const publishable = listPublishableWorkspaces(); +const publishableNames = new Set(publishable.map(workspace => workspace.name)); +let pendingCount = 0; + +for (const workspace of publishable) { + const packed = packagesByName.get(workspace.name); + const needsRelease = Boolean(packed); + if (needsRelease) pendingCount += 1; + + setAzureOutput(`${workspace.prefix}NeedsRelease`, needsRelease ? "true" : "false"); + setAzureOutput(`${workspace.prefix}ReleaseTag`, packed ? packed.tag : workspace.tag); + setAzureOutput( + `${workspace.prefix}ReleaseVersion`, + packed ? packed.version : workspace.version, + ); +} + +setAzureOutput("releaseCommit", manifest.releaseCommit); +console.log(`Pending releases: ${pendingCount}/${publishable.length}`); + +for (const pkg of manifest.packages || []) { + if (!publishableNames.has(pkg.name)) { + console.log( + `##vso[task.logissue type=warning]${pkg.name} was packed but is no longer a publishable workspace on this commit.`, + ); + } +} + +if (pendingCount === 0) { + console.log( + "##vso[task.logissue type=error]No packages are pending release, but the CD pipeline was triggered.", + ); + process.exit(1); +}