fix(release): npm needs an explicit dist-tag for a prerelease, chosen from what the registry holds

.github/workflows/release.yml

@@ -134,6 +134,8 @@ jobs:
     needs: gate
     if: needs.gate.outputs.package == 'typescript'
     runs-on: ubuntu-latest
+    outputs:
+      dist_tag: ${{ steps.dist_tag.outputs.tag }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
@@ -159,6 +161,21 @@ jobs:
             python3 tools/check_artefact.py npm "$tarball" "$VERSION"
           done
 
+      - name: The dist-tag, from what the registry already holds
+        # npm 11 refuses a prerelease without `--tag`, and a fixed tag is wrong one way or the
+        # other: `latest` would put every candidate over the last final, `next` would leave an
+        # unpinned install on the version published by hand. `tools/npm_dist_tag.py` decides, and
+        # it runs here because the publishing job must not check out this repository.
+        id: dist_tag
+        env:
+          VERSION: ${{ needs.gate.outputs.version }}
+        run: |
+          set -euo pipefail
+          # Public metadata, no credential. A failed lookup stops the release: an empty list would
+          # let a prerelease take `latest` from a final.
+          published=$(npm view "@smart-data-engines/sde" versions --json)
+          python3 tools/npm_dist_tag.py "$VERSION" "$published" >> "$GITHUB_OUTPUT"
+
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
           name: npm-tarball
@@ -195,17 +212,32 @@ jobs:
 
       - name: Publish
         # No NODE_AUTH_TOKEN and no .npmrc. Trusted publishing generates the provenance attestation
-        # by itself, so `--provenance` is not passed: npm does it and it cannot be forgotten.
+        # by itself, so `--provenance` is not passed: npm does it and it cannot be forgotten. The tag
+        # is always explicit, chosen by the build job from the registry.
+        env:
+          DIST_TAG: ${{ needs.build-npm.outputs.dist_tag }}
         run: |
           set -euo pipefail
+          test -n "$DIST_TAG"
           tarball=$(ls ./*.tgz)
-          npm publish "$tarball" --access public
+          npm publish "$tarball" --access public --tag "$DIST_TAG"
 
-      - name: It is actually there
+      - name: It is actually there, under its tag
         env:
           VERSION: ${{ needs.gate.outputs.version }}
+          DIST_TAG: ${{ needs.build-npm.outputs.dist_tag }}
         run: |
           set -euo pipefail
-          published=$(npm view "@smart-data-engines/sde@$VERSION" version)
-          test "$published" = "$VERSION"
-          echo "@smart-data-engines/sde@$published is on the registry"
+          # Bounded: the registry's metadata can trail the publish by seconds, and a check that
+          # waits for ever is not a check.
+          for attempt in 1 2 3 4 5 6 7 8 9 10; do
+            published=$(npm view "@smart-data-engines/sde@$VERSION" version 2>/dev/null || true)
+            tagged=$(npm view "@smart-data-engines/sde" "dist-tags.$DIST_TAG" 2>/dev/null || true)
+            if [ "$published" = "$VERSION" ] && [ "$tagged" = "$VERSION" ]; then
+              echo "@smart-data-engines/sde@$published is on the registry as $DIST_TAG"
+              exit 0
+            fi
+            sleep 6
+          done
+          echo "::error::after 10 attempts the registry shows version '$published' and $DIST_TAG '$tagged', not $VERSION"
+          exit 1

docs/publishing.md

@@ -572,6 +572,19 @@ upload:
 | the artefact is missing a licence, `py.typed`, or `dist/` | All three have actually been missing, on 8 September, and none of it was visible from a green suite — the suite runs the source tree and a user runs the artefact. The worst would have put an importable-looking package with no code in it under our own scope. |
 | the artefact records a version other than the tag's | The gate agreeing with the manifest does not prove the *build* used it, and what a user installs is the number inside the file. |
 
+**The npm dist-tag is chosen, not defaulted.** npm 11 refuses a prerelease without `--tag`, and it
+does so in the publishing job, after the gate and the reviewer have both said yes. A fixed tag would
+be wrong in one direction or the other. So the build job reads the versions the registry holds, and
+`tools/npm_dist_tag.py` picks the tag:
+- a final version gets `latest`;
+- a prerelease gets `latest` while no final exists, because an unpinned `pip install` gets the
+  newest prerelease on PyPI too;
+- a prerelease gets `next` once a final exists.
+
+The script refuses a version already published, and any version lower than one already published,
+because it would move a tag backwards. The publishing job passes the tag explicitly and then checks
+that the registry shows the version under it.
+
 `tools/check_artefact.py` carries a control in the same run: one member that cannot exist must be
 reported absent. A checker stuck on "present" would find every required file and report a flawless
 package, which is the shape of good news worth distrusting.
@@ -598,11 +611,17 @@ package, which is the shape of good news worth distrusting.
 
    ```bash
    cd typescript
-   npm login                      # 2FA, the auth-and-writes mode from section 1
-   npm publish --access public    # prepack builds; this is the 0.1.0-dev.0 already on PyPI
+   npm login                                  # 2FA, the auth-and-writes mode from section 1
+   npm publish --access public --tag latest   # prepack builds; the 0.1.0-dev.0 already on PyPI
    npm view @smart-data-engines/sde version
    ```
 
+   **`--tag latest` is not optional.** npm 11 refuses to publish a prerelease without an explicit
+   tag: "You must specify a tag using --tag when publishing a prerelease version"
+   (`lib/commands/publish.js` in npm 11.15.0). It is `latest` because no final version exists, which
+   is what `tools/npm_dist_tag.py` would choose too. The first release candidate through the
+   workflow then takes `latest` from it.
+
    Then configure the publisher, either on the package page or in one command:
 
    ```bash

python/tests/test_release.py

@@ -51,6 +51,7 @@ def _tool(name: str) -> ModuleType:
 
 release_tag = _tool("release_tag")
 check_artefact = _tool("check_artefact")
+npm_dist_tag = _tool("npm_dist_tag")
 
 
 # --------------------------------------------------------------------------------------------------
@@ -369,3 +370,104 @@ def test_the_security_document_counts_the_required_checks_the_ruleset_requires()
             f"the ruleset requires {count} checks ({WORDS[count]}) and the document says {word!r}. "
             f"Adding a matrix entry or an analysis changes the first number and not the second."
         )
+
+
+# --------------------------------------------------------------------------------------------------
+# The npm dist-tag
+# --------------------------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("version", "published", "tag"),
+    [
+        # The first candidate after the version published by hand: no final yet, so `latest`,
+        # which is what an unpinned `pip install` gets on PyPI too.
+        ("0.1.0-rc.1", ["0.1.0-dev.0"], "latest"),
+        ("0.1.0-rc.2", ["0.1.0-dev.0", "0.1.0-rc.1"], "latest"),
+        ("0.1.0", ["0.1.0-dev.0", "0.1.0-rc.1"], "latest"),
+        # Once a final exists, a candidate must not take `latest` from it.
+        ("0.2.0-rc.1", ["0.1.0-dev.0", "0.1.0"], "next"),
+        ("0.2.0", ["0.1.0", "0.2.0-rc.1"], "latest"),
+        # Semantic Versioning precedence, not string order: rc.10 is after rc.9, and a longer
+        # prerelease ranks above its prefix.
+        ("0.2.0-rc.10", ["0.1.0", "0.2.0-rc.9"], "next"),
+        ("0.2.0-rc.1.1", ["0.1.0", "0.2.0-rc.1"], "next"),
+    ],
+)
+def test_the_dist_tag_follows_what_an_unpinned_install_should_get(
+    version: str, published: list[str], tag: str
+) -> None:
+    assert npm_dist_tag.choose(version, published)[0] == tag
+
+
+@pytest.mark.parametrize(
+    ("version", "published", "reason"),
+    [
+        ("0.1.0-rc.1", ["0.1.0-dev.0", "0.1.0-rc.1"], "already on the registry"),
+        ("0.1.0-rc.1", ["0.1.0-rc.2"], "lower than 0.1.0-rc.2"),
+        ("0.1.0-rc.9", ["0.1.0-rc.10"], "lower than 0.1.0-rc.10"),
+        ("0.1.1", ["0.2.0"], "lower than 0.2.0"),
+        ("0.1.1-rc.1", ["0.1.0", "0.2.0"], "lower than 0.2.0"),
+        ("0.1", ["0.1.0-dev.0"], "not a semantic version"),
+        ("01.0.0", ["0.1.0-dev.0"], "not a semantic version"),
+    ],
+)
+def test_a_version_that_would_move_a_tag_backwards_or_repeat_is_refused(
+    version: str, published: list[str], reason: str
+) -> None:
+    with pytest.raises(npm_dist_tag.Refused, match=re.escape(reason)):
+        npm_dist_tag.choose(version, published)
+
+
+def test_the_registry_answer_is_read_as_npm_prints_it() -> None:
+    """`npm view <package> versions --json` prints a string when the registry holds one version."""
+    assert npm_dist_tag.published_versions('"0.1.0-dev.0"') == ["0.1.0-dev.0"]
+    assert npm_dist_tag.published_versions('["0.1.0-dev.0", "0.1.0-rc.1"]') == [
+        "0.1.0-dev.0",
+        "0.1.0-rc.1",
+    ]
+    for broken in ("[]", "", "not json", "{}", "[1]", '["0.1"]'):
+        with pytest.raises(npm_dist_tag.Refused):
+            npm_dist_tag.published_versions(broken)
+
+
+def test_the_script_prints_one_tag_for_github_output_and_refuses_with_status_1(
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    assert npm_dist_tag.main(["npm_dist_tag.py", "0.1.0-rc.1", '"0.1.0-dev.0"']) == 0
+    captured = capsys.readouterr()
+    assert captured.out == "tag=latest\n"
+    assert "no final version is published" in captured.err
+    assert npm_dist_tag.main(["npm_dist_tag.py", "0.1.0-rc.1", "[]"]) == 1
+    assert capsys.readouterr().out == ""
+
+
+def test_no_npm_publish_in_this_repository_goes_without_an_explicit_tag() -> None:
+    """npm 11 refuses a prerelease without `--tag`, after the gate and the reviewer have said yes.
+
+    The release workflow published with `npm publish "$tarball" --access public`, and the runbook
+    told a person to type the same for the first, hand-made publish. Both were prereleases. So
+    every `npm publish` a workflow runs or a document tells someone to type must name its tag.
+    """
+    places = [ROOT / ".github" / "workflows" / "release.yml", ROOT / "docs" / "publishing.md"]
+    found = 0
+    for path in places:
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if re.search(r"\bnpm publish\b", line) and not line.lstrip().startswith(("#", "//")):
+                if "`npm publish`" in line or "npm publish --provenance" in line:
+                    continue  # prose naming the command, not an invocation
+                found += 1
+                assert "--tag" in line, f"{path.relative_to(ROOT)}: {line.strip()}"
+    assert found >= 2, "the check found no invocation to look at, so it would pass on anything"
+
+
+def test_the_publish_job_takes_the_tag_the_build_job_chose_from_the_registry() -> None:
+    workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8")
+    build = workflow.split("  build-npm:", 1)[1].split("\n  publish-npm:", 1)[0]
+    publish = workflow.split("  publish-npm:", 1)[1]
+    assert "dist_tag: ${{ steps.dist_tag.outputs.tag }}" in build
+    assert 'npm view "@smart-data-engines/sde" versions --json' in build
+    assert 'python3 tools/npm_dist_tag.py "$VERSION" "$published" >> "$GITHUB_OUTPUT"' in build
+    assert publish.count("DIST_TAG: ${{ needs.build-npm.outputs.dist_tag }}") == 2
+    assert 'npm publish "$tarball" --access public --tag "$DIST_TAG"' in publish
+    assert '"dist-tags.$DIST_TAG"' in publish

tools/npm_dist_tag.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Which npm dist-tag does this version get, given what the registry already holds?
+
+npm needs the answer spelled out. Since npm 11 `npm publish` of a prerelease without `--tag` is an
+error ("You must specify a tag using --tag when publishing a prerelease version", in
+`lib/commands/publish.js` of npm 11.15.0, the version the release workflow installs), so the first
+release-candidate run of `release.yml` would have failed at the publish step, after the gate and the
+reviewer had both said yes. A literal `--tag latest` would publish every candidate over the last
+final, and a literal `--tag next` would leave an unpinned `npm install` on whatever was published by
+hand before any candidate existed.
+
+So the tag follows what an unpinned install should get, the way `pip` behaves on PyPI:
+
+- a final version is `latest`;
+- a prerelease is `latest` while no final version exists - pip installs the newest prerelease of a
+  project that has no final release, and an npm user should get the same version;
+- a prerelease is `next` once a final exists, so `latest` stays on the final.
+
+**Refused, because a person has to decide:**
+- a version already on the registry, which npm will not accept again;
+- any version lower than one already published. As `latest` or as `next` it would move the tag
+  backwards, and a maintenance release of an older line needs a tag of its own that nobody has
+  chosen yet;
+- a registry that lists nothing. The package exists before this runs - trusted publishing is
+  configured on an existing package, so the first publish is by hand (docs/publishing.md §5.3) -
+  and an empty list here means the lookup failed, not that the package is new.
+
+Usage, with the output of `npm view <package> versions --json` (a list, or a string when the
+registry holds one version):
+
+    python3 tools/npm_dist_tag.py 0.1.0-rc.1 '["0.1.0-dev.0"]'
+
+Prints `tag=` for `$GITHUB_OUTPUT` and says why on stderr. Exit status 1 on any refusal.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from typing import Any
+
+#: Semantic Versioning 2.0.0, as npm uses it: no leading zeros in numbers, build metadata allowed.
+SEMVER = re.compile(
+    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
+    r"(?:-(?P<pre>(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*)"
+    r"(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*))*))?"
+    r"(?:\+(?P<build>[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$"
+)
+
+
+class Refused(Exception):
+    """The version gets no tag from this script, and the message says why."""
+
+
+def parse(version: str) -> tuple[tuple[int, int, int], tuple[str, ...]]:
+    match = SEMVER.match(version)
+    if match is None:
+        raise Refused(f"{version!r} is not a semantic version, so npm would not accept it either.")
+    core = (int(match["major"]), int(match["minor"]), int(match["patch"]))
+    pre = tuple(match["pre"].split(".")) if match["pre"] else ()
+    return core, pre
+
+
+def _identifier_key(identifier: str) -> tuple[int, int, str]:
+    # Numeric identifiers compare numerically and rank below alphanumeric ones (SemVer 11.4).
+    if identifier.isdigit():
+        return (0, int(identifier), "")
+    return (1, 0, identifier)
+
+
+def precedence(version: str) -> tuple[Any, ...]:
+    """A sort key with Semantic Versioning precedence; build metadata does not take part."""
+    core, pre = parse(version)
+    if not pre:
+        # Without a prerelease a version ranks above every prerelease of its core (SemVer 11.3).
+        return (core, 1, ())
+    return (core, 0, tuple(_identifier_key(item) for item in pre))
+
+
+def published_versions(raw: str) -> list[str]:
+    try:
+        value = json.loads(raw)
+    except json.JSONDecodeError:
+        raise Refused(f"the registry's version list is not JSON: {raw[:200]!r}") from None
+    if isinstance(value, str):
+        value = [value]
+    if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+        raise Refused(f"the registry's version list is not a list of strings: {raw[:200]!r}")
+    if not value:
+        raise Refused(
+            "the registry lists no version of this package. It exists before any tag publishes "
+            "it - the first publish is by hand, because trusted publishing is configured on an "
+            "existing package (docs/publishing.md 5.3) - so an empty list means the lookup failed."
+        )
+    for item in value:
+        parse(item)
+    return value
+
+
+def choose(version: str, published: list[str]) -> tuple[str, str]:
+    """The dist-tag for `version`, and the sentence that explains it."""
+    _, pre = parse(version)
+    if version in published:
+        raise Refused(
+            f"{version} is already on the registry, and npm never accepts a version number twice. "
+            f"Bump the manifest, merge it, and tag that commit."
+        )
+    key = precedence(version)
+    higher = sorted((v for v in published if precedence(v) > key), key=precedence)
+    if higher:
+        raise Refused(
+            f"{version} is lower than {higher[-1]}, already published. As `latest` or `next` it "
+            f"would move the tag backwards; a release of an older line needs a dist-tag of its "
+            f"own, and choosing it is not this script's decision."
+        )
+    finals = sorted((v for v in published if not parse(v)[1]), key=precedence)
+    if not pre:
+        return "latest", f"{version} is a final version, so it becomes `latest`."
+    if not finals:
+        return "latest", (
+            f"{version} is a prerelease and no final version is published, so it becomes `latest`: "
+            f"an unpinned install gets the newest prerelease, as pip does on PyPI."
+        )
+    return "next", (
+        f"{version} is a prerelease and {finals[-1]} is the final on `latest`, so it becomes "
+        f"`next`."
+    )
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 3:
+        print("usage: npm_dist_tag.py VERSION PUBLISHED_VERSIONS_JSON", file=sys.stderr)
+        return 2
+    try:
+        tag, why = choose(argv[1], published_versions(argv[2]))
+    except Refused as refusal:
+        print(f"::error::{refusal}", file=sys.stderr)
+        return 1
+    print(why, file=sys.stderr)
+    print(f"tag={tag}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))