diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 78ca603b..762af3e8 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -2,7 +2,9 @@ name: Publish uipath-ipc to PyPI # Publishes uipath-ipc to public PyPI via Trusted Publishing (OIDC) — no tokens. # ADO stays primary; on a green publish it POSTs a repository_dispatch (publish-pypi) -# with the released commit SHA, and this builds & publishes that commit. Only master +# with the released commit SHA and the exact version CI built (release or .devN +# prerelease), and this builds that commit at that version and publishes it — so +# public PyPI matches the internal feed (no stripped-to-clean surprise). Only master # publishes (dispatch runs the default-branch file; the build refuses non-master # commits) and there is no workflow_dispatch, so it can't fire by accident. Prereqs: # a PyPI Trusted Publisher (UiPath/coreipc, workflow cd.yml, env pypi) + protected `pypi` env. @@ -21,7 +23,6 @@ concurrency: env: PACKAGE_DIR: src/Clients/python/uipath-ipc - CSPROJ: src/UiPath.CoreIpc/UiPath.CoreIpc.csproj jobs: build: @@ -30,9 +31,8 @@ jobs: outputs: version: ${{ steps.resolve.outputs.version }} steps: - # Untrusted input (client_payload.sha) is read via env, never inlined into - # a run: script, and validated as a 40-char hex commit — script-injection - # hardening + rejects tags/branch-names (we require an exact commit). + # Untrusted inputs (client_payload.*) are read via env, never inlined into + # a run: script (script-injection hardening). - name: Resolve & validate commit SHA id: ref env: @@ -73,44 +73,40 @@ jobs: with: python-version: '3.12' - # Version comes from the SAME commit we build (single source of truth) and - # is validated as a clean PEP 440 RELEASE version. A CI/dev build (local - # +segment) or a non-PEP-440 string (e.g. .NET-style '2.5.2-rc1') is - # REJECTED rather than silently mangled into the wrong release. - - name: Resolve & validate version (PEP 440, no local segment) + # The version to publish is decided by ADO (the exact version CI built and + # pushed to the internal feed) and passed in the dispatch. We accept PEP 440 + # RELEASES and PRE-RELEASES (e.g. 2.5.3, 2.5.3.dev123, 2.5.3rc1); we reject + # only local (+...) segments, which PyPI forbids. + - name: Resolve & validate version (PEP 440) id: resolve + env: + DISPATCH_VERSION: ${{ github.event.client_payload.version }} run: | set -euo pipefail python -m pip install --quiet --upgrade packaging - python - "$CSPROJ" <<'PY' - import os, sys, xml.etree.ElementTree as ET + RAW="$DISPATCH_VERSION" python - <<'PY' + import os, sys from packaging.version import Version, InvalidVersion - root = ET.parse(sys.argv[1]).getroot() - vals = [e.text.strip() for e in root.iter() - if e.tag.split('}')[-1] == 'Version' and (e.text or '').strip()] - if not vals: - sys.exit(f"::error::No found in {sys.argv[1]}") - raw = vals[0] - + raw = os.environ.get("RAW", "").strip() + if not raw: + sys.exit("::error::No client_payload.version provided.") if "+" in raw: - sys.exit(f"::error::Version '{raw}' has a PEP 440 local segment (+...); " - "that is a CI/dev build, not a release — refusing to publish.") + sys.exit(f"::error::Version '{raw}' has a PEP 440 local segment (+...); PyPI forbids it — refusing.") try: - v = Version(raw) # rejects .NET-style '2.5.2-rc1' + v = Version(raw) except InvalidVersion: - sys.exit(f"::error::Version '{raw}' is not a valid PEP 440 release version. " - "Set a clean version (e.g. 2.5.2 or 2.5.2rc1) before publishing.") + sys.exit(f"::error::Version '{raw}' is not a valid PEP 440 version.") if v.local: sys.exit(f"::error::Version '{raw}' has a local segment — refusing.") with open(os.environ["GITHUB_OUTPUT"], "a") as fh: fh.write(f"version={raw}\n") - print(f"Publishing version: {raw}") + print(f"Publishing version: {raw}" + (" (pre-release)" if v.is_prerelease else "")) PY - name: Stamp version into pyproject.toml - # Reuse the repo's own stamper (single source of truth). With a clean - # dash-free version its to_pep440() is the identity. + # The version is already PEP 440 (no dash), so stamp-python-version.py's + # to_pep440() writes it verbatim. run: | set -euo pipefail python src/CI/stamp-python-version.py \ diff --git a/src/CI/azp-publish.yaml b/src/CI/azp-publish.yaml index 88c75f31..46ef8185 100644 --- a/src/CI/azp-publish.yaml +++ b/src/CI/azp-publish.yaml @@ -6,9 +6,12 @@ # behind their approval-gated environments. Builds nothing itself. # # Parameters: -# - buildId (required) the CI build/run id to publish from. -# - publishNuGet / publishNpm / publishPyPI (default: all on) — which -# packages to publish. At least one must be selected. +# - buildId (required) the CI build/run id to publish from. +# - publishNuGet / publishNpm / publishPython (default: all on) — which +# packages to publish to the INTERNAL (UiPath-Internal) feeds. +# - publishToPublicPyPI (default off) — ALSO publish Python to PUBLIC PyPI +# (master builds only; via the GitHub Action). Separate from publishPython +# so you can publish internally without touching public PyPI. # # The `ci` pipeline resource below is hardcoded to the CI pipeline (named # "CI", running azp-start.yaml). It resolves the CI definition id for the @@ -36,11 +39,16 @@ parameters: type: boolean default: true - - name: publishPyPI + - name: publishPython displayName: 'Publish Python (wheel + sdist) → UiPath-Internal' type: boolean default: true + - name: publishToPublicPyPI + displayName: 'ALSO publish Python → PUBLIC PyPI (master builds only)' + type: boolean + default: false + resources: pipelines: - pipeline: ci # referenced as $(resources.pipeline.ci.pipelineID) @@ -73,7 +81,7 @@ stages: exit 1 displayName: 'buildId is required' -- ${{ if not(or(eq(parameters.publishNuGet, true), eq(parameters.publishNpm, true), eq(parameters.publishPyPI, true))) }}: +- ${{ if not(or(eq(parameters.publishNuGet, true), eq(parameters.publishNpm, true), eq(parameters.publishPython, true), eq(parameters.publishToPublicPyPI, true))) }}: - stage: Nothing_Selected displayName: '❌ Nothing selected' jobs: @@ -83,9 +91,9 @@ stages: steps: - checkout: none - script: | - echo "##vso[task.logissue type=error]Select at least one package to publish (NuGet, NPM, or PyPI)." + echo "##vso[task.logissue type=error]Select at least one target (NuGet, NPM, Python, or public PyPI)." exit 1 - displayName: 'At least one package required' + displayName: 'At least one target required' # ===================================================================== # Publish stages — only emitted when a real buildId was provided. Each runs @@ -98,7 +106,7 @@ stages: # ===================================================================== - ${{ if and(not(in(parameters.buildId, '', '0')), eq(parameters.publishNuGet, true)) }}: - stage: Publish_NuGet - displayName: '🚚 Publish NuGet' + displayName: '🚚 Publish NuGet → UiPath-Internal' dependsOn: [] jobs: - deployment: Publish_NuGet_Package @@ -118,7 +126,7 @@ stages: - ${{ if and(not(in(parameters.buildId, '', '0')), eq(parameters.publishNpm, true)) }}: - stage: Publish_NPM - displayName: '🚚 Publish NPM' + displayName: '🚚 Publish NPM → uipath-ipc-deps' dependsOn: [] jobs: - deployment: Publish_NPM_Packages @@ -136,12 +144,14 @@ stages: reuseArtifactsFromBuildId: ${{ parameters.buildId }} sourcePipelineId: $(resources.pipeline.ci.pipelineID) -- ${{ if and(not(in(parameters.buildId, '', '0')), eq(parameters.publishPyPI, true)) }}: - - stage: Publish_PyPI - displayName: '🚚 Publish PyPI' +# Publish Python to the INTERNAL feed (UiPath-Internal). NOT public PyPI — +# that is the separate publishToPublicPyPI lever + Publish_PublicPyPI stage below. +- ${{ if and(not(in(parameters.buildId, '', '0')), eq(parameters.publishPython, true)) }}: + - stage: Publish_Python + displayName: '🚚 Publish Python → UiPath-Internal' dependsOn: [] jobs: - - deployment: Publish_PyPI_Package + - deployment: Publish_Python_Package displayName: '📦 Publish Python wheel + sdist to UiPath-Internal' environment: 'PyPI-Packages' pool: @@ -156,21 +166,31 @@ stages: reuseArtifactsFromBuildId: ${{ parameters.buildId }} sourcePipelineId: $(resources.pipeline.ci.pipelineID) -# Whisper to GitHub to publish the same release to public PyPI. dependsOn every -# emitted publish stage with condition: succeeded(), so it fires only if all of them -# succeeded — no PyPI publish on any ADO failure. It resolves the published buildId's -# exact commit (ADO REST) and gates upstream — only a genuine NEW MASTER release -# whispers (branch builds and already-published versions are skipped), so the GitHub -# Action is never triggered just to refuse. Needs var group `github-dispatch`. -- ${{ if and(not(in(parameters.buildId, '', '0')), eq(parameters.publishPyPI, true)) }}: - - stage: Notify_GitHub - displayName: '📣 Notify GitHub → PyPI (master, new version only)' +# ===================================================================== +# Publish Python to PUBLIC PyPI — via GitHub Actions (Trusted Publishing/OIDC). +# ===================================================================== +# Only emitted when `publishToPublicPyPI` is set (separate lever from the +# internal-feed publish above). It resolves the build's branch + commit (ADO +# REST) and reads the EXACT version from the wheel CI built — so public PyPI +# carries the same version as the internal feed (release, or .devN prerelease), +# never a stripped-to-clean surprise. It then: +# - non-master build -> skips (green): the GitHub Action is never fired +# just to refuse. +# - version already on PyPI -> ends YELLOW (SucceededWithIssues): the +# at-most-once semantic hit an existing version; no failure, no no-op run. +# - new master version -> POSTs the repository_dispatch (sha + version). +# dependsOn the emitted internal publish stages + condition: succeeded() => +# no public publish on ANY ADO failure. Needs var group `github-dispatch`. +- ${{ if and(not(in(parameters.buildId, '', '0')), eq(parameters.publishToPublicPyPI, true)) }}: + - stage: Publish_PublicPyPI + displayName: '📣 Publish Python → PUBLIC PyPI (master, new version only)' dependsOn: - ${{ if eq(parameters.publishNuGet, true) }}: - Publish_NuGet - ${{ if eq(parameters.publishNpm, true) }}: - Publish_NPM - - Publish_PyPI + - ${{ if eq(parameters.publishPython, true) }}: + - Publish_Python condition: succeeded() variables: - group: github-dispatch # supplies secret GITHUB_DISPATCH_TOKEN @@ -181,18 +201,29 @@ stages: vmImage: 'ubuntu-latest' steps: - checkout: none + # The published version = the EXACT version of the wheel CI built, so + # public PyPI matches the internal feed. Pull that artifact to read it. + - task: DownloadPipelineArtifact@2 + displayName: 'Download Python package from build ${{ parameters.buildId }}' + inputs: + buildType: specific + project: $(System.TeamProject) + pipeline: $(resources.pipeline.ci.pipelineID) + buildVersionToDownload: specific + buildId: ${{ parameters.buildId }} + artifactName: 'Python package' + targetPath: '$(Pipeline.Workspace)/pypkg' - script: | set -euo pipefail - # 1. Resolve the published build's branch + commit (ADO REST). + # 1. Resolve the build's branch + commit (ADO REST). build=$(curl -sSf -H "Authorization: Bearer $SYSTEM_ACCESSTOKEN" \ "${SYSTEM_COLLECTIONURI}${SYSTEM_TEAMPROJECT}/_apis/build/builds/${BUILD_ID}?api-version=7.1") SHA=$(printf '%s' "$build" | jq -r '.sourceVersion') BRANCH=$(printf '%s' "$build" | jq -r '.sourceBranch') echo "buildId ${BUILD_ID}: branch=${BRANCH} commit=${SHA}" - # 2. GATE — public PyPI gets MASTER releases only. For anything else, - # skip the whisper entirely so the GitHub Action is never triggered - # just to refuse (saves a wasted runner). + # 2. GATE — public PyPI gets MASTER builds only. Anything else: skip + # (green) so the GitHub Action is never triggered just to refuse. if [ "$BRANCH" != "refs/heads/master" ]; then echo "Not a master build (${BRANCH}) — skipping public PyPI dispatch." exit 0 @@ -202,34 +233,43 @@ stages: exit 1 fi - # 3. GATE — skip if this version is already on PyPI, so we never fire a - # no-op run. Version = from the csproj AT that commit - # (public repo → unauthenticated raw read). - VERSION=$(curl -sSf -H "Accept: application/vnd.github.raw" \ - "https://api.github.com/repos/UiPath/coreipc/contents/src/UiPath.CoreIpc/UiPath.CoreIpc.csproj?ref=${SHA}" \ - | grep -oPm1 '(?<=)[^<]+' | tr -d '[:space:]') + # 3. Read the EXACT version from the wheel CI built (from its + # metadata, so it is un-escaped): release or .devN prerelease. + WHEEL=$(ls "$PYPKG_DIR"/*.whl 2>/dev/null | head -1 || true) + if [ -z "$WHEEL" ]; then + echo "##vso[task.logissue type=error]No wheel found in the Python artifact." + exit 1 + fi + VERSION=$(unzip -p "$WHEEL" '*.dist-info/METADATA' | sed -n 's/^Version: //p' | head -1) if [ -z "$VERSION" ]; then - echo "##vso[task.logissue type=error]Could not read from csproj at ${SHA}." + echo "##vso[task.logissue type=error]Could not read Version from the wheel metadata." exit 1 fi + echo "Wheel version: ${VERSION}" + + # 4. GATE — at-most-once. If PyPI already has this version, end + # YELLOW (SucceededWithIssues) to flag the collision; do NOT fail, + # and do NOT fire a no-op GitHub Action run. code=$(curl -sS -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/uipath-ipc/${VERSION}/json") if [ "$code" = "200" ]; then - echo "uipath-ipc ${VERSION} already on PyPI — nothing to publish, skipping dispatch." + echo "##vso[task.logissue type=warning]uipath-ipc ${VERSION} is already on PyPI — at-most-once: skipping public publish." + echo "##vso[task.complete result=SucceededWithIssues;]already on PyPI" exit 0 fi - # 4. New master release → whisper to GitHub to publish it. + # 5. New master version → whisper to GitHub to publish it. echo "Dispatching publish-pypi for uipath-ipc ${VERSION} (${SHA})" curl -sSf -X POST \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $GITHUB_DISPATCH_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ https://api.github.com/repos/UiPath/coreipc/dispatches \ - -d "{\"event_type\":\"publish-pypi\",\"client_payload\":{\"sha\":\"${SHA}\"}}" - displayName: 'Gate on master + unpublished version, then dispatch' + -d "{\"event_type\":\"publish-pypi\",\"client_payload\":{\"sha\":\"${SHA}\",\"version\":\"${VERSION}\"}}" + displayName: 'Gate on master + version, then dispatch' env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) SYSTEM_COLLECTIONURI: $(System.CollectionUri) SYSTEM_TEAMPROJECT: $(System.TeamProject) BUILD_ID: ${{ parameters.buildId }} GITHUB_DISPATCH_TOKEN: $(GITHUB_DISPATCH_TOKEN) + PYPKG_DIR: $(Pipeline.Workspace)/pypkg diff --git a/src/CI/stamp-python-version.py b/src/CI/stamp-python-version.py index 9fc1c8b1..c00eb542 100644 --- a/src/CI/stamp-python-version.py +++ b/src/CI/stamp-python-version.py @@ -1,11 +1,20 @@ """Rewrite the Python package's pyproject.toml version line to match the pipeline's $(FullVersion). -Converts the .NET-flavoured version produced by azp-initialization.yaml -to a PEP 440-valid string for Python packaging: +Converts the .NET-flavoured version produced by azp-initialization.yaml to a +PEP 440-valid string for Python packaging: - "2.5.1" -> "2.5.1" (release) - "2.5.1-20260528-08" -> "2.5.1+20260528.08" (local version) + "2.5.3" -> "2.5.3" (release, unchanged) + "2.5.3-20260724-01" -> "2.5.3.dev2026072401" (dev pre-release) + +A build carrying a SemVer suffix (a non-release CI build) becomes a PEP 440 +DEV release — a genuine pre-release that PyPI accepts and that pip/uv skip by +default — rather than a local ("+") segment. A local segment is wrong here on +two counts: PyPI rejects it on upload, and pip/uv treat it as a FINAL release +(so a CI build would masquerade as the real release, and land on public PyPI +stripped to a clean release). The suffix's digits form the monotonic .devN +number. NOTE: .devN sorts BEFORE its base release, so bump the base version in +the csproj right after cutting a release, or later dev builds look "older". The wheel built right after this step will carry the new version. @@ -24,7 +33,11 @@ def to_pep440(full_version: str) -> str: if "-" not in full_version: return full_version base, rest = full_version.split("-", 1) - return f"{base}+{rest.replace('-', '.')}" + # The pipeline's FullVersion suffix is the build number (digits), e.g. + # "20260724-01" -> ".dev2026072401". Strip any non-digits and drop leading + # zeros via int() so it is a valid, monotonic PEP 440 dev number. + digits = re.sub(r"\D", "", rest) or "0" + return f"{base}.dev{int(digits)}" def main() -> int: