diff --git a/.pipelines/foundry-local-packaging.yml b/.pipelines/foundry-local-packaging.yml deleted file mode 100644 index bedd8a3d1..000000000 --- a/.pipelines/foundry-local-packaging.yml +++ /dev/null @@ -1,269 +0,0 @@ -# Foundry Local Packaging Pipeline -# -# Builds Foundry Local Core from neutron-server (windows.ai.toolkit project), -# then packages the C# and JS SDKs from this repo using the built Core. -# -# Produces artifacts: flc-nuget, flc-nuget-winml, flc-wheels, flc-wheels-winml, -# cs-sdk, cs-sdk-winml, js-sdk, js-sdk-winml, python-sdk, python-sdk-winml, -# rust-sdk, rust-sdk-winml - -# CI: only auto-run on pushes to main / release branches. Without an explicit -# trigger block, ADO defaults to "every commit on every branch", which fires -# this expensive pipeline on every dev branch push. -# -# TODO: This prevents auto-run on all pushes but doesn't trigger commits to `main` and `release` -# branches. Re-enable when triggering is fixed. -# trigger: -# branches: -# include: -# - main -# - releases/* - -pr: -- main -- releases/* - -name: $(Date:yyyyMMdd).$(Rev:r) - -parameters: -- name: version - displayName: 'Package version (sdk_v2/...)' - type: string - default: '2.0.0' -- name: versionV1 - displayName: 'Package version (sdk/...)' - type: string - default: '1.3.0' -- name: prereleaseId - displayName: 'Pre-release identifier (e.g. rc1, beta).' - type: string - default: 'none' -- name: isRelease - displayName: 'Release build' - type: boolean - default: false -- name: buildV1 - displayName: 'Force build v1 sdk. If unchecked, auto-detected from PR changes.' - type: boolean - default: false -- name: neutronServerBranch - displayName: 'Foundry Local Core branch (windows.ai.toolkit/neutron-server)' - type: string - default: 'dev/FoundryLocalCore/main' - -variables: -- group: FoundryLocal-ESRP-Signing -# C++ SDK (sdk_v2/cpp) native dependency versions. Must match cmake defaults -# in sdk_v2/deps_versions.json. -- name: cppOrtVersion - value: '1.26.0' -- name: cppGenaiVersion - value: '0.14.1' -- name: cppWinmlVersion - value: '2.1.70' -- name: cppBuildConfig - value: 'RelWithDebInfo' - -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release - -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - parameters: - settings: - networkIsolationPolicy: Permissive - pool: - # default all windows jobs, individual jobs override - name: onnxruntime-Win-CPU-2022 - os: windows - sdl: - binskim: - break: false - scanOutputDirectoryOnly: true - stages: - # ── Detect Changed Paths ── - # Decides whether the legacy sdk/ branch should run. sdk_v2 always builds, - # so there is no buildV2 flag here. Emits one output variable on the - # `setflags` job: `buildV1` ("true"/"false"). - # - # Rules: - # * `buildV1` parameter set true => buildV1 = true. - # * Otherwise => buildV1 = true iff this branch has changes under sdk/ - # relative to its base. Base is the PR target branch when available, - # else origin/main. If that range is empty (e.g. post-merge CI build - # of main itself), fall back to the latest commit's diff so a merge - # that touched sdk/ still triggers v1. Branches forked off something - # other than main and built outside a PR will over-trigger; that's - # acceptable — release branches aren't expected to touch sdk/. - - stage: detect_changes - displayName: 'Detect Changed Paths' - dependsOn: [] - jobs: - - job: setflags - displayName: 'Compute build flags' - pool: - name: onnxruntime-Win-CPU-2022 - os: windows - steps: - - checkout: self - fetchDepth: 0 - - task: PowerShell@2 - name: setflags - displayName: 'Detect changed paths and set buildV1' - inputs: - targetType: inline - pwsh: true - script: | - $forceV1 = "${{ parameters.buildV1 }}" -eq "True" - - if ($forceV1) { - $buildV1 = $true - Write-Host "Manual override: buildV1=$buildV1" - } else { - # Read via env var so PowerShell doesn't mis-parse the literal - # '$(System.PullRequest.TargetBranch)' as a subexpression when - # ADO leaves it unresolved on non-PR runs. Env var is empty - # (not undefined) when the variable isn't set. - $prTarget = $env:SYSTEM_PULLREQUEST_TARGETBRANCH - if (-not [string]::IsNullOrEmpty($prTarget)) { - # Typically "refs/heads/"; normalize to a short name for git fetch + origin/ refs. - $baseBranch = $prTarget -replace '^refs/heads/', '' - Write-Host "Base branch (from PR target): $baseBranch" - } else { - $baseBranch = "main" - Write-Host "Base branch (fallback): $baseBranch" - } - - git fetch --no-tags --depth=50 origin $baseBranch 2>&1 | Out-Null - $diffRange = "origin/$baseBranch...HEAD" - $changed = @(git diff --name-only $diffRange) - Write-Host "Changed files vs $diffRange ($($changed.Count)):" - $changed | ForEach-Object { Write-Host " $_" } - - # On a build of the base branch itself (e.g. post-merge CI on main), - # the range above is empty. Fall back to the latest commit's diff so - # a merge that touched sdk/ still triggers v1. - if ($changed.Count -eq 0) { - $diffRange = "HEAD~1...HEAD" - $changed = @(git diff --name-only $diffRange) - Write-Host "Empty range vs base; falling back to $diffRange ($($changed.Count)):" - $changed | ForEach-Object { Write-Host " $_" } - } - - $buildV1 = @($changed | Where-Object { $_ -like 'sdk/*' }).Count -gt 0 - Write-Host "Auto-detected: buildV1=$buildV1" - } - - $v1 = $buildV1.ToString().ToLower() - Write-Host "##vso[task.setvariable variable=buildV1;isOutput=true]$v1" - - # ── Compute Version ── - # A single version string is computed once and shared across all stages. - # This prevents timestamp drift between standard and WinML builds. - # Outputs (written to the `version-info` pipeline artifact): - # sdk_v2 (no suffix; FLC is legacy-only and not emitted here): - # sdkVersion.txt – semver for C# (e.g. X.Y.Z-dev.202604061234) - # pyVersion.txt – PEP 440 for Python (e.g. X.Y.Z.dev202604061234) - # Legacy sdk/ (.v1. suffix): - # sdkVersion.v1.txt – semver for JS, C#, Rust - # pyVersion.v1.txt – PEP 440 for Python - # flcVersion.v1.txt – NuGet/FLC style - - stage: compute_version - displayName: 'Compute Version' - dependsOn: [] - jobs: - - job: version - displayName: 'Compute Version' - pool: - name: onnxruntime-Win-CPU-2022 - os: windows - templateContext: - outputs: - - output: pipelineArtifact - artifactName: 'version-info' - targetPath: '$(Build.ArtifactStagingDirectory)/version-info' - steps: - - checkout: none - - task: PowerShell@2 - displayName: 'Compute and write version files' - inputs: - targetType: inline - script: | - $preId = "${{ parameters.prereleaseId }}" - $ts = Get-Date -Format "yyyyMMddHHmm" - $commitId = "$(Build.SourceVersion)".Substring(0, 8) - - function Compute-Versions([string]$base, [bool]$includeFlc) { - if ($preId -ne '' -and $preId -ne 'none') { - $result = @{ - sdk = "$base-$preId" - py = "$base$preId" - } - if ($includeFlc) { $result.flc = "$base-$preId" } - return $result - } elseif ("${{ parameters.isRelease }}" -ne "True") { - $result = @{ - sdk = "$base-dev.$ts" - py = "$base.dev$ts" - } - if ($includeFlc) { $result.flc = "$base-dev-$ts-$commitId" } - return $result - } else { - $result = @{ - sdk = $base - py = $base - } - if ($includeFlc) { $result.flc = $base } - return $result - } - } - - $v2 = Compute-Versions "${{ parameters.version }}" $false - $v1 = Compute-Versions "${{ parameters.versionV1 }}" $true - - $outDir = "$(Build.ArtifactStagingDirectory)/version-info" - New-Item -ItemType Directory -Path $outDir -Force | Out-Null - - # sdk_v2 versions (canonical, no suffix). No flcVersion — v2 - # does not consume Foundry Local Core. - Set-Content -Path "$outDir/sdkVersion.txt" -Value $v2.sdk -NoNewline - Set-Content -Path "$outDir/pyVersion.txt" -Value $v2.py -NoNewline - - # Legacy sdk/ versions (.v1. suffix). - Set-Content -Path "$outDir/sdkVersion.v1.txt" -Value $v1.sdk -NoNewline - Set-Content -Path "$outDir/pyVersion.v1.txt" -Value $v1.py -NoNewline - Set-Content -Path "$outDir/flcVersion.v1.txt" -Value $v1.flc -NoNewline - - Write-Host "v2 SDK version: $($v2.sdk)" - Write-Host "v2 Python version: $($v2.py)" - Write-Host "v1 SDK version: $($v1.sdk)" - Write-Host "v1 Python version: $($v1.py)" - Write-Host "v1 FLC version: $($v1.flc)" - - # -- sdk_v1 (sdk/) C++ Core (FLC) + C# / JS / Python / Rust SDKs -- - # Entire subgraph is gated by a single artificial root stage (`v1_gate`) - # defined inside stages-sdk-v1.yml. When detect_changes.buildV1 is false, - # v1_gate is skipped and every downstream v1 stage skips via the default - # `succeeded()` cascade. - - template: v1/templates/stages-sdk-v1.yml - parameters: - gateCondition: eq(dependencies.detect_changes.outputs['setflags.setflags.buildV1'], 'true') - gateDependsOn: [detect_changes] - version: ${{ parameters.versionV1 }} - prereleaseId: ${{ parameters.prereleaseId }} - isRelease: ${{ parameters.isRelease }} - neutronServerBranch: ${{ parameters.neutronServerBranch }} - - - # ── sdk_v2 (C++ native + C# SDK + Python SDK) ── - # Built unconditionally on every run; no detect_changes gate. - - template: v2/templates/stages-sdk-v2.yml - parameters: - buildConfig: $(cppBuildConfig) - ortVersion: $(cppOrtVersion) - genaiVersion: $(cppGenaiVersion) - winmlVersion: $(cppWinmlVersion) diff --git a/.pipelines/v1/foundry-local-packaging.yml b/.pipelines/v1/foundry-local-packaging.yml new file mode 100644 index 000000000..926460f99 --- /dev/null +++ b/.pipelines/v1/foundry-local-packaging.yml @@ -0,0 +1,140 @@ +# Foundry Local Packaging Pipeline (v1) +# +# Builds Foundry Local Core from neutron-server (windows.ai.toolkit project), +# then packages the C# and JS SDKs from this repo using the built Core. +# +# Produces artifacts: flc-nuget, flc-nuget-winml, flc-wheels, flc-wheels-winml, +# cs-sdk, cs-sdk-winml, js-sdk, js-sdk-winml, python-sdk, python-sdk-winml, +# rust-sdk, rust-sdk-winml + +# CI: only auto-run on pushes to main / release branches. Without an explicit +# trigger block, ADO defaults to "every commit on every branch", which fires +# this expensive pipeline on every dev branch push. +# +# TODO: This prevents auto-run on all pushes but doesn't trigger commits to `main` and `release` +# branches. Re-enable when triggering is fixed. +# trigger: +# branches: +# include: +# - main +# - releases/* + +# PR validation only. CI (push) is disabled to avoid duplicate runs when +# branch policy and YAML triggers both fire for the same commit. +trigger: none + +pr: + branches: + include: + - main + - releases/* + paths: + include: + - .pipelines/v1/** + - sdk/** + +name: $(Date:yyyyMMdd).$(Rev:r) + +parameters: +- name: versionV1 + displayName: 'Package version (sdk/...)' + type: string + default: '1.3.0' +- name: prereleaseId + displayName: 'Pre-release identifier (e.g. rc1, beta).' + type: string + default: 'none' +- name: isRelease + displayName: 'Release build' + type: boolean + default: false +- name: neutronServerBranch + displayName: 'Foundry Local Core branch (windows.ai.toolkit/neutron-server)' + type: string + default: 'dev/FoundryLocalCore/main' + +variables: +- group: FoundryLocal-ESRP-Signing + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + settings: + networkIsolationPolicy: Permissive + pool: + # default all windows jobs, individual jobs override + name: onnxruntime-Win-CPU-2022 + os: windows + sdl: + binskim: + break: false + scanOutputDirectoryOnly: true + stages: + # ── Compute Version ── + # Computes the legacy sdk/ version files consumed by stages-sdk-v1.yml. + - stage: compute_version + displayName: 'Compute Version' + dependsOn: [] + jobs: + - job: version + displayName: 'Compute Version' + pool: + name: onnxruntime-Win-CPU-2022 + os: windows + templateContext: + outputs: + - output: pipelineArtifact + artifactName: 'version-info' + targetPath: '$(Build.ArtifactStagingDirectory)/version-info' + steps: + - checkout: none + - task: PowerShell@2 + displayName: 'Compute and write version files' + inputs: + targetType: inline + script: | + $preId = "${{ parameters.prereleaseId }}" + $ts = Get-Date -Format "yyyyMMddHHmm" + $commitId = "$(Build.SourceVersion)".Substring(0, 8) + + if ($preId -ne '' -and $preId -ne 'none') { + $sdkVersion = "${{ parameters.versionV1 }}-$preId" + $pyVersion = "${{ parameters.versionV1 }}$preId" + $flcVersion = "${{ parameters.versionV1 }}-$preId" + } elseif ("${{ parameters.isRelease }}" -ne "True") { + $sdkVersion = "${{ parameters.versionV1 }}-dev.$ts" + $pyVersion = "${{ parameters.versionV1 }}.dev$ts" + $flcVersion = "${{ parameters.versionV1 }}-dev-$ts-$commitId" + } else { + $sdkVersion = "${{ parameters.versionV1 }}" + $pyVersion = "${{ parameters.versionV1 }}" + $flcVersion = "${{ parameters.versionV1 }}" + } + + $outDir = "$(Build.ArtifactStagingDirectory)/version-info" + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + + Set-Content -Path "$outDir/sdkVersion.v1.txt" -Value $sdkVersion -NoNewline + Set-Content -Path "$outDir/pyVersion.v1.txt" -Value $pyVersion -NoNewline + Set-Content -Path "$outDir/flcVersion.v1.txt" -Value $flcVersion -NoNewline + + Write-Host "v1 SDK version: $sdkVersion" + Write-Host "v1 Python version: $pyVersion" + Write-Host "v1 FLC version: $flcVersion" + + # -- sdk_v1 (sdk/) C++ Core (FLC) + C# / JS / Python / Rust SDKs -- + # Entire subgraph is rooted at the artificial `v1_gate` stage defined + # inside stages-sdk-v1.yml. + - template: templates/stages-sdk-v1.yml + parameters: + version: ${{ parameters.versionV1 }} + prereleaseId: ${{ parameters.prereleaseId }} + isRelease: ${{ parameters.isRelease }} + neutronServerBranch: ${{ parameters.neutronServerBranch }} diff --git a/.pipelines/v1/templates/build-core-steps.yml b/.pipelines/v1/templates/build-core-steps.yml index e2f13aed3..41f70c66c 100644 --- a/.pipelines/v1/templates/build-core-steps.yml +++ b/.pipelines/v1/templates/build-core-steps.yml @@ -175,3 +175,4 @@ steps: Copy-Item -Destination $destDir -Force Write-Host "Staged binaries:" Get-ChildItem $destDir | ForEach-Object { Write-Host " $($_.Name)" } + diff --git a/.pipelines/v1/templates/build-cs-steps.yml b/.pipelines/v1/templates/build-cs-steps.yml index b9bfd962a..e9e6ca773 100644 --- a/.pipelines/v1/templates/build-cs-steps.yml +++ b/.pipelines/v1/templates/build-cs-steps.yml @@ -197,3 +197,4 @@ steps: inlineOperation: | [{"keyCode":"CP-401405","operationSetCode":"NuGetSign","parameters":[],"toolName":"sign","toolVersion":"6.2.9304.0"},{"keyCode":"CP-401405","operationSetCode":"NuGetVerify","parameters":[],"toolName":"sign","toolVersion":"6.2.9304.0"}] + diff --git a/.pipelines/v1/templates/build-js-addon-steps.yml b/.pipelines/v1/templates/build-js-addon-steps.yml index dce0f7fce..26eaac07b 100644 --- a/.pipelines/v1/templates/build-js-addon-steps.yml +++ b/.pipelines/v1/templates/build-js-addon-steps.yml @@ -43,3 +43,4 @@ steps: Copy-Item "build/Release/foundry_local_napi.node" "$destDir/foundry_local_napi.node" -Force Write-Host "Built addon for $platformKey -> $destDir/foundry_local_napi.node" Get-Item "$destDir/foundry_local_napi.node" | ForEach-Object { Write-Host " Size: $($_.Length) bytes" } + diff --git a/.pipelines/v1/templates/build-js-steps.yml b/.pipelines/v1/templates/build-js-steps.yml index 1b2a48263..521267bc7 100644 --- a/.pipelines/v1/templates/build-js-steps.yml +++ b/.pipelines/v1/templates/build-js-steps.yml @@ -219,3 +219,4 @@ steps: New-Item -ItemType Directory -Path $destDir -Force | Out-Null Copy-Item "$(repoRoot)/sdk/js/*.tgz" "$destDir/" + diff --git a/.pipelines/v1/templates/build-python-steps.yml b/.pipelines/v1/templates/build-python-steps.yml index 6309782cc..b8e36bf98 100644 --- a/.pipelines/v1/templates/build-python-steps.yml +++ b/.pipelines/v1/templates/build-python-steps.yml @@ -225,3 +225,4 @@ steps: Write-Host "Staged wheels:" Get-ChildItem $destDir | ForEach-Object { Write-Host " $($_.Name)" } + diff --git a/.pipelines/v1/templates/build-rust-steps.yml b/.pipelines/v1/templates/build-rust-steps.yml index 8402b9aac..31add02dc 100644 --- a/.pipelines/v1/templates/build-rust-steps.yml +++ b/.pipelines/v1/templates/build-rust-steps.yml @@ -219,3 +219,4 @@ steps: Write-Host "Staged crates:" Get-ChildItem $destDir | ForEach-Object { Write-Host " $($_.Name)" } + diff --git a/.pipelines/templates/checkout-steps.yml b/.pipelines/v1/templates/checkout-steps.yml similarity index 99% rename from .pipelines/templates/checkout-steps.yml rename to .pipelines/v1/templates/checkout-steps.yml index 601eacf56..bcdfe96c5 100644 --- a/.pipelines/templates/checkout-steps.yml +++ b/.pipelines/v1/templates/checkout-steps.yml @@ -72,3 +72,4 @@ steps: Pop-Location Write-Host "Checked out ${{ parameters.repoName }} at $(git -C $repoDir rev-parse HEAD)" + diff --git a/.pipelines/v1/templates/package-core-steps.yml b/.pipelines/v1/templates/package-core-steps.yml index fb9edc514..5cd5d344d 100644 --- a/.pipelines/v1/templates/package-core-steps.yml +++ b/.pipelines/v1/templates/package-core-steps.yml @@ -343,3 +343,4 @@ steps: New-Item -ItemType Directory -Path $outDir -Force | Out-Null [System.IO.File]::WriteAllText("$outDir/$fileName", $json, [System.Text.UTF8Encoding]::new($false)) Write-Host "Wrote $fileName to $outDir" + diff --git a/.pipelines/v1/templates/stages-sdk-v1.yml b/.pipelines/v1/templates/stages-sdk-v1.yml index 7bbd1f368..b965b7d6a 100644 --- a/.pipelines/v1/templates/stages-sdk-v1.yml +++ b/.pipelines/v1/templates/stages-sdk-v1.yml @@ -3,16 +3,9 @@ # Produces NuGet/wheel/JS artifacts for Foundry Local Core (FLC) plus the # v1 C# / JS / Python / Rust SDKs in both base and WinML flavors. # -# Assumes the caller has already emitted: -# * a `compute_version` stage publishing the `version-info` pipeline -# artifact (containing sdkVersion.v1.txt, pyVersion.v1.txt, flcVersion.v1.txt) -# * any stage referenced by `gateDependsOn` (typically `detect_changes`) -# -# The entire subgraph is gated by a single artificial root stage (`v1_gate`) -# whose runtime condition is supplied via `gateCondition`. When v1_gate is -# skipped, every downstream v1 stage skips via ADO's default `succeeded()` -# cascade on dependsOn. See stages-sdk-v2.yml for the rationale behind this -# pattern. +# Assumes the caller has already emitted a `compute_version` stage +# publishing the `version-info` pipeline artifact +# (sdkVersion.v1.txt, pyVersion.v1.txt, flcVersion.v1.txt). parameters: - name: version @@ -25,35 +18,12 @@ parameters: default: false - name: neutronServerBranch type: string -- name: gateCondition - type: string - default: 'succeeded()' -- name: gateDependsOn - type: object - default: [] stages: -# -- Artificial root: gates the entire v1 subgraph -- -- stage: v1_gate - displayName: 'sdk_v1 build gate' - condition: ${{ parameters.gateCondition }} - dependsOn: ${{ parameters.gateDependsOn }} - jobs: - - job: noop - displayName: 'sdk_v1 build enabled' - pool: - name: onnxruntime-Win-CPU-2022 - os: windows - steps: - - checkout: none - - script: echo "sdk_v1 build enabled" - displayName: 'sdk_v1 build enabled' - - stage: build_core displayName: 'Build Core' dependsOn: - - v1_gate - compute_version jobs: - job: flc_win_x64 @@ -67,11 +37,11 @@ stages: artifactName: 'flc-win-x64' targetPath: '$(Build.ArtifactStagingDirectory)/native' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-core-steps.yml @@ -90,11 +60,11 @@ stages: artifactName: 'flc-win-arm64' targetPath: '$(Build.ArtifactStagingDirectory)/native' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-core-steps.yml @@ -113,11 +83,11 @@ stages: artifactName: 'flc-linux-x64' targetPath: '$(Build.ArtifactStagingDirectory)/native' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-core-steps.yml @@ -137,11 +107,11 @@ stages: artifactName: 'flc-linux-arm64' targetPath: '$(Build.ArtifactStagingDirectory)/native' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-core-steps.yml @@ -162,11 +132,11 @@ stages: artifactName: 'flc-osx-arm64' targetPath: '$(Build.ArtifactStagingDirectory)/native' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared # AcesShared macOS agents don't have git-lfs pre-installed @@ -204,7 +174,7 @@ stages: artifactName: 'deps-versions-standard' targetPath: '$(Build.ArtifactStagingDirectory)/deps-versions' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} @@ -266,7 +236,6 @@ stages: - stage: build_cs displayName: 'Build C# SDK' dependsOn: - - v1_gate - build_core jobs: - job: cs_sdk @@ -291,7 +260,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/cs-sdk' steps: - checkout: self - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-cs-steps.yml @@ -306,7 +275,7 @@ stages: # ── Build JS Node-API Addon (all platforms) ── - stage: build_js_addon displayName: 'Build JS Addon' - dependsOn: [v1_gate] + dependsOn: [] jobs: - job: js_addon_win_x64 displayName: 'Addon win32-x64' @@ -436,7 +405,6 @@ stages: - stage: build_js displayName: 'Build JS SDK' dependsOn: - - v1_gate - build_core - build_js_addon jobs: @@ -462,7 +430,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/js-sdk' steps: - checkout: self - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared @@ -527,7 +495,6 @@ stages: - stage: build_python displayName: 'Build Python SDK' dependsOn: - - v1_gate - build_core jobs: - job: python_sdk @@ -552,7 +519,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/python-sdk' steps: - checkout: self - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-python-steps.yml @@ -568,7 +535,6 @@ stages: - stage: build_rust displayName: 'Build Rust SDK' dependsOn: - - v1_gate - build_core - build_core_winml jobs: @@ -597,7 +563,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/rust-sdk' steps: - checkout: self - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-rust-steps.yml @@ -658,7 +624,6 @@ stages: - stage: test_cs displayName: 'Test C#' dependsOn: - - v1_gate - build_cs jobs: - job: test_cs_win_x64 @@ -677,7 +642,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -703,7 +668,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -730,7 +695,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -761,7 +726,7 @@ stages: displayName: 'Install Git LFS' - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -775,7 +740,6 @@ stages: - stage: test_js displayName: 'Test JS' dependsOn: - - v1_gate - build_js jobs: - job: test_js_win_x64 @@ -794,7 +758,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-js-steps.yml @@ -819,7 +783,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -846,7 +810,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -877,7 +841,7 @@ stages: displayName: 'Install Git LFS' - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-js-steps.yml @@ -890,7 +854,6 @@ stages: - stage: test_python displayName: 'Test Python' dependsOn: - - v1_gate - build_python jobs: - job: test_python_win_x64 @@ -912,7 +875,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-python-steps.yml @@ -941,7 +904,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -972,7 +935,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -1007,7 +970,7 @@ stages: displayName: 'Install Git LFS' - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-python-steps.yml @@ -1021,7 +984,6 @@ stages: - stage: test_rust displayName: 'Test Rust' dependsOn: - - v1_gate - build_rust jobs: - job: test_rust_win_x64 @@ -1040,7 +1002,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-rust-steps.yml @@ -1065,7 +1027,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-rust-steps.yml @@ -1091,7 +1053,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-rust-steps.yml @@ -1121,7 +1083,7 @@ stages: displayName: 'Install Git LFS' - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-rust-steps.yml @@ -1134,7 +1096,6 @@ stages: - stage: build_core_winml displayName: 'Build Core (WinML)' dependsOn: - - v1_gate - compute_version jobs: - job: flc_winml_win_x64 @@ -1148,11 +1109,11 @@ stages: artifactName: 'flc-winml-win-x64' targetPath: '$(Build.ArtifactStagingDirectory)/native' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-core-steps.yml @@ -1172,11 +1133,11 @@ stages: artifactName: 'flc-winml-win-arm64' targetPath: '$(Build.ArtifactStagingDirectory)/native' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-core-steps.yml @@ -1209,7 +1170,7 @@ stages: artifactName: 'deps-versions-winml' targetPath: '$(Build.ArtifactStagingDirectory)/deps-versions' steps: - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: neutron-server ref: refs/heads/${{ parameters.neutronServerBranch }} @@ -1250,7 +1211,6 @@ stages: - stage: build_cs_winml displayName: 'Build C# SDK (WinML)' dependsOn: - - v1_gate - build_core_winml jobs: - job: cs_sdk_winml @@ -1275,7 +1235,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/cs-sdk-winml' steps: - checkout: self - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-cs-steps.yml @@ -1292,7 +1252,6 @@ stages: - stage: build_js_winml displayName: 'Build JS SDK (WinML)' dependsOn: - - v1_gate - build_core_winml - build_js_addon jobs: @@ -1318,7 +1277,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/js-sdk' steps: - checkout: self - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared @@ -1383,7 +1342,6 @@ stages: - stage: build_python_winml displayName: 'Build Python SDK (WinML)' dependsOn: - - v1_gate - build_core_winml jobs: - job: python_sdk_winml @@ -1408,7 +1366,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/python-sdk-winml' steps: - checkout: self - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: build-python-steps.yml @@ -1425,7 +1383,6 @@ stages: - stage: test_cs_winml displayName: 'Test C# (WinML)' dependsOn: - - v1_gate - build_cs_winml jobs: - job: test_cs_winml_win_x64 @@ -1444,7 +1401,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared basePath: '$(Agent.BuildDirectory)' @@ -1458,7 +1415,6 @@ stages: - stage: test_js_winml displayName: 'Test JS (WinML)' dependsOn: - - v1_gate - build_js_winml jobs: - job: test_js_winml_win_x64 @@ -1477,7 +1433,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-js-steps.yml @@ -1490,7 +1446,6 @@ stages: - stage: test_python_winml displayName: 'Test Python (WinML)' dependsOn: - - v1_gate - build_python_winml jobs: - job: test_python_winml_win_x64 @@ -1512,7 +1467,7 @@ stages: steps: - checkout: self clean: true - - template: ../../templates/checkout-steps.yml@self + - template: checkout-steps.yml parameters: repoName: test-data-shared - template: test-python-steps.yml @@ -1520,4 +1475,5 @@ stages: isWinML: true flcWheelsDir: '$(Pipeline.Workspace)/flc-wheels-winml' sdkWheelsDir: '$(Pipeline.Workspace)/python-sdk-winml' - depsVersionsDir: '$(Pipeline.Workspace)/deps-versions-winml' \ No newline at end of file + depsVersionsDir: '$(Pipeline.Workspace)/deps-versions-winml' + diff --git a/.pipelines/v1/templates/test-cs-steps.yml b/.pipelines/v1/templates/test-cs-steps.yml index 605b36cf2..2f9eb4028 100644 --- a/.pipelines/v1/templates/test-cs-steps.yml +++ b/.pipelines/v1/templates/test-cs-steps.yml @@ -147,3 +147,4 @@ steps: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } env: TF_BUILD: 'true' + diff --git a/.pipelines/v1/templates/test-js-steps.yml b/.pipelines/v1/templates/test-js-steps.yml index 2d007f70b..64dfbb012 100644 --- a/.pipelines/v1/templates/test-js-steps.yml +++ b/.pipelines/v1/templates/test-js-steps.yml @@ -154,3 +154,4 @@ steps: env: TF_BUILD: 'true' FOUNDRY_TEST_DATA_DIR: $(testDataDir) + diff --git a/.pipelines/v1/templates/test-python-steps.yml b/.pipelines/v1/templates/test-python-steps.yml index cf2643982..6e77c151b 100644 --- a/.pipelines/v1/templates/test-python-steps.yml +++ b/.pipelines/v1/templates/test-python-steps.yml @@ -178,3 +178,4 @@ steps: env: TF_BUILD: 'true' FOUNDRY_TEST_DATA_DIR: $(testDataDir) + diff --git a/.pipelines/v1/templates/test-rust-steps.yml b/.pipelines/v1/templates/test-rust-steps.yml index 4d382b1de..67c893a46 100644 --- a/.pipelines/v1/templates/test-rust-steps.yml +++ b/.pipelines/v1/templates/test-rust-steps.yml @@ -134,3 +134,4 @@ steps: env: TF_BUILD: 'true' FOUNDRY_TEST_DATA_DIR: $(testDataDir) + diff --git a/.pipelines/v1/templates/update-deps-versions-steps.yml b/.pipelines/v1/templates/update-deps-versions-steps.yml index 6f0ebcc34..fae88114a 100644 --- a/.pipelines/v1/templates/update-deps-versions-steps.yml +++ b/.pipelines/v1/templates/update-deps-versions-steps.yml @@ -42,3 +42,4 @@ steps: if ($isWinML -and $deps.'windows-ai-machinelearning') { Write-Host " Windows AI ML: $($deps.'windows-ai-machinelearning'.version)" } + diff --git a/.pipelines/v2/foundry-local-packaging.yml b/.pipelines/v2/foundry-local-packaging.yml new file mode 100644 index 000000000..f9d913662 --- /dev/null +++ b/.pipelines/v2/foundry-local-packaging.yml @@ -0,0 +1,128 @@ +# Foundry Local Packaging Pipeline (v2) +# +# Builds and packages sdk_v2 artifacts only: +# - Native runtime artifacts (base + WinML) +# - C# SDK (base + WinML) +# - Python SDK (base + WinML) +# - JS SDK + +# PR validation only. CI (push) is disabled to avoid duplicate runs when +# branch policy and YAML triggers both fire for the same commit. +trigger: none + +pr: + branches: + include: + - main + - releases/* + paths: + include: + - .pipelines/v2/** + - sdk_v2/** + +name: $(Date:yyyyMMdd).$(Rev:r) + +parameters: +- name: version + displayName: 'Package version (sdk_v2/...)' + type: string + default: '2.0.0' +- name: prereleaseId + displayName: 'Pre-release identifier (e.g. rc1, beta).' + type: string + default: 'none' +- name: isRelease + displayName: 'Release build' + type: boolean + default: false + +variables: +- group: FoundryLocal-ESRP-Signing +# C++ SDK (sdk_v2/cpp) native dependency versions. Must match cmake defaults +# in sdk_v2/deps_versions.json. +- name: cppOrtVersion + value: '1.26.0' +- name: cppGenaiVersion + value: '0.14.1' +- name: cppWinmlVersion + value: '2.1.70' +- name: cppBuildConfig + value: 'RelWithDebInfo' + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: test-data-shared + type: git + name: AIFoundryLocal/test-data-shared + ref: refs/heads/main + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + settings: + networkIsolationPolicy: Permissive + pool: + # default all windows jobs, individual jobs override + name: onnxruntime-Win-CPU-2022 + os: windows + sdl: + sourceRepositoriesToScan: + exclude: + - repository: test-data-shared + binskim: + break: false + scanOutputDirectoryOnly: true + stages: + - stage: compute_version + displayName: 'Compute Version' + dependsOn: [] + jobs: + - job: version + displayName: 'Compute Version' + pool: + name: onnxruntime-Win-CPU-2022 + os: windows + templateContext: + outputs: + - output: pipelineArtifact + artifactName: 'version-info' + targetPath: '$(Build.ArtifactStagingDirectory)/version-info' + steps: + - checkout: none + - task: PowerShell@2 + displayName: 'Compute and write version files' + inputs: + targetType: inline + script: | + $preId = "${{ parameters.prereleaseId }}" + $ts = Get-Date -Format "yyyyMMddHHmm" + + if ($preId -ne '' -and $preId -ne 'none') { + $sdkVersion = "${{ parameters.version }}-$preId" + $pyVersion = "${{ parameters.version }}$preId" + } elseif ("${{ parameters.isRelease }}" -ne "True") { + $sdkVersion = "${{ parameters.version }}-dev.$ts" + $pyVersion = "${{ parameters.version }}.dev$ts" + } else { + $sdkVersion = "${{ parameters.version }}" + $pyVersion = "${{ parameters.version }}" + } + + $outDir = "$(Build.ArtifactStagingDirectory)/version-info" + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + Set-Content -Path "$outDir/sdkVersion.txt" -Value $sdkVersion -NoNewline + Set-Content -Path "$outDir/pyVersion.txt" -Value $pyVersion -NoNewline + + Write-Host "v2 SDK version: $sdkVersion" + Write-Host "v2 Python version: $pyVersion" + + - template: templates/stages-sdk-v2.yml + parameters: + buildConfig: $(cppBuildConfig) + ortVersion: $(cppOrtVersion) + genaiVersion: $(cppGenaiVersion) + winmlVersion: $(cppWinmlVersion) diff --git a/.pipelines/v2/templates/stages-build-native.yml b/.pipelines/v2/templates/stages-build-native.yml index 09a75c33e..a99fe846a 100644 --- a/.pipelines/v2/templates/stages-build-native.yml +++ b/.pipelines/v2/templates/stages-build-native.yml @@ -298,3 +298,4 @@ stages: ortVersion: ${{ parameters.ortVersion }} genaiVersion: ${{ parameters.genaiVersion }} variant: winml + diff --git a/.pipelines/v2/templates/stages-cs.yml b/.pipelines/v2/templates/stages-cs.yml index c271dda13..8a3008f9a 100644 --- a/.pipelines/v2/templates/stages-cs.yml +++ b/.pipelines/v2/templates/stages-cs.yml @@ -66,6 +66,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/cs-sdk' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-cs.yml parameters: @@ -98,6 +99,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/cs-sdk' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-cs.yml parameters: @@ -128,10 +130,10 @@ stages: targetPath: '$(Pipeline.Workspace)/${{ parameters._config_base.nativeArtifact }}' steps: - checkout: self + path: s/Foundry-Local clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-cs.yml parameters: flNugetDir: '$(Pipeline.Workspace)/${{ parameters._config_base.nativeArtifact }}' @@ -158,10 +160,10 @@ stages: targetPath: '$(Pipeline.Workspace)/${{ parameters._config_winml.nativeArtifact }}' steps: - checkout: self + path: s/Foundry-Local clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-cs.yml parameters: flNugetDir: '$(Pipeline.Workspace)/${{ parameters._config_winml.nativeArtifact }}' @@ -191,10 +193,10 @@ stages: targetPath: '$(Pipeline.Workspace)/${{ parameters._config_base.nativeArtifact }}' steps: - checkout: self + path: s/Foundry-Local clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-cs.yml parameters: flNugetDir: '$(Pipeline.Workspace)/${{ parameters._config_base.nativeArtifact }}' @@ -222,6 +224,7 @@ stages: targetPath: '$(Pipeline.Workspace)/${{ parameters._config_base.nativeArtifact }}' steps: - checkout: self + path: s/Foundry-Local clean: true - bash: | set -euo pipefail @@ -230,12 +233,17 @@ stages: fi git lfs install displayName: 'Install git-lfs (macOS)' - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-cs.yml parameters: flNugetDir: '$(Pipeline.Workspace)/${{ parameters._config_base.nativeArtifact }}' isWinML: false testDataSharedDir: '$(Build.SourcesDirectory)/test-data-shared' - additionalTestArgs: '--settings $(Build.SourcesDirectory)/sdk_v2/cs/test/FoundryLocal.Tests/sequential.runsettings' + additionalTestArgs: '--settings $(Build.Repository.LocalPath)/sdk_v2/cs/test/FoundryLocal.Tests/sequential.runsettings' + + + + + + diff --git a/.pipelines/v2/templates/stages-js.yml b/.pipelines/v2/templates/stages-js.yml index e434f7c74..9c0fe99c6 100644 --- a/.pipelines/v2/templates/stages-js.yml +++ b/.pipelines/v2/templates/stages-js.yml @@ -51,6 +51,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/js-prebuild' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-js.yml parameters: @@ -90,6 +91,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/js-prebuild' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-js.yml parameters: @@ -128,6 +130,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/js-prebuild' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-js.yml parameters: @@ -168,6 +171,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/js-prebuild' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-js.yml parameters: @@ -197,10 +201,10 @@ stages: targetPath: '$(Pipeline.Workspace)/js-prebuild-win-x64' steps: - checkout: self + path: s/Foundry-Local clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-js.yml parameters: rid: 'win-x64' @@ -223,10 +227,10 @@ stages: targetPath: '$(Pipeline.Workspace)/js-prebuild-linux-x64' steps: - checkout: self + path: s/Foundry-Local clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-js.yml parameters: rid: 'linux-x64' @@ -251,6 +255,7 @@ stages: targetPath: '$(Pipeline.Workspace)/js-prebuild-osx-arm64' steps: - checkout: self + path: s/Foundry-Local clean: true - bash: | set -euo pipefail @@ -259,9 +264,8 @@ stages: fi git lfs install displayName: 'Install git-lfs (macOS)' - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-js.yml parameters: rid: 'osx-arm64' @@ -307,7 +311,13 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/js-sdk-v2' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-pack-js.yml parameters: outputDir: '$(Build.ArtifactStagingDirectory)/js-sdk-v2' + + + + + diff --git a/.pipelines/v2/templates/stages-python.yml b/.pipelines/v2/templates/stages-python.yml index 7a07c7dae..11655812c 100644 --- a/.pipelines/v2/templates/stages-python.yml +++ b/.pipelines/v2/templates/stages-python.yml @@ -49,6 +49,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/python-sdk' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-python.yml parameters: @@ -81,6 +82,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/python-sdk' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-python.yml parameters: @@ -114,6 +116,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/python-sdk' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-python.yml parameters: @@ -148,6 +151,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/python-sdk' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-python.yml parameters: @@ -177,10 +181,10 @@ stages: targetPath: '$(Pipeline.Workspace)/python-sdk-base-win-x64' steps: - checkout: self + path: s/Foundry-Local clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-python.yml parameters: wheelDir: '$(Pipeline.Workspace)/python-sdk-base-win-x64' @@ -203,10 +207,10 @@ stages: targetPath: '$(Pipeline.Workspace)/python-sdk-base-linux-x64' steps: - checkout: self + path: s/Foundry-Local clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-python.yml parameters: wheelDir: '$(Pipeline.Workspace)/python-sdk-base-linux-x64' @@ -231,6 +235,7 @@ stages: targetPath: '$(Pipeline.Workspace)/python-sdk-base-osx-arm64' steps: - checkout: self + path: s/Foundry-Local clean: true - bash: | set -euo pipefail @@ -239,9 +244,8 @@ stages: fi git lfs install displayName: 'Install git-lfs (macOS)' - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-python.yml parameters: wheelDir: '$(Pipeline.Workspace)/python-sdk-base-osx-arm64' @@ -278,6 +282,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/python-sdk' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-python.yml parameters: @@ -310,6 +315,7 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)/python-sdk' steps: - checkout: self + path: s/Foundry-Local clean: true - template: steps-build-python.yml parameters: @@ -335,12 +341,17 @@ stages: targetPath: '$(Pipeline.Workspace)/python-sdk-winml-win-x64' steps: - checkout: self + path: s/Foundry-Local clean: true - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared + - checkout: test-data-shared + lfs: true - template: steps-test-python.yml parameters: wheelDir: '$(Pipeline.Workspace)/python-sdk-winml-win-x64' testDataSharedDir: '$(Build.SourcesDirectory)/test-data-shared' isWinML: true + + + + + diff --git a/.pipelines/v2/templates/stages-sdk-v2.yml b/.pipelines/v2/templates/stages-sdk-v2.yml index 5a1c59bab..34794ea01 100644 --- a/.pipelines/v2/templates/stages-sdk-v2.yml +++ b/.pipelines/v2/templates/stages-sdk-v2.yml @@ -53,3 +53,4 @@ stages: # ── JS SDK (single multi-platform tarball) ── - template: stages-js.yml + diff --git a/.pipelines/v2/templates/steps-build-cs.yml b/.pipelines/v2/templates/steps-build-cs.yml index 1c77af191..abf05f678 100644 --- a/.pipelines/v2/templates/steps-build-cs.yml +++ b/.pipelines/v2/templates/steps-build-cs.yml @@ -69,7 +69,6 @@ steps: - @@ -88,7 +87,7 @@ steps: targetType: inline pwsh: true script: | - $proj = "$(Build.SourcesDirectory)/sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj" + $proj = "$(Build.Repository.LocalPath)/sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj" if (-not (Test-Path $proj)) { throw "Project not found: $proj" } dotnet restore $proj ` --configfile "$(customNugetConfig)" ` @@ -103,7 +102,7 @@ steps: targetType: inline pwsh: true script: | - dotnet build "$(Build.SourcesDirectory)/sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj" ` + dotnet build "$(Build.Repository.LocalPath)/sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj" ` --no-restore --configuration Release ` /p:UseWinML=${{ parameters.isWinML }} ` /p:FoundryLocalRuntimeVersion=$(packageVersion) ` @@ -122,7 +121,7 @@ steps: EsrpClientId: '$(esrpClientId)' AuthAKVName: '$(esrpAkvName)' AuthSignCertName: '$(esrpSignCertName)' - FolderPath: '$(Build.SourcesDirectory)/sdk_v2/cs/src/bin/Release' + FolderPath: '$(Build.Repository.LocalPath)/sdk_v2/cs/src/bin/Release' # Pattern is passed to .NET Directory.EnumerateFiles with SearchOption.AllDirectories, # so it must be a plain filename (no '/' or '**'). Recursion across TFM subfolders is # provided by the task itself when UseMinimatch=false. @@ -141,7 +140,7 @@ steps: targetType: inline pwsh: true script: | - dotnet pack "$(Build.SourcesDirectory)/sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj" ` + dotnet pack "$(Build.Repository.LocalPath)/sdk_v2/cs/src/Microsoft.AI.Foundry.Local.csproj" ` --no-build --no-restore --configuration Release ` --output "${{ parameters.outputDir }}" ` /p:PackageVersion=$(packageVersion) ` @@ -171,3 +170,5 @@ steps: signConfigType: inlineSignParams inlineOperation: | [{"keyCode":"CP-401405","operationSetCode":"NuGetSign","parameters":[],"toolName":"sign","toolVersion":"6.2.9304.0"},{"keyCode":"CP-401405","operationSetCode":"NuGetVerify","parameters":[],"toolName":"sign","toolVersion":"6.2.9304.0"}] + + diff --git a/.pipelines/v2/templates/steps-build-js.yml b/.pipelines/v2/templates/steps-build-js.yml index 5e6ab1e77..a4814c363 100644 --- a/.pipelines/v2/templates/steps-build-js.yml +++ b/.pipelines/v2/templates/steps-build-js.yml @@ -51,6 +51,24 @@ steps: inputs: versionSpec: '20.x' +- task: PowerShell@2 + displayName: 'Configure npm registry (AIFoundryLocal_PublicPackages)' + inputs: + targetType: inline + pwsh: true + script: | + $npmrc = "$(Agent.TempDirectory)/foundrylocal-$(System.JobId).npmrc" + $registry = 'https://pkgs.dev.azure.com/aiinfra/AIFoundryLocal/_packaging/AIFoundryLocal_PublicPackages/npm/registry/' + Set-Content -Path $npmrc -Value "registry=$registry`nalways-auth=true" -Encoding UTF8 + Write-Host "##vso[task.setvariable variable=npmUserConfig]$npmrc" + Write-Host "##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]$npmrc" + Write-Host "npm userconfig: $npmrc" + +- task: npmAuthenticate@0 + displayName: 'Authenticate npm with Azure Artifacts' + inputs: + workingFile: '$(npmUserConfig)' + # Map ADO RID -> Node-style platform-arch key used inside the npm tarball # (prebuilds/-/) so downstream consumers find the addon under # the directory matching `${process.platform}-${process.arch}` at runtime. @@ -85,8 +103,22 @@ steps: displayName: 'npm ci' inputs: command: custom - workingDir: '$(Build.SourcesDirectory)/sdk_v2/js' - customCommand: 'ci --no-audit --no-fund --ignore-scripts' + workingDir: '$(Build.Repository.LocalPath)/sdk_v2/js' + customCommand: 'ci --no-audit --no-fund --ignore-scripts --include=dev' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' + +- task: PowerShell@2 + displayName: 'Ensure node-gyp tooling is installed' + inputs: + targetType: inline + pwsh: true + script: | + Set-Location "$(Build.Repository.LocalPath)/sdk_v2/js" + npm install --no-save --ignore-scripts --include=dev --no-audit --no-fund node-gyp node-api-headers + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' # binding.gyp resolves the foundry_local lib + gsl/public include dirs # via FOUNDRY_LOCAL_LIB_DIR and FOUNDRY_LOCAL_INCLUDE_DIR when set. The @@ -98,29 +130,37 @@ steps: $arch = '${{ parameters.targetArch }}' $args = if ($arch -eq 'native') { 'rebuild' } else { "rebuild --arch=$arch" } Write-Host "node-gyp $args" - Set-Location "$(Build.SourcesDirectory)/sdk_v2/js" - npx --no-install node-gyp $args.Split(' ') + Set-Location "$(Build.Repository.LocalPath)/sdk_v2/js" + $nodeGyp = Join-Path (Get-Location) 'node_modules/node-gyp/bin/node-gyp.js' + if (-not (Test-Path $nodeGyp)) { throw "node-gyp not found at $nodeGyp" } + & node $nodeGyp $args.Split(' ') if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Build Node-API addon' env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' FOUNDRY_LOCAL_LIB_DIR: '${{ parameters.nativeArtifactDir }}' FOUNDRY_LOCAL_INCLUDE_DIR: '${{ parameters.includeArtifactDir }}' # Force the copy_addon_to_prebuilds destination to the target-arch dir. # Without this, print-prebuild-dir.mjs uses the host Node.js process.arch # (x64) and the addon lands in prebuilds/win32-x64/ even when we are # cross-compiling to arm64. - FOUNDRY_LOCAL_PREBUILD_DIR: '$(Build.SourcesDirectory)/sdk_v2/js/prebuilds/$(prebuildDir)' + FOUNDRY_LOCAL_PREBUILD_DIR: '$(Build.Repository.LocalPath)/sdk_v2/js/prebuilds/$(prebuildDir)' - ${{ if or(eq(parameters.rid, 'linux-x64'), eq(parameters.rid, 'osx-arm64')) }}: - bash: | set -euo pipefail - cd "$(Build.SourcesDirectory)/sdk_v2/js" - npx --no-install node-gyp rebuild + cd "$(Build.Repository.LocalPath)/sdk_v2/js" + if [ ! -f ./node_modules/node-gyp/bin/node-gyp.js ]; then + echo "node-gyp not found at ./node_modules/node-gyp/bin/node-gyp.js" >&2 + exit 1 + fi + node ./node_modules/node-gyp/bin/node-gyp.js rebuild displayName: 'Build Node-API addon' env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' FOUNDRY_LOCAL_LIB_DIR: '${{ parameters.nativeArtifactDir }}' FOUNDRY_LOCAL_INCLUDE_DIR: '${{ parameters.includeArtifactDir }}' - FOUNDRY_LOCAL_PREBUILD_DIR: '$(Build.SourcesDirectory)/sdk_v2/js/prebuilds/$(prebuildDir)' + FOUNDRY_LOCAL_PREBUILD_DIR: '$(Build.Repository.LocalPath)/sdk_v2/js/prebuilds/$(prebuildDir)' # Stage prebuilds/-/ with the two .node addons + foundry_local # shared library. binding.gyp's copy_addon_to_prebuilds target already @@ -132,7 +172,7 @@ steps: if (Test-Path $dst) { Remove-Item -Recurse -Force $dst } New-Item -ItemType Directory -Force -Path $dst | Out-Null - $srcAddon = "$(Build.SourcesDirectory)/sdk_v2/js/prebuilds/$(prebuildDir)" + $srcAddon = "$(Build.Repository.LocalPath)/sdk_v2/js/prebuilds/$(prebuildDir)" foreach ($file in @('foundry_local_node.node', 'foundry_local_preload.node')) { $path = Join-Path $srcAddon $file if (-not (Test-Path $path)) { throw "Addon not found: $path" } @@ -155,7 +195,7 @@ steps: rm -rf "$dst" mkdir -p "$dst" - srcAddon="$(Build.SourcesDirectory)/sdk_v2/js/prebuilds/$(prebuildDir)" + srcAddon="$(Build.Repository.LocalPath)/sdk_v2/js/prebuilds/$(prebuildDir)" for file in foundry_local_node.node foundry_local_preload.node; do path="$srcAddon/$file" if [ ! -f "$path" ]; then echo "Addon not found: $path" >&2; exit 1; fi @@ -220,3 +260,5 @@ steps: signConfigType: inlineSignParams inlineOperation: | [{"keyCode":"CP-401337","operationSetCode":"MacAppDeveloperSign","parameters":[{"parameterName":"hardening","parameterValue":"--options=runtime"}],"toolName":"sign","toolVersion":"1.0"}] + + diff --git a/.pipelines/v2/templates/steps-build-linux.yml b/.pipelines/v2/templates/steps-build-linux.yml index b16b13fdd..deee978e6 100644 --- a/.pipelines/v2/templates/steps-build-linux.yml +++ b/.pipelines/v2/templates/steps-build-linux.yml @@ -16,6 +16,9 @@ parameters: default: false steps: +- checkout: self + path: s/Foundry-Local + clean: true - bash: | set -euo pipefail @@ -43,10 +46,9 @@ steps: displayName: 'Append version define' - ${{ if eq(parameters.runTests, true) }}: - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared - basePath: '$(Agent.BuildDirectory)' + - checkout: test-data-shared + lfs: true + path: test-data-shared - bash: | set -euo pipefail @@ -54,7 +56,7 @@ steps: --config ${{ parameters.buildConfig }} \ --cmake_extra_defines $(cmakeFetchDefines) displayName: 'Configure and build' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)/vcpkg @@ -63,7 +65,7 @@ steps: set -euo pipefail python3 build.py --test --config ${{ parameters.buildConfig }} displayName: 'Run tests' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)/vcpkg FOUNDRY_TEST_DATA_DIR: $(Agent.BuildDirectory)/test-data-shared @@ -85,7 +87,7 @@ steps: # side); test/example binaries are not part of the redistributable surface. - bash: | set -euo pipefail - src='$(Build.SourcesDirectory)/sdk_v2/cpp/build/Linux/${{ parameters.buildConfig }}/bin' + src='$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp/build/Linux/${{ parameters.buildConfig }}/bin' dst='$(Build.ArtifactStagingDirectory)/native' mkdir -p "$dst" @@ -97,3 +99,7 @@ steps: cp -P "$primary" "$dst/" echo " staged libfoundry_local.so" displayName: 'Stage native artifacts' + + + + diff --git a/.pipelines/v2/templates/steps-build-macos.yml b/.pipelines/v2/templates/steps-build-macos.yml index 278bdbff8..b7635b830 100644 --- a/.pipelines/v2/templates/steps-build-macos.yml +++ b/.pipelines/v2/templates/steps-build-macos.yml @@ -14,6 +14,9 @@ parameters: default: false steps: +- checkout: self + path: s/Foundry-Local + clean: true - bash: | set -euo pipefail @@ -38,7 +41,7 @@ steps: genaiVersion: ${{ parameters.genaiVersion }} winmlVersion: '' includeWinml: false - shell: bash + shell: pwsh # Bake the pipeline-computed version into the binary so FoundryLocalGetVersionString() # matches the .nupkg version instead of the cmake default. @@ -51,10 +54,9 @@ steps: displayName: 'Append version define' - ${{ if eq(parameters.runTests, true) }}: - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared - basePath: '$(Agent.BuildDirectory)' + - checkout: test-data-shared + lfs: true + path: test-data-shared - bash: | set -euo pipefail @@ -62,7 +64,7 @@ steps: --config ${{ parameters.buildConfig }} \ --cmake_extra_defines $(cmakeFetchDefines) displayName: 'Configure and build' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)/vcpkg @@ -71,7 +73,7 @@ steps: set -euo pipefail python3 build.py --test --config ${{ parameters.buildConfig }} displayName: 'Run tests' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)/vcpkg FOUNDRY_TEST_DATA_DIR: $(Agent.BuildDirectory)/test-data-shared @@ -90,7 +92,7 @@ steps: # are not part of the redistributable surface. - bash: | set -euo pipefail - src='$(Build.SourcesDirectory)/sdk_v2/cpp/build/macOS/${{ parameters.buildConfig }}/bin' + src='$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp/build/macOS/${{ parameters.buildConfig }}/bin' dst='$(Build.ArtifactStagingDirectory)/native' mkdir -p "$dst" @@ -102,3 +104,7 @@ steps: cp -P "$primary" "$dst/" echo " staged libfoundry_local.dylib" displayName: 'Stage native artifacts' + + + + diff --git a/.pipelines/v2/templates/steps-build-python.yml b/.pipelines/v2/templates/steps-build-python.yml index fe81d69cd..606da3404 100644 --- a/.pipelines/v2/templates/steps-build-python.yml +++ b/.pipelines/v2/templates/steps-build-python.yml @@ -37,12 +37,33 @@ parameters: steps: -- task: UsePythonVersion@0 - displayName: 'Use Python 3.12' - inputs: - versionSpec: '3.12' - addToPath: true - architecture: '${{ parameters.pythonArchitecture }}' +- ${{ if ne(parameters.rid, 'linux-x64') }}: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.12' + inputs: + versionSpec: '3.12' + addToPath: true + architecture: '${{ parameters.pythonArchitecture }}' + +- ${{ if eq(parameters.rid, 'linux-x64') }}: + - task: PowerShell@2 + displayName: 'Use preinstalled system Python (Linux)' + inputs: + targetType: inline + pwsh: true + script: | + # Hosted Linux images may not have Python in Agent.ToolsDirectory, + # and UsePythonVersion then attempts a blocked GitHub download path. + # Use the preinstalled system python3 instead. + $pythonCmd = if (Get-Command python3.12 -ErrorAction SilentlyContinue) { 'python3.12' } else { 'python3' } + if (-not (Get-Command $pythonCmd -ErrorAction SilentlyContinue)) { + throw "Neither python3.12 nor python3 is available on PATH" + } + & $pythonCmd --version + $pythonPath = (& which $pythonCmd).Trim() + if (-not $pythonPath) { throw "Unable to resolve path for $pythonCmd" } + Write-Host "System python: $pythonPath" + Write-Host "##vso[task.setvariable variable=buildPythonExe]$pythonPath" - task: PowerShell@2 displayName: 'Set package version from pyVersion.txt' @@ -62,7 +83,7 @@ steps: targetType: inline pwsh: true script: | - $pyproject = "$(Build.SourcesDirectory)/sdk_v2/python/pyproject.toml" + $pyproject = "$(Build.Repository.LocalPath)/sdk_v2/python/pyproject.toml" $content = Get-Content $pyproject -Raw # Replace the first occurrence of `version = "..."` inside the # [project] table. The backend shim handles the package-name @@ -93,7 +114,7 @@ steps: targetType: inline pwsh: true script: | - $dest = "$(Build.SourcesDirectory)/sdk_v2/python/src/foundry_local_sdk/_native/${{ parameters.rid }}" + $dest = "$(Build.Repository.LocalPath)/sdk_v2/python/src/foundry_local_sdk/_native/${{ parameters.rid }}" if (Test-Path $dest) { Remove-Item -Recurse -Force $dest } New-Item -ItemType Directory -Force -Path $dest | Out-Null @@ -132,7 +153,7 @@ steps: targetType: inline pwsh: true script: | - $py = "$(Build.SourcesDirectory)/sdk_v2/python" + $py = "$(Build.Repository.LocalPath)/sdk_v2/python" # Remove stale, untagged cffi extension from prior dev compiles. # The wheel build below produces a properly ABI-tagged variant; the # untagged copy is bonus baggage we don't want shipped. @@ -152,17 +173,58 @@ steps: } } +- ${{ if eq(parameters.rid, 'linux-x64') }}: + - task: PowerShell@2 + displayName: 'Create Linux build venv' + inputs: + targetType: inline + pwsh: true + script: | + $basePython = "$(buildPythonExe)" + if (-not $basePython) { throw "buildPythonExe is not set" } + + $venvDir = "$(Pipeline.Workspace)/.venv-python-build" + if (Test-Path $venvDir) { Remove-Item -Recurse -Force $venvDir } + + & $basePython -m venv $venvDir + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + $venvPython = Join-Path $venvDir 'bin/python' + if (-not (Test-Path $venvPython)) { throw "Venv python not found at $venvPython" } + + & $venvPython --version + Write-Host "##vso[task.setvariable variable=buildPythonExe]$venvPython" + +- task: PipAuthenticate@1 + displayName: 'Authenticate pip with Azure Artifacts' + inputs: + artifactFeeds: 'AIFoundryLocal/AIFoundryLocal_PublicPackages' + - task: PowerShell@2 displayName: 'Install build tooling' inputs: targetType: inline pwsh: true script: | - python -m pip install --upgrade pip - python -m pip install --upgrade build setuptools wheel "cffi>=1.16" + # Require the authenticated pip URL injected by PipAuthenticate@1. + $indexUrl = $env:PIP_INDEX_URL + if (-not $indexUrl -and $env:PIP_EXTRA_INDEX_URL) { + $indexUrl = ($env:PIP_EXTRA_INDEX_URL -split '\s+')[0] + } + if (-not $indexUrl) { throw 'PipAuthenticate did not provide an authenticated pip index URL (PIP_INDEX_URL/PIP_EXTRA_INDEX_URL missing).' } + Write-Host "Using pip index: $indexUrl" + $pythonExe = if ('${{ parameters.rid }}' -eq 'linux-x64') { '$(buildPythonExe)' } else { 'python' } + if (-not $pythonExe) { throw 'Python executable is not set.' } + + & $pythonExe -m pip install --upgrade pip --index-url $indexUrl + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & $pythonExe -m pip install --upgrade build setuptools wheel "cffi>=1.16" --index-url $indexUrl if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - ${{ if eq(parameters.targetArch, 'arm64') }}: + - task: NuGetAuthenticate@1 + displayName: 'Authenticate NuGet feeds' + - task: PowerShell@2 displayName: 'Locate MSVC arm64 cross-tools (vswhere)' inputs: @@ -224,12 +286,12 @@ steps: New-Item -ItemType Directory -Force -Path $stageRoot | Out-Null # nuget.exe is on PATH on Microsoft-hosted Windows agents and on most - # self-hosted ones. The package is on nuget.org; -Source pins it - # explicitly so a misconfigured local nuget.config can't redirect us. + # self-hosted ones. Pull through the AIFoundryLocal feed (upstreamed + # to nuget.org) to avoid direct nuget.org network dependency. & nuget install pythonarm64 ` -Version 3.12.10 ` -OutputDirectory $stageRoot ` - -Source 'https://api.nuget.org/v3/index.json' ` + -Source 'https://pkgs.dev.azure.com/aiinfra/AIFoundryLocal/_packaging/AIFoundryLocal_PublicPackages/nuget/v3/index.json' ` -ExcludeVersion ` -NonInteractive if ($LASTEXITCODE -ne 0) { throw "nuget install pythonarm64 failed ($LASTEXITCODE)" } @@ -285,7 +347,7 @@ steps: $extraArgs += '--config-setting=--build-option=--plat-name=win_arm64' Write-Host "Cross-compile: forcing wheel plat tag win_arm64" } - Push-Location "$(Build.SourcesDirectory)/sdk_v2/python" + Push-Location "$(Build.Repository.LocalPath)/sdk_v2/python" try { if ($targetArch -eq 'arm64') { # Run vcvarsall and the build in a single child cmd.exe process so the cross @@ -325,7 +387,9 @@ steps: cmd /v:on /c $cmdLine if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } else { - python -m build --wheel --outdir $outDir @extraArgs + $pythonExe = if ('${{ parameters.rid }}' -eq 'linux-x64') { '$(buildPythonExe)' } else { 'python' } + if (-not $pythonExe) { throw 'Python executable is not set.' } + & $pythonExe -m build --wheel --outdir $outDir @extraArgs if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } } finally { @@ -340,3 +404,5 @@ steps: script: | Get-ChildItem "${{ parameters.outputDir }}" -Filter '*.whl' | ForEach-Object { Write-Host ("{0,12} {1}" -f $_.Length, $_.Name) } + + diff --git a/.pipelines/v2/templates/steps-build-windows.yml b/.pipelines/v2/templates/steps-build-windows.yml index 0e17f07b6..fc26296a9 100644 --- a/.pipelines/v2/templates/steps-build-windows.yml +++ b/.pipelines/v2/templates/steps-build-windows.yml @@ -37,6 +37,9 @@ parameters: default: false steps: +- checkout: self + path: s/Foundry-Local + clean: true # Windows hosted agents don't have python on PATH by default — the launcher # stub redirects to the Microsoft Store. UsePythonVersion installs and adds it @@ -78,10 +81,9 @@ steps: # Tests need shared model files. - ${{ if eq(parameters.runTests, true) }}: - - template: ../../templates/checkout-steps.yml@self - parameters: - repoName: test-data-shared - basePath: '$(Agent.BuildDirectory)' + - checkout: test-data-shared + lfs: true + path: test-data-shared - ${{ if and(eq(parameters.arch, 'x64'), eq(parameters.useWinml, false)) }}: - script: >- @@ -91,7 +93,7 @@ steps: --cmake_generator "Visual Studio 17 2022" --cmake_extra_defines $(cmakeFetchDefines) displayName: 'Configure and build (x64)' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg PKG_CONFIG: $(Build.BinariesDirectory)\tools\pkg-config.bat @@ -105,7 +107,7 @@ steps: --use_winml --winml_sdk_version ${{ parameters.winmlVersion }} --cmake_extra_defines $(cmakeFetchDefines) displayName: 'Configure and build (x64, WinML)' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg PKG_CONFIG: $(Build.BinariesDirectory)\tools\pkg-config.bat @@ -118,7 +120,7 @@ steps: --cmake_generator "Visual Studio 17 2022" --cmake_extra_defines $(cmakeFetchDefines) displayName: 'Configure and build (arm64 cross-compile)' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg PKG_CONFIG: $(Build.BinariesDirectory)\tools\pkg-config.bat @@ -132,7 +134,7 @@ steps: --use_winml --winml_sdk_version ${{ parameters.winmlVersion }} --cmake_extra_defines $(cmakeFetchDefines) displayName: 'Configure and build (arm64 cross-compile, WinML)' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg PKG_CONFIG: $(Build.BinariesDirectory)\tools\pkg-config.bat @@ -140,7 +142,7 @@ steps: - ${{ if and(eq(parameters.runTests, true), eq(parameters.arch, 'x64')) }}: - script: python build.py --test --config ${{ parameters.buildConfig }} displayName: 'Run tests' - workingDirectory: $(Build.SourcesDirectory)/sdk_v2/cpp + workingDirectory: $(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp env: VCPKG_ROOT: $(Build.BinariesDirectory)\vcpkg FOUNDRY_TEST_DATA_DIR: $(Agent.BuildDirectory)\test-data-shared @@ -162,8 +164,8 @@ steps: targetType: inline pwsh: true script: | - $binDir = '$(Build.SourcesDirectory)/sdk_v2/cpp/build/Windows/${{ parameters.buildConfig }}/bin/${{ parameters.buildConfig }}' - $linkDir = '$(Build.SourcesDirectory)/sdk_v2/cpp/build/Windows/${{ parameters.buildConfig }}/${{ parameters.buildConfig }}' + $binDir = '$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp/build/Windows/${{ parameters.buildConfig }}/bin/${{ parameters.buildConfig }}' + $linkDir = '$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp/build/Windows/${{ parameters.buildConfig }}/${{ parameters.buildConfig }}' $dst = '$(Build.ArtifactStagingDirectory)/native' New-Item -ItemType Directory -Force -Path $dst | Out-Null @@ -189,7 +191,7 @@ steps: - task: CopyFiles@2 displayName: 'Stage public headers' inputs: - SourceFolder: '$(Build.SourcesDirectory)/sdk_v2/cpp/include' + SourceFolder: '$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp/include' Contents: '**' TargetFolder: '$(Build.ArtifactStagingDirectory)/include' @@ -202,6 +204,10 @@ steps: - task: CopyFiles@2 displayName: 'Stage ms-gsl headers' inputs: - SourceFolder: '$(Build.SourcesDirectory)/sdk_v2/cpp/build/Windows/${{ parameters.buildConfig }}/vcpkg_installed/x64-windows/include/gsl' + SourceFolder: '$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cpp/build/Windows/${{ parameters.buildConfig }}/vcpkg_installed/x64-windows/include/gsl' Contents: '**' TargetFolder: '$(Build.ArtifactStagingDirectory)/include/gsl' + + + + diff --git a/.pipelines/v2/templates/steps-pack-js.yml b/.pipelines/v2/templates/steps-pack-js.yml index 439ab41fd..51a2e2f1e 100644 --- a/.pipelines/v2/templates/steps-pack-js.yml +++ b/.pipelines/v2/templates/steps-pack-js.yml @@ -27,8 +27,26 @@ steps: inputs: versionSpec: '20.x' +- task: PowerShell@2 + displayName: 'Configure npm registry (AIFoundryLocal_PublicPackages)' + inputs: + targetType: inline + pwsh: true + script: | + $npmrc = "$(Agent.TempDirectory)/foundrylocal-$(System.JobId).npmrc" + $registry = 'https://pkgs.dev.azure.com/aiinfra/AIFoundryLocal/_packaging/AIFoundryLocal_PublicPackages/npm/registry/' + Set-Content -Path $npmrc -Value "registry=$registry`nalways-auth=true" -Encoding UTF8 + Write-Host "##vso[task.setvariable variable=npmUserConfig]$npmrc" + Write-Host "##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]$npmrc" + Write-Host "npm userconfig: $npmrc" + +- task: npmAuthenticate@0 + displayName: 'Authenticate npm with Azure Artifacts' + inputs: + workingFile: '$(npmUserConfig)' + - pwsh: | - $dst = "$(Build.SourcesDirectory)/sdk_v2/js/prebuilds" + $dst = "$(Build.Repository.LocalPath)/sdk_v2/js/prebuilds" if (Test-Path $dst) { Remove-Item -Recurse -Force $dst } New-Item -ItemType Directory -Force -Path $dst | Out-Null @@ -53,28 +71,46 @@ steps: displayName: 'npm ci' inputs: command: custom - workingDir: '$(Build.SourcesDirectory)/sdk_v2/js' - customCommand: 'ci --no-audit --no-fund --ignore-scripts' + workingDir: '$(Build.Repository.LocalPath)/sdk_v2/js' + customCommand: 'ci --no-audit --no-fund --ignore-scripts --include=dev' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' + +- pwsh: | + Set-Location "$(Build.Repository.LocalPath)/sdk_v2/js" + npm install --no-save --ignore-scripts --include=dev --no-audit --no-fund typescript + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + displayName: 'Ensure TypeScript tooling is installed' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' - pwsh: | $version = (Get-Content "$(Pipeline.Workspace)/version-info/sdkVersion.txt" -Raw).Trim() Write-Host "Stamping JS package version: $version" - Set-Location "$(Build.SourcesDirectory)/sdk_v2/js" + Set-Location "$(Build.Repository.LocalPath)/sdk_v2/js" npm version $version --no-git-tag-version --allow-same-version if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Stamp package version' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' - pwsh: | - Set-Location "$(Build.SourcesDirectory)/sdk_v2/js" + Set-Location "$(Build.Repository.LocalPath)/sdk_v2/js" npm run build:ts if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Build TypeScript' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' - pwsh: | $dst = '${{ parameters.outputDir }}' New-Item -ItemType Directory -Force -Path $dst | Out-Null - Set-Location "$(Build.SourcesDirectory)/sdk_v2/js" + Set-Location "$(Build.Repository.LocalPath)/sdk_v2/js" npm pack --pack-destination $dst if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Get-ChildItem $dst | ForEach-Object { Write-Host " $($_.Name) $($_.Length) bytes" } displayName: 'npm pack' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' + + diff --git a/.pipelines/v2/templates/steps-pack-nuget.yml b/.pipelines/v2/templates/steps-pack-nuget.yml index bde97d772..c0ba7dbc7 100644 --- a/.pipelines/v2/templates/steps-pack-nuget.yml +++ b/.pipelines/v2/templates/steps-pack-nuget.yml @@ -57,10 +57,16 @@ steps: $version = (Get-Content "$(Pipeline.Workspace)/version-info/sdkVersion.txt" -Raw).Trim() Write-Host "Packing version: $version" + $packScript = "$(Build.SourcesDirectory)/sdk_v2/cpp/nuget/pack.py" + if (-not (Test-Path $packScript)) { + throw "pack.py not found at expected path: $packScript" + } + Write-Host "Using pack script: $packScript" + $outDir = "$(Build.ArtifactStagingDirectory)/nuget" New-Item -ItemType Directory -Force -Path $outDir | Out-Null - python "$(Build.SourcesDirectory)/sdk_v2/cpp/nuget/pack.py" ` + python "$packScript" ` --version "$version" ` --package_id "Microsoft.AI.Foundry.Local.Runtime" ` --ort_version "${{ parameters.ortVersion }}" ` @@ -86,10 +92,16 @@ steps: $version = (Get-Content "$(Pipeline.Workspace)/version-info/sdkVersion.txt" -Raw).Trim() Write-Host "Packing version: $version" + $packScript = "$(Build.SourcesDirectory)/sdk_v2/cpp/nuget/pack.py" + if (-not (Test-Path $packScript)) { + throw "pack.py not found at expected path: $packScript" + } + Write-Host "Using pack script: $packScript" + $outDir = "$(Build.ArtifactStagingDirectory)/nuget" New-Item -ItemType Directory -Force -Path $outDir | Out-Null - python "$(Build.SourcesDirectory)/sdk_v2/cpp/nuget/pack.py" ` + python "$packScript" ` --version "$version" ` --package_id "Microsoft.AI.Foundry.Local.Runtime.WinML" ` --ort_version "${{ parameters.ortVersion }}" ` @@ -100,3 +112,5 @@ steps: Write-Host "Generated packages:" Get-ChildItem $outDir -Filter '*.nupkg' | ForEach-Object { Write-Host " $($_.Name)" } + + diff --git a/.pipelines/v2/templates/steps-prefetch-nuget.yml b/.pipelines/v2/templates/steps-prefetch-nuget.yml index 661075294..9b55993c9 100644 --- a/.pipelines/v2/templates/steps-prefetch-nuget.yml +++ b/.pipelines/v2/templates/steps-prefetch-nuget.yml @@ -10,7 +10,7 @@ # winmlVersion – Microsoft.Windows.AI.MachineLearning version (Windows only) # includeWinml – Download WinML and emit WINML_EP_CATALOG_FETCH_URL # includeOrtGpuLinux – Also download Microsoft.ML.OnnxRuntime.Gpu.Linux (Linux only) -# shell – 'pwsh' (Windows/macOS) or 'bash' (Linux) +# shell – 'pwsh' (Windows/macOS) or 'bash' (Linux only) parameters: - name: ortVersion @@ -32,6 +32,9 @@ parameters: steps: +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet' + # Fail fast if the pipeline-pinned versions drift from the cmake source of # truth in sdk_v2/deps_versions.json. - task: PowerShell@2 @@ -41,8 +44,12 @@ steps: pwsh: true script: | $ErrorActionPreference = 'Stop' - $depsFile = Join-Path "$(Build.SourcesDirectory)" "sdk_v2/deps_versions.json" - if (-not (Test-Path $depsFile)) { throw "deps_versions.json not found at $depsFile" } + $depsFile = Join-Path "$(Pipeline.Workspace)/s/Foundry-Local" "sdk_v2/deps_versions.json" + if (-not (Test-Path $depsFile)) { + throw "deps_versions.json not found at $depsFile" + } + + Write-Host "Using deps_versions.json at: $depsFile" $deps = Get-Content $depsFile -Raw | ConvertFrom-Json $expected = @{ 'onnxruntime' = '${{ parameters.ortVersion }}' @@ -61,11 +68,14 @@ steps: if ($errors.Count -gt 0) { Write-Host "Version drift detected between pipeline literals and sdk_v2/deps_versions.json:" $errors | ForEach-Object { Write-Host $_ } - throw "Bump the matching cpp*Version variable in .pipelines/foundry-local-packaging.yml" + throw "Bump the matching cpp*Version variable in .pipelines/v2/foundry-local-packaging.yml" } Write-Host "Pinned versions agree with sdk_v2/deps_versions.json." - ${{ if eq(parameters.shell, 'pwsh') }}: + - task: NuGetAuthenticate@1 + displayName: 'Authenticate NuGet feeds' + - task: PowerShell@2 displayName: 'Pre-download NuGet packages' inputs: @@ -76,11 +86,56 @@ steps: $cacheDir = "$(Build.BinariesDirectory)/nuget_packages" New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null - # All four packages are public on nuget.org. Foundry Local Core's - # nuget.config maps everything except Microsoft.Telemetry* to nuget.org - # (see .pipelines/templates/build-core-steps.yml), so we follow the - # same source of truth here. - $feed = "https://www.nuget.org/api/v2/package" + $nugetCmd = Get-Command nuget -ErrorAction SilentlyContinue + if (-not $nugetCmd) { + throw "nuget CLI not found on PATH. This step uses nuget install to fetch package artifacts." + } + + function Install-NuGetPackage { + param( + [string]$Id, + [string]$Version, + [string]$OutDir + ) + + $pkgDir = Join-Path $OutDir $Id + if (Test-Path $pkgDir) { + Remove-Item -Recurse -Force $pkgDir + } + + $nugetArgs = @( + 'install', $Id, + '-Version', $Version, + '-Source', 'https://pkgs.dev.azure.com/aiinfra/AIFoundryLocal/_packaging/AIFoundryLocal_PublicPackages/nuget/v3/index.json', + '-OutputDirectory', $OutDir, + '-ExcludeVersion', + '-NonInteractive', + '-DirectDownload', + '-PackageSaveMode', 'nupkg' + ) + + Write-Host "Running: nuget $($nugetArgs -join ' ')" + # nuget writes status lines to stdout; forward them to logs but keep + # this function's output channel reserved for the return path. + & nuget $nugetArgs | ForEach-Object { Write-Host $_ } + if ($LASTEXITCODE -ne 0) { + throw "nuget install failed for $Id $Version" + } + + $nupkg = Get-ChildItem -Path $pkgDir -Filter '*.nupkg' -File -Recurse -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 + if (-not $nupkg) { + throw "nuget install did not produce a .nupkg under $pkgDir" + } + + $resolvedPath = $nupkg | Select-Object -ExpandProperty FullName -First 1 + if ([string]::IsNullOrWhiteSpace($resolvedPath)) { + throw "nuget install produced an invalid path under $pkgDir" + } + + return $resolvedPath + } $packages = @( @{ key = 'genai'; id = 'Microsoft.ML.OnnxRuntimeGenAI.Foundry'; version = '${{ parameters.genaiVersion }}' }, @@ -92,30 +147,28 @@ steps: $defines = @() foreach ($pkg in $packages) { - $url = "$feed/$($pkg.id)/$($pkg.version)" - $out = "$cacheDir/$($pkg.key).nupkg" - Write-Host "Downloading $($pkg.id) $($pkg.version)" - Invoke-WebRequest -Uri $url -OutFile $out -UseBasicParsing - if (-not (Test-Path $out)) { throw "$($pkg.key) download failed" } - Write-Host " -> $out ($((Get-Item $out).Length) bytes)" + Write-Host "Downloading $($pkg.id) $($pkg.version) via nuget install" + $out = Install-NuGetPackage -Id $pkg.id -Version $pkg.version -OutDir $cacheDir + if ([string]::IsNullOrWhiteSpace($out)) { throw "$($pkg.key) download failed" } + $downloaded = Get-Item -LiteralPath $out -ErrorAction Stop + Write-Host " -> $($downloaded.FullName) ($($downloaded.Length) bytes)" switch ($pkg.key) { - 'genai' { $defines += "GENAI_FETCH_URL=$out" } - 'ort' { $defines += "ORT_FETCH_URL=$out" } - 'ort_gpu_linux' { $defines += "ORT_GPU_LINUX_FETCH_URL=$out" } + 'genai' { $defines += "GENAI_FETCH_URL=$($downloaded.FullName)" } + 'ort' { $defines += "ORT_FETCH_URL=$($downloaded.FullName)" } + 'ort_gpu_linux' { $defines += "ORT_GPU_LINUX_FETCH_URL=$($downloaded.FullName)" } } } if ($${{ parameters.includeWinml }}) { # WinML 2.x (Microsoft.Windows.AI.MachineLearning) is reg-free — a single self-contained # native package with no transitive Windows App SDK Foundation dependency to resolve. - $winmlUrl = "$feed/Microsoft.Windows.AI.MachineLearning/${{ parameters.winmlVersion }}" - $winmlOut = "$cacheDir/winml.nupkg" - Write-Host "Downloading Microsoft.Windows.AI.MachineLearning ${{ parameters.winmlVersion }}" - Invoke-WebRequest -Uri $winmlUrl -OutFile $winmlOut - if (-not (Test-Path $winmlOut)) { throw "WinML download failed" } - Write-Host " -> $winmlOut ($((Get-Item $winmlOut).Length) bytes)" - $defines += "WINML_EP_CATALOG_FETCH_URL=$winmlOut" + Write-Host "Downloading Microsoft.Windows.AI.MachineLearning ${{ parameters.winmlVersion }} via nuget install" + $winmlOut = Install-NuGetPackage -Id 'Microsoft.Windows.AI.MachineLearning' -Version '${{ parameters.winmlVersion }}' -OutDir $cacheDir + if ([string]::IsNullOrWhiteSpace($winmlOut)) { throw "WinML download failed" } + $winmlDownloaded = Get-Item -LiteralPath $winmlOut -ErrorAction Stop + Write-Host " -> $($winmlDownloaded.FullName) ($($winmlDownloaded.Length) bytes)" + $defines += "WINML_EP_CATALOG_FETCH_URL=$($winmlDownloaded.FullName)" } $joined = ($defines | ForEach-Object { "`"$_`"" }) -join ' ' @@ -123,16 +176,28 @@ steps: Write-Host "cmakeFetchDefines = $joined" - ${{ if eq(parameters.shell, 'bash') }}: + - task: NuGetAuthenticate@1 + displayName: 'Authenticate NuGet feeds' + - bash: | set -euo pipefail cacheDir="$(Build.BinariesDirectory)/nuget_packages" mkdir -p "$cacheDir" - # All four packages are public on nuget.org. Foundry Local Core's - # nuget.config maps everything except Microsoft.Telemetry* to nuget.org - # (see .pipelines/templates/build-core-steps.yml), so we follow the - # same source of truth here. - feed="https://www.nuget.org/api/v2/package" + if [ "${{ parameters.includeOrtGpuLinux }}" != "True" ]; then + echo "ERROR: the bash prefetch branch is reserved for Linux native builds; use shell=pwsh for non-Linux platforms." >&2 + exit 1 + fi + + if [ -z "${SYSTEM_ACCESSTOKEN:-}" ]; then + echo "ERROR: SYSTEM_ACCESSTOKEN is not available for authenticated package download." >&2 + exit 1 + fi + + # Download package content from the feed's V3 flat-container endpoint. + # This avoids direct nuget.org access and avoids nuget.exe + mono on + # hosted Linux agents. + flatBase="https://pkgs.dev.azure.com/aiinfra/AIFoundryLocal/_packaging/AIFoundryLocal_PublicPackages/nuget/v3/flat2" declare -a entries=( "genai:Microsoft.ML.OnnxRuntimeGenAI.Foundry:${{ parameters.genaiVersion }}" @@ -151,9 +216,13 @@ steps: for entry in "${entries[@]}"; do IFS=: read -r key id version <<< "$entry" out="$cacheDir/${key}.nupkg" - url="$feed/${id}/${version}" + lowerId=$(printf '%s' "$id" | tr '[:upper:]' '[:lower:]') + url="$flatBase/${lowerId}/${version}/${lowerId}.${version}.nupkg" echo "Downloading $id $version" - curl -fSL -o "$out" "$url" + curl -fSL \ + -H "Authorization: Bearer $SYSTEM_ACCESSTOKEN" \ + -o "$out" \ + "$url" echo " -> $out ($(stat -c%s "$out" 2>/dev/null || stat -f%z "$out") bytes)" case "$key" in @@ -167,3 +236,7 @@ steps: echo "##vso[task.setvariable variable=cmakeFetchDefines]$joined" echo "cmakeFetchDefines = $joined" displayName: 'Pre-download NuGet packages' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + diff --git a/.pipelines/v2/templates/steps-test-cs.yml b/.pipelines/v2/templates/steps-test-cs.yml index 73274e275..a39b4b8c6 100644 --- a/.pipelines/v2/templates/steps-test-cs.yml +++ b/.pipelines/v2/templates/steps-test-cs.yml @@ -67,7 +67,6 @@ steps: - @@ -110,9 +109,11 @@ steps: targetType: inline pwsh: true script: | - $proj = "$(Build.SourcesDirectory)/sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj" - $rid = dotnet msbuild $proj -getProperty:NETCoreSdkRuntimeIdentifier - if ($LASTEXITCODE -ne 0 -or -not $rid) { throw "Failed to determine RuntimeIdentifier" } + $proj = "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj" + $dotnetInfo = (dotnet --info | Out-String) + $m = [regex]::Match($dotnetInfo, '(?m)^\s*RID:\s*(\S+)\s*$') + if (-not $m.Success) { throw "Failed to determine RuntimeIdentifier from dotnet --info" } + $rid = $m.Groups[1].Value.Trim() Write-Host "Restoring for RuntimeIdentifier: $rid" dotnet restore $proj ` @@ -142,7 +143,7 @@ steps: # The redirect file foundry_local.native.cfg is a local-dev override that, if # present, takes priority in DllLoader and would silently point the loader at # an arbitrary path on the agent. Sweep it out of every bin folder before tests. - $root = "$(Build.SourcesDirectory)/sdk_v2/cs" + $root = "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cs" $stale = Get-ChildItem -Path $root -Recurse -Force -Filter 'foundry_local.native.cfg' -ErrorAction SilentlyContinue if ($stale) { Write-Host "Removing stale redirect files:" @@ -157,7 +158,7 @@ steps: targetType: inline pwsh: true script: | - $proj = "$(Build.SourcesDirectory)/sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj" + $proj = "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/cs/test/FoundryLocal.Tests/Microsoft.AI.Foundry.Local.Tests.csproj" # Test TFMs: # * net9.0 covers the .NET (Core) runtime test surface. @@ -185,3 +186,5 @@ steps: env: TF_BUILD: 'true' FOUNDRY_TEST_DATA_DIR: ${{ parameters.testDataSharedDir }} + + diff --git a/.pipelines/v2/templates/steps-test-js.yml b/.pipelines/v2/templates/steps-test-js.yml index d161da06e..c21e739e2 100644 --- a/.pipelines/v2/templates/steps-test-js.yml +++ b/.pipelines/v2/templates/steps-test-js.yml @@ -23,10 +23,28 @@ steps: inputs: versionSpec: '20.x' +- task: PowerShell@2 + displayName: 'Configure npm registry (AIFoundryLocal_PublicPackages)' + inputs: + targetType: inline + pwsh: true + script: | + $npmrc = "$(Agent.TempDirectory)/foundrylocal-$(System.JobId).npmrc" + $registry = 'https://pkgs.dev.azure.com/aiinfra/AIFoundryLocal/_packaging/AIFoundryLocal_PublicPackages/npm/registry/' + Set-Content -Path $npmrc -Value "registry=$registry`nalways-auth=true" -Encoding UTF8 + Write-Host "##vso[task.setvariable variable=npmUserConfig]$npmrc" + Write-Host "##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]$npmrc" + Write-Host "npm userconfig: $npmrc" + +- task: npmAuthenticate@0 + displayName: 'Authenticate npm with Azure Artifacts' + inputs: + workingFile: '$(npmUserConfig)' + - ${{ if eq(parameters.rid, 'win-x64') }}: - pwsh: | $src = "${{ parameters.prebuildArtifactDir }}/prebuilds" - $dst = "$(Build.SourcesDirectory)/sdk_v2/js/prebuilds" + $dst = "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js/prebuilds" if (-not (Test-Path $src)) { throw "Prebuild artifact missing: $src" } if (Test-Path $dst) { Remove-Item -Recurse -Force $dst } New-Item -ItemType Directory -Force -Path $dst | Out-Null @@ -38,7 +56,7 @@ steps: - bash: | set -euo pipefail src="${{ parameters.prebuildArtifactDir }}/prebuilds" - dst="$(Build.SourcesDirectory)/sdk_v2/js/prebuilds" + dst="$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js/prebuilds" if [ ! -d "$src" ]; then echo "Prebuild artifact missing: $src" >&2; exit 1; fi rm -rf "$dst" mkdir -p "$dst" @@ -51,40 +69,60 @@ steps: displayName: 'npm ci (runs install-native postinstall)' inputs: command: custom - workingDir: '$(Build.SourcesDirectory)/sdk_v2/js' - customCommand: 'ci --no-audit --no-fund' + workingDir: '$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js' + customCommand: 'ci --no-audit --no-fund --include=dev' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' + + - pwsh: | + Set-Location "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js" + npm install --no-save --ignore-scripts --include=dev --no-audit --no-fund typescript + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + displayName: 'Ensure TypeScript tooling is installed' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' - pwsh: | - Set-Location "$(Build.SourcesDirectory)/sdk_v2/js" + Set-Location "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js" npm run build:ts if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Build TypeScript' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' - pwsh: | - Set-Location "$(Build.SourcesDirectory)/sdk_v2/js" + Set-Location "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js" npx --no-install vitest run --reporter=verbose if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Run vitest' env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' FOUNDRY_TEST_DATA_DIR: '${{ parameters.testDataSharedDir }}' - ${{ if or(eq(parameters.rid, 'linux-x64'), eq(parameters.rid, 'osx-arm64')) }}: - bash: | set -euo pipefail - cd "$(Build.SourcesDirectory)/sdk_v2/js" - npm ci --no-audit --no-fund + cd "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js" + npm ci --no-audit --no-fund --include=dev displayName: 'npm ci (runs install-native postinstall)' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' - bash: | set -euo pipefail - cd "$(Build.SourcesDirectory)/sdk_v2/js" + cd "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js" npm run build:ts displayName: 'Build TypeScript' + env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' - bash: | set -euo pipefail - cd "$(Build.SourcesDirectory)/sdk_v2/js" + cd "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/js" npx --no-install vitest run --reporter=verbose displayName: 'Run vitest' env: + NPM_CONFIG_USERCONFIG: '$(npmUserConfig)' FOUNDRY_TEST_DATA_DIR: '${{ parameters.testDataSharedDir }}' + + diff --git a/.pipelines/v2/templates/steps-test-python.yml b/.pipelines/v2/templates/steps-test-python.yml index 9396a7cce..10c9c2c0b 100644 --- a/.pipelines/v2/templates/steps-test-python.yml +++ b/.pipelines/v2/templates/steps-test-python.yml @@ -50,6 +50,11 @@ steps: Write-Host "##vso[task.setvariable variable=venvPy]$venvPy" Write-Host "venv python: $venvPy" +- task: PipAuthenticate@1 + displayName: 'Authenticate pip with Azure Artifacts' + inputs: + artifactFeeds: 'AIFoundryLocal/AIFoundryLocal_PublicPackages' + - task: PowerShell@2 displayName: 'Install built wheel + test deps' inputs: @@ -68,6 +73,10 @@ steps: $wheel = $wheels[0].FullName Write-Host "Installing wheel: $wheel" + # Use the Azure Artifacts feed as the sole package index for test installs. + & "$(venvPy)" -m pip config set global.index-url https://pkgs.dev.azure.com/aiinfra/AIFoundryLocal/_packaging/AIFoundryLocal_PublicPackages/pypi/simple/ + & "$(venvPy)" -m pip config unset global.extra-index-url + if ($LASTEXITCODE -ne 0) { Write-Host 'No existing extra-index-url configured; continuing.' } & "$(venvPy)" -m pip install --upgrade pip & "$(venvPy)" -m pip install $wheel if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } @@ -80,7 +89,7 @@ steps: targetType: inline pwsh: true script: | - Push-Location "$(Build.SourcesDirectory)/sdk_v2/python" + Push-Location "$(Pipeline.Workspace)/s/Foundry-Local/sdk_v2/python" try { # Override pythonpath = ["src"] from pyproject.toml so pytest # imports the installed wheel (which contains the compiled cffi @@ -95,3 +104,5 @@ steps: env: TF_BUILD: 'true' FOUNDRY_TEST_DATA_DIR: ${{ parameters.testDataSharedDir }} + +