diff --git a/.azure-pipelines/esrp/sign.yml b/.azure-pipelines/esrp/sign.yml new file mode 100644 index 00000000000000..b4d14d2713ee8f --- /dev/null +++ b/.azure-pipelines/esrp/sign.yml @@ -0,0 +1,106 @@ +# Reusable step template for ESRP code signing via EsrpCodeSigning@6. +# +# For macOS, ESRP requires files to be submitted as a zip archive. +# Set 'useArchive: true' to automatically handle the +# copy → zip → sign → extract cycle. For Windows/Linux where ESRP +# can sign files directly in a folder, leave it as false (default). +# +parameters: + - name: displayName + type: string + - name: folderPath + type: string + - name: pattern + type: string + - name: inlineOperation + type: string + # When true, matching files are copied to a staging dir, zipped, + # signed, and extracted back to folderPath. + - name: useArchive + type: boolean + default: false + # ESRP connection parameters (defaults use pipeline variables) + - name: connectedServiceName + type: string + default: $(esrpAppConnectionName) + - name: appRegistrationClientId + type: string + default: $(esrpClientId) + - name: appRegistrationTenantId + type: string + default: $(esrpTenantId) + - name: authAkvName + type: string + default: $(esrpKeyVaultName) + - name: authSignCertName + type: string + default: $(esrpSignReqCertName) + - name: serviceEndpointUrl + type: string + default: $(esrpEndpointUrl) + +steps: + - ${{ if eq(parameters.useArchive, true) }}: + - task: DeleteFiles@1 + displayName: 'Clean staging dir for ${{ parameters.displayName }}' + inputs: + SourceFolder: '$(Agent.TempDirectory)/esrp-staging' + Contents: '*' + RemoveSourceFolder: true + - task: CopyFiles@2 + displayName: 'Collect files for ${{ parameters.displayName }}' + inputs: + SourceFolder: '${{ parameters.folderPath }}' + Contents: '${{ parameters.pattern }}' + TargetFolder: '$(Agent.TempDirectory)/esrp-staging/contents' + - task: ArchiveFiles@2 + displayName: 'Archive files for ${{ parameters.displayName }}' + inputs: + rootFolderOrFile: '$(Agent.TempDirectory)/esrp-staging/contents' + includeRootFolder: false + archiveType: zip + archiveFile: '$(Agent.TempDirectory)/esrp-staging/archive.zip' + - task: EsrpCodeSigning@6 + displayName: '${{ parameters.displayName }}' + inputs: + connectedServiceName: '${{ parameters.connectedServiceName }}' + useMSIAuthentication: true + appRegistrationClientId: '${{ parameters.appRegistrationClientId }}' + appRegistrationTenantId: '${{ parameters.appRegistrationTenantId }}' + authAkvName: '${{ parameters.authAkvName }}' + authSignCertName: '${{ parameters.authSignCertName }}' + serviceEndpointUrl: '${{ parameters.serviceEndpointUrl }}' + folderPath: '$(Agent.TempDirectory)/esrp-staging' + pattern: 'archive.zip' + useMinimatch: true + signConfigType: inlineSignParams + inlineOperation: ${{ parameters.inlineOperation }} + - task: ExtractFiles@1 + displayName: 'Extract signed files for ${{ parameters.displayName }}' + inputs: + archiveFilePatterns: '$(Agent.TempDirectory)/esrp-staging/archive.zip' + destinationFolder: '${{ parameters.folderPath }}' + overwriteExistingFiles: true + - task: DeleteFiles@1 + displayName: 'Clean up staging dir for ${{ parameters.displayName }}' + condition: always() + inputs: + SourceFolder: '$(Agent.TempDirectory)/esrp-staging' + Contents: '*' + RemoveSourceFolder: true + - ${{ else }}: + - task: EsrpCodeSigning@6 + displayName: '${{ parameters.displayName }}' + inputs: + connectedServiceName: '${{ parameters.connectedServiceName }}' + useMSIAuthentication: true + appRegistrationClientId: '${{ parameters.appRegistrationClientId }}' + appRegistrationTenantId: '${{ parameters.appRegistrationTenantId }}' + authAkvName: '${{ parameters.authAkvName }}' + authSignCertName: '${{ parameters.authSignCertName }}' + serviceEndpointUrl: '${{ parameters.serviceEndpointUrl }}' + folderPath: '${{ parameters.folderPath }}' + pattern: '${{ parameters.pattern }}' + useMinimatch: true + signConfigType: inlineSignParams + inlineOperation: ${{ parameters.inlineOperation }} diff --git a/.azure-pipelines/esrp/windows/esrpsign.sh b/.azure-pipelines/esrp/windows/esrpsign.sh new file mode 100755 index 00000000000000..ee2ed2db5d9e2d --- /dev/null +++ b/.azure-pipelines/esrp/windows/esrpsign.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# +# Sign Windows files using the ESRP client (Authenticode). +# Usage: esrpsign.sh [file2 ...] +# +# Required environment variables: +# ESRP_TOOL - Path to ESRPClient.exe +# ESRP_AUTH - Path to the ESRP auth JSON file +# SYSTEM_ACCESSTOKEN - ADO system access token (OAuth bearer) +# +# Optional environment variables: +# ESRP_KEYCODE - Signing key code (default: CP-231522) +# +# The script generates the auth and input JSON files and sets the +# following ESRP client environment variables automatically: +# ESRP_AUTH_CONFIG - Path to the auth JSON file +# ESRP_POLICY_CONFIG - Path to the policy JSON file +# ESRP_SESSION_CONFIG - Not set; ESRP client defaults are used +# +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "usage: esrpsign.sh [file ...]" >&2 + exit 1 +fi + +if [ -z "${ESRP_TOOL:-}" ]; then + echo "error: ESRP_TOOL environment variable must be set" >&2 + exit 1 +fi +if [ -z "${ESRP_AUTH:-}" ]; then + echo "error: ESRP_AUTH environment variable must be set" >&2 + exit 1 +fi +if [ -z "${SYSTEM_ACCESSTOKEN:-}" ]; then + echo "error: SYSTEM_ACCESSTOKEN environment variable must be set" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/../../scripts/windows/utils.sh" + +# Check for overriden key code, otherwise use default (Microsoft Third-Party/OSS) +ESRP_KEYCODE="${ESRP_KEYCODE:-CP-231522}" + +# Create work dir and resolve its Windows path by cd-ing into it. +WORK_DIR="$(mktemp -d)" +WORK_DIR_WIN="$(cd "$WORK_DIR" && pwd -W | sed 's|/|\\|g')" + +echo "==> ESRP signing tool: $ESRP_TOOL" +echo "==> Working directory: $WORK_DIR" + +if [ ! -f "$ESRP_TOOL" ]; then + echo "error: ESRPClient.exe not found at $ESRP_TOOL" >&2 + exit 1 +fi + +# Build the SignRequestFiles JSON array +echo "==> Preparing files for signing ($# file(s))..." +files_json="" +for file in "$@"; do + if [ ! -f "$file" ]; then + echo "error: file not found: $file" >&2 + exit 1 + fi + + abs_path="$(cd "$(dirname "$file")" && pwd)/$(basename "$file")" + win_path="$(to_windows_path "$abs_path")" + # Escape backslashes for JSON + win_path_escaped="${win_path//\\/\\\\}" + echo " - $win_path" + + if [ -n "$files_json" ]; then + files_json+="," + fi + files_json+=" + { + \"SourceLocation\": \"$win_path_escaped\", + \"DestinationLocation\": \"$win_path_escaped\" + }" +done + +# Generate the input JSON +input_json="$WORK_DIR/input.json" +output_json="$WORK_DIR/output.json" + +echo "==> Generating input JSON: $input_json" +cat > "$input_json" <<-EOF + { + "Version": "1.0.0", + "SignBatches": [ + { + "SourceLocationType": "UNC", + "DestinationLocationType": "UNC", + "SignRequestFiles": [$files_json + ], + "SigningInfo": { + "Operations": [ + { + "KeyCode": "$ESRP_KEYCODE", + "OperationCode": "SigntoolSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "OpusName": "Microsoft", + "OpusInfo": "https://www.microsoft.com", + "FileDigest": "/fd SHA256", + "PageHash": "/NPH", + "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + }, + { + "KeyCode": "$ESRP_KEYCODE", + "OperationCode": "SigntoolVerify", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": {} + } + ] + } + } + ] + } +EOF + +# Generate policy JSON +echo "==> Generating policy JSON..." +policy_json="$WORK_DIR/policy.json" +cat > "$policy_json" <<-EOF + { + "Version": "1.0.0", + "Intent": "ProductRelease", + "ContentType": "Binaries", + "ContentOrigin": "1stParty", + "ProductState": "Current", + "Audience": "ExternalBroad" + } +EOF + +# Use auth JSON from ESRP_AUTH +export ESRP_AUTH_CONFIG="$(to_windows_path "$ESRP_AUTH")" +export ESRP_POLICY_CONFIG="$WORK_DIR_WIN\\policy.json" + +# The ADO system access token is referenced in the auth JSON via the environment +# variable - export this so the ESRP client can pick it up when it runs. +export SYSTEM_ACCESSTOKEN + +# Print generated JSON files for debugging +echo "==> Auth JSON:" +cat "$ESRP_AUTH" +echo "" +echo "==> Policy JSON:" +cat "$policy_json" +echo "" +echo "==> Input JSON:" +cat "$input_json" +echo "" + +# Sign the files +esrp_tool_win="$(to_windows_path "$ESRP_TOOL")" +input_json_win="$WORK_DIR_WIN\\input.json" +output_json_win="$WORK_DIR_WIN\\output.json" + +echo "==> ESRP_AUTH_CONFIG=$ESRP_AUTH_CONFIG" +echo "==> ESRP_POLICY_CONFIG=$ESRP_POLICY_CONFIG" +echo "==> Running: $esrp_tool_win sign -i $input_json_win -o $output_json_win" +"$esrp_tool_win" sign \ + -i "$input_json_win" \ + -o "$output_json_win" + +echo "==> Signing complete." +echo "==> Output JSON:" +cat "$output_json" diff --git a/.azure-pipelines/esrp/windows/setup.yml b/.azure-pipelines/esrp/windows/setup.yml new file mode 100644 index 00000000000000..0653b868d0a728 --- /dev/null +++ b/.azure-pipelines/esrp/windows/setup.yml @@ -0,0 +1,69 @@ +parameters: + - name: serviceConnectionName + type: string + - name: esrpClientId + type: string + - name: keyVaultName + type: string + - name: signCertName + type: string + +steps: + - task: EsrpClientTool@4 + name: esrpinstall + displayName: 'Install ESRP client' + - task: AzureCLI@2 + displayName: 'Set up ESRP environment' + inputs: + azureSubscription: ${{ parameters.serviceConnectionName }} + addSpnToEnvironment: true + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + # Resolve ESRP client tool path (passed via env to avoid PS subexpression issues) + $esrpTool = "$env:ESRPCLIENT_TOOLPATH\$env:ESRPCLIENT_TOOLNAME" + if (-not (Test-Path $esrpTool)) { Write-Error "ESRPClient.exe not found at $esrpTool"; exit 1 } + Write-Host "Found ESRP client: $esrpTool" + Write-Host "##vso[task.setvariable variable=ESRP_TOOL]$esrpTool" + + # Derive the service connection GUID from the ENDPOINT_URL_* env vars + # that the agent emits for the bound connection. Filter out the + # built-in SystemVssConnection which is always present. + $scId = (Get-ChildItem env:ENDPOINT_URL_*).Name ` + -replace '^ENDPOINT_URL_','' | + Where-Object { $_ -ne 'SYSTEMVSSCONNECTION' } + if (-not $scId) { Write-Error "Could not derive service connection GUID"; exit 1 } + Write-Host "Resolved service connection GUID: $scId" + + # servicePrincipalId and tenantId are provided by addSpnToEnvironment + $authJson = @{ + Version = "1.0.0" + AuthenticationType = "AAD_MSI_WIF" + EsrpClientId = "${{ parameters.esrpClientId }}" + ClientId = $env:servicePrincipalId + TenantId = $env:tenantId + AADAuthorityBaseUri = "https://login.microsoftonline.com/" + FederatedTokenData = @{ + JobId = "$(System.JobId)" + PlanId = "$(System.PlanId)" + ProjectId = "$(System.TeamProjectId)" + Hub = "$(System.HostType)" + Uri = "$(System.CollectionUri)" + ServiceConnectionId = $scId + SystemAccessToken = "SYSTEM_ACCESSTOKEN" + } + RequestSigningCert = @{ + GetCertFromKeyVault = $true + KeyVaultName = "${{ parameters.keyVaultName }}" + KeyVaultCertName = "${{ parameters.signCertName }}" + } + } | ConvertTo-Json -Depth 4 + + $authPath = "$(Agent.TempDirectory)\esrp-auth.json" + $authJson | Set-Content -Path $authPath -Encoding UTF8 + Write-Host "Generated ESRP auth JSON: $authPath" + Write-Host "##vso[task.setvariable variable=ESRP_AUTH]$authPath" + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + ESRPCLIENT_TOOLPATH: $(esrpinstall.esrpclient.toolpath) + ESRPCLIENT_TOOLNAME: $(esrpinstall.esrpclient.toolname) diff --git a/.azure-pipelines/patches/.gitattributes b/.azure-pipelines/patches/.gitattributes new file mode 100644 index 00000000000000..ef9170ec0077a5 --- /dev/null +++ b/.azure-pipelines/patches/.gitattributes @@ -0,0 +1 @@ +*.patch whitespace=-trailing-space,-blank-at-eof diff --git a/.azure-pipelines/patches/windows/build-extra/0000-installer-publisher.patch b/.azure-pipelines/patches/windows/build-extra/0000-installer-publisher.patch new file mode 100644 index 00000000000000..aaef8b8a35b6d1 --- /dev/null +++ b/.azure-pipelines/patches/windows/build-extra/0000-installer-publisher.patch @@ -0,0 +1,13 @@ +diff --git a/installer/install.iss b/installer/install.iss +index 70787b7..137f660 100644 +--- a/installer/install.iss ++++ b/installer/install.iss +@@ -65,7 +65,7 @@ SignTool=signtool + ; Installer-related + AllowNoIcons=yes + AppName={#APP_NAME} +-AppPublisher=The Git Development Community ++AppPublisher=The Git Client Team at Microsoft + AppPublisherURL={#APP_URL} + AppSupportURL={#APP_CONTACT_URL} + AppVersion={#APP_VERSION} diff --git a/.azure-pipelines/patches/windows/build-extra/0001-installer-vsintegration.patch b/.azure-pipelines/patches/windows/build-extra/0001-installer-vsintegration.patch new file mode 100644 index 00000000000000..6797e6ab6c88ba --- /dev/null +++ b/.azure-pipelines/patches/windows/build-extra/0001-installer-vsintegration.patch @@ -0,0 +1,50 @@ +diff --git a/installer/helpers.inc.iss b/installer/helpers.inc.iss +index 3e3788d..fc81be7 100644 +--- a/installer/helpers.inc.iss ++++ b/installer/helpers.inc.iss +@@ -224,3 +224,25 @@ begin + DeleteFile(OutPath); + DeleteFile(ErrPath); + end; ++ ++procedure CustomPostInstall(); ++begin ++ if not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\15.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\16.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\17.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\18.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\19.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or ++ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\20.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) then ++ LogError('Could not register TeamFoundation\GitSourceControl'); ++end; ++ ++procedure CustomPostUninstall(); ++begin ++ if not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\15.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\16.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\17.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\18.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\19.0\TeamFoundation\GitSourceControl','GitPath') or ++ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\20.0\TeamFoundation\GitSourceControl','GitPath') then ++ LogError('Could not register TeamFoundation\GitSourceControl'); ++end; +diff --git a/installer/install.iss b/installer/install.iss +index 70787b7..74d8375 100644 +--- a/installer/install.iss ++++ b/installer/install.iss +@@ -3603,6 +3603,7 @@ begin + Install a scheduled task to try to auto-update Git for Windows + } + ++ CustomPostInstall(); + if IsComponentInstalled('autoupdate') then begin + WizardForm.StatusLabel.Caption:='Set up daily up to date check'; + InstallAutoUpdater(); +@@ -3943,6 +3944,7 @@ begin + Remove the scheduled task to try to auto-update Git for Windows + } + ++ CustomPostUninstall(); + if IsComponentInstalled('autoupdate') then + UninstallAutoUpdater(); + diff --git a/.azure-pipelines/patches/windows/build-extra/0002-installer-default-components.patch b/.azure-pipelines/patches/windows/build-extra/0002-installer-default-components.patch new file mode 100644 index 00000000000000..2135ecf925bc84 --- /dev/null +++ b/.azure-pipelines/patches/windows/build-extra/0002-installer-default-components.patch @@ -0,0 +1,17 @@ +diff --git a/installer/install.iss b/installer/install.iss +index 70787b7..37d79b0 100644 +--- a/installer/install.iss ++++ b/installer/install.iss +@@ -1925,6 +1925,12 @@ begin + GetDefaultsFromGitConfig('system'); + + ChosenOptions:=''; ++ if (ExpandConstant('{param:components|/}')='/') then begin ++ WizardSelectComponents('autoupdate'); ++#ifdef WITH_SCALAR ++ WizardSelectComponents('scalar'); ++#endif ++ end; + + PrevPageID:=wpSelectProgramGroup; + diff --git a/.azure-pipelines/patches/windows/build-extra/0003-installer-fork-from-microsoft-git.patch b/.azure-pipelines/patches/windows/build-extra/0003-installer-fork-from-microsoft-git.patch new file mode 100644 index 00000000000000..28af996716dc03 --- /dev/null +++ b/.azure-pipelines/patches/windows/build-extra/0003-installer-fork-from-microsoft-git.patch @@ -0,0 +1,29 @@ +diff --git a/git-update-git-for-windows.config b/git-update-git-for-windows.config +new file mode 100644 +index 0000000..bfd0744 +--- /dev/null ++++ b/git-update-git-for-windows.config +@@ -0,0 +1,2 @@ ++[update] ++ fromFork = microsoft/git +diff --git a/installer/install.iss b/installer/install.iss +index 70787b7..71d5e72 100644 +--- a/installer/install.iss ++++ b/installer/install.iss +@@ -126,6 +126,7 @@ Filename: {app}\ReleaseNotes.html; Description: View Release Notes; Flags: shell + [Files] + ; Install files that might be in use during setup under a different name. + #include "file-list.iss" ++Source: {#SourcePath}\..\git-update-git-for-windows.config; DestDir: {app}\{#MINGW_BITNESS}\bin; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore + Source: {#SourcePath}\ReleaseNotes.html; DestDir: {app}; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore + Source: {#SourcePath}\..\LICENSE.txt; DestDir: {app}; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore + Source: {#SourcePath}\NOTICE.txt; DestDir: {app}; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore; Check: ParamIsSet('VSNOTICE') +@@ -275,6 +276,8 @@ Type: files; Name: {app}\etc\rebase.db.i386 + Type: files; Name: {app}\etc\install-options.txt + Type: dirifempty; Name: {app}\{#MINGW_BITNESS}\libexec\git-core + Type: dirifempty; Name: {app}\{#MINGW_BITNESS}\libexec ++Type: files; Name: {app}\{#MINGW_BITNESS}\bin\git-update-git-for-windows.config ++Type: dirifempty; Name: {app}\{#MINGW_BITNESS}\bin + Type: dirifempty; Name: {app}\{#MINGW_BITNESS} + Type: dirifempty; Name: {app} + diff --git a/.azure-pipelines/patches/windows/git-sdk/0000-update-recently-seen.patch b/.azure-pipelines/patches/windows/git-sdk/0000-update-recently-seen.patch new file mode 100644 index 00000000000000..cf73dfd5f3f449 --- /dev/null +++ b/.azure-pipelines/patches/windows/git-sdk/0000-update-recently-seen.patch @@ -0,0 +1,12 @@ +diff --git a/bin/git-update-git-for-windows b/bin/git-update-git-for-windows +index 29444d9..6705da1 100644 +--- a/bin/git-update-git-for-windows ++++ b/bin/git-update-git-for-windows +@@ -4,6 +4,7 @@ + # release. If versions differ, the bit matched installer is downloaded and run + # when confirmation to do so is given. + ++use_recently_seen=no + + # Compare version strings + # Prints -1, 0 or 1 to stdout diff --git a/.azure-pipelines/release.yml b/.azure-pipelines/release.yml index 0e0ce01ac71978..b1da29c2494041 100644 --- a/.azure-pipelines/release.yml +++ b/.azure-pipelines/release.yml @@ -10,6 +10,10 @@ resources: ref: refs/tags/release parameters: + - name: 'esrp' + type: boolean + default: false # TODO: change default to true after testing + displayName: 'Enable ESRP code signing' - name: 'github' type: boolean default: false # TODO: change default to true after testing @@ -26,21 +30,39 @@ parameters: - id: windows_x64 jobName: 'Windows (x64)' pool: GitClientPME-1ESHostedPool-intel-pc + poolArch: amd64 image: win-x86_64-ado1es os: windows toolchain: x86_64 + cpu_arch: x86_64 mingwprefix: mingw64 + msystem: MINGW64 + sdk_repo: git-for-windows/git-sdk-64 - id: windows_arm64 jobName: 'Windows (ARM64)' pool: GitClientPME-1ESHostedPool-arm64-pc + poolArch: arm64 image: win-arm64-ado1es os: windows toolchain: clang-aarch64 + cpu_arch: aarch64 mingwprefix: clangarm64 + msystem: CLANGARM64 + sdk_repo: git-for-windows/git-sdk-arm64 - # No matrix for macOS as we build both x64 and ARM64 in the same job - # and produce a universal binary. + - name: macos_matrix + type: object + default: + - id: macos_universal + jobName: 'macOS (x64 + ARM64)' + pool: 'Azure Pipelines' + # macOS-latest is an Intel x86_64 Mac Pro, which can't host the + # arm64 Homebrew the universal-binary build needs. We need to + # explictly target the newer Apple Silicon machines that can run + # both x86_64 and arm64 builds. + image: macOS-15-arm64 + os: macos - name: linux_matrix type: object @@ -48,6 +70,7 @@ parameters: - id: linux_x64 jobName: 'Linux (x64)' pool: GitClientPME-1ESHostedPool-intel-pc + poolArch: amd64 image: ubuntu-x86_64-ado1es os: linux cc_arch: x86_64 @@ -56,14 +79,24 @@ parameters: - id: linux_arm64 jobName: 'Linux (ARM64)' pool: GitClientPME-1ESHostedPool-arm64-pc + poolArch: arm64 image: ubuntu-arm64-ado1es os: linux cc_arch: aarch64 deb_arch: arm64 variables: + - name: 'esrpAppConnectionName' + value: '1ESGitClient-ESRP-App' - name: 'githubConnectionName' value: 'GitHub-MicrosoftGit' + # ESRP signing variables set in the pipeline settings: + # - esrpEndpointUrl + # - esrpMI + # - esrpClientId + # - esrpTenantId + # - esrpKeyVaultName + # - esrpSignReqCertName extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelines @@ -85,20 +118,15 @@ extends: image: ubuntu-x86_64-ado1es os: linux steps: + - checkout: self + fetchDepth: 0 + fetchTags: true - task: Bash@3 displayName: 'Resolve version and tag information' name: info inputs: - targetType: inline - script: | - # TODO: determine git_version, tag_name, and tag_sha - # TODO: error if the current commit is not an annotated tag - git_version=TODO_GITVER - tag_name=TODO_TAGNAME - tag_sha=TODO_TAGSHA - echo "##vso[task.setvariable variable=git_version;isOutput=true;isReadOnly=true]$git_version" - echo "##vso[task.setvariable variable=tag_name;isOutput=true;isReadOnly=true]$tag_name" - echo "##vso[task.setvariable variable=tag_sha;isOutput=true;isReadOnly=true]$tag_sha" + targetType: filePath + filePath: .azure-pipelines/scripts/resolve-version.sh - stage: build displayName: 'Build' @@ -114,12 +142,15 @@ extends: name: ${{ dim.pool }} image: ${{ dim.image }} os: ${{ dim.os }} + hostArchitecture: ${{ dim.poolArch }} variables: tag_name: $[stageDependencies.prereqs.prebuild.outputs['info.tag_name']] tag_sha: $[stageDependencies.prereqs.prebuild.outputs['info.tag_sha']] git_version: $[stageDependencies.prereqs.prebuild.outputs['info.git_version']] toolchain: ${{ dim.toolchain }} mingwprefix: ${{ dim.mingwprefix }} + sdk_repo: ${{ dim.sdk_repo }} + cpu_arch: ${{ dim.cpu_arch }} templateContext: outputs: - output: pipelineArtifact @@ -127,44 +158,599 @@ extends: artifactName: '${{ dim.id }}' steps: - checkout: self - # TODO: add tasks to set up Git for Windows SDK - # TODO: add tasks to build Git and installers - - script: | - echo $(mingwprefix) - echo $(toolchain) - displayName: 'Dummy build' - # TODO: put final artifacts under $(Build.ArtifactStagingDirectory)/_final - - script: | - echo "TODO" > $(Build.ArtifactStagingDirectory)/_final/placeholder.txt + # Add Git Bash to the PATH so Bash tasks can find it + - task: BatchScript@1 + displayName: 'Add Git Bash to PATH' + inputs: + filename: ./.azure-pipelines/scripts/windows/setup-git-bash.cmd + # Install Azure CLI on arm64 (not pre-installed on these agents) + - ${{ if eq(dim.poolArch, 'arm64') }}: + - powershell: | + $ProgressPreference = 'SilentlyContinue' + $msi = "$env:TEMP\AzureCLI.msi" + Write-Host "Downloading Azure CLI (x64)..." + Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile $msi + Write-Host "Installing Azure CLI..." + Start-Process msiexec.exe -ArgumentList "/i", $msi, "/quiet", "/norestart" -Wait + $azPath = "C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\wbin" + Write-Host "##vso[task.prependpath]$azPath" + Write-Host "Azure CLI installed." + displayName: 'Install Azure CLI (x64 on ARM64)' + # Install VS 2022 Build Tools on x64 so cv2pdb-strip can locate + # mspdb140.dll. We do not need to do this on ARM64 since we use + # clang's llvm-strip there instead. + - ${{ if eq(dim.poolArch, 'amd64') }}: + - task: PowerShell@2 + displayName: 'Setup cv2pdb (x64)' + inputs: + filePath: ./.azure-pipelines/scripts/windows/setup-cv2pdb-x64.ps1 + - task: Bash@3 + displayName: 'Install Git for Windows SDK' + inputs: + filePath: ./.azure-pipelines/scripts/windows/setup-git-sdk.sh + arguments: '$(sdk_repo) $(mingwprefix) "$(Agent.TempDirectory)\gitsdk"' + env: + BOOTSTRAP_DIR: '$(Build.SourcesDirectory)' + - task: Bash@3 + displayName: 'Clone build-extra into SDK' + inputs: + targetType: inline + script: | + set -euo pipefail + # The please.sh + signtool.sh scripts the build + # relies on live in build-extra; the SDK ships + # without them. Partial clone to keep this fast. + git clone --filter=blob:none --single-branch -b main \ + https://github.com/git-for-windows/build-extra \ + /usr/src/build-extra + # Setup ESRP code signing for Windows (sets ESRP_TOOL, + # ESRP_AUTH) before the build steps so that the build + # itself (Inno Setup, makepkg-mingw) can invoke ESRP + # via the `git signtool` alias for in-line signing of + # individual binaries. + - ${{ if eq(parameters.esrp, true) }}: + - template: .azure-pipelines/esrp/windows/setup.yml@self + parameters: + serviceConnectionName: $(esrpAppConnectionName) + esrpClientId: $(esrpClientId) + keyVaultName: $(esrpKeyVaultName) + signCertName: $(esrpSignReqCertName) + - task: Bash@3 + displayName: 'Configure git signtool alias for ESRP' + inputs: + targetType: inline + script: | + set -euo pipefail + # please.sh, makepkg-mingw, and Inno Setup's + # release.sh all detect this alias and route + # their per-file code-signing through it; see + # build-extra's please.sh + installer/release.sh. + script="$(cygpath -au "$BUILD_SOURCESDIRECTORY/.azure-pipelines/esrp/windows/esrpsign.sh")" + git config --global alias.signtool "!sh \"$script\"" + git config --global --get alias.signtool + - task: Bash@3 + displayName: 'Apply Windows build patches' + inputs: + targetType: inline + script: | + set -euo pipefail + apply="$(cygpath -au "$BUILD_SOURCESDIRECTORY/.azure-pipelines/scripts/apply-patches.sh")" + patches="$(cygpath -au "$BUILD_SOURCESDIRECTORY/.azure-pipelines/patches/windows")" + bash "$apply" "$patches/build-extra" /usr/src/build-extra + bash "$apply" "$patches/git-sdk" "/$(mingwprefix)" + - task: Bash@3 + displayName: 'Build mingw-w64-git package' + ${{ if eq(parameters.esrp, true) }}: + env: + ESRP_TOOL: $(ESRP_TOOL) + ESRP_AUTH: $(ESRP_AUTH) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + targetType: inline + script: | + set -euo pipefail + set -x + + # Detach stdin so descendants like the git-extra + # post_install hook (which runs `for s in $(grep + # -l PAT $(find /mingw*/bin/ ...))` and falls back + # to reading stdin when /mingw*/bin/ is absent + # and find produces empty output) cannot block + # the build waiting for input. Bash@3 leaves the + # task's stdin pipe open with no writer; the + # GitHub Actions runner closes it for the same + # reason (see actions/runner ProcessInvoker.cs). + exec /usr/bin/git + chmod +x /usr/bin/git + + USER_NAME='microsoft-git-build' + USER_EMAIL='microsoft-git-build@users.noreply.github.com' + git config --global user.name "$USER_NAME" + git config --global user.email "$USER_EMAIL" + export PACKAGER="$USER_NAME <$USER_EMAIL>" + + sh -x /usr/src/build-extra/please.sh build-mingw-w64-git \ + --only-"$(cpu_arch)" \ + --build-src-pkg \ + -o artifacts \ + HEAD + + # NOTE: the GitHub workflow additionally GPG-signs + # each tarball and creates a MINGW-packages.bundle + # for downstream Pacman consumers; both are + # intentionally out of scope for the initial port + # and tracked as follow-ups. + - task: Bash@3 + displayName: 'Build installer and portable Git' + env: + # `please.sh make_installers_from_mingw_w64_git` invokes + # build-extra's installer/release.sh, which requires + # MSYSTEM to select the architecture branch. Bash@3 does + # not source /etc/profile, so we export it explicitly. + MSYSTEM: ${{ dim.msystem }} + ${{ if eq(parameters.esrp, true) }}: + ESRP_TOOL: $(ESRP_TOOL) + ESRP_AUTH: $(ESRP_AUTH) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + targetType: inline + script: | + set -euo pipefail + set -x + + # Detach stdin so descendants like the git-extra + # post_install hook (which runs `for s in $(grep + # -l PAT $(find /mingw*/bin/ ...))` and falls back + # to reading stdin when /mingw*/bin/ is absent + # and find produces empty output) cannot block + # the build waiting for input. Bash@3 leaves the + # task's stdin pipe open with no writer; the + # GitHub Actions runner closes it for the same + # reason (see actions/runner ProcessInvoker.cs). + exec /tmp/setx.sh + export BASH_ENV=/tmp/setx.sh + + # please.sh make_installers_from_mingw_w64_git + # --include-pdbs reads PDB archives from + # cached-source-packages/. + mkdir -p "$b/cached-source-packages" + cp artifacts/*-pdb* "$b/cached-source-packages/" + + # The --pkg=... list excludes the optional pieces + # the workflow drops (signatures, archimport, cvs, + # p4, gitweb, doc-man); keep the same filter so + # the resulting .exe size is comparable. + pkg_args=$( + ls artifacts/mingw-w64-$(toolchain)-*.tar.* \ + | sed '/\.sig$/d;/archimport/d;/cvs/d;/p4/d;/gitweb/d;/doc-man/d;s/^/--pkg=/' \ + | tr '\n' ' ' + ) + + for type in installer portable; do + eval sh -x "$b"/please.sh make_installers_from_mingw_w64_git --include-pdbs \ + --version="$(git_version)" \ + -o artifacts --"$type" \ + $pkg_args + + # The installer .exe is signed inline by Inno + # Setup via the `git signtool` alias; the + # portable .exe is a 7z self-extractor that + # bypasses that path, so sign it explicitly. + if test "$type" = portable && \ + test -n "$(git config alias.signtool)" + then + git signtool artifacts/PortableGit-*.exe + fi + done + - task: Bash@3 + displayName: 'Stage installer artifacts for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + + # Compute SHA-256 over the (possibly signed) + # binaries; if ESRP signing ran, this picks up + # the post-sign bytes, which is what we want to + # publish in the release notes. + openssl dgst -sha256 \ + artifacts/Git-*.exe \ + artifacts/PortableGit-*.exe \ + | sed 's/.* //' >artifacts/sha-256.txt + + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + cp artifacts/Git-*.exe \ + artifacts/PortableGit-*.exe \ + artifacts/sha-256.txt \ + "$(Build.ArtifactStagingDirectory)/_final/" # - # macOS build job (universal) + # macOS build jobs # - - job: macos_universal - displayName: 'macOS (x64 + ARM64)' - pool: - name: 'Azure Pipelines' - image: macOS-latest - os: macos - variables: - tag_name: $[stageDependencies.prereqs.prebuild.outputs['info.tag_name']] - tag_sha: $[stageDependencies.prereqs.prebuild.outputs['info.tag_sha']] - git_version: $[stageDependencies.prereqs.prebuild.outputs['info.git_version']] - templateContext: - outputs: - - output: pipelineArtifact - targetPath: '$(Build.ArtifactStagingDirectory)/_final' - artifactName: 'macos_universal' - steps: - - checkout: self - # TODO: add tasks to set up build environment - # TODO: add tasks to build Git and installers - - script: | - echo "Hello, Mac!" - displayName: 'Dummy build' - # TODO: put final artifacts under $(Build.ArtifactStagingDirectory)/_final - - script: | - echo "TODO" > $(Build.ArtifactStagingDirectory)/_final/placeholder.txt + - ${{ each dim in parameters.macos_matrix }}: + - job: ${{ dim.id }} + displayName: ${{ dim.jobName }} + pool: + name: ${{ dim.pool }} + image: ${{ dim.image }} + os: ${{ dim.os }} + variables: + tag_name: $[stageDependencies.prereqs.prebuild.outputs['info.tag_name']] + tag_sha: $[stageDependencies.prereqs.prebuild.outputs['info.tag_sha']] + git_version: $[stageDependencies.prereqs.prebuild.outputs['info.git_version']] + templateContext: + outputs: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/_final' + artifactName: '${{ dim.id }}' + # macOS build flow: + # + # 1. Configure for a universal build and produce + # Git's own dist tarballs (`make dist dist-doc`). + # 2. Extract the source tarball into payload/, copy + # config.mak in, run `make payload` to compile and + # install into stage/git-universal-/. + # 3. Mirror stage/ into build-artifacts/ (which is + # what the macos-installer Makefile's `pkg` target + # consumes - see note in the build step). + # 4. ESRP-sign Mach-O files in build-artifacts/. + # 5. `make pkg` -> unsigned .pkg in disk-image/. + # 6. ESRP-sign and ESRP-notarize the .pkg in place. + # 7. `make image` wraps disk-image/ contents in a DMG. + # 8. Stage the .pkg and .dmg under _final/ for upload. + steps: + - checkout: self + - task: Bash@3 + displayName: 'Disable Spotlight indexing' + inputs: + targetType: inline + script: | + # Disable Spotlight indexing to prevent file + # locking issues. + set -euo pipefail + sudo mdutil -i off / || true + - task: Bash@3 + displayName: 'Install build dependencies' + inputs: + targetType: inline + script: | + set -euo pipefail + + # The agent's native arm64 Homebrew lives at + # /opt/homebrew. Install a separate x86_64 Homebrew + # under /usr/local so we can fetch the x86_64 build + # of gettext/libintl alongside the arm64 one. + arch -x86_64 /bin/bash -c \ + "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + arch -x86_64 /usr/local/bin/brew install gettext + + # Native (arm64) build dependencies. + brew install automake asciidoc xmlto docbook + brew link --force gettext + + # Make a universal libintl.a out of the two + # arch-specific copies, dropped at the workspace + # root so the build's LDFLAGS = -L"$(pwd)" can find + # it. libintl depends on iconv, but we deliberately + # use the system's universal /usr/lib/libiconv.dylib + # rather than Homebrew's libiconv (which exports + # _libiconv* symbols, while Homebrew's gettext was + # built against system iconv with _iconv* symbols). + lipo -create \ + -output libintl.a \ + /usr/local/opt/gettext/lib/libintl.a \ + /opt/homebrew/opt/gettext/lib/libintl.a + - task: Bash@3 + displayName: 'Configure universal build' + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + # Git's GIT-VERSION-GEN expects .rc rather than -rc + BUILD_VERSION="$(echo "$VERSION" | sed 's/-rc/.rc/g')" + echo "$BUILD_VERSION" >version + + # HOST_CPU is a bit of a lie and is only used in + # 'git version --build-options'; we'll fix that in + # code. The two -arch flags are what actually drive + # the universal build. + cat >config.mak <>config.mak <>config.mak <>config.mak + - task: Bash@3 + displayName: 'Build payload via macos-installer' + env: + # The macos-installer Makefile derives BUILD_DIR from + # $(GITHUB_WORKSPACE), which is unset in ADO. Point it + # at the worktree root. + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + BUILD_VERSION="$(echo "$VERSION" | sed 's/-rc/.rc/g')" + + # The asciidoc/xmlto build steps need the catalogs + # from Homebrew docbook. + export XML_CATALOG_FILES="$(brew --prefix)/etc/xml/catalog" + + # `git commit` (in dist-doc) forks a detached + # `git maintenance run --auto` that keeps writing + # into .git/ after the commit returns, which then + # races with dist-doc's `rm -fr .doc-tmp-dir` and + # produces "Directory not empty". Disable + # auto-maintenance for every git invocation in + # this build. + export GIT_CONFIG_PARAMETERS="'maintenance.auto=false'" + + make -j"$(sysctl -n hw.physicalcpu)" GIT-VERSION-FILE dist dist-doc + + # Recover the source-tree commit OID from the dist + # tarball; the macos-installer Makefile bakes it + # into 'git version --build-options'. + # `git get-tar-commit-id` reads only the leading + # pax header and then closes its stdin, which + # makes `gunzip -c` exit 141 (SIGPIPE) under the + # outer `set -o pipefail`. Disable pipefail for + # the duration of this one pipeline. + GIT_BUILT_FROM_COMMIT="$( + set +o pipefail + gunzip -c "git-$BUILD_VERSION.tar.gz" | + git get-tar-commit-id + )" + export GIT_BUILT_FROM_COMMIT + export VERSION + + mkdir payload manpages + tar -xf "git-$BUILD_VERSION.tar.gz" -C payload + tar -xf "git-manpages-$BUILD_VERSION.tar.gz" -C manpages + + # The actual compile happens inside the extracted + # tree, against a copy of the config.mak we wrote + # at the worktree root in the previous step. + cp config.mak "payload/git-$BUILD_VERSION/config.mak" + + make -C .github/macos-installer V=1 payload + + # NOTE: the macos-installer Makefile produces + # the install tree at stage/git-universal-/ + # but its `pkg` target packages from + # build-artifacts/. Mirror the GitHub workflow + # by copying the tree across so the rest of the + # pipeline (signing, pkg) finds it where the + # Makefile expects. + # + # FUTURE: this duplication exists only because + # .github/macos-installer/Makefile hardcodes + # both DESTDIR (=stage/...) and ARTIFACTDIR + # (=build-artifacts). Overriding ARTIFACTDIR on + # the `make pkg` line below to point at stage/ + # would let us drop the cp entirely. Worth + # cleaning up alongside moving the macOS + # installer Makefiles out of .github/ (they are + # build infrastructure, not GitHub-specific). + mkdir -p .github/macos-installer/build-artifacts + cp -R "stage/git-universal-$BUILD_VERSION/." \ + .github/macos-installer/build-artifacts/ + # ESRP-sign the universal Mach-O binaries inside the + # payload tree. The existing esrp/sign.yml template's + # CopyFiles@2 step uses minimatch globs and the only + # reliable way to detect Mach-O is by file content + # (file --mime), so we pre-filter into a staging dir, + # let the template zip/sign/extract that staging dir, + # then copy the signed binaries back over the payload. + - ${{ if eq(parameters.esrp, true) }}: + # ESRP ADO tasks require .NET, which the macOS pool + # image does not provide by default. + - task: UseDotNet@2 + displayName: 'Install .NET for ESRP' + inputs: + packageType: sdk + version: '8.x' + - task: Bash@3 + displayName: 'Stage Mach-O binaries for signing' + inputs: + targetType: inline + script: | + set -euo pipefail + + # Sign the install tree (build-artifacts/) - + # this is what `make pkg` packages. Signing + # the source tree under payload/ would have + # no effect on the resulting .pkg. + install_tree=".github/macos-installer/build-artifacts/usr/local/git" + stage_dir="$(Build.ArtifactStagingDirectory)/macos-tosign/binaries" + + rm -rf "$stage_dir" + mkdir -p "$stage_dir" + + pushd "$install_tree" + find . -type f -exec file --mime {} + \ + | sed -n '/mach/s/: .*//p' \ + | while IFS= read -r f; do + rel="${f#./}" + tgt="$stage_dir/$rel" + mkdir -p "$(dirname "$tgt")" + cp -- "$f" "$tgt" + done + popd + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'ESRP-sign Mach-O binaries' + folderPath: '$(Build.ArtifactStagingDirectory)/macos-tosign/binaries' + pattern: '**/*' + useArchive: true # Required for macOS signing + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppDeveloperSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "Hardening": "Enable" + } + } + ] + - task: Bash@3 + displayName: 'Copy signed binaries back to install tree' + inputs: + targetType: inline + script: | + set -euo pipefail + + cp -R "$(Build.ArtifactStagingDirectory)/macos-tosign/binaries"/* \ + .github/macos-installer/build-artifacts/usr/local/git/ + - task: Bash@3 + displayName: 'Build unsigned installer pkg' + env: + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + export VERSION + + # Leave APPLE_INSTALLER_IDENTITY undefined so the + # Makefile's `pkg` target produces an unsigned .pkg + # (the `ifdef APPLE_INSTALLER_IDENTITY` branch in + # pkg_cmd is skipped). ESRP signs it in the next + # step. + make -C .github/macos-installer V=1 pkg + - ${{ if eq(parameters.esrp, true) }}: + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'ESRP-sign installer pkg' + folderPath: '.github/macos-installer/disk-image' + pattern: '*.pkg' + useArchive: true # Required for macOS signing + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppDeveloperSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "Hardening": "Enable" + } + } + ] + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'ESRP-notarize installer pkg' + folderPath: '.github/macos-installer/disk-image' + pattern: '*.pkg' + useArchive: true # Required for macOS notarization + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppNotarize", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "BundleId": "com.git.pkg" + } + } + ] + - task: Bash@3 + displayName: 'Build DMG' + env: + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + export VERSION + + # Builds .github/macos-installer/git--universal.dmg + # from the contents of disk-image/, which now contains + # the signed and notarized .pkg. + make -C .github/macos-installer V=1 image + - task: Bash@3 + displayName: 'Stage installer artifacts for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + ls -la .github/macos-installer/ \ + .github/macos-installer/disk-image/ || true + # The .pkg lands either directly under disk-image/ + # or, after ESRP MacAppNotarize re-packs it, inside + # disk-image/.zip.unzipped/. Find it. + pkg=$(find .github/macos-installer/disk-image \ + -name 'git-*-universal.pkg' -type f \ + | head -1) + mv .github/macos-installer/git-*-universal.dmg \ + "$pkg" \ + "$(Build.ArtifactStagingDirectory)/_final/" # # Linux build jobs @@ -176,6 +762,7 @@ extends: name: ${{ dim.pool }} image: ${{ dim.image }} os: ${{ dim.os }} + hostArchitecture: ${{ dim.poolArch }} variables: tag_name: $[stageDependencies.prereqs.prebuild.outputs['info.tag_name']] tag_sha: $[stageDependencies.prereqs.prebuild.outputs['info.tag_sha']] @@ -189,21 +776,185 @@ extends: artifactName: '${{ dim.id }}' steps: - checkout: self - # TODO: add tasks to set up build environment - # TODO: add tasks to build Git and installers - - script: | - echo $(cc_arch) - echo $(deb_arch) - displayName: 'Dummy build' - # TODO: put final artifacts under $(Build.ArtifactStagingDirectory)/_final - - script: | - echo "TODO" > $(Build.ArtifactStagingDirectory)/_final/placeholder.txt + - task: Bash@3 + displayName: 'Log build environment' + inputs: + targetType: inline + script: | + lsb_release -a || true + uname -a + id + - task: Bash@3 + displayName: 'Install build dependencies' + inputs: + targetType: inline + script: | + set -euo pipefail + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends \ + build-essential \ + tcl tk gettext asciidoc xmlto \ + libcurl4-gnutls-dev libpcre2-dev zlib1g-dev libexpat-dev \ + curl ca-certificates + - task: Bash@3 + displayName: 'Build microsoft-git Debian package' + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + # Git's GIT-VERSION-GEN expects .rc rather than -rc + BUILD_VERSION="$(echo "$VERSION" | sed 's/-rc/.rc/g')" + echo "$BUILD_VERSION" >version + make GIT-VERSION-FILE + + PKGNAME="microsoft-git_${VERSION}_$(deb_arch)" + PKGDIR="$(Build.ArtifactStagingDirectory)/pkgroot" + rm -rf "$PKGDIR" + mkdir -p "$PKGDIR/DEBIAN" + + DESTDIR="$PKGDIR" make -j"$(nproc)" V=1 DEVELOPER=1 \ + USE_LIBPCRE=1 \ + USE_CURL_FOR_IMAP_SEND=1 NO_OPENSSL=1 \ + NO_CROSS_DIRECTORY_HARDLINKS=1 \ + ASCIIDOC8=1 ASCIIDOC_NO_ROFF=1 \ + ASCIIDOC='TZ=UTC asciidoc' \ + prefix=/usr/local \ + gitexecdir=/usr/local/lib/git-core \ + libexecdir=/usr/local/lib/git-core \ + htmldir=/usr/local/share/doc/git/html \ + install install-doc install-html + + # Based on https://packages.ubuntu.com/xenial/vcs/git + cat >"$PKGDIR/DEBIAN/control" < + Description: Git client built from the https://github.com/microsoft/git repository, + specialized in supporting monorepo scenarios. Includes the Scalar CLI. + CTRL + + mkdir -p "$(Build.ArtifactStagingDirectory)/app" + dpkg-deb -Zxz --build "$PKGDIR" \ + "$(Build.ArtifactStagingDirectory)/app/$PKGNAME.deb" + - ${{ if eq(parameters.esrp, true) }}: + # ESRP ADO tasks require .NET, so we install it here since the + # Linux images do not have it by default. + - task: UseDotNet@2 + displayName: 'Install .NET for ESRP' + inputs: + packageType: sdk + version: '8.x' + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'Example sign Debian package' + folderPath: '$(Build.ArtifactStagingDirectory)/app' + pattern: '**/*.deb' + inlineOperation: | + [ + { + "KeyCode": "CP-453387-Pgp", + "OperationCode": "LinuxSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": {} + } + ] + - task: Bash@3 + displayName: 'Stage Debian package for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + mv "$(Build.ArtifactStagingDirectory)/app/microsoft-git_$(git_version)_$(deb_arch).deb" \ + "$(Build.ArtifactStagingDirectory)/_final/" - stage: release displayName: 'Release' dependsOn: [prereqs, build] jobs: + # + # Windows validation jobs + # + - ${{ each dim in parameters.windows_matrix }}: + - job: validate_${{ dim.id }} + displayName: 'Validate ${{ dim.jobName }}' + pool: + name: ${{ dim.pool }} + image: ${{ dim.image }} + os: ${{ dim.os }} + hostArchitecture: ${{ dim.poolArch }} + templateContext: + inputs: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} + steps: + # TODO: add artifact validation steps + - script: | + dir $(Pipeline.Workspace)\assets\${{ dim.id }} + displayName: 'Validate artifacts' + + # + # macOS validation jobs + # + - ${{ each dim in parameters.macos_matrix }}: + - job: validate_${{ dim.id }} + displayName: 'Validate ${{ dim.jobName }}' + pool: + name: ${{ dim.pool }} + image: ${{ dim.image }} + os: ${{ dim.os }} + templateContext: + inputs: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} + steps: + # TODO: add artifact validation steps + - script: | + ls $(Pipeline.Workspace)/assets/${{ dim.id }} + displayName: 'Validate artifacts' + + # + # Linux validation jobs + # + - ${{ each dim in parameters.linux_matrix }}: + - job: validate_${{ dim.id }} + displayName: 'Validate ${{ dim.jobName }}' + pool: + name: ${{ dim.pool }} + image: ${{ dim.image }} + os: ${{ dim.os }} + hostArchitecture: ${{ dim.poolArch }} + templateContext: + inputs: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} + steps: + # TODO: add artifact validation steps + - script: | + ls $(Pipeline.Workspace)/assets/${{ dim.id }} + displayName: 'Validate artifacts' + + # + # GitHub release publishing + # - job: github + dependsOn: + - ${{ each dim in parameters.windows_matrix }}: + - validate_${{ dim.id }} + - ${{ each dim in parameters.macos_matrix }}: + - validate_${{ dim.id }} + - ${{ each dim in parameters.linux_matrix }}: + - validate_${{ dim.id }} displayName: 'Publish GitHub release' condition: and(succeeded(), eq('${{ parameters.github }}', true)) pool: @@ -218,21 +969,18 @@ extends: type: releaseJob isProduction: true inputs: - - input: pipelineArtifact - artifactName: 'windows_x64' - targetPath: $(Pipeline.Workspace)/assets/windows_x64 - - input: pipelineArtifact - artifactName: 'windows_arm64' - targetPath: $(Pipeline.Workspace)/assets/windows_arm64 - - input: pipelineArtifact - artifactName: 'macos_universal' - targetPath: $(Pipeline.Workspace)/assets/macos_universal - - input: pipelineArtifact - artifactName: 'linux_x64' - targetPath: $(Pipeline.Workspace)/assets/linux_x64 - - input: pipelineArtifact - artifactName: 'linux_arm64' - targetPath: $(Pipeline.Workspace)/assets/linux_arm64 + - ${{ each dim in parameters.windows_matrix }}: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} + - ${{ each dim in parameters.macos_matrix }}: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} + - ${{ each dim in parameters.linux_matrix }}: + - input: pipelineArtifact + artifactName: '${{ dim.id }}' + targetPath: $(Pipeline.Workspace)/assets/${{ dim.id }} steps: - task: GitHubRelease@1 displayName: 'Create Draft GitHub Release' diff --git a/.azure-pipelines/scripts/apply-patches.sh b/.azure-pipelines/scripts/apply-patches.sh new file mode 100755 index 00000000000000..f1325a7caa0064 --- /dev/null +++ b/.azure-pipelines/scripts/apply-patches.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Apply all numbered patches from a directory to a target tree. +# +# Patches are applied in lexicographic order, so name them with +# zero-padded numeric prefixes (e.g. 0000-foo.patch, 0001-bar.patch). +# +# Arguments: +# $1 patches_dir Directory containing *.patch files +# $2 target_dir Directory to apply patches in (need not be a +# git repository; git apply works on any tree) + +set -euo pipefail + +if test $# -ne 2 +then + echo "Usage: $0 " >&2 + exit 1 +fi + +patches_dir="$1" +target_dir="$2" + +if test ! -d "$patches_dir" +then + echo "Patches directory not found: $patches_dir" >&2 + exit 1 +fi + +if test ! -d "$target_dir" +then + echo "Target directory not found: $target_dir" >&2 + exit 1 +fi + +shopt -s nullglob +patches=("$patches_dir"/*.patch) +if test ${#patches[@]} -eq 0 +then + echo "No patches found in $patches_dir" + exit 0 +fi + +cd "$target_dir" +for patch in "${patches[@]}" +do + echo "Applying $(basename "$patch")..." + # Use patch(1) rather than `git apply` because the latter is + # strict about context whitespace; CRLF/LF mismatches between + # patch context (as authored) and the working tree (which may + # be CRLF on Windows checkouts) trip it up. patch is more + # forgiving by default. + # + # This matches the convention used by msys2/MINGW-packages + # PKGBUILDs and git-for-windows/build-extra's get-sources.sh. + command patch -p1 -i "$patch" +done diff --git a/.azure-pipelines/scripts/resolve-version.sh b/.azure-pipelines/scripts/resolve-version.sh new file mode 100755 index 00000000000000..6169ae767bae1f --- /dev/null +++ b/.azure-pipelines/scripts/resolve-version.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Resolve version and tag information from the current HEAD commit. +# Validates that HEAD is an annotated version tag matching GIT-VERSION-GEN. +# +# Sets the following ADO output variables (via ##vso): +# git_version - Version string without "v" prefix (e.g., 2.53.0.vfs.0.0) +# tag_name - Full tag name (e.g., v2.53.0.vfs.0.0) +# tag_sha - Commit SHA of HEAD +# +# Also updates the build number to include the tag name. +# +set -euo pipefail + +echo "HEAD: $(git rev-parse HEAD)" + +# Determine the tag pointing at HEAD +tag_name=$(git describe --exact-match --match "v[0-9]*vfs*" HEAD 2>/dev/null) || { + echo "##vso[task.logissue type=error]HEAD is not tagged with a version tag" + exit 1 +} + +# Verify the tag is annotated (not lightweight) +tag_type=$(git cat-file -t "refs/tags/$tag_name") +if [ "$tag_type" != "tag" ]; then + echo "##vso[task.logissue type=error]Tag $tag_name is not annotated (type: $tag_type)" + exit 1 +fi + +tag_sha=$(git rev-parse HEAD) +git_version="${tag_name#v}" + +# Verify the version matches GIT-VERSION-GEN +make GIT-VERSION-FILE +expected_version="${git_version//-rc/.rc}" +actual_version=$(sed -n 's/^GIT_VERSION *= *//p' < GIT-VERSION-FILE) +if [ "$expected_version" != "$actual_version" ]; then + echo "##vso[task.logissue type=error]GIT-VERSION-FILE ($actual_version) does not match tag $tag_name ($expected_version)" + exit 1 +fi + +echo "Git version: $git_version" +echo "Tag name: $tag_name" +echo "Tag SHA: $tag_sha" +echo "##vso[task.setvariable variable=git_version;isOutput=true;isReadOnly=true]$git_version" +echo "##vso[task.setvariable variable=tag_name;isOutput=true;isReadOnly=true]$tag_name" +echo "##vso[task.setvariable variable=tag_sha;isOutput=true;isReadOnly=true]$tag_sha" +echo "##vso[build.updatebuildnumber]${tag_name} (${BUILD_BUILDNUMBER:-unknown})" diff --git a/.azure-pipelines/scripts/windows/setup-cv2pdb-x64.ps1 b/.azure-pipelines/scripts/windows/setup-cv2pdb-x64.ps1 new file mode 100644 index 00000000000000..a035f43897e4b8 --- /dev/null +++ b/.azure-pipelines/scripts/windows/setup-cv2pdb-x64.ps1 @@ -0,0 +1,99 @@ +# Set up cv2pdb-strip support on Windows x64 agents. +# +# build-extra's please.sh runs cv2pdb-strip during the strip phase of +# build-mingw-w64-git. cv2pdb-strip loads mspdb140.dll via PATH +# lookup, and the DLL is part of the MSVC C++ toolchain +# (Microsoft.VisualStudio.Component.VC.Tools.x86.x64) which is not +# present on the 1ES image by default. +# +# Install VS 2022 Build Tools with that single component (the +# smallest selection that ships the DLL), locate mspdb140.dll via +# vswhere with a filesystem fallback, and prepend its directory to +# PATH for subsequent tasks via the `##vso[task.prependpath]` logging +# command. +# +# This script is intended to be invoked by a PowerShell@2 task with +# `filePath:`. It takes no arguments and writes diagnostics to stdout +# so install failures can be diagnosed from the task log. + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +$bootstrapper = "$env:TEMP\vs_BuildTools.exe" +Write-Host "Downloading VS 2022 Build Tools bootstrapper..." +Invoke-WebRequest -Uri 'https://aka.ms/vs/17/release/vs_BuildTools.exe' ` + -OutFile $bootstrapper + +$vsArgs = @( + '--quiet', '--wait', '--norestart', '--nocache', + '--add', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64' +) +Write-Host "Installing VS Build Tools (args: $($vsArgs -join ' '))..." +$start = Get-Date +$p = Start-Process -FilePath $bootstrapper -ArgumentList $vsArgs -Wait -PassThru +$elapsed = (Get-Date) - $start +Write-Host ("Installer exited with code {0} after {1:N0}s" -f ` + $p.ExitCode, $elapsed.TotalSeconds) + +Write-Host "" +Write-Host "===== Installer logs in `$env:TEMP =====" +$logs = Get-ChildItem $env:TEMP -Filter 'dd_*.log' -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending +if ($logs) { + foreach ($log in $logs | Select-Object -First 5) { + Write-Host "----- $($log.FullName) (last 50 lines) -----" + Get-Content $log.FullName -Tail 50 -ErrorAction SilentlyContinue + } +} else { + Write-Host "(no dd_*.log files found in `$env:TEMP)" +} + +Write-Host "" +Write-Host "===== vswhere -all -prerelease (every install) =====" +$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" +if (-not (Test-Path $vswhere)) { + Write-Host "vswhere not found at $vswhere" +} else { + & $vswhere -all -prerelease -format json | + Out-String | Write-Host +} + +Write-Host "" +Write-Host "===== Filesystem search for mspdb*.dll =====" +$roots = @( + "${env:ProgramFiles(x86)}\Microsoft Visual Studio", + "${env:ProgramFiles}\Microsoft Visual Studio" +) | Where-Object { Test-Path $_ } +$hits = foreach ($r in $roots) { + Get-ChildItem -Path $r -Filter 'mspdb*.dll' -Recurse -File ` + -ErrorAction SilentlyContinue +} +if ($hits) { + $hits | ForEach-Object { Write-Host $_.FullName } +} else { + Write-Host "(no mspdb*.dll under any VS install root)" +} + +# 3010 = reboot required, treated as success. +if ($p.ExitCode -notin 0,3010) { + throw "VS Build Tools installer exited with code $($p.ExitCode)" +} + +Write-Host "" +Write-Host "===== Locate mspdb140.dll via vswhere -find =====" +$mspdb = & $vswhere -latest ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -find 'VC\Tools\MSVC\**\bin\Hostx64\x64\mspdb140.dll' | + Select-Object -First 1 +if (-not $mspdb) { + # Fall back to filesystem hits we already have. + $mspdb = $hits | + Where-Object { $_.Name -ieq 'mspdb140.dll' } | + Select-Object -First 1 -ExpandProperty FullName +} +if (-not $mspdb) { + throw "mspdb140.dll not found after install (see logs above)" +} +$dir = Split-Path -Parent $mspdb +Write-Host "Found mspdb140.dll at $mspdb" +Write-Host "##vso[task.prependpath]$dir" diff --git a/.azure-pipelines/scripts/windows/setup-git-bash.cmd b/.azure-pipelines/scripts/windows/setup-git-bash.cmd new file mode 100644 index 00000000000000..b3ef5518cfc85d --- /dev/null +++ b/.azure-pipelines/scripts/windows/setup-git-bash.cmd @@ -0,0 +1,13 @@ +@echo off +setlocal enabledelayedexpansion +set "agentgit=%AGENT_HOMEDIRECTORY%\externals\git" +set "gitcopy=%AGENT_TEMPDIRECTORY%\git" +echo Copying !agentgit! to !gitcopy!... +xcopy /E /I /Q "!agentgit!" "!gitcopy!" +if not exist "!gitcopy!\usr\bin\sh.exe" ( + echo ##vso[task.logissue type=error]Could not find sh.exe at !gitcopy!\usr\bin\sh.exe + exit /b 1 +) +echo Copying !gitcopy!\usr\bin\sh.exe to !gitcopy!\usr\bin\bash.exe... +copy /Y "!gitcopy!\usr\bin\sh.exe" "!gitcopy!\usr\bin\bash.exe" +echo ##vso[task.prependpath]!gitcopy!\usr\bin diff --git a/.azure-pipelines/scripts/windows/setup-git-sdk.sh b/.azure-pipelines/scripts/windows/setup-git-sdk.sh new file mode 100755 index 00000000000000..c123ec3682a9b5 --- /dev/null +++ b/.azure-pipelines/scripts/windows/setup-git-sdk.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Materialise the build-installers flavour of the Git for Windows SDK. +# +# Performs a partial + bare clone of the given Git SDK repository, +# then runs build-extra's please.sh to sparse-checkout just the +# build-installers subset into the requested SDK output directory. +# +# Environment: +# BOOTSTRAP_DIR (optional) - directory for transient bootstrap clones +# (the bare git-sdk fetch and build-extra +# checkout used to drive please.sh). +# Falls back to TEMP, then TMP, then errors +# if none are set. +# +# Arguments: +# $1 sdk_repo e.g. git-for-windows/git-sdk-64 +# $2 mingwprefix e.g. mingw64 or clangarm64 +# $3 sdk_output_dir Windows or MSYS path where the SDK will be installed +# +# See: +# https://github.com/git-for-windows/git-sdk-64/blob/main/.github/workflows/ci-artifacts.yml +# https://github.com/git-for-windows/build-extra/blob/main/please.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$SCRIPT_DIR/utils.sh" + +if test $# -ne 3 +then + echo "Usage: $0 " >&2 + exit 1 +fi + +sdk_repo="$1" +mingwprefix="$2" +sdk_output="$3" + +bootstrap_dir="${BOOTSTRAP_DIR:-${TEMP:-${TMP:-}}}" +if test -z "$bootstrap_dir" +then + echo "BOOTSTRAP_DIR (or TEMP/TMP) must be set" >&2 + exit 1 +fi + +bootstrap="$(to_unix_path "$bootstrap_dir")" +sdk="$(to_unix_path "$sdk_output")" + +sdk_bare="$bootstrap/sdk-bare.git" +bootstrap_be="$bootstrap/build-extra-bootstrap" + +git init --bare "$sdk_bare" +git --git-dir="$sdk_bare" remote add origin "https://github.com/$sdk_repo" +git --git-dir="$sdk_bare" config remote.origin.promisor true +git --git-dir="$sdk_bare" config remote.origin.partialCloneFilter blob:none +git --git-dir="$sdk_bare" fetch --depth=1 origin HEAD +git --git-dir="$sdk_bare" update-ref --no-deref HEAD FETCH_HEAD + +# please.sh is the bootstrap; build-extra gets cloned again into the SDK +# in a separate task so `please.sh build-mingw-w64-git` can find it at +# /usr/src/build-extra under the SDK's bash. +git clone --depth=1 --single-branch -b main \ + https://github.com/git-for-windows/build-extra \ + "$bootstrap_be" + +# Architecture is auto-detected from the bare clone's HEAD tree +# (clangarm64/ vs usr/x86_64-pc-msys/). +bash "$bootstrap_be/please.sh" create-sdk-artifact \ + --sdk="$sdk_bare" --out="$sdk" build-installers + +# Expose the SDK's bash and the matching MinGW toolchain to subsequent +# tasks. +echo "##vso[task.prependpath]$(to_windows_path "$sdk/usr/bin")" +echo "##vso[task.prependpath]$(to_windows_path "$sdk/$mingwprefix/bin")" diff --git a/.azure-pipelines/scripts/windows/utils.sh b/.azure-pipelines/scripts/windows/utils.sh new file mode 100755 index 00000000000000..f94c380a2b49af --- /dev/null +++ b/.azure-pipelines/scripts/windows/utils.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Utilities for bash scripts running on Windows. +# +# Source this file from another bash script: +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# . "$SCRIPT_DIR/utils.sh" +# +# Functions: +# to_windows_path - output a Windows-form (D:\foo) path. +# to_unix_path - output an MSYS-form (/d/foo) path. + +# Convert a path to Windows form for tools that demand backslashes +# (e.g. ESRPClient.exe, ##vso[task.prependpath]). +# Useful when a script may run before the full Git for Windows SDK +# (which provides cygpath) is available. Falls back to pure-shell +# parsing when cygpath is not on PATH. +to_windows_path () { + local drive rest root + if command -v cygpath >/dev/null 2>&1; then + cygpath -w "$1" + return + fi + case "$1" in + /[A-Za-z]/*) + # /d/path -> D:\path + drive=$(echo "$1" | cut -c2 | tr 'a-z' 'A-Z') + rest=$(echo "$1" | cut -c3-) + echo "${drive}:${rest}" | sed 's|/|\\|g' + ;; + /*) + # Absolute path under MSYS root + root=$(cd / && pwd -W) + echo "${root}${1}" | sed 's|/|\\|g' + ;; + *) + # Relative or already-Windows: just flip slashes + echo "$1" | sed 's|/|\\|g' + ;; + esac +} + +# Convert a path to MSYS form for bash-friendly handling. Inverse of +# to_windows_path. +# Useful when a script may run before the full Git for Windows SDK +# (which provides cygpath) is available. Falls back to pure-shell +# parsing when cygpath is not on PATH. +to_unix_path () { + local p drive rest + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$1" + return + fi + # Normalize separators to forward slashes first. + p="${1//\\//}" + case "$p" in + [A-Za-z]:/*) + # D:/path -> /d/path + drive=$(echo "$p" | cut -c1 | tr 'A-Z' 'a-z') + rest=$(echo "$p" | cut -c3-) + echo "/${drive}${rest}" + ;; + *) + echo "$p" + ;; + esac +}