From fd6db8b978ac6ce74459dba57695fc981c3a49b4 Mon Sep 17 00:00:00 2001 From: Trent Mohay Date: Mon, 13 Apr 2026 10:50:12 +1000 Subject: [PATCH 1/2] Back porting "Enforce Basic Auth for git operations" (#1875) --- source/Calamari/ArgoCD/ArgoCDModule.cs | 17 +- .../ArgoCD/Git/GitHttpSmartSubTransport.cs | 305 ++++++++++++++++++ 2 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 source/Calamari/ArgoCD/Git/GitHttpSmartSubTransport.cs diff --git a/source/Calamari/ArgoCD/ArgoCDModule.cs b/source/Calamari/ArgoCD/ArgoCDModule.cs index 6b5e4b63b2..1c6395680a 100644 --- a/source/Calamari/ArgoCD/ArgoCDModule.cs +++ b/source/Calamari/ArgoCD/ArgoCDModule.cs @@ -3,11 +3,26 @@ using Calamari.ArgoCD.Git; using Calamari.ArgoCD.Git.GitVendorApiAdapters; using Calamari.ArgoCD.GitHub; +using LibGit2Sharp; namespace Calamari.ArgoCD { public class ArgoCDModule : Module { + static ArgoCDModule() + { + // Note this cannot be set in the RepositoryFactory as it causes tests to fail, due to the following issue. + + // LibGit2Sharp custom sub-transports are registered by calling a static registration + // method on GlobalSettings. Additionally, if you try and register a multiple transports + // with the same scheme, it throws an exception. It's not ideal, but it's what we've got + // to work with. + // + // Using the type constructor to make sure that these methods are only called once. + GlobalSettings.RegisterSmartSubtransport("http"); + GlobalSettings.RegisterSmartSubtransport("https"); + } + protected override void Load(ContainerBuilder builder) { builder.RegisterType().AsSelf().InstancePerLifetimeScope(); @@ -22,5 +37,5 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); } + } } -} \ No newline at end of file diff --git a/source/Calamari/ArgoCD/Git/GitHttpSmartSubTransport.cs b/source/Calamari/ArgoCD/Git/GitHttpSmartSubTransport.cs new file mode 100644 index 0000000000..e412a2c502 --- /dev/null +++ b/source/Calamari/ArgoCD/Git/GitHttpSmartSubTransport.cs @@ -0,0 +1,305 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using LibGit2Sharp; + +namespace Calamari.ArgoCD.Git +{ + /// + /// We found some credential caching issues in the default sub-transport implemented + /// in LibGit2Sharp, and there was no easy way to fix it in the upstream library + /// without breaking other existing authentication or exhausting sockets. Read + /// more here: https://github.com/libgit2/libgit2sharp/issues/1894 + /// + /// For now, we're have implemented this sub-transport to get around these issues. + /// This code is pretty much taken wholesale from the LibGit2Sharp ManagedHttpSmartSubtransport + /// https://github.com/OctopusDeploy/libgit2sharp/blob/master/LibGit2Sharp/Core/ManagedHttpSmartSubtransport.cs + /// with a few changes (and simplifications) around authentication. We currently only + /// support basic auth, but it seems to work nicely for what we need. + /// + /// + /// This was taken (almost) verbatim from the OctopusDeploy server code base Octopus.Core.Git.GitHttpSmartSubTransport. + /// https://github.com/OctopusDeploy/OctopusDeploy/blob/2026.2.3071/source/Octopus.Core/Git/GitHttpSmartSubTransport.cs + /// + public class GitHttpSmartSubTransport : RpcSmartSubtransport + { + protected override SmartSubtransportStream Action(string url, GitSmartSubtransportAction action) + { + string endpointUrl; + string? contentType = null; + var isPost = false; + + switch (action) + { + case GitSmartSubtransportAction.UploadPackList: + endpointUrl = string.Concat(url, "/info/refs?service=git-upload-pack"); + break; + + case GitSmartSubtransportAction.UploadPack: + endpointUrl = string.Concat(url, "/git-upload-pack"); + contentType = "application/x-git-upload-pack-request"; + isPost = true; + break; + + case GitSmartSubtransportAction.ReceivePackList: + endpointUrl = string.Concat(url, "/info/refs?service=git-receive-pack"); + break; + + case GitSmartSubtransportAction.ReceivePack: + endpointUrl = string.Concat(url, "/git-receive-pack"); + contentType = "application/x-git-receive-pack-request"; + isPost = true; + break; + + default: + throw new InvalidOperationException(); + } + + return new GitHttpSmartSubTransportStream(this, endpointUrl, isPost, contentType); + } + + class GitHttpSmartSubTransportStream : SmartSubtransportStream + { + static readonly int MAX_REDIRECTS = 7; + static readonly HttpClient HttpClient; + + readonly MemoryStream postBuffer = new(); + HttpResponseMessage? response; + Stream? responseStream; + + static GitHttpSmartSubTransportStream() + { + var handler = new SocketsHttpHandler + { + PooledConnectionLifetime = TimeSpan.FromMinutes(5), + AllowAutoRedirect = false + }; + +#pragma warning disable HttpClientInstantiation + HttpClient = new HttpClient(handler, false) + { + DefaultRequestHeaders = + { + // This worked fine when it was on, but git.exe doesn't specify this header, so we don't either. + ExpectContinue = false + } + }; +#pragma warning restore HttpClientInstantiation + } + + public GitHttpSmartSubTransportStream(GitHttpSmartSubTransport parent, + string endpointUrl, + bool isPost, + string? contentType) + : base(parent) + { + EndpointUrl = new Uri(endpointUrl); + IsPost = isPost; + ContentType = contentType; + } + + Uri EndpointUrl { get; set; } + bool IsPost { get; set; } + string? ContentType { get; set; } + + string GetUserAgent() + { + var userAgent = GlobalSettings.GetUserAgent(); + + if (string.IsNullOrEmpty(userAgent)) + { + userAgent = "LibGit2Sharp " + GlobalSettings.Version.InformationalVersion; + } + + return userAgent; + } + + HttpRequestMessage CreateRequest(Uri endpointUrl, bool isPost) + { + var verb = isPost ? new HttpMethod("POST") : new HttpMethod("GET"); + var request = new HttpRequestMessage(verb, endpointUrl); + request.Headers.Add("User-Agent", $"git/2.0 ({GetUserAgent()})"); + request.Headers.Remove("Expect"); + + return request; + } + + HttpResponseMessage GetResponseWithRedirects() + { + var url = EndpointUrl; + var credentials = GetCredentials(); + + for (var retries = 0; retries < MAX_REDIRECTS; retries++) + { + var request = CreateRequest(url, IsPost); + + // This is the main difference between out implementation and the LibGit2Sharp + // library. Rather than first making an unauthorized request and then adding + // credentials to a CredentialCache if the request is rejected, we always + // add credentials to the first request. + if (credentials is UsernamePasswordCredentials usernamePasswordCredentials) + { + request.Headers.Authorization = GetBasicAuthenticationHeader( + usernamePasswordCredentials.Username, + usernamePasswordCredentials.Password + ); + } + + if (IsPost && postBuffer.Length > 0) + { + var bufferDup = new MemoryStream(postBuffer.GetBuffer(), 0, (int)postBuffer.Length); + + request.Content = new StreamContent(bufferDup); + request.Content.Headers.Add("Content-Type", ContentType); + } + + // This was originally configured with HttpCompletionOption.ResponseContentRead which would + // cause HttpClient timeout and buffer errors. Using the Stack Overflow post linked below + // as a guide, this has been changed to ResponseHeadersRead. + // + // The caller of this method was reads the content from the HttpResponseMessage as a stream + // anyway, so loads the response as it goes + // https://stackoverflow.com/questions/18720435/httpclient-buffer-size-limit-exceeded?rq=1 + var response = HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead) + .GetAwaiter() + .GetResult(); + + switch (response.StatusCode) + { + case HttpStatusCode.OK: + return response; + case HttpStatusCode.Unauthorized: + case HttpStatusCode.Forbidden: + case HttpStatusCode.NotFound: + var responseContent = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + throw new Exception(responseContent); + case HttpStatusCode.Moved or HttpStatusCode.Redirect: + url = new Uri(response.Headers.GetValues("Location").First()); + continue; + default: + throw new Exception($"Unexpected HTTP response: {response.StatusCode}"); + } + } + + throw new Exception("Too many redirects"); + } + + Credentials GetCredentials() + { + var ret = SmartTransport.AcquireCredentials( + out var cred, + null, + typeof(UsernamePasswordCredentials)); + + // GitErrorCode.PassThrough is returned when the credentialsProvider returns null + // (https://github.com/libgit2/libgit2sharp/blob/5085a0c6173cdb2a3fde205330b327a8eb0a26c4/LibGit2Sharp/RemoteCallbacks.cs#L294) + if (ret != 0 && ret != (int)GitErrorCode.PassThrough) + { + throw new InvalidOperationException("Authentication cancelled"); + } + + return cred; + } + + AuthenticationHeaderValue GetBasicAuthenticationHeader(string username, string password) + { + var authorizationHeaderValue = EncodeAuthorizationHeaderValue($"{username}:{password}"); + return new AuthenticationHeaderValue("Basic", authorizationHeaderValue); + } + + string EncodeAuthorizationHeaderValue(string input) + { + var bytes = Encoding.UTF8.GetBytes(input); + return Convert.ToBase64String(bytes); + } + + public override int Write(Stream dataStream, long length) + { + var buffer = new byte[4096]; + long writeTotal = 0; + + while (length > 0) + { + var readLen = dataStream.Read(buffer, 0, (int)Math.Min(buffer.Length, length)); + + if (readLen == 0) + { + break; + } + + postBuffer.Write(buffer, 0, readLen); + length -= readLen; + writeTotal += readLen; + } + + if (writeTotal < length) + { + throw new EndOfStreamException("Could not write buffer (short read)"); + } + + return 0; + } + + public override int Read(Stream dataStream, long length, out long readTotal) + { + var buffer = new byte[4096]; + readTotal = 0; + + if (responseStream == null) + { + response = GetResponseWithRedirects(); + responseStream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult(); + } + + while (length > 0) + { + var readLen = responseStream.Read(buffer, 0, (int)Math.Min(buffer.Length, length)); + + if (readLen == 0) + { + break; + } + + dataStream.Write(buffer, 0, readLen); + readTotal += readLen; + length -= readLen; + } + + return 0; + } + + protected override void Free() + { + if (responseStream != null) + { + responseStream.Dispose(); + responseStream = null; + } + + if (response != null) + { + response.Dispose(); + response = null; + } + + base.Free(); + } + } + + /// + /// Copy of internal enum representing error codes presented by LibGit2Sharp + /// (https://github.com/libgit2/libgit2sharp/blob/5085a0c6173cdb2a3fde205330b327a8eb0a26c4/LibGit2Sharp/Core/GitErrorCode.cs#L122) + /// + enum GitErrorCode + { + /// + /// Skip and passthrough the given ODB backend. + /// + PassThrough = -30, + } + } +} \ No newline at end of file From 821fc12a4fc9ec384bab2f9ee692579c31cdd629 Mon Sep 17 00:00:00 2001 From: Trent Mohay Date: Fri, 29 May 2026 15:05:29 +1000 Subject: [PATCH 2/2] Backport fixes for handling "unstructured" text in 2026.1 (#1960) --- build/_build.csproj | 2 +- .../Calamari.AzureAppService.csproj | 4 +- ...i.ConsolidateCalamariPackages.Tests.csproj | 4 +- ...alamari.ConsolidateCalamariPackages.csproj | 4 +- ...elmValuesImageReplaceStepVariablesTests.cs | 178 ++++++++++++++++++ source/Calamari.Tests/Calamari.Tests.csproj | 2 +- .../UpdateArgoCDAppDeploymentConfig.cs | 3 +- .../HelmValuesImageReplaceStepVariables.cs | 10 +- source/Calamari/Calamari.csproj | 4 +- 9 files changed, 201 insertions(+), 10 deletions(-) create mode 100644 source/Calamari.Tests/ArgoCD/Helm/HelmValuesImageReplaceStepVariablesTests.cs diff --git a/build/_build.csproj b/build/_build.csproj index b178179720..d4a839b8e6 100644 --- a/build/_build.csproj +++ b/build/_build.csproj @@ -24,7 +24,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/source/Calamari.AzureAppService/Calamari.AzureAppService.csproj b/source/Calamari.AzureAppService/Calamari.AzureAppService.csproj index aa124dd30b..1f444dd39b 100644 --- a/source/Calamari.AzureAppService/Calamari.AzureAppService.csproj +++ b/source/Calamari.AzureAppService/Calamari.AzureAppService.csproj @@ -17,7 +17,9 @@ - + + NU1902 + diff --git a/source/Calamari.ConsolidateCalamariPackages.Tests/Calamari.ConsolidateCalamariPackages.Tests.csproj b/source/Calamari.ConsolidateCalamariPackages.Tests/Calamari.ConsolidateCalamariPackages.Tests.csproj index 078fa3263e..a2ff998be7 100644 --- a/source/Calamari.ConsolidateCalamariPackages.Tests/Calamari.ConsolidateCalamariPackages.Tests.csproj +++ b/source/Calamari.ConsolidateCalamariPackages.Tests/Calamari.ConsolidateCalamariPackages.Tests.csproj @@ -15,7 +15,9 @@ - + + NU1902 + diff --git a/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj b/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj index 7a93745308..11096e249b 100644 --- a/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj +++ b/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj @@ -19,7 +19,9 @@ - + + NU1902 + diff --git a/source/Calamari.Tests/ArgoCD/Helm/HelmValuesImageReplaceStepVariablesTests.cs b/source/Calamari.Tests/ArgoCD/Helm/HelmValuesImageReplaceStepVariablesTests.cs new file mode 100644 index 0000000000..028ab96656 --- /dev/null +++ b/source/Calamari.Tests/ArgoCD/Helm/HelmValuesImageReplaceStepVariablesTests.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using Calamari.ArgoCD; +using Calamari.ArgoCD.Conventions; +using Calamari.ArgoCD.Models; +using Calamari.Testing.Helpers; +using FluentAssertions; +using FluentAssertions.Execution; +using NUnit.Framework; + +namespace Calamari.Tests.ArgoCD.Helm; + +public class HelmValuesImageReplaceStepVariablesTests +{ + const string DefaultRegistry = "docker.io"; + readonly InMemoryLog log = new(); + + [Test] + public void UnstructuredValue_UpdatesTag_TracksWithFriendlyName() + { + const string yaml = @" +image: + tag: 1.0 +"; + var replacer = new HelmValuesImageReplaceStepVariables(yaml, DefaultRegistry, log); + var images = new List + { + new(ContainerImageReference.FromReferenceString("nginx:1.27.1", DefaultRegistry), "image.tag") + }; + + var result = replacer.UpdateImages(images); + + using var scope = new AssertionScope(); + result.UpdatedImageReferences.Should().BeEquivalentTo(["nginx:1.27.1"]); + result.UpdatedContents.Should().Contain("tag: 1.27.1"); + } + + [Test] + public void UnstructuredValue_AlreadyAtTarget_TracksWithFriendlyName() + { + const string yaml = @" +image: + tag: 1.27.1 +"; + var replacer = new HelmValuesImageReplaceStepVariables(yaml, DefaultRegistry, log); + var images = new List + { + new(ContainerImageReference.FromReferenceString("nginx:1.27.1", DefaultRegistry), "image.tag") + }; + + var result = replacer.UpdateImages(images); + + using var scope = new AssertionScope(); + result.UpdatedImageReferences.Should().BeEmpty(); + } + + [Test] + public void StructuredValue_UpdatesFullRef() + { + const string yaml = @" +image: + name: nginx:1.0 +"; + var replacer = new HelmValuesImageReplaceStepVariables(yaml, DefaultRegistry, log); + var images = new List + { + new(ContainerImageReference.FromReferenceString("nginx:1.27.1", DefaultRegistry), "image.name") + }; + + var result = replacer.UpdateImages(images); + + using var scope = new AssertionScope(); + result.UpdatedImageReferences.Should().HaveCount(1); + result.UpdatedContents.Should().Contain("name: nginx:1.27.1"); + } + + [Test] + public void StructuredValue_AlreadyAtTarget_TracksWithFriendlyName() + { + const string yaml = @" +image: + name: nginx:1.27.1 +"; + var replacer = new HelmValuesImageReplaceStepVariables(yaml, DefaultRegistry, log); + var images = new List + { + new(ContainerImageReference.FromReferenceString("nginx:1.27.1", DefaultRegistry), "image.name") + }; + + var result = replacer.UpdateImages(images); + + using var scope = new AssertionScope(); + result.UpdatedImageReferences.Should().BeEmpty(); + } + + [Test] + public void TwoImagesWithSameTag_OnlyUpdatesConfiguredPath() + { + const string yaml = @" +nginx: + tag: 1.0 +redis: + tag: 1.0 +"; + + var replacer = new HelmValuesImageReplaceStepVariables(yaml, DefaultRegistry, log); + var images = new List + { + new(ContainerImageReference.FromReferenceString("nginx:1.27.1", DefaultRegistry), "nginx.tag") + }; + + var result = replacer.UpdateImages(images); + + using var scope = new AssertionScope(); + result.UpdatedImageReferences.Should().BeEquivalentTo(["nginx:1.27.1"]); + result.UpdatedContents.Should().Contain($"nginx:{Environment.NewLine} tag: 1.27.1"); + result.UpdatedContents.Should().Contain($"redis:{Environment.NewLine} tag: 1.0"); + } + + [Test] + public void NoHelmReference_SkipsImage() + { + const string yaml = @" +image: + tag: 1.0 +"; + var replacer = new HelmValuesImageReplaceStepVariables(yaml, DefaultRegistry, log); + var images = new List + { + new(ContainerImageReference.FromReferenceString("nginx:1.27.1", DefaultRegistry)) + }; + + var result = replacer.UpdateImages(images); + + using var scope = new AssertionScope(); + result.UpdatedImageReferences.Should().BeEmpty(); + } + + [Test] + public void PathNotFoundInYaml_SkipsImage() + { + const string yaml = @" +image: + tag: 1.0 +"; + var replacer = new HelmValuesImageReplaceStepVariables(yaml, DefaultRegistry, log); + var images = new List + { + new(ContainerImageReference.FromReferenceString("nginx:1.27.1", DefaultRegistry), "nonexistent.path") + }; + + var result = replacer.UpdateImages(images); + + using var scope = new AssertionScope(); + result.UpdatedImageReferences.Should().BeEmpty(); + result.UpdatedContents.Should().Be(yaml); + } + + [Test] + public void StructuredValue_MismatchedImageName_DoesNotUpdate() + { + const string yaml = @" +image: + name: alpine:3.18 +"; + var replacer = new HelmValuesImageReplaceStepVariables(yaml, DefaultRegistry, log); + var images = new List + { + new(ContainerImageReference.FromReferenceString("nginx:1.27.1", DefaultRegistry), "image.name") + }; + + var result = replacer.UpdateImages(images); + + using var scope = new AssertionScope(); + result.UpdatedImageReferences.Should().BeEmpty(); + result.UpdatedContents.Should().Contain("name: alpine:3.18"); + } +} diff --git a/source/Calamari.Tests/Calamari.Tests.csproj b/source/Calamari.Tests/Calamari.Tests.csproj index b121284417..94b43b7483 100644 --- a/source/Calamari.Tests/Calamari.Tests.csproj +++ b/source/Calamari.Tests/Calamari.Tests.csproj @@ -15,7 +15,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppDeploymentConfig.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppDeploymentConfig.cs index 82503f14dc..fccf3c4cc1 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppDeploymentConfig.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppDeploymentConfig.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Linq; +using Octopus.CoreUtilities.Extensions; namespace Calamari.ArgoCD.Conventions { @@ -18,7 +19,7 @@ public UpdateArgoCDAppDeploymentConfig(GitCommitParameters commitParameters, Lis public bool HasStepBasedHelmValueReferences() { - return ImageReferences.Any(ir => ir.HelmReference is not null) && UseHelmReferenceFromContainer; + return ImageReferences.Any(ir => !ir.HelmReference.IsNullOrEmpty()) && UseHelmReferenceFromContainer; } } } diff --git a/source/Calamari/ArgoCD/HelmValuesImageReplaceStepVariables.cs b/source/Calamari/ArgoCD/HelmValuesImageReplaceStepVariables.cs index 24bd71c1ba..fbb4b0b999 100644 --- a/source/Calamari/ArgoCD/HelmValuesImageReplaceStepVariables.cs +++ b/source/Calamari/ArgoCD/HelmValuesImageReplaceStepVariables.cs @@ -33,13 +33,17 @@ public ImageReplacementResult UpdateImages(IReadOnlyCollection - + + NU1902 +