From f9b5d626c5486d1f02b3bd45879f61834b7aeecc Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 11 Mar 2020 10:36:56 -0400 Subject: [PATCH 01/86] load and print machine setup info from .setup_info (#364) --- src/Runner.Common/Constants.cs | 1 + src/Runner.Common/HostContext.cs | 7 +++++ src/Runner.Worker/JobExtension.cs | 49 +++++++++++++++++++++++++++++++ src/Test/L0/TestHostContext.cs | 7 +++++ 4 files changed, 64 insertions(+) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 0d333464bbf..0a8261c5525 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -25,6 +25,7 @@ public enum WellKnownConfigFile CredentialStore, Certificates, Options, + SetupInfo, } public static class Constants diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 1a44e8588ed..99e152b1d6f 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -322,6 +322,13 @@ public string GetConfigFile(WellKnownConfigFile configFile) GetDirectory(WellKnownDirectory.Root), ".options"); break; + + case WellKnownConfigFile.SetupInfo: + path = Path.Combine( + GetDirectory(WellKnownDirectory.Root), + ".setup_info"); + break; + default: throw new NotSupportedException($"Unexpected well known config file: '{configFile}'"); } diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index d7b6a99b113..3db3e61d059 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Runtime.Serialization; using System.Threading.Tasks; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Pipelines.ObjectTemplating; @@ -14,6 +15,16 @@ namespace GitHub.Runner.Worker { + [DataContract] + public class SetupInfo + { + [DataMember] + public string Group { get; set; } + + [DataMember] + public string Detail { get; set; } + } + [ServiceLocator(Default = typeof(JobExtension))] public interface IJobExtension : IRunnerService @@ -49,6 +60,44 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.Start(); context.Debug($"Starting: Set up job"); context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'"); + + var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo); + if (File.Exists(setupInfoFile)) + { + Trace.Info($"Load machine setup info from {setupInfoFile}"); + try + { + var setupInfo = IOUtil.LoadObject>(setupInfoFile); + if (setupInfo?.Count > 0) + { + foreach (var info in setupInfo) + { + if (!string.IsNullOrEmpty(info?.Detail)) + { + var groupName = info.Group; + if (string.IsNullOrEmpty(groupName)) + { + groupName = "Machine Setup Info"; + } + + context.Output($"##[group]{groupName}"); + var multiLines = info.Detail.Replace("\r\n", "\n").TrimEnd('\n').Split('\n'); + foreach (var line in multiLines) + { + context.Output(line); + } + context.Output("##[endgroup]"); + } + } + } + } + catch (Exception ex) + { + context.Output($"Fail to load and print machine setup info: {ex.Message}"); + Trace.Error(ex); + } + } + var repoFullName = context.GetGitHubContext("repository"); ArgUtil.NotNull(repoFullName, nameof(repoFullName)); context.Debug($"Primary repository: {repoFullName}"); diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index 88c38b7c76a..3d3c99c736b 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -279,6 +279,13 @@ public string GetConfigFile(WellKnownConfigFile configFile) GetDirectory(WellKnownDirectory.Root), ".options"); break; + + case WellKnownConfigFile.SetupInfo: + path = Path.Combine( + GetDirectory(WellKnownDirectory.Root), + ".setup_info"); + break; + default: throw new NotSupportedException($"Unexpected well known config file: '{configFile}'"); } From 53fb6297cb0c2ed6ff8530ccbc01594ada71f596 Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Thu, 12 Mar 2020 02:52:46 +0100 Subject: [PATCH 02/86] Change problem matchers output to debug (#363) --- src/Runner.Worker/ExecutionContext.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index f68b98f515e..22e1226abb8 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -736,7 +736,7 @@ public void AddMatchers(IssueMatchersConfig config) var owners = config.Matchers.Select(x => $"'{x.Owner}'"); var joinedOwners = string.Join(", ", owners); // todo: loc - this.Output($"Added matchers: {joinedOwners}. Problem matchers scan action output for known warning or error strings and report these inline."); + this.Debug($"Added matchers: {joinedOwners}. Problem matchers scan action output for known warning or error strings and report these inline."); } } @@ -778,7 +778,7 @@ public void RemoveMatchers(IEnumerable owners) owners = removedMatchers.Select(x => $"'{x.Owner}'"); var joinedOwners = string.Join(", ", owners); // todo: loc - this.Output($"Removed matchers: {joinedOwners}"); + this.Debug($"Removed matchers: {joinedOwners}"); } } From c8890d0f3f64b5ed79e619623a66e2c2f3351adf Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 12 Mar 2020 20:47:25 -0400 Subject: [PATCH 03/86] Expose job name as $GITHUB_JOB (#366) --- src/Runner.Worker/ExecutionContext.cs | 5 +++++ src/Runner.Worker/GitHubContext.cs | 1 + 2 files changed, 6 insertions(+) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 22e1226abb8..d390a9e8680 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -599,8 +599,13 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation var githubAccessToken = new StringContextData(Variables.Get("system.github.token")); var base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{githubAccessToken}")); HostContext.SecretMasker.AddValue(base64EncodedToken); + var githubJob = Variables.Get("system.github.job"); var githubContext = new GitHubContext(); githubContext["token"] = githubAccessToken; + if (!string.IsNullOrEmpty(githubJob)) + { + githubContext["job"] = new StringContextData(githubJob); + } var githubDictionary = ExpressionValues["github"].AssertDictionary("github"); foreach (var pair in githubDictionary) { diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index 0316fad854b..454f5e21111 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -14,6 +14,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "event_name", "event_path", "head_ref", + "job", "ref", "repository", "run_id", From 2d6042421f9f07e72a9113c0d7261b85f75b9f5c Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Sat, 14 Mar 2020 17:54:58 -0400 Subject: [PATCH 04/86] add support for job outputs. (#365) * add support for job outputs. --- src/Runner.Worker/ExecutionContext.cs | 8 +- src/Runner.Worker/JobExtension.cs | 54 +++++ src/Runner.Worker/JobRunner.cs | 2 +- src/Runner.Worker/action_yaml.json | 4 +- .../Expressions2/ExpressionConstants.cs | 1 + .../Expressions2/Sdk/Functions/FromJson.cs | 24 ++ .../Schema/MappingDefinition.cs | 14 +- .../ObjectTemplating/Schema/PropertyValue.cs | 34 ++- .../ObjectTemplating/Schema/TemplateSchema.cs | 77 ++++--- .../ObjectTemplating/TemplateConstants.cs | 5 + .../ObjectTemplating/TemplateEvaluator.cs | 17 +- .../ObjectTemplating/TemplateReader.cs | 17 +- .../Pipelines/AgentJobRequestMessage.cs | 11 +- .../PipelineTemplateConstants.cs | 2 + .../PipelineTemplateEvaluator.cs | 37 ++++ src/Sdk/DTPipelines/workflow-v1.0.json | 209 ++++++++++++++---- src/Sdk/DTWebApi/WebApi/JobEvent.cs | 24 +- src/Test/L0/Listener/JobDispatcherL0.cs | 2 +- src/Test/L0/Listener/RunnerL0.cs | 2 +- src/Test/L0/Worker/ActionCommandManagerL0.cs | 2 +- src/Test/L0/Worker/ExecutionContextL0.cs | 6 +- src/Test/L0/Worker/JobExtensionL0.cs | 2 +- src/Test/L0/Worker/JobRunnerL0.cs | 2 +- src/Test/L0/Worker/StepsRunnerL0.cs | 3 + src/Test/L0/Worker/WorkerL0.cs | 2 +- 25 files changed, 445 insertions(+), 116 deletions(-) create mode 100644 src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/FromJson.cs diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index d390a9e8680..b48936b08fe 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -47,7 +47,7 @@ public interface IExecutionContext : IRunnerService PlanFeatures Features { get; } Variables Variables { get; } Dictionary IntraActionState { get; } - HashSet OutputVariables { get; } + Dictionary JobOutputs { get; } IDictionary EnvironmentVariables { get; } IDictionary Scopes { get; } IList FileTable { get; } @@ -110,7 +110,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private readonly TimelineRecord _record = new TimelineRecord(); private readonly Dictionary _detailRecords = new Dictionary(); private readonly object _loggerLock = new object(); - private readonly HashSet _outputvariables = new HashSet(StringComparer.OrdinalIgnoreCase); private readonly object _matchersLock = new object(); private event OnMatcherChanged _onMatcherChanged; @@ -140,7 +139,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public List Endpoints { get; private set; } public Variables Variables { get; private set; } public Dictionary IntraActionState { get; private set; } - public HashSet OutputVariables => _outputvariables; + public Dictionary JobOutputs { get; private set; } public IDictionary EnvironmentVariables { get; private set; } public IDictionary Scopes { get; private set; } public IList FileTable { get; private set; } @@ -557,6 +556,9 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // Environment variables shared across all actions EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer); + // Job Outputs + JobOutputs = new Dictionary(StringComparer.OrdinalIgnoreCase); + // Service container info ServiceContainers = new List(); diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 3db3e61d059..c0de945a0d8 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -6,6 +6,8 @@ using System.Runtime.Serialization; using System.Threading.Tasks; using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.Pipelines.ObjectTemplating; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; @@ -291,6 +293,58 @@ public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestM context.Start(); context.Debug("Starting: Complete job"); + // Evaluate job outputs + if (message.JobOutputs != null && message.JobOutputs.Type != TokenType.Null) + { + try + { + context.Output($"Evaluate and set job outputs"); + + // Populate env context for each step + Trace.Info("Initialize Env context for evaluating job outputs"); +#if OS_WINDOWS + var envContext = new DictionaryContextData(); +#else + var envContext = new CaseSensitiveDictionaryContextData(); +#endif + context.ExpressionValues["env"] = envContext; + foreach (var pair in context.EnvironmentVariables) + { + envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty); + } + + Trace.Info("Initialize steps context for evaluating job outputs"); + context.ExpressionValues["steps"] = context.StepsContext.GetScope(context.ScopeName); + + var templateEvaluator = context.ToPipelineTemplateEvaluator(); + var outputs = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues); + foreach (var output in outputs) + { + if (string.IsNullOrEmpty(output.Value)) + { + context.Debug($"Skip output '{output.Key}' since it's empty"); + continue; + } + + if (!string.Equals(output.Value, HostContext.SecretMasker.MaskSecrets(output.Value))) + { + context.Warning($"Skip output '{output.Key}' since it may contain secret."); + continue; + } + + context.Output($"Set output '{output.Key}'"); + jobContext.JobOutputs[output.Key] = output.Value; + } + } + catch (Exception ex) + { + context.Result = TaskResult.Failed; + context.Error($"Fail to evaluate job outputs"); + context.Error(ex); + jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, TaskResult.Failed); + } + } + if (context.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false) { Trace.Info("Support log upload starting."); diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 585885ffe39..a94d7dd3d49 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -231,7 +231,7 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution } Trace.Info("Raising job completed event."); - var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result); + var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs); var completeJobRetryLimit = 5; var exceptions = new List(); diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index a30de160674..c9eb2d38d2b 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -90,10 +90,8 @@ "github", "strategy", "matrix", - "steps", "job", - "runner", - "env" + "runner" ], "string": {} }, diff --git a/src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs b/src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs index a0291cee0b8..7974c85bc03 100644 --- a/src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs +++ b/src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs @@ -15,6 +15,7 @@ static ExpressionConstants() AddFunction("join", 1, 2); AddFunction("startsWith", 2, 2); AddFunction("toJson", 1, 1); + AddFunction("fromJson", 1, 1); AddFunction("hashFiles", 1, 1); } diff --git a/src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/FromJson.cs b/src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/FromJson.cs new file mode 100644 index 00000000000..347c704672e --- /dev/null +++ b/src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/FromJson.cs @@ -0,0 +1,24 @@ +using System; +using System.IO; +using GitHub.DistributedTask.Pipelines.ContextData; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace GitHub.DistributedTask.Expressions2.Sdk.Functions +{ + internal sealed class FromJson : Function + { + protected sealed override Object EvaluateCore( + EvaluationContext context, + out ResultMemory resultMemory) + { + resultMemory = null; + var json = Parameters[0].Evaluate(context).ConvertToString(); + using (var stringReader = new StringReader(json)) + using (var jsonReader = new JsonTextReader(stringReader) { DateParseHandling = DateParseHandling.None, FloatParseHandling = FloatParseHandling.Double }) + { + var token = JToken.ReadFrom(jsonReader); + return token.ToPipelineContextData(); + } + } + }} diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs index 8e43e53edd3..3da980185ca 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs @@ -30,8 +30,7 @@ internal MappingDefinition(MappingToken definition) foreach (var propertiesPair in properties) { var propertyName = propertiesPair.Key.AssertString($"{TemplateConstants.Definition} {TemplateConstants.Mapping} {TemplateConstants.Properties} key"); - var propertyValue = propertiesPair.Value.AssertString($"{TemplateConstants.Definition} {TemplateConstants.Mapping} {TemplateConstants.Properties} value"); - Properties.Add(propertyName.Value, new PropertyValue(propertyValue.Value)); + Properties.Add(propertyName.Value, new PropertyValue(propertiesPair.Value)); } break; @@ -85,7 +84,7 @@ internal override void Validate( } else { - throw new ArgumentException($"Property '{TemplateConstants.LooseKeyType}' is defined but '{TemplateConstants.LooseValueType}' is not defined"); + throw new ArgumentException($"Property '{TemplateConstants.LooseKeyType}' is defined but '{TemplateConstants.LooseValueType}' is not defined on '{name}'"); } } // Otherwise validate loose value type not be defined @@ -95,9 +94,14 @@ internal override void Validate( } // Lookup each property - foreach (var property in Properties.Values) + foreach (var property in Properties) { - schema.GetDefinition(property.Type); + if (String.IsNullOrEmpty(property.Value.Type)) + { + throw new ArgumentException($"Type not specified for the '{property.Key}' property on the '{name}' type"); + } + + schema.GetDefinition(property.Value.Type); } if (!String.IsNullOrEmpty(Inherits)) diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/PropertyValue.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/PropertyValue.cs index 5a95b0171df..4064159aa0e 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/PropertyValue.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/PropertyValue.cs @@ -1,18 +1,40 @@ using System; +using GitHub.DistributedTask.ObjectTemplating.Tokens; namespace GitHub.DistributedTask.ObjectTemplating.Schema { internal sealed class PropertyValue { - internal PropertyValue() + internal PropertyValue(TemplateToken token) { - } - - internal PropertyValue(String type) - { - Type = type; + if (token is StringToken stringToken) + { + Type = stringToken.Value; + } + else + { + var mapping = token.AssertMapping($"{TemplateConstants.MappingPropertyValue}"); + foreach (var mappingPair in mapping) + { + var mappingKey = mappingPair.Key.AssertString($"{TemplateConstants.MappingPropertyValue} key"); + switch (mappingKey.Value) + { + case TemplateConstants.Type: + Type = mappingPair.Value.AssertString($"{TemplateConstants.MappingPropertyValue} {TemplateConstants.Type}").Value; + break; + case TemplateConstants.Required: + Required = mappingPair.Value.AssertBoolean($"{TemplateConstants.MappingPropertyValue} {TemplateConstants.Required}").Value; + break; + default: + mappingKey.AssertUnexpectedValue($"{TemplateConstants.MappingPropertyValue} key"); + break; + } + } + } } internal String Type { get; set; } + + internal Boolean Required { get; set; } } } diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/TemplateSchema.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/TemplateSchema.cs index 9ac6b2453e9..699af9ba9cc 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/TemplateSchema.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/TemplateSchema.cs @@ -312,8 +312,8 @@ private static TemplateSchema Schema // template-schema mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Version, new PropertyValue(TemplateConstants.NonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.Definitions, new PropertyValue(TemplateConstants.Definitions)); + mappingDefinition.Properties.Add(TemplateConstants.Version, new PropertyValue(new StringToken(null, null, null, TemplateConstants.NonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.Definitions, new PropertyValue(new StringToken(null, null, null, TemplateConstants.Definitions))); schema.Definitions.Add(TemplateConstants.TemplateSchema, mappingDefinition); // definitions @@ -335,9 +335,9 @@ private static TemplateSchema Schema // null-definition mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(TemplateConstants.String)); - mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(TemplateConstants.SequenceOfNonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.Null, new PropertyValue(TemplateConstants.NullDefinitionProperties)); + mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(new StringToken(null, null, null, TemplateConstants.String))); + mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceOfNonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.Null, new PropertyValue(new StringToken(null, null, null, TemplateConstants.NullDefinitionProperties))); schema.Definitions.Add(TemplateConstants.NullDefinition, mappingDefinition); // null-definition-properties @@ -346,9 +346,9 @@ private static TemplateSchema Schema // boolean-definition mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(TemplateConstants.String)); - mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(TemplateConstants.SequenceOfNonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.Boolean, new PropertyValue(TemplateConstants.BooleanDefinitionProperties)); + mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(new StringToken(null, null, null, TemplateConstants.String))); + mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceOfNonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.Boolean, new PropertyValue(new StringToken(null, null, null, TemplateConstants.BooleanDefinitionProperties))); schema.Definitions.Add(TemplateConstants.BooleanDefinition, mappingDefinition); // boolean-definition-properties @@ -357,9 +357,9 @@ private static TemplateSchema Schema // number-definition mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(TemplateConstants.String)); - mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(TemplateConstants.SequenceOfNonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.Number, new PropertyValue(TemplateConstants.NumberDefinitionProperties)); + mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(new StringToken(null, null, null, TemplateConstants.String))); + mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceOfNonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.Number, new PropertyValue(new StringToken(null, null, null, TemplateConstants.NumberDefinitionProperties))); schema.Definitions.Add(TemplateConstants.NumberDefinition, mappingDefinition); // number-definition-properties @@ -368,55 +368,68 @@ private static TemplateSchema Schema // string-definition mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(TemplateConstants.String)); - mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(TemplateConstants.SequenceOfNonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.String, new PropertyValue(TemplateConstants.StringDefinitionProperties)); + mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(new StringToken(null, null, null, TemplateConstants.String))); + mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceOfNonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.String, new PropertyValue(new StringToken(null, null, null, TemplateConstants.StringDefinitionProperties))); schema.Definitions.Add(TemplateConstants.StringDefinition, mappingDefinition); // string-definition-properties mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Constant, new PropertyValue(TemplateConstants.NonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.IgnoreCase, new PropertyValue(TemplateConstants.Boolean)); - mappingDefinition.Properties.Add(TemplateConstants.RequireNonEmpty, new PropertyValue(TemplateConstants.Boolean)); + mappingDefinition.Properties.Add(TemplateConstants.Constant, new PropertyValue(new StringToken(null, null, null, TemplateConstants.NonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.IgnoreCase, new PropertyValue(new StringToken(null, null, null,TemplateConstants.Boolean))); + mappingDefinition.Properties.Add(TemplateConstants.RequireNonEmpty, new PropertyValue(new StringToken(null, null, null, TemplateConstants.Boolean))); schema.Definitions.Add(TemplateConstants.StringDefinitionProperties, mappingDefinition); // sequence-definition mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(TemplateConstants.String)); - mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(TemplateConstants.SequenceOfNonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.Sequence, new PropertyValue(TemplateConstants.SequenceDefinitionProperties)); + mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(new StringToken(null, null, null, TemplateConstants.String))); + mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceOfNonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.Sequence, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceDefinitionProperties))); schema.Definitions.Add(TemplateConstants.SequenceDefinition, mappingDefinition); // sequence-definition-properties mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.ItemType, new PropertyValue(TemplateConstants.NonEmptyString)); + mappingDefinition.Properties.Add(TemplateConstants.ItemType, new PropertyValue(new StringToken(null, null, null, TemplateConstants.NonEmptyString))); schema.Definitions.Add(TemplateConstants.SequenceDefinitionProperties, mappingDefinition); // mapping-definition mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(TemplateConstants.String)); - mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(TemplateConstants.SequenceOfNonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.Mapping, new PropertyValue(TemplateConstants.MappingDefinitionProperties)); + mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(new StringToken(null, null, null, TemplateConstants.String))); + mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceOfNonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.Mapping, new PropertyValue(new StringToken(null, null, null, TemplateConstants.MappingDefinitionProperties))); schema.Definitions.Add(TemplateConstants.MappingDefinition, mappingDefinition); // mapping-definition-properties mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Properties, new PropertyValue(TemplateConstants.Properties)); - mappingDefinition.Properties.Add(TemplateConstants.LooseKeyType, new PropertyValue(TemplateConstants.NonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.LooseValueType, new PropertyValue(TemplateConstants.NonEmptyString)); + mappingDefinition.Properties.Add(TemplateConstants.Properties, new PropertyValue(new StringToken(null, null, null, TemplateConstants.Properties))); + mappingDefinition.Properties.Add(TemplateConstants.LooseKeyType, new PropertyValue(new StringToken(null, null, null, TemplateConstants.NonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.LooseValueType, new PropertyValue(new StringToken(null, null, null, TemplateConstants.NonEmptyString))); schema.Definitions.Add(TemplateConstants.MappingDefinitionProperties, mappingDefinition); // properties mappingDefinition = new MappingDefinition(); mappingDefinition.LooseKeyType = TemplateConstants.NonEmptyString; - mappingDefinition.LooseValueType = TemplateConstants.NonEmptyString; + mappingDefinition.LooseValueType = TemplateConstants.PropertyValue; schema.Definitions.Add(TemplateConstants.Properties, mappingDefinition); + // property-value + oneOfDefinition = new OneOfDefinition(); + oneOfDefinition.OneOf.Add(TemplateConstants.NonEmptyString); + oneOfDefinition.OneOf.Add(TemplateConstants.MappingPropertyValue); + schema.Definitions.Add(TemplateConstants.PropertyValue, oneOfDefinition); + + // mapping-property-value + mappingDefinition = new MappingDefinition(); + mappingDefinition.Properties.Add(TemplateConstants.Type, new PropertyValue(new StringToken(null, null, null, TemplateConstants.NonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.Required, new PropertyValue(new StringToken(null, null, null, TemplateConstants.Boolean))); + schema.Definitions.Add(TemplateConstants.MappingPropertyValue, mappingDefinition); + + // one-of-definition mappingDefinition = new MappingDefinition(); - mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(TemplateConstants.String)); - mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(TemplateConstants.SequenceOfNonEmptyString)); - mappingDefinition.Properties.Add(TemplateConstants.OneOf, new PropertyValue(TemplateConstants.SequenceOfNonEmptyString)); + mappingDefinition.Properties.Add(TemplateConstants.Description, new PropertyValue(new StringToken(null, null, null, TemplateConstants.String))); + mappingDefinition.Properties.Add(TemplateConstants.Context, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceOfNonEmptyString))); + mappingDefinition.Properties.Add(TemplateConstants.OneOf, new PropertyValue(new StringToken(null, null, null, TemplateConstants.SequenceOfNonEmptyString))); schema.Definitions.Add(TemplateConstants.OneOfDefinition, mappingDefinition); // non-empty-string @@ -477,4 +490,4 @@ private void Validate() private static readonly Regex s_definitionNameRegex = new Regex("^[a-zA-Z_][a-zA-Z0-9_-]*$", RegexOptions.Compiled); private static TemplateSchema s_schema; } -} +} \ No newline at end of file diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateConstants.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateConstants.cs index 72ebae5ab22..21e70e4b9e6 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateConstants.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateConstants.cs @@ -22,9 +22,11 @@ internal static class TemplateConstants internal const String ItemType = "item-type"; internal const String LooseKeyType = "loose-key-type"; internal const String LooseValueType = "loose-value-type"; + internal const String MaxConstant = "MAX"; internal const String Mapping = "mapping"; internal const String MappingDefinition = "mapping-definition"; internal const String MappingDefinitionProperties = "mapping-definition-properties"; + internal const String MappingPropertyValue = "mapping-property-value"; internal const String NonEmptyString = "non-empty-string"; internal const String Null = "null"; internal const String NullDefinition = "null-definition"; @@ -35,7 +37,9 @@ internal static class TemplateConstants internal const String OneOf = "one-of"; internal const String OneOfDefinition = "one-of-definition"; internal const String OpenExpression = "${{"; + internal const String PropertyValue = "property-value"; internal const String Properties = "properties"; + internal const String Required = "required"; internal const String RequireNonEmpty = "require-non-empty"; internal const String Scalar = "scalar"; internal const String ScalarDefinition = "scalar-definition"; @@ -43,6 +47,7 @@ internal static class TemplateConstants internal const String Sequence = "sequence"; internal const String SequenceDefinition = "sequence-definition"; internal const String SequenceDefinitionProperties = "sequence-definition-properties"; + internal const String Type = "type"; internal const String SequenceOfNonEmptyString = "sequence-of-non-empty-string"; internal const String String = "string"; internal const String StringDefinition = "string-definition"; diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs index 48670a9f3a6..63f5163194f 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs @@ -47,7 +47,7 @@ internal static TemplateToken Evaluate( var evaluator = new TemplateEvaluator(context, template, removeBytes); try { - var availableContext = new HashSet(context.ExpressionValues.Keys); + var availableContext = new HashSet(context.ExpressionValues.Keys.Concat(context.ExpressionFunctions.Select(x => $"{x.Name}({x.MinParameters},{x.MaxParameters})"))); var definitionInfo = new DefinitionInfo(context.Schema, type, availableContext); result = evaluator.Evaluate(definitionInfo); @@ -182,12 +182,14 @@ private void HandleMappingWithWellKnownProperties( } var keys = new HashSet(StringComparer.OrdinalIgnoreCase); + var hasExpressionKey = false; while (m_unraveler.AllowScalar(definition.Expand, out ScalarToken nextKeyScalar)) { // Expression if (nextKeyScalar is ExpressionToken) { + hasExpressionKey = true; var anyDefinition = new DefinitionInfo(definition, TemplateConstants.Any); mapping.Add(nextKeyScalar, Evaluate(anyDefinition)); continue; @@ -268,6 +270,19 @@ private void HandleMappingWithWellKnownProperties( String listToDeDuplicate = String.Join(", ", nonDuplicates); m_context.Error(mapping, TemplateStrings.UnableToDetermineOneOf(listToDeDuplicate)); } + else if (mappingDefinitions.Count == 1 && !hasExpressionKey) + { + foreach (var property in mappingDefinitions[0].Properties) + { + if (property.Value.Required) + { + if (!keys.Contains(property.Key)) + { + m_context.Error(mapping, $"Required property is missing: {property.Key}"); + } + } + } + } m_unraveler.ReadMappingEnd(); } diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs index 56b149c3f08..eab601bc09f 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs @@ -178,14 +178,15 @@ private void HandleMappingWithWellKnownProperties( } var keys = new HashSet(StringComparer.OrdinalIgnoreCase); + var hasExpressionKey = false; while (m_objectReader.AllowLiteral(out LiteralToken rawLiteral)) { var nextKeyScalar = ParseScalar(rawLiteral, definition.AllowedContext); - // Expression if (nextKeyScalar is ExpressionToken) { + hasExpressionKey = true; // Legal if (definition.AllowedContext.Length > 0) { @@ -280,7 +281,19 @@ private void HandleMappingWithWellKnownProperties( String listToDeDuplicate = String.Join(", ", nonDuplicates); m_context.Error(mapping, TemplateStrings.UnableToDetermineOneOf(listToDeDuplicate)); } - + else if (mappingDefinitions.Count == 1 && !hasExpressionKey) + { + foreach (var property in mappingDefinitions[0].Properties) + { + if (property.Value.Required) + { + if (!keys.Contains(property.Key)) + { + m_context.Error(mapping, $"Required property is missing: {property.Key}"); + } + } + } + } ExpectMappingEnd(); } diff --git a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs index 36a8864cdc5..576c3cb6ff7 100644 --- a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs +++ b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs @@ -40,7 +40,8 @@ public AgentJobRequestMessage( WorkspaceOptions workspaceOptions, IEnumerable steps, IEnumerable scopes, - IList fileTable) + IList fileTable, + TemplateToken jobOutputs) { this.MessageType = JobRequestMessageTypes.PipelineAgentJobRequest; this.Plan = plan; @@ -52,6 +53,7 @@ public AgentJobRequestMessage( this.Timeline = timeline; this.Resources = jobResources; this.Workspace = workspaceOptions; + this.JobOutputs = jobOutputs; m_variables = new Dictionary(variables, StringComparer.OrdinalIgnoreCase); m_maskHints = new List(maskHints); @@ -138,6 +140,13 @@ public TemplateToken JobServiceContainers private set; } + [DataMember(EmitDefaultValue = false)] + public TemplateToken JobOutputs + { + get; + private set; + } + [DataMember] public Int64 RequestId { diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index 0675b993c9a..72853f5b2ed 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -29,7 +29,9 @@ public sealed class PipelineTemplateConstants public const String Include = "include"; public const String Inputs = "inputs"; public const String Job = "job"; + public const String JobOutputs = "job-outputs"; public const String Jobs = "jobs"; + public const String Labels = "labels"; public const String Lfs = "lfs"; public const String Matrix = "matrix"; public const String MaxParallel = "max-parallel"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index 1d10a3adcec..ea9193ee16c 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -231,6 +231,42 @@ public JobContainer EvaluateJobContainer( return result; } + public Dictionary EvaluateJobOutput( + TemplateToken token, + DictionaryContextData contextData) + { + var result = default(Dictionary); + + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(contextData); + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobOutputs, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = new Dictionary(StringComparer.OrdinalIgnoreCase); + var mapping = token.AssertMapping("outputs"); + foreach (var pair in mapping) + { + // Literal key + var key = pair.Key.AssertString("output key"); + + // Literal value + var value = pair.Value.AssertString("output value"); + result[key.Value] = value.Value; + } + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + + return result; + } + public IList> EvaluateJobServiceContainers( TemplateToken token, DictionaryContextData contextData) @@ -364,6 +400,7 @@ private TemplateContext CreateContext(DictionaryContextData contextData) PipelineTemplateConstants.GitHub, PipelineTemplateConstants.Strategy, PipelineTemplateConstants.Matrix, + PipelineTemplateConstants.Needs, PipelineTemplateConstants.Secrets, PipelineTemplateConstants.Steps, PipelineTemplateConstants.Inputs, diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index 21e6d2b63b5..203a686ad00 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -38,6 +38,7 @@ "context": [ "github", "strategy", + "needs", "matrix", "secrets", "steps", @@ -66,6 +67,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "inputs", @@ -89,7 +91,8 @@ "context": [ "github", "strategy", - "matrix" + "matrix", + "needs" ], "one-of": [ "string", @@ -112,6 +115,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "job", @@ -143,16 +147,20 @@ "mapping": { "properties": { "needs": "needs", - "if": "string", + "if": "job-if", "strategy": "strategy", "name": "string-strategy-context", - "runs-on": "runs-on", + "runs-on": { + "type": "runs-on", + "required": true + }, "timeout-minutes": "number-strategy-context", "cancel-timeout-minutes": "number-strategy-context", - "continue-on-error": "boolean", + "continue-on-error": "boolean-strategy-context", "container": "container", "services": "services", "env": "job-env", + "outputs": "job-outputs", "steps": "steps" } } @@ -165,9 +173,22 @@ ] }, + "job-if": { + "context": [ + "github", + "needs", + "always(0,0)", + "failure(0,MAX)", + "cancelled(0,0)", + "success(0,MAX)" + ], + "string": {} + }, + "strategy": { "context": [ - "github" + "github", + "needs" ], "mapping": { "properties": { @@ -233,24 +254,23 @@ "context": [ "github", "strategy", - "matrix" + "matrix", + "needs" ], "one-of": [ - "runs-on-string", + "non-empty-string", + "sequence-of-non-empty-string", "runs-on-mapping" ] }, - "runs-on-string": { - "string": { - "require-non-empty": true - } - }, - "runs-on-mapping": { "mapping": { "properties": { - "pool": "non-empty-string" + "pool": { + "type": "non-empty-string", + "required": true + } } } }, @@ -260,7 +280,8 @@ "github", "secrets", "strategy", - "matrix" + "matrix", + "needs" ], "mapping": { "loose-key-type": "non-empty-string", @@ -268,6 +289,13 @@ } }, + "job-outputs": { + "mapping": { + "loose-key-type": "non-empty-string", + "loose-value-type": "string-runner-context" + } + }, + "steps": { "sequence": { "item-type": "steps-item" @@ -301,9 +329,12 @@ "properties": { "name": "string-steps-context", "id": "non-empty-string", - "if": "string", + "if": "step-if", "timeout-minutes": "number-steps-context", - "run": "string-steps-context", + "run": { + "type": "string-steps-context", + "required": true + }, "continue-on-error": "boolean-steps-context", "env": "step-env", "working-directory": "string-steps-context", @@ -317,9 +348,12 @@ "properties": { "name": "string-steps-context-in-template", "id": "non-empty-string", - "if": "string", + "if": "step-if-in-template", "timeout-minutes": "number-steps-context-in-template", - "run": "string-steps-context-in-template", + "run": { + "type": "string-steps-context-in-template", + "required": true + }, "continue-on-error": "boolean-steps-context-in-template", "env": "step-env-in-template", "working-directory": "string-steps-context-in-template", @@ -333,10 +367,13 @@ "properties": { "name": "string-steps-context", "id": "non-empty-string", - "if": "string", + "if": "step-if", "continue-on-error": "boolean-steps-context", "timeout-minutes": "number-steps-context", - "uses": "non-empty-string", + "uses": { + "type": "non-empty-string", + "required": true + }, "with": "step-with", "env": "step-env" } @@ -348,16 +385,56 @@ "properties": { "name": "string-steps-context-in-template", "id": "non-empty-string", - "if": "string", + "if": "step-if-in-template", "continue-on-error": "boolean-steps-context-in-template", "timeout-minutes": "number-steps-context-in-template", - "uses": "non-empty-string", + "uses": { + "type": "non-empty-string", + "required": true + }, "with": "step-with-in-template", "env": "step-env-in-template" } } }, + "step-if": { + "context": [ + "github", + "strategy", + "matrix", + "needs", + "steps", + "job", + "runner", + "env", + "always(0,0)", + "failure(0,0)", + "cancelled(0,0)", + "success(0,0)" + ], + "string": {} + }, + + "step-if-in-template": { + "context": [ + "github", + "strategy", + "matrix", + "needs", + "steps", + "inputs", + "job", + "runner", + "env", + "always(0,0)", + "failure(0,0)", + "cancelled(0,0)", + "success(0,0)" + ], + "string": {} + }, + "steps-template-reference": { "mapping": { "properties": { @@ -383,6 +460,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "job", @@ -400,6 +478,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "inputs", @@ -418,6 +497,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "job", @@ -435,6 +515,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "inputs", @@ -453,6 +534,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "job", @@ -469,7 +551,8 @@ "context": [ "github", "strategy", - "matrix" + "matrix", + "needs" ], "one-of": [ "string", @@ -493,7 +576,8 @@ "context": [ "github", "strategy", - "matrix" + "matrix", + "needs" ], "mapping": { "loose-key-type": "non-empty-string", @@ -505,7 +589,8 @@ "context": [ "github", "strategy", - "matrix" + "matrix", + "needs" ], "one-of": [ "non-empty-string", @@ -521,23 +606,24 @@ }, "step-with-in-template": { - "context": [ - "github", - "strategy", - "matrix", - "secrets", - "steps", - "inputs", - "job", - "runner", - "env" - ], - "mapping": { - "loose-key-type": "non-empty-string", - "loose-value-type": "string" - } - }, - + "context": [ + "github", + "strategy", + "matrix", + "needs", + "secrets", + "steps", + "inputs", + "job", + "runner", + "env" + ], + "mapping": { + "loose-key-type": "non-empty-string", + "loose-value-type": "string" + } + }, + "non-empty-string": { "string": { "require-non-empty": true @@ -550,11 +636,22 @@ } }, + "boolean-strategy-context": { + "context": [ + "github", + "strategy", + "matrix", + "needs" + ], + "boolean": {} + }, + "number-strategy-context": { "context": [ "github", "strategy", - "matrix" + "matrix", + "needs" ], "number": {} }, @@ -563,7 +660,8 @@ "context": [ "github", "strategy", - "matrix" + "matrix", + "needs" ], "string": {} }, @@ -573,6 +671,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "job", @@ -587,6 +686,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "inputs", @@ -602,6 +702,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "job", @@ -616,6 +717,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "inputs", @@ -626,11 +728,27 @@ "number": {} }, + "string-runner-context": { + "context": [ + "github", + "strategy", + "matrix", + "needs", + "secrets", + "steps", + "job", + "runner", + "env" + ], + "string": {} + }, + "string-steps-context": { "context": [ "github", "strategy", "matrix", + "needs", "secrets", "steps", "job", @@ -645,6 +763,7 @@ "github", "strategy", "matrix", + "needs", "secrets", "steps", "inputs", diff --git a/src/Sdk/DTWebApi/WebApi/JobEvent.cs b/src/Sdk/DTWebApi/WebApi/JobEvent.cs index 3bfbdd53b59..7566cde947a 100644 --- a/src/Sdk/DTWebApi/WebApi/JobEvent.cs +++ b/src/Sdk/DTWebApi/WebApi/JobEvent.cs @@ -31,7 +31,7 @@ protected JobEvent(String name) } protected JobEvent( - String name, + String name, Guid jobId) { this.Name = name; @@ -123,11 +123,12 @@ public JobCompletedEvent( Int64 requestId, Guid jobId, TaskResult result, - IDictionary outputVariables) + Dictionary outputs) : base(JobEventTypes.JobCompleted, jobId) { this.RequestId = requestId; this.Result = result; + this.Outputs = outputs; } [DataMember(EmitDefaultValue = false)] @@ -143,6 +144,13 @@ public TaskResult Result get; set; } + + [DataMember(EmitDefaultValue = false)] + public IDictionary Outputs + { + get; + set; + } } [DataContract] @@ -153,9 +161,9 @@ protected TaskEvent(string name) : base(name) } protected TaskEvent( - string name, - Guid jobId, - Guid taskId) + string name, + Guid jobId, + Guid taskId) : base(name, jobId) { TaskId = taskId; @@ -185,9 +193,9 @@ public override Boolean CanConvert(Type objectType) } public override Object ReadJson( - JsonReader reader, - Type objectType, - Object existingValue, + JsonReader reader, + Type objectType, + Object existingValue, JsonSerializer serializer) { var eventObject = JObject.Load(reader); diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index b81606d54c1..6ee79360e24 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -33,7 +33,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage() TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); + var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); return result; } diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index 07e80e9ce3c..5d8bc3e0f9d 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -43,7 +43,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); + return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); } private JobCancelMessage CreateJobCancelMessage() diff --git a/src/Test/L0/Worker/ActionCommandManagerL0.cs b/src/Test/L0/Worker/ActionCommandManagerL0.cs index a6cdc086e8a..2ef9f5c5c3e 100644 --- a/src/Test/L0/Worker/ActionCommandManagerL0.cs +++ b/src/Test/L0/Worker/ActionCommandManagerL0.cs @@ -150,7 +150,7 @@ public void EchoProcessCommandDebugOn() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 513d286ccea..190bcda7611 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -25,7 +25,7 @@ public void AddIssue_CountWarningsErrors() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -101,7 +101,7 @@ public void Debug_Multilines() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -152,7 +152,7 @@ public void RegisterPostJobAction_ShareState() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 209f915e950..b2c5dbdddd6 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -100,7 +100,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " }; Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null); + _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null); GitHubContext github = new GitHubContext(); github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner"); _message.ContextData.Add("github", github); diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index a88e6b8a799..5746547204a 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -63,7 +63,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = new Timeline(Guid.NewGuid()); Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); + _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); _message.Variables[Constants.Variables.System.Culture] = "en-US"; _message.Resources.Endpoints.Add(new ServiceEndpoint() { diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index e3ef3d2f77a..fd215b71890 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -55,6 +55,9 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " _ec.Setup(x => x.PostJobSteps).Returns(new Stack()); + var trace = hc.GetTrace(); + _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { trace.Info($"[{tag}]{message}"); }); + _stepsRunner = new StepsRunner(); _stepsRunner.Initialize(hc); return hc; diff --git a/src/Test/L0/Worker/WorkerL0.cs b/src/Test/L0/Worker/WorkerL0.cs index b48542e7970..ed930f184c6 100644 --- a/src/Test/L0/Worker/WorkerL0.cs +++ b/src/Test/L0/Worker/WorkerL0.cs @@ -67,7 +67,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) new Pipelines.ContextData.DictionaryContextData() }, }; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null, null); return jobRequest; } From aa9f5bf0700f22fa50b8b08f318658b54a2a2f38 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Mon, 16 Mar 2020 14:56:07 -0400 Subject: [PATCH 05/86] adr step output and conclusion (#274) --- docs/adrs/0274-step-outcome-and-conclusion.md | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/adrs/0274-step-outcome-and-conclusion.md diff --git a/docs/adrs/0274-step-outcome-and-conclusion.md b/docs/adrs/0274-step-outcome-and-conclusion.md new file mode 100644 index 00000000000..afc9ff3136b --- /dev/null +++ b/docs/adrs/0274-step-outcome-and-conclusion.md @@ -0,0 +1,62 @@ +# ADR 0274: Step outcome and conclusion + +**Date**: 2020-01-13 + +**Status**: Accepted + +## Context + +This ADR proposes adding `steps..outcome` and `steps..conclusion` to the steps context. + +This allows downstream a step to run based on whether a previous step succeeded or failed. + +Reminder, currently the steps contains `steps..outputs`. + +## Decision + +For steps that have completed, populate `steps..outcome` and `steps..conclusion` with one of the following values: + +- `success` +- `failure` +- `cancelled` +- `skipped` + +When a continue-on-error step fails, the outcome will be `failure` even though the final conclusion is `success`. + +### Example + +```yaml +steps: + + - id: experimental + continue-on-error: true + run: ./build.sh experimental + + - if: ${{ steps.experimental.outcome == 'success' }} + run: ./publish.sh experimental +``` + +### Terminology + +The runs API uses the term `conclusion`. + +Therefore we use a different term `outcome` for the value prior to continue-on-error. + +The following is a snippet from the runs API response payload: + +```json + "steps": [ + { + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "number": 1, + "started_at": "2020-01-09T11:06:16.000-05:00", + "completed_at": "2020-01-09T11:06:18.000-05:00" + }, +``` + +## Consequences + +- Update runner +- Update [docs](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/contexts-and-expression-syntax-for-github-actions#steps-context) \ No newline at end of file From 41f4ca3414b9e48c663b04d2ace044ee355d834e Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Mon, 16 Mar 2020 22:19:57 -0400 Subject: [PATCH 06/86] grammar (#373) --- docs/contribute.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contribute.md b/docs/contribute.md index 78cbfb57842..8d68a446f15 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -44,7 +44,7 @@ Sample developer flow: ```bash git clone https://github.com/actions/runner cd ./src -./dev.(sh/cmd) layout # the runner that build from source is in {root}/_layout +./dev.(sh/cmd) layout # the runner that built from source is in {root}/_layout ./dev.(sh/cmd) build # {root}/_layout will get updated ./dev.(sh/cmd) test # run all unit tests before git commit/push From a5eb8cb5c44e52a2a760db12c16c65c893cb4f0d Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 17 Mar 2020 19:58:12 -0400 Subject: [PATCH 07/86] set CI=true when launch process in actions runner. (#374) --- src/Runner.Sdk/ProcessInvoker.cs | 8 +++ src/Test/L0/ProcessInvokerL0.cs | 97 ++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/src/Runner.Sdk/ProcessInvoker.cs b/src/Runner.Sdk/ProcessInvoker.cs index 5841469144e..a56b1475ae7 100644 --- a/src/Runner.Sdk/ProcessInvoker.cs +++ b/src/Runner.Sdk/ProcessInvoker.cs @@ -271,6 +271,14 @@ public async Task ExecuteAsync( // Indicate GitHub Actions process. _proc.StartInfo.Environment["GITHUB_ACTIONS"] = "true"; + // Set CI=true when no one else already set it. + // CI=true is common set in most CI provider in GitHub + if (!_proc.StartInfo.Environment.ContainsKey("CI") && + Environment.GetEnvironmentVariable("CI") == null) + { + _proc.StartInfo.Environment["CI"] = "true"; + } + // Hook up the events. _proc.EnableRaisingEvents = true; _proc.Exited += ProcessExitedHandler; diff --git a/src/Test/L0/ProcessInvokerL0.cs b/src/Test/L0/ProcessInvokerL0.cs index 0679349cece..1aca0bba28a 100644 --- a/src/Test/L0/ProcessInvokerL0.cs +++ b/src/Test/L0/ProcessInvokerL0.cs @@ -8,6 +8,7 @@ using GitHub.Runner.Common.Util; using System.Threading.Channels; using GitHub.Runner.Sdk; +using System.Linq; namespace GitHub.Runner.Common.Tests { @@ -81,6 +82,102 @@ public async Task SuccessExitsWithCodeZero() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public async Task SetCIEnv() + { + using (TestHostContext hc = new TestHostContext(this)) + { + var existingCI = Environment.GetEnvironmentVariable("CI"); + try + { + // Clear out CI and make sure process invoker sets it. + Environment.SetEnvironmentVariable("CI", null); + + Tracing trace = hc.GetTrace(); + + Int32 exitCode = -1; + var processInvoker = new ProcessInvokerWrapper(); + processInvoker.Initialize(hc); + var stdout = new List(); + var stderr = new List(); + processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stdout.Add(e.Data); + }; + processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stderr.Add(e.Data); + }; +#if OS_WINDOWS + exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %CI%\"", null, CancellationToken.None); +#else + exitCode = await processInvoker.ExecuteAsync("", "bash", "-c \"echo $CI\"", null, CancellationToken.None); +#endif + + trace.Info("Exit Code: {0}", exitCode); + Assert.Equal(0, exitCode); + + Assert.Equal("true", stdout.First(x => !string.IsNullOrWhiteSpace(x))); + } + finally + { + Environment.SetEnvironmentVariable("CI", existingCI); + } + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public async Task KeepExistingCIEnv() + { + using (TestHostContext hc = new TestHostContext(this)) + { + var existingCI = Environment.GetEnvironmentVariable("CI"); + try + { + // Clear out CI and make sure process invoker sets it. + Environment.SetEnvironmentVariable("CI", null); + + Tracing trace = hc.GetTrace(); + + Int32 exitCode = -1; + var processInvoker = new ProcessInvokerWrapper(); + processInvoker.Initialize(hc); + var stdout = new List(); + var stderr = new List(); + processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stdout.Add(e.Data); + }; + processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stderr.Add(e.Data); + }; +#if OS_WINDOWS + exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %CI%\"", new Dictionary() { { "CI", "false" } }, CancellationToken.None); +#else + exitCode = await processInvoker.ExecuteAsync("", "bash", "-c \"echo $CI\"", new Dictionary() { { "CI", "false" } }, CancellationToken.None); +#endif + + trace.Info("Exit Code: {0}", exitCode); + Assert.Equal(0, exitCode); + + Assert.Equal("false", stdout.First(x => !string.IsNullOrWhiteSpace(x))); + } + finally + { + Environment.SetEnvironmentVariable("CI", existingCI); + } + } + } + #if !OS_WINDOWS //Run a process that normally takes 20sec to finish and cancel it. [Fact] From 88875ca1b071c0cf999ad0a82596ed1290b6ac83 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 17 Mar 2020 21:18:42 -0400 Subject: [PATCH 08/86] set steps..outcome and steps..conclusion. (#372) --- src/Runner.Worker/ExecutionContext.cs | 9 +++ src/Runner.Worker/StepsContext.cs | 15 ++++- src/Runner.Worker/StepsRunner.cs | 1 + src/Test/L0/Worker/StepsRunnerL0.cs | 83 ++++++++++++++++++++++++++- 4 files changed, 102 insertions(+), 6 deletions(-) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index b48936b08fe..667e93e8c6d 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -39,6 +39,7 @@ public interface IExecutionContext : IRunnerService string ContextName { get; } Task ForceCompleted { get; } TaskResult? Result { get; set; } + TaskResult? Outcome { get; set; } string ResultCode { get; set; } TaskResult? CommandResult { get; set; } CancellationToken CancellationToken { get; } @@ -171,6 +172,8 @@ public TaskResult? Result } } + public TaskResult? Outcome { get; set; } + public TaskResult? CommandResult { get; set; } private string ContextType => _record.RecordType; @@ -346,6 +349,12 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = _logger.End(); + if (!string.IsNullOrEmpty(ContextName)) + { + StepsContext.SetOutcome(ScopeName, ContextName, (Outcome ?? Result ?? TaskResult.Succeeded).ToActionResult().ToString()); + StepsContext.SetConclusion(ScopeName, ContextName, (Result ?? TaskResult.Succeeded).ToActionResult().ToString()); + } + return Result.Value; } diff --git a/src/Runner.Worker/StepsContext.cs b/src/Runner.Worker/StepsContext.cs index 41ea72961d2..d9add5a09be 100644 --- a/src/Runner.Worker/StepsContext.cs +++ b/src/Runner.Worker/StepsContext.cs @@ -56,13 +56,22 @@ public void SetOutput( } } - public void SetResult( + public void SetConclusion( string scopeName, string stepName, - string result) + string conclusion) { var step = GetStep(scopeName, stepName); - step["result"] = new StringContextData(result); + step["conclusion"] = new StringContextData(conclusion); + } + + public void SetOutcome( + string scopeName, + string stepName, + string outcome) + { + var step = GetStep(scopeName, stepName); + step["outcome"] = new StringContextData(outcome); } private DictionaryContextData GetStep(string scopeName, string stepName) diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index f6953067236..ec1d7069c2a 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -351,6 +351,7 @@ private async Task RunStepAsync(IStep step, CancellationToken jobCancellationTok if (continueOnError) { + step.ExecutionContext.Outcome = step.ExecutionContext.Result; step.ExecutionContext.Result = TaskResult.Succeeded; Trace.Info($"Updated step result (continue on error)"); } diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index fd215b71890..23534813f4b 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -11,6 +11,7 @@ using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.Runner.Common.Util; namespace GitHub.Runner.Common.Tests.Worker { @@ -57,7 +58,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " var trace = hc.GetTrace(); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { trace.Info($"[{tag}]{message}"); }); - + _stepsRunner = new StepsRunner(); _stepsRunner.Initialize(hc); return hc; @@ -516,7 +517,78 @@ public async Task PopulateEnvContextAfterSetupStepsContext() } } - private Mock CreateStep(TestHostContext hc, TaskResult result, string condition, Boolean continueOnError = false, MappingToken env = null, string name = "Test", bool setOutput = false) + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task StepContextOutcome() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange. + var step1 = CreateStep(hc, TaskResult.Succeeded, "success()", contextName: "step1"); + var step2 = CreateStep(hc, TaskResult.Failed, "steps.step1.outcome == 'success'", continueOnError: true, contextName: "step2"); + var step3 = CreateStep(hc, TaskResult.Succeeded, "steps.step1.outcome == 'success' && steps.step2.outcome == 'failure'", contextName: "step3"); + + _ec.Object.Result = null; + + _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object, step3.Object })); + + // Act. + await _stepsRunner.RunAsync(jobContext: _ec.Object); + + // Assert. + Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); + + step1.Verify(x => x.RunAsync(), Times.Once); + step2.Verify(x => x.RunAsync(), Times.Once); + step3.Verify(x => x.RunAsync(), Times.Once); + + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Failed.ToActionResult().ToString(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["conclusion"].AssertString("")); + + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task StepContextConclusion() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange. + var step1 = CreateStep(hc, TaskResult.Succeeded, "false", contextName: "step1"); + var step2 = CreateStep(hc, TaskResult.Failed, "steps.step1.conclusion == 'skipped'", continueOnError: true, contextName: "step2"); + var step3 = CreateStep(hc, TaskResult.Succeeded, "steps.step1.outcome == 'skipped' && steps.step2.outcome == 'failure' && steps.step2.conclusion == 'success'", contextName: "step3"); + + _ec.Object.Result = null; + + _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object, step3.Object })); + + // Act. + await _stepsRunner.RunAsync(jobContext: _ec.Object); + + // Assert. + Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); + + step1.Verify(x => x.RunAsync(), Times.Never); + step2.Verify(x => x.RunAsync(), Times.Once); + step3.Verify(x => x.RunAsync(), Times.Once); + + Assert.Equal(TaskResult.Skipped.ToActionResult().ToString(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Skipped.ToActionResult().ToString(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Failed.ToActionResult().ToString(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["conclusion"].AssertString("")); + } + } + + private Mock CreateStep(TestHostContext hc, TaskResult result, string condition, Boolean continueOnError = false, MappingToken env = null, string name = "Test", bool setOutput = false, string contextName = null) { // Setup the step. var step = new Mock(); @@ -527,7 +599,8 @@ private Mock CreateStep(TestHostContext hc, TaskResult result, st { Name = name, Id = Guid.NewGuid(), - Environment = env + Environment = env, + ContextName = contextName ?? "Test" }); // Setup the step execution context. @@ -539,6 +612,7 @@ private Mock CreateStep(TestHostContext hc, TaskResult result, st stepContext.Setup(x => x.ExpressionValues).Returns(_contexts); stepContext.Setup(x => x.JobContext).Returns(_jobContext); stepContext.Setup(x => x.StepsContext).Returns(_stepContext); + stepContext.Setup(x => x.ContextName).Returns(step.Object.Action.ContextName); stepContext.Setup(x => x.Complete(It.IsAny(), It.IsAny(), It.IsAny())) .Callback((TaskResult? r, string currentOperation, string resultCode) => { @@ -546,6 +620,9 @@ private Mock CreateStep(TestHostContext hc, TaskResult result, st { stepContext.Object.Result = r; } + + _stepContext.SetOutcome("", stepContext.Object.ContextName, (stepContext.Object.Outcome ?? stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult().ToString()); + _stepContext.SetConclusion("", stepContext.Object.ContextName, (stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult().ToString()); }); var trace = hc.GetTrace(); stepContext.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { trace.Info($"[{tag}]{message}"); }); From b0a71481f064718ad4a7adde014f6545bbb8c60c Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 17 Mar 2020 23:40:37 -0400 Subject: [PATCH 09/86] support defaults. (#369) --- src/Runner.Worker/ExecutionContext.cs | 6 +++ src/Runner.Worker/Handlers/ScriptHandler.cs | 42 ++++++++++++++++-- src/Runner.Worker/JobExtension.cs | 20 +++++++++ .../Pipelines/AgentJobRequestMessage.cs | 34 ++++++++++++++- .../PipelineTemplateConstants.cs | 2 + .../PipelineTemplateEvaluator.cs | 36 ++++++++++++++++ src/Sdk/DTPipelines/workflow-v1.0.json | 43 +++++++++++++++++++ src/Test/L0/Listener/JobDispatcherL0.cs | 2 +- src/Test/L0/Listener/RunnerL0.cs | 2 +- src/Test/L0/Worker/ActionCommandManagerL0.cs | 2 +- src/Test/L0/Worker/ExecutionContextL0.cs | 6 +-- src/Test/L0/Worker/JobExtensionL0.cs | 2 +- src/Test/L0/Worker/JobRunnerL0.cs | 2 +- src/Test/L0/Worker/WorkerL0.cs | 2 +- 14 files changed, 187 insertions(+), 14 deletions(-) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 667e93e8c6d..6464e080704 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -48,6 +48,7 @@ public interface IExecutionContext : IRunnerService PlanFeatures Features { get; } Variables Variables { get; } Dictionary IntraActionState { get; } + IDictionary> JobDefaults { get; } Dictionary JobOutputs { get; } IDictionary EnvironmentVariables { get; } IDictionary Scopes { get; } @@ -140,6 +141,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public List Endpoints { get; private set; } public Variables Variables { get; private set; } public Dictionary IntraActionState { get; private set; } + public IDictionary> JobDefaults { get; private set; } public Dictionary JobOutputs { get; private set; } public IDictionary EnvironmentVariables { get; private set; } public IDictionary Scopes { get; private set; } @@ -270,6 +272,7 @@ public IExecutionContext CreateChild(Guid recordId, string displayName, string r child.IntraActionState = intraActionState; } child.EnvironmentVariables = EnvironmentVariables; + child.JobDefaults = JobDefaults; child.Scopes = Scopes; child.FileTable = FileTable; child.StepsContext = StepsContext; @@ -565,6 +568,9 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // Environment variables shared across all actions EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer); + // Job defaults shared across all actions + JobDefaults = new Dictionary>(StringComparer.OrdinalIgnoreCase); + // Job Outputs JobOutputs = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/Runner.Worker/Handlers/ScriptHandler.cs b/src/Runner.Worker/Handlers/ScriptHandler.cs index ccae1350182..89ac15a030e 100644 --- a/src/Runner.Worker/Handlers/ScriptHandler.cs +++ b/src/Runner.Worker/Handlers/ScriptHandler.cs @@ -58,12 +58,21 @@ public override void PrintActionDetails(ActionRunStage stage) string shellCommandPath = null; bool validateShellOnHost = !(StepHost is ContainerStepHost); string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse()); - Inputs.TryGetValue("shell", out var shell); + string shell = null; + if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell)) + { + // TODO: figure out how defaults interact with template later + // for now, we won't check job.defaults if we are inside a template. + if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults)) + { + runDefaults.TryGetValue("shell", out shell); + } + } if (string.IsNullOrEmpty(shell)) { #if OS_WINDOWS shellCommand = "pwsh"; - if(validateShellOnHost) + if (validateShellOnHost) { shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath); if (string.IsNullOrEmpty(shellCommandPath)) @@ -139,11 +148,36 @@ public async Task RunAsync(ActionRunStage stage) Inputs.TryGetValue("script", out var contents); contents = contents ?? string.Empty; - Inputs.TryGetValue("workingDirectory", out var workingDirectory); + string workingDirectory = null; + if (!Inputs.TryGetValue("workingDirectory", out workingDirectory)) + { + // TODO: figure out how defaults interact with template later + // for now, we won't check job.defaults if we are inside a template. + if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults)) + { + if (runDefaults.TryGetValue("working-directory", out workingDirectory)) + { + ExecutionContext.Debug("Overwrite 'working-directory' base on job defaults."); + } + } + } var workspaceDir = githubContext["workspace"] as StringContextData; workingDirectory = Path.Combine(workspaceDir, workingDirectory ?? string.Empty); - Inputs.TryGetValue("shell", out var shell); + string shell = null; + if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell)) + { + // TODO: figure out how defaults interact with template later + // for now, we won't check job.defaults if we are inside a template. + if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults)) + { + if (runDefaults.TryGetValue("shell", out shell)) + { + ExecutionContext.Debug("Overwrite 'shell' base on job defaults."); + } + } + } + var isContainerStepHost = StepHost is ContainerStepHost; string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse()); diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index c0de945a0d8..343b5d87e4d 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -161,6 +161,26 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } + // Evaluate the job defaults + context.Debug("Evaluating job defaults"); + foreach (var token in message.Defaults) + { + var defaults = token.AssertMapping("defaults"); + if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase))) + { + context.JobDefaults["run"] = new Dictionary(StringComparer.OrdinalIgnoreCase); + var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)); + var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues); + foreach (var pair in jobDefaults) + { + if (!string.IsNullOrEmpty(pair.Value)) + { + context.JobDefaults["run"][pair.Key] = pair.Value; + } + } + } + } + // Build up 2 lists of steps, pre-job, job // Download actions not already in the cache Trace.Info("Downloading actions"); diff --git a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs index 576c3cb6ff7..c94ff59132d 100644 --- a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs +++ b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs @@ -41,7 +41,8 @@ public AgentJobRequestMessage( IEnumerable steps, IEnumerable scopes, IList fileTable, - TemplateToken jobOutputs) + TemplateToken jobOutputs, + IList defaults) { this.MessageType = JobRequestMessageTypes.PipelineAgentJobRequest; this.Plan = plan; @@ -69,6 +70,11 @@ public AgentJobRequestMessage( m_environmentVariables = new List(environmentVariables); } + if (defaults?.Count > 0) + { + m_defaults = new List(defaults); + } + this.ContextData = new Dictionary(StringComparer.OrdinalIgnoreCase); if (contextData?.Count > 0) { @@ -213,6 +219,21 @@ public IList EnvironmentVariables } } + /// + /// Gets the hierarchy of defaults to overlay, last wins. + /// + public IList Defaults + { + get + { + if (m_defaults == null) + { + m_defaults = new List(); + } + return m_defaults; + } + } + /// /// Gets the collection of variables associated with the current context. /// @@ -252,6 +273,9 @@ public IList Scopes } } + /// + /// Gets the table of files used when parsing the pipeline (e.g. yaml files) + /// public IList FileTable { get @@ -372,6 +396,11 @@ private void OnSerializing(StreamingContext context) m_environmentVariables = null; } + if (m_defaults?.Count == 0) + { + m_defaults = null; + } + if (m_fileTable?.Count == 0) { m_fileTable = null; @@ -406,6 +435,9 @@ private void OnSerializing(StreamingContext context) [DataMember(Name = "EnvironmentVariables", EmitDefaultValue = false)] private List m_environmentVariables; + [DataMember(Name = "Defaults", EmitDefaultValue = false)] + private List m_defaults; + [DataMember(Name = "FileTable", EmitDefaultValue = false)] private List m_fileTable; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index 72853f5b2ed..86db411704a 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -14,6 +14,7 @@ public sealed class PipelineTemplateConstants public const String Clean = "clean"; public const String Container = "container"; public const String ContinueOnError = "continue-on-error"; + public const String Defaults = "defaults"; public const String Env = "env"; public const String Event = "event"; public const String EventPattern = "github.event"; @@ -29,6 +30,7 @@ public sealed class PipelineTemplateConstants public const String Include = "include"; public const String Inputs = "inputs"; public const String Job = "job"; + public const String JobDefaultsRun = "job-defaults-run"; public const String JobOutputs = "job-outputs"; public const String Jobs = "jobs"; public const String Labels = "labels"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index ea9193ee16c..d60fcc5296b 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -267,6 +267,42 @@ public Dictionary EvaluateJobOutput( return result; } + public Dictionary EvaluateJobDefaultsRun( + TemplateToken token, + DictionaryContextData contextData) + { + var result = default(Dictionary); + + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(contextData); + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobDefaultsRun, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = new Dictionary(StringComparer.OrdinalIgnoreCase); + var mapping = token.AssertMapping("defaults run"); + foreach (var pair in mapping) + { + // Literal key + var key = pair.Key.AssertString("defaults run key"); + + // Literal value + var value = pair.Value.AssertString("defaults run value"); + result[key.Value] = value.Value; + } + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + + return result; + } + public IList> EvaluateJobServiceContainers( TemplateToken token, DictionaryContextData contextData) diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index 203a686ad00..5c8c4c43e14 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -9,6 +9,7 @@ "properties": { "on": "any", "name": "string", + "defaults": "workflow-defaults", "env": "workflow-env", "jobs": "jobs" } @@ -125,6 +126,23 @@ "string": {} }, + "workflow-defaults": { + "mapping": { + "properties": { + "run": "workflow-defaults-run" + } + } + }, + + "workflow-defaults-run": { + "mapping": { + "properties": { + "shell": "non-empty-string", + "working-directory": "non-empty-string" + } + } + }, + "workflow-env": { "context": [ "github", @@ -161,6 +179,7 @@ "services": "services", "env": "job-env", "outputs": "job-outputs", + "defaults": "job-defaults", "steps": "steps" } } @@ -289,6 +308,30 @@ } }, + "job-defaults": { + "mapping": { + "properties": { + "run": "job-defaults-run" + } + } + }, + + "job-defaults-run": { + "context": [ + "github", + "strategy", + "matrix", + "needs", + "env" + ], + "mapping": { + "properties": { + "shell": "non-empty-string", + "working-directory": "non-empty-string" + } + } + }, + "job-outputs": { "mapping": { "loose-key-type": "non-empty-string", diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index 6ee79360e24..00a7b5155f1 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -33,7 +33,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage() TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); + var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); return result; } diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index 5d8bc3e0f9d..32a21521bd8 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -43,7 +43,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); + return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); } private JobCancelMessage CreateJobCancelMessage() diff --git a/src/Test/L0/Worker/ActionCommandManagerL0.cs b/src/Test/L0/Worker/ActionCommandManagerL0.cs index 2ef9f5c5c3e..568c1a86ad9 100644 --- a/src/Test/L0/Worker/ActionCommandManagerL0.cs +++ b/src/Test/L0/Worker/ActionCommandManagerL0.cs @@ -150,7 +150,7 @@ public void EchoProcessCommandDebugOn() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 190bcda7611..aef52d402eb 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -25,7 +25,7 @@ public void AddIssue_CountWarningsErrors() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -101,7 +101,7 @@ public void Debug_Multilines() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -152,7 +152,7 @@ public void RegisterPostJobAction_ShareState() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index b2c5dbdddd6..bedd24c363e 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -100,7 +100,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " }; Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null); + _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null); GitHubContext github = new GitHubContext(); github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner"); _message.ContextData.Add("github", github); diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index 5746547204a..de09a4c96da 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -63,7 +63,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = new Timeline(Guid.NewGuid()); Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null); + _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); _message.Variables[Constants.Variables.System.Culture] = "en-US"; _message.Resources.Endpoints.Add(new ServiceEndpoint() { diff --git a/src/Test/L0/Worker/WorkerL0.cs b/src/Test/L0/Worker/WorkerL0.cs index ed930f184c6..80e5eaa0aa4 100644 --- a/src/Test/L0/Worker/WorkerL0.cs +++ b/src/Test/L0/Worker/WorkerL0.cs @@ -67,7 +67,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) new Pipelines.ContextData.DictionaryContextData() }, }; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null, null, null); return jobRequest; } From dfaf6e06ee862f94e0daf11ac5e5d197b913080c Mon Sep 17 00:00:00 2001 From: eric sciple Date: Wed, 18 Mar 2020 12:08:51 -0400 Subject: [PATCH 10/86] switch hashFiles to extension function (#362) --- src/Runner.Worker/ActionManifestManager.cs | 40 ++-- src/Runner.Worker/ActionRunner.cs | 20 +- src/Runner.Worker/ExecutionContext.cs | 39 ++-- src/Runner.Worker/ExpressionManager.cs | 162 ------------- .../Expressions/AlwaysFunction.cs | 25 ++ .../Expressions/CancelledFunction.cs | 31 +++ .../Expressions/FailureFunction.cs | 31 +++ .../HashFilesFunction.cs} | 44 ++-- .../Expressions/SuccessFunction.cs | 31 +++ .../Handlers/ContainerActionHandler.cs | 8 +- src/Runner.Worker/JobExtension.cs | 10 +- src/Runner.Worker/StepsRunner.cs | 97 +++++--- src/Runner.Worker/action_yaml.json | 3 +- .../Expressions2/ExpressionConstants.cs | 9 +- .../Expressions2/Sdk/Functions/HashFiles.cs | 122 ---------- .../ObjectTemplating/Schema/Definition.cs | 38 +++- .../Schema/MappingDefinition.cs | 2 +- .../Schema/OneOfDefinition.cs | 2 +- .../ObjectTemplating/TemplateEvaluator.cs | 20 +- .../ObjectTemplating/TemplateException.cs | 8 + .../ObjectTemplating/TemplateReader.cs | 17 +- .../TemplateValidationErrors.cs | 20 +- .../Tokens/ExpressionToken.cs | 43 +++- .../Tokens/TemplateTokenExtensions.cs | 40 ++++ .../PipelineTemplateConstants.cs | 3 + .../PipelineTemplateConverter.cs | 14 ++ .../PipelineTemplateEvaluator.cs | 215 +++++++++++------- .../PipelineTemplateSchemaFactory.cs | 28 ++- src/Sdk/DTPipelines/workflow-v1.0.json | 210 +++++++++++------ src/Test/L0/Worker/ActionManagerL0.cs | 6 +- src/Test/L0/Worker/ActionManifestManagerL0.cs | 30 +-- src/Test/L0/Worker/ActionRunnerL0.cs | 4 +- .../ConditionFunctionsL0.cs} | 83 +++---- src/Test/L0/Worker/JobExtensionL0.cs | 3 - src/Test/L0/Worker/JobRunnerL0.cs | 3 - src/Test/L0/Worker/StepsRunnerL0.cs | 22 +- 36 files changed, 825 insertions(+), 658 deletions(-) delete mode 100644 src/Runner.Worker/ExpressionManager.cs create mode 100644 src/Runner.Worker/Expressions/AlwaysFunction.cs create mode 100644 src/Runner.Worker/Expressions/CancelledFunction.cs create mode 100644 src/Runner.Worker/Expressions/FailureFunction.cs rename src/Runner.Worker/{ExpressionFunctions/HashFiles.cs => Expressions/HashFilesFunction.cs} (87%) create mode 100644 src/Runner.Worker/Expressions/SuccessFunction.cs delete mode 100644 src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/HashFiles.cs rename src/Test/L0/Worker/{ExpressionManagerL0.cs => Expressions/ConditionFunctionsL0.cs} (63%) diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 980b87e17ca..9b94faaf85d 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -22,11 +22,11 @@ public interface IActionManifestManager : IRunnerService { ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile); - List EvaluateContainerArguments(IExecutionContext executionContext, SequenceToken token, IDictionary contextData); + List EvaluateContainerArguments(IExecutionContext executionContext, SequenceToken token, IDictionary extraExpressionValues); - Dictionary EvaluateContainerEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary contextData); + Dictionary EvaluateContainerEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary extraExpressionValues); - string EvaluateDefaultInput(IExecutionContext executionContext, string inputName, TemplateToken token, IDictionary contextData); + string EvaluateDefaultInput(IExecutionContext executionContext, string inputName, TemplateToken token); } public sealed class ActionManifestManager : RunnerService, IActionManifestManager @@ -54,7 +54,7 @@ public override void Initialize(IHostContext hostContext) public ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile) { - var context = CreateContext(executionContext, null); + var context = CreateContext(executionContext); ActionDefinitionData actionDefinition = new ActionDefinitionData(); try { @@ -133,13 +133,13 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani public List EvaluateContainerArguments( IExecutionContext executionContext, SequenceToken token, - IDictionary contextData) + IDictionary extraExpressionValues) { var result = new List(); if (token != null) { - var context = CreateContext(executionContext, contextData); + var context = CreateContext(executionContext, extraExpressionValues); try { var evaluateResult = TemplateEvaluator.Evaluate(context, "container-runs-args", token, 0, null, omitHeader: true); @@ -172,13 +172,13 @@ public List EvaluateContainerArguments( public Dictionary EvaluateContainerEnvironment( IExecutionContext executionContext, MappingToken token, - IDictionary contextData) + IDictionary extraExpressionValues) { var result = new Dictionary(StringComparer.OrdinalIgnoreCase); if (token != null) { - var context = CreateContext(executionContext, contextData); + var context = CreateContext(executionContext, extraExpressionValues); try { var evaluateResult = TemplateEvaluator.Evaluate(context, "container-runs-env", token, 0, null, omitHeader: true); @@ -216,13 +216,12 @@ public Dictionary EvaluateContainerEnvironment( public string EvaluateDefaultInput( IExecutionContext executionContext, string inputName, - TemplateToken token, - IDictionary contextData) + TemplateToken token) { string result = ""; if (token != null) { - var context = CreateContext(executionContext, contextData); + var context = CreateContext(executionContext); try { var evaluateResult = TemplateEvaluator.Evaluate(context, "input-default-context", token, 0, null, omitHeader: true); @@ -247,7 +246,7 @@ public string EvaluateDefaultInput( private TemplateContext CreateContext( IExecutionContext executionContext, - IDictionary contextData) + IDictionary extraExpressionValues = null) { var result = new TemplateContext { @@ -261,14 +260,27 @@ private TemplateContext CreateContext( TraceWriter = executionContext.ToTemplateTraceWriter(), }; - if (contextData?.Count > 0) + // Expression values from execution context + foreach (var pair in executionContext.ExpressionValues) { - foreach (var pair in contextData) + result.ExpressionValues[pair.Key] = pair.Value; + } + + // Extra expression values + if (extraExpressionValues?.Count > 0) + { + foreach (var pair in extraExpressionValues) { result.ExpressionValues[pair.Key] = pair.Value; } } + // Expression functions from execution context + foreach (var item in executionContext.ExpressionFunctions) + { + result.ExpressionFunctions.Add(item); + } + // Add the file table if (_fileTable?.Count > 0) { diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index 3b7e83e95db..7272303bcf3 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -26,7 +26,7 @@ public enum ActionRunStage public interface IActionRunner : IStep, IRunnerService { ActionRunStage Stage { get; set; } - Boolean TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context); + bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context); Pipelines.ActionStep Action { get; set; } } @@ -142,7 +142,7 @@ public async Task RunAsync() // Load the inputs. ExecutionContext.Debug("Loading inputs"); var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); - var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues); + var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions); foreach (KeyValuePair input in inputs) { @@ -162,13 +162,7 @@ public async Task RunAsync() string key = input.Key.AssertString("action input name").Value; if (!inputs.ContainsKey(key)) { - var evaluateContext = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var data in ExecutionContext.ExpressionValues) - { - evaluateContext[data.Key] = data.Value; - } - - inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value, evaluateContext); + inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value); } } } @@ -293,10 +287,14 @@ private string GenerateDisplayName(ActionStep action, DictionaryContextData cont return displayName; } // Try evaluating fully - var templateEvaluator = context.ToPipelineTemplateEvaluator(); try { - didFullyEvaluate = templateEvaluator.TryEvaluateStepDisplayName(tokenToParse, contextData, out displayName); + if (tokenToParse.CheckHasRequiredContext(contextData, context.ExpressionFunctions)) + { + var templateEvaluator = context.ToPipelineTemplateEvaluator(); + displayName = templateEvaluator.EvaluateStepDisplayName(tokenToParse, contextData, context.ExpressionFunctions); + didFullyEvaluate = true; + } } catch (TemplateValidationException e) { diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 6464e080704..25cbeefb10a 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -1,14 +1,15 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Collections.Specialized; using System.Globalization; using System.IO; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using System.Web; -using GitHub.Runner.Worker.Container; -using GitHub.Services.WebApi; +using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.Pipelines.ObjectTemplating; @@ -16,12 +17,11 @@ using GitHub.Runner.Common.Util; using GitHub.Runner.Common; using GitHub.Runner.Sdk; +using GitHub.Runner.Worker.Container; +using GitHub.Services.WebApi; using Newtonsoft.Json; -using System.Text; -using System.Collections; using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; using Pipelines = GitHub.DistributedTask.Pipelines; -using GitHub.DistributedTask.Expressions2; namespace GitHub.Runner.Worker { @@ -55,6 +55,7 @@ public interface IExecutionContext : IRunnerService IList FileTable { get; } StepsContext StepsContext { get; } DictionaryContextData ExpressionValues { get; } + IList ExpressionFunctions { get; } List PrependPath { get; } ContainerInfo Container { get; set; } List ServiceContainers { get; } @@ -148,6 +149,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public IList FileTable { get; private set; } public StepsContext StepsContext { get; private set; } public DictionaryContextData ExpressionValues { get; } = new DictionaryContextData(); + public IList ExpressionFunctions { get; } = new List(); public bool WriteDebug { get; private set; } public List PrependPath { get; private set; } public ContainerInfo Container { get; set; } @@ -280,6 +282,10 @@ public IExecutionContext CreateChild(Guid recordId, string displayName, string r { child.ExpressionValues[pair.Key] = pair.Value; } + foreach (var item in ExpressionFunctions) + { + child.ExpressionFunctions.Add(item); + } child._cancellationTokenSource = new CancellationTokenSource(); child.WriteDebug = WriteDebug; child._parentExecutionContext = this; @@ -593,12 +599,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // File table FileTable = new List(message.FileTable ?? new string[0]); - // Expression functions - if (Variables.GetBoolean("System.HashFilesV2") == true) - { - ExpressionConstants.UpdateFunction("hashFiles", 1, byte.MaxValue); - } - // Expression values if (message.ContextData?.Count > 0) { @@ -915,11 +915,19 @@ public static void Debug(this IExecutionContext context, string message) } } - public static PipelineTemplateEvaluator ToPipelineTemplateEvaluator(this IExecutionContext context) + public static IEnumerable> ToExpressionState(this IExecutionContext context) { - var templateTrace = context.ToTemplateTraceWriter(); - var schema = new PipelineTemplateSchemaFactory().CreateSchema(); - return new PipelineTemplateEvaluator(templateTrace, schema, context.FileTable); + return new[] { new KeyValuePair(nameof(IExecutionContext), context) }; + } + + public static PipelineTemplateEvaluator ToPipelineTemplateEvaluator(this IExecutionContext context, ObjectTemplating.ITraceWriter traceWriter = null) + { + if (traceWriter == null) + { + traceWriter = context.ToTemplateTraceWriter(); + } + var schema = PipelineTemplateSchemaFactory.GetSchema(); + return new PipelineTemplateEvaluator(traceWriter, schema, context.FileTable); } public static ObjectTemplating.ITraceWriter ToTemplateTraceWriter(this IExecutionContext context) @@ -934,6 +942,7 @@ internal sealed class TemplateTraceWriter : ObjectTemplating.ITraceWriter internal TemplateTraceWriter(IExecutionContext executionContext) { + ArgUtil.NotNull(executionContext, nameof(executionContext)); _executionContext = executionContext; } diff --git a/src/Runner.Worker/ExpressionManager.cs b/src/Runner.Worker/ExpressionManager.cs deleted file mode 100644 index b5218a806fd..00000000000 --- a/src/Runner.Worker/ExpressionManager.cs +++ /dev/null @@ -1,162 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using GitHub.DistributedTask.Expressions2; -using GitHub.DistributedTask.Expressions2.Sdk; -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common; -using GitHub.Runner.Common.Util; -using GitHub.Runner.Sdk; -using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; -using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; - -namespace GitHub.Runner.Worker -{ - [ServiceLocator(Default = typeof(ExpressionManager))] - public interface IExpressionManager : IRunnerService - { - ConditionResult Evaluate(IExecutionContext context, string condition, bool hostTracingOnly = false); - } - - public sealed class ExpressionManager : RunnerService, IExpressionManager - { - public ConditionResult Evaluate(IExecutionContext executionContext, string condition, bool hostTracingOnly = false) - { - ArgUtil.NotNull(executionContext, nameof(executionContext)); - - ConditionResult result = new ConditionResult(); - var expressionTrace = new TraceWriter(Trace, hostTracingOnly ? null : executionContext); - var tree = Parse(executionContext, expressionTrace, condition); - var expressionResult = tree.Evaluate(expressionTrace, HostContext.SecretMasker, state: executionContext, options: null); - result.Value = expressionResult.IsTruthy; - result.Trace = expressionTrace.Trace; - - return result; - } - - private static IExpressionNode Parse(IExecutionContext executionContext, TraceWriter expressionTrace, string condition) - { - ArgUtil.NotNull(executionContext, nameof(executionContext)); - - if (string.IsNullOrWhiteSpace(condition)) - { - condition = $"{PipelineTemplateConstants.Success}()"; - } - - var parser = new ExpressionParser(); - var namedValues = executionContext.ExpressionValues.Keys.Select(x => new NamedValueInfo(x)).ToArray(); - var functions = new IFunctionInfo[] - { - new FunctionInfo(name: Constants.Expressions.Always, minParameters: 0, maxParameters: 0), - new FunctionInfo(name: Constants.Expressions.Cancelled, minParameters: 0, maxParameters: 0), - new FunctionInfo(name: Constants.Expressions.Failure, minParameters: 0, maxParameters: 0), - new FunctionInfo(name: Constants.Expressions.Success, minParameters: 0, maxParameters: 0), - }; - return parser.CreateTree(condition, expressionTrace, namedValues, functions) ?? new SuccessNode(); - } - - private sealed class TraceWriter : DistributedTask.Expressions2.ITraceWriter - { - private readonly IExecutionContext _executionContext; - private readonly Tracing _trace; - private readonly StringBuilder _traceBuilder = new StringBuilder(); - - public string Trace => _traceBuilder.ToString(); - - public TraceWriter(Tracing trace, IExecutionContext executionContext) - { - ArgUtil.NotNull(trace, nameof(trace)); - _trace = trace; - _executionContext = executionContext; - } - - public void Info(string message) - { - _trace.Info(message); - _executionContext?.Debug(message); - _traceBuilder.AppendLine(message); - } - - public void Verbose(string message) - { - _trace.Verbose(message); - _executionContext?.Debug(message); - } - } - - private sealed class AlwaysNode : Function - { - protected override Object EvaluateCore(EvaluationContext context, out ResultMemory resultMemory) - { - resultMemory = null; - return true; - } - } - - private sealed class CancelledNode : Function - { - protected sealed override object EvaluateCore(EvaluationContext evaluationContext, out ResultMemory resultMemory) - { - resultMemory = null; - var executionContext = evaluationContext.State as IExecutionContext; - ArgUtil.NotNull(executionContext, nameof(executionContext)); - ActionResult jobStatus = executionContext.JobContext.Status ?? ActionResult.Success; - return jobStatus == ActionResult.Cancelled; - } - } - - private sealed class FailureNode : Function - { - protected sealed override object EvaluateCore(EvaluationContext evaluationContext, out ResultMemory resultMemory) - { - resultMemory = null; - var executionContext = evaluationContext.State as IExecutionContext; - ArgUtil.NotNull(executionContext, nameof(executionContext)); - ActionResult jobStatus = executionContext.JobContext.Status ?? ActionResult.Success; - return jobStatus == ActionResult.Failure; - } - } - - private sealed class SuccessNode : Function - { - protected sealed override object EvaluateCore(EvaluationContext evaluationContext, out ResultMemory resultMemory) - { - resultMemory = null; - var executionContext = evaluationContext.State as IExecutionContext; - ArgUtil.NotNull(executionContext, nameof(executionContext)); - ActionResult jobStatus = executionContext.JobContext.Status ?? ActionResult.Success; - return jobStatus == ActionResult.Success; - } - } - - private sealed class ContextValueNode : NamedValue - { - protected override Object EvaluateCore(EvaluationContext evaluationContext, out ResultMemory resultMemory) - { - resultMemory = null; - var jobContext = evaluationContext.State as IExecutionContext; - ArgUtil.NotNull(jobContext, nameof(jobContext)); - return jobContext.ExpressionValues[Name]; - } - } - } - - public class ConditionResult - { - public ConditionResult(bool value = false, string trace = null) - { - this.Value = value; - this.Trace = trace; - } - - public bool Value { get; set; } - public string Trace { get; set; } - - public static implicit operator ConditionResult(bool value) - { - return new ConditionResult(value); - } - } -} diff --git a/src/Runner.Worker/Expressions/AlwaysFunction.cs b/src/Runner.Worker/Expressions/AlwaysFunction.cs new file mode 100644 index 00000000000..1101e191707 --- /dev/null +++ b/src/Runner.Worker/Expressions/AlwaysFunction.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.Expressions2.Sdk; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; +using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; + +namespace GitHub.Runner.Worker.Expressions +{ + public sealed class AlwaysFunction : Function + { + protected override Object EvaluateCore(EvaluationContext context, out ResultMemory resultMemory) + { + resultMemory = null; + return true; + } + } +} diff --git a/src/Runner.Worker/Expressions/CancelledFunction.cs b/src/Runner.Worker/Expressions/CancelledFunction.cs new file mode 100644 index 00000000000..ae676e8d69b --- /dev/null +++ b/src/Runner.Worker/Expressions/CancelledFunction.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.Expressions2.Sdk; +using GitHub.DistributedTask.ObjectTemplating; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; +using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; + +namespace GitHub.Runner.Worker.Expressions +{ + public sealed class CancelledFunction : Function + { + protected sealed override object EvaluateCore(EvaluationContext evaluationContext, out ResultMemory resultMemory) + { + resultMemory = null; + var templateContext = evaluationContext.State as TemplateContext; + ArgUtil.NotNull(templateContext, nameof(templateContext)); + var executionContext = templateContext.State[nameof(IExecutionContext)] as IExecutionContext; + ArgUtil.NotNull(executionContext, nameof(executionContext)); + ActionResult jobStatus = executionContext.JobContext.Status ?? ActionResult.Success; + return jobStatus == ActionResult.Cancelled; + } + } +} diff --git a/src/Runner.Worker/Expressions/FailureFunction.cs b/src/Runner.Worker/Expressions/FailureFunction.cs new file mode 100644 index 00000000000..4c8aa569e2e --- /dev/null +++ b/src/Runner.Worker/Expressions/FailureFunction.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.Expressions2.Sdk; +using GitHub.DistributedTask.ObjectTemplating; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; +using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; + +namespace GitHub.Runner.Worker.Expressions +{ + public sealed class FailureFunction : Function + { + protected sealed override object EvaluateCore(EvaluationContext evaluationContext, out ResultMemory resultMemory) + { + resultMemory = null; + var templateContext = evaluationContext.State as TemplateContext; + ArgUtil.NotNull(templateContext, nameof(templateContext)); + var executionContext = templateContext.State[nameof(IExecutionContext)] as IExecutionContext; + ArgUtil.NotNull(executionContext, nameof(executionContext)); + ActionResult jobStatus = executionContext.JobContext.Status ?? ActionResult.Success; + return jobStatus == ActionResult.Failure; + } + } +} diff --git a/src/Runner.Worker/ExpressionFunctions/HashFiles.cs b/src/Runner.Worker/Expressions/HashFilesFunction.cs similarity index 87% rename from src/Runner.Worker/ExpressionFunctions/HashFiles.cs rename to src/Runner.Worker/Expressions/HashFilesFunction.cs index 915533ba23b..ecbe00ce2cd 100644 --- a/src/Runner.Worker/ExpressionFunctions/HashFiles.cs +++ b/src/Runner.Worker/Expressions/HashFilesFunction.cs @@ -8,28 +8,9 @@ using System.Threading; using System.Collections.Generic; -namespace GitHub.Runner.Worker.Handlers +namespace GitHub.Runner.Worker.Expressions { - public class FunctionTrace : ITraceWriter - { - private GitHub.DistributedTask.Expressions2.ITraceWriter _trace; - - public FunctionTrace(GitHub.DistributedTask.Expressions2.ITraceWriter trace) - { - _trace = trace; - } - public void Info(string message) - { - _trace.Info(message); - } - - public void Verbose(string message) - { - _trace.Info(message); - } - } - - public sealed class HashFiles : Function + public sealed class HashFilesFunction : Function { protected sealed override Object EvaluateCore( EvaluationContext context, @@ -82,7 +63,7 @@ protected sealed override Object EvaluateCore( string node = Path.Combine(runnerRoot, "externals", "node12", "bin", $"node{IOUtil.ExeExtension}"); string hashFilesScript = Path.Combine(binDir, "hashFiles"); var hashResult = string.Empty; - var p = new ProcessInvoker(new FunctionTrace(context.Trace)); + var p = new ProcessInvoker(new HashFilesTrace(context.Trace)); p.ErrorDataReceived += ((_, data) => { if (!string.IsNullOrEmpty(data.Data) && data.Data.StartsWith("__OUTPUT__") && data.Data.EndsWith("__OUTPUT__")) @@ -122,5 +103,24 @@ protected sealed override Object EvaluateCore( return hashResult; } + + private sealed class HashFilesTrace : ITraceWriter + { + private GitHub.DistributedTask.Expressions2.ITraceWriter _trace; + + public HashFilesTrace(GitHub.DistributedTask.Expressions2.ITraceWriter trace) + { + _trace = trace; + } + public void Info(string message) + { + _trace.Info(message); + } + + public void Verbose(string message) + { + _trace.Info(message); + } + } } } \ No newline at end of file diff --git a/src/Runner.Worker/Expressions/SuccessFunction.cs b/src/Runner.Worker/Expressions/SuccessFunction.cs new file mode 100644 index 00000000000..3d161abb55a --- /dev/null +++ b/src/Runner.Worker/Expressions/SuccessFunction.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.Expressions2.Sdk; +using GitHub.DistributedTask.ObjectTemplating; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; +using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; + +namespace GitHub.Runner.Worker.Expressions +{ + public sealed class SuccessFunction : Function + { + protected sealed override object EvaluateCore(EvaluationContext evaluationContext, out ResultMemory resultMemory) + { + resultMemory = null; + var templateContext = evaluationContext.State as TemplateContext; + ArgUtil.NotNull(templateContext, nameof(templateContext)); + var executionContext = templateContext.State[nameof(IExecutionContext)] as IExecutionContext; + ArgUtil.NotNull(executionContext, nameof(executionContext)); + ActionResult jobStatus = executionContext.JobContext.Status ?? ActionResult.Success; + return jobStatus == ActionResult.Success; + } + } +} diff --git a/src/Runner.Worker/Handlers/ContainerActionHandler.cs b/src/Runner.Worker/Handlers/ContainerActionHandler.cs index a623da9688f..8c4f22602b4 100644 --- a/src/Runner.Worker/Handlers/ContainerActionHandler.cs +++ b/src/Runner.Worker/Handlers/ContainerActionHandler.cs @@ -97,14 +97,14 @@ public async Task RunAsync(ActionRunStage stage) } } - var evaluateContext = new Dictionary(StringComparer.OrdinalIgnoreCase); - evaluateContext["inputs"] = inputsContext; + var extraExpressionValues = new Dictionary(StringComparer.OrdinalIgnoreCase); + extraExpressionValues["inputs"] = inputsContext; var manifestManager = HostContext.GetService(); if (Data.Arguments != null) { container.ContainerEntryPointArgs = ""; - var evaluatedArgs = manifestManager.EvaluateContainerArguments(ExecutionContext, Data.Arguments, evaluateContext); + var evaluatedArgs = manifestManager.EvaluateContainerArguments(ExecutionContext, Data.Arguments, extraExpressionValues); foreach (var arg in evaluatedArgs) { if (!string.IsNullOrEmpty(arg)) @@ -124,7 +124,7 @@ public async Task RunAsync(ActionRunStage stage) if (Data.Environment != null) { - var evaluatedEnv = manifestManager.EvaluateContainerEnvironment(ExecutionContext, Data.Environment, evaluateContext); + var evaluatedEnv = manifestManager.EvaluateContainerEnvironment(ExecutionContext, Data.Environment, extraExpressionValues); foreach (var env in evaluatedEnv) { if (!this.Environment.ContainsKey(env.Key)) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 343b5d87e4d..fe3a1ed24d8 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -132,7 +132,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel var templateEvaluator = context.ToPipelineTemplateEvaluator(); foreach (var token in message.EnvironmentVariables) { - var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer); + var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, jobContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer); foreach (var pair in environmentVariables) { context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty; @@ -142,7 +142,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel // Evaluate the job container context.Debug("Evaluating job container"); - var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues); + var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions); if (container != null) { jobContext.Container = new Container.ContainerInfo(HostContext, container); @@ -150,7 +150,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel // Evaluate the job service containers context.Debug("Evaluating job service containers"); - var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues); + var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues, jobContext.ExpressionFunctions); if (serviceContainers?.Count > 0) { foreach (var pair in serviceContainers) @@ -170,7 +170,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel { context.JobDefaults["run"] = new Dictionary(StringComparer.OrdinalIgnoreCase); var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)); - var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues); + var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues, jobContext.ExpressionFunctions); foreach (var pair in jobDefaults) { if (!string.IsNullOrEmpty(pair.Value)) @@ -337,7 +337,7 @@ public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestM context.ExpressionValues["steps"] = context.StepsContext.GetScope(context.ScopeName); var templateEvaluator = context.ToPipelineTemplateEvaluator(); - var outputs = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues); + var outputs = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues, context.ExpressionFunctions); foreach (var output in outputs) { if (string.IsNullOrEmpty(output.Value)) diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index ec1d7069c2a..3e758c6d53b 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -1,8 +1,6 @@ -using GitHub.DistributedTask.WebApi; -using Pipelines = GitHub.DistributedTask.Pipelines; -using GitHub.Runner.Common.Util; using System; using System.Collections.Generic; +using System.Text; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.Expressions2; @@ -10,8 +8,13 @@ using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.Pipelines.ObjectTemplating; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; +using GitHub.Runner.Worker.Expressions; +using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; +using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker { @@ -63,11 +66,7 @@ public async Task RunAsync(IExecutionContext jobContext) } var step = jobContext.JobSteps.Dequeue(); - IStep nextStep = null; - if (jobContext.JobSteps.Count > 0) - { - nextStep = jobContext.JobSteps.Peek(); - } + var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null; Trace.Info($"Processing step: DisplayName='{step.DisplayName}'"); ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext)); @@ -76,6 +75,13 @@ public async Task RunAsync(IExecutionContext jobContext) // Start step.ExecutionContext.Start(); + // Expression functions + step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo(PipelineTemplateConstants.Always, 0, 0)); + step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo(PipelineTemplateConstants.Cancelled, 0, 0)); + step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo(PipelineTemplateConstants.Failure, 0, 0)); + step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo(PipelineTemplateConstants.Success, 0, 0)); + step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo(PipelineTemplateConstants.HashFiles, 1, byte.MaxValue)); + // Initialize scope if (InitializeScope(step, scopeInputs)) { @@ -99,14 +105,13 @@ public async Task RunAsync(IExecutionContext jobContext) // Evaluate and merge action's env block to env context var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); - var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer); + var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer); foreach (var env in actionEnvironment) { envContext[env.Key] = new StringContextData(env.Value ?? string.Empty); } } - var expressionManager = HostContext.GetService(); try { // Register job cancellation call back only if job cancellation token not been fire before each step run @@ -120,28 +125,29 @@ public async Task RunAsync(IExecutionContext jobContext) jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'."); - ConditionResult conditionReTestResult; + var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only + var conditionReTestResult = false; if (HostContext.RunnerShutdownToken.IsCancellationRequested) { step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown."); - conditionReTestResult = false; } else { try { - conditionReTestResult = expressionManager.Evaluate(step.ExecutionContext, step.Condition, hostTracingOnly: true); + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter); + var condition = new BasicExpressionToken(null, null, null, step.Condition); + conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); } catch (Exception ex) { // Cancel the step since we get exception while re-evaluate step condition. Trace.Info("Caught exception from expression when re-test condition on job cancellation."); step.ExecutionContext.Error(ex); - conditionReTestResult = false; } } - if (!conditionReTestResult.Value) + if (!conditionReTestResult) { // Cancel the step. Trace.Info("Cancel current running step."); @@ -161,34 +167,35 @@ public async Task RunAsync(IExecutionContext jobContext) // Evaluate condition. step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'"); - Exception conditionEvaluateError = null; - ConditionResult conditionResult; + var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext); + var conditionResult = false; + var conditionEvaluateError = default(Exception); if (HostContext.RunnerShutdownToken.IsCancellationRequested) { step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown."); - conditionResult = false; } else { try { - conditionResult = expressionManager.Evaluate(step.ExecutionContext, step.Condition); + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter); + var condition = new BasicExpressionToken(null, null, null, step.Condition); + conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); } catch (Exception ex) { Trace.Info("Caught exception from expression."); Trace.Error(ex); - conditionResult = false; conditionEvaluateError = ex; } } // no evaluate error but condition is false - if (!conditionResult.Value && conditionEvaluateError == null) + if (!conditionResult && conditionEvaluateError == null) { // Condition == false Trace.Info("Skipping step due to condition evaluation."); - CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionResult.Trace); + CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace); } else if (conditionEvaluateError != null) { @@ -248,7 +255,7 @@ private async Task RunStepAsync(IStep step, CancellationToken jobCancellationTok var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); try { - timeoutMinutes = templateEvaluator.EvaluateStepTimeout(step.Timeout, step.ExecutionContext.ExpressionValues); + timeoutMinutes = templateEvaluator.EvaluateStepTimeout(step.Timeout, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions); } catch (Exception ex) { @@ -339,7 +346,7 @@ private async Task RunStepAsync(IStep step, CancellationToken jobCancellationTok var continueOnError = false; try { - continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues); + continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions); } catch (Exception ex) { @@ -392,7 +399,7 @@ private bool InitializeScope(IStep step, Dictionary var inputs = default(DictionaryContextData); try { - inputs = templateEvaluator.EvaluateStepScopeInputs(scope.Inputs, executionContext.ExpressionValues); + inputs = templateEvaluator.EvaluateStepScopeInputs(scope.Inputs, executionContext.ExpressionValues, executionContext.ExpressionFunctions); } catch (Exception ex) { @@ -448,7 +455,7 @@ private void CompleteStep(IStep step, IStep nextStep, TaskResult? result = null, var outputs = default(DictionaryContextData); try { - outputs = templateEvaluator.EvaluateStepScopeOutputs(scope.Outputs, executionContext.ExpressionValues); + outputs = templateEvaluator.EvaluateStepScopeOutputs(scope.Outputs, executionContext.ExpressionValues, executionContext.ExpressionFunctions); } catch (Exception ex) { @@ -476,5 +483,43 @@ private void CompleteStep(IStep step, IStep nextStep, TaskResult? result = null, executionContext.Complete(result, resultCode: resultCode); } + + private sealed class ConditionTraceWriter : ObjectTemplating::ITraceWriter + { + private readonly IExecutionContext _executionContext; + private readonly Tracing _trace; + private readonly StringBuilder _traceBuilder = new StringBuilder(); + + public string Trace => _traceBuilder.ToString(); + + public ConditionTraceWriter(Tracing trace, IExecutionContext executionContext) + { + ArgUtil.NotNull(trace, nameof(trace)); + _trace = trace; + _executionContext = executionContext; + } + + public void Error(string format, params Object[] args) + { + var message = StringUtil.Format(format, args); + _trace.Error(message); + _executionContext?.Debug(message); + } + + public void Info(string format, params Object[] args) + { + var message = StringUtil.Format(format, args); + _trace.Info(message); + _executionContext?.Debug(message); + _traceBuilder.AppendLine(message); + } + + public void Verbose(string format, params Object[] args) + { + var message = StringUtil.Format(format, args); + _trace.Verbose(message); + _executionContext?.Debug(message); + } + } } } diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index c9eb2d38d2b..10e691694c1 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -91,7 +91,8 @@ "strategy", "matrix", "job", - "runner" + "runner", + "hashFiles(1,255)" ], "string": {} }, diff --git a/src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs b/src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs index 7974c85bc03..99e19debf52 100644 --- a/src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs +++ b/src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs @@ -5,7 +5,7 @@ namespace GitHub.DistributedTask.Expressions2 { - public static class ExpressionConstants + internal static class ExpressionConstants { static ExpressionConstants() { @@ -16,7 +16,6 @@ static ExpressionConstants() AddFunction("startsWith", 2, 2); AddFunction("toJson", 1, 1); AddFunction("fromJson", 1, 1); - AddFunction("hashFiles", 1, 1); } private static void AddFunction(String name, Int32 minParameters, Int32 maxParameters) @@ -25,12 +24,6 @@ private static void AddFunction(String name, Int32 minParameters, Int32 maxPa WellKnownFunctions.Add(name, new FunctionInfo(name, minParameters, maxParameters)); } - public static void UpdateFunction(String name, Int32 minParameters, Int32 maxParameters) - where T : Function, new() - { - WellKnownFunctions[name] = new FunctionInfo(name, minParameters, maxParameters); - } - internal static readonly String False = "false"; internal static readonly String Infinity = "Infinity"; internal static readonly Int32 MaxDepth = 50; diff --git a/src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/HashFiles.cs b/src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/HashFiles.cs deleted file mode 100644 index 82862e000bc..00000000000 --- a/src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/HashFiles.cs +++ /dev/null @@ -1,122 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Minimatch; -using System.IO; -using System.Security.Cryptography; -using GitHub.DistributedTask.Expressions2.Sdk; -using GitHub.DistributedTask.Pipelines.ContextData; -using GitHub.DistributedTask.Pipelines.ObjectTemplating; -namespace GitHub.DistributedTask.Expressions2.Sdk.Functions -{ - internal sealed class HashFiles : Function - { - protected sealed override Object EvaluateCore( - EvaluationContext context, - out ResultMemory resultMemory) - { - resultMemory = null; - - // hashFiles() only works on the runner and only works with files under GITHUB_WORKSPACE - // Since GITHUB_WORKSPACE is set by runner, I am using that as the fact of this code runs on server or runner. - if (context.State is ObjectTemplating.TemplateContext templateContext && - templateContext.ExpressionValues.TryGetValue(PipelineTemplateConstants.GitHub, out var githubContextData) && - githubContextData is DictionaryContextData githubContext && - githubContext.TryGetValue(PipelineTemplateConstants.Workspace, out var workspace) == true && - workspace is StringContextData workspaceData) - { - string searchRoot = workspaceData.Value; - string pattern = Parameters[0].Evaluate(context).ConvertToString(); - - // Convert slashes on Windows - if (s_isWindows) - { - pattern = pattern.Replace('\\', '/'); - } - - // Root the pattern - if (!Path.IsPathRooted(pattern)) - { - var patternRoot = s_isWindows ? searchRoot.Replace('\\', '/').TrimEnd('/') : searchRoot.TrimEnd('/'); - pattern = string.Concat(patternRoot, "/", pattern); - } - - // Get all files - context.Trace.Info($"Search root directory: '{searchRoot}'"); - context.Trace.Info($"Search pattern: '{pattern}'"); - var files = Directory.GetFiles(searchRoot, "*", SearchOption.AllDirectories) - .Select(x => s_isWindows ? x.Replace('\\', '/') : x) - .OrderBy(x => x, StringComparer.Ordinal) - .ToList(); - if (files.Count == 0) - { - throw new ArgumentException($"hashFiles('{ExpressionUtility.StringEscape(pattern)}') failed. Directory '{searchRoot}' is empty"); - } - else - { - context.Trace.Info($"Found {files.Count} files"); - } - - // Match - var matcher = new Minimatcher(pattern, s_minimatchOptions); - files = matcher.Filter(files) - .Select(x => s_isWindows ? x.Replace('/', '\\') : x) - .ToList(); - if (files.Count == 0) - { - throw new ArgumentException($"hashFiles('{ExpressionUtility.StringEscape(pattern)}') failed. Search pattern '{pattern}' doesn't match any file under '{searchRoot}'"); - } - else - { - context.Trace.Info($"{files.Count} matches to hash"); - } - - // Hash each file - List filesSha256 = new List(); - foreach (var file in files) - { - context.Trace.Info($"Hash {file}"); - using (SHA256 sha256hash = SHA256.Create()) - { - using (var fileStream = File.OpenRead(file)) - { - filesSha256.AddRange(sha256hash.ComputeHash(fileStream)); - } - } - } - - // Hash the hashes - using (SHA256 sha256hash = SHA256.Create()) - { - var hashBytes = sha256hash.ComputeHash(filesSha256.ToArray()); - StringBuilder hashString = new StringBuilder(); - for (int i = 0; i < hashBytes.Length; i++) - { - hashString.Append(hashBytes[i].ToString("x2")); - } - var result = hashString.ToString(); - context.Trace.Info($"Final hash result: '{result}'"); - return result; - } - } - else - { - throw new InvalidOperationException("'hashfiles' expression function is only supported under runner context."); - } - } - - private static readonly bool s_isWindows = Environment.OSVersion.Platform != PlatformID.Unix && Environment.OSVersion.Platform != PlatformID.MacOSX; - - // Only support basic globbing (* ? and []) and globstar (**) - private static readonly Options s_minimatchOptions = new Options - { - Dot = true, - NoBrace = true, - NoCase = s_isWindows, - NoComment = true, - NoExt = true, - NoNegate = true, - }; - } -} \ No newline at end of file diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/Definition.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/Definition.cs index 259724c2d76..e74656fee87 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/Definition.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/Definition.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using GitHub.DistributedTask.ObjectTemplating.Tokens; @@ -22,10 +23,27 @@ protected Definition(MappingToken definition) { var context = definition[i].Value.AssertSequence($"{TemplateConstants.Context}"); definition.RemoveAt(i); - Context = context - .Select(x => x.AssertString($"{TemplateConstants.Context} item").Value) - .Distinct() - .ToArray(); + var readerContext = new HashSet(StringComparer.OrdinalIgnoreCase); + var evaluatorContext = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (TemplateToken item in context) + { + var itemStr = item.AssertString($"{TemplateConstants.Context} item").Value; + readerContext.Add(itemStr); + + // Remove min/max parameter info + var paramIndex = itemStr.IndexOf('('); + if (paramIndex > 0) + { + evaluatorContext.Add(String.Concat(itemStr.Substring(0, paramIndex + 1), ")")); + } + else + { + evaluatorContext.Add(itemStr); + } + } + + ReaderContext = readerContext.ToArray(); + EvaluatorContext = evaluatorContext.ToArray(); } else if (String.Equals(definitionKey.Value, TemplateConstants.Description, StringComparison.Ordinal)) { @@ -40,7 +58,17 @@ protected Definition(MappingToken definition) internal abstract DefinitionType DefinitionType { get; } - internal String[] Context { get; private set; } = new String[0]; + /// + /// Used by the template reader to determine allowed expression values and functions. + /// Also used by the template reader to validate function min/max parameters. + /// + internal String[] ReaderContext { get; private set; } = new String[0]; + + /// + /// Used by the template evaluator to determine allowed expression values and functions. + /// The min/max parameter info is omitted. + /// + internal String[] EvaluatorContext { get; private set; } = new String[0]; internal abstract void Validate( TemplateSchema schema, diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs index 3da980185ca..2d63c4008cd 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs @@ -108,7 +108,7 @@ internal override void Validate( { var inherited = schema.GetDefinition(Inherits); - if (inherited.Context.Length > 0) + if (inherited.ReaderContext.Length > 0) { throw new NotSupportedException($"Property '{TemplateConstants.Context}' is not supported on inhertied definitions"); } diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/OneOfDefinition.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/OneOfDefinition.cs index 200933ebf6c..671f13fb59a 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/OneOfDefinition.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/OneOfDefinition.cs @@ -62,7 +62,7 @@ internal override void Validate( { var nestedDefinition = schema.GetDefinition(nestedType); - if (nestedDefinition.Context.Length > 0) + if (nestedDefinition.ReaderContext.Length > 0) { throw new ArgumentException($"'{name}' is a one-of definition and references another definition that defines context. This is currently not supported."); } diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs index 63f5163194f..915fc3cbc2b 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs @@ -47,7 +47,16 @@ internal static TemplateToken Evaluate( var evaluator = new TemplateEvaluator(context, template, removeBytes); try { - var availableContext = new HashSet(context.ExpressionValues.Keys.Concat(context.ExpressionFunctions.Select(x => $"{x.Name}({x.MinParameters},{x.MaxParameters})"))); + var availableContext = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var key in context.ExpressionValues.Keys) + { + availableContext.Add(key); + } + foreach (var function in context.ExpressionFunctions) + { + availableContext.Add($"{function.Name}()"); + } + var definitionInfo = new DefinitionInfo(context.Schema, type, availableContext); result = evaluator.Evaluate(definitionInfo); @@ -393,14 +402,13 @@ public DefinitionInfo( Definition = m_schema.GetDefinition(name); // Determine whether to expand - if (Definition.Context.Length > 0) + m_allowedContext = Definition.EvaluatorContext; + if (Definition.EvaluatorContext.Length > 0) { - m_allowedContext = Definition.Context; Expand = m_availableContext.IsSupersetOf(m_allowedContext); } else { - m_allowedContext = new String[0]; Expand = false; } } @@ -416,9 +424,9 @@ public DefinitionInfo( Definition = m_schema.GetDefinition(name); // Determine whether to expand - if (Definition.Context.Length > 0) + if (Definition.EvaluatorContext.Length > 0) { - m_allowedContext = new HashSet(parent.m_allowedContext.Concat(Definition.Context)).ToArray(); + m_allowedContext = new HashSet(parent.m_allowedContext.Concat(Definition.EvaluatorContext), StringComparer.OrdinalIgnoreCase).ToArray(); Expand = m_availableContext.IsSupersetOf(m_allowedContext); } else diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateException.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateException.cs index cc9d57c691f..835b75ebd80 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateException.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateException.cs @@ -49,6 +49,14 @@ public TemplateValidationException(IEnumerable errors) m_errors = new List(errors ?? Enumerable.Empty()); } + public TemplateValidationException( + String message, + IEnumerable errors) + : this(message) + { + m_errors = new List(errors ?? Enumerable.Empty()); + } + public TemplateValidationException(String message) : base(message) { diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs index eab601bc09f..886bea4c3d2 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs @@ -780,15 +780,8 @@ public DefinitionInfo( // Lookup the definition Definition = m_schema.GetDefinition(name); - // Determine whether to expand - if (Definition.Context.Length > 0) - { - AllowedContext = Definition.Context; - } - else - { - AllowedContext = new String[0]; - } + // Record allowed context + AllowedContext = Definition.ReaderContext; } public DefinitionInfo( @@ -800,10 +793,10 @@ public DefinitionInfo( // Lookup the definition Definition = m_schema.GetDefinition(name); - // Determine whether to expand - if (Definition.Context.Length > 0) + // Record allowed context + if (Definition.ReaderContext.Length > 0) { - AllowedContext = new HashSet(parent.AllowedContext.Concat(Definition.Context)).ToArray(); + AllowedContext = new HashSet(parent.AllowedContext.Concat(Definition.ReaderContext), StringComparer.OrdinalIgnoreCase).ToArray(); } else { diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateValidationErrors.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateValidationErrors.cs index 4b1e738d0e3..4ada3c8e610 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateValidationErrors.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateValidationErrors.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.ComponentModel; +using System.Linq; using System.Runtime.Serialization; namespace GitHub.DistributedTask.ObjectTemplating @@ -41,7 +42,7 @@ public void Add(String messagePrefix, Exception ex) { for (int i = 0; i < 50; i++) { - String message = !String.IsNullOrEmpty(messagePrefix) ? $"{messagePrefix} {ex.Message}" : ex.Message; + String message = !String.IsNullOrEmpty(messagePrefix) ? $"{messagePrefix} {ex.Message}" : ex.ToString(); Add(new TemplateValidationError(message)); if (ex.InnerException == null) { @@ -88,6 +89,23 @@ public void Check() } } + /// + /// Throws if any errors. + /// The error message prefix + /// + public void Check(String prefix) + { + if (String.IsNullOrEmpty(prefix)) + { + this.Check(); + } + else if (m_errors.Count > 0) + { + var message = $"{prefix.Trim()} {String.Join(",", m_errors.Select(e => e.Message))}"; + throw new TemplateValidationException(message, m_errors); + } + } + public void Clear() { m_errors.Clear(); diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/ExpressionToken.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/ExpressionToken.cs index 0709e236cd7..11f5f1bbfdd 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/ExpressionToken.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/ExpressionToken.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Generic; using System.ComponentModel; -using System.Linq; +using System.Globalization; using System.Runtime.Serialization; +using System.Text.RegularExpressions; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Expressions2.Sdk; using GitHub.Services.WebApi.Internal; @@ -35,11 +37,29 @@ internal static Boolean IsValidExpression( String[] allowedContext, out Exception ex) { - // Create dummy allowed contexts - INamedValueInfo[] namedValues = null; + // Create dummy named values and functions + var namedValues = new List(); + var functions = new List(); if (allowedContext?.Length > 0) { - namedValues = allowedContext.Select(x => new NamedValueInfo(x)).ToArray(); + foreach (var contextItem in allowedContext) + { + var match = s_function.Match(contextItem); + if (match.Success) + { + var functionName = match.Groups[1].Value; + var minParameters = Int32.Parse(match.Groups[2].Value, NumberStyles.None, CultureInfo.InvariantCulture); + var maxParametersRaw = match.Groups[3].Value; + var maxParameters = String.Equals(maxParametersRaw, TemplateConstants.MaxConstant, StringComparison.Ordinal) + ? Int32.MaxValue + : Int32.Parse(maxParametersRaw, NumberStyles.None, CultureInfo.InvariantCulture); + functions.Add(new FunctionInfo(functionName, minParameters, maxParameters)); + } + else + { + namedValues.Add(new NamedValueInfo(contextItem)); + } + } } // Parse @@ -47,7 +67,7 @@ internal static Boolean IsValidExpression( ExpressionNode root = null; try { - root = new ExpressionParser().CreateTree(expression, null, namedValues, null) as ExpressionNode; + root = new ExpressionParser().CreateTree(expression, null, namedValues, functions) as ExpressionNode; result = true; ex = null; @@ -60,5 +80,18 @@ internal static Boolean IsValidExpression( return result; } + + private sealed class DummyFunction : Function + { + protected override Object EvaluateCore( + EvaluationContext context, + out ResultMemory resultMemory) + { + resultMemory = null; + return null; + } + } + + private static readonly Regex s_function = new Regex(@"^([a-zA-Z0-9_]+)\(([0-9]+),([0-9]+|MAX)\)$", RegexOptions.Compiled); } } diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenExtensions.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenExtensions.cs index 7b368404e81..c8c0eabb7c4 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenExtensions.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenExtensions.cs @@ -1,5 +1,8 @@ using System; using System.Collections.Generic; +using System.Linq; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.Expressions2.Sdk; namespace GitHub.DistributedTask.ObjectTemplating.Tokens { @@ -106,6 +109,43 @@ internal static void AssertUnexpectedValue( throw new ArgumentException($"Error while reading '{objectDescription}'. Unexpected value '{literal.ToString()}'"); } + /// + /// Traverses the token and checks whether all required expression values + /// and functions are provided. + /// + public static bool CheckHasRequiredContext( + this TemplateToken token, + IReadOnlyObject expressionValues, + IList expressionFunctions) + { + var expressionTokens = token.Traverse() + .OfType() + .ToArray(); + var parser = new ExpressionParser(); + foreach (var expressionToken in expressionTokens) + { + var tree = parser.ValidateSyntax(expressionToken.Expression, null); + foreach (var node in tree.Traverse()) + { + if (node is NamedValue namedValue) + { + if (expressionValues?.Keys.Any(x => string.Equals(x, namedValue.Name, StringComparison.OrdinalIgnoreCase)) != true) + { + return false; + } + } + else if (node is Function function && + !ExpressionConstants.WellKnownFunctions.ContainsKey(function.Name) && + expressionFunctions?.Any(x => string.Equals(x.Name, function.Name, StringComparison.OrdinalIgnoreCase)) != true) + { + return false; + } + } + } + + return true; + } + /// /// Returns all tokens (depth first) /// diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index 86db411704a..464cd9aad38 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -24,6 +24,7 @@ public sealed class PipelineTemplateConstants public const String FetchDepth = "fetch-depth"; public const String GeneratedId = "generated-id"; public const String GitHub = "github"; + public const String HashFiles = "hashFiles"; public const String Id = "id"; public const String If = "if"; public const String Image = "image"; @@ -31,6 +32,7 @@ public sealed class PipelineTemplateConstants public const String Inputs = "inputs"; public const String Job = "job"; public const String JobDefaultsRun = "job-defaults-run"; + public const String JobIfResult = "job-if-result"; public const String JobOutputs = "job-outputs"; public const String Jobs = "jobs"; public const String Labels = "labels"; @@ -60,6 +62,7 @@ public sealed class PipelineTemplateConstants public const String Shell = "shell"; public const String Skipped = "skipped"; public const String StepEnv = "step-env"; + public const String StepIfResult = "step-if-result"; public const String Steps = "steps"; public const String StepsScopeInputs = "steps-scope-inputs"; public const String StepsScopeOutputs = "steps-scope-outputs"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 951d0869f4e..43be43d3375 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -16,6 +16,20 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating { internal static class PipelineTemplateConverter { + internal static Boolean ConvertToIfResult( + TemplateContext context, + TemplateToken ifResult) + { + var expression = ifResult.Traverse().FirstOrDefault(x => x is ExpressionToken); + if (expression != null) + { + throw new ArgumentException($"Unexpected type '{expression.GetType().Name}' encountered while reading 'if'."); + } + + var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult); + return evaluationResult.IsTruthy; + } + internal static Boolean? ConvertToStepContinueOnError( TemplateContext context, TemplateToken token, diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index d60fcc5296b..a36f5b7e3aa 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Threading; using GitHub.DistributedTask.Expressions2; -using GitHub.DistributedTask.Expressions2.Sdk; +using GitHub.DistributedTask.Expressions2.Sdk.Functions; using GitHub.DistributedTask.ObjectTemplating; using GitHub.DistributedTask.ObjectTemplating.Schema; using GitHub.DistributedTask.ObjectTemplating.Tokens; @@ -14,6 +14,9 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating { + /// + /// Evaluates parts of the workflow DOM. For example, a job strategy or step inputs. + /// [EditorBrowsable(EditorBrowsableState.Never)] public class PipelineTemplateEvaluator { @@ -50,13 +53,14 @@ public PipelineTemplateEvaluator( public DictionaryContextData EvaluateStepScopeInputs( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(DictionaryContextData); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsScopeInputs, token, 0, null, omitHeader: true); @@ -76,13 +80,14 @@ public DictionaryContextData EvaluateStepScopeInputs( public DictionaryContextData EvaluateStepScopeOutputs( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(DictionaryContextData); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsScopeOutputs, token, 0, null, omitHeader: true); @@ -102,13 +107,14 @@ public DictionaryContextData EvaluateStepScopeOutputs( public Boolean EvaluateStepContinueOnError( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(Boolean?); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.BooleanStepsContext, token, 0, null, omitHeader: true); @@ -126,16 +132,44 @@ public Boolean EvaluateStepContinueOnError( return result ?? false; } + public String EvaluateStepDisplayName( + TemplateToken token, + DictionaryContextData contextData, + IList expressionFunctions) + { + var result = default(String); + + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(contextData, expressionFunctions); + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StringStepsContext, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = PipelineTemplateConverter.ConvertToStepDisplayName(context, token); + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + + return result; + } + public Dictionary EvaluateStepEnvironment( TemplateToken token, DictionaryContextData contextData, + IList expressionFunctions, StringComparer keyComparer) { var result = default(Dictionary); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepEnv, token, 0, null, omitHeader: true); @@ -153,15 +187,44 @@ public Dictionary EvaluateStepEnvironment( return result ?? new Dictionary(keyComparer); } + public Boolean EvaluateStepIf( + TemplateToken token, + DictionaryContextData contextData, + IList expressionFunctions, + IEnumerable> expressionState) + { + var result = default(Boolean?); + + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(contextData, expressionFunctions, expressionState); + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepIfResult, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = PipelineTemplateConverter.ConvertToIfResult(context, token); + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + + return result ?? throw new InvalidOperationException("Step if cannot be null"); + } + public Dictionary EvaluateStepInputs( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(Dictionary); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepWith, token, 0, null, omitHeader: true); @@ -181,13 +244,14 @@ public Dictionary EvaluateStepInputs( public Int32 EvaluateStepTimeout( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(Int32?); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.NumberStepsContext, token, 0, null, omitHeader: true); @@ -207,13 +271,14 @@ public Int32 EvaluateStepTimeout( public JobContainer EvaluateJobContainer( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(JobContainer); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Container, token, 0, null, omitHeader: true); @@ -233,13 +298,14 @@ public JobContainer EvaluateJobContainer( public Dictionary EvaluateJobOutput( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(Dictionary); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobOutputs, token, 0, null, omitHeader: true); @@ -269,13 +335,14 @@ public Dictionary EvaluateJobOutput( public Dictionary EvaluateJobDefaultsRun( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(Dictionary); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobDefaultsRun, token, 0, null, omitHeader: true); @@ -305,13 +372,14 @@ public Dictionary EvaluateJobDefaultsRun( public IList> EvaluateJobServiceContainers( TemplateToken token, - DictionaryContextData contextData) + DictionaryContextData contextData, + IList expressionFunctions) { var result = default(List>); if (token != null && token.Type != TokenType.Null) { - var context = CreateContext(contextData); + var context = CreateContext(contextData, expressionFunctions); try { token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Services, token, 0, null, omitHeader: true); @@ -329,62 +397,10 @@ public IList> EvaluateJobServiceContainers( return result; } - public Boolean TryEvaluateStepDisplayName( - TemplateToken token, + private TemplateContext CreateContext( DictionaryContextData contextData, - out String stepName) - { - stepName = default(String); - var context = CreateContext(contextData); - - if (token != null && token.Type != TokenType.Null) - { - // We should only evaluate basic expressions if we are sure we have context on all the Named Values and functions - // Otherwise return and use a default name - if (token is BasicExpressionToken expressionToken) - { - ExpressionNode root = null; - try - { - root = new ExpressionParser().ValidateSyntax(expressionToken.Expression, null) as ExpressionNode; - } - catch (Exception exception) - { - context.Errors.Add(exception); - context.Errors.Check(); - } - foreach (var node in root.Traverse()) - { - if (node is NamedValue namedValue && !contextData.ContainsKey(namedValue.Name)) - { - return false; - } - else if (node is Function function && - !context.ExpressionFunctions.Any(item => String.Equals(item.Name, function.Name)) && - !ExpressionConstants.WellKnownFunctions.ContainsKey(function.Name)) - { - return false; - } - } - } - - try - { - token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StringStepsContext, token, 0, null, omitHeader: true); - context.Errors.Check(); - stepName = PipelineTemplateConverter.ConvertToStepDisplayName(context, token); - } - catch (Exception ex) when (!(ex is TemplateValidationException)) - { - context.Errors.Add(ex); - } - - context.Errors.Check(); - } - return true; - } - - private TemplateContext CreateContext(DictionaryContextData contextData) + IList expressionFunctions, + IEnumerable> expressionState = null) { var result = new TemplateContext { @@ -407,7 +423,7 @@ private TemplateContext CreateContext(DictionaryContextData contextData) } } - // Add named context + // Add named values if (contextData != null) { foreach (var pair in contextData) @@ -416,14 +432,46 @@ private TemplateContext CreateContext(DictionaryContextData contextData) } } - // Compat for new agent against old server - foreach (var name in s_contextNames) + // Add functions + var functionNames = new HashSet(StringComparer.OrdinalIgnoreCase); + if (expressionFunctions?.Count > 0) + { + foreach (var function in expressionFunctions) + { + result.ExpressionFunctions.Add(function); + functionNames.Add(function.Name); + } + } + + // Add missing expression values and expression functions. + // This solves the following problems: + // - Compat for new agent against old server (new contexts not sent down in job message) + // - Evaluating early when all referenced contexts are available, even though all allowed + // contexts may not yet be available. For example, evaluating step display name can often + // be performed early. + foreach (var name in s_expressionValueNames) { if (!result.ExpressionValues.ContainsKey(name)) { result.ExpressionValues[name] = null; } } + foreach (var name in s_expressionFunctionNames) + { + if (!functionNames.Contains(name)) + { + result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + } + } + + // Add state + if (expressionState != null) + { + foreach (var pair in expressionState) + { + result.State[pair.Key] = pair.Value; + } + } return result; } @@ -431,9 +479,10 @@ private TemplateContext CreateContext(DictionaryContextData contextData) private readonly ITraceWriter m_trace; private readonly TemplateSchema m_schema; private readonly IList m_fileTable; - private readonly String[] s_contextNames = new[] + private readonly String[] s_expressionValueNames = new[] { PipelineTemplateConstants.GitHub, + PipelineTemplateConstants.Needs, PipelineTemplateConstants.Strategy, PipelineTemplateConstants.Matrix, PipelineTemplateConstants.Needs, @@ -444,5 +493,13 @@ private TemplateContext CreateContext(DictionaryContextData contextData) PipelineTemplateConstants.Runner, PipelineTemplateConstants.Env, }; + private readonly String[] s_expressionFunctionNames = new[] + { + PipelineTemplateConstants.Always, + PipelineTemplateConstants.Cancelled, + PipelineTemplateConstants.Failure, + PipelineTemplateConstants.HashFiles, + PipelineTemplateConstants.Success, + }; } } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateSchemaFactory.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateSchemaFactory.cs index 55db1ea13f4..47048322f2a 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateSchemaFactory.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateSchemaFactory.cs @@ -2,25 +2,35 @@ using System.ComponentModel; using System.IO; using System.Reflection; +using System.Threading; +using System.Threading.Tasks; using GitHub.DistributedTask.ObjectTemplating.Schema; namespace GitHub.DistributedTask.Pipelines.ObjectTemplating { [EditorBrowsable(EditorBrowsableState.Never)] - public sealed class PipelineTemplateSchemaFactory + public static class PipelineTemplateSchemaFactory { - public TemplateSchema CreateSchema() + public static TemplateSchema GetSchema() { - var assembly = Assembly.GetExecutingAssembly(); - var json = default(String); - using (var stream = assembly.GetManifestResourceStream("GitHub.DistributedTask.Pipelines.ObjectTemplating.workflow-v1.0.json")) - using (var streamReader = new StreamReader(stream)) + if (s_schema == null) { - json = streamReader.ReadToEnd(); + var assembly = Assembly.GetExecutingAssembly(); + var json = default(String); + using (var stream = assembly.GetManifestResourceStream("GitHub.DistributedTask.Pipelines.ObjectTemplating.workflow-v1.0.json")) + using (var streamReader = new StreamReader(stream)) + { + json = streamReader.ReadToEnd(); + } + + var objectReader = new JsonObjectReader(null, json); + var schema = TemplateSchema.Load(objectReader); + Interlocked.CompareExchange(ref s_schema, schema, null); } - var objectReader = new JsonObjectReader(null, json); - return TemplateSchema.Load(objectReader); + return s_schema; } + + private static TemplateSchema s_schema; } } diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index 5c8c4c43e14..f1e364c5cc7 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -38,8 +38,8 @@ "steps-scope-input-value": { "context": [ "github", - "strategy", "needs", + "strategy", "matrix", "secrets", "steps", @@ -66,9 +66,9 @@ "steps-scope-output-value": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "inputs", @@ -91,9 +91,9 @@ "description": "Default input values for a steps template", "context": [ "github", + "needs", "strategy", - "matrix", - "needs" + "matrix" ], "one-of": [ "string", @@ -114,9 +114,9 @@ "description": "Output values for a steps template", "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "job", @@ -204,6 +204,25 @@ "string": {} }, + "job-if-result": { + "context": [ + "github", + "needs", + "always(0,0)", + "failure(0,MAX)", + "cancelled(0,0)", + "success(0,MAX)" + ], + "one-of": [ + "null", + "boolean", + "number", + "string", + "sequence", + "mapping" + ] + }, + "strategy": { "context": [ "github", @@ -272,9 +291,9 @@ "runs-on": { "context": [ "github", + "needs", "strategy", - "matrix", - "needs" + "matrix" ], "one-of": [ "non-empty-string", @@ -297,10 +316,10 @@ "job-env": { "context": [ "github", - "secrets", + "needs", "strategy", "matrix", - "needs" + "secrets" ], "mapping": { "loose-key-type": "non-empty-string", @@ -444,9 +463,9 @@ "step-if": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "steps", "job", "runner", @@ -454,7 +473,8 @@ "always(0,0)", "failure(0,0)", "cancelled(0,0)", - "success(0,0)" + "success(0,0)", + "hashFiles(1,255)" ], "string": {} }, @@ -462,9 +482,9 @@ "step-if-in-template": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "steps", "inputs", "job", @@ -473,11 +493,63 @@ "always(0,0)", "failure(0,0)", "cancelled(0,0)", - "success(0,0)" + "success(0,0)", + "hashFiles(1,255)" ], "string": {} }, + "step-if-result": { + "context": [ + "github", + "strategy", + "matrix", + "steps", + "job", + "runner", + "env", + "always(0,0)", + "failure(0,0)", + "cancelled(0,0)", + "success(0,0)", + "hashFiles(1,255)" + ], + "one-of": [ + "null", + "boolean", + "number", + "string", + "sequence", + "mapping" + ] + }, + + "step-if-result-in-template": { + "context": [ + "github", + "strategy", + "matrix", + "steps", + "inputs", + "job", + "runner", + "env", + "always(0,0)", + "failure(0,0)", + "cancelled(0,0)", + "success(0,0)", + "hashFiles(1,255)" + ], + "one-of": [ + "null", + "boolean", + "number", + "string", + "sequence", + "mapping" + ] + }, + "steps-template-reference": { "mapping": { "properties": { @@ -501,9 +573,9 @@ "steps-template-reference-inputs": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "job", @@ -519,9 +591,9 @@ "steps-template-reference-inputs-in-template": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "inputs", @@ -538,14 +610,15 @@ "step-env": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "mapping": { "loose-key-type": "non-empty-string", @@ -556,15 +629,16 @@ "step-env-in-template": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "inputs", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "mapping": { "loose-key-type": "non-empty-string", @@ -575,14 +649,35 @@ "step-with": { "context": [ "github", + "needs", "strategy", "matrix", + "secrets", + "steps", + "job", + "runner", + "env", + "hashFiles(1,255)" + ], + "mapping": { + "loose-key-type": "non-empty-string", + "loose-value-type": "string" + } + }, + + "step-with-in-template": { + "context": [ + "github", "needs", + "strategy", + "matrix", "secrets", "steps", + "inputs", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "mapping": { "loose-key-type": "non-empty-string", @@ -593,9 +688,9 @@ "container": { "context": [ "github", + "needs", "strategy", - "matrix", - "needs" + "matrix" ], "one-of": [ "string", @@ -618,9 +713,9 @@ "services": { "context": [ "github", + "needs", "strategy", - "matrix", - "needs" + "matrix" ], "mapping": { "loose-key-type": "non-empty-string", @@ -631,9 +726,9 @@ "services-container": { "context": [ "github", + "needs", "strategy", - "matrix", - "needs" + "matrix" ], "one-of": [ "non-empty-string", @@ -648,25 +743,6 @@ } }, - "step-with-in-template": { - "context": [ - "github", - "strategy", - "matrix", - "needs", - "secrets", - "steps", - "inputs", - "job", - "runner", - "env" - ], - "mapping": { - "loose-key-type": "non-empty-string", - "loose-value-type": "string" - } - }, - "non-empty-string": { "string": { "require-non-empty": true @@ -682,9 +758,9 @@ "boolean-strategy-context": { "context": [ "github", + "needs", "strategy", - "matrix", - "needs" + "matrix" ], "boolean": {} }, @@ -692,9 +768,9 @@ "number-strategy-context": { "context": [ "github", + "needs", "strategy", - "matrix", - "needs" + "matrix" ], "number": {} }, @@ -702,9 +778,9 @@ "string-strategy-context": { "context": [ "github", + "needs", "strategy", - "matrix", - "needs" + "matrix" ], "string": {} }, @@ -712,14 +788,15 @@ "boolean-steps-context": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "boolean": {} }, @@ -727,15 +804,16 @@ "boolean-steps-context-in-template": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "inputs", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "boolean": {} }, @@ -743,14 +821,15 @@ "number-steps-context": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "number": {} }, @@ -758,15 +837,16 @@ "number-steps-context-in-template": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "inputs", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "number": {} }, @@ -774,9 +854,9 @@ "string-runner-context": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "job", @@ -789,14 +869,15 @@ "string-steps-context": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "string": {} }, @@ -804,15 +885,16 @@ "string-steps-context-in-template": { "context": [ "github", + "needs", "strategy", "matrix", - "needs", "secrets", "steps", "inputs", "job", "runner", - "env" + "env", + "hashFiles(1,255)" ], "string": {} } diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 51e09bff167..b5a2ed5db76 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -1,4 +1,6 @@ -using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common.Util; using GitHub.Runner.Worker; @@ -1600,6 +1602,8 @@ private void Setup([CallerMemberName] string name = "") _ec = new Mock(); _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); _ec.Setup(x => x.Variables).Returns(new Variables(_hc, new Dictionary())); + _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); + _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"[{tag}]{message}"); }); _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, string message) => { _hc.GetTrace().Info($"[{issue.Type}]{issue.Message ?? message}"); }); _ec.Setup(x => x.GetGitHubContext("workspace")).Returns(Path.Combine(_workFolder, "actions", "actions")); diff --git a/src/Test/L0/Worker/ActionManifestManagerL0.cs b/src/Test/L0/Worker/ActionManifestManagerL0.cs index 75761070634..ca789b7f38e 100644 --- a/src/Test/L0/Worker/ActionManifestManagerL0.cs +++ b/src/Test/L0/Worker/ActionManifestManagerL0.cs @@ -1,7 +1,9 @@ +using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Expressions; using Moq; using System; using System.Collections.Generic; @@ -533,26 +535,26 @@ public void Evaluate_Default_Input() var actionManifest = new ActionManifestManager(); actionManifest.Initialize(_hc); - var githubContext = new DictionaryContextData(); - githubContext.Add("ref", new StringContextData("refs/heads/master")); - - var evaluateContext = new Dictionary(StringComparer.OrdinalIgnoreCase); - evaluateContext["github"] = githubContext; - evaluateContext["strategy"] = new DictionaryContextData(); - evaluateContext["matrix"] = new DictionaryContextData(); - evaluateContext["steps"] = new DictionaryContextData(); - evaluateContext["job"] = new DictionaryContextData(); - evaluateContext["runner"] = new DictionaryContextData(); - evaluateContext["env"] = new DictionaryContextData(); + _ec.Object.ExpressionValues["github"] = new DictionaryContextData + { + { "ref", new StringContextData("refs/heads/master") }, + }; + _ec.Object.ExpressionValues["strategy"] = new DictionaryContextData(); + _ec.Object.ExpressionValues["matrix"] = new DictionaryContextData(); + _ec.Object.ExpressionValues["steps"] = new DictionaryContextData(); + _ec.Object.ExpressionValues["job"] = new DictionaryContextData(); + _ec.Object.ExpressionValues["runner"] = new DictionaryContextData(); + _ec.Object.ExpressionValues["env"] = new DictionaryContextData(); + _ec.Object.ExpressionFunctions.Add(new FunctionInfo("hashFiles", 1, 255)); //Act - var result = actionManifest.EvaluateDefaultInput(_ec.Object, "testInput", new StringToken(null, null, null, "defaultValue"), evaluateContext); + var result = actionManifest.EvaluateDefaultInput(_ec.Object, "testInput", new StringToken(null, null, null, "defaultValue")); //Assert Assert.Equal("defaultValue", result); //Act - result = actionManifest.EvaluateDefaultInput(_ec.Object, "testInput", new BasicExpressionToken(null, null, null, "github.ref"), evaluateContext); + result = actionManifest.EvaluateDefaultInput(_ec.Object, "testInput", new BasicExpressionToken(null, null, null, "github.ref")); //Assert Assert.Equal("refs/heads/master", result); @@ -575,6 +577,8 @@ private void Setup([CallerMemberName] string name = "") _ec.Setup(x => x.WriteDebug).Returns(true); _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); _ec.Setup(x => x.Variables).Returns(new Variables(_hc, new Dictionary())); + _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); + _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"{tag}{message}"); }); _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, string message) => { _hc.GetTrace().Info($"[{issue.Type}]{issue.Message ?? message}"); }); } diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index b0d0c0ff4ea..24ff73f4f04 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -1,4 +1,5 @@ -using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; @@ -322,6 +323,7 @@ private void Setup([CallerMemberName] string name = "") _ec = new Mock(); _ec.Setup(x => x.ExpressionValues).Returns(_context); + _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); _ec.Setup(x => x.IntraActionState).Returns(new Dictionary()); _ec.Setup(x => x.EnvironmentVariables).Returns(new Dictionary()); _ec.Setup(x => x.SetGitHubContext(It.IsAny(), It.IsAny())); diff --git a/src/Test/L0/Worker/ExpressionManagerL0.cs b/src/Test/L0/Worker/Expressions/ConditionFunctionsL0.cs similarity index 63% rename from src/Test/L0/Worker/ExpressionManagerL0.cs rename to src/Test/L0/Worker/Expressions/ConditionFunctionsL0.cs index 9bdcdeeeeed..4ffcdc9dc42 100644 --- a/src/Test/L0/Worker/ExpressionManagerL0.cs +++ b/src/Test/L0/Worker/Expressions/ConditionFunctionsL0.cs @@ -1,20 +1,20 @@ using System; using System.Collections.Generic; using System.Runtime.CompilerServices; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.ObjectTemplating; +using GitHub.DistributedTask.Pipelines.ObjectTemplating; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Expressions; using Moq; using Xunit; -using GitHub.DistributedTask.Expressions2; -using GitHub.DistributedTask.Pipelines.ContextData; -namespace GitHub.Runner.Common.Tests.Worker +namespace GitHub.Runner.Common.Tests.Worker.Expressions { - public sealed class ExpressionManagerL0 + public sealed class ConditionFunctionsL0 { - private Mock _ec; - private ExpressionManager _expressionManager; - private DictionaryContextData _expressions; + private TemplateContext _templateContext; private JobContext _jobContext; [Fact] @@ -38,7 +38,7 @@ public void AlwaysFunction() _jobContext.Status = variableSet.JobStatus; // Act. - bool actual = _expressionManager.Evaluate(_ec.Object, "always()").Value; + bool actual = Evaluate("always()"); // Assert. Assert.Equal(variableSet.Expected, actual); @@ -68,7 +68,7 @@ public void CancelledFunction() _jobContext.Status = variableSet.JobStatus; // Act. - bool actual = _expressionManager.Evaluate(_ec.Object, "cancelled()").Value; + bool actual = Evaluate("cancelled()"); // Assert. Assert.Equal(variableSet.Expected, actual); @@ -97,7 +97,7 @@ public void FailureFunction() _jobContext.Status = variableSet.JobStatus; // Act. - bool actual = _expressionManager.Evaluate(_ec.Object, "failure()").Value; + bool actual = Evaluate("failure()"); // Assert. Assert.Equal(variableSet.Expected, actual); @@ -126,37 +126,7 @@ public void SuccessFunction() _jobContext.Status = variableSet.JobStatus; // Act. - bool actual = _expressionManager.Evaluate(_ec.Object, "success()").Value; - - // Assert. - Assert.Equal(variableSet.Expected, actual); - } - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public void ContextNamedValue() - { - using (TestHostContext hc = CreateTestContext()) - { - // Arrange. - var variableSets = new[] - { - new { Condition = "github.ref == 'refs/heads/master'", VariableName = "ref", VariableValue = "refs/heads/master", Expected = true }, - new { Condition = "github['ref'] == 'refs/heads/master'", VariableName = "ref", VariableValue = "refs/heads/master", Expected = true }, - new { Condition = "github.nosuch || '' == ''", VariableName = "ref", VariableValue = "refs/heads/master", Expected = true }, - new { Condition = "github['ref'] == 'refs/heads/release'", VariableName = "ref", VariableValue = "refs/heads/master", Expected = false }, - new { Condition = "github.ref == 'refs/heads/release'", VariableName = "ref", VariableValue = "refs/heads/master", Expected = false }, - }; - foreach (var variableSet in variableSets) - { - InitializeExecutionContext(hc); - _ec.Object.ExpressionValues["github"] = new GitHubContext() { { variableSet.VariableName, new StringContextData(variableSet.VariableValue) } }; - - // Act. - bool actual = _expressionManager.Evaluate(_ec.Object, variableSet.Condition).Value; + bool actual = Evaluate("success()"); // Assert. Assert.Equal(variableSet.Expected, actual); @@ -166,21 +136,34 @@ public void ContextNamedValue() private TestHostContext CreateTestContext([CallerMemberName] String testName = "") { - var hc = new TestHostContext(this, testName); - _expressionManager = new ExpressionManager(); - _expressionManager.Initialize(hc); - return hc; + return new TestHostContext(this, testName); } private void InitializeExecutionContext(TestHostContext hc) { - _expressions = new DictionaryContextData(); _jobContext = new JobContext(); - _ec = new Mock(); - _ec.SetupAllProperties(); - _ec.Setup(x => x.ExpressionValues).Returns(_expressions); - _ec.Setup(x => x.JobContext).Returns(_jobContext); + var executionContext = new Mock(); + executionContext.SetupAllProperties(); + executionContext.Setup(x => x.JobContext).Returns(_jobContext); + + _templateContext = new TemplateContext(); + _templateContext.State[nameof(IExecutionContext)] = executionContext.Object; + } + + private bool Evaluate(string expression) + { + var parser = new ExpressionParser(); + var functions = new IFunctionInfo[] + { + new FunctionInfo(PipelineTemplateConstants.Always, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Cancelled, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Failure, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Success, 0, 0), + }; + var tree = parser.CreateTree(expression, null, null, functions); + var result = tree.Evaluate(null, null, _templateContext, null); + return result.IsTruthy; } } } diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index bedd24c363e..a8c4573e61f 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -22,7 +22,6 @@ public sealed class JobExtensionL0 private Mock _jobServerQueue; private Mock _config; private Mock _logger; - private Mock _express; private Mock _containerProvider; private Mock _diagnosticLogManager; @@ -35,7 +34,6 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " _jobServerQueue = new Mock(); _config = new Mock(); _logger = new Mock(); - _express = new Mock(); _containerProvider = new Mock(); _diagnosticLogManager = new Mock(); _directoryManager = new Mock(); @@ -108,7 +106,6 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " hc.SetSingleton(_actionManager.Object); hc.SetSingleton(_config.Object); hc.SetSingleton(_jobServerQueue.Object); - hc.SetSingleton(_express.Object); hc.SetSingleton(_containerProvider.Object); hc.SetSingleton(_directoryManager.Object); hc.SetSingleton(_diagnosticLogManager.Object); diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index de09a4c96da..198d378b9b9 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -53,9 +53,6 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " } _tokenSource = new CancellationTokenSource(); - var expressionManager = new ExpressionManager(); - expressionManager.Initialize(hc); - hc.SetSingleton(expressionManager); _jobRunner = new JobRunner(); _jobRunner.Initialize(hc); diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index 23534813f4b..c2996fab51f 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -1,17 +1,17 @@ -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Worker; -using Moq; -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Tasks; +using Moq; using Xunit; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common.Util; +using GitHub.Runner.Worker; namespace GitHub.Runner.Common.Tests.Worker { @@ -27,9 +27,6 @@ public sealed class StepsRunnerL0 private TestHostContext CreateTestContext([CallerMemberName] String testName = "") { var hc = new TestHostContext(this, testName); - var expressionManager = new ExpressionManager(); - expressionManager.Initialize(hc); - hc.SetSingleton(expressionManager); Dictionary variablesToCopy = new Dictionary(); _variables = new Variables( hostContext: hc, @@ -49,6 +46,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " _contexts["runner"] = new DictionaryContextData(); _contexts["job"] = _jobContext; _ec.Setup(x => x.ExpressionValues).Returns(_contexts); + _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); _ec.Setup(x => x.JobContext).Returns(_jobContext); _stepContext = new StepsContext(); @@ -383,16 +381,11 @@ public async Task TreatsConditionErrorAsFailure() { using (TestHostContext hc = CreateTestContext()) { - var expressionManager = new Mock(); - expressionManager.Object.Initialize(hc); - hc.SetSingleton(expressionManager.Object); - expressionManager.Setup(x => x.Evaluate(It.IsAny(), It.IsAny(), It.IsAny())).Throws(new Exception()); - // Arrange. var variableSets = new[] { - new[] { CreateStep(hc, TaskResult.Succeeded, "success()") }, - new[] { CreateStep(hc, TaskResult.Succeeded, "success()") }, + new[] { CreateStep(hc, TaskResult.Succeeded, "fromJson('not json')") }, + new[] { CreateStep(hc, TaskResult.Succeeded, "fromJson('not json')") }, }; foreach (var variableSet in variableSets) { @@ -610,6 +603,7 @@ private Mock CreateStep(TestHostContext hc, TaskResult result, st stepContext.Setup(x => x.Variables).Returns(_variables); stepContext.Setup(x => x.EnvironmentVariables).Returns(_env); stepContext.Setup(x => x.ExpressionValues).Returns(_contexts); + stepContext.Setup(x => x.ExpressionFunctions).Returns(new List()); stepContext.Setup(x => x.JobContext).Returns(_jobContext); stepContext.Setup(x => x.StepsContext).Returns(_stepContext); stepContext.Setup(x => x.ContextName).Returns(step.Object.Action.ContextName); From 178a618e015cd4a6d6453f2f92c20fdb98191dd1 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 20 Mar 2020 13:02:07 -0400 Subject: [PATCH 11/86] expose GITHUB_REPOSITORY_OWNER. (#378) --- .../Container/DockerCommandManager.cs | 14 ++++++++++++++ src/Runner.Worker/GitHubContext.cs | 1 + 2 files changed, 15 insertions(+) diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index 6451a568d08..737c24852bc 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -130,6 +130,13 @@ public async Task DockerCreate(IExecutionContext context, ContainerInfo // Watermark for GitHub Action environment dockerOptions.Add("-e GITHUB_ACTIONS=true"); + // Set CI=true when no one else already set it. + // CI=true is common set in most CI provider in GitHub + if (!container.ContainerEnvironmentVariables.ContainsKey("CI")) + { + dockerOptions.Add("-e CI=true"); + } + foreach (var volume in container.MountVolumes) { // replace `"` with `\"` and add `"{0}"` to all path. @@ -189,6 +196,13 @@ public async Task DockerRun(IExecutionContext context, ContainerInfo contai // Watermark for GitHub Action environment dockerOptions.Add("-e GITHUB_ACTIONS=true"); + // Set CI=true when no one else already set it. + // CI=true is common set in most CI provider in GitHub + if (!container.ContainerEnvironmentVariables.ContainsKey("CI")) + { + dockerOptions.Add("-e CI=true"); + } + if (!string.IsNullOrEmpty(container.ContainerEntryPoint)) { dockerOptions.Add($"--entrypoint \"{container.ContainerEntryPoint}\""); diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index 454f5e21111..541199b836d 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -17,6 +17,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "job", "ref", "repository", + "repository_owner", "run_id", "run_number", "sha", From ab001a700403c0b9c22405e9a8beebbd9d969db0 Mon Sep 17 00:00:00 2001 From: David Kale Date: Mon, 23 Mar 2020 18:53:01 -0400 Subject: [PATCH 12/86] Add expanded volumes strings to container mounts (#384) --- src/Runner.Worker/Container/ContainerInfo.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Runner.Worker/Container/ContainerInfo.cs b/src/Runner.Worker/Container/ContainerInfo.cs index a1cc2782c52..695364c4a96 100644 --- a/src/Runner.Worker/Container/ContainerInfo.cs +++ b/src/Runner.Worker/Container/ContainerInfo.cs @@ -61,6 +61,7 @@ public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container, foreach (var volume in container.Volumes) { UserMountVolumes[volume] = volume; + MountVolumes.Add(new MountVolume(volume)); } } From 9fc0686dc26c62ada68c9630c941055af21e050e Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Tue, 24 Mar 2020 16:25:11 -0400 Subject: [PATCH 13/86] prepare 2.168.0 runner release. --- releaseNote.md | 35 ++++++++++++++++------------------- src/runnerversion | 2 +- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index f580cc3edf0..18452f965f9 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,27 +1,24 @@ ## Features - - Expose whether debug is on/off via RUNNER_DEBUG. (#253) - - Upload log on runner when worker get killed due to cancellation timeout. (#255) - - Update config.sh/cmd --help documentation (#282) - - Set http_proxy and related env vars for job/service containers (#304) - - Set both http_proxy and HTTP_PROXY env for runner/worker processes. (#298) + - Update Runner Register GitHub API URL to Support Org-level Runner (#339 #345 #352) + - Preserve workflow file/line/column for better error messages (#356) + - Switch to use token service instead of SPS for exchanging oauth token. (#325) + - Load and print machine setup info from .setup_info (#364) + - Expose job name as $GITHUB_JOB (#366) + - Add support for job outputs. (#365) + - Set CI=true when launch process in actions runner. (#374) + - Set steps..outcome and steps..conclusion. (#372) + - Add support for workflow/job defaults. (#369) + - Expose GITHUB_REPOSITORY_OWNER and ${{github.repository_owner}}. (#378) ## Bugs - - Verify runner Windows service hash started successfully after configuration (#236) - - Detect source file path in L0 without using env. (#257) - - Handle escaped '%' in commands data section (#200) - - Allow container to be null/empty during matrix expansion (#266) - - Translate problem matcher file to host path (#272) - - Change hashFiles() expression function to use @actions/glob. (#268) - - Default post-job action's condition to always(). (#293) - - Support action.yaml file as action's entry file (#288) - - Trace javascript action exit code to debug instead of user logs (#290) - - Change prompt message when removing a runner to lines up with GitHub.com UI (#303) - - Include step.env as part of env context. (#300) - - Update Base64 Encoders to deal with suffixes (#284) + - Use authenticate endpoint for testing runner connection. (#311) + - Commands translate file path from container action (#331) + - Change problem matchers output to debug (#363) + - Switch hashFiles to extension function (#362) + - Add expanded volumes strings to container mounts (#384) ## Misc - - Move .sln file under ./src (#238) - - Treat warnings as errors during compile (#249) + - Add runner auth documentation (#357) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows diff --git a/src/runnerversion b/src/runnerversion index af6ddeb49fd..2973ad9e470 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.165.2 +2.168.0 From dff1024cd3ab8f5b852750be7e1ddc605c3a45da Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 24 Mar 2020 21:51:37 -0400 Subject: [PATCH 14/86] cache actions on premises (#381) --- src/Runner.Common/ConfigurationStore.cs | 27 ++++++++++ .../Configuration/ConfigurationManager.cs | 6 +-- src/Runner.Worker/ActionManager.cs | 52 ++++++++++++------- src/Test/L0/Worker/ActionManagerL0.cs | 51 ++++++++++++++++++ 4 files changed, 114 insertions(+), 22 deletions(-) diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs index da66d7f8df1..fc32ad436b1 100644 --- a/src/Runner.Common/ConfigurationStore.cs +++ b/src/Runner.Common/ConfigurationStore.cs @@ -15,6 +15,9 @@ namespace GitHub.Runner.Common [DataContract] public sealed class RunnerSettings { + [DataMember(Name = "IsHostedServer", EmitDefaultValue = false)] + private bool? _isHostedServer; + [DataMember(EmitDefaultValue = false)] public int AgentId { get; set; } @@ -42,6 +45,21 @@ public sealed class RunnerSettings [DataMember(EmitDefaultValue = false)] public string MonitorSocketAddress { get; set; } + [IgnoreDataMember] + public bool IsHostedServer + { + get + { + // Old runners do not have this property. Hosted runners likely don't have this property either. + return _isHostedServer ?? true; + } + + set + { + _isHostedServer = value; + } + } + /// // Computed property for convenience. Can either return: // 1. If runner was configured at the repo level, returns something like: "myorg/myrepo" @@ -69,6 +87,15 @@ public string RepoOrOrgName return repoOrOrgName; } } + + [OnSerializing] + private void OnSerializing(StreamingContext context) + { + if (_isHostedServer.HasValue && _isHostedServer.Value) + { + _isHostedServer = null; + } + } } [ServiceLocator(Default = typeof(ConfigurationStore))] diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 8d99f09c26f..1c12a73249e 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -86,7 +86,6 @@ public async Task ConfigureAsync(CommandSettings command) RunnerSettings runnerSettings = new RunnerSettings(); - bool isHostedServer = false; // Loop getting url and creds until you can connect ICredentialProvider credProvider = null; VssCredentials creds = null; @@ -117,7 +116,7 @@ public async Task ConfigureAsync(CommandSettings command) try { // Determine the service deployment type based on connection data. (Hosted/OnPremises) - isHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds); + runnerSettings.IsHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds); // Validate can connect. await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds); @@ -248,7 +247,7 @@ public async Task ConfigureAsync(CommandSettings command) { UriBuilder configServerUrl = new UriBuilder(runnerSettings.ServerUrl); UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl); - if (!isHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0) + if (!runnerSettings.IsHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0) { oauthEndpointUrlBuilder.Scheme = configServerUrl.Scheme; oauthEndpointUrlBuilder.Host = configServerUrl.Host; @@ -381,7 +380,6 @@ public async Task UnconfigureAsync(CommandSettings command) } // Determine the service deployment type based on connection data. (Hosted/OnPremises) - bool isHostedServer = await IsHostedServer(settings.ServerUrl, creds); await _runnerServer.ConnectAsync(new Uri(settings.ServerUrl), creds); var agents = await _runnerServer.GetAgentsAsync(settings.PoolId, settings.AgentName); diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 28ab955912e..7cfbe7b5955 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -58,8 +58,14 @@ public async Task> PrepareActionsAsync(IExecutionContex executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is depreciated. Please remove it from the repository's secrets"); } - // Clear the cache (local runner) - IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken); + // Clear the cache (for self-hosted runners) + // Note, temporarily avoid this step for the on-premises product, to avoid rate limiting. + var configurationStore = HostContext.GetService(); + var isHostedServer = configurationStore.GetSettings().IsHostedServer; + if (isHostedServer) + { + IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken); + } foreach (var action in actions) { @@ -448,7 +454,8 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref)); string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref); - if (File.Exists(destDirectory + ".completed")) + string watermarkFile = destDirectory + ".completed"; + if (File.Exists(watermarkFile)) { executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'."); return; @@ -498,24 +505,33 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { - var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); - if (string.IsNullOrEmpty(authToken)) - { - // TODO: Depreciate the PREVIEW_ACTION_TOKEN - authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); - } - - if (!string.IsNullOrEmpty(authToken)) + var configurationStore = HostContext.GetService(); + var isHostedServer = configurationStore.GetSettings().IsHostedServer; + if (isHostedServer) { - HostContext.SecretMasker.AddValue(authToken); - var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}")); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); + var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); + if (string.IsNullOrEmpty(authToken)) + { + // TODO: Depreciate the PREVIEW_ACTION_TOKEN + authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); + } + + if (!string.IsNullOrEmpty(authToken)) + { + HostContext.SecretMasker.AddValue(authToken); + var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}")); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); + } + else + { + var accessToken = executionContext.GetGitHubContext("token"); + var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); + } } else { - var accessToken = executionContext.GetGitHubContext("token"); - var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); + // Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated. } httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent); @@ -610,7 +626,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } Trace.Verbose("Create watermark file indicate action download succeed."); - File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString()); + File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString()); executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'."); Trace.Info("Finished getting action repository."); diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index b5a2ed5db76..34b64880d3c 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -111,6 +111,57 @@ public async void PrepareActions_DownloadActionFromGraph() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_SkipDownloadActionFromGraphWhenCached_OnPremises() + { + try + { + // Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "actions/no-such-action", + Ref = "master", + RepositoryType = "GitHub" + } + } + }; + _configurationStore.Object.GetSettings().IsHostedServer = false; + var actionDirectory = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/no-such-action", "master"); + Directory.CreateDirectory(actionDirectory); + var watermarkFile = $"{actionDirectory}.completed"; + File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString()); + var actionFile = Path.Combine(actionDirectory, "action.yml"); + File.WriteAllText(actionFile, @" +name: ""no-such-action"" +runs: + using: node12 + main: no-such-action.js +"); + var testFile = Path.Combine(actionDirectory, "test-file"); + File.WriteAllText(testFile, "asdf"); + + // Act + await _actionManager.PrepareActionsAsync(_ec.Object, actions); + + // Assert + Assert.True(File.Exists(testFile)); + } + finally + { + Teardown(); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] From e23d68f6e21d0e92ad8613b9a273f54631ac6626 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Wed, 25 Mar 2020 15:11:52 -0400 Subject: [PATCH 15/86] add github.url and github.api_url for ghes alpha (#386) --- src/Runner.Worker/GitHubContext.cs | 2 ++ src/Runner.Worker/JobExtension.cs | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index 541199b836d..7a707d1a9b0 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -10,6 +10,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa { "action", "actor", + "api_url", // temp for GHES alpha release "base_ref", "event_name", "event_path", @@ -21,6 +22,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "run_id", "run_number", "sha", + "url", // temp for GHES alpha release "workflow", "workspace", }; diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index fe3a1ed24d8..927b111f721 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Runtime.Serialization; @@ -127,6 +128,17 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory)); context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory)); + // Temporary hack for GHES alpha + var configurationStore = HostContext.GetService(); + var runnerSettings = configurationStore.GetSettings(); + if (!runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl)) + { + var url = new Uri(runnerSettings.GitHubUrl); + var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}"; + context.SetGitHubContext("url", $"{url.Scheme}://{url.Host}{portInfo}"); + context.SetGitHubContext("api_url", $"{url.Scheme}://api.{url.Host}{portInfo}"); + } + // Evaluate the job-level environment variables context.Debug("Evaluating job-level environment variables"); var templateEvaluator = context.ToPipelineTemplateEvaluator(); From 3d70ef2da1798bb22fde31ceef8b9ea8cc328adb Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 26 Mar 2020 23:01:17 -0400 Subject: [PATCH 16/86] update workflow schema file. (#388) --- .../Pipelines/ObjectTemplating/PipelineTemplateConstants.cs | 1 + .../Pipelines/ObjectTemplating/YamlObjectReader.cs | 4 ++-- src/Sdk/DTPipelines/workflow-v1.0.json | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index 464cd9aad38..d1c886dd891 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -8,6 +8,7 @@ public sealed class PipelineTemplateConstants { public const String Always = "always"; public const String BooleanStepsContext = "boolean-steps-context"; + public const String BooleanStrategyContext = "boolean-strategy-context"; public const String CancelTimeoutMinutes = "cancel-timeout-minutes"; public const String Cancelled = "cancelled"; public const String Checkout = "checkout"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs index 881b70ae245..982a9c487f0 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Globalization; using System.IO; using System.Linq; @@ -12,7 +12,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating /// /// Converts a YAML file into a TemplateToken /// - public sealed class YamlObjectReader : IObjectReader + internal sealed class YamlObjectReader : IObjectReader { internal YamlObjectReader( Int32? fileId, diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index f1e364c5cc7..29847005473 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -739,7 +739,7 @@ "container-env": { "mapping": { "loose-key-type": "non-empty-string", - "loose-value-type": "string" + "loose-value-type": "string-runner-context" } }, From b0a1294ef5751746eb2b4815c4e3e8a6d575833c Mon Sep 17 00:00:00 2001 From: eric sciple Date: Fri, 27 Mar 2020 00:16:02 -0400 Subject: [PATCH 17/86] Fix API URL for GHES (#390) --- src/Runner.Worker/JobExtension.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 927b111f721..232ec18cf10 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -136,7 +136,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel var url = new Uri(runnerSettings.GitHubUrl); var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}"; context.SetGitHubContext("url", $"{url.Scheme}://{url.Host}{portInfo}"); - context.SetGitHubContext("api_url", $"{url.Scheme}://api.{url.Host}{portInfo}"); + context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3"); } // Evaluate the job-level environment variables From dec260920fc8a1c7e92bed4c1d1be9dd012fe86e Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Mon, 30 Mar 2020 07:45:06 -0400 Subject: [PATCH 18/86] spelling: deprecate (#394) --- src/Runner.Worker/ActionManager.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 7cfbe7b5955..e73aa17dd00 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -51,11 +51,11 @@ public async Task> PrepareActionsAsync(IExecutionContex List containerSetupSteps = new List(); IEnumerable actions = steps.OfType(); - // TODO: Depreciate the PREVIEW_ACTION_TOKEN + // TODO: Deprecate the PREVIEW_ACTION_TOKEN // Log even if we aren't using it to ensure users know. if (!string.IsNullOrEmpty(executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"))) { - executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is depreciated. Please remove it from the repository's secrets"); + executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is deprecated. Please remove it from the repository's secrets"); } // Clear the cache (for self-hosted runners) @@ -512,7 +512,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); if (string.IsNullOrEmpty(authToken)) { - // TODO: Depreciate the PREVIEW_ACTION_TOKEN + // TODO: Deprecate the PREVIEW_ACTION_TOKEN authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); } From be325f26a6bde5400bf38f4021fcd3175231a38a Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 30 Mar 2020 14:48:02 -0400 Subject: [PATCH 19/86] support config with GHES url. (#393) --- .../Configuration/ConfigurationManager.cs | 24 +++++++++++++------ .../Configuration/ConfigurationManagerL0.cs | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 1c12a73249e..a6e980a5dfe 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -94,8 +94,7 @@ public async Task ConfigureAsync(CommandSettings command) { // Get the URL var inputUrl = command.GetUrl(); - if (!inputUrl.Contains("github.com", StringComparison.OrdinalIgnoreCase) && - !inputUrl.Contains("github.localhost", StringComparison.OrdinalIgnoreCase)) + if (inputUrl.Contains("codedev.ms", StringComparison.OrdinalIgnoreCase)) { runnerSettings.ServerUrl = inputUrl; // Get the credentials @@ -198,7 +197,7 @@ public async Task ConfigureAsync(CommandSettings command) } else { - // Create a new agent. + // Create a new agent. agent = CreateNewAgent(runnerSettings.AgentName, publicKey); try @@ -290,7 +289,7 @@ public async Task ConfigureAsync(CommandSettings command) { // there are two exception messages server send that indicate clock skew. // 1. The bearer token expired on {jwt.ValidTo}. Current server time is {DateTime.UtcNow}. - // 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}. + // 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}. Trace.Error("Catch exception during test agent connection."); Trace.Error(ex); throw new Exception("The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again."); @@ -402,7 +401,7 @@ public async Task UnconfigureAsync(CommandSettings command) _term.WriteLine("Cannot connect to server, because config files are missing. Skipping removing runner from the server."); } - //delete credential config files + //delete credential config files currentAction = "Removing .credentials"; if (hasCredentials) { @@ -416,7 +415,7 @@ public async Task UnconfigureAsync(CommandSettings command) _term.WriteLine("Does not exist. Skipping " + currentAction); } - //delete settings config file + //delete settings config file currentAction = "Removing .runner"; if (isConfigured) { @@ -519,8 +518,19 @@ private async Task IsHostedServer(string serverUrl, VssCredentials credent private async Task GetTenantCredential(string githubUrl, string githubToken, string runnerEvent) { + var githubApiUrl = ""; var gitHubUrlBuilder = new UriBuilder(githubUrl); - var githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runner-registration"; + if (string.Equals(gitHubUrlBuilder.Host, "github.com", StringComparison.OrdinalIgnoreCase) || + string.Equals(gitHubUrlBuilder.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) || + string.Equals(gitHubUrlBuilder.Host, "github.localhost", StringComparison.OrdinalIgnoreCase)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runner-registration"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/actions/runner-registration"; + } + using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index e9d05435362..c47d5be21c0 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -37,7 +37,7 @@ public class ConfigurationManagerL0 private Mock _rsaKeyManager; private string _expectedToken = "expectedToken"; - private string _expectedServerUrl = "https://localhost"; + private string _expectedServerUrl = "https://codedev.ms"; private string _expectedAgentName = "expectedAgentName"; private string _expectedPoolName = "poolName"; private string _expectedAuthType = "pat"; From a5f06b3ec2c138a4a3a03b674e87fa262e1f766f Mon Sep 17 00:00:00 2001 From: Bryan MacFarlane Date: Mon, 30 Mar 2020 17:46:32 -0400 Subject: [PATCH 20/86] ADR 397: Configuration time custom labels (#397) Since configuring self-hosted runners is commonly automated via scripts, the labels need to be able to be created during configuration. The runner currently registers the built-in labels (os, arch) during registration but does not accept labels via command line args to extend the set registered. --- docs/adrs/0397-runner-registration-labels.md | 56 ++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 docs/adrs/0397-runner-registration-labels.md diff --git a/docs/adrs/0397-runner-registration-labels.md b/docs/adrs/0397-runner-registration-labels.md new file mode 100644 index 00000000000..d949ddfb833 --- /dev/null +++ b/docs/adrs/0397-runner-registration-labels.md @@ -0,0 +1,56 @@ +# ADR 0397: Support adding custom labels during runner config +**Date**: 2020-03-30 + +**Status**: Approved + +## Context + +Since configuring self-hosted runners is commonly automated via scripts, the labels need to be able to be created during configuration. The runner currently registers the built-in labels (os, arch) during registration but does not accept labels via command line args to extend the set registered. + +See Issue: https://github.com/actions/runner/issues/262 + +This is another version of [ADR275](https://github.com/actions/runner/pull/275) + +## Decision + +This ADR proposes that we add a `--labels` option to `config`, which could be used to add custom additional labels to the configured runner. + +For example, to add a single extra label the operator could run: +```bash +./config.sh --labels mylabel +``` +> Note: the current runner command line parsing and envvar override algorithm only supports a single argument (key). + +This would add the label `mylabel` to the runner, and enable users to select the runner in their workflow using this label: +```yaml +runs-on: [self-hosted, mylabel] +``` + +To add multiple labels the operator could run: +```bash +./config.sh --labels mylabel,anotherlabel +``` +> Note: the current runner command line parsing and envvar override algorithm only supports a single argument (key). + +This would add the label `mylabel` and `anotherlabel` to the runner, and enable users to select the runner in their workflow using this label: +```yaml +runs-on: [self-hosted, mylabel, anotherlabel] +``` + +It would not be possible to remove labels from an existing runner using `config.sh`, instead labels would have to be removed using the GitHub UI. + +The labels argument will split on commas, trim and discard empty strings. That effectively means don't use commans in unattended config label names. Alternatively we could choose to escape commans but it's a nice to have. + +## Replace + +If an existing runner exists and the option to replace is chosen (interactively of via unattend as in this scenario), then the labels will be replaced / overwritten (not merged). + +## Overriding built-in labels + +Note that it is possible to register "built-in" hosted labels like `ubuntu-latest` and is not considered an error. This is an effective way for the org / runner admin to dictate by policy through registration that this set of runners will be used without having to edit all the workflow files now and in the future. + +We will also not make other restrictions such as limiting explicitly adding os / arch labels and validating. We will assume that explicit labels were added for a reason and not restricting offers the most flexibility and future proofing / compat. + +## Consequences + +The ability to add custom labels to a self-hosted runner would enable most scenarios where job runner selection based on runner capabilities or characteristics are required. From 0e8777ebda3e9aa9a26fe22900b7131ae8bbd34d Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 31 Mar 2020 10:11:02 -0400 Subject: [PATCH 21/86] ADR for wrapper action (#361) * wrapper action adr. * rename * updates. * Update 0361-wrapper-action.md --- docs/adrs/0361-wrapper-action.md | 75 ++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 docs/adrs/0361-wrapper-action.md diff --git a/docs/adrs/0361-wrapper-action.md b/docs/adrs/0361-wrapper-action.md new file mode 100644 index 00000000000..52adbf173c1 --- /dev/null +++ b/docs/adrs/0361-wrapper-action.md @@ -0,0 +1,75 @@ +# ADR 361: Wrapper Action + +**Date**: 2020-03-06 + +**Status**: Pending + +## Context + +In addition to action's regular execution, action author may wants their action has a chance to participate in: +- Job initialize + My Action will collect machine resource usage (CPU/RAM/Disk) during a workflow job execution, we need to start perf recorder at the begin of the job. +- Job cleanup + My Action will dirty local workspace or machine environment during execution, we need to cleanup these changes at the end of the job. + Ex: `actions/checkout@v2` will write `github.token` into local `.git/config` during execution, it has post job cleanup defined to undo the changes. + +## Decision + +### Add `pre` and `post` execution to action + +Node Action Example: + +```yaml + name: 'My action with pre' + description: 'My action with pre' + runs: + using: 'node12' + pre: 'setup.js' + pre-if: 'success()' // Optional + main: 'index.js' + post: 'cleanup.js' + post-if: 'success()' // Optional +``` + +Container Action Example: + +```yaml + name: 'My action with pre' + description: 'My action with pre' + runs: + using: 'docker' + image: 'mycontainer:latest' + pre-entrypoint: 'setup.sh' + pre-if: 'success()' // Optional + entrypoint: 'entrypoint.sh' + post-entrypoint: 'cleanup.sh' + post-if: 'success()' // Optional +``` + +Both `pre` and `post` will has default `pre-if/post-if` sets to `always()`. +Setting `pre` to `always()` will make sure no matter what condition evaluate result the `main` gets at runtime, the `pre` has always run already. +`pre` executes in order of how the steps are defined. +`pre` will always be added to job steps list during job setup. +> Action referenced from local repository (`./my-action`) won't get `pre` setup correctly since the repository haven't checkout during job initialize. +> We can't use GitHub api to download the repository since there is a about 3 mins delay between `git push` and the new commit available to download using GitHub api. + +`post` will be pushed into a `poststeps` stack lazily when the action's `pre` or `main` execution passed `if` condition check and about to run, you can't have an action that only contains a `post`, we will pop and run each `post` after all `pre` and `main` finished. +> Currently `post` works for both repository action (`org/repo@v1`) and local action (`./my-action`) + +Valid action: +- only has `main` +- has `pre` and `main` +- has `main` and `post` +- has `pre`, `main` and `post` + +Invalid action: +- only has `pre` +- only has `post` +- has `pre` and `post` + +Potential downside of introducing `pre`: + +- Extra magic wrt step order. Users should control the step order. Especially when we introduce templates. +- Eliminates the possibility to lazily download the action tarball, since `pre` always run by default, we have to download the tarball to check whether action defined a `pre` +- `pre` doesn't work with local action, we suggested customer use local action for testing their action changes, ex CI for their action, to avoid delay between `git push` and GitHub repo tarball download api. +- Condition on the `pre` can't be controlled using dynamic step outputs. `pre` executes too early. From ba69b5bc932f885bd80c056652c479457f36effd Mon Sep 17 00:00:00 2001 From: eric sciple Date: Wed, 1 Apr 2020 17:29:57 -0400 Subject: [PATCH 22/86] Fix runner config IsHostedServer detection for GHES alpha (#401) --- .../Configuration/ConfigurationManager.cs | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index a6e980a5dfe..e2b8ca31cd1 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -115,7 +115,7 @@ public async Task ConfigureAsync(CommandSettings command) try { // Determine the service deployment type based on connection data. (Hosted/OnPremises) - runnerSettings.IsHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds); + runnerSettings.IsHostedServer = runnerSettings.GitHubUrl == null || IsHostedServer(new UriBuilder(runnerSettings.GitHubUrl)); // Validate can connect. await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds); @@ -246,14 +246,6 @@ public async Task ConfigureAsync(CommandSettings command) { UriBuilder configServerUrl = new UriBuilder(runnerSettings.ServerUrl); UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl); - if (!runnerSettings.IsHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0) - { - oauthEndpointUrlBuilder.Scheme = configServerUrl.Scheme; - oauthEndpointUrlBuilder.Host = configServerUrl.Host; - oauthEndpointUrlBuilder.Port = configServerUrl.Port; - Trace.Info($"Set oauth endpoint url's scheme://host:port component to match runner configure url's scheme://host:port: '{oauthEndpointUrlBuilder.Uri.AbsoluteUri}'."); - } - var credentialData = new CredentialData { Scheme = Constants.Configuration.OAuth, @@ -495,34 +487,18 @@ private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey) return agent; } - private async Task IsHostedServer(string serverUrl, VssCredentials credentials) + private bool IsHostedServer(UriBuilder gitHubUrl) { - // Determine the service deployment type based on connection data. (Hosted/OnPremises) - var locationServer = HostContext.GetService(); - VssConnection connection = VssUtil.CreateConnection(new Uri(serverUrl), credentials); - await locationServer.ConnectAsync(connection); - try - { - var connectionData = await locationServer.GetConnectionDataAsync(); - Trace.Info($"Server deployment type: {connectionData.DeploymentType}"); - return connectionData.DeploymentType.HasFlag(DeploymentFlags.Hosted); - } - catch (Exception ex) - { - // Since the DeploymentType is Enum, deserialization exception means there is a new Enum member been added. - // It's more likely to be Hosted since OnPremises is always behind and customer can update their agent if are on-prem - Trace.Error(ex); - return true; - } + return string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) || + string.Equals(gitHubUrl.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) || + string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase); } private async Task GetTenantCredential(string githubUrl, string githubToken, string runnerEvent) { var githubApiUrl = ""; var gitHubUrlBuilder = new UriBuilder(githubUrl); - if (string.Equals(gitHubUrlBuilder.Host, "github.com", StringComparison.OrdinalIgnoreCase) || - string.Equals(gitHubUrlBuilder.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) || - string.Equals(gitHubUrlBuilder.Host, "github.localhost", StringComparison.OrdinalIgnoreCase)) + if (IsHostedServer(gitHubUrlBuilder)) { githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runner-registration"; } From 83b5742278fb9a152df2d2d3e4ee0dcc2cebe7fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Apr 2020 13:49:48 -0400 Subject: [PATCH 23/86] Bump acorn from 6.4.0 to 6.4.1 in /src/Misc/expressionFunc/hashFiles (#371) Bumps [acorn](https://github.com/acornjs/acorn) from 6.4.0 to 6.4.1. - [Release notes](https://github.com/acornjs/acorn/releases) - [Commits](https://github.com/acornjs/acorn/compare/6.4.0...6.4.1) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- src/Misc/expressionFunc/hashFiles/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Misc/expressionFunc/hashFiles/package-lock.json b/src/Misc/expressionFunc/hashFiles/package-lock.json index 5938e85d99f..909e53360a8 100644 --- a/src/Misc/expressionFunc/hashFiles/package-lock.json +++ b/src/Misc/expressionFunc/hashFiles/package-lock.json @@ -258,9 +258,9 @@ "dev": true }, "acorn": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.0.tgz", - "integrity": "sha512-gac8OEcQ2Li1dxIEWGZzsp2BitJxwkwcOm0zHAJLcPJaVvm58FRnk6RkuLRpU1EujipU2ZFODv2P9DLMfnV8mw==", + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", "dev": true }, "acorn-jsx": { From 1f52dfa6368bb175d1ccb898fd579cc46ce17e04 Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Tue, 7 Apr 2020 14:00:37 -0400 Subject: [PATCH 24/86] bump dev-dependency version. --- .../hashFiles/package-lock.json | 1075 +++++++++++------ .../expressionFunc/hashFiles/package.json | 2 +- 2 files changed, 673 insertions(+), 404 deletions(-) diff --git a/src/Misc/expressionFunc/hashFiles/package-lock.json b/src/Misc/expressionFunc/hashFiles/package-lock.json index 909e53360a8..75c7d515389 100644 --- a/src/Misc/expressionFunc/hashFiles/package-lock.json +++ b/src/Misc/expressionFunc/hashFiles/package-lock.json @@ -19,130 +19,150 @@ } }, "@babel/code-frame": { - "version": "7.5.5", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.5.5.tgz", - "integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==", + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.8.3.tgz", + "integrity": "sha512-a9gxpmdXtZEInkCSHUJDLHZVBgb1QS0jhss4cPP93EW7s+uC5bikET2twEF3KV+7rDblJcmNvTR7VJejqd2C2g==", "dev": true, "requires": { - "@babel/highlight": "^7.0.0" + "@babel/highlight": "^7.8.3" } }, "@babel/generator": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.7.4.tgz", - "integrity": "sha512-m5qo2WgdOJeyYngKImbkyQrnUN1mPceaG5BV+G0E3gWsa4l/jCSryWJdM2x8OuGAOyh+3d5pVYfZWCiNFtynxg==", + "version": "7.9.4", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.9.4.tgz", + "integrity": "sha512-rjP8ahaDy/ouhrvCoU1E5mqaitWrxwuNGU+dy1EpaoK48jZay4MdkskKGIMHLZNewg8sAsqpGSREJwP0zH3YQA==", "dev": true, "requires": { - "@babel/types": "^7.7.4", + "@babel/types": "^7.9.0", "jsesc": "^2.5.1", "lodash": "^4.17.13", "source-map": "^0.5.0" } }, "@babel/helper-function-name": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.7.4.tgz", - "integrity": "sha512-AnkGIdiBhEuiwdoMnKm7jfPfqItZhgRaZfMg1XX3bS25INOnLPjPG1Ppnajh8eqgt5kPJnfqrRHqFqmjKDZLzQ==", + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.8.3.tgz", + "integrity": "sha512-BCxgX1BC2hD/oBlIFUgOCQDOPV8nSINxCwM3o93xP4P9Fq6aV5sgv2cOOITDMtCfQ+3PvHp3l689XZvAM9QyOA==", "dev": true, "requires": { - "@babel/helper-get-function-arity": "^7.7.4", - "@babel/template": "^7.7.4", - "@babel/types": "^7.7.4" + "@babel/helper-get-function-arity": "^7.8.3", + "@babel/template": "^7.8.3", + "@babel/types": "^7.8.3" } }, "@babel/helper-get-function-arity": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.7.4.tgz", - "integrity": "sha512-QTGKEdCkjgzgfJ3bAyRwF4yyT3pg+vDgan8DSivq1eS0gwi+KGKE5x8kRcbeFTb/673mkO5SN1IZfmCfA5o+EA==", + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.8.3.tgz", + "integrity": "sha512-FVDR+Gd9iLjUMY1fzE2SR0IuaJToR4RkCDARVfsBBPSP53GEqSFjD8gNyxg246VUyc/ALRxFaAK8rVG7UT7xRA==", "dev": true, "requires": { - "@babel/types": "^7.7.4" + "@babel/types": "^7.8.3" } }, "@babel/helper-split-export-declaration": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.7.4.tgz", - "integrity": "sha512-guAg1SXFcVr04Guk9eq0S4/rWS++sbmyqosJzVs8+1fH5NI+ZcmkaSkc7dmtAFbHFva6yRJnjW3yAcGxjueDug==", + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.8.3.tgz", + "integrity": "sha512-3x3yOeyBhW851hroze7ElzdkeRXQYQbFIb7gLK1WQYsw2GWDay5gAJNw1sWJ0VFP6z5J1whqeXH/WCdCjZv6dA==", "dev": true, "requires": { - "@babel/types": "^7.7.4" + "@babel/types": "^7.8.3" } }, + "@babel/helper-validator-identifier": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.9.0.tgz", + "integrity": "sha512-6G8bQKjOh+of4PV/ThDm/rRqlU7+IGoJuofpagU5GlEl29Vv0RGqqt86ZGRV8ZuSOY3o+8yXl5y782SMcG7SHw==", + "dev": true + }, "@babel/highlight": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.5.0.tgz", - "integrity": "sha512-7dV4eu9gBxoM0dAnj/BCFDW9LFU0zvTrkq0ugM7pnHEgguOEeOz1so2ZghEdzviYzQEED0r4EAgpsBChKy1TRQ==", + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.9.0.tgz", + "integrity": "sha512-lJZPilxX7Op3Nv/2cvFdnlepPXDxi29wxteT57Q965oc5R9v86ztx0jfxVrTcBk8C2kcPkkDa2Z4T3ZsPPVWsQ==", "dev": true, "requires": { + "@babel/helper-validator-identifier": "^7.9.0", "chalk": "^2.0.0", - "esutils": "^2.0.2", "js-tokens": "^4.0.0" } }, "@babel/parser": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.7.4.tgz", - "integrity": "sha512-jIwvLO0zCL+O/LmEJQjWA75MQTWwx3c3u2JOTDK5D3/9egrWRRA0/0hk9XXywYnXZVVpzrBYeIQTmhwUaePI9g==", + "version": "7.9.4", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.9.4.tgz", + "integrity": "sha512-bC49otXX6N0/VYhgOMh4gnP26E9xnDZK3TmbNpxYzzz9BQLBosQwfyOe9/cXUU3txYhTzLCbcqd5c8y/OmCjHA==", "dev": true }, "@babel/runtime": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.7.4.tgz", - "integrity": "sha512-r24eVUUr0QqNZa+qrImUk8fn5SPhHq+IfYvIoIMg0do3GdK9sMdiLKP3GYVVaxpPKORgm8KRKaNTEhAjgIpLMw==", + "version": "7.9.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.9.2.tgz", + "integrity": "sha512-NE2DtOdufG7R5vnfQUTehdTfNycfUANEtCa9PssN9O/xmTzP4E08UI797ixaei6hBEVL9BI/PsdJS5x7mWoB9Q==", "dev": true, "requires": { - "regenerator-runtime": "^0.13.2" + "regenerator-runtime": "^0.13.4" } }, "@babel/runtime-corejs3": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.7.4.tgz", - "integrity": "sha512-BBIEhzk8McXDcB3IbOi8zQPzzINUp4zcLesVlBSOcyGhzPUU8Xezk5GAG7Sy5GVhGmAO0zGd2qRSeY2g4Obqxw==", + "version": "7.9.2", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.9.2.tgz", + "integrity": "sha512-HHxmgxbIzOfFlZ+tdeRKtaxWOMUoCG5Mu3wKeUmOxjYrwb3AAHgnmtCUbPPK11/raIWLIBK250t8E2BPO0p7jA==", "dev": true, "requires": { "core-js-pure": "^3.0.0", - "regenerator-runtime": "^0.13.2" + "regenerator-runtime": "^0.13.4" } }, "@babel/template": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.7.4.tgz", - "integrity": "sha512-qUzihgVPguAzXCK7WXw8pqs6cEwi54s3E+HrejlkuWO6ivMKx9hZl3Y2fSXp9i5HgyWmj7RKP+ulaYnKM4yYxw==", + "version": "7.8.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.8.6.tgz", + "integrity": "sha512-zbMsPMy/v0PWFZEhQJ66bqjhH+z0JgMoBWuikXybgG3Gkd/3t5oQ1Rw2WQhnSrsOmsKXnZOx15tkC4qON/+JPg==", "dev": true, "requires": { - "@babel/code-frame": "^7.0.0", - "@babel/parser": "^7.7.4", - "@babel/types": "^7.7.4" + "@babel/code-frame": "^7.8.3", + "@babel/parser": "^7.8.6", + "@babel/types": "^7.8.6" } }, "@babel/traverse": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.7.4.tgz", - "integrity": "sha512-P1L58hQyupn8+ezVA2z5KBm4/Zr4lCC8dwKCMYzsa5jFMDMQAzaBNy9W5VjB+KAmBjb40U7a/H6ao+Xo+9saIw==", + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.9.0.tgz", + "integrity": "sha512-jAZQj0+kn4WTHO5dUZkZKhbFrqZE7K5LAQ5JysMnmvGij+wOdr+8lWqPeW0BcF4wFwrEXXtdGO7wcV6YPJcf3w==", "dev": true, "requires": { - "@babel/code-frame": "^7.5.5", - "@babel/generator": "^7.7.4", - "@babel/helper-function-name": "^7.7.4", - "@babel/helper-split-export-declaration": "^7.7.4", - "@babel/parser": "^7.7.4", - "@babel/types": "^7.7.4", + "@babel/code-frame": "^7.8.3", + "@babel/generator": "^7.9.0", + "@babel/helper-function-name": "^7.8.3", + "@babel/helper-split-export-declaration": "^7.8.3", + "@babel/parser": "^7.9.0", + "@babel/types": "^7.9.0", "debug": "^4.1.0", "globals": "^11.1.0", "lodash": "^4.17.13" + }, + "dependencies": { + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + } } }, "@babel/types": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.7.4.tgz", - "integrity": "sha512-cz5Ji23KCi4T+YIE/BolWosrJuSmoZeN1EFnRtBwF+KKLi8GG/Z2c2hOJJeCXPk4mwk4QFvTmwIodJowXgttRA==", + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.9.0.tgz", + "integrity": "sha512-BS9JKfXkzzJl8RluW4JGknzpiUV7ZrvTayM6yfqLTVBEnFtyowVIOu6rqxRd5cVO6yGoWf4T8u8dgK9oB+GCng==", "dev": true, "requires": { - "esutils": "^2.0.2", + "@babel/helper-validator-identifier": "^7.9.0", "lodash": "^4.17.13", "to-fast-properties": "^2.0.0" } }, + "@types/color-name": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", + "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", + "dev": true + }, "@types/eslint-visitor-keys": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", @@ -258,39 +278,50 @@ "dev": true }, "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.1.tgz", + "integrity": "sha512-add7dgA5ppRPxCFJoAGfMDi7PIBXq1RtGo7BhbLaxwrXPOmw8gq48Y9ozT01hUKy9byMjlR20EJhu5zlkErEkg==", "dev": true }, "acorn-jsx": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.1.0.tgz", - "integrity": "sha512-tMUqwBWfLFbJbizRmEcWSLw6HnFzfdJs2sOJEOwwtVPMoH/0Ay+E703oZz78VSXZiiDcZrQ5XKjPIUQixhmgVw==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", + "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", "dev": true }, "ajv": { - "version": "6.10.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", - "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==", + "version": "6.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.0.tgz", + "integrity": "sha512-D6gFiFA0RRLyUbvijN74DWAjXSFxWKaWP7mldxkVhyhAV3+SWA9HEJPHQ2c9soIeTFJqcSdFDGFgdqs1iUU2Hw==", "dev": true, "requires": { - "fast-deep-equal": "^2.0.1", + "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "ansi-escapes": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", - "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", - "dev": true + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.1.tgz", + "integrity": "sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA==", + "dev": true, + "requires": { + "type-fest": "^0.11.0" + }, + "dependencies": { + "type-fest": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.11.0.tgz", + "integrity": "sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==", + "dev": true + } + } }, "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, "ansi-styles": { @@ -322,13 +353,24 @@ } }, "array-includes": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.0.3.tgz", - "integrity": "sha1-GEtI9i2S10UrsxsyMWXH+L0CJm0=", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.1.tgz", + "integrity": "sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ==", "dev": true, "requires": { - "define-properties": "^1.1.2", - "es-abstract": "^1.7.0" + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0", + "is-string": "^1.0.5" + } + }, + "array.prototype.flat": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz", + "integrity": "sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" } }, "ast-types-flow": { @@ -344,25 +386,21 @@ "dev": true }, "axobject-query": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.1.1.tgz", - "integrity": "sha512-lF98xa/yvy6j3fBHAgQXIYl+J4eZadOSqsPojemUqClzNbBV38wWGpUbQbVEyf4eUF5yF7eHmGgGA2JiHyjeqw==", - "dev": true, - "requires": { - "@babel/runtime": "^7.7.4", - "@babel/runtime-corejs3": "^7.7.4" - } + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.1.2.tgz", + "integrity": "sha512-ICt34ZmrVt8UQnvPl6TVyDTkmhXmAyAT4Jh5ugfGUX4MOrZ+U/ZY6/sdylRw3qGNr9Ub5AJsaHeDMzNLehRdOQ==", + "dev": true }, "babel-eslint": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.0.3.tgz", - "integrity": "sha512-z3U7eMY6r/3f3/JB9mTsLjyxrv0Yb1zb8PCWCLpguxfCzBIZUwy23R1t/XKewP+8mEN2Ck8Dtr4q20z6ce6SoA==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", + "integrity": "sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", - "@babel/parser": "^7.0.0", - "@babel/traverse": "^7.0.0", - "@babel/types": "^7.0.0", + "@babel/parser": "^7.7.0", + "@babel/traverse": "^7.7.0", + "@babel/types": "^7.7.0", "eslint-visitor-keys": "^1.0.0", "resolve": "^1.12.0" } @@ -405,12 +443,12 @@ "dev": true }, "cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", "dev": true, "requires": { - "restore-cursor": "^2.0.0" + "restore-cursor": "^3.1.0" } }, "cli-width": { @@ -452,9 +490,9 @@ "dev": true }, "core-js-pure": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.4.7.tgz", - "integrity": "sha512-Am3uRS8WCdTFA3lP7LtKR0PxgqYzjAMGKXaZKSNSC/8sqU0Wfq8R/YzoRs2rqtOVEunfgH+0q3O0BKOg0AvjPw==", + "version": "3.6.4", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.6.4.tgz", + "integrity": "sha512-epIhRLkXdgv32xIUFaaAry2wdxZYBi6bgM7cB136dzzXXa+dFyRLTZeLUJxnd8ShrmyVXBub63n2NHo2JAt8Cw==", "dev": true }, "cross-fetch": { @@ -489,9 +527,9 @@ } }, "damerau-levenshtein": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.5.tgz", - "integrity": "sha512-CBCRqFnpu715iPmw1KrdOrzRqbdFwQTwAWyyyYS42+iAgHCuXZ+/TdMgQkUENPomxEz9z1BEzuQU2Xw0kUuAgA==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.6.tgz", + "integrity": "sha512-JVrozIeElnj3QzfUIt8tB8YMluBJom4Vw9qTPpjGYQ9fYlB3D/rb6OordUxf3xeFB35LKWs0xqcO5U6ySvBtug==", "dev": true }, "debug": { @@ -528,9 +566,9 @@ } }, "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, "error-ex": { @@ -543,21 +581,22 @@ } }, "es-abstract": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.16.2.tgz", - "integrity": "sha512-jYo/J8XU2emLXl3OLwfwtuFfuF2w6DYPs+xy9ZfVyPkDcrauu6LYrw/q2TyCtrbc/KUdCiC5e9UajRhgNkVopA==", + "version": "1.17.5", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", + "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", "dev": true, "requires": { "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", "has": "^1.0.3", "has-symbols": "^1.0.1", - "is-callable": "^1.1.4", - "is-regex": "^1.0.4", + "is-callable": "^1.1.5", + "is-regex": "^1.0.5", "object-inspect": "^1.7.0", "object-keys": "^1.1.1", - "string.prototype.trimleft": "^2.1.0", - "string.prototype.trimright": "^2.1.0" + "object.assign": "^4.1.0", + "string.prototype.trimleft": "^2.1.1", + "string.prototype.trimright": "^2.1.1" } }, "es-to-primitive": { @@ -578,65 +617,48 @@ "dev": true }, "eslint": { - "version": "5.16.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", - "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-6.8.0.tgz", + "integrity": "sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", - "ajv": "^6.9.1", + "ajv": "^6.10.0", "chalk": "^2.1.0", "cross-spawn": "^6.0.5", "debug": "^4.0.1", "doctrine": "^3.0.0", - "eslint-scope": "^4.0.3", - "eslint-utils": "^1.3.1", - "eslint-visitor-keys": "^1.0.0", - "espree": "^5.0.1", + "eslint-scope": "^5.0.0", + "eslint-utils": "^1.4.3", + "eslint-visitor-keys": "^1.1.0", + "espree": "^6.1.2", "esquery": "^1.0.1", "esutils": "^2.0.2", "file-entry-cache": "^5.0.1", "functional-red-black-tree": "^1.0.1", - "glob": "^7.1.2", - "globals": "^11.7.0", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", "ignore": "^4.0.6", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", - "inquirer": "^6.2.2", - "js-yaml": "^3.13.0", + "inquirer": "^7.0.0", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.3.0", - "lodash": "^4.17.11", + "lodash": "^4.17.14", "minimatch": "^3.0.4", "mkdirp": "^0.5.1", "natural-compare": "^1.4.0", - "optionator": "^0.8.2", - "path-is-inside": "^1.0.2", + "optionator": "^0.8.3", "progress": "^2.0.0", "regexpp": "^2.0.1", - "semver": "^5.5.1", - "strip-ansi": "^4.0.0", - "strip-json-comments": "^2.0.1", + "semver": "^6.1.2", + "strip-ansi": "^5.2.0", + "strip-json-comments": "^3.0.1", "table": "^5.2.3", - "text-table": "^0.2.0" - }, - "dependencies": { - "eslint-scope": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", - "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", - "dev": true, - "requires": { - "esrecurse": "^4.1.0", - "estraverse": "^4.1.1" - } - }, - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true - } + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" } }, "eslint-config-prettier": { @@ -649,13 +671,13 @@ } }, "eslint-import-resolver-node": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.2.tgz", - "integrity": "sha512-sfmTqJfPSizWu4aymbPr4Iidp5yKm8yDkHp+Ir3YiTHiiDfxh69mOUsmiqW6RZ9zRXFaF64GtYmN7e+8GHBv6Q==", + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.3.tgz", + "integrity": "sha512-b8crLDo0M5RSe5YG8Pu2DYBj71tSB6OvXkfzwbJU2w7y8P4/yo0MyF8jU26IEuEuHF2K5/gcAJE3LhQGqBBbVg==", "dev": true, "requires": { "debug": "^2.6.9", - "resolve": "^1.5.0" + "resolve": "^1.13.1" }, "dependencies": { "debug": { @@ -676,12 +698,12 @@ } }, "eslint-module-utils": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.4.1.tgz", - "integrity": "sha512-H6DOj+ejw7Tesdgbfs4jeS4YMFrT8uI8xwd1gtQqXssaR0EQ26L+2O/w6wkYFy2MymON0fTwHmXBvvfLNZVZEw==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.0.tgz", + "integrity": "sha512-6j9xxegbqe8/kZY8cYpcp0xhbK0EgJlg3g9mib3/miLaExuuwc3n5UEfSnU6hWMbT0FAYVvDbL9RrRgpUeQIvA==", "dev": true, "requires": { - "debug": "^2.6.8", + "debug": "^2.6.9", "pkg-dir": "^2.0.0" }, "dependencies": { @@ -720,16 +742,10 @@ } } }, - "eslint-plugin-eslint-plugin": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-eslint-plugin/-/eslint-plugin-eslint-plugin-2.1.0.tgz", - "integrity": "sha512-kT3A/ZJftt28gbl/Cv04qezb/NQ1dwYIbi8lyf806XMxkus7DvOVCLIfTXMrorp322Pnoez7+zabXH29tADIDg==", - "dev": true - }, "eslint-plugin-flowtype": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-flowtype/-/eslint-plugin-flowtype-4.5.2.tgz", - "integrity": "sha512-ByV0EtEQOqiCl6bsrtXtTGnXlIXoyvDrvUq3Nz28huODAhnRDuMotyTrwP+TjAKZMPWbtaNGFHMoUxW3DktGOw==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-flowtype/-/eslint-plugin-flowtype-4.7.0.tgz", + "integrity": "sha512-M+hxhSCk5QBEValO5/UqrS4UunT+MgplIJK5wA1sCtXjzBcZkpTGRwxmLHhGpbHcrmQecgt6ZL/KDdXWqGB7VA==", "dev": true, "requires": { "lodash": "^4.17.15" @@ -794,6 +810,27 @@ "semver": "5.5.0" } }, + "ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true + }, + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "requires": { + "restore-cursor": "^2.0.0" + } + }, "eslint-scope": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", @@ -804,18 +841,106 @@ "estraverse": "^4.1.1" } }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "inquirer": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", + "dev": true, + "requires": { + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^2.0.0", + "lodash": "^4.17.12", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "dev": true + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + }, "semver": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", "dev": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "dependencies": { + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + } + } } } }, "eslint-plugin-graphql": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-graphql/-/eslint-plugin-graphql-3.1.0.tgz", - "integrity": "sha512-87HGS00aeBqGFiQZQGzSPzk1D59w+124F8CRIDATh3LJqce5RCTuUI4tcIqPeyY95YPBCIKwISksWUuA0nrgNw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-graphql/-/eslint-plugin-graphql-3.1.1.tgz", + "integrity": "sha512-VNu2AipS8P1BAnE/tcJ2EmBWjFlCnG+1jKdUlFNDQjocWZlFiPpMu9xYNXePoEXK+q+jG51M/6PdhOjEgJZEaQ==", "dev": true, "requires": { "graphql-config": "^2.0.1", @@ -823,22 +948,23 @@ } }, "eslint-plugin-import": { - "version": "2.18.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.18.2.tgz", - "integrity": "sha512-5ohpsHAiUBRNaBWAF08izwUGlbrJoJJ+W9/TBwsGoR1MnlgfwMIKrFeSjWbt6moabiXW9xNvtFz+97KHRfI4HQ==", + "version": "2.20.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.20.2.tgz", + "integrity": "sha512-FObidqpXrR8OnCh4iNsxy+WACztJLXAHBO5hK79T1Hc77PgQZkyDGA5Ag9xAvRpglvLNxhH/zSmZ70/pZ31dHg==", "dev": true, "requires": { "array-includes": "^3.0.3", + "array.prototype.flat": "^1.2.1", "contains-path": "^0.1.0", "debug": "^2.6.9", "doctrine": "1.5.0", "eslint-import-resolver-node": "^0.3.2", - "eslint-module-utils": "^2.4.0", + "eslint-module-utils": "^2.4.1", "has": "^1.0.3", "minimatch": "^3.0.4", "object.values": "^1.1.0", "read-pkg-up": "^2.0.0", - "resolve": "^1.11.0" + "resolve": "^1.12.0" }, "dependencies": { "debug": { @@ -879,51 +1005,12 @@ } }, "eslint-plugin-jest": { - "version": "22.21.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-22.21.0.tgz", - "integrity": "sha512-OaqnSS7uBgcGiqXUiEnjoqxPNKvR4JWG5mSRkzVoR6+vDwlqqp11beeql1hYs0HTbdhiwrxWLxbX0Vx7roG3Ew==", + "version": "23.8.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-23.8.2.tgz", + "integrity": "sha512-xwbnvOsotSV27MtAe7s8uGWOori0nUsrXh2f1EnpmXua8sDfY6VZhHAhHg2sqK7HBNycRQExF074XSZ7DvfoFg==", "dev": true, "requires": { - "@typescript-eslint/experimental-utils": "^1.13.0" - }, - "dependencies": { - "@typescript-eslint/experimental-utils": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-1.13.0.tgz", - "integrity": "sha512-zmpS6SyqG4ZF64ffaJ6uah6tWWWgZ8m+c54XXgwFtUv0jNz8aJAVx8chMCvnk7yl6xwn8d+d96+tWp7fXzTuDg==", - "dev": true, - "requires": { - "@types/json-schema": "^7.0.3", - "@typescript-eslint/typescript-estree": "1.13.0", - "eslint-scope": "^4.0.0" - } - }, - "@typescript-eslint/typescript-estree": { - "version": "1.13.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-1.13.0.tgz", - "integrity": "sha512-b5rCmd2e6DCC6tCTN9GSUAuxdYwCM/k/2wdjHGrIRGPSJotWMCe/dGpi66u42bhuh8q3QBzqM4TMA1GUUCJvdw==", - "dev": true, - "requires": { - "lodash.unescape": "4.0.1", - "semver": "5.5.0" - } - }, - "eslint-scope": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", - "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", - "dev": true, - "requires": { - "esrecurse": "^4.1.0", - "estraverse": "^4.1.1" - } - }, - "semver": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", - "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", - "dev": true - } + "@typescript-eslint/experimental-utils": "^2.5.0" } }, "eslint-plugin-jsx-a11y": { @@ -941,33 +1028,43 @@ "emoji-regex": "^7.0.2", "has": "^1.0.3", "jsx-ast-utils": "^2.2.1" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + } } }, "eslint-plugin-prettier": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.1.tgz", - "integrity": "sha512-A+TZuHZ0KU0cnn56/9mfR7/KjUJ9QNVXUhwvRFSR7PGPe0zQR6PTkmyqg1AtUUEOzTqeRsUwyKFh0oVZKVCrtA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.2.tgz", + "integrity": "sha512-GlolCC9y3XZfv3RQfwGew7NnuFDKsfI4lbvRK+PIIo23SFH+LemGs4cKwzAaRa+Mdb+lQO/STaIayno8T5sJJA==", "dev": true, "requires": { "prettier-linter-helpers": "^1.0.0" } }, "eslint-plugin-react": { - "version": "7.17.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.17.0.tgz", - "integrity": "sha512-ODB7yg6lxhBVMeiH1c7E95FLD4E/TwmFjltiU+ethv7KPdCwgiFuOZg9zNRHyufStTDLl/dEFqI2Q1VPmCd78A==", + "version": "7.19.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.19.0.tgz", + "integrity": "sha512-SPT8j72CGuAP+JFbT0sJHOB80TX/pu44gQ4vXH/cq+hQTiY2PuZ6IHkqXJV6x1b28GDdo1lbInjKUrrdUf0LOQ==", "dev": true, "requires": { - "array-includes": "^3.0.3", + "array-includes": "^3.1.1", "doctrine": "^2.1.0", - "eslint-plugin-eslint-plugin": "^2.1.0", "has": "^1.0.3", "jsx-ast-utils": "^2.2.3", - "object.entries": "^1.1.0", - "object.fromentries": "^2.0.1", - "object.values": "^1.1.0", + "object.entries": "^1.1.1", + "object.fromentries": "^2.0.2", + "object.values": "^1.1.1", "prop-types": "^15.7.2", - "resolve": "^1.13.1" + "resolve": "^1.15.1", + "semver": "^6.3.0", + "string.prototype.matchall": "^4.0.2", + "xregexp": "^4.3.0" }, "dependencies": { "doctrine": { @@ -982,12 +1079,12 @@ } }, "eslint-plugin-relay": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-relay/-/eslint-plugin-relay-1.4.1.tgz", - "integrity": "sha512-yb+p+4AxZTi2gXN7cZRfXMBFlRa5j6TtiVeq3yHXyy+tlgYNpxi/dDrP1+tcUTNP9vdaJovnfGZ5jp6kMiH9eg==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-relay/-/eslint-plugin-relay-1.7.0.tgz", + "integrity": "sha512-JmAMQFr9CxXFLo5BppdN/sleofrE1J/cERIgkFqnYdTq0KAeUNGnz3jO41cqcp1y92/D+KJdmEKFsPfnqnDByQ==", "dev": true, "requires": { - "graphql": "^14.0.0" + "graphql": "^14.0.0 | ^15.0.0-rc.1" } }, "eslint-rule-documentation": { @@ -1022,14 +1119,14 @@ "dev": true }, "espree": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", - "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", "dev": true, "requires": { - "acorn": "^6.0.7", - "acorn-jsx": "^5.0.0", - "eslint-visitor-keys": "^1.0.0" + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" } }, "esprima": { @@ -1039,12 +1136,20 @@ "dev": true }, "esquery": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.0.1.tgz", - "integrity": "sha512-SmiyZ5zIWH9VM+SRUReLS5Q8a7GxtRdxEBVZpm98rJM7Sb+A9DVCndXfkeFUd3byderg+EbDkfnevfCwynWaNA==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.2.0.tgz", + "integrity": "sha512-weltsSqdeWIX9G2qQZz7KlTRJdkkOCTPgLYJUz1Hacf48R4YOwGPHO3+ORfWedqJKbq5WQmsgK90n+pFLIKt/Q==", "dev": true, "requires": { - "estraverse": "^4.0.0" + "estraverse": "^5.0.0" + }, + "dependencies": { + "estraverse": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.0.0.tgz", + "integrity": "sha512-j3acdrMzqrxmJTNj5dbr1YbjacrYgAxVMeF0gK16E3j494mOe7xygM/ZLIguEQ0ETwAg2hlJCtHRGav+y0Ny5A==", + "dev": true + } } }, "esrecurse": { @@ -1080,9 +1185,9 @@ } }, "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", + "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==", "dev": true }, "fast-diff": { @@ -1092,9 +1197,9 @@ "dev": true }, "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true }, "fast-levenshtein": { @@ -1104,9 +1209,9 @@ "dev": true }, "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", "dev": true, "requires": { "escape-string-regexp": "^1.0.5" @@ -1142,9 +1247,9 @@ } }, "flatted": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz", - "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", "dev": true }, "fs.realpath": { @@ -1185,11 +1290,23 @@ "path-is-absolute": "^1.0.0" } }, + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, "globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } }, "graceful-fs": { "version": "4.2.3", @@ -1198,18 +1315,15 @@ "dev": true }, "graphql": { - "version": "14.5.8", - "resolved": "https://registry.npmjs.org/graphql/-/graphql-14.5.8.tgz", - "integrity": "sha512-MMwmi0zlVLQKLdGiMfWkgQD7dY/TUKt4L+zgJ/aR0Howebod3aNgP5JkgvAULiR2HPVZaP2VEElqtdidHweLkg==", - "dev": true, - "requires": { - "iterall": "^1.2.2" - } + "version": "15.0.0-rc.2", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-15.0.0-rc.2.tgz", + "integrity": "sha512-X9ZybETBiZ5zndyXm/Yn3dd0nJqiCNZ7w06lnd0zMiCtBR/KQGgxJmnf47Y/P/Fy7JXM4QDF+MeeoH724yc3DQ==", + "dev": true }, "graphql-config": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/graphql-config/-/graphql-config-2.2.1.tgz", - "integrity": "sha512-U8+1IAhw9m6WkZRRcyj8ZarK96R6lQBQ0an4lp76Ps9FyhOXENC5YQOxOFGm5CxPrX2rD0g3Je4zG5xdNJjwzQ==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/graphql-config/-/graphql-config-2.2.2.tgz", + "integrity": "sha512-mtv1ejPyyR2mJUUZNhljggU+B/Xl8tJJWf+h145hB+1Y48acSghFalhNtXfPBcYl2tJzpb+lGxfj3O7OjaiMgw==", "dev": true, "requires": { "graphql-import": "^0.7.1", @@ -1260,9 +1374,9 @@ "dev": true }, "hosted-git-info": { - "version": "2.8.5", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.5.tgz", - "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==", + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", + "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", "dev": true }, "iconv-lite": { @@ -1313,37 +1427,98 @@ "dev": true }, "inquirer": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", - "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.1.0.tgz", + "integrity": "sha512-5fJMWEmikSYu0nv/flMc475MhGbB7TSPd/2IpFV4I4rMklboCH2rQjYY5kKiYGHqUF9gvaambupcJFFG9dvReg==", "dev": true, "requires": { - "ansi-escapes": "^3.2.0", - "chalk": "^2.4.2", - "cli-cursor": "^2.1.0", + "ansi-escapes": "^4.2.1", + "chalk": "^3.0.0", + "cli-cursor": "^3.1.0", "cli-width": "^2.0.0", "external-editor": "^3.0.3", - "figures": "^2.0.0", - "lodash": "^4.17.12", - "mute-stream": "0.0.7", - "run-async": "^2.2.0", - "rxjs": "^6.4.0", - "string-width": "^2.1.0", - "strip-ansi": "^5.1.0", + "figures": "^3.0.0", + "lodash": "^4.17.15", + "mute-stream": "0.0.8", + "run-async": "^2.4.0", + "rxjs": "^6.5.3", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", "through": "^2.3.6" }, "dependencies": { + "ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "requires": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "requires": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.0" + } + }, + "supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" } } } }, + "internal-slot": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.2.tgz", + "integrity": "sha512-2cQNfwhAfJIkU4KZPkDI+Gj5yNNnbqi40W9Gge6dfnk4TocEVm00B3bdiL+JINrbGJil2TeHvM4rETGzk/f/0g==", + "dev": true, + "requires": { + "es-abstract": "^1.17.0-next.1", + "has": "^1.0.3", + "side-channel": "^1.0.2" + } + }, "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -1351,15 +1526,15 @@ "dev": true }, "is-callable": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", - "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==", + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", + "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", "dev": true }, "is-date-object": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", - "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", + "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", "dev": true }, "is-extglob": { @@ -1369,9 +1544,9 @@ "dev": true }, "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, "is-glob": { @@ -1390,14 +1565,20 @@ "dev": true }, "is-regex": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", - "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", + "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", "dev": true, "requires": { - "has": "^1.0.1" + "has": "^1.0.3" } }, + "is-string": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", + "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", + "dev": true + }, "is-symbol": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", @@ -1419,12 +1600,6 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, - "iterall": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/iterall/-/iterall-1.2.2.tgz", - "integrity": "sha512-yynBb1g+RFUPY64fTrFv7nsjRrENBQJaX2UL+2Szc9REFrSNm1rpSXHGzhmAy7a9uv3vlvgBlXnf9RqmPH1/DA==", - "dev": true - }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -1529,9 +1704,9 @@ } }, "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, "minimatch": { @@ -1543,18 +1718,18 @@ } }, "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", "dev": true }, "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "dev": true, "requires": { - "minimist": "0.0.8" + "minimist": "^1.2.5" } }, "ms": { @@ -1564,9 +1739,9 @@ "dev": true }, "mute-stream": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", - "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, "natural-compare": { @@ -1638,37 +1813,37 @@ } }, "object.entries": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.0.tgz", - "integrity": "sha512-l+H6EQ8qzGRxbkHOd5I/aHRhHDKoQXQ8g0BYt4uSweQU1/J6dZUOyWh9a2Vky35YCKjzmgxOzta2hH6kf9HuXA==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.1.tgz", + "integrity": "sha512-ilqR7BgdyZetJutmDPfXCDffGa0/Yzl2ivVNpbx/g4UeWrCdRnFDUBrKJGLhGieRHDATnyZXWBeCb29k9CJysQ==", "dev": true, "requires": { "define-properties": "^1.1.3", - "es-abstract": "^1.12.0", + "es-abstract": "^1.17.0-next.1", "function-bind": "^1.1.1", "has": "^1.0.3" } }, "object.fromentries": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.1.tgz", - "integrity": "sha512-PUQv8Hbg3j2QX0IQYv3iAGCbGcu4yY4KQ92/dhA4sFSixBmSmp13UpDLs6jGK8rBtbmhNNIK99LD2k293jpiGA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.2.tgz", + "integrity": "sha512-r3ZiBH7MQppDJVLx6fhD618GKNG40CZYH9wgwdhKxBDDbQgjeWGGd4AtkZad84d291YxvWe7bJGuE65Anh0dxQ==", "dev": true, "requires": { "define-properties": "^1.1.3", - "es-abstract": "^1.15.0", + "es-abstract": "^1.17.0-next.1", "function-bind": "^1.1.1", "has": "^1.0.3" } }, "object.values": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.0.tgz", - "integrity": "sha512-8mf0nKLAoFX6VlNVdhGj31SVYpaNFtUnuoOXWyFEstsWRgU837AK+JYM0iAxwkSzGRbwn8cbFmgbyxj1j4VbXg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.1.tgz", + "integrity": "sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA==", "dev": true, "requires": { "define-properties": "^1.1.3", - "es-abstract": "^1.12.0", + "es-abstract": "^1.17.0-next.1", "function-bind": "^1.1.1", "has": "^1.0.3" } @@ -1683,12 +1858,12 @@ } }, "onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", + "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", "dev": true, "requires": { - "mimic-fn": "^1.0.0" + "mimic-fn": "^2.1.0" } }, "optionator": { @@ -1765,12 +1940,6 @@ "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, - "path-is-inside": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", - "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", - "dev": true - }, "path-key": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", @@ -1852,9 +2021,9 @@ "dev": true }, "react-is": { - "version": "16.12.0", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.12.0.tgz", - "integrity": "sha512-rPCkf/mWBtKc97aLL9/txD8DZdemK0vkA3JMLShjlJB3Pj3s+lpf1KaBzMfQrAmhMQB0n1cU/SUGgKKBCe837Q==", + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "dev": true }, "read-pkg": { @@ -1910,9 +2079,9 @@ } }, "p-limit": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.1.tgz", - "integrity": "sha512-85Tk+90UCVWvbDavCLKPOLC9vvY8OwEX/RtKF+/1OADJMVlFfEHOiMTPVyxg7mk/dKa+ipdHm0OUkTvCpMTuwg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", "dev": true, "requires": { "p-try": "^2.0.0" @@ -1972,11 +2141,21 @@ } }, "regenerator-runtime": { - "version": "0.13.3", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz", - "integrity": "sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==", + "version": "0.13.5", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz", + "integrity": "sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==", "dev": true }, + "regexp.prototype.flags": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz", + "integrity": "sha512-2+Q0C5g951OlYlJz6yu5/M33IcsESLlLfsyIaLJaG4FA2r4yP8MvVMJUUP/fVBkSpbbbZlS5gynbEWLipiiXiQ==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + } + }, "regexpp": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", @@ -1984,9 +2163,9 @@ "dev": true }, "resolve": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.13.1.tgz", - "integrity": "sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==", + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.1.tgz", + "integrity": "sha512-84oo6ZTtoTUpjgNEr5SJyzQhzL72gaRodsSfyxC/AXRvwu0Yse9H8eF9IpGo7b8YetZhlI6v7ZQ6bKBFV/6S7w==", "dev": true, "requires": { "path-parse": "^1.0.6" @@ -1999,12 +2178,12 @@ "dev": true }, "restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", "dev": true, "requires": { - "onetime": "^2.0.0", + "onetime": "^5.1.0", "signal-exit": "^3.0.2" } }, @@ -2018,18 +2197,18 @@ } }, "run-async": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", - "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.0.tgz", + "integrity": "sha512-xJTbh/d7Lm7SBhc1tNvTpeCHaEzoyxPrqNlvSdMfBTYwaY++UJFyXUOxAtsRUXjlqOfj8luNaR9vjCh4KeV+pg==", "dev": true, "requires": { "is-promise": "^2.1.0" } }, "rxjs": { - "version": "6.5.3", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.3.tgz", - "integrity": "sha512-wuYsAYYFdWTAnAaPoKGNhfpWwKZbJW+HgAJ+mImp+Epl7BG8oNWBCTyRM8gba9k4lk8BgWdoYm21Mo/RYhhbgA==", + "version": "6.5.5", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz", + "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==", "dev": true, "requires": { "tslib": "^1.9.0" @@ -2062,10 +2241,20 @@ "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", "dev": true }, + "side-channel": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.2.tgz", + "integrity": "sha512-7rL9YlPHg7Ancea1S96Pa8/QWb4BtXL/TZvS6B8XFetGBeuhAsfmUspK6DokBeZ64+Kj9TCNRD/30pVz1BvQNA==", + "dev": true, + "requires": { + "es-abstract": "^1.17.0-next.1", + "object-inspect": "^1.7.0" + } + }, "signal-exit": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, "slice-ansi": { @@ -2077,6 +2266,14 @@ "ansi-styles": "^3.2.0", "astral-regex": "^1.0.0", "is-fullwidth-code-point": "^2.0.0" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + } } }, "source-map": { @@ -2124,48 +2321,96 @@ "dev": true }, "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", "dev": true, "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + } + } + }, + "string.prototype.matchall": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.2.tgz", + "integrity": "sha512-N/jp6O5fMf9os0JU3E72Qhf590RSRZU/ungsL/qJUYVTNv7hTG0P/dbPjxINVN9jpscu3nzYwKESU3P3RY5tOg==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0", + "has-symbols": "^1.0.1", + "internal-slot": "^1.0.2", + "regexp.prototype.flags": "^1.3.0", + "side-channel": "^1.0.2" + } + }, + "string.prototype.trimend": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.0.tgz", + "integrity": "sha512-EEJnGqa/xNfIg05SxiPSqRS7S9qwDhYts1TSLR1BQfYUfPe1stofgGKvwERK9+9yf+PpfBMlpBaCHucXGPQfUA==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" } }, "string.prototype.trimleft": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.0.tgz", - "integrity": "sha512-FJ6b7EgdKxxbDxc79cOlok6Afd++TTs5szo+zJTUyow3ycrRfJVE2pq3vcN53XexvKZu/DJMDfeI/qMiZTrjTw==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", + "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", "dev": true, "requires": { "define-properties": "^1.1.3", - "function-bind": "^1.1.1" + "es-abstract": "^1.17.5", + "string.prototype.trimstart": "^1.0.0" } }, "string.prototype.trimright": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.0.tgz", - "integrity": "sha512-fXZTSV55dNBwv16uw+hh5jkghxSnc5oHq+5K/gXgizHwAvMetdAJlHqqoFC1FSDVPYWLkAKl2cxpUT41sV7nSg==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", + "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", "dev": true, "requires": { "define-properties": "^1.1.3", - "function-bind": "^1.1.1" + "es-abstract": "^1.17.5", + "string.prototype.trimend": "^1.0.0" + } + }, + "string.prototype.trimstart": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.0.tgz", + "integrity": "sha512-iCP8g01NFYiiBOnwG1Xc3WZLyoo+RuBymwIlWncShXDDJYWN6DbnM3odslBJdgCdRlq94B5s63NWAZlcn2CS4w==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" } }, "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", "dev": true, "requires": { - "ansi-regex": "^3.0.0" + "ansi-regex": "^4.1.0" }, "dependencies": { "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", "dev": true } } @@ -2177,9 +2422,9 @@ "dev": true }, "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.0.tgz", + "integrity": "sha512-e6/d0eBu7gHtdCqFt0xJr642LdToM5/cN4Qb9DbHjVx1CP5RyeM+zH7pbecEmDv/lBqb0QH+6Uqq75rxFPkM0w==", "dev": true }, "supports-color": { @@ -2192,9 +2437,9 @@ } }, "svg-element-attributes": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/svg-element-attributes/-/svg-element-attributes-1.3.0.tgz", - "integrity": "sha512-M4rTTZ186MY4/d3a4XNNuEptXOTIz5qeasp2D7gWVwIDa9e2wF1ccrFs9x7ZW6Sp4+ebCOt9GMCpccC3wt3srg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/svg-element-attributes/-/svg-element-attributes-1.3.1.tgz", + "integrity": "sha512-Bh05dSOnJBf3miNMqpsormfNtfidA/GxQVakhtn0T4DECWKeXQRQUceYjJ+OxYiiLdGe4Jo9iFV8wICFapFeIA==", "dev": true }, "table": { @@ -2209,6 +2454,18 @@ "string-width": "^3.0.0" }, "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, "string-width": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", @@ -2219,15 +2476,6 @@ "is-fullwidth-code-point": "^2.0.0", "strip-ansi": "^5.1.0" } - }, - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "dev": true, - "requires": { - "ansi-regex": "^4.1.0" - } } } }, @@ -2282,6 +2530,12 @@ "prelude-ls": "~1.1.2" } }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true + }, "typescript": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.2.tgz", @@ -2297,6 +2551,12 @@ "punycode": "^2.1.0" } }, + "v8-compile-cache": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", + "integrity": "sha512-usZBT3PW+LOjM25wbqIlZwPeJV+3OSz3M1k1Ws8snlW39dZyYL9lOGC5FgPVHfk0jKmjiDV8Z0mIbVQPiwFs7g==", + "dev": true + }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -2342,6 +2602,15 @@ "requires": { "mkdirp": "^0.5.1" } + }, + "xregexp": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.3.0.tgz", + "integrity": "sha512-7jXDIFXh5yJ/orPn4SXjuVrWWoi4Cr8jfV1eHv9CixKSbU+jY4mxfrBwAuDvupPNKpMUY+FeIqsVw/JLT9+B8g==", + "dev": true, + "requires": { + "@babel/runtime-corejs3": "^7.8.3" + } } } } diff --git a/src/Misc/expressionFunc/hashFiles/package.json b/src/Misc/expressionFunc/hashFiles/package.json index de7df8837ee..b650ba428ee 100644 --- a/src/Misc/expressionFunc/hashFiles/package.json +++ b/src/Misc/expressionFunc/hashFiles/package.json @@ -27,7 +27,7 @@ "@types/node": "^12.7.12", "@typescript-eslint/parser": "^2.8.0", "@zeit/ncc": "^0.20.5", - "eslint": "^5.16.0", + "eslint": "^6.8.0", "eslint-plugin-github": "^2.0.0", "prettier": "^1.19.1", "typescript": "^3.6.4" From 2cdde6cb16b3b9b589eb0c8beeb8f26d4a43a87b Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Tue, 7 Apr 2020 14:04:37 -0400 Subject: [PATCH 25/86] fix L0 test. --- src/Misc/dotnet-install.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Misc/dotnet-install.sh b/src/Misc/dotnet-install.sh index 31303d94dea..0c20299a59b 100755 --- a/src/Misc/dotnet-install.sh +++ b/src/Misc/dotnet-install.sh @@ -172,7 +172,7 @@ get_current_os_name() { return 0 elif [ "$uname" = "FreeBSD" ]; then echo "freebsd" - return 0 + return 0 elif [ "$uname" = "Linux" ]; then local linux_platform_name linux_platform_name="$(get_linux_platform_name)" || { echo "linux" && return 0 ; } @@ -728,11 +728,12 @@ downloadcurl() { # Append feed_credential as late as possible before calling curl to avoid logging feed_credential remote_path="${remote_path}${feed_credential}" + local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " local failed=false if [ -z "$out_path" ]; then - curl --retry 10 -sSL -f --create-dirs "$remote_path" || failed=true + curl $curl_options "$remote_path" || failed=true else - curl --retry 10 -sSL -f --create-dirs -o "$out_path" "$remote_path" || failed=true + curl $curl_options -o "$out_path" "$remote_path" || failed=true fi if [ "$failed" = true ]; then say_verbose "Curl download failed" @@ -748,12 +749,12 @@ downloadwget() { # Append feed_credential as late as possible before calling wget to avoid logging feed_credential remote_path="${remote_path}${feed_credential}" - + local wget_options="--tries 20 --waitretry 2 --connect-timeout 15 " local failed=false if [ -z "$out_path" ]; then - wget -q --tries 10 -O - "$remote_path" || failed=true + wget -q $wget_options -O - "$remote_path" || failed=true else - wget --tries 10 -O "$out_path" "$remote_path" || failed=true + wget $wget_options -O "$out_path" "$remote_path" || failed=true fi if [ "$failed" = true ]; then say_verbose "Wget download failed" From d90273a068f8c8156078c1e85c9d5ed47406ed7b Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 8 Apr 2020 11:17:54 -0400 Subject: [PATCH 26/86] Raise warning when volume mount root. (#413) --- src/Runner.Worker/ContainerOperationProvider.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index a476b54646b..c105e2162f1 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -47,7 +47,7 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec condition: $"{PipelineTemplateConstants.Always}()", displayName: "Stop containers", data: data); - + executionContext.Debug($"Register post job cleanup for stopping/deleting containers."); executionContext.RegisterPostJobStep(nameof(StopContainersAsync), postJobStep); @@ -180,6 +180,11 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta foreach (var volume in container.UserMountVolumes) { Trace.Info($"User provided volume: {volume.Value}"); + var mount = new MountVolume(volume.Value); + if (string.Equals(mount.SourceVolumePath, "/", StringComparison.OrdinalIgnoreCase)) + { + executionContext.Warning($"Volume mount {volume.Value} is going to mount '/' into the container which may cause file ownership change in the entire file system and cause Actions Runner to lose permission to access the disk."); + } } // Pull down docker image with retry up to 3 times From 7817e1a97608d0a326b6c98d1157a67eba49603b Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Wed, 8 Apr 2020 11:32:56 -0400 Subject: [PATCH 27/86] Prepare 2.169.0 runner release for GHES Alpha. --- releaseNote.md | 23 +++++------------------ src/runnerversion | 2 +- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 18452f965f9..6a9eb4606b0 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,24 +1,11 @@ ## Features - - Update Runner Register GitHub API URL to Support Org-level Runner (#339 #345 #352) - - Preserve workflow file/line/column for better error messages (#356) - - Switch to use token service instead of SPS for exchanging oauth token. (#325) - - Load and print machine setup info from .setup_info (#364) - - Expose job name as $GITHUB_JOB (#366) - - Add support for job outputs. (#365) - - Set CI=true when launch process in actions runner. (#374) - - Set steps..outcome and steps..conclusion. (#372) - - Add support for workflow/job defaults. (#369) - - Expose GITHUB_REPOSITORY_OWNER and ${{github.repository_owner}}. (#378) - + - Runner support for GHES Alpha (#381 #386 #390 #393 $401) + - Allow secrets context in Container.env (#388) ## Bugs - - Use authenticate endpoint for testing runner connection. (#311) - - Commands translate file path from container action (#331) - - Change problem matchers output to debug (#363) - - Switch hashFiles to extension function (#362) - - Add expanded volumes strings to container mounts (#384) - + - Raise warning when volume mount root. (#413) + - Fix typo (#394) ## Misc - - Add runner auth documentation (#357) + - N/A ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows diff --git a/src/runnerversion b/src/runnerversion index 2973ad9e470..5f7924ad30f 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.168.0 +2.169.0 From baa6ded3bc446cadbed72ce4da91df26740d5a29 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 9 Apr 2020 14:33:16 -0400 Subject: [PATCH 28/86] Better Kusto Tracing for self-hosted runner. (#405) --- src/Runner.Common/HostContext.cs | 17 ++++++-- src/Runner.Common/RunnerServer.cs | 6 +-- .../Configuration/ConfigurationManager.cs | 2 +- src/Runner.Listener/JobDispatcher.cs | 32 +++++++++++---- src/Runner.Listener/Runner.cs | 2 +- src/Runner.Sdk/Util/VssUtil.cs | 4 +- src/Runner.Worker/ActionManager.cs | 2 +- src/Runner.Worker/Worker.cs | 2 +- .../DTWebApi/WebApi/TaskAgentHttpClient.cs | 28 ++++++++++++- .../WebApi/Jwt/JsonWebTokenUtilities.cs | 2 +- src/Test/L0/Listener/JobDispatcherL0.cs | 40 +++++++++---------- src/Test/L0/TestHostContext.cs | 2 +- 12 files changed, 95 insertions(+), 44 deletions(-) diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 99e152b1d6f..4da520913bc 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -24,7 +24,7 @@ public interface IHostContext : IDisposable CancellationToken RunnerShutdownToken { get; } ShutdownReason RunnerShutdownReason { get; } ISecretMasker SecretMasker { get; } - ProductInfoHeaderValue UserAgent { get; } + List UserAgents { get; } RunnerWebProxy WebProxy { get; } string GetDirectory(WellKnownDirectory directory); string GetConfigFile(WellKnownConfigFile configFile); @@ -54,7 +54,7 @@ public sealed class HostContext : EventListener, IObserver, private readonly ConcurrentDictionary _serviceInstances = new ConcurrentDictionary(); private readonly ConcurrentDictionary _serviceTypes = new ConcurrentDictionary(); private readonly ISecretMasker _secretMasker = new SecretMasker(); - private readonly ProductInfoHeaderValue _userAgent = new ProductInfoHeaderValue($"GitHubActionsRunner-{BuildConstants.RunnerPackage.PackageName}", BuildConstants.RunnerPackage.Version); + private readonly List _userAgents = new List() { new ProductInfoHeaderValue($"GitHubActionsRunner-{BuildConstants.RunnerPackage.PackageName}", BuildConstants.RunnerPackage.Version) }; private CancellationTokenSource _runnerShutdownTokenSource = new CancellationTokenSource(); private object _perfLock = new object(); private Tracing _trace; @@ -72,7 +72,7 @@ public sealed class HostContext : EventListener, IObserver, public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token; public ShutdownReason RunnerShutdownReason { get; private set; } public ISecretMasker SecretMasker => _secretMasker; - public ProductInfoHeaderValue UserAgent => _userAgent; + public List UserAgents => _userAgents; public RunnerWebProxy WebProxy => _webProxy; public HostContext(string hostType, string logFile = null) { @@ -189,6 +189,17 @@ public HostContext(string hostType, string logFile = null) { _trace.Info($"No proxy settings were found based on environmental variables (http_proxy/https_proxy/HTTP_PROXY/HTTPS_PROXY)"); } + + var credFile = GetConfigFile(WellKnownConfigFile.Credentials); + if (File.Exists(credFile)) + { + var credData = IOUtil.LoadObject(credFile); + if (credData != null && + credData.Data.TryGetValue("clientId", out var clientId)) + { + _userAgents.Add(new ProductInfoHeaderValue($"RunnerId", clientId)); + } + } } public string GetDirectory(WellKnownDirectory directory) diff --git a/src/Runner.Common/RunnerServer.cs b/src/Runner.Common/RunnerServer.cs index 7b244db0ed7..cbdcb898c29 100644 --- a/src/Runner.Common/RunnerServer.cs +++ b/src/Runner.Common/RunnerServer.cs @@ -41,7 +41,7 @@ public interface IRunnerServer : IRunnerService // job request Task GetAgentRequestAsync(int poolId, long requestId, CancellationToken cancellationToken); - Task RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, CancellationToken cancellationToken); + Task RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId, CancellationToken cancellationToken); Task FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, CancellationToken cancellationToken); // agent package @@ -300,10 +300,10 @@ public Task GetAgentMessageAsync(Int32 poolId, Guid sessionId, // JobRequest //----------------------------------------------------------------- - public Task RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, CancellationToken cancellationToken = default(CancellationToken)) + public Task RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId = null, CancellationToken cancellationToken = default(CancellationToken)) { CheckConnection(RunnerConnectionType.JobRequest); - return _requestTaskAgentClient.RenewAgentRequestAsync(poolId, requestId, lockToken, cancellationToken: cancellationToken); + return _requestTaskAgentClient.RenewAgentRequestAsync(poolId, requestId, lockToken, orchestrationId: orchestrationId, cancellationToken: cancellationToken); } public Task FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, CancellationToken cancellationToken = default(CancellationToken)) diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index e2b8ca31cd1..c2d4a60cd18 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -511,7 +511,7 @@ private async Task GetTenantCredential(string githubUrl, strin using (var httpClient = new HttpClient(httpClientHandler)) { httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("RemoteAuth", githubToken); - httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent); + httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); var bodyObject = new Dictionary() { diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index 00d31b11649..706044c55ae 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -12,6 +12,7 @@ using GitHub.Services.Common; using GitHub.Runner.Common; using GitHub.Runner.Sdk; +using GitHub.Services.WebApi.Jwt; namespace GitHub.Runner.Listener { @@ -86,15 +87,30 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce } } + var orchestrationId = string.Empty; + var systemConnection = jobRequestMessage.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + if (systemConnection?.Authorization != null && + systemConnection.Authorization.Parameters.TryGetValue("AccessToken", out var accessToken) && + !string.IsNullOrEmpty(accessToken)) + { + var jwt = JsonWebToken.Create(accessToken); + var claims = jwt.ExtractClaims(); + orchestrationId = claims.FirstOrDefault(x => string.Equals(x.Type, "orchid", StringComparison.OrdinalIgnoreCase))?.Value; + if (!string.IsNullOrEmpty(orchestrationId)) + { + Trace.Info($"Pull OrchestrationId {orchestrationId} from JWT claims"); + } + } + WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId); if (runOnce) { Trace.Info("Start dispatcher for one time used runner."); - newDispatch.WorkerDispatch = RunOnceAsync(jobRequestMessage, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token); + newDispatch.WorkerDispatch = RunOnceAsync(jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token); } else { - newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token); + newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, orchestrationId, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token, newDispatch.WorkerCancelTimeoutKillTokenSource.Token); } _jobInfos.TryAdd(newDispatch.JobId, newDispatch); @@ -284,11 +300,11 @@ private async Task EnsureDispatchFinished(WorkerDispatcher jobDispatch, bool can } } - private async Task RunOnceAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken) + private async Task RunOnceAsync(Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken) { try { - await RunAsync(message, previousJobDispatch, jobRequestCancellationToken, workerCancelTimeoutKillToken); + await RunAsync(message, orchestrationId, previousJobDispatch, jobRequestCancellationToken, workerCancelTimeoutKillToken); } finally { @@ -297,7 +313,7 @@ private async Task RunOnceAsync(Pipelines.AgentJobRequestMessage message, Worker } } - private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken) + private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orchestrationId, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken) { Busy = true; try @@ -328,7 +344,7 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDisp // start renew job request Trace.Info($"Start renew job request {requestId} for job {message.JobId}."); - Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token); + Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, orchestrationId, firstJobRequestRenewed, lockRenewalTokenSource.Token); // wait till first renew succeed or job request is canceled // not even start worker if the first renew fail @@ -607,7 +623,7 @@ await processChannel.SendAsync( } } - public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToken, TaskCompletionSource firstJobRequestRenewed, CancellationToken token) + public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId, TaskCompletionSource firstJobRequestRenewed, CancellationToken token) { var runnerServer = HostContext.GetService(); TaskAgentJobRequest request = null; @@ -620,7 +636,7 @@ public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToke { try { - request = await runnerServer.RenewAgentRequestAsync(poolId, requestId, lockToken, token); + request = await runnerServer.RenewAgentRequestAsync(poolId, requestId, lockToken, orchestrationId, token); Trace.Info($"Successfully renew job request {requestId}, job is valid till {request.LockedUntil.Value}"); diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 5ca2ef21c32..bcc982edace 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -37,7 +37,7 @@ public async Task ExecuteCommand(CommandSettings command) { try { - VssUtil.InitializeVssClientSettings(HostContext.UserAgent, HostContext.WebProxy); + VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy); _inConfigStage = true; _completedCommand.Reset(); diff --git a/src/Runner.Sdk/Util/VssUtil.cs b/src/Runner.Sdk/Util/VssUtil.cs index b5b6ce7b382..3b4e1b3edba 100644 --- a/src/Runner.Sdk/Util/VssUtil.cs +++ b/src/Runner.Sdk/Util/VssUtil.cs @@ -14,10 +14,10 @@ namespace GitHub.Runner.Sdk { public static class VssUtil { - public static void InitializeVssClientSettings(ProductInfoHeaderValue additionalUserAgent, IWebProxy proxy) + public static void InitializeVssClientSettings(List additionalUserAgents, IWebProxy proxy) { var headerValues = new List(); - headerValues.Add(additionalUserAgent); + headerValues.AddRange(additionalUserAgents); headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})")); if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index e73aa17dd00..4d01a789433 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -534,7 +534,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont // Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated. } - httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent); + httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); using (var result = await httpClient.GetStreamAsync(archiveLink)) { await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token); diff --git a/src/Runner.Worker/Worker.cs b/src/Runner.Worker/Worker.cs index 8db8424d23d..1c83c434292 100644 --- a/src/Runner.Worker/Worker.cs +++ b/src/Runner.Worker/Worker.cs @@ -40,7 +40,7 @@ public async Task RunAsync(string pipeIn, string pipeOut) // Validate args. ArgUtil.NotNullOrEmpty(pipeIn, nameof(pipeIn)); ArgUtil.NotNullOrEmpty(pipeOut, nameof(pipeOut)); - VssUtil.InitializeVssClientSettings(HostContext.UserAgent, HostContext.WebProxy); + VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy); var jobRunner = HostContext.CreateService(); using (var channel = HostContext.CreateService()) diff --git a/src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs b/src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs index 79d9bd481f1..c97fea0a4d1 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs @@ -95,6 +95,7 @@ public Task RenewAgentRequestAsync( Int64 requestId, Guid lockToken, DateTime? expiresOn = null, + string orchestrationId = null, Object userState = null, CancellationToken cancellationToken = default(CancellationToken)) { @@ -104,7 +105,30 @@ public Task RenewAgentRequestAsync( LockedUntil = expiresOn, }; - return UpdateAgentRequestAsync(poolId, requestId, lockToken, request, userState, cancellationToken); + var additionalHeaders = new Dictionary(); + if (!string.IsNullOrEmpty(orchestrationId)) + { + additionalHeaders["X-VSS-OrchestrationId"] = orchestrationId; + } + + HttpMethod httpMethod = new HttpMethod("PATCH"); + Guid locationId = new Guid("fc825784-c92a-4299-9221-998a02d1b54f"); + object routeValues = new { poolId = poolId, requestId = requestId }; + HttpContent content = new ObjectContent(request, new VssJsonMediaTypeFormatter(true)); + + List> queryParams = new List>(); + queryParams.Add("lockToken", lockToken.ToString()); + + return SendAsync( + httpMethod, + additionalHeaders, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(5.1, 1), + queryParameters: queryParams, + userState: userState, + cancellationToken: cancellationToken, + content: content); } public Task ReplaceAgentAsync( @@ -171,5 +195,5 @@ protected async Task SendAsync( } private readonly ApiResourceVersion m_currentApiVersion = new ApiResourceVersion(3.0, 1); - } + } } diff --git a/src/Sdk/WebApi/WebApi/Jwt/JsonWebTokenUtilities.cs b/src/Sdk/WebApi/WebApi/Jwt/JsonWebTokenUtilities.cs index d296666b75c..5287dbf65c4 100644 --- a/src/Sdk/WebApi/WebApi/Jwt/JsonWebTokenUtilities.cs +++ b/src/Sdk/WebApi/WebApi/Jwt/JsonWebTokenUtilities.cs @@ -96,7 +96,7 @@ internal static IEnumerable TranslateFromJwtClaims(IDictionary ExtractClaims(this JsonWebToken token) + public static IEnumerable ExtractClaims(this JsonWebToken token) { ArgumentUtility.CheckForNull(token, nameof(token)); diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index 00a7b5155f1..a8062b20650 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -73,7 +73,7 @@ public async void DispatchesJobRequest() Assert.NotNull(sessionIdProperty); sessionIdProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); - _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(request)); + _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(request)); _runnerServer.Setup(x => x.FinishAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(new TaskAgentJobRequest())); @@ -112,7 +112,7 @@ public async void DispatcherRenewJobRequest() hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configurationStore.Object); _configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1 }); - _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(() => { count++; @@ -139,10 +139,10 @@ public async void DispatcherRenewJobRequest() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully); - _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); + _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); } } @@ -170,7 +170,7 @@ public async void DispatcherRenewJobRequestStopOnJobNotFoundExceptions() hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configurationStore.Object); _configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1 }); - _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(() => { count++; @@ -197,11 +197,11 @@ public async void DispatcherRenewJobRequestStopOnJobNotFoundExceptions() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.False(cancellationTokenSource.IsCancellationRequested); - _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); + _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); } } @@ -229,7 +229,7 @@ public async void DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions() hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configurationStore.Object); _configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1 }); - _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(() => { count++; @@ -256,11 +256,11 @@ public async void DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.False(cancellationTokenSource.IsCancellationRequested); - _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); + _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); } } @@ -288,7 +288,7 @@ public async void DispatcherRenewJobRequestRecoverFromExceptions() hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configurationStore.Object); _configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1 }); - _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(() => { count++; @@ -315,11 +315,11 @@ public async void DispatcherRenewJobRequestRecoverFromExceptions() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.True(cancellationTokenSource.IsCancellationRequested); - _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(8)); + _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(8)); _runnerServer.Verify(x => x.RefreshConnectionAsync(RunnerConnectionType.JobRequest, It.IsAny()), Times.Exactly(3)); _runnerServer.Verify(x => x.SetConnectionTimeout(RunnerConnectionType.JobRequest, It.IsAny()), Times.Once); } @@ -349,7 +349,7 @@ public async void DispatcherRenewJobRequestFirstRenewRetrySixTimes() hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configurationStore.Object); _configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1 }); - _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(() => { count++; @@ -372,11 +372,11 @@ public async void DispatcherRenewJobRequestFirstRenewRetrySixTimes() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.False(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should failed."); Assert.False(cancellationTokenSource.IsCancellationRequested); - _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(6)); + _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(6)); } } @@ -404,7 +404,7 @@ public async void DispatcherRenewJobRequestStopOnExpiredRequest() hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configurationStore.Object); _configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1 }); - _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(() => { count++; @@ -436,11 +436,11 @@ public async void DispatcherRenewJobRequestStopOnExpiredRequest() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.False(cancellationTokenSource.IsCancellationRequested); - _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); + _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); _runnerServer.Verify(x => x.RefreshConnectionAsync(RunnerConnectionType.JobRequest, It.IsAny()), Times.Exactly(3)); _runnerServer.Verify(x => x.SetConnectionTimeout(RunnerConnectionType.JobRequest, It.IsAny()), Times.Never); } @@ -481,7 +481,7 @@ public async void DispatchesOneTimeJobRequest() Assert.NotNull(sessionIdProperty); sessionIdProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); - _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(request)); + _runnerServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(request)); _runnerServer.Setup(x => x.FinishAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(new TaskAgentJobRequest())); diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index 3d3c99c736b..546b3cc8e98 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -86,7 +86,7 @@ public StartupType StartupType } } - public ProductInfoHeaderValue UserAgent => new ProductInfoHeaderValue("L0Test", "0.0"); + public List UserAgents => new List() { new ProductInfoHeaderValue("L0Test", "0.0") }; public RunnerWebProxy WebProxy => new RunnerWebProxy(); From 2bd0b1af0e4daa0ed3d8115c79ba6aa6633dd732 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 9 Apr 2020 16:13:06 -0400 Subject: [PATCH 29/86] launch middle man process on macOS to workaround SIP limit (#416) --- src/Misc/layoutbin/macos-run-invoker.js | 13 +++++++++++++ src/Runner.Worker/Handlers/ScriptHandler.cs | 10 ++++++++++ 2 files changed, 23 insertions(+) create mode 100644 src/Misc/layoutbin/macos-run-invoker.js diff --git a/src/Misc/layoutbin/macos-run-invoker.js b/src/Misc/layoutbin/macos-run-invoker.js new file mode 100644 index 00000000000..624e775e966 --- /dev/null +++ b/src/Misc/layoutbin/macos-run-invoker.js @@ -0,0 +1,13 @@ +const { spawn } = require('child_process'); +// argv[0] = node +// argv[1] = macos-run-invoker.js +var shell = process.argv[2]; +var args = process.argv.slice(3); +console.log(`::debug::macos-run-invoker: ${shell}`); +console.log(`::debug::macos-run-invoker: ${JSON.stringify(args)}`); +var launch = spawn(shell, args, { stdio: 'inherit' }); +launch.on('exit', function (code) { + if (code !== 0) { + process.exit(code); + } +}); diff --git a/src/Runner.Worker/Handlers/ScriptHandler.cs b/src/Runner.Worker/Handlers/ScriptHandler.cs index 89ac15a030e..051cd5fc78e 100644 --- a/src/Runner.Worker/Handlers/ScriptHandler.cs +++ b/src/Runner.Worker/Handlers/ScriptHandler.cs @@ -259,6 +259,16 @@ public async Task RunAsync(ActionRunStage stage) // dump out the command var fileName = isContainerStepHost ? shellCommand : commandPath; +#if OS_OSX + if (Environment.ContainsKey("DYLD_INSERT_LIBRARIES")) // We don't check `isContainerStepHost` because we don't support container on macOS + { + // launch `node macOSRunInvoker.js shell args` instead of `shell args` to avoid macOS SIP remove `DYLD_INSERT_LIBRARIES` when launch process + string node12 = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node12", "bin", $"node{IOUtil.ExeExtension}"); + string macOSRunInvoker = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "macos-run-invoker.js"); + arguments = $"\"{macOSRunInvoker.Replace("\"", "\\\"")}\" \"{fileName.Replace("\"", "\\\"")}\" {arguments}"; + fileName = node12; + } +#endif ExecutionContext.Debug($"{fileName} {arguments}"); using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager)) From a20ad4e12114d0f18292d1e01926f10646f07dbb Mon Sep 17 00:00:00 2001 From: chenrui Date: Sat, 11 Apr 2020 13:59:20 -0400 Subject: [PATCH 30/86] Update `releaseVersion` to v2.168.0 (#420) v2.168.0 is the latest release version, update the meta file to reflect that. --- releaseVersion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releaseVersion b/releaseVersion index 8164268dacd..2973ad9e470 100644 --- a/releaseVersion +++ b/releaseVersion @@ -1 +1 @@ -2.164.0 +2.168.0 From f9baec4b32458d6f0f858a463ba264558d23bddf Mon Sep 17 00:00:00 2001 From: Lokesh Gopu Date: Mon, 13 Apr 2020 21:33:13 -0400 Subject: [PATCH 31/86] Added support for custom labels (#414) * Added support for custom labels * ignore case * Added interactive config for labels * Fixing L0s * pr comments --- src/Runner.Common/Constants.cs | 1 + src/Runner.Listener/CommandSettings.cs | 25 +++++++- .../Configuration/ConfigurationManager.cs | 37 ++++++++---- .../Configuration/PromptManager.cs | 24 ++++++-- .../Configuration/Validators.cs | 16 +++++ .../Generated/TaskAgentHttpClientBase.cs | 12 ++-- src/Sdk/DTWebApi/WebApi/AgentLabel.cs | 59 +++++++++++++++++++ src/Sdk/DTWebApi/WebApi/LabelType.cs | 14 +++++ src/Sdk/DTWebApi/WebApi/TaskAgent.cs | 8 +-- src/Test/L0/Listener/CommandSettingsL0.cs | 36 +++++++---- .../Configuration/ConfigurationManagerL0.cs | 10 +++- 11 files changed, 199 insertions(+), 43 deletions(-) create mode 100644 src/Sdk/DTWebApi/WebApi/AgentLabel.cs create mode 100644 src/Sdk/DTWebApi/WebApi/LabelType.cs diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 0a8261c5525..e67b6d42ef9 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -87,6 +87,7 @@ public static class CommandLine public static class Args { public static readonly string Auth = "auth"; + public static readonly string Labels = "labels"; public static readonly string MonitorSocketAddress = "monitorsocketaddress"; public static readonly string Name = "name"; public static readonly string Pool = "pool"; diff --git a/src/Runner.Listener/CommandSettings.cs b/src/Runner.Listener/CommandSettings.cs index 07a6c334ba9..0d80c1d5e4c 100644 --- a/src/Runner.Listener/CommandSettings.cs +++ b/src/Runner.Listener/CommandSettings.cs @@ -39,6 +39,7 @@ public sealed class CommandSettings private readonly string[] validArgs = { Constants.Runner.CommandLine.Args.Auth, + Constants.Runner.CommandLine.Args.Labels, Constants.Runner.CommandLine.Args.MonitorSocketAddress, Constants.Runner.CommandLine.Args.Name, Constants.Runner.CommandLine.Args.Pool, @@ -249,6 +250,24 @@ public string GetStartupType() return GetArg(Constants.Runner.CommandLine.Args.StartupType); } + public ISet GetLabels() + { + var labelSet = new HashSet(StringComparer.OrdinalIgnoreCase); + string labels = GetArgOrPrompt( + name: Constants.Runner.CommandLine.Args.Labels, + description: $"This runner will have the following labels: 'self-hosted', '{VarUtil.OS}', '{VarUtil.OSArchitecture}' \nEnter any additional labels (ex. label-1,label-2):", + defaultValue: string.Empty, + validator: Validators.LabelsValidator, + isOptional: true); + + if (!string.IsNullOrEmpty(labels)) + { + labelSet = labels.Split(',').Where(x => !string.IsNullOrEmpty(x)).ToHashSet(StringComparer.OrdinalIgnoreCase); + } + + return labelSet; + } + // // Private helpers. // @@ -280,7 +299,8 @@ private string GetArgOrPrompt( string name, string description, string defaultValue, - Func validator) + Func validator, + bool isOptional = false) { // Check for the arg in the command line parser. ArgUtil.NotNull(validator, nameof(validator)); @@ -311,7 +331,8 @@ private string GetArgOrPrompt( secret: Constants.Runner.CommandLine.Args.Secrets.Any(x => string.Equals(x, name, StringComparison.OrdinalIgnoreCase)), defaultValue: defaultValue, validator: validator, - unattended: Unattended); + unattended: Unattended, + isOptional: isOptional); } private string GetEnvArg(string name) diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index c2d4a60cd18..289ccaa1a55 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -166,6 +166,9 @@ public async Task ConfigureAsync(CommandSettings command) _term.WriteLine(); + var userLabels = command.GetLabels(); + _term.WriteLine(); + var agents = await _runnerServer.GetAgentsAsync(runnerSettings.PoolId, runnerSettings.AgentName); Trace.Verbose("Returns {0} agents", agents.Count); agent = agents.FirstOrDefault(); @@ -175,7 +178,7 @@ public async Task ConfigureAsync(CommandSettings command) if (command.GetReplace()) { // Update existing agent with new PublicKey, agent version. - agent = UpdateExistingAgent(agent, publicKey); + agent = UpdateExistingAgent(agent, publicKey, userLabels); try { @@ -198,7 +201,7 @@ public async Task ConfigureAsync(CommandSettings command) else { // Create a new agent. - agent = CreateNewAgent(runnerSettings.AgentName, publicKey); + agent = CreateNewAgent(runnerSettings.AgentName, publicKey, userLabels); try { @@ -448,7 +451,7 @@ private ICredentialProvider GetCredentialProvider(CommandSettings command, strin } - private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey) + private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, ISet userLabels) { ArgUtil.NotNull(agent, nameof(agent)); agent.Authorization = new TaskAgentAuthorization @@ -456,18 +459,25 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey) PublicKey = new TaskAgentPublicKey(publicKey.Exponent, publicKey.Modulus), }; - // update - update instead of delete so we don't lose labels etc... + // update should replace the existing labels agent.Version = BuildConstants.RunnerPackage.Version; agent.OSDescription = RuntimeInformation.OSDescription; + + agent.Labels.Clear(); - agent.Labels.Add("self-hosted"); - agent.Labels.Add(VarUtil.OS); - agent.Labels.Add(VarUtil.OSArchitecture); + agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); + agent.Labels.Add(new AgentLabel(VarUtil.OS, LabelType.System)); + agent.Labels.Add(new AgentLabel(VarUtil.OSArchitecture, LabelType.System)); + foreach (var userLabel in userLabels) + { + agent.Labels.Add(new AgentLabel(userLabel, LabelType.User)); + } + return agent; } - private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey) + private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet userLabels) { TaskAgent agent = new TaskAgent(agentName) { @@ -480,9 +490,14 @@ private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey) OSDescription = RuntimeInformation.OSDescription, }; - agent.Labels.Add("self-hosted"); - agent.Labels.Add(VarUtil.OS); - agent.Labels.Add(VarUtil.OSArchitecture); + agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); + agent.Labels.Add(new AgentLabel(VarUtil.OS, LabelType.System)); + agent.Labels.Add(new AgentLabel(VarUtil.OSArchitecture, LabelType.System)); + + foreach (var userLabel in userLabels) + { + agent.Labels.Add(new AgentLabel(userLabel, LabelType.User)); + } return agent; } diff --git a/src/Runner.Listener/Configuration/PromptManager.cs b/src/Runner.Listener/Configuration/PromptManager.cs index 977786c231d..274a248787d 100644 --- a/src/Runner.Listener/Configuration/PromptManager.cs +++ b/src/Runner.Listener/Configuration/PromptManager.cs @@ -20,7 +20,8 @@ string ReadValue( bool secret, string defaultValue, Func validator, - bool unattended); + bool unattended, + bool isOptional = false); } public sealed class PromptManager : RunnerService, IPromptManager @@ -56,7 +57,8 @@ public string ReadValue( bool secret, string defaultValue, Func validator, - bool unattended) + bool unattended, + bool isOptional = false) { Trace.Info(nameof(ReadValue)); ArgUtil.NotNull(validator, nameof(validator)); @@ -85,18 +87,28 @@ public string ReadValue( { _terminal.Write($"[press Enter for {defaultValue}] "); } + else if (isOptional){ + _terminal.Write($"[press Enter to skip] "); + } // Read and trim the value. value = secret ? _terminal.ReadSecret() : _terminal.ReadLine(); value = value?.Trim() ?? string.Empty; // Return the default if not specified. - if (string.IsNullOrEmpty(value) && !string.IsNullOrEmpty(defaultValue)) + if (string.IsNullOrEmpty(value)) { - Trace.Info($"Falling back to the default: '{defaultValue}'"); - return defaultValue; + if (!string.IsNullOrEmpty(defaultValue)) + { + Trace.Info($"Falling back to the default: '{defaultValue}'"); + return defaultValue; + } + else if (isOptional) + { + return string.Empty; + } } - + // Return the value if it is not empty and it is valid. // Otherwise try the loop again. if (!string.IsNullOrEmpty(value)) diff --git a/src/Runner.Listener/Configuration/Validators.cs b/src/Runner.Listener/Configuration/Validators.cs index c0cd1ef0ed0..79d9682215c 100644 --- a/src/Runner.Listener/Configuration/Validators.cs +++ b/src/Runner.Listener/Configuration/Validators.cs @@ -1,6 +1,7 @@ using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; +using System.Linq; using System.IO; using System.Security.Principal; @@ -46,6 +47,21 @@ public static bool BoolValidator(string value) string.Equals(value, "N", StringComparison.CurrentCultureIgnoreCase); } + public static bool LabelsValidator(string labels) + { + if (!string.IsNullOrEmpty(labels)) + { + var labelSet = labels.Split(',').Where(x => !string.IsNullOrEmpty(x)).ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (labelSet.Any(x => x.Length > 256)) + { + return false; + } + } + + return true; + } + public static bool NonEmptyValidator(string value) { return !string.IsNullOrEmpty(value); diff --git a/src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs b/src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs index 14327a6f878..d5f9e2b7cf2 100644 --- a/src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs +++ b/src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs @@ -82,7 +82,7 @@ public virtual Task AddAgentAsync( httpMethod, locationId, routeValues: routeValues, - version: new ApiResourceVersion(5.1, 1), + version: new ApiResourceVersion(6.0, 2), userState: userState, cancellationToken: cancellationToken, content: content); @@ -109,7 +109,7 @@ public virtual async Task DeleteAgentAsync( httpMethod, locationId, routeValues: routeValues, - version: new ApiResourceVersion(5.1, 1), + version: new ApiResourceVersion(6.0, 2), userState: userState, cancellationToken: cancellationToken).ConfigureAwait(false)) { @@ -164,7 +164,7 @@ public virtual Task GetAgentAsync( httpMethod, locationId, routeValues: routeValues, - version: new ApiResourceVersion(5.1, 1), + version: new ApiResourceVersion(6.0, 2), queryParameters: queryParams, userState: userState, cancellationToken: cancellationToken); @@ -227,7 +227,7 @@ public virtual Task> GetAgentsAsync( httpMethod, locationId, routeValues: routeValues, - version: new ApiResourceVersion(5.1, 1), + version: new ApiResourceVersion(6.0, 2), queryParameters: queryParams, userState: userState, cancellationToken: cancellationToken); @@ -257,7 +257,7 @@ public virtual Task ReplaceAgentAsync( httpMethod, locationId, routeValues: routeValues, - version: new ApiResourceVersion(5.1, 1), + version: new ApiResourceVersion(6.0, 2), userState: userState, cancellationToken: cancellationToken, content: content); @@ -287,7 +287,7 @@ public virtual Task UpdateAgentAsync( httpMethod, locationId, routeValues: routeValues, - version: new ApiResourceVersion(5.1, 1), + version: new ApiResourceVersion(6.0, 2), userState: userState, cancellationToken: cancellationToken, content: content); diff --git a/src/Sdk/DTWebApi/WebApi/AgentLabel.cs b/src/Sdk/DTWebApi/WebApi/AgentLabel.cs new file mode 100644 index 00000000000..6d98caed1c9 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/AgentLabel.cs @@ -0,0 +1,59 @@ +using System.Runtime.Serialization; +using Newtonsoft.Json; + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public class AgentLabel + { + [JsonConstructor] + public AgentLabel() + { + } + + public AgentLabel(string name) + { + this.Name = name; + this.Type = LabelType.System; + } + + public AgentLabel(string name, LabelType type) + { + this.Name = name; + this.Type = type; + } + + private AgentLabel(AgentLabel labelToBeCloned) + { + this.Id = labelToBeCloned.Id; + this.Name = labelToBeCloned.Name; + this.Type = labelToBeCloned.Type; + } + + [DataMember] + public int Id + { + get; + set; + } + + [DataMember] + public string Name + { + get; + set; + } + + [DataMember] + public LabelType Type + { + get; + set; + } + + public AgentLabel Clone() + { + return new AgentLabel(this); + } + } +} diff --git a/src/Sdk/DTWebApi/WebApi/LabelType.cs b/src/Sdk/DTWebApi/WebApi/LabelType.cs new file mode 100644 index 00000000000..dd135020a92 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/LabelType.cs @@ -0,0 +1,14 @@ +using System.Runtime.Serialization; + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public enum LabelType + { + [EnumMember] + System = 0, + + [EnumMember] + User = 1 + } +} diff --git a/src/Sdk/DTWebApi/WebApi/TaskAgent.cs b/src/Sdk/DTWebApi/WebApi/TaskAgent.cs index 1ca8ae94b46..f97322e1382 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAgent.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAgent.cs @@ -51,7 +51,7 @@ private TaskAgent(TaskAgent agentToBeCloned) if (agentToBeCloned.m_labels != null && agentToBeCloned.m_labels.Count > 0) { - m_labels = new HashSet(agentToBeCloned.m_labels, StringComparer.OrdinalIgnoreCase); + m_labels = new HashSet(agentToBeCloned.m_labels); } } @@ -118,13 +118,13 @@ public TaskAgentAuthorization Authorization /// /// The labels of the runner /// - public ISet Labels + public ISet Labels { get { if (m_labels == null) { - m_labels = new HashSet(StringComparer.OrdinalIgnoreCase); + m_labels = new HashSet(); } return m_labels; } @@ -164,6 +164,6 @@ Object ICloneable.Clone() private PropertiesCollection m_properties; [DataMember(IsRequired = false, EmitDefaultValue = false, Name = "Labels")] - private HashSet m_labels; + private HashSet m_labels; } } diff --git a/src/Test/L0/Listener/CommandSettingsL0.cs b/src/Test/L0/Listener/CommandSettingsL0.cs index 9ef40dd0cd0..b35729d4949 100644 --- a/src/Test/L0/Listener/CommandSettingsL0.cs +++ b/src/Test/L0/Listener/CommandSettingsL0.cs @@ -317,7 +317,8 @@ public void PassesUnattendedToReadValue() false, // secret Environment.MachineName, // defaultValue Validators.NonEmptyValidator, // validator - true)) // unattended + true, // unattended + false)) // isOptional .Returns("some runner"); // Act. @@ -344,7 +345,8 @@ public void PromptsForRunnerName() false, // secret Environment.MachineName, // defaultValue Validators.NonEmptyValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some runner"); // Act. @@ -371,7 +373,8 @@ public void PromptsForAuth() false, // secret "some default auth", // defaultValue Validators.AuthSchemeValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some auth"); // Act. @@ -398,7 +401,8 @@ public void PromptsForRunnerRegisterToken() true, // secret string.Empty, // defaultValue Validators.NonEmptyValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some token"); // Act. @@ -475,7 +479,8 @@ public void PromptsForToken() true, // secret string.Empty, // defaultValue Validators.NonEmptyValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some token"); // Act. @@ -502,7 +507,8 @@ public void PromptsForRunnerDeletionToken() true, // secret string.Empty, // defaultValue Validators.NonEmptyValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some token"); // Act. @@ -529,7 +535,8 @@ public void PromptsForUrl() false, // secret string.Empty, // defaultValue Validators.ServerUrlValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some url"); // Act. @@ -556,7 +563,8 @@ public void PromptsForWindowsLogonAccount() false, // secret "some default account", // defaultValue Validators.NTAccountValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some windows logon account"); // Act. @@ -584,7 +592,8 @@ public void PromptsForWindowsLogonPassword() true, // secret string.Empty, // defaultValue Validators.NonEmptyValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some windows logon password"); // Act. @@ -611,7 +620,8 @@ public void PromptsForWork() false, // secret "_work", // defaultValue Validators.NonEmptyValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some work"); // Act. @@ -640,7 +650,8 @@ public void PromptsWhenEmpty() false, // secret string.Empty, // defaultValue Validators.ServerUrlValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some url"); // Act. @@ -669,7 +680,8 @@ public void PromptsWhenInvalid() false, // secret string.Empty, // defaultValue Validators.ServerUrlValidator, // validator - false)) // unattended + false, // unattended + false)) // isOptional .Returns("some url"); // Act. diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index c47d5be21c0..9cf3ad5adae 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -145,6 +145,8 @@ public async Task CanEnsureConfigure() IConfigurationManager configManager = new ConfigurationManager(); configManager.Initialize(tc); + var userLabels = "userlabel1,userlabel2"; + trace.Info("Preparing command line arguments"); var command = new CommandSettings( tc, @@ -156,7 +158,8 @@ public async Task CanEnsureConfigure() "--pool", _expectedPoolName, "--work", _expectedWorkFolder, "--auth", _expectedAuthType, - "--token", _expectedToken + "--token", _expectedToken, + "--labels", userLabels }); trace.Info("Constructed."); _store.Setup(x => x.IsConfigured()).Returns(false); @@ -178,7 +181,10 @@ public async Task CanEnsureConfigure() // validate GetAgentPoolsAsync gets called twice with automation pool type _runnerServer.Verify(x => x.GetAgentPoolsAsync(It.IsAny(), It.Is(p => p == TaskAgentPoolType.Automation)), Times.Exactly(2)); - _runnerServer.Verify(x => x.AddAgentAsync(It.IsAny(), It.Is(a => a.Labels.Contains("self-hosted") && a.Labels.Contains(VarUtil.OS) && a.Labels.Contains(VarUtil.OSArchitecture))), Times.Once); + var expectedLabels = new List() { "self-hosted", VarUtil.OS, VarUtil.OSArchitecture}; + expectedLabels.AddRange(userLabels.Split(",").ToList()); + + _runnerServer.Verify(x => x.AddAgentAsync(It.IsAny(), It.Is(a => a.Labels.Select(x => x.Name).ToHashSet().SetEquals(expectedLabels))), Times.Once); } } } From d5c7097d2c4f3f40a029124bbf1bc4c9994929f5 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 13 Apr 2020 21:46:30 -0400 Subject: [PATCH 32/86] support 'pre' execution for actions. (#389) --- src/Runner.Worker/ActionManager.cs | 84 ++++++-- src/Runner.Worker/ActionManifestManager.cs | 22 +- src/Runner.Worker/ActionRunner.cs | 23 +-- .../ContainerOperationProvider.cs | 2 +- src/Runner.Worker/ExecutionContext.cs | 23 ++- .../Handlers/ContainerActionHandler.cs | 6 +- .../Handlers/NodeScriptActionHandler.cs | 6 +- .../Handlers/RunnerPluginHandler.cs | 2 +- src/Runner.Worker/JobExtension.cs | 27 ++- src/Runner.Worker/action_yaml.json | 4 + src/Test/L0/Listener/MessageListenerL0.cs | 2 +- src/Test/L0/Worker/ActionManagerL0.cs | 126 ++++++++++-- src/Test/L0/Worker/ActionManifestManagerL0.cs | 188 +++++++++++++++++- src/Test/L0/Worker/ExecutionContextL0.cs | 93 ++++++++- src/Test/L0/Worker/JobExtensionL0.cs | 4 +- src/Test/TestData/dockerfileaction_init.yml | 27 +++ .../dockerfileaction_init_default.yml | 26 +++ src/Test/TestData/nodeaction_init.yml | 22 ++ src/Test/TestData/nodeaction_init_default.yml | 21 ++ 19 files changed, 640 insertions(+), 68 deletions(-) create mode 100644 src/Test/TestData/dockerfileaction_init.yml create mode 100644 src/Test/TestData/dockerfileaction_init_default.yml create mode 100644 src/Test/TestData/nodeaction_init.yml create mode 100644 src/Test/TestData/nodeaction_init_default.yml diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 4d01a789433..5338416590c 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -21,11 +21,24 @@ namespace GitHub.Runner.Worker { + public class PrepareResult + { + public PrepareResult(List containerSetupSteps, Dictionary preStepTracker) + { + this.ContainerSetupSteps = containerSetupSteps; + this.PreStepTracker = preStepTracker; + } + + public List ContainerSetupSteps { get; set; } + + public Dictionary PreStepTracker { get; set; } + } + [ServiceLocator(Default = typeof(ActionManager))] public interface IActionManager : IRunnerService { Dictionary CachedActionContainers { get; } - Task> PrepareActionsAsync(IExecutionContext executionContext, IEnumerable steps); + Task PrepareActionsAsync(IExecutionContext executionContext, IEnumerable steps); Definition LoadAction(IExecutionContext executionContext, Pipelines.ActionStep action); } @@ -39,7 +52,7 @@ public sealed class ActionManager : RunnerService, IActionManager private readonly Dictionary _cachedActionContainers = new Dictionary(); public Dictionary CachedActionContainers => _cachedActionContainers; - public async Task> PrepareActionsAsync(IExecutionContext executionContext, IEnumerable steps) + public async Task PrepareActionsAsync(IExecutionContext executionContext, IEnumerable steps) { ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(steps, nameof(steps)); @@ -49,6 +62,7 @@ public async Task> PrepareActionsAsync(IExecutionContex Dictionary> imagesToBuild = new Dictionary>(StringComparer.OrdinalIgnoreCase); Dictionary imagesToBuildInfo = new Dictionary(StringComparer.OrdinalIgnoreCase); List containerSetupSteps = new List(); + Dictionary preStepTracker = new Dictionary(); IEnumerable actions = steps.OfType(); // TODO: Deprecate the PREVIEW_ACTION_TOKEN @@ -117,6 +131,22 @@ public async Task> PrepareActionsAsync(IExecutionContex imagesToBuildInfo[setupInfo.ActionRepository] = setupInfo; } } + + var repoAction = action.Reference as Pipelines.RepositoryPathReference; + if (repoAction.RepositoryType != Pipelines.PipelineConstants.SelfAlias) + { + var definition = LoadAction(executionContext, action); + if (definition.Data.Execution.HasPre) + { + var actionRunner = HostContext.CreateService(); + actionRunner.Action = action; + actionRunner.Stage = ActionRunStage.Pre; + actionRunner.Condition = definition.Data.Execution.InitCondition; + + Trace.Info($"Add 'pre' execution for {action.Id}"); + preStepTracker[action.Id] = actionRunner; + } + } } } @@ -153,7 +183,7 @@ public async Task> PrepareActionsAsync(IExecutionContex } #endif - return containerSetupSteps; + return new PrepareResult(containerSetupSteps, preStepTracker); } public Definition LoadAction(IExecutionContext executionContext, Pipelines.ActionStep action) @@ -245,14 +275,19 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio Trace.Info($"Action container env: {StringUtil.ConvertToJson(containerAction.Environment)}."); } + if (!string.IsNullOrEmpty(containerAction.Pre)) + { + Trace.Info($"Action container pre entrypoint: {containerAction.Pre}."); + } + if (!string.IsNullOrEmpty(containerAction.EntryPoint)) { Trace.Info($"Action container entrypoint: {containerAction.EntryPoint}."); } - if (!string.IsNullOrEmpty(containerAction.Cleanup)) + if (!string.IsNullOrEmpty(containerAction.Post)) { - Trace.Info($"Action container cleanup entrypoint: {containerAction.Cleanup}."); + Trace.Info($"Action container post entrypoint: {containerAction.Post}."); } if (CachedActionContainers.TryGetValue(action.Id, out var container)) @@ -264,8 +299,9 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio else if (definition.Data.Execution.ExecutionType == ActionExecutionType.NodeJS) { var nodeAction = definition.Data.Execution as NodeJSActionExecutionData; + Trace.Info($"Action pre node.js file: {nodeAction.Pre ?? "N/A"}."); Trace.Info($"Action node.js file: {nodeAction.Script}."); - Trace.Info($"Action cleanup node.js file: {nodeAction.Cleanup ?? "N/A"}."); + Trace.Info($"Action post node.js file: {nodeAction.Post ?? "N/A"}."); } else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Plugin) { @@ -281,7 +317,7 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio if (!string.IsNullOrEmpty(plugin.PostPluginTypeName)) { - pluginAction.Cleanup = plugin.PostPluginTypeName; + pluginAction.Post = plugin.PostPluginTypeName; Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}."); } } @@ -788,7 +824,8 @@ public sealed class ContainerActionExecutionData : ActionExecutionData { public override ActionExecutionType ExecutionType => ActionExecutionType.Container; - public override bool HasCleanup => !string.IsNullOrEmpty(Cleanup); + public override bool HasPre => !string.IsNullOrEmpty(Pre); + public override bool HasPost => !string.IsNullOrEmpty(Post); public string Image { get; set; } @@ -798,51 +835,66 @@ public sealed class ContainerActionExecutionData : ActionExecutionData public MappingToken Environment { get; set; } - public string Cleanup { get; set; } + public string Pre { get; set; } + + public string Post { get; set; } } public sealed class NodeJSActionExecutionData : ActionExecutionData { public override ActionExecutionType ExecutionType => ActionExecutionType.NodeJS; - public override bool HasCleanup => !string.IsNullOrEmpty(Cleanup); + public override bool HasPre => !string.IsNullOrEmpty(Pre); + public override bool HasPost => !string.IsNullOrEmpty(Post); public string Script { get; set; } - public string Cleanup { get; set; } + public string Pre { get; set; } + + public string Post { get; set; } } public sealed class PluginActionExecutionData : ActionExecutionData { public override ActionExecutionType ExecutionType => ActionExecutionType.Plugin; - public override bool HasCleanup => !string.IsNullOrEmpty(Cleanup); + public override bool HasPre => false; + + public override bool HasPost => !string.IsNullOrEmpty(Post); public string Plugin { get; set; } - public string Cleanup { get; set; } + public string Post { get; set; } } public sealed class ScriptActionExecutionData : ActionExecutionData { public override ActionExecutionType ExecutionType => ActionExecutionType.Script; - - public override bool HasCleanup => false; + public override bool HasPre => false; + public override bool HasPost => false; } public abstract class ActionExecutionData { + private string _initCondition = $"{Constants.Expressions.Always}()"; private string _cleanupCondition = $"{Constants.Expressions.Always}()"; public abstract ActionExecutionType ExecutionType { get; } - public abstract bool HasCleanup { get; } + public abstract bool HasPre { get; } + public abstract bool HasPost { get; } public string CleanupCondition { get { return _cleanupCondition; } set { _cleanupCondition = value; } } + + public string InitCondition + { + get { return _initCondition; } + set { _initCondition = value; } + } } public class ContainerSetupInfo diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 9b94faaf85d..4e9149d26b6 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -305,6 +305,9 @@ private ActionExecutionData ConvertRuns( var envToken = default(MappingToken); var mainToken = default(StringToken); var pluginToken = default(StringToken); + var preToken = default(StringToken); + var preEntrypointToken = default(StringToken); + var preIfToken = default(StringToken); var postToken = default(StringToken); var postEntrypointToken = default(StringToken); var postIfToken = default(StringToken); @@ -343,6 +346,15 @@ private ActionExecutionData ConvertRuns( case "post-if": postIfToken = run.Value.AssertString("post-if"); break; + case "pre": + preToken = run.Value.AssertString("pre"); + break; + case "pre-entrypoint": + preEntrypointToken = run.Value.AssertString("pre-entrypoint"); + break; + case "pre-if": + preIfToken = run.Value.AssertString("pre-if"); + break; default: Trace.Info($"Ignore run property {runsKey}."); break; @@ -365,7 +377,9 @@ private ActionExecutionData ConvertRuns( Arguments = argsToken, EntryPoint = entrypointToken?.Value, Environment = envToken, - Cleanup = postEntrypointToken?.Value, + Pre = preEntrypointToken?.Value, + InitCondition = preIfToken?.Value ?? "always()", + Post = postEntrypointToken?.Value, CleanupCondition = postIfToken?.Value ?? "always()" }; } @@ -374,14 +388,16 @@ private ActionExecutionData ConvertRuns( { if (string.IsNullOrEmpty(mainToken?.Value)) { - throw new ArgumentNullException($"Entry javascript fils is not provided."); + throw new ArgumentNullException($"Entry javascript file is not provided."); } else { return new NodeJSActionExecutionData() { Script = mainToken.Value, - Cleanup = postToken?.Value, + Pre = preToken?.Value, + InitCondition = preIfToken?.Value ?? "always()", + Post = postToken?.Value, CleanupCondition = postIfToken?.Value ?? "always()" }; } diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index 7272303bcf3..8331e99158f 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -18,6 +18,7 @@ namespace GitHub.Runner.Worker { public enum ActionRunStage { + Pre, Main, Post, } @@ -81,20 +82,18 @@ public async Task RunAsync() ActionExecutionData handlerData = definition.Data?.Execution; ArgUtil.NotNull(handlerData, nameof(handlerData)); + if (handlerData.HasPre && + Action.Reference is Pipelines.RepositoryPathReference repoAction && + string.Equals(repoAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase)) + { + ExecutionContext.Warning($"`pre` execution is not supported for local action from '{repoAction.Path}'"); + } + // The action has post cleanup defined. // we need to create timeline record for them and add them to the step list that StepRunner is using - if (handlerData.HasCleanup && Stage == ActionRunStage.Main) + if (handlerData.HasPost && (Stage == ActionRunStage.Pre || Stage == ActionRunStage.Main)) { - string postDisplayName = null; - if (this.DisplayName.StartsWith(PipelineTemplateConstants.RunDisplayPrefix)) - { - postDisplayName = $"Post {this.DisplayName.Substring(PipelineTemplateConstants.RunDisplayPrefix.Length)}"; - } - else - { - postDisplayName = $"Post {this.DisplayName}"; - } - + string postDisplayName = $"Post {this.DisplayName}"; var repositoryReference = Action.Reference as RepositoryPathReference; var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}"; var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" : @@ -108,7 +107,7 @@ public async Task RunAsync() actionRunner.Condition = handlerData.CleanupCondition; actionRunner.DisplayName = postDisplayName; - ExecutionContext.RegisterPostJobStep($"{actionRunner.Action.Name}_post", actionRunner); + ExecutionContext.RegisterPostJobStep(actionRunner); } IStepHost stepHost = HostContext.CreateService(); diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index c105e2162f1..2a27a731ae5 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -49,7 +49,7 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec data: data); executionContext.Debug($"Register post job cleanup for stopping/deleting containers."); - executionContext.RegisterPostJobStep(nameof(StopContainersAsync), postJobStep); + executionContext.RegisterPostJobStep(postJobStep); // Check whether we are inside a container. // Our container feature requires to map working directory from host to the container. diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 25cbeefb10a..48c8d096fad 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -103,7 +103,7 @@ public interface IExecutionContext : IRunnerService // others void ForceTaskComplete(); - void RegisterPostJobStep(string refName, IStep step); + void RegisterPostJobStep(IStep step); } public sealed class ExecutionContext : RunnerService, IExecutionContext @@ -161,6 +161,9 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext // Only job level ExecutionContext has PostJobSteps public Stack PostJobSteps { get; private set; } + // Only job level ExecutionContext has StepsWithPostRegistered + public HashSet StepsWithPostRegistered { get; private set; } + public bool EchoOnActionCommand { get; set; } @@ -248,9 +251,15 @@ public void ForceTaskComplete() }); } - public void RegisterPostJobStep(string refName, IStep step) + public void RegisterPostJobStep(IStep step) { - step.ExecutionContext = Root.CreatePostChild(step.DisplayName, refName, IntraActionState); + if (step is IActionRunner actionRunner && !Root.StepsWithPostRegistered.Add(actionRunner.Action.Id)) + { + Trace.Info($"'post' of '{actionRunner.DisplayName}' already push to post step stack."); + return; + } + + step.ExecutionContext = Root.CreatePostChild(step.DisplayName, IntraActionState); Root.PostJobSteps.Push(step); } @@ -647,6 +656,9 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // PostJobSteps for job ExecutionContext PostJobSteps = new Stack(); + // StepsWithPostRegistered for job ExecutionContext + StepsWithPostRegistered = new HashSet(); + // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, @@ -847,7 +859,7 @@ private void JobServerQueueThrottling_EventReceived(object sender, ThrottlingEve } } - private IExecutionContext CreatePostChild(string displayName, string refName, Dictionary intraActionState) + private IExecutionContext CreatePostChild(string displayName, Dictionary intraActionState) { if (!_expandedForPostJob) { @@ -856,7 +868,8 @@ private IExecutionContext CreatePostChild(string displayName, string refName, Di _childTimelineRecordOrder = _childTimelineRecordOrder * 2; } - return CreateChild(Guid.NewGuid(), displayName, refName, null, null, intraActionState, _childTimelineRecordOrder - Root.PostJobSteps.Count); + var newGuid = Guid.NewGuid(); + return CreateChild(newGuid, displayName, newGuid.ToString("N"), null, null, intraActionState, _childTimelineRecordOrder - Root.PostJobSteps.Count); } } diff --git a/src/Runner.Worker/Handlers/ContainerActionHandler.cs b/src/Runner.Worker/Handlers/ContainerActionHandler.cs index 8c4f22602b4..10059ec6853 100644 --- a/src/Runner.Worker/Handlers/ContainerActionHandler.cs +++ b/src/Runner.Worker/Handlers/ContainerActionHandler.cs @@ -82,9 +82,13 @@ public async Task RunAsync(ActionRunStage stage) container.ContainerEntryPoint = Inputs.GetValueOrDefault("entryPoint"); } } + else if (stage == ActionRunStage.Pre) + { + container.ContainerEntryPoint = Data.Pre; + } else if (stage == ActionRunStage.Post) { - container.ContainerEntryPoint = Data.Cleanup; + container.ContainerEntryPoint = Data.Post; } // create inputs context for template evaluation diff --git a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs index fb3b15448aa..c28f3de9373 100644 --- a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs +++ b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs @@ -60,9 +60,13 @@ public async Task RunAsync(ActionRunStage stage) { target = Data.Script; } + else if (stage == ActionRunStage.Pre) + { + target = Data.Pre; + } else if (stage == ActionRunStage.Post) { - target = Data.Cleanup; + target = Data.Post; } ArgUtil.NotNullOrEmpty(target, nameof(target)); diff --git a/src/Runner.Worker/Handlers/RunnerPluginHandler.cs b/src/Runner.Worker/Handlers/RunnerPluginHandler.cs index c082fe9fcf3..6b73b19f175 100644 --- a/src/Runner.Worker/Handlers/RunnerPluginHandler.cs +++ b/src/Runner.Worker/Handlers/RunnerPluginHandler.cs @@ -31,7 +31,7 @@ public async Task RunAsync(ActionRunStage stage) } else if (stage == ActionRunStage.Post) { - plugin = Data.Cleanup; + plugin = Data.Post; } ArgUtil.NotNullOrEmpty(plugin, nameof(plugin)); diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 232ec18cf10..2554039b42d 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -197,8 +197,8 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel // Download actions not already in the cache Trace.Info("Downloading actions"); var actionManager = HostContext.GetService(); - var prepareSteps = await actionManager.PrepareActionsAsync(context, message.Steps); - preJobSteps.AddRange(prepareSteps); + var prepareResult = await actionManager.PrepareActionsAsync(context, message.Steps); + preJobSteps.AddRange(prepareResult.ContainerSetupSteps); // Add start-container steps, record and stop-container steps if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0) @@ -239,9 +239,23 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel actionRunner.TryEvaluateDisplayName(contextData, context); jobSteps.Add(actionRunner); + + if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep)) + { + Trace.Info($"Adding pre-{action.DisplayName}."); + preStep.TryEvaluateDisplayName(contextData, context); + preStep.DisplayName = $"Pre {preStep.DisplayName}"; + preJobSteps.Add(preStep); + } } } + var intraActionStates = new Dictionary>(); + foreach (var preStep in prepareResult.PreStepTracker) + { + intraActionStates[preStep.Key] = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + // Create execution context for pre-job steps foreach (var step in preJobSteps) { @@ -252,6 +266,12 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel Guid stepId = Guid.NewGuid(); extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N")); } + else if (step is IActionRunner actionStep) + { + ArgUtil.NotNull(actionStep, step.DisplayName); + Guid stepId = Guid.NewGuid(); + actionStep.ExecutionContext = jobContext.CreateChild(stepId, actionStep.DisplayName, stepId.ToString("N"), null, null, intraActionStates[actionStep.Action.Id]); + } } // Create execution context for job steps @@ -260,7 +280,8 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel if (step is IActionRunner actionStep) { ArgUtil.NotNull(actionStep, step.DisplayName); - actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName); + intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState); + actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName, intraActionState); } } diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index 10e691694c1..7a8b847d31f 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -43,6 +43,8 @@ "entrypoint": "non-empty-string", "args": "container-runs-args", "env": "container-runs-env", + "pre-entrypoint": "non-empty-string", + "pre-if": "non-empty-string", "post-entrypoint": "non-empty-string", "post-if": "non-empty-string" } @@ -67,6 +69,8 @@ "properties": { "using": "non-empty-string", "main": "non-empty-string", + "pre": "non-empty-string", + "pre-if": "non-empty-string", "post": "non-empty-string", "post-if": "non-empty-string" } diff --git a/src/Test/L0/Listener/MessageListenerL0.cs b/src/Test/L0/Listener/MessageListenerL0.cs index ea358e73efa..ba0f0ee7716 100644 --- a/src/Test/L0/Listener/MessageListenerL0.cs +++ b/src/Test/L0/Listener/MessageListenerL0.cs @@ -660,7 +660,7 @@ public async void CreateSessionWithOriginalGetMessageMigtateToMigrated() _settings.AgentId)) .Returns(async () => { - await Task.Delay(10); + await Task.Delay(100); return "https://t.server"; }); diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 34b64880d3c..59d7ed17c0b 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -56,7 +56,7 @@ public async void PrepareActions_PullImageFromDockerHub() }; //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; //Assert Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); @@ -217,7 +217,7 @@ public async void PrepareActions_SkipDownloadActionForSelfRepo() }; //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; Assert.True(steps.Count == 0); } @@ -256,7 +256,7 @@ public async void PrepareActions_RepositoryActionWithDockerfile() var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithdockerfile"); //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); Assert.Equal(Path.Combine(actionDir, "Dockerfile"), (steps[0].Data as ContainerSetupInfo).Container.Dockerfile); @@ -296,7 +296,7 @@ public async void PrepareActions_RepositoryActionWithDockerfileInRelativePath() var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithdockerfileinrelativepath"); //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); @@ -335,7 +335,7 @@ public async void PrepareActions_RepositoryActionWithActionfile_Dockerfile() var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithdockerfileinrelativepath"); //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); @@ -375,7 +375,7 @@ public async void PrepareActions_RepositoryActionWithActionfile_DockerfileRelati var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "RepositoryActionWithActionfile_DockerfileRelativePath"); //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); @@ -415,7 +415,7 @@ public async void PrepareActions_RepositoryActionWithActionfile_DockerHubImage() var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "RepositoryActionWithActionfile_DockerHubImage"); //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); Assert.Equal("ubuntu:18.04", (steps[0].Data as ContainerSetupInfo).Container.Image); @@ -454,7 +454,7 @@ public async void PrepareActions_RepositoryActionWithActionYamlFile_DockerHubIma var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "RepositoryActionWithActionYamlFile_DockerHubImage"); //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; Assert.Equal((steps[0].Data as ContainerSetupInfo).StepIds[0], actionId); Assert.Equal("ubuntu:18.04", (steps[0].Data as ContainerSetupInfo).Container.Image); @@ -493,7 +493,7 @@ public async void PrepareActions_RepositoryActionWithActionfileAndDockerfile() var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithactionfileanddockerfile"); //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); @@ -610,7 +610,7 @@ public async void PrepareActions_NotPullOrBuildImagesMultipleTimes() }; //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; //Assert Assert.Equal(actionId1, (steps[0].Data as ContainerSetupInfo).StepIds[0]); @@ -671,7 +671,7 @@ public async void PrepareActions_RepositoryActionWithActionfile_Node() }; //Act - var steps = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; // node.js based action doesn't need any extra steps to build/pull containers. Assert.True(steps.Count == 0); @@ -682,6 +682,104 @@ public async void PrepareActions_RepositoryActionWithActionfile_Node() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithInvalidWrapperActionfile_Node() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithInvalidWrapperActionfile_Node", + RepositoryType = "GitHub" + } + } + }; + + //Act + try + { + await _actionManager.PrepareActionsAsync(_ec.Object, actions); + } + catch (ArgumentException) + { + var traceFile = Path.GetTempFileName(); + File.Copy(_hc.TraceFileName, traceFile, true); + Assert.Contains("Entry javascript file is not provided.", File.ReadAllText(traceFile)); + } + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithWrapperActionfile_PreSteps() + { + try + { + //Arrange + Setup(); + + _hc.EnqueueInstance(new Mock().Object); + _hc.EnqueueInstance(new Mock().Object); + + var actionId1 = Guid.NewGuid(); + var actionId2 = Guid.NewGuid(); + _hc.GetTrace().Info(actionId1); + _hc.GetTrace().Info(actionId2); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action1", + Id = actionId1, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithWrapperActionfile_Node", + RepositoryType = "GitHub" + } + }, + new Pipelines.ActionStep() + { + Name = "action2", + Id = actionId2, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithWrapperActionfile_Docker", + RepositoryType = "GitHub" + } + } + }; + + //Act + var preResult = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + Assert.Equal(2, preResult.PreStepTracker.Count); + Assert.NotNull(preResult.PreStepTracker[actionId1]); + Assert.NotNull(preResult.PreStepTracker[actionId2]); + } + finally + { + Teardown(); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -1426,7 +1524,7 @@ public void LoadsNodeActionDefinition_Cleanup() Assert.NotNull((definition.Data.Execution as NodeJSActionExecutionData)); Assert.Equal("task.js", (definition.Data.Execution as NodeJSActionExecutionData).Script); - Assert.Equal("cleanup.js", (definition.Data.Execution as NodeJSActionExecutionData).Cleanup); + Assert.Equal("cleanup.js", (definition.Data.Execution as NodeJSActionExecutionData).Post); } finally { @@ -1506,7 +1604,7 @@ public void LoadsContainerActionDefinitionDockerfile_Cleanup() Assert.NotNull((definition.Data.Execution as ContainerActionExecutionData)); // execution.Node Assert.Equal("image:1234", (definition.Data.Execution as ContainerActionExecutionData).Image); Assert.Equal("main.sh", (definition.Data.Execution as ContainerActionExecutionData).EntryPoint); - Assert.Equal("cleanup.sh", (definition.Data.Execution as ContainerActionExecutionData).Cleanup); + Assert.Equal("cleanup.sh", (definition.Data.Execution as ContainerActionExecutionData).Post); foreach (var arg in (definition.Data.Execution as ContainerActionExecutionData).Arguments) { @@ -1595,7 +1693,7 @@ public void LoadsPluginActionDefinition() Assert.NotNull((definition.Data.Execution as PluginActionExecutionData)); Assert.Equal("plugin.class, plugin", (definition.Data.Execution as PluginActionExecutionData).Plugin); - Assert.Equal("plugin.cleanup, plugin", (definition.Data.Execution as PluginActionExecutionData).Cleanup); + Assert.Equal("plugin.cleanup, plugin", (definition.Data.Execution as PluginActionExecutionData).Post); } finally { diff --git a/src/Test/L0/Worker/ActionManifestManagerL0.cs b/src/Test/L0/Worker/ActionManifestManagerL0.cs index ca789b7f38e..07f99a0aec8 100644 --- a/src/Test/L0/Worker/ActionManifestManagerL0.cs +++ b/src/Test/L0/Worker/ActionManifestManagerL0.cs @@ -65,6 +65,52 @@ public void Load_ContainerAction_Dockerfile() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Load_ContainerAction_Dockerfile_Pre() + { + try + { + //Arrange + Setup(); + + var actionManifest = new ActionManifestManager(); + actionManifest.Initialize(_hc); + + //Act + var result = actionManifest.Load(_ec.Object, Path.Combine(TestUtil.GetTestDataPath(), "dockerfileaction_init.yml")); + + //Assert + + Assert.Equal("Hello World", result.Name); + Assert.Equal("Greet the world and record the time", result.Description); + Assert.Equal(2, result.Inputs.Count); + Assert.Equal("greeting", result.Inputs[0].Key.AssertString("key").Value); + Assert.Equal("Hello", result.Inputs[0].Value.AssertString("value").Value); + Assert.Equal("entryPoint", result.Inputs[1].Key.AssertString("key").Value); + Assert.Equal("", result.Inputs[1].Value.AssertString("value").Value); + + Assert.Equal(ActionExecutionType.Container, result.Execution.ExecutionType); + + var containerAction = result.Execution as ContainerActionExecutionData; + + Assert.Equal("Dockerfile", containerAction.Image); + Assert.Equal("main.sh", containerAction.EntryPoint); + Assert.Equal("init.sh", containerAction.Pre); + Assert.Equal("success()", containerAction.InitCondition); + Assert.Equal("bzz", containerAction.Arguments[0].ToString()); + Assert.Equal("Token", containerAction.Environment[0].Key.ToString()); + Assert.Equal("foo", containerAction.Environment[0].Value.ToString()); + Assert.Equal("Url", containerAction.Environment[1].Key.ToString()); + Assert.Equal("bar", containerAction.Environment[1].Value.ToString()); + } + finally + { + Teardown(); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -97,7 +143,7 @@ public void Load_ContainerAction_Dockerfile_Post() Assert.Equal("Dockerfile", containerAction.Image); Assert.Equal("main.sh", containerAction.EntryPoint); - Assert.Equal("cleanup.sh", containerAction.Cleanup); + Assert.Equal("cleanup.sh", containerAction.Post); Assert.Equal("failure()", containerAction.CleanupCondition); Assert.Equal("bzz", containerAction.Arguments[0].ToString()); Assert.Equal("Token", containerAction.Environment[0].Key.ToString()); @@ -111,6 +157,52 @@ public void Load_ContainerAction_Dockerfile_Post() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Load_ContainerAction_Dockerfile_Pre_DefaultCondition() + { + try + { + //Arrange + Setup(); + + var actionManifest = new ActionManifestManager(); + actionManifest.Initialize(_hc); + + //Act + var result = actionManifest.Load(_ec.Object, Path.Combine(TestUtil.GetTestDataPath(), "dockerfileaction_init_default.yml")); + + //Assert + + Assert.Equal("Hello World", result.Name); + Assert.Equal("Greet the world and record the time", result.Description); + Assert.Equal(2, result.Inputs.Count); + Assert.Equal("greeting", result.Inputs[0].Key.AssertString("key").Value); + Assert.Equal("Hello", result.Inputs[0].Value.AssertString("value").Value); + Assert.Equal("entryPoint", result.Inputs[1].Key.AssertString("key").Value); + Assert.Equal("", result.Inputs[1].Value.AssertString("value").Value); + + Assert.Equal(ActionExecutionType.Container, result.Execution.ExecutionType); + + var containerAction = result.Execution as ContainerActionExecutionData; + + Assert.Equal("Dockerfile", containerAction.Image); + Assert.Equal("main.sh", containerAction.EntryPoint); + Assert.Equal("init.sh", containerAction.Pre); + Assert.Equal("always()", containerAction.InitCondition); + Assert.Equal("bzz", containerAction.Arguments[0].ToString()); + Assert.Equal("Token", containerAction.Environment[0].Key.ToString()); + Assert.Equal("foo", containerAction.Environment[0].Value.ToString()); + Assert.Equal("Url", containerAction.Environment[1].Key.ToString()); + Assert.Equal("bar", containerAction.Environment[1].Value.ToString()); + } + finally + { + Teardown(); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -143,7 +235,7 @@ public void Load_ContainerAction_Dockerfile_Post_DefaultCondition() Assert.Equal("Dockerfile", containerAction.Image); Assert.Equal("main.sh", containerAction.EntryPoint); - Assert.Equal("cleanup.sh", containerAction.Cleanup); + Assert.Equal("cleanup.sh", containerAction.Post); Assert.Equal("always()", containerAction.CleanupCondition); Assert.Equal("bzz", containerAction.Arguments[0].ToString()); Assert.Equal("Token", containerAction.Environment[0].Key.ToString()); @@ -323,6 +415,94 @@ public void Load_NodeAction() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Load_NodeAction_Pre() + { + try + { + //Arrange + Setup(); + + var actionManifest = new ActionManifestManager(); + actionManifest.Initialize(_hc); + + //Act + var result = actionManifest.Load(_ec.Object, Path.Combine(TestUtil.GetTestDataPath(), "nodeaction_init.yml")); + + //Assert + Assert.Equal("Hello World", result.Name); + Assert.Equal("Greet the world and record the time", result.Description); + Assert.Equal(2, result.Inputs.Count); + Assert.Equal("greeting", result.Inputs[0].Key.AssertString("key").Value); + Assert.Equal("Hello", result.Inputs[0].Value.AssertString("value").Value); + Assert.Equal("entryPoint", result.Inputs[1].Key.AssertString("key").Value); + Assert.Equal("", result.Inputs[1].Value.AssertString("value").Value); + Assert.Equal(1, result.Deprecated.Count); + + Assert.True(result.Deprecated.ContainsKey("greeting")); + result.Deprecated.TryGetValue("greeting", out string value); + Assert.Equal("This property has been deprecated", value); + + Assert.Equal(ActionExecutionType.NodeJS, result.Execution.ExecutionType); + + var nodeAction = result.Execution as NodeJSActionExecutionData; + + Assert.Equal("main.js", nodeAction.Script); + Assert.Equal("init.js", nodeAction.Pre); + Assert.Equal("cancelled()", nodeAction.InitCondition); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Load_NodeAction_Init_DefaultCondition() + { + try + { + //Arrange + Setup(); + + var actionManifest = new ActionManifestManager(); + actionManifest.Initialize(_hc); + + //Act + var result = actionManifest.Load(_ec.Object, Path.Combine(TestUtil.GetTestDataPath(), "nodeaction_init_default.yml")); + + //Assert + Assert.Equal("Hello World", result.Name); + Assert.Equal("Greet the world and record the time", result.Description); + Assert.Equal(2, result.Inputs.Count); + Assert.Equal("greeting", result.Inputs[0].Key.AssertString("key").Value); + Assert.Equal("Hello", result.Inputs[0].Value.AssertString("value").Value); + Assert.Equal("entryPoint", result.Inputs[1].Key.AssertString("key").Value); + Assert.Equal("", result.Inputs[1].Value.AssertString("value").Value); + Assert.Equal(1, result.Deprecated.Count); + + Assert.True(result.Deprecated.ContainsKey("greeting")); + result.Deprecated.TryGetValue("greeting", out string value); + Assert.Equal("This property has been deprecated", value); + + Assert.Equal(ActionExecutionType.NodeJS, result.Execution.ExecutionType); + + var nodeAction = result.Execution as NodeJSActionExecutionData; + + Assert.Equal("main.js", nodeAction.Script); + Assert.Equal("init.js", nodeAction.Pre); + Assert.Equal("always()", nodeAction.InitCondition); + } + finally + { + Teardown(); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -358,7 +538,7 @@ public void Load_NodeAction_Cleanup() var nodeAction = result.Execution as NodeJSActionExecutionData; Assert.Equal("main.js", nodeAction.Script); - Assert.Equal("cleanup.js", nodeAction.Cleanup); + Assert.Equal("cleanup.js", nodeAction.Post); Assert.Equal("cancelled()", nodeAction.CleanupCondition); } finally @@ -402,7 +582,7 @@ public void Load_NodeAction_Cleanup_DefaultCondition() var nodeAction = result.Execution as NodeJSActionExecutionData; Assert.Equal("main.js", nodeAction.Script); - Assert.Equal("cleanup.js", nodeAction.Cleanup); + Assert.Equal("cleanup.js", nodeAction.Post); Assert.Equal("always()", nodeAction.CleanupCondition); } finally diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index aef52d402eb..38fe135ee4c 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -199,20 +199,20 @@ public void RegisterPostJobAction_ShareState() var postRunner1 = hc.CreateService(); - postRunner1.Action = new Pipelines.ActionStep() { Name = "post1", DisplayName = "Test 1", Reference = new Pipelines.RepositoryPathReference() { Name = "actions/action" } }; + postRunner1.Action = new Pipelines.ActionStep() { Id = Guid.NewGuid(), Name = "post1", DisplayName = "Test 1", Reference = new Pipelines.RepositoryPathReference() { Name = "actions/action" } }; postRunner1.Stage = ActionRunStage.Post; postRunner1.Condition = "always()"; postRunner1.DisplayName = "post1"; var postRunner2 = hc.CreateService(); - postRunner2.Action = new Pipelines.ActionStep() { Name = "post2", DisplayName = "Test 2", Reference = new Pipelines.RepositoryPathReference() { Name = "actions/action" } }; + postRunner2.Action = new Pipelines.ActionStep() { Id = Guid.NewGuid(), Name = "post2", DisplayName = "Test 2", Reference = new Pipelines.RepositoryPathReference() { Name = "actions/action" } }; postRunner2.Stage = ActionRunStage.Post; postRunner2.Condition = "always()"; postRunner2.DisplayName = "post2"; - action1.RegisterPostJobStep("post1", postRunner1); - action2.RegisterPostJobStep("post2", postRunner2); + action1.RegisterPostJobStep(postRunner1); + action2.RegisterPostJobStep(postRunner2); Assert.NotNull(jobContext.JobSteps); Assert.NotNull(jobContext.PostJobSteps); @@ -238,6 +238,91 @@ public void RegisterPostJobAction_ShareState() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void RegisterPostJobAction_NotRegisterPostTwice() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange: Create a job request message. + TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); + TimelineReference timeline = new TimelineReference(); + Guid jobId = Guid.NewGuid(); + string jobName = "some job name"; + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); + jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); + jobRequest.Variables["ACTIONS_STEP_DEBUG"] = "true"; + + // Arrange: Setup the paging logger. + var pagingLogger1 = new Mock(); + var pagingLogger2 = new Mock(); + var pagingLogger3 = new Mock(); + var pagingLogger4 = new Mock(); + var pagingLogger5 = new Mock(); + var jobServerQueue = new Mock(); + jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.IsAny())); + jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny())).Callback((Guid id, string msg) => { hc.GetTrace().Info(msg); }); + + var actionRunner1 = new ActionRunner(); + actionRunner1.Initialize(hc); + var actionRunner2 = new ActionRunner(); + actionRunner2.Initialize(hc); + + hc.EnqueueInstance(pagingLogger1.Object); + hc.EnqueueInstance(pagingLogger2.Object); + hc.EnqueueInstance(pagingLogger3.Object); + hc.EnqueueInstance(pagingLogger4.Object); + hc.EnqueueInstance(pagingLogger5.Object); + hc.EnqueueInstance(actionRunner1 as IActionRunner); + hc.EnqueueInstance(actionRunner2 as IActionRunner); + hc.SetSingleton(jobServerQueue.Object); + + var jobContext = new Runner.Worker.ExecutionContext(); + jobContext.Initialize(hc); + + // Act. + jobContext.InitializeJob(jobRequest, CancellationToken.None); + + var action1 = jobContext.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null); + var action2 = jobContext.CreateChild(Guid.NewGuid(), "action_1_main", "action_1_main", null, null); + + var actionId = Guid.NewGuid(); + var postRunner1 = hc.CreateService(); + postRunner1.Action = new Pipelines.ActionStep() { Id = actionId, Name = "post1", DisplayName = "Test 1", Reference = new Pipelines.RepositoryPathReference() { Name = "actions/action" } }; + postRunner1.Stage = ActionRunStage.Post; + postRunner1.Condition = "always()"; + postRunner1.DisplayName = "post1"; + + + var postRunner2 = hc.CreateService(); + postRunner2.Action = new Pipelines.ActionStep() { Id = actionId, Name = "post2", DisplayName = "Test 2", Reference = new Pipelines.RepositoryPathReference() { Name = "actions/action" } }; + postRunner2.Stage = ActionRunStage.Post; + postRunner2.Condition = "always()"; + postRunner2.DisplayName = "post2"; + + action1.RegisterPostJobStep(postRunner1); + action2.RegisterPostJobStep(postRunner2); + + Assert.NotNull(jobContext.JobSteps); + Assert.NotNull(jobContext.PostJobSteps); + Assert.Equal(1, jobContext.PostJobSteps.Count); + var post1 = jobContext.PostJobSteps.Pop(); + + Assert.Equal("post1", (post1 as IActionRunner).Action.Name); + + Assert.Equal(ActionRunStage.Post, (post1 as IActionRunner).Stage); + + Assert.Equal("always()", (post1 as IActionRunner).Condition); + } + } + private TestHostContext CreateTestContext([CallerMemberName] String testName = "") { var hc = new TestHostContext(this, testName); diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index a8c4573e61f..0101db135ad 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -141,7 +141,7 @@ public async Task JobExtensionBuildStepsList() jobExtension.Initialize(hc); _actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny(), It.IsAny>())) - .Returns(Task.FromResult(new List())); + .Returns(Task.FromResult(new PrepareResult(new List(), new Dictionary()))); List result = await jobExtension.InitializeJob(_jobEc, _message); @@ -176,7 +176,7 @@ public async Task JobExtensionBuildPreStepsList() jobExtension.Initialize(hc); _actionManager.Setup(x => x.PrepareActionsAsync(It.IsAny(), It.IsAny>())) - .Returns(Task.FromResult(new List() { new JobExtensionRunner(null, "", "prepare1", null), new JobExtensionRunner(null, "", "prepare2", null) })); + .Returns(Task.FromResult(new PrepareResult(new List() { new JobExtensionRunner(null, "", "prepare1", null), new JobExtensionRunner(null, "", "prepare2", null) }, new Dictionary()))); List result = await jobExtension.InitializeJob(_jobEc, _message); diff --git a/src/Test/TestData/dockerfileaction_init.yml b/src/Test/TestData/dockerfileaction_init.yml new file mode 100644 index 00000000000..3407f58a938 --- /dev/null +++ b/src/Test/TestData/dockerfileaction_init.yml @@ -0,0 +1,27 @@ +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'Test Corporation' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'docker' + image: 'Dockerfile' + args: + - 'bzz' + entrypoint: 'main.sh' + env: + Token: foo + Url: bar + pre-entrypoint: 'init.sh' + pre-if: 'success()' \ No newline at end of file diff --git a/src/Test/TestData/dockerfileaction_init_default.yml b/src/Test/TestData/dockerfileaction_init_default.yml new file mode 100644 index 00000000000..923fb8beb2e --- /dev/null +++ b/src/Test/TestData/dockerfileaction_init_default.yml @@ -0,0 +1,26 @@ +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'Test Corporation' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'docker' + image: 'Dockerfile' + args: + - 'bzz' + entrypoint: 'main.sh' + env: + Token: foo + Url: bar + pre-entrypoint: 'init.sh' \ No newline at end of file diff --git a/src/Test/TestData/nodeaction_init.yml b/src/Test/TestData/nodeaction_init.yml new file mode 100644 index 00000000000..c1140b3289c --- /dev/null +++ b/src/Test/TestData/nodeaction_init.yml @@ -0,0 +1,22 @@ +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'Test Corporation' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + deprecationMessage: 'This property has been deprecated' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'main.js' + pre: 'init.js' + pre-if: 'cancelled()' \ No newline at end of file diff --git a/src/Test/TestData/nodeaction_init_default.yml b/src/Test/TestData/nodeaction_init_default.yml new file mode 100644 index 00000000000..8d300a11a7c --- /dev/null +++ b/src/Test/TestData/nodeaction_init_default.yml @@ -0,0 +1,21 @@ +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'Test Corporation' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + deprecationMessage: 'This property has been deprecated' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'main.js' + pre: 'init.js' \ No newline at end of file From 117ec1fff97fea432c38e8f0728e45b12fd0d500 Mon Sep 17 00:00:00 2001 From: Lokesh Gopu Date: Tue, 14 Apr 2020 12:14:26 -0400 Subject: [PATCH 33/86] Fix optional parameter for unattended (#425) --- src/Runner.Listener/Configuration/PromptManager.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Runner.Listener/Configuration/PromptManager.cs b/src/Runner.Listener/Configuration/PromptManager.cs index 274a248787d..3b765ef82e3 100644 --- a/src/Runner.Listener/Configuration/PromptManager.cs +++ b/src/Runner.Listener/Configuration/PromptManager.cs @@ -72,6 +72,10 @@ public string ReadValue( { return defaultValue; } + else if (isOptional) + { + return string.Empty; + } // Otherwise throw. throw new Exception($"Invalid configuration provided for {argName}. Terminating unattended configuration."); From c126b52fe5bb86c38032439abc64ae790bb01f38 Mon Sep 17 00:00:00 2001 From: David Kale Date: Tue, 14 Apr 2020 14:36:39 -0400 Subject: [PATCH 34/86] ArgumentNullException: Value cannot be null, for anonymous volume mounts (#426) * Dont check if path starts with null * Check SourceVolumePath not MountVolume obj * Prefer string.IsNullOrEmpty --- src/Runner.Worker/Handlers/StepHost.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/Handlers/StepHost.cs b/src/Runner.Worker/Handlers/StepHost.cs index 08edc91a24f..548a74f9554 100644 --- a/src/Runner.Worker/Handlers/StepHost.cs +++ b/src/Runner.Worker/Handlers/StepHost.cs @@ -110,9 +110,9 @@ public string ResolvePathForStepHost(string path) // try to resolve path inside container if the request path is part of the mount volume #if OS_WINDOWS - if (Container.MountVolumes.Exists(x => path.StartsWith(x.SourceVolumePath, StringComparison.OrdinalIgnoreCase))) + if (Container.MountVolumes.Exists(x => !string.IsNullOrEmpty(x.SourceVolumePath) && path.StartsWith(x.SourceVolumePath, StringComparison.OrdinalIgnoreCase))) #else - if (Container.MountVolumes.Exists(x => path.StartsWith(x.SourceVolumePath))) + if (Container.MountVolumes.Exists(x => !string.IsNullOrEmpty(x.SourceVolumePath) && path.StartsWith(x.SourceVolumePath))) #endif { return Container.TranslateToContainerPath(path); From b2dcdc21dc8c540f9e4214992c1d13fff83a7a6c Mon Sep 17 00:00:00 2001 From: Bryan MacFarlane Date: Fri, 17 Apr 2020 11:08:45 -0400 Subject: [PATCH 35/86] Sample scripts to automate scaleable runners (#427) --- docs/automate.md | 57 +++++++++++++++ scripts/create-latest-svc.sh | 135 +++++++++++++++++++++++++++++++++++ scripts/delete.sh | 83 +++++++++++++++++++++ scripts/remove-svc.sh | 76 ++++++++++++++++++++ 4 files changed, 351 insertions(+) create mode 100644 docs/automate.md create mode 100755 scripts/create-latest-svc.sh create mode 100755 scripts/delete.sh create mode 100755 scripts/remove-svc.sh diff --git a/docs/automate.md b/docs/automate.md new file mode 100644 index 00000000000..11a87a3ec3c --- /dev/null +++ b/docs/automate.md @@ -0,0 +1,57 @@ +# Automate Configuring Self-Hosted Runners + + +## Export PAT + +Before running any of these sample scripts, create a GitHub PAT and export it before running the script + +```bash +export RUNNER_CFG_PAT=yourPAT +``` + +## Create running as a service + +**Scenario**: Run on a machine or VM (not container) which automates: + + - Resolving latest released runner + - Download and extract latest + - Acquire a registration token + - Configure the runner + - Run as a systemd (linux) or Launchd (osx) service + +:point_right: [Sample script here](../scripts/create-latest-svc.sh) :point_left: + +Run as a one-liner. NOTE: replace with yourorg/yourrepo (repo level) or just yourorg (org level) +```bash +curl -s https://raw.githubusercontent.com/actions/runner/automate/scripts/create-latest-svc.sh | bash -s yourorg/yourrepo +``` + +## Uninstall running as service + +**Scenario**: Run on a machine or VM (not container) which automates: + + - Stops and uninstalls the systemd (linux) or Launchd (osx) service + - Acquires a removal token + - Removes the runner + +:point_right: [Sample script here](../scripts/remove-svc.sh) :point_left: + +Repo level one liner. NOTE: replace with yourorg/yourrepo (repo level) or just yourorg (org level) +```bash +curl -s https://raw.githubusercontent.com/actions/runner/automate/scripts/remove-svc.sh | bash -s yourorg/yourrepo +``` + +### Delete an offline runner + +**Scenario**: Deletes a registered runner that is offline: + + - Ensures the runner is offline + - Resolves id from name + - Deletes the runner + +:point_right: [Sample script here](../scripts/delete.sh) :point_left: + +Repo level one-liner. NOTE: replace with yourorg/yourrepo (repo level) or just yourorg (org level) and replace runnername +```bash +curl -s https://raw.githubusercontent.com/actions/runner/automate/scripts/delete.sh | bash -s yourorg/yourrepo runnername +``` diff --git a/scripts/create-latest-svc.sh b/scripts/create-latest-svc.sh new file mode 100755 index 00000000000..f056c51aded --- /dev/null +++ b/scripts/create-latest-svc.sh @@ -0,0 +1,135 @@ +#/bin/bash + +set -e + +# +# Downloads latest releases (not pre-release) runner +# Configures as a service +# +# Examples: +# RUNNER_CFG_PAT= ./create-latest-svc.sh myuser/myrepo +# RUNNER_CFG_PAT= ./create-latest-svc.sh myorg +# +# Usage: +# export RUNNER_CFG_PAT= +# ./create-latest-svc scope [name] [user] +# +# scope required repo (:owner/:repo) or org (:organization) +# name optional defaults to hostname +# user optional user svc will run as. defaults to current +# +# Notes: +# PATS over envvars are more secure +# Should be used on VMs and not containers +# Works on OSX and Linux +# Assumes x64 arch +# + +runner_scope=${1} +runner_name=${2:-$(hostname)} +svc_user=${3:-$USER} + +echo "Configuring runner @ ${runner_scope}" +sudo echo + +#--------------------------------------- +# Validate Environment +#--------------------------------------- +runner_plat=linux +[ ! -z "$(which sw_vers)" ] && runner_plat=osx; + +function fatal() +{ + echo "error: $1" >&2 + exit 1 +} + +if [ -z "${runner_scope}" ]; then fatal "supply scope as argument 1"; fi +if [ -z "${RUNNER_CFG_PAT}" ]; then fatal "RUNNER_CFG_PAT must be set before calling"; fi + +which curl || fatal "curl required. Please install in PATH with apt-get, brew, etc" +which jq || fatal "jq required. Please install in PATH with apt-get, brew, etc" + +# bail early if there's already a runner there. also sudo early +if [ -d ./runner ]; then + fatal "Runner already exists. Use a different directory or delete ./runner" +fi + +sudo -u ${svc_user} mkdir runner + +# TODO: validate not in a container +# TODO: validate systemd or osx svc installer + +#-------------------------------------- +# Get a config token +#-------------------------------------- +echo +echo "Generating a registration token..." + +# if the scope has a slash, it's an repo runner +base_api_url="https://api.github.com/orgs" +if [[ "$runner_scope" == *\/* ]]; then + base_api_url="https://api.github.com/repos" +fi + +export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token') + +if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi + +#--------------------------------------- +# Download latest released and extract +#--------------------------------------- +echo +echo "Downloading latest runner ..." + +latest_version_label=$(curl -s -X GET 'https://api.github.com/repos/actions/runner/releases/latest' | jq -r '.tag_name') +latest_version=$(echo ${latest_version_label:1}) +runner_file="actions-runner-${runner_plat}-x64-${latest_version}.tar.gz" + +if [ -f "${runner_file}" ]; then + echo "${runner_file} exists. skipping download." +else + runner_url="https://github.com/actions/runner/releases/download/${latest_version_label}/${runner_file}" + + echo "Downloading ${latest_version_label} for ${runner_plat} ..." + echo $runner_url + + curl -O -L ${runner_url} +fi + +ls -la *.tar.gz + +#--------------------------------------------------- +# extract to runner directory in this directory +#--------------------------------------------------- +echo +echo "Extracting ${runner_file} to ./runner" + +tar xzf "./${runner_file}" -C runner + +# export of pass +sudo chown -R $svc_user ./runner + +pushd ./runner + +#--------------------------------------- +# Unattend config +#--------------------------------------- +runner_url="https://github.com/${runner_scope}" +echo +echo "Configuring ${runner_name} @ $runner_url" +echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name" +sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN --name $runner_name + +#--------------------------------------- +# Configuring as a service +#--------------------------------------- +echo +echo "Configuring as a service ..." +prefix="" +if [ "${runner_plat}" == "linux" ]; then + prefix="sudo " +fi + +${prefix}./svc.sh install ${svc_user} +${prefix}./svc.sh start diff --git a/scripts/delete.sh b/scripts/delete.sh new file mode 100755 index 00000000000..96cf3a61e29 --- /dev/null +++ b/scripts/delete.sh @@ -0,0 +1,83 @@ +#/bin/bash + +set -e + +# +# Force deletes a runner from the service +# The caller should have already ensured the runner is gone and/or stopped +# +# Examples: +# RUNNER_CFG_PAT= ./delete.sh myuser/myrepo myname +# RUNNER_CFG_PAT= ./delete.sh myorg +# +# Usage: +# export RUNNER_CFG_PAT= +# ./delete.sh scope name +# +# scope required repo (:owner/:repo) or org (:organization) +# name optional defaults to hostname. name to delete +# +# Notes: +# PATS over envvars are more secure +# Works on OSX and Linux +# Assumes x64 arch +# + +runner_scope=${1} +runner_name=${2} + +echo "Deleting runner ${runner_name} @ ${runner_scope}" + +function fatal() +{ + echo "error: $1" >&2 + exit 1 +} + +if [ -z "${runner_scope}" ]; then fatal "supply scope as argument 1"; fi +if [ -z "${runner_name}" ]; then fatal "supply name as argument 2"; fi +if [ -z "${RUNNER_CFG_PAT}" ]; then fatal "RUNNER_CFG_PAT must be set before calling"; fi + +which curl || fatal "curl required. Please install in PATH with apt-get, brew, etc" +which jq || fatal "jq required. Please install in PATH with apt-get, brew, etc" + +base_api_url="https://api.github.com/orgs" +if [[ "$runner_scope" == *\/* ]]; then + base_api_url="https://api.github.com/repos" +fi + + +#-------------------------------------- +# Ensure offline +#-------------------------------------- +runner_status=$(curl -s -X GET ${base_api_url}/${runner_scope}/actions/runners?per_page=100 -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" \ + | jq -M -j ".runners | .[] | [select(.name == \"${runner_name}\")] | .[0].status") + +if [ -z "${runner_status}" ]; then + fatal "Could not find runner with name ${runner_name}" +fi + +echo "Status: ${runner_status}" + +if [ "${runner_status}" != "offline" ]; then + fatal "Runner should be offline before removing" +fi + +#-------------------------------------- +# Get id of runner to remove +#-------------------------------------- +runner_id=$(curl -s -X GET ${base_api_url}/${runner_scope}/actions/runners?per_page=100 -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" \ + | jq -M -j ".runners | .[] | [select(.name == \"${runner_name}\")] | .[0].id") + +if [ -z "${runner_id}" ]; then + fatal "Could not find runner with name ${runner_name}" +fi + +echo "Removing id ${runner_id}" + +#-------------------------------------- +# Remove the runner +#-------------------------------------- +curl -s -X DELETE ${base_api_url}/${runner_scope}/actions/runners/${runner_id} -H "authorization: token ${RUNNER_CFG_PAT}" + +echo "Done." diff --git a/scripts/remove-svc.sh b/scripts/remove-svc.sh new file mode 100755 index 00000000000..c55d0075d36 --- /dev/null +++ b/scripts/remove-svc.sh @@ -0,0 +1,76 @@ +#/bin/bash + +set -e + +# +# Removes a runner running as a service +# Must be run on the machine where the service is run +# +# Examples: +# RUNNER_CFG_PAT= ./remove-svc.sh myuser/myrepo +# RUNNER_CFG_PAT= ./remove-svc.sh myorg +# +# Usage: +# export RUNNER_CFG_PAT= +# ./remove-svc scope name +# +# scope required repo (:owner/:repo) or org (:organization) +# name optional defaults to hostname. name to uninstall and remove +# +# Notes: +# PATS over envvars are more secure +# Should be used on VMs and not containers +# Works on OSX and Linux +# Assumes x64 arch +# + +runner_scope=${1} +runner_name=${2:-$(hostname)} + +echo "Uninstalling runner ${runner_name} @ ${runner_scope}" +sudo echo + +function fatal() +{ + echo "error: $1" >&2 + exit 1 +} + +if [ -z "${runner_scope}" ]; then fatal "supply scope as argument 1"; fi +if [ -z "${RUNNER_CFG_PAT}" ]; then fatal "RUNNER_CFG_PAT must be set before calling"; fi + +which curl || fatal "curl required. Please install in PATH with apt-get, brew, etc" +which jq || fatal "jq required. Please install in PATH with apt-get, brew, etc" + +runner_plat=linux +[ ! -z "$(which sw_vers)" ] && runner_plat=osx; + +#-------------------------------------- +# Get a remove token +#-------------------------------------- +echo +echo "Generating a removal token..." + +# if the scope has a slash, it's an repo runner +base_api_url="https://api.github.com/orgs" +if [[ "$runner_scope" == *\/* ]]; then + base_api_url="https://api.github.com/repos" +fi + +export REMOVE_TOKEN=$(curl -s -X POST ${base_api_url}/${runner_scope}/actions/runners/remove-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token') + +if [ -z "$REMOVE_TOKEN" ]; then fatal "Failed to get a token"; fi + +#--------------------------------------- +# Stop and uninstall the service +#--------------------------------------- +echo +echo "Uninstall the service ..." +pushd ./runner +prefix="" +if [ "${runner_plat}" == "linux" ]; then + prefix="sudo " +fi +${prefix}./svc.sh stop +${prefix}./svc.sh uninstall +${prefix}./config.sh remove --token $REMOVE_TOKEN From c5fa9fb0625ce3f3c2830244fd5ce73275483cd9 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 17 Apr 2020 11:53:51 -0400 Subject: [PATCH 36/86] Print node version in debug instead of output. (#433) --- src/Runner.Worker/Handlers/StepHost.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/Handlers/StepHost.cs b/src/Runner.Worker/Handlers/StepHost.cs index 548a74f9554..0907eaed23e 100644 --- a/src/Runner.Worker/Handlers/StepHost.cs +++ b/src/Runner.Worker/Handlers/StepHost.cs @@ -149,14 +149,14 @@ public async Task DetermineNodeRuntimeVersion(IExecutionContext executio throw new NotSupportedException(msg); } nodeExternal = "node12_alpine"; - executionContext.Output($"Container distribution is alpine. Running JavaScript Action with external tool: {nodeExternal}"); + executionContext.Debug($"Container distribution is alpine. Running JavaScript Action with external tool: {nodeExternal}"); return nodeExternal; } } } // Optimistically use the default nodeExternal = "node12"; - executionContext.Output($"Running JavaScript Action with default external tool: {nodeExternal}"); + executionContext.Debug($"Running JavaScript Action with default external tool: {nodeExternal}"); return nodeExternal; } From 97883c8cd503e32d86af6c016b6c1eb9c4ab80be Mon Sep 17 00:00:00 2001 From: Jan Pazdziora Date: Sun, 19 Apr 2020 22:53:06 +0200 Subject: [PATCH 37/86] Fix spelling of RHEL and CentOS. (#436) --- docs/start/envlinux.md | 2 +- src/Misc/layoutbin/installdependencies.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/start/envlinux.md b/docs/start/envlinux.md index 4e27cf2ec2d..4ae20148d34 100644 --- a/docs/start/envlinux.md +++ b/docs/start/envlinux.md @@ -40,7 +40,7 @@ Debian based OS (Debian, Ubuntu, Linux Mint) - libssl1.1, libssl1.0.2 or libssl1.0.0 - libicu63, libicu60, libicu57 or libicu55 -Fedora based OS (Fedora, Redhat, Centos, Oracle Linux 7) +Fedora based OS (Fedora, Red Hat Enterprise Linux, CentOS, Oracle Linux 7) - lttng-ust - openssl-libs diff --git a/src/Misc/layoutbin/installdependencies.sh b/src/Misc/layoutbin/installdependencies.sh index 18b6fcbe99f..50a19982e0c 100755 --- a/src/Misc/layoutbin/installdependencies.sh +++ b/src/Misc/layoutbin/installdependencies.sh @@ -9,7 +9,7 @@ fi # Determine OS type # Debian based OS (Debian, Ubuntu, Linux Mint) has /etc/debian_version -# Fedora based OS (Fedora, Redhat, Centos, Oracle Linux 7) has /etc/redhat-release +# Fedora based OS (Fedora, Red Hat Enterprise Linux, CentOS, Oracle Linux 7) has /etc/redhat-release # SUSE based OS (OpenSUSE, SUSE Enterprise) has ID_LIKE=suse in /etc/os-release function print_errormessage() @@ -116,12 +116,12 @@ then elif [ -e /etc/redhat-release ] then echo "The current OS is Fedora based" - echo "--------Redhat Version--------" + echo "--Fedora/RHEL/CentOS Version--" cat /etc/redhat-release echo "------------------------------" # use dnf on fedora - # use yum on centos and redhat + # use yum on centos and rhel if [ -e /etc/fedora-release ] then command -v dnf @@ -191,7 +191,7 @@ then redhatRelease=$( Date: Mon, 20 Apr 2020 04:11:44 +0200 Subject: [PATCH 38/86] Make release notes code blocks copy-paste-able (#430) The release notes used C-style comments (`//`) instead of shell-style (`#`), causing the code snippet to not be easily copy-paste-able. Additionally, the code fence for Windows had the extra `powershell` added, as well as a comment to direct users towards `powershell` over `cmd`. --- releaseNote.md | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 6a9eb4606b0..25703dc58f2 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -8,13 +8,15 @@ - N/A ## Windows x64 -We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows -``` -// Create a folder under the drive root +We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. + +The following snipped needs to be run on `powershell`: +``` powershell +# Create a folder under the drive root mkdir \actions-runner ; cd \actions-runner -// Download the latest runner package +# Download the latest runner package Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v/actions-runner-win-x64-.zip -OutFile actions-runner-win-x64-.zip -// Extract the installer +# Extract the installer Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-x64-.zip", "$PWD") ``` @@ -22,44 +24,44 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ; ## OSX ``` bash -// Create a folder +# Create a folder mkdir actions-runner && cd actions-runner -// Download the latest runner package +# Download the latest runner package curl -O -L https://github.com/actions/runner/releases/download/v/actions-runner-osx-x64-.tar.gz -// Extract the installer +# Extract the installer tar xzf ./actions-runner-osx-x64-.tar.gz ``` ## Linux x64 ``` bash -// Create a folder +# Create a folder mkdir actions-runner && cd actions-runner -// Download the latest runner package +# Download the latest runner package curl -O -L https://github.com/actions/runner/releases/download/v/actions-runner-linux-x64-.tar.gz -// Extract the installer +# Extract the installer tar xzf ./actions-runner-linux-x64-.tar.gz ``` ## Linux arm64 (Pre-release) ``` bash -// Create a folder +# Create a folder mkdir actions-runner && cd actions-runner -// Download the latest runner package +# Download the latest runner package curl -O -L https://github.com/actions/runner/releases/download/v/actions-runner-linux-arm64-.tar.gz -// Extract the installer +# Extract the installer tar xzf ./actions-runner-linux-arm64-.tar.gz ``` ## Linux arm (Pre-release) ``` bash -// Create a folder +# Create a folder mkdir actions-runner && cd actions-runner -// Download the latest runner package +# Download the latest runner package curl -O -L https://github.com/actions/runner/releases/download/v/actions-runner-linux-arm-.tar.gz -// Extract the installer +# Extract the installer tar xzf ./actions-runner-linux-arm-.tar.gz ``` From d5c54f981946e997f1fe09afe670daa94e54528a Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 21 Apr 2020 19:42:53 -0400 Subject: [PATCH 39/86] Raise warning when action input does not match action.yml. (#429) --- src/Runner.Worker/ActionRunner.cs | 14 +++++++- src/Test/L0/Worker/ActionRunnerL0.cs | 53 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index 8331e99158f..b0c245ac317 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -143,8 +143,10 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions); + var userInputs = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (KeyValuePair input in inputs) { + userInputs.Add(input.Key); string message = ""; if (definition.Data?.Deprecated?.TryGetValue(input.Key, out message) == true) { @@ -152,13 +154,15 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && } } + var validInputs = new HashSet(StringComparer.OrdinalIgnoreCase); // Merge the default inputs from the definition if (definition.Data?.Inputs != null) { var manifestManager = HostContext.GetService(); - foreach (var input in (definition.Data?.Inputs)) + foreach (var input in definition.Data.Inputs) { string key = input.Key.AssertString("action input name").Value; + validInputs.Add(key); if (!inputs.ContainsKey(key)) { inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value); @@ -166,6 +170,14 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && } } + foreach (var input in userInputs) + { + if (!validInputs.Contains(input)) + { + ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']"); + } + } + // Load the action environment. ExecutionContext.Debug("Loading env"); var environment = new Dictionary(VarUtil.EnvironmentVariableKeyComparer); diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index 24ff73f4f04..73f215a3ffe 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -278,6 +278,59 @@ public void EvaluateDisplayNameWithoutContext() Assert.Equal("${{ matrix.node }}", _actionRunner.DisplayName); } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void WarnInvalidInputs() + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actionInputs = new MappingToken(null, null, null); + actionInputs.Add(new StringToken(null, null, null, "input1"), new StringToken(null, null, null, "test1")); + actionInputs.Add(new StringToken(null, null, null, "input2"), new StringToken(null, null, null, "test2")); + actionInputs.Add(new StringToken(null, null, null, "invalid1"), new StringToken(null, null, null, "invalid1")); + actionInputs.Add(new StringToken(null, null, null, "invalid2"), new StringToken(null, null, null, "invalid2")); + var action = new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.ContainerRegistryReference() + { + Image = "ubuntu:16.04" + }, + Inputs = actionInputs + }; + + _actionRunner.Action = action; + + Dictionary finialInputs = new Dictionary(); + _handlerFactory.Setup(x => x.Create(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Callback((IExecutionContext executionContext, Pipelines.ActionStepDefinitionReference actionReference, IStepHost stepHost, ActionExecutionData data, Dictionary inputs, Dictionary environment, Variables runtimeVariables, string taskDirectory) => + { + finialInputs = inputs; + }) + .Returns(new Mock().Object); + + //Act + await _actionRunner.RunAsync(); + + foreach (var input in finialInputs) + { + _hc.GetTrace().Info($"Input: {input.Key}={input.Value}"); + } + + //Assert + Assert.Equal("test1", finialInputs["input1"]); + Assert.Equal("test2", finialInputs["input2"]); + Assert.Equal("github", finialInputs["input3"]); + Assert.Equal("invalid1", finialInputs["invalid1"]); + Assert.Equal("invalid2", finialInputs["invalid2"]); + + _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Unexpected input 'invalid1'")), It.IsAny()), Times.Once); + _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Unexpected input 'invalid2'")), It.IsAny()), Times.Once); + } + private void Setup([CallerMemberName] string name = "") { _ecTokenSource?.Dispose(); From 3f7a01af939560ceb1222bca97d1ca4bee1ed843 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 21 Apr 2020 22:07:55 -0400 Subject: [PATCH 40/86] add secret masker for trimming double qoutes. (#440) --- src/Runner.Common/HostContext.cs | 1 + src/Sdk/DTLogging/Logging/ValueEncoders.cs | 14 ++++++++++++++ src/Test/L0/HostContextL0.cs | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 4da520913bc..b9a44fa2bc6 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -89,6 +89,7 @@ public HostContext(string hostType, string logFile = null) this.SecretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape); this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape); this.SecretMasker.AddValueEncoder(ValueEncoders.XmlDataEscape); + this.SecretMasker.AddValueEncoder(ValueEncoders.TrimDoubleQuotes); // Create the trace manager. if (string.IsNullOrEmpty(logFile)) diff --git a/src/Sdk/DTLogging/Logging/ValueEncoders.cs b/src/Sdk/DTLogging/Logging/ValueEncoders.cs index 77478799178..6a96c17206b 100644 --- a/src/Sdk/DTLogging/Logging/ValueEncoders.cs +++ b/src/Sdk/DTLogging/Logging/ValueEncoders.cs @@ -60,6 +60,20 @@ public static String XmlDataEscape(String value) return SecurityElement.Escape(value); } + public static String TrimDoubleQuotes(String value) + { + var trimmed = string.Empty; + if (!string.IsNullOrEmpty(value) && + value.Length > 8 && + value.StartsWith('"') && + value.EndsWith('"')) + { + trimmed = value.Substring(1, value.Length - 2); + } + + return trimmed; + } + private static string Base64StringEscapeShift(String value, int shift) { var bytes = Encoding.UTF8.GetBytes(value); diff --git a/src/Test/L0/HostContextL0.cs b/src/Test/L0/HostContextL0.cs index 4b5bbf6d172..9e5c529016f 100644 --- a/src/Test/L0/HostContextL0.cs +++ b/src/Test/L0/HostContextL0.cs @@ -85,6 +85,8 @@ public void DefaultSecretMaskers() _hc.SecretMasker.AddValue("Pass word 123!"); _hc.SecretMasker.AddValue("Pass123!"); _hc.SecretMasker.AddValue("Pass'word'123!"); + _hc.SecretMasker.AddValue("\"Password123!!\""); + _hc.SecretMasker.AddValue("\"short\""); // Assert. Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Password123!123")); @@ -99,6 +101,9 @@ public void DefaultSecretMaskers() Assert.Equal("YWJjOlBh***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abc:Password123!")))); Assert.Equal("YWJjZDpQ***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abcd:Password123!")))); Assert.Equal("YWJjZGU6***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abcde:Password123!")))); + Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Password123!!123")); + Assert.Equal("123short123", _hc.SecretMasker.MaskSecrets("123short123")); + Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123\"short\"123")); } finally { From f798f5606bc5d619ea4581a56e22dcc1e79e33b7 Mon Sep 17 00:00:00 2001 From: PJ Quirk Date: Thu, 23 Apr 2020 17:02:13 -0400 Subject: [PATCH 41/86] Use the API_URL and munge action URLs for GHES (#437) * First pass at logic for GHES, not all correct * Need to mock out file downloading * Allowed for mocking of HTTP responses * Added test for builtin GHES action download * More tests * Don't retry on action 404 * Remove commented out code * Add a using statement back, because Windows * Make windows happy again * Another windows fix * Always delete the cache since it isn't fully implemented * Use RunnerService base class * Add examples, update URL path * Remove forceDotCom * Fix a bug * Remove a test that's no longer relevant * PR feedback * Add missing return * More trace info * Use the new agreed-upon format * Use the auth token since we're hitting GHES directly * Fixing tests on windows * Fixed one more test --- src/Runner.Common/HostContext.cs | 16 +- src/Runner.Common/HttpClientHandlerFactory.cs | 19 ++ src/Runner.Worker/ActionManager.cs | 171 +++++++---- src/Runner.Worker/ActionNotFoundException.cs | 33 ++ src/Test/L0/RunnerWebProxyL0.cs | 4 +- src/Test/L0/Worker/ActionManagerL0.cs | 287 +++++++++++++++--- 6 files changed, 426 insertions(+), 104 deletions(-) create mode 100644 src/Runner.Common/HttpClientHandlerFactory.cs create mode 100644 src/Runner.Worker/ActionNotFoundException.cs diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index b9a44fa2bc6..8126f8c957c 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -1,19 +1,18 @@ -using GitHub.Runner.Common.Util; -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Tracing; using System.Globalization; using System.IO; using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; using System.Reflection; using System.Runtime.Loader; using System.Threading; using System.Threading.Tasks; -using System.Diagnostics; -using System.Net.Http; -using System.Diagnostics.Tracing; using GitHub.DistributedTask.Logging; -using System.Net.Http.Headers; using GitHub.Runner.Sdk; namespace GitHub.Runner.Common @@ -615,9 +614,8 @@ public static class HostContextExtension { public static HttpClientHandler CreateHttpClientHandler(this IHostContext context) { - HttpClientHandler clientHandler = new HttpClientHandler(); - clientHandler.Proxy = context.WebProxy; - return clientHandler; + var handlerFactory = context.GetService(); + return handlerFactory.CreateClientHandler(context.WebProxy); } } diff --git a/src/Runner.Common/HttpClientHandlerFactory.cs b/src/Runner.Common/HttpClientHandlerFactory.cs new file mode 100644 index 00000000000..f507dd7af39 --- /dev/null +++ b/src/Runner.Common/HttpClientHandlerFactory.cs @@ -0,0 +1,19 @@ +using System.Net.Http; +using GitHub.Runner.Sdk; + +namespace GitHub.Runner.Common +{ + [ServiceLocator(Default = typeof(HttpClientHandlerFactory))] + public interface IHttpClientHandlerFactory : IRunnerService + { + HttpClientHandler CreateClientHandler(RunnerWebProxy webProxy); + } + + public class HttpClientHandlerFactory : RunnerService, IHttpClientHandlerFactory + { + public HttpClientHandler CreateClientHandler(RunnerWebProxy webProxy) + { + return new HttpClientHandler() { Proxy = webProxy }; + } + } +} \ No newline at end of file diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 5338416590c..6865a7852f7 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -1,21 +1,19 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Threading; +using System.Threading.Tasks; using GitHub.DistributedTask.ObjectTemplating.Tokens; -using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using GitHub.Runner.Worker.Container; using GitHub.Services.Common; -using Newtonsoft.Json; using Pipelines = GitHub.DistributedTask.Pipelines; using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; @@ -73,13 +71,7 @@ public async Task PrepareActionsAsync(IExecutionContext execution } // Clear the cache (for self-hosted runners) - // Note, temporarily avoid this step for the on-premises product, to avoid rate limiting. - var configurationStore = HostContext.GetService(); - var isHostedServer = configurationStore.GetSettings().IsHostedServer; - if (isHostedServer) - { - IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken); - } + IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken); foreach (var action in actions) { @@ -490,7 +482,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref)); string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref); - string watermarkFile = destDirectory + ".completed"; + string watermarkFile = GetWatermarkFilePath(destDirectory); if (File.Exists(watermarkFile)) { executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'."); @@ -504,27 +496,84 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'"); } + var configurationStore = HostContext.GetService(); + var isHostedServer = configurationStore.GetSettings().IsHostedServer; + if (isHostedServer) + { + string apiUrl = GetApiUrl(executionContext); + string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref); + Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); + await DownloadRepositoryActionAsync(executionContext, archiveLink, destDirectory); + return; + } + else + { + string apiUrl = GetApiUrl(executionContext); + + // URLs to try: + var archiveLinks = new List { + // A built-in action or an action the user has created, on their GHES instance + // Example: https://my-ghes/api/v3/repos/my-org/my-action/tarball/v1 + BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref), + + // A community action, synced to their GHES instance + // Example: https://my-ghes/api/v3/repos/actions-community/some-org-some-action/tarball/v1 + BuildLinkToActionArchive(apiUrl, $"actions-community/{repositoryReference.Name.Replace("/", "-")}", repositoryReference.Ref) + }; + + foreach (var archiveLink in archiveLinks) + { + Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); + try + { + await DownloadRepositoryActionAsync(executionContext, archiveLink, destDirectory); + return; + } + catch (ActionNotFoundException) + { + Trace.Info($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}' at {archiveLink}"); + continue; + } + } + throw new ActionNotFoundException($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}'. Paths attempted: {string.Join(", ", archiveLinks)}"); + } + } + + private string GetApiUrl(IExecutionContext executionContext) + { + string apiUrl = executionContext.GetGitHubContext("api_url"); + if (!string.IsNullOrEmpty(apiUrl)) + { + return apiUrl; + } + // Once the api_url is set for hosted, we can remove this fallback (it doesn't make sense for GHES) + return "https://api.github.com"; + } + + private static string BuildLinkToActionArchive(string apiUrl, string repository, string @ref) + { #if OS_WINDOWS - string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/zipball/{repositoryReference.Ref}"; + return $"{apiUrl}/repos/{repository}/zipball/{@ref}"; #else - string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/tarball/{repositoryReference.Ref}"; + return $"{apiUrl}/repos/{repository}/tarball/{@ref}"; #endif - Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); + } + private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, string link, string destDirectory) + { //download and extract action in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); Directory.CreateDirectory(tempDirectory); - #if OS_WINDOWS string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip"); #else string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz"); #endif - Trace.Info($"Save archive '{archiveLink}' into {archiveFile}."); + + Trace.Info($"Save archive '{link}' into {archiveFile}."); try { - int retryCount = 0; // Allow up to 20 * 60s for any action to be downloaded from github graph. @@ -541,64 +590,76 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { - var configurationStore = HostContext.GetService(); - var isHostedServer = configurationStore.GetSettings().IsHostedServer; - if (isHostedServer) + var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); + if (string.IsNullOrEmpty(authToken)) { - var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); - if (string.IsNullOrEmpty(authToken)) - { - // TODO: Deprecate the PREVIEW_ACTION_TOKEN - authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); - } + // TODO: Deprecate the PREVIEW_ACTION_TOKEN + authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); + } - if (!string.IsNullOrEmpty(authToken)) - { - HostContext.SecretMasker.AddValue(authToken); - var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}")); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); - } - else - { - var accessToken = executionContext.GetGitHubContext("token"); - var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); - } + if (!string.IsNullOrEmpty(authToken)) + { + HostContext.SecretMasker.AddValue(authToken); + var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}")); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); } else { - // Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated. + var accessToken = executionContext.GetGitHubContext("token"); + var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); } httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); - using (var result = await httpClient.GetStreamAsync(archiveLink)) + using (var response = await httpClient.GetAsync(link)) { - await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token); - await fs.FlushAsync(actionDownloadCancellation.Token); - - // download succeed, break out the retry loop. - break; + if (response.IsSuccessStatusCode) + { + using (var result = await response.Content.ReadAsStreamAsync()) + { + await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token); + await fs.FlushAsync(actionDownloadCancellation.Token); + + // download succeed, break out the retry loop. + break; + } + } + else if (response.StatusCode == HttpStatusCode.NotFound) + { + // It doesn't make sense to retry in this case, so just stop + throw new ActionNotFoundException(new Uri(link)); + } + else + { + // Something else bad happened, let's go to our retry logic + response.EnsureSuccessStatusCode(); + } } } } catch (OperationCanceledException) when (executionContext.CancellationToken.IsCancellationRequested) { - Trace.Info($"Action download has been cancelled."); + Trace.Info("Action download has been cancelled."); + throw; + } + catch (ActionNotFoundException) + { + Trace.Info($"The action at '{link}' does not exist"); throw; } catch (Exception ex) when (retryCount < 2) { retryCount++; - Trace.Error($"Fail to download archive '{archiveLink}' -- Attempt: {retryCount}"); + Trace.Error($"Fail to download archive '{link}' -- Attempt: {retryCount}"); Trace.Error(ex); if (actionDownloadTimeout.Token.IsCancellationRequested) { // action download didn't finish within timeout - executionContext.Warning($"Action '{archiveLink}' didn't finish download within {timeoutSeconds} seconds."); + executionContext.Warning($"Action '{link}' didn't finish download within {timeoutSeconds} seconds."); } else { - executionContext.Warning($"Failed to download action '{archiveLink}'. Error {ex.Message}"); + executionContext.Warning($"Failed to download action '{link}'. Error: {ex.Message}"); } } } @@ -612,7 +673,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile)); - executionContext.Debug($"Download '{archiveLink}' to '{archiveFile}'"); + executionContext.Debug($"Download '{link}' to '{archiveFile}'"); var stagingDirectory = Path.Combine(tempDirectory, "_staging"); Directory.CreateDirectory(stagingDirectory); @@ -662,6 +723,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } Trace.Verbose("Create watermark file indicate action download succeed."); + string watermarkFile = GetWatermarkFilePath(destDirectory); File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString()); executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'."); @@ -686,6 +748,8 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } } + private string GetWatermarkFilePath(string directory) => directory + ".completed"; + private ActionContainer PrepareRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction) { var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference; @@ -931,4 +995,3 @@ public class ActionContainer public string ActionRepository { get; set; } } } - diff --git a/src/Runner.Worker/ActionNotFoundException.cs b/src/Runner.Worker/ActionNotFoundException.cs new file mode 100644 index 00000000000..9e67af44fc5 --- /dev/null +++ b/src/Runner.Worker/ActionNotFoundException.cs @@ -0,0 +1,33 @@ +using System; +using System.Runtime.Serialization; + +namespace GitHub.Runner.Worker +{ + public class ActionNotFoundException : Exception + { + public ActionNotFoundException(Uri actionUri) + : base(FormatMessage(actionUri)) + { + } + + public ActionNotFoundException(string message) + : base(message) + { + } + + public ActionNotFoundException(string message, System.Exception inner) + : base(message, inner) + { + } + + protected ActionNotFoundException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + + private static string FormatMessage(Uri actionUri) + { + return $"An action could not be found at the URI '{actionUri}'"; + } + } +} \ No newline at end of file diff --git a/src/Test/L0/RunnerWebProxyL0.cs b/src/Test/L0/RunnerWebProxyL0.cs index b83371d6ad5..3c1704f6cb1 100644 --- a/src/Test/L0/RunnerWebProxyL0.cs +++ b/src/Test/L0/RunnerWebProxyL0.cs @@ -16,7 +16,9 @@ public sealed class RunnerWebProxyL0 private static readonly List SkippedFiles = new List() { "Runner.Common\\HostContext.cs", - "Runner.Common/HostContext.cs" + "Runner.Common/HostContext.cs", + "Runner.Common\\HttpClientHandlerFactory.cs", + "Runner.Common/HttpClientHandlerFactory.cs" }; [Fact] diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 59d7ed17c0b..90d203147f5 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -1,19 +1,21 @@ -using GitHub.DistributedTask.Expressions2; -using GitHub.DistributedTask.ObjectTemplating.Tokens; -using GitHub.DistributedTask.Pipelines.ContextData; -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common.Util; -using GitHub.Runner.Worker; -using GitHub.Runner.Worker.Container; -using Moq; -using System; +using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; -using System.Reflection; +using System.Net; +using System.Net.Http; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Sdk; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using Moq; +using Moq.Protected; using Xunit; using Pipelines = GitHub.DistributedTask.Pipelines; @@ -114,47 +116,175 @@ public async void PrepareActions_DownloadActionFromGraph() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_SkipDownloadActionFromGraphWhenCached_OnPremises() + public async void PrepareActions_DownloadBuiltInActionFromGraph_OnPremises() { try { // Arrange Setup(); - var actionId = Guid.NewGuid(); + const string ActionName = "actions/sample-action"; var actions = new List { new Pipelines.ActionStep() { Name = "action", - Id = actionId, + Id = Guid.NewGuid(), Reference = new Pipelines.RepositoryPathReference() { - Name = "actions/no-such-action", + Name = ActionName, Ref = "master", RepositoryType = "GitHub" } } }; + + // Return a valid action from GHES via mock + const string ApiUrl = "https://ghes.example.com/api/v3"; + string expectedArchiveLink = GetLinkToActionArchive(ApiUrl, ActionName, "master"); + string archiveFile = await CreateRepoArchive(); + using var stream = File.OpenRead(archiveFile); + var mockClientHandler = new Mock(); + mockClientHandler.Protected().Setup>("SendAsync", ItExpr.Is(m => m.RequestUri == new Uri(expectedArchiveLink)), ItExpr.IsAny()) + .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(stream) }); + + var mockHandlerFactory = new Mock(); + mockHandlerFactory.Setup(p => p.CreateClientHandler(It.IsAny())).Returns(mockClientHandler.Object); + _hc.SetSingleton(mockHandlerFactory.Object); + + _ec.Setup(x => x.GetGitHubContext("api_url")).Returns(ApiUrl); _configurationStore.Object.GetSettings().IsHostedServer = false; - var actionDirectory = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/no-such-action", "master"); - Directory.CreateDirectory(actionDirectory); - var watermarkFile = $"{actionDirectory}.completed"; - File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString()); - var actionFile = Path.Combine(actionDirectory, "action.yml"); - File.WriteAllText(actionFile, @" -name: ""no-such-action"" -runs: - using: node12 - main: no-such-action.js -"); - var testFile = Path.Combine(actionDirectory, "test-file"); - File.WriteAllText(testFile, "asdf"); - // Act + //Act await _actionManager.PrepareActionsAsync(_ec.Object, actions); - // Assert - Assert.True(File.Exists(testFile)); + //Assert + var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master.completed"); + Assert.True(File.Exists(watermarkFile)); + + var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master", "action.yml"); + Assert.True(File.Exists(actionYamlFile)); + _hc.GetTrace().Info(File.ReadAllText(actionYamlFile)); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_DownloadCommunityActionFromGraph_OnPremises() + { + try + { + // Arrange + Setup(); + const string ActionName = "ownerName/sample-action"; + const string MungedActionName = "actions-community/ownerName-sample-action"; + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = Guid.NewGuid(), + Reference = new Pipelines.RepositoryPathReference() + { + Name = ActionName, + Ref = "master", + RepositoryType = "GitHub" + } + } + }; + + // Return a valid action from GHES via mock + const string ApiUrl = "https://ghes.example.com/api/v3"; + string builtInArchiveLink = GetLinkToActionArchive(ApiUrl, ActionName, "master"); + string mungedArchiveLink = GetLinkToActionArchive(ApiUrl, MungedActionName, "master"); + string archiveFile = await CreateRepoArchive(); + using var stream = File.OpenRead(archiveFile); + var mockClientHandler = new Mock(); + mockClientHandler.Protected().Setup>("SendAsync", ItExpr.Is(m => m.RequestUri == new Uri(builtInArchiveLink)), ItExpr.IsAny()) + .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.NotFound)); + mockClientHandler.Protected().Setup>("SendAsync", ItExpr.Is(m => m.RequestUri == new Uri(mungedArchiveLink)), ItExpr.IsAny()) + .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(stream) }); + + var mockHandlerFactory = new Mock(); + mockHandlerFactory.Setup(p => p.CreateClientHandler(It.IsAny())).Returns(mockClientHandler.Object); + _hc.SetSingleton(mockHandlerFactory.Object); + + _ec.Setup(x => x.GetGitHubContext("api_url")).Returns(ApiUrl); + _configurationStore.Object.GetSettings().IsHostedServer = false; + + //Act + await _actionManager.PrepareActionsAsync(_ec.Object, actions); + + //Assert + var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master.completed"); + Assert.True(File.Exists(watermarkFile)); + + var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master", "action.yml"); + Assert.True(File.Exists(actionYamlFile)); + _hc.GetTrace().Info(File.ReadAllText(actionYamlFile)); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_DownloadUnknownActionFromGraph_OnPremises() + { + try + { + // Arrange + Setup(); + const string ActionName = "ownerName/sample-action"; + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = Guid.NewGuid(), + Reference = new Pipelines.RepositoryPathReference() + { + Name = ActionName, + Ref = "master", + RepositoryType = "GitHub" + } + } + }; + + // Return a valid action from GHES via mock + const string ApiUrl = "https://ghes.example.com/api/v3"; + string archiveLink = GetLinkToActionArchive(ApiUrl, ActionName, "master"); + string archiveFile = await CreateRepoArchive(); + using var stream = File.OpenRead(archiveFile); + var mockClientHandler = new Mock(); + mockClientHandler.Protected().Setup>("SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.NotFound)); + + var mockHandlerFactory = new Mock(); + mockHandlerFactory.Setup(p => p.CreateClientHandler(It.IsAny())).Returns(mockClientHandler.Object); + _hc.SetSingleton(mockHandlerFactory.Object); + + _ec.Setup(x => x.GetGitHubContext("api_url")).Returns(ApiUrl); + _configurationStore.Object.GetSettings().IsHostedServer = false; + + //Act + Func action = async () => await _actionManager.PrepareActionsAsync(_ec.Object, actions); + + //Assert + await Assert.ThrowsAsync(action); + + var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master.completed"); + Assert.False(File.Exists(watermarkFile)); + + var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), ActionName, "master", "action.yml"); + Assert.False(File.Exists(actionYamlFile)); } finally { @@ -862,7 +992,7 @@ public void LoadsContainerActionDefinitionDockerfile() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -962,7 +1092,7 @@ public void LoadsContainerActionDefinitionRegistry() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1061,7 +1191,7 @@ public void LoadsNodeActionDefinition() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1129,7 +1259,7 @@ public void LoadsNodeActionDefinitionYaml() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1211,7 +1341,7 @@ public void LoadsContainerActionDefinitionDockerfile_SelfRepo() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1310,7 +1440,7 @@ public void LoadsContainerActionDefinitionRegistry_SelfRepo() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1408,7 +1538,7 @@ public void LoadsNodeActionDefinition_SelfRepo() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1476,7 +1606,7 @@ public void LoadsNodeActionDefinition_Cleanup() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1547,7 +1677,7 @@ public void LoadsContainerActionDefinitionDockerfile_Cleanup() name: 'Hello World' description: 'Greet the world and record the time' author: 'GitHub' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1647,7 +1777,7 @@ public void LoadsPluginActionDefinition() name: 'Hello World' description: 'Greet the world and record the time' author: 'Test Corporation' -inputs: +inputs: greeting: # id of input description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' required: true @@ -1737,6 +1867,82 @@ private void CreateSelfRepoAction(string yamlContent, out Pipelines.ActionStep i }; } + /// + /// Creates a sample action in an archive on disk, similar to the archive + /// retrieved from GitHub's or GHES' repository API. + /// + /// The path on disk to the archive. +#if OS_WINDOWS + private Task CreateRepoArchive() +#else + private async Task CreateRepoArchive() +#endif + { + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world' +author: 'GitHub' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'task.js' +"; + CreateAction(yamlContent: Content, instance: out _, directory: out string directory); + + var tempDir = _hc.GetDirectory(WellKnownDirectory.Temp); + Directory.CreateDirectory(tempDir); + var archiveFile = Path.Combine(tempDir, Path.GetRandomFileName()); + var trace = _hc.GetTrace(); + +#if OS_WINDOWS + ZipFile.CreateFromDirectory(directory, archiveFile, CompressionLevel.Fastest, includeBaseDirectory: true); + return Task.FromResult(archiveFile); +#else + string tar = WhichUtil.Which("tar", require: true, trace: trace); + + // tar -xzf + using (var processInvoker = new ProcessInvokerWrapper()) + { + processInvoker.Initialize(_hc); + processInvoker.OutputDataReceived += new EventHandler((sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + trace.Info(args.Data); + } + }); + + processInvoker.ErrorDataReceived += new EventHandler((sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + trace.Error(args.Data); + } + }); + + string cwd = Path.GetDirectoryName(directory); + string inputDirectory = Path.GetFileName(directory); + int exitCode = await processInvoker.ExecuteAsync(_hc.GetDirectory(WellKnownDirectory.Bin), tar, $"-czf \"{archiveFile}\" -C \"{cwd}\" \"{inputDirectory}\"", null, CancellationToken.None); + if (exitCode != 0) + { + throw new NotSupportedException($"Can't use 'tar -czf' to create archive file: {archiveFile}. return code: {exitCode}."); + } + } + return archiveFile; +#endif + } + + private static string GetLinkToActionArchive(string apiUrl, string repository, string @ref) + { +#if OS_WINDOWS + return $"{apiUrl}/repos/{repository}/zipball/{@ref}"; +#else + return $"{apiUrl}/repos/{repository}/tarball/{@ref}"; +#endif + } + private void Setup([CallerMemberName] string name = "") { _ecTokenSource?.Dispose(); @@ -1772,6 +1978,7 @@ private void Setup([CallerMemberName] string name = "") _hc.SetSingleton(_dockerManager.Object); _hc.SetSingleton(_pluginManager.Object); _hc.SetSingleton(actionManifest); + _hc.SetSingleton(new HttpClientHandlerFactory()); _configurationStore = new Mock(); _configurationStore From 2fadf430e4329abee503bba39f40d2b412458c1b Mon Sep 17 00:00:00 2001 From: eric sciple Date: Fri, 24 Apr 2020 13:38:59 -0400 Subject: [PATCH 42/86] post-alpha fixes for github.url github.api_url and github.graphql_url (#451) --- src/Runner.Worker/GitHubContext.cs | 5 +++-- src/Runner.Worker/JobExtension.cs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index 7a707d1a9b0..afc08708902 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -10,10 +10,11 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa { "action", "actor", - "api_url", // temp for GHES alpha release + "api_url", "base_ref", "event_name", "event_path", + "graphql_url", "head_ref", "job", "ref", @@ -22,7 +23,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "run_id", "run_number", "sha", - "url", // temp for GHES alpha release + "url", "workflow", "workspace", }; diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 2554039b42d..02745013c1f 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -131,7 +131,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel // Temporary hack for GHES alpha var configurationStore = HostContext.GetService(); var runnerSettings = configurationStore.GetSettings(); - if (!runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl)) + if (string.IsNullOrEmpty(context.GetGitHubContext("url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl)) { var url = new Uri(runnerSettings.GitHubUrl); var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}"; From 1470a3b6e205d2c58d4e31753d90374b45fdbde1 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 27 Apr 2020 23:02:00 -0400 Subject: [PATCH 43/86] better error when runner removed from service. (#441) --- src/Runner.Listener/MessageListener.cs | 14 ++++++++++++++ .../WebApi/WebApi/OAuth/VssOAuthTokenProvider.cs | 11 ++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index d1e43e283f1..056fe00f5ff 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -150,6 +150,20 @@ public async Task CreateSessionAsync(CancellationToken token) Trace.Error("Catch exception during create session."); Trace.Error(ex); + if (ex is VssOAuthTokenRequestException && creds.Federated is VssOAuthCredential vssOAuthCred) + { + // Check whether we get 401 because the runner registration already removed by the service. + // If the runner registration get deleted, we can't exchange oauth token. + Trace.Error("Test oauth app registration."); + var oauthTokenProvider = new VssOAuthTokenProvider(vssOAuthCred, new Uri(serverUrl)); + var authError = await oauthTokenProvider.ValidateCredentialAsync(token); + if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase)) + { + _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure."); + return false; + } + } + if (!IsSessionCreationExceptionRetriable(ex)) { if (_useMigratedCredentials) diff --git a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenProvider.cs b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenProvider.cs index e910b259f6b..84122a49434 100644 --- a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenProvider.cs +++ b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenProvider.cs @@ -119,6 +119,15 @@ protected override String AuthenticationScheme } } + public async Task ValidateCredentialAsync(CancellationToken cancellationToken) + { + var tokenHttpClient = new VssOAuthTokenHttpClient(this.SignInUrl); + var tokenResponse = await tokenHttpClient.GetTokenAsync(this.Grant, this.ClientCredential, this.TokenParameters, cancellationToken); + + // return the underlying authentication error + return tokenResponse.Error; + } + /// /// Issues a token request to the configured secure token service. On success, the access token issued by the /// token service is returned to the caller @@ -131,7 +140,7 @@ protected override async Task OnGetTokenAsync( CancellationToken cancellationToken) { if (this.SignInUrl == null || - this.Grant == null || + this.Grant == null || this.ClientCredential == null) { return null; From 70729fb3c497e6eaae9e66d972a87d6d5aa2c734 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 27 Apr 2020 23:44:17 -0400 Subject: [PATCH 44/86] Help trace worker crash in Kusto. (#450) * Help trace worker crash in Kusto. * more * feedback. --- src/Runner.Common/Constants.cs | 3 +++ src/Runner.Listener/JobDispatcher.cs | 5 +++-- src/Runner.Worker/ActionCommandManager.cs | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index e67b6d42ef9..8533e931a25 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -137,6 +137,9 @@ public static class ReturnCode public const int RunnerUpdating = 3; public const int RunOnceRunnerUpdating = 4; } + + public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry"; + public static readonly string WorkerCrash = "WORKER_CRASH"; } public static class RunnerEvent diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index 706044c55ae..45f509e8551 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -858,7 +858,6 @@ private async Task TryUploadUnfinishedLogs(Pipelines.AgentJobRequestMessage mess } } - // TODO: We need send detailInfo back to DT in order to add an issue for the job private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequestMessage message, Guid lockToken, TaskResult result, string detailInfo = null) { Trace.Entering(); @@ -952,8 +951,10 @@ private async Task LogWorkerProcessUnhandledException(Pipelines.AgentJobRequestM ArgUtil.NotNull(timeline, nameof(timeline)); TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job"); ArgUtil.NotNull(jobRecord, nameof(jobRecord)); + var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = errorMessage }; + unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash; jobRecord.ErrorCount++; - jobRecord.Issues.Add(new Issue() { Type = IssueType.Error, Message = errorMessage }); + jobRecord.Issues.Add(unhandledExceptionIssue); await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None); } catch (Exception ex) diff --git a/src/Runner.Worker/ActionCommandManager.cs b/src/Runner.Worker/ActionCommandManager.cs index 29bd4a03b05..132e5cce586 100644 --- a/src/Runner.Worker/ActionCommandManager.cs +++ b/src/Runner.Worker/ActionCommandManager.cs @@ -486,7 +486,10 @@ public void ProcessCommand(IExecutionContext context, string inputLine, ActionCo foreach (var property in command.Properties) { - issue.Data[property.Key] = property.Value; + if (!string.Equals(property.Key, Constants.Runner.InternalTelemetryIssueDataKey, StringComparison.OrdinalIgnoreCase)) + { + issue.Data[property.Key] = property.Value; + } } context.AddIssue(issue); From c7768d4a7be68be6493139db2897bac69c4cbedd Mon Sep 17 00:00:00 2001 From: Brian Cristante <33549821+brcrista@users.noreply.github.com> Date: Mon, 27 Apr 2020 23:53:53 -0400 Subject: [PATCH 45/86] Adapt create-latest-svc.sh to work with GHES (#452) * Add README * Adapt create-latest-svc to work with GHES * Fix inverted conditions --- scripts/README.md | 4 +++ scripts/create-latest-svc.sh | 52 ++++++++++++++++++++++-------------- 2 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000000..0c13018b4ae --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,4 @@ +# Sample scripts for self-hosted runners + +Here are some examples to work from if you'd like to automate your use of self-hosted runners. +See the docs [here](../docs/automate.md). \ No newline at end of file diff --git a/scripts/create-latest-svc.sh b/scripts/create-latest-svc.sh index f056c51aded..031e3cb4974 100755 --- a/scripts/create-latest-svc.sh +++ b/scripts/create-latest-svc.sh @@ -7,27 +7,29 @@ set -e # Configures as a service # # Examples: -# RUNNER_CFG_PAT= ./create-latest-svc.sh myuser/myrepo -# RUNNER_CFG_PAT= ./create-latest-svc.sh myorg +# RUNNER_CFG_PAT= ./create-latest-svc.sh myuser/myrepo my.ghe.deployment.net +# RUNNER_CFG_PAT= ./create-latest-svc.sh myorg my.ghe.deployment.net # # Usage: # export RUNNER_CFG_PAT= -# ./create-latest-svc scope [name] [user] +# ./create-latest-svc scope [ghe_domain] [name] [user] +# +# scope required repo (:owner/:repo) or org (:organization) +# ghe_domain optional the fully qualified domain name of your GitHub Enterprise Server deployment +# name optional defaults to hostname +# user optional user svc will run as. defaults to current # -# scope required repo (:owner/:repo) or org (:organization) -# name optional defaults to hostname -# user optional user svc will run as. defaults to current -# # Notes: # PATS over envvars are more secure # Should be used on VMs and not containers -# Works on OSX and Linux +# Works on OSX and Linux # Assumes x64 arch # runner_scope=${1} -runner_name=${2:-$(hostname)} -svc_user=${3:-$USER} +ghe_hostname=${2} +runner_name=${3:-$(hostname)} +svc_user=${4:-$USER} echo "Configuring runner @ ${runner_scope}" sudo echo @@ -51,9 +53,9 @@ which curl || fatal "curl required. Please install in PATH with apt-get, brew, which jq || fatal "jq required. Please install in PATH with apt-get, brew, etc" # bail early if there's already a runner there. also sudo early -if [ -d ./runner ]; then +if [ -d ./runner ]; then fatal "Runner already exists. Use a different directory or delete ./runner" -fi +fi sudo -u ${svc_user} mkdir runner @@ -66,15 +68,20 @@ sudo -u ${svc_user} mkdir runner echo echo "Generating a registration token..." -# if the scope has a slash, it's an repo runner -base_api_url="https://api.github.com/orgs" +base_api_url="https://api.github.com" +if [ -n "${ghe_hostname}" ]; then + base_api_url="https://${ghe_hostname}/api/v3" +fi + +# if the scope has a slash, it's a repo runner +orgs_or_repos="orgs" if [[ "$runner_scope" == *\/* ]]; then - base_api_url="https://api.github.com/repos" + orgs_or_repos="repos" fi -export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token') +export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${orgs_or_repos}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token') -if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi +if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi #--------------------------------------- # Download latest released and extract @@ -82,6 +89,7 @@ if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi echo echo "Downloading latest runner ..." +# For the GHES Alpha, download the runner from github.com latest_version_label=$(curl -s -X GET 'https://api.github.com/repos/actions/runner/releases/latest' | jq -r '.tag_name') latest_version=$(echo ${latest_version_label:1}) runner_file="actions-runner-${runner_plat}-x64-${latest_version}.tar.gz" @@ -116,6 +124,10 @@ pushd ./runner # Unattend config #--------------------------------------- runner_url="https://github.com/${runner_scope}" +if [ -n "${ghe_hostname}" ]; then + runner_url="https://${ghe_hostname}/${runner_scope}" +fi + echo echo "Configuring ${runner_name} @ $runner_url" echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name" @@ -127,9 +139,9 @@ sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNE echo echo "Configuring as a service ..." prefix="" -if [ "${runner_plat}" == "linux" ]; then - prefix="sudo " -fi +if [ "${runner_plat}" == "linux" ]; then +prefix="sudo " +fi ${prefix}./svc.sh install ${svc_user} ${prefix}./svc.sh start From a246b3b29d853e73db4f7be04cb25f85b3d98e27 Mon Sep 17 00:00:00 2001 From: Justin Hutchings Date: Thu, 7 May 2020 08:17:34 -0700 Subject: [PATCH 46/86] Add CodeQL Analysis workflow (#459) * Add CodeQL Analysis workflow * Fix path * Add manual build step Import manual build step from build.yml --- .github/workflows/codeql.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..783bbefdc62 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,35 @@ +name: "Code Scanning - Action" + +on: + push: + schedule: + - cron: '0 0 * * 0' + +jobs: + CodeQL-Build: + + strategy: + fail-fast: false + + + # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + # Override language selection by uncommenting this and choosing your languages + # with: + # languages: go, javascript, csharp, python, cpp, java + + - name: Manual build + run : | + ./dev.sh layout Release linux-x64 + working-directory: src + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 33d2d2c328bed8c2041fee0a90c5f615bb313f74 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Mon, 11 May 2020 11:41:16 -0400 Subject: [PATCH 47/86] update checkout@v1 for GHES (#470) --- src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs b/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs index 39fc9dc8440..5c83accce07 100644 --- a/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs +++ b/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs @@ -80,7 +80,12 @@ public async Task GetSourceAsync( // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); executionContext.Output($"Syncing repository: {repoFullName}"); - Uri repositoryUrl = new Uri($"https://github.com/{repoFullName}"); + + // Repository URL + var githubUrl = executionContext.GetGitHubContext("url"); + var githubUri = new Uri(!string.IsNullOrEmpty(githubUrl) ? githubUrl : "https://github.com"); + var portInfo = githubUri.IsDefaultPort ? string.Empty : $":{githubUri.Port}"; + Uri repositoryUrl = new Uri($"{githubUri.Scheme}://{githubUri.Host}{portInfo}/{repoFullName}"); if (!repositoryUrl.IsAbsoluteUri) { throw new InvalidOperationException("Repository url need to be an absolute uri."); From 01c9a8a8afe66d2d9ad2146b4646b647265f111c Mon Sep 17 00:00:00 2001 From: PJ Quirk Date: Mon, 11 May 2020 12:23:02 -0400 Subject: [PATCH 48/86] Remove community actions munging and add dotcom fallback for downloading actions (#469) * Fallback to dotcom rather than munged community actions * Encapsulate the link and the auth details * Rename the method to be clearer * Remove BOM --- src/Runner.Worker/ActionManager.cs | 92 +++++++++++++++++---------- src/Test/L0/Worker/ActionManagerL0.cs | 7 +- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 6865a7852f7..11242ab82f3 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -46,7 +46,7 @@ public sealed class ActionManager : RunnerService, IActionManager //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k). private const int _defaultCopyBufferSize = 81920; - + private const string _dotcomApiUrl = "https://api.github.com"; private readonly Dictionary _cachedActionContainers = new Dictionary(); public Dictionary CachedActionContainers => _cachedActionContainers; @@ -503,7 +503,8 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont string apiUrl = GetApiUrl(executionContext); string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref); Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); - await DownloadRepositoryActionAsync(executionContext, archiveLink, destDirectory); + var downloadDetails = new ActionDownloadDetails(archiveLink, ConfigureAuthorizationFromContext); + await DownloadRepositoryActionAsync(executionContext, downloadDetails, destDirectory); return; } else @@ -511,31 +512,35 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont string apiUrl = GetApiUrl(executionContext); // URLs to try: - var archiveLinks = new List { + var downloadAttempts = new List { // A built-in action or an action the user has created, on their GHES instance // Example: https://my-ghes/api/v3/repos/my-org/my-action/tarball/v1 - BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref), - - // A community action, synced to their GHES instance - // Example: https://my-ghes/api/v3/repos/actions-community/some-org-some-action/tarball/v1 - BuildLinkToActionArchive(apiUrl, $"actions-community/{repositoryReference.Name.Replace("/", "-")}", repositoryReference.Ref) + new ActionDownloadDetails( + BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref), + ConfigureAuthorizationFromContext), + + // The same action, on GitHub.com + // Example: https://api.github.com/repos/my-org/my-action/tarball/v1 + new ActionDownloadDetails( + BuildLinkToActionArchive(_dotcomApiUrl, repositoryReference.Name, repositoryReference.Ref), + configureAuthorization: (e,h) => { /* no authorization for dotcom */ }) }; - foreach (var archiveLink in archiveLinks) + foreach (var downloadAttempt in downloadAttempts) { - Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); + Trace.Info($"Download archive '{downloadAttempt.ArchiveLink}' to '{destDirectory}'."); try { - await DownloadRepositoryActionAsync(executionContext, archiveLink, destDirectory); + await DownloadRepositoryActionAsync(executionContext, downloadAttempt, destDirectory); return; } catch (ActionNotFoundException) { - Trace.Info($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}' at {archiveLink}"); + Trace.Info($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}' at {downloadAttempt.ArchiveLink}"); continue; } } - throw new ActionNotFoundException($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}'. Paths attempted: {string.Join(", ", archiveLinks)}"); + throw new ActionNotFoundException($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}'. Paths attempted: {string.Join(", ", downloadAttempts.Select(d => d.ArchiveLink))}"); } } @@ -547,7 +552,7 @@ private string GetApiUrl(IExecutionContext executionContext) return apiUrl; } // Once the api_url is set for hosted, we can remove this fallback (it doesn't make sense for GHES) - return "https://api.github.com"; + return _dotcomApiUrl; } private static string BuildLinkToActionArchive(string apiUrl, string repository, string @ref) @@ -559,7 +564,7 @@ private static string BuildLinkToActionArchive(string apiUrl, string repository, #endif } - private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, string link, string destDirectory) + private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, string destDirectory) { //download and extract action in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); @@ -571,6 +576,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz"); #endif + string link = actionDownloadDetails.ArchiveLink; Trace.Info($"Save archive '{link}' into {archiveFile}."); try { @@ -590,25 +596,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { - var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); - if (string.IsNullOrEmpty(authToken)) - { - // TODO: Deprecate the PREVIEW_ACTION_TOKEN - authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); - } - - if (!string.IsNullOrEmpty(authToken)) - { - HostContext.SecretMasker.AddValue(authToken); - var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}")); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); - } - else - { - var accessToken = executionContext.GetGitHubContext("token"); - var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); - } + actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient); httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); using (var response = await httpClient.GetAsync(link)) @@ -748,6 +736,29 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } } + private void ConfigureAuthorizationFromContext(IExecutionContext executionContext, HttpClient httpClient) + { + var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); + if (string.IsNullOrEmpty(authToken)) + { + // TODO: Deprecate the PREVIEW_ACTION_TOKEN + authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); + } + + if (!string.IsNullOrEmpty(authToken)) + { + HostContext.SecretMasker.AddValue(authToken); + var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}")); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); + } + else + { + var accessToken = executionContext.GetGitHubContext("token"); + var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); + } + } + private string GetWatermarkFilePath(string directory) => directory + ".completed"; private ActionContainer PrepareRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction) @@ -855,6 +866,19 @@ private ActionContainer PrepareRepositoryActionAsync(IExecutionContext execution throw new InvalidOperationException($"Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under '{fullPath}'. Did you forget to run actions/checkout before running your local action?"); } } + + private class ActionDownloadDetails + { + public string ArchiveLink { get; } + + public Action ConfigureAuthorization { get; } + + public ActionDownloadDetails(string archiveLink, Action configureAuthorization) + { + ArchiveLink = archiveLink; + ConfigureAuthorization = configureAuthorization; + } + } } public sealed class Definition diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 90d203147f5..9c41c0341f4 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -174,14 +174,13 @@ public async void PrepareActions_DownloadBuiltInActionFromGraph_OnPremises() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_DownloadCommunityActionFromGraph_OnPremises() + public async void PrepareActions_DownloadActionFromDotCom_OnPremises() { try { // Arrange Setup(); const string ActionName = "ownerName/sample-action"; - const string MungedActionName = "actions-community/ownerName-sample-action"; var actions = new List { new Pipelines.ActionStep() @@ -200,13 +199,13 @@ public async void PrepareActions_DownloadCommunityActionFromGraph_OnPremises() // Return a valid action from GHES via mock const string ApiUrl = "https://ghes.example.com/api/v3"; string builtInArchiveLink = GetLinkToActionArchive(ApiUrl, ActionName, "master"); - string mungedArchiveLink = GetLinkToActionArchive(ApiUrl, MungedActionName, "master"); + string dotcomArchiveLink = GetLinkToActionArchive("https://api.github.com", ActionName, "master"); string archiveFile = await CreateRepoArchive(); using var stream = File.OpenRead(archiveFile); var mockClientHandler = new Mock(); mockClientHandler.Protected().Setup>("SendAsync", ItExpr.Is(m => m.RequestUri == new Uri(builtInArchiveLink)), ItExpr.IsAny()) .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.NotFound)); - mockClientHandler.Protected().Setup>("SendAsync", ItExpr.Is(m => m.RequestUri == new Uri(mungedArchiveLink)), ItExpr.IsAny()) + mockClientHandler.Protected().Setup>("SendAsync", ItExpr.Is(m => m.RequestUri == new Uri(dotcomArchiveLink)), ItExpr.IsAny()) .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(stream) }); var mockHandlerFactory = new Mock(); From 911135e66ccc49a44f3708d31853bbeda6296e23 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 11 May 2020 12:36:16 -0400 Subject: [PATCH 49/86] add help info for '--labels' (#472) --- src/Runner.Listener/Runner.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index bcc982edace..2fd57d23c9c 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -466,6 +466,7 @@ private void PrintUsage(CommandSettings command) --url string Repository to add the runner to. Required if unattended --token string Registration token. Required if unattended --name string Name of the runner to configure (default {Environment.MachineName ?? "myrunner"}) + --labels string Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}' --work string Relative runner work directory (default {Constants.Path.WorkDirectory}) --replace Replace any existing runner with the same name (default false)"); #if OS_WINDOWS @@ -478,7 +479,9 @@ private void PrintUsage(CommandSettings command) Configure a runner non-interactively: .{separator}config.{ext} --unattended --url --token Configure a runner non-interactively, replacing any existing runner with the same name: - .{separator}config.{ext} --unattended --url --token --replace [--name ]"); + .{separator}config.{ext} --unattended --url --token --replace [--name ] + Configure a runner non-interactively with three extra labels: + .{separator}config.{ext} --unattended --url --token --labels L1,L2,L3"); #if OS_WINDOWS _term.WriteLine($@" Configure a runner to run as a service:"); _term.WriteLine($@" .{separator}config.{ext} --url --token --runasservice"); From 6922f3cb8684bf1151884422b246687232804aaf Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 11 May 2020 12:36:35 -0400 Subject: [PATCH 50/86] sps/token migration tweak, ActionResult casing. (#462) --- src/Runner.Listener/MessageListener.cs | 49 +++++++++++++++++--- src/Runner.Worker/ExecutionContext.cs | 4 +- src/Runner.Worker/JobContext.cs | 2 +- src/Runner.Worker/StepsContext.cs | 8 ++-- src/Test/L0/Worker/ExecutionContextL0.cs | 57 +++++++++++++++++++++++- src/Test/L0/Worker/StepsRunnerL0.cs | 28 ++++++------ 6 files changed, 119 insertions(+), 29 deletions(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 056fe00f5ff..5718654b767 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -164,9 +164,30 @@ public async Task CreateSessionAsync(CancellationToken token) } } + if (ex is TaskAgentSessionConflictException) + { + try + { + var newCred = await GetNewOAuthAuthorizationSetting(token, true); + if (newCred != null) + { + await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), newCred); + Trace.Info("Updated connection to use migrated credential for next CreateSession call."); + _useMigratedCredentials = true; + _authorizationUrlMigrationBackgroundTask = null; + _needToCheckAuthorizationUrlUpdate = false; + } + } + catch (Exception e) + { + Trace.Error("Fail to refresh connection with new authorization url."); + Trace.Error(e); + } + } + if (!IsSessionCreationExceptionRetriable(ex)) { - if (_useMigratedCredentials) + if (_useMigratedCredentials && !(ex is TaskAgentSessionConflictException)) { // migrated credentials might cause lose permission during permission check, // we will force to use original credential and try again @@ -516,14 +537,11 @@ ex is AccessDeniedException || } } - private async Task GetNewOAuthAuthorizationSetting(CancellationToken token) + private async Task GetNewOAuthAuthorizationSetting(CancellationToken token, bool adhoc = false) { Trace.Info("Start checking oauth authorization url update."); while (true) { - var backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromMinutes(30), TimeSpan.FromMinutes(45)); - await HostContext.Delay(backoff, token); - try { var migratedAuthorizationUrl = await _runnerServer.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId); @@ -538,7 +556,14 @@ private async Task GetNewOAuthAuthorizationSetting(CancellationT { // We don't need to update credentials. Trace.Info("No needs to update authorization url"); - await Task.Delay(TimeSpan.FromMilliseconds(-1), token); + if (adhoc) + { + return null; + } + else + { + await Task.Delay(TimeSpan.FromMilliseconds(-1), token); + } } var keyManager = HostContext.GetService(); @@ -572,7 +597,7 @@ private async Task GetNewOAuthAuthorizationSetting(CancellationT Trace.Verbose("No authorization url updates"); } } - catch (Exception ex) + catch (Exception ex) when (!token.IsCancellationRequested) { Trace.Error("Fail to get/test new authorization url."); Trace.Error(ex); @@ -588,6 +613,16 @@ private async Task GetNewOAuthAuthorizationSetting(CancellationT Trace.Error(e); } } + + if (adhoc) + { + return null; + } + else + { + var backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromMinutes(30), TimeSpan.FromMinutes(45)); + await HostContext.Delay(backoff, token); + } } } } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 48c8d096fad..f47629bf3d4 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -369,8 +369,8 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = if (!string.IsNullOrEmpty(ContextName)) { - StepsContext.SetOutcome(ScopeName, ContextName, (Outcome ?? Result ?? TaskResult.Succeeded).ToActionResult().ToString()); - StepsContext.SetConclusion(ScopeName, ContextName, (Result ?? TaskResult.Succeeded).ToActionResult().ToString()); + StepsContext.SetOutcome(ScopeName, ContextName, (Outcome ?? Result ?? TaskResult.Succeeded).ToActionResult()); + StepsContext.SetConclusion(ScopeName, ContextName, (Result ?? TaskResult.Succeeded).ToActionResult()); } return Result.Value; diff --git a/src/Runner.Worker/JobContext.cs b/src/Runner.Worker/JobContext.cs index 05d31ce281b..d824fbe91f2 100644 --- a/src/Runner.Worker/JobContext.cs +++ b/src/Runner.Worker/JobContext.cs @@ -21,7 +21,7 @@ public ActionResult? Status } set { - this["status"] = new StringContextData(value.ToString()); + this["status"] = new StringContextData(value.ToString().ToLowerInvariant()); } } diff --git a/src/Runner.Worker/StepsContext.cs b/src/Runner.Worker/StepsContext.cs index d9add5a09be..bcd3a6217d5 100644 --- a/src/Runner.Worker/StepsContext.cs +++ b/src/Runner.Worker/StepsContext.cs @@ -59,19 +59,19 @@ public void SetOutput( public void SetConclusion( string scopeName, string stepName, - string conclusion) + ActionResult conclusion) { var step = GetStep(scopeName, stepName); - step["conclusion"] = new StringContextData(conclusion); + step["conclusion"] = new StringContextData(conclusion.ToString().ToLowerInvariant()); } public void SetOutcome( string scopeName, string stepName, - string outcome) + ActionResult outcome) { var step = GetStep(scopeName, stepName); - step["outcome"] = new StringContextData(outcome); + step["outcome"] = new StringContextData(outcome.ToString().ToLowerInvariant()); } private DictionaryContextData GetStep(string scopeName, string stepName) diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 38fe135ee4c..7dbbf099f79 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -1,4 +1,5 @@ -using GitHub.DistributedTask.WebApi; +using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker; using Moq; using System; @@ -323,6 +324,60 @@ public void RegisterPostJobAction_NotRegisterPostTwice() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void ActionResult_Lowercase() + { + using (TestHostContext hc = CreateTestContext()) + { + TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); + TimelineReference timeline = new TimelineReference(); + Guid jobId = Guid.NewGuid(); + string jobName = "some job name"; + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); + jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); + jobRequest.Variables["ACTIONS_STEP_DEBUG"] = "true"; + + // Arrange: Setup the paging logger. + var pagingLogger1 = new Mock(); + var jobServerQueue = new Mock(); + hc.EnqueueInstance(pagingLogger1.Object); + hc.SetSingleton(jobServerQueue.Object); + + var jobContext = new Runner.Worker.ExecutionContext(); + jobContext.Initialize(hc); + + // Act. + jobContext.InitializeJob(jobRequest, CancellationToken.None); + + jobContext.StepsContext.SetConclusion(null, "step1", ActionResult.Success); + var conclusion1 = (jobContext.StepsContext.GetScope(null)["step1"] as DictionaryContextData)["conclusion"].ToString(); + Assert.Equal(conclusion1, conclusion1.ToLowerInvariant()); + + jobContext.StepsContext.SetOutcome(null, "step2", ActionResult.Cancelled); + var outcome1 = (jobContext.StepsContext.GetScope(null)["step2"] as DictionaryContextData)["outcome"].ToString(); + Assert.Equal(outcome1, outcome1.ToLowerInvariant()); + + jobContext.StepsContext.SetConclusion(null, "step3", ActionResult.Failure); + var conclusion2 = (jobContext.StepsContext.GetScope(null)["step3"] as DictionaryContextData)["conclusion"].ToString(); + Assert.Equal(conclusion2, conclusion2.ToLowerInvariant()); + + jobContext.StepsContext.SetOutcome(null, "step4", ActionResult.Skipped); + var outcome2 = (jobContext.StepsContext.GetScope(null)["step4"] as DictionaryContextData)["outcome"].ToString(); + Assert.Equal(outcome2, outcome2.ToLowerInvariant()); + + jobContext.JobContext.Status = ActionResult.Success; + Assert.Equal(jobContext.JobContext["status"].ToString(), jobContext.JobContext["status"].ToString().ToLowerInvariant()); + } + } + private TestHostContext CreateTestContext([CallerMemberName] String testName = "") { var hc = new TestHostContext(this, testName); diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index c2996fab51f..2d7cb9fb0c4 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -536,12 +536,12 @@ public async Task StepContextOutcome() step2.Verify(x => x.RunAsync(), Times.Once); step3.Verify(x => x.RunAsync(), Times.Once); - Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["outcome"].AssertString("")); - Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["conclusion"].AssertString("")); - Assert.Equal(TaskResult.Failed.ToActionResult().ToString(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["outcome"].AssertString("")); - Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["conclusion"].AssertString("")); - Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["outcome"].AssertString("")); - Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Failed.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["conclusion"].AssertString("")); } } @@ -572,12 +572,12 @@ public async Task StepContextConclusion() step2.Verify(x => x.RunAsync(), Times.Once); step3.Verify(x => x.RunAsync(), Times.Once); - Assert.Equal(TaskResult.Skipped.ToActionResult().ToString(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["outcome"].AssertString("")); - Assert.Equal(TaskResult.Skipped.ToActionResult().ToString(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["conclusion"].AssertString("")); - Assert.Equal(TaskResult.Failed.ToActionResult().ToString(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["outcome"].AssertString("")); - Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["conclusion"].AssertString("")); - Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["outcome"].AssertString("")); - Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Skipped.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Skipped.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step1"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Failed.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step2"].AssertDictionary("")["conclusion"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["outcome"].AssertString("")); + Assert.Equal(TaskResult.Succeeded.ToActionResult().ToString().ToLowerInvariant(), _stepContext.GetScope(null)["step3"].AssertDictionary("")["conclusion"].AssertString("")); } } @@ -615,8 +615,8 @@ private Mock CreateStep(TestHostContext hc, TaskResult result, st stepContext.Object.Result = r; } - _stepContext.SetOutcome("", stepContext.Object.ContextName, (stepContext.Object.Outcome ?? stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult().ToString()); - _stepContext.SetConclusion("", stepContext.Object.ContextName, (stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult().ToString()); + _stepContext.SetOutcome("", stepContext.Object.ContextName, (stepContext.Object.Outcome ?? stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult()); + _stepContext.SetConclusion("", stepContext.Object.ContextName, (stepContext.Object.Result ?? TaskResult.Succeeded).ToActionResult()); }); var trace = hc.GetTrace(); stepContext.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { trace.Info($"[{tag}]{message}"); }); From 462b5117c88d2251cd5a88b805f62fb0b5c82f08 Mon Sep 17 00:00:00 2001 From: David Kale Date: Mon, 11 May 2020 13:57:31 -0400 Subject: [PATCH 51/86] docker build using -f instead of implied default (#471) * pass -f to docker build * Wrong place * build path * Also pass docker context path * Tidy up format * PR Feedback --- src/Runner.Worker/ActionManager.cs | 7 ++++++- src/Runner.Worker/Container/DockerCommandManager.cs | 6 +++--- src/Runner.Worker/Handlers/ContainerActionHandler.cs | 7 ++++++- src/Test/L0/Worker/ActionManagerL0.cs | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 11242ab82f3..e60bb50ee88 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -430,7 +430,12 @@ private async Task BuildActionContainerAsync(IExecutionContext executionContext, var imageName = $"{dockerManger.DockerInstanceLabel}:{Guid.NewGuid().ToString("N")}"; while (retryCount < 3) { - buildExitCode = await dockerManger.DockerBuild(executionContext, setupInfo.Container.WorkingDirectory, Directory.GetParent(setupInfo.Container.Dockerfile).FullName, imageName); + buildExitCode = await dockerManger.DockerBuild( + executionContext, + setupInfo.Container.WorkingDirectory, + setupInfo.Container.Dockerfile, + Directory.GetParent(setupInfo.Container.Dockerfile).FullName, + imageName); if (buildExitCode == 0) { break; diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index 737c24852bc..fd2d1051764 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -17,7 +17,7 @@ public interface IDockerCommandManager : IRunnerService string DockerInstanceLabel { get; } Task DockerVersion(IExecutionContext context); Task DockerPull(IExecutionContext context, string image); - Task DockerBuild(IExecutionContext context, string workingDirectory, string dockerFile, string tag); + Task DockerBuild(IExecutionContext context, string workingDirectory, string dockerFile, string dockerContext, string tag); Task DockerCreate(IExecutionContext context, ContainerInfo container); Task DockerRun(IExecutionContext context, ContainerInfo container, EventHandler stdoutDataReceived, EventHandler stderrDataReceived); Task DockerStart(IExecutionContext context, string containerId); @@ -87,9 +87,9 @@ public async Task DockerPull(IExecutionContext context, string image) return await ExecuteDockerCommandAsync(context, "pull", image, context.CancellationToken); } - public async Task DockerBuild(IExecutionContext context, string workingDirectory, string dockerFile, string tag) + public async Task DockerBuild(IExecutionContext context, string workingDirectory, string dockerFile, string dockerContext, string tag) { - return await ExecuteDockerCommandAsync(context, "build", $"-t {tag} \"{dockerFile}\"", workingDirectory, context.CancellationToken); + return await ExecuteDockerCommandAsync(context, "build", $"-t {tag} -f \"{dockerFile}\" \"{dockerContext}\"", workingDirectory, context.CancellationToken); } public async Task DockerCreate(IExecutionContext context, ContainerInfo container) diff --git a/src/Runner.Worker/Handlers/ContainerActionHandler.cs b/src/Runner.Worker/Handlers/ContainerActionHandler.cs index 10059ec6853..6e93d191929 100644 --- a/src/Runner.Worker/Handlers/ContainerActionHandler.cs +++ b/src/Runner.Worker/Handlers/ContainerActionHandler.cs @@ -52,7 +52,12 @@ public async Task RunAsync(ActionRunStage stage) ExecutionContext.Output($"Dockerfile for action: '{dockerFile}'."); var imageName = $"{dockerManger.DockerInstanceLabel}:{ExecutionContext.Id.ToString("N")}"; - var buildExitCode = await dockerManger.DockerBuild(ExecutionContext, ExecutionContext.GetGitHubContext("workspace"), Directory.GetParent(dockerFile).FullName, imageName); + var buildExitCode = await dockerManger.DockerBuild( + ExecutionContext, + ExecutionContext.GetGitHubContext("workspace"), + dockerFile, + Directory.GetParent(dockerFile).FullName, + imageName); if (buildExitCode != 0) { throw new InvalidOperationException($"Docker build failed with exit code {buildExitCode}"); diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 9c41c0341f4..5be09a0efcc 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -1966,7 +1966,7 @@ private void Setup([CallerMemberName] string name = "") _dockerManager.Setup(x => x.DockerPull(_ec.Object, "ubuntu:16.04")).Returns(Task.FromResult(0)); _dockerManager.Setup(x => x.DockerPull(_ec.Object, "ubuntu:100.04")).Returns(Task.FromResult(1)); - _dockerManager.Setup(x => x.DockerBuild(_ec.Object, It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)); + _dockerManager.Setup(x => x.DockerBuild(_ec.Object, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)); _pluginManager = new Mock(); _pluginManager.Setup(x => x.GetPluginAction(It.IsAny())).Returns(new RunnerPluginActionInfo() { PluginTypeName = "plugin.class, plugin", PostPluginTypeName = "plugin.cleanup, plugin" }); From 7a6523602242d4a7cdaea81f09010fddf1f3999f Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Mon, 11 May 2020 15:05:59 -0400 Subject: [PATCH 52/86] prepare 2.262.0 runner release. --- releaseNote.md | 19 ++++++++++++++----- releaseVersion | 2 +- src/runnerversion | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 25703dc58f2..43a9063ffbe 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,11 +1,20 @@ ## Features - - Runner support for GHES Alpha (#381 #386 #390 #393 $401) - - Allow secrets context in Container.env (#388) + - Sample scripts to automate scalable runners (#427) + - Raise warning when action input does not match action.yml. (#429) + - Add secret masker for trimming double quotes. (#440) + - Use the API_URL and munge action URLs for GHES (#437 #469) + - Help trace worker crash in Kusto. (#450) + - update checkout@v1 for GHES (#470) ## Bugs - - Raise warning when volume mount root. (#413) - - Fix typo (#394) + - Print node version in debug instead of output. (#433) + - Better error when runner removed from service. (#441) + - Add help info for '--labels' config option (#472) + - Sps/token migration fix, job.status/steps.outcome/steps.conclusion case match with GitHub check suites conclusion. (#462) + - Docker build using -f instead of implied default (#471) ## Misc - - N/A + - Make release notes code blocks copy-paste-able (#430) + - Fix spelling of RHEL and CentOS. (#436) + - Add CodeQL Analysis workflow (#459) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/releaseVersion b/releaseVersion index 2973ad9e470..ef96e25e847 100644 --- a/releaseVersion +++ b/releaseVersion @@ -1 +1 @@ -2.168.0 + diff --git a/src/runnerversion b/src/runnerversion index 5f7924ad30f..93c284c977a 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.169.0 +2.262.0 From 09cf59c1e0c51654d406012fab87c96aa51f50b6 Mon Sep 17 00:00:00 2001 From: Brian Cristante <33549821+brcrista@users.noreply.github.com> Date: Mon, 11 May 2020 17:14:02 -0400 Subject: [PATCH 53/86] Use an env var to point to an Actions Service dev instance (#468) * Use an environment variable for the testing backdoor * Make the env var a boolean We'll still have to pass the URL on the command line * Empty commit to rerun CI --- src/Runner.Listener/Configuration/ConfigurationManager.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 289ccaa1a55..9a98c7d6505 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -92,9 +92,11 @@ public async Task ConfigureAsync(CommandSettings command) _term.WriteSection("Authentication"); while (true) { - // Get the URL + // When testing against a dev deployment of Actions Service, set this environment variable + var useDevActionsServiceUrl = Environment.GetEnvironmentVariable("USE_DEV_ACTIONS_SERVICE_URL"); var inputUrl = command.GetUrl(); - if (inputUrl.Contains("codedev.ms", StringComparison.OrdinalIgnoreCase)) + if (inputUrl.Contains("codedev.ms", StringComparison.OrdinalIgnoreCase) + || useDevActionsServiceUrl != null) { runnerSettings.ServerUrl = inputUrl; // Get the credentials From abf59bdcb6be5f146e58af87eb771a2ffe5edc90 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 12 May 2020 16:08:50 -0400 Subject: [PATCH 54/86] Fix configure as service with runner name has space. (#474) --- src/Misc/layoutbin/darwin.svc.sh.template | 1 + src/Misc/layoutbin/systemd.svc.sh.template | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Misc/layoutbin/darwin.svc.sh.template b/src/Misc/layoutbin/darwin.svc.sh.template index 5210eb94d81..8d2f96512f8 100644 --- a/src/Misc/layoutbin/darwin.svc.sh.template +++ b/src/Misc/layoutbin/darwin.svc.sh.template @@ -1,6 +1,7 @@ #!/bin/bash SVC_NAME="{{SvcNameVar}}" +SVC_NAME=${SVC_NAME// /_} SVC_DESCRIPTION="{{SvcDescription}}" user_id=`id -u` diff --git a/src/Misc/layoutbin/systemd.svc.sh.template b/src/Misc/layoutbin/systemd.svc.sh.template index a3d2390cf27..cbec3319754 100644 --- a/src/Misc/layoutbin/systemd.svc.sh.template +++ b/src/Misc/layoutbin/systemd.svc.sh.template @@ -1,6 +1,7 @@ #!/bin/bash SVC_NAME="{{SvcNameVar}}" +SVC_NAME=${SVC_NAME// /_} SVC_DESCRIPTION="{{SvcDescription}}" SVC_CMD=$1 From cd8e4ddba19f79874650ec54ba786c51d771f54e Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 12 May 2020 16:09:13 -0400 Subject: [PATCH 55/86] Validate inputs only for repo action, no warning for small delay. (#476) * validate inputs only for repo action, no warning for small delay. * l0 --- src/Runner.Worker/ActionRunner.cs | 10 +++++++--- src/Runner.Worker/ExecutionContext.cs | 6 ++++-- src/Test/L0/Worker/ActionRunnerL0.cs | 5 +++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index b0c245ac317..78c7943c755 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -170,11 +170,15 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && } } - foreach (var input in userInputs) + // Validate inputs only for actions with action.yml + if (Action.Reference.Type == Pipelines.ActionSourceType.Repository) { - if (!validInputs.Contains(input)) + foreach (var input in userInputs) { - ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']"); + if (!validInputs.Contains(input)) + { + ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']"); + } } } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index f47629bf3d4..54dbe38260a 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -109,6 +109,7 @@ public interface IExecutionContext : IRunnerService public sealed class ExecutionContext : RunnerService, IExecutionContext { private const int _maxIssueCount = 10; + private const int _throttlingDelayReportThreshold = 10 * 1000; // Don't report throttling with less than 10 seconds delay private readonly TimelineRecord _record = new TimelineRecord(); private readonly Dictionary _detailRecords = new Dictionary(); @@ -335,7 +336,7 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = } // report total delay caused by server throttling. - if (_totalThrottlingDelayInMilliseconds > 0) + if (_totalThrottlingDelayInMilliseconds > _throttlingDelayReportThreshold) { this.Warning($"The job has experienced {TimeSpan.FromMilliseconds(_totalThrottlingDelayInMilliseconds).TotalSeconds} seconds total delay caused by server throttling."); } @@ -851,7 +852,8 @@ private void JobServerQueueThrottling_EventReceived(object sender, ThrottlingEve { Interlocked.Add(ref _totalThrottlingDelayInMilliseconds, Convert.ToInt64(data.Delay.TotalMilliseconds)); - if (!_throttlingReported) + if (!_throttlingReported && + _totalThrottlingDelayInMilliseconds > _throttlingDelayReportThreshold) { this.Warning(string.Format("The job is currently being throttled by the server. You may experience delays in console line output, job status reporting, and action log uploads.")); diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index 73f215a3ffe..f8185d26dd3 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -295,9 +295,10 @@ public async void WarnInvalidInputs() { Name = "action", Id = actionId, - Reference = new Pipelines.ContainerRegistryReference() + Reference = new Pipelines.RepositoryPathReference() { - Image = "ubuntu:16.04" + Name = "actions/runner", + Ref = "v1" }, Inputs = actionInputs }; From 73307c0a307c7bc3d108063185bd688ede093f70 Mon Sep 17 00:00:00 2001 From: Quan TRAN Date: Thu, 14 May 2020 17:09:20 +0200 Subject: [PATCH 56/86] jq returns "null" if the field does not exist (#478) --- scripts/create-latest-svc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create-latest-svc.sh b/scripts/create-latest-svc.sh index 031e3cb4974..de04ee12cae 100755 --- a/scripts/create-latest-svc.sh +++ b/scripts/create-latest-svc.sh @@ -81,7 +81,7 @@ fi export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${orgs_or_repos}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token') -if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi +if [ "null" == "$RUNNER_TOKEN" -o -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi #--------------------------------------- # Download latest released and extract From b45c1b9440cd282ba5e719b110ec47226630e7ca Mon Sep 17 00:00:00 2001 From: eric sciple Date: Mon, 18 May 2020 13:02:30 -0400 Subject: [PATCH 57/86] switch GITHUB_URL to GITHUB_SERVER_URL (#482) --- src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs | 2 +- src/Runner.Worker/GitHubContext.cs | 2 +- src/Runner.Worker/JobExtension.cs | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs b/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs index 5c83accce07..5f776723ce1 100644 --- a/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs +++ b/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs @@ -82,7 +82,7 @@ public async Task GetSourceAsync( executionContext.Output($"Syncing repository: {repoFullName}"); // Repository URL - var githubUrl = executionContext.GetGitHubContext("url"); + var githubUrl = executionContext.GetGitHubContext("server_url"); var githubUri = new Uri(!string.IsNullOrEmpty(githubUrl) ? githubUrl : "https://github.com"); var portInfo = githubUri.IsDefaultPort ? string.Empty : $":{githubUri.Port}"; Uri repositoryUrl = new Uri($"{githubUri.Scheme}://{githubUri.Host}{portInfo}/{repoFullName}"); diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index afc08708902..ac6566ad919 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -22,8 +22,8 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "repository_owner", "run_id", "run_number", + "server_url", "sha", - "url", "workflow", "workspace", }; diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 02745013c1f..235cc90638e 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -131,12 +131,13 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel // Temporary hack for GHES alpha var configurationStore = HostContext.GetService(); var runnerSettings = configurationStore.GetSettings(); - if (string.IsNullOrEmpty(context.GetGitHubContext("url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl)) + if (string.IsNullOrEmpty(context.GetGitHubContext("server_url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl)) { var url = new Uri(runnerSettings.GitHubUrl); var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}"; - context.SetGitHubContext("url", $"{url.Scheme}://{url.Host}{portInfo}"); + context.SetGitHubContext("server_url", $"{url.Scheme}://{url.Host}{portInfo}"); context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3"); + context.SetGitHubContext("graphql_url", $"{url.Scheme}://{url.Host}{portInfo}/api/graphql"); } // Evaluate the job-level environment variables From 4fc87ddfc6c1f8891c3d1d504de8910864160465 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 19 May 2020 16:15:03 -0400 Subject: [PATCH 58/86] fix problem matcher for GHES (#488) --- src/Runner.Worker/Handlers/OutputManager.cs | 19 ++++++++++++++----- src/Test/L0/Worker/OutputManagerL0.cs | 12 ++++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/Runner.Worker/Handlers/OutputManager.cs b/src/Runner.Worker/Handlers/OutputManager.cs index 42478e44da8..a0c136c3f58 100644 --- a/src/Runner.Worker/Handlers/OutputManager.cs +++ b/src/Runner.Worker/Handlers/OutputManager.cs @@ -352,15 +352,24 @@ private string GetRepositoryPath(string filePath, int recursion = 0) if (File.Exists(gitConfigPath)) { // Check if the config contains the workflow repository url - var qualifiedRepository = _executionContext.GetGitHubContext("repository"); - var configMatch = $"url = https://github.com/{qualifiedRepository}"; + var serverUrl = _executionContext.GetGitHubContext("server_url"); + serverUrl = !string.IsNullOrEmpty(serverUrl) ? serverUrl : "https://github.com"; + var host = new Uri(serverUrl, UriKind.Absolute).Host; + var nameWithOwner = _executionContext.GetGitHubContext("repository"); + var patterns = new[] { + $"url = {serverUrl}/{nameWithOwner}", + $"url = git@{host}:{nameWithOwner}.git", + }; var content = File.ReadAllText(gitConfigPath); foreach (var line in content.Split("\n").Select(x => x.Trim())) { - if (String.Equals(line, configMatch, StringComparison.OrdinalIgnoreCase)) + foreach (var pattern in patterns) { - repositoryPath = directoryPath; - break; + if (String.Equals(line, pattern, StringComparison.OrdinalIgnoreCase)) + { + repositoryPath = directoryPath; + break; + } } } } diff --git a/src/Test/L0/Worker/OutputManagerL0.cs b/src/Test/L0/Worker/OutputManagerL0.cs index 8b50c08b5bf..bcd2936f798 100644 --- a/src/Test/L0/Worker/OutputManagerL0.cs +++ b/src/Test/L0/Worker/OutputManagerL0.cs @@ -686,14 +686,17 @@ public async void MatcherFile() // /workflow-repo/nested-other-repo // /other-repo // /other-repo/nested-workflow-repo + // /workflow-repo-using-ssh var workflowRepository = Path.Combine(workspaceDirectory, "workflow-repo"); var nestedOtherRepository = Path.Combine(workspaceDirectory, "workflow-repo", "nested-other-repo"); var otherRepository = Path.Combine(workspaceDirectory, workflowRepository, "nested-other-repo"); var nestedWorkflowRepository = Path.Combine(workspaceDirectory, "other-repo", "nested-workflow-repo"); + var workflowRepositoryUsingSsh = Path.Combine(workspaceDirectory, "workflow-repo-using-ssh"); await CreateRepository(hostContext, workflowRepository, "https://github.com/my-org/workflow-repo"); await CreateRepository(hostContext, nestedOtherRepository, "https://github.com/my-org/other-repo"); await CreateRepository(hostContext, otherRepository, "https://github.com/my-org/other-repo"); await CreateRepository(hostContext, nestedWorkflowRepository, "https://github.com/my-org/workflow-repo"); + await CreateRepository(hostContext, workflowRepositoryUsingSsh, "git@github.com:my-org/workflow-repo.git"); // Create test files var file_noRepository = Path.Combine(workspaceDirectory, "no-repo.txt"); @@ -703,7 +706,8 @@ public async void MatcherFile() var file_nestedOtherRepository = Path.Combine(nestedOtherRepository, "nested-other-repo"); var file_otherRepository = Path.Combine(otherRepository, "other-repo.txt"); var file_nestedWorkflowRepository = Path.Combine(nestedWorkflowRepository, "nested-workflow-repo.txt"); - foreach (var file in new[] { file_noRepository, file_workflowRepository, file_workflowRepository_nestedDirectory, file_workflowRepository_failsafe, file_nestedOtherRepository, file_otherRepository, file_nestedWorkflowRepository }) + var file_workflowRepositoryUsingSsh = Path.Combine(workflowRepositoryUsingSsh, "workflow-repo-using-ssh.txt"); + foreach (var file in new[] { file_noRepository, file_workflowRepository, file_workflowRepository_nestedDirectory, file_workflowRepository_failsafe, file_nestedOtherRepository, file_otherRepository, file_nestedWorkflowRepository, file_workflowRepositoryUsingSsh }) { Directory.CreateDirectory(Path.GetDirectoryName(file)); File.WriteAllText(file, ""); @@ -718,8 +722,9 @@ public async void MatcherFile() Process($"{file_nestedOtherRepository}: some error 6"); Process($"{file_otherRepository}: some error 7"); Process($"{file_nestedWorkflowRepository}: some error 8"); + Process($"{file_workflowRepositoryUsingSsh}: some error 9"); - Assert.Equal(8, _issues.Count); + Assert.Equal(9, _issues.Count); Assert.Equal("some error 1", _issues[0].Item1.Message); Assert.False(_issues[0].Item1.Data.ContainsKey("file")); @@ -744,6 +749,9 @@ public async void MatcherFile() Assert.Equal("some error 8", _issues[7].Item1.Message); Assert.Equal(file_nestedWorkflowRepository.Substring(nestedWorkflowRepository.Length + 1).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), _issues[7].Item1.Data["file"]); + + Assert.Equal("some error 9", _issues[8].Item1.Message); + Assert.Equal(file_workflowRepositoryUsingSsh.Substring(workflowRepositoryUsingSsh.Length + 1).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), _issues[8].Item1.Data["file"]); } Environment.SetEnvironmentVariable("RUNNER_TEST_GET_REPOSITORY_PATH_FAILSAFE", ""); From 6f260012a3fece14640dff9eda89bfb096bdde08 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 21 May 2020 11:09:50 -0400 Subject: [PATCH 59/86] Fix inputs validation warning, fix post step display name, fix worker crash due to error in step.env (#490) --- src/Runner.Worker/ActionRunner.cs | 14 ++ src/Runner.Worker/ExecutionContext.cs | 8 +- src/Runner.Worker/JobRunner.cs | 6 + src/Runner.Worker/StepsRunner.cs | 193 ++++++++++++++------------ src/runnerversion | 2 +- 5 files changed, 131 insertions(+), 92 deletions(-) diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index 78c7943c755..0c4cc76ce59 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -94,6 +94,13 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && if (handlerData.HasPost && (Stage == ActionRunStage.Pre || Stage == ActionRunStage.Main)) { string postDisplayName = $"Post {this.DisplayName}"; + if (Stage == ActionRunStage.Pre && + this.DisplayName.StartsWith("Pre ", StringComparison.OrdinalIgnoreCase)) + { + // Trim the leading `Pre ` from the display name. + // Otherwise, we will get `Post Pre xxx` as DisplayName for the Post step. + postDisplayName = $"Post {this.DisplayName.Substring("Pre ".Length)}"; + } var repositoryReference = Action.Reference as RepositoryPathReference; var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}"; var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" : @@ -155,6 +162,13 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && } var validInputs = new HashSet(StringComparer.OrdinalIgnoreCase); + if (handlerData.ExecutionType == ActionExecutionType.Container) + { + // container action always accept 'entryPoint' and 'args' as inputs + // https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepswithargs + validInputs.Add("entryPoint"); + validInputs.Add("args"); + } // Merge the default inputs from the definition if (definition.Data?.Inputs != null) { diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 54dbe38260a..7bfe0932a58 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -364,7 +364,11 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = } } - _cancellationTokenSource?.Dispose(); + if (Root != this) + { + // only dispose TokenSource for step level ExecutionContext + _cancellationTokenSource?.Dispose(); + } _logger.End(); @@ -852,7 +856,7 @@ private void JobServerQueueThrottling_EventReceived(object sender, ThrottlingEve { Interlocked.Add(ref _totalThrottlingDelayInMilliseconds, Convert.ToInt64(data.Delay.TotalMilliseconds)); - if (!_throttlingReported && + if (!_throttlingReported && _totalThrottlingDelayInMilliseconds > _throttlingDelayReportThreshold) { this.Warning(string.Format("The job is currently being throttled by the server. You may experience delays in console line output, job status reporting, and action log uploads.")); diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index a94d7dd3d49..58cf26bf48a 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -254,6 +254,12 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution Trace.Error(ex); return TaskResult.Failed; } + catch (TaskOrchestrationPlanTerminatedException ex) + { + Trace.Error($"TaskOrchestrationPlanTerminatedException received, while attempting to raise JobCompletedEvent for job {message.JobId}."); + Trace.Error(ex); + return TaskResult.Failed; + } catch (Exception ex) { Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}."); diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 3e758c6d53b..485a4cdf980 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -98,124 +98,139 @@ public async Task RunAsync(IExecutionContext jobContext) envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty); } + bool evaluateStepEnvFailed = false; if (step is IActionRunner actionStep) { // Set GITHUB_ACTION step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name); - // Evaluate and merge action's env block to env context - var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); - var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer); - foreach (var env in actionEnvironment) + try { - envContext[env.Key] = new StringContextData(env.Value ?? string.Empty); + // Evaluate and merge action's env block to env context + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); + var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer); + foreach (var env in actionEnvironment) + { + envContext[env.Key] = new StringContextData(env.Value ?? string.Empty); + } + } + catch (Exception ex) + { + // fail the step since there is an evaluate error. + Trace.Info("Caught exception from expression for step.env"); + evaluateStepEnvFailed = true; + step.ExecutionContext.Error(ex); + CompleteStep(step, nextStep, TaskResult.Failed); } } - try + if (!evaluateStepEnvFailed) { - // Register job cancellation call back only if job cancellation token not been fire before each step run - if (!jobContext.CancellationToken.IsCancellationRequested) + try { - // Test the condition again. The job was canceled after the condition was originally evaluated. - jobCancelRegister = jobContext.CancellationToken.Register(() => + // Register job cancellation call back only if job cancellation token not been fire before each step run + if (!jobContext.CancellationToken.IsCancellationRequested) { - // mark job as cancelled - jobContext.Result = TaskResult.Canceled; - jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); - - step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'."); - var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only - var conditionReTestResult = false; - if (HostContext.RunnerShutdownToken.IsCancellationRequested) - { - step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown."); - } - else + // Test the condition again. The job was canceled after the condition was originally evaluated. + jobCancelRegister = jobContext.CancellationToken.Register(() => { - try + // mark job as cancelled + jobContext.Result = TaskResult.Canceled; + jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); + + step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'."); + var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only + var conditionReTestResult = false; + if (HostContext.RunnerShutdownToken.IsCancellationRequested) { - var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter); - var condition = new BasicExpressionToken(null, null, null, step.Condition); - conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); + step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown."); } - catch (Exception ex) + else { - // Cancel the step since we get exception while re-evaluate step condition. - Trace.Info("Caught exception from expression when re-test condition on job cancellation."); - step.ExecutionContext.Error(ex); + try + { + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter); + var condition = new BasicExpressionToken(null, null, null, step.Condition); + conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); + } + catch (Exception ex) + { + // Cancel the step since we get exception while re-evaluate step condition. + Trace.Info("Caught exception from expression when re-test condition on job cancellation."); + step.ExecutionContext.Error(ex); + } } - } - if (!conditionReTestResult) + if (!conditionReTestResult) + { + // Cancel the step. + Trace.Info("Cancel current running step."); + step.ExecutionContext.CancelToken(); + } + }); + } + else + { + if (jobContext.Result != TaskResult.Canceled) { - // Cancel the step. - Trace.Info("Cancel current running step."); - step.ExecutionContext.CancelToken(); + // mark job as cancelled + jobContext.Result = TaskResult.Canceled; + jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); } - }); - } - else - { - if (jobContext.Result != TaskResult.Canceled) - { - // mark job as cancelled - jobContext.Result = TaskResult.Canceled; - jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); } - } - // Evaluate condition. - step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'"); - var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext); - var conditionResult = false; - var conditionEvaluateError = default(Exception); - if (HostContext.RunnerShutdownToken.IsCancellationRequested) - { - step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown."); - } - else - { - try + // Evaluate condition. + step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'"); + var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext); + var conditionResult = false; + var conditionEvaluateError = default(Exception); + if (HostContext.RunnerShutdownToken.IsCancellationRequested) { - var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter); - var condition = new BasicExpressionToken(null, null, null, step.Condition); - conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); + step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown."); } - catch (Exception ex) + else { - Trace.Info("Caught exception from expression."); - Trace.Error(ex); - conditionEvaluateError = ex; + try + { + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter); + var condition = new BasicExpressionToken(null, null, null, step.Condition); + conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); + } + catch (Exception ex) + { + Trace.Info("Caught exception from expression."); + Trace.Error(ex); + conditionEvaluateError = ex; + } } - } - // no evaluate error but condition is false - if (!conditionResult && conditionEvaluateError == null) - { - // Condition == false - Trace.Info("Skipping step due to condition evaluation."); - CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace); - } - else if (conditionEvaluateError != null) - { - // fail the step since there is an evaluate error. - step.ExecutionContext.Error(conditionEvaluateError); - CompleteStep(step, nextStep, TaskResult.Failed); - } - else - { - // Run the step. - await RunStepAsync(step, jobContext.CancellationToken); - CompleteStep(step, nextStep); + // no evaluate error but condition is false + if (!conditionResult && conditionEvaluateError == null) + { + // Condition == false + Trace.Info("Skipping step due to condition evaluation."); + CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace); + } + else if (conditionEvaluateError != null) + { + // fail the step since there is an evaluate error. + step.ExecutionContext.Error(conditionEvaluateError); + CompleteStep(step, nextStep, TaskResult.Failed); + } + else + { + // Run the step. + await RunStepAsync(step, jobContext.CancellationToken); + CompleteStep(step, nextStep); + } } - } - finally - { - if (jobCancelRegister != null) + finally { - jobCancelRegister?.Dispose(); - jobCancelRegister = null; + if (jobCancelRegister != null) + { + jobCancelRegister?.Dispose(); + jobCancelRegister = null; + } } } } diff --git a/src/runnerversion b/src/runnerversion index 93c284c977a..127f551ee12 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.262.0 +2.262.1 From 11435857e4cd9795fa5ee68c20b674e5f96052ab Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Thu, 21 May 2020 15:49:10 -0400 Subject: [PATCH 60/86] prepare 2.263.0 runner release. --- releaseNote.md | 22 ++++++++-------------- src/runnerversion | 2 +- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 43a9063ffbe..b75e056e3ba 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,20 +1,14 @@ ## Features - - Sample scripts to automate scalable runners (#427) - - Raise warning when action input does not match action.yml. (#429) - - Add secret masker for trimming double quotes. (#440) - - Use the API_URL and munge action URLs for GHES (#437 #469) - - Help trace worker crash in Kusto. (#450) - - update checkout@v1 for GHES (#470) + - N/A ## Bugs - - Print node version in debug instead of output. (#433) - - Better error when runner removed from service. (#441) - - Add help info for '--labels' config option (#472) - - Sps/token migration fix, job.status/steps.outcome/steps.conclusion case match with GitHub check suites conclusion. (#462) - - Docker build using -f instead of implied default (#471) + - Handle `jq` returns "null" if the field does not exist in create-latest-svc.sh (#478) + - Switch GITHUB_URL to GITHUB_SERVER_URL (#482) + - Fix problem matcher for GHES (#488) + - Fix container action inputs validation warning (#490) + - Fix post step display name (#490) + - Fix worker crash due to exception from evaluating step.env (#490) ## Misc - - Make release notes code blocks copy-paste-able (#430) - - Fix spelling of RHEL and CentOS. (#436) - - Add CodeQL Analysis workflow (#459) + - N/A ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/runnerversion b/src/runnerversion index 127f551ee12..f9d36d71d1d 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.262.1 +2.263.0 From 416a7ac4b89c1bc5312e9ba93608f7a02eb72e5b Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 2 Jun 2020 17:21:50 -0400 Subject: [PATCH 61/86] prepare to switch to service resolves archive download info (#508) --- src/Runner.Worker/ActionManager.cs | 305 +++- src/Test/L0/Worker/ActionManagerL0.cs | 1953 ++++++++++++++++++++++--- 2 files changed, 2088 insertions(+), 170 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index e60bb50ee88..3004e680290 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -73,6 +73,11 @@ public async Task PrepareActionsAsync(IExecutionContext execution // Clear the cache (for self-hosted runners) IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken); + // todo: Remove when feature flag DistributedTask.NewActionMetadata is removed + var newActionMetadata = executionContext.Variables.GetBoolean("DistributedTask.NewActionMetadata") ?? false; + + var repositoryActions = new List(); + foreach (var action in actions) { if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry) @@ -90,7 +95,8 @@ public async Task PrepareActionsAsync(IExecutionContext execution Trace.Info($"Action {action.Name} ({action.Id}) needs to pull image '{containerReference.Image}'"); imagesToPull[containerReference.Image].Add(action.Id); } - else if (action.Reference.Type == Pipelines.ActionSourceType.Repository) + // todo: Remove when feature flag DistributedTask.NewActionMetadata is removed + else if (action.Reference.Type == Pipelines.ActionSourceType.Repository && !newActionMetadata) { // only download the repository archive await DownloadRepositoryActionAsync(executionContext, action); @@ -124,6 +130,81 @@ public async Task PrepareActionsAsync(IExecutionContext execution } } + var repoAction = action.Reference as Pipelines.RepositoryPathReference; + if (repoAction.RepositoryType != Pipelines.PipelineConstants.SelfAlias) + { + var definition = LoadAction(executionContext, action); + if (definition.Data.Execution.HasPre) + { + var actionRunner = HostContext.CreateService(); + actionRunner.Action = action; + actionRunner.Stage = ActionRunStage.Pre; + actionRunner.Condition = definition.Data.Execution.InitCondition; + + Trace.Info($"Add 'pre' execution for {action.Id}"); + preStepTracker[action.Id] = actionRunner; + } + } + } + else if (action.Reference.Type == Pipelines.ActionSourceType.Repository && newActionMetadata) + { + repositoryActions.Add(action); + } + } + + if (repositoryActions.Count > 0) + { + // Get the download info + var downloadInfos = await GetDownloadInfoAsync(executionContext, repositoryActions); + + // Download each action + foreach (var action in repositoryActions) + { + var lookupKey = GetDownloadInfoLookupKey(action); + if (string.IsNullOrEmpty(lookupKey)) + { + continue; + } + + if (!downloadInfos.TryGetValue(lookupKey, out var downloadInfo)) + { + throw new Exception($"Missing download info for {lookupKey}"); + } + + await DownloadRepositoryActionAsync(executionContext, downloadInfo); + } + + // More preparation based on content in the repository (action.yml) + foreach (var action in repositoryActions) + { + var setupInfo = PrepareRepositoryActionAsync(executionContext, action); + if (setupInfo != null) + { + if (!string.IsNullOrEmpty(setupInfo.Image)) + { + if (!imagesToPull.ContainsKey(setupInfo.Image)) + { + imagesToPull[setupInfo.Image] = new List(); + } + + Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to pull image '{setupInfo.Image}'"); + imagesToPull[setupInfo.Image].Add(action.Id); + } + else + { + ArgUtil.NotNullOrEmpty(setupInfo.ActionRepository, nameof(setupInfo.ActionRepository)); + + if (!imagesToBuild.ContainsKey(setupInfo.ActionRepository)) + { + imagesToBuild[setupInfo.ActionRepository] = new List(); + } + + Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to build image '{setupInfo.Dockerfile}'"); + imagesToBuild[setupInfo.ActionRepository].Add(action.Id); + imagesToBuildInfo[setupInfo.ActionRepository] = setupInfo; + } + } + var repoAction = action.Reference as Pipelines.RepositoryPathReference; if (repoAction.RepositoryType != Pipelines.PipelineConstants.SelfAlias) { @@ -464,6 +545,127 @@ private async Task BuildActionContainerAsync(IExecutionContext executionContext, } } + // This implementation is temporary and will be removed when we switch to a REST API call to the service to resolve the download info + private async Task RepoExistsAsync(IExecutionContext executionContext, Pipelines.RepositoryPathReference repositoryReference, string authorization) + { + var apiUrl = GetApiUrl(executionContext); + var repoUrl = $"{apiUrl}/repos/{repositoryReference.Name}"; + for (var attempt = 1; attempt <= 3; attempt++) + { + executionContext.Debug($"Checking whether repo exists: {repoUrl}"); + try + { + using (var httpClientHandler = HostContext.CreateHttpClientHandler()) + using (var httpClient = new HttpClient(httpClientHandler)) + { + httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(authorization); + httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); + using (var response = await httpClient.GetAsync(repoUrl)) + { + if (response.IsSuccessStatusCode) + { + return true; + } + else if (response.StatusCode == HttpStatusCode.NotFound) + { + return false; + } + else + { + // Throw + response.EnsureSuccessStatusCode(); + } + } + } + } + catch (Exception ex) + { + if (attempt < 3) + { + executionContext.Debug($"Failed checking whether repo '{repositoryReference.Name}' exists: {ex.Message}"); + } + else + { + executionContext.Error($"Failed checking whether repo '{repositoryReference.Name}' exists: {ex.Message}"); + throw; + } + } + } + + return false; // Never reaches here + } + + // This implementation is temporary and will be replaced with a REST API call to the service to resolve + private async Task> GetDownloadInfoAsync(IExecutionContext executionContext, List actions) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + var configurationStore = HostContext.GetService(); + var runnerSettings = configurationStore.GetSettings(); + var apiUrl = GetApiUrl(executionContext); + var accessToken = executionContext.GetGitHubContext("token"); + var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); + var authorization = $"Basic {base64EncodingToken}"; + + foreach (var action in actions) + { + var lookupKey = GetDownloadInfoLookupKey(action); + if (string.IsNullOrEmpty(lookupKey) || result.ContainsKey(lookupKey)) + { + continue; + } + + var repositoryReference = action.Reference as Pipelines.RepositoryPathReference; + ArgUtil.NotNull(repositoryReference, nameof(repositoryReference)); + + var downloadInfo = default(ActionDownloadInfo); + + if (runnerSettings.IsHostedServer) + { + downloadInfo = new ActionDownloadInfo + { + NameWithOwner = repositoryReference.Name, + Ref = repositoryReference.Ref, + ArchiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref), + Authorization = authorization, + }; + } + // Test whether the repo exists in the instance + else if (await RepoExistsAsync(executionContext, repositoryReference, authorization)) + { + downloadInfo = new ActionDownloadInfo + { + NameWithOwner = repositoryReference.Name, + Ref = repositoryReference.Ref, + ArchiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref), + Authorization = authorization, + }; + } + // Fallback to dotcom + else + { + downloadInfo = new ActionDownloadInfo + { + NameWithOwner = repositoryReference.Name, + Ref = repositoryReference.Ref, + ArchiveLink = BuildLinkToActionArchive(_dotcomApiUrl, repositoryReference.Name, repositoryReference.Ref), + Authorization = null, + }; + } + + result.Add(lookupKey, downloadInfo); + } + + // Register secrets + foreach (var downloadInfo in result.Values) + { + HostContext.SecretMasker.AddValue(downloadInfo.Authorization); + } + + return result; + } + + // todo: Remove when feature flag DistributedTask.NewActionMetadata is removed private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction) { Trace.Entering(); @@ -509,7 +711,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref); Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); var downloadDetails = new ActionDownloadDetails(archiveLink, ConfigureAuthorizationFromContext); - await DownloadRepositoryActionAsync(executionContext, downloadDetails, destDirectory); + await DownloadRepositoryActionAsync(executionContext, downloadDetails, null, destDirectory); return; } else @@ -536,7 +738,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont Trace.Info($"Download archive '{downloadAttempt.ArchiveLink}' to '{destDirectory}'."); try { - await DownloadRepositoryActionAsync(executionContext, downloadAttempt, destDirectory); + await DownloadRepositoryActionAsync(executionContext, downloadAttempt, null, destDirectory); return; } catch (ActionNotFoundException) @@ -549,6 +751,33 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } } + private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadInfo downloadInfo) + { + Trace.Entering(); + ArgUtil.NotNull(executionContext, nameof(executionContext)); + ArgUtil.NotNull(downloadInfo, nameof(downloadInfo)); + ArgUtil.NotNullOrEmpty(downloadInfo.NameWithOwner, nameof(downloadInfo.NameWithOwner)); + ArgUtil.NotNullOrEmpty(downloadInfo.Ref, nameof(downloadInfo.Ref)); + + string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), downloadInfo.NameWithOwner.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), downloadInfo.Ref); + string watermarkFile = GetWatermarkFilePath(destDirectory); + if (File.Exists(watermarkFile)) + { + executionContext.Debug($"Action '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' already downloaded at '{destDirectory}'."); + return; + } + else + { + // make sure we get a clean folder ready to use. + IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken); + Directory.CreateDirectory(destDirectory); + executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}'"); + } + + Trace.Info($"Download archive '{downloadInfo.ArchiveLink}' to '{destDirectory}'."); + await DownloadRepositoryActionAsync(executionContext, null, downloadInfo, destDirectory); + } + private string GetApiUrl(IExecutionContext executionContext) { string apiUrl = executionContext.GetGitHubContext("api_url"); @@ -569,7 +798,8 @@ private static string BuildLinkToActionArchive(string apiUrl, string repository, #endif } - private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, string destDirectory) + // todo: Remove the parameter "actionDownloadDetails" when feature flag DistributedTask.NewActionMetadata is removed + private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, ActionDownloadInfo downloadInfo, string destDirectory) { //download and extract action in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); @@ -581,7 +811,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz"); #endif - string link = actionDownloadDetails.ArchiveLink; + string link = downloadInfo != null ? downloadInfo.ArchiveLink : actionDownloadDetails.ArchiveLink; Trace.Info($"Save archive '{link}' into {archiveFile}."); try { @@ -601,7 +831,16 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { - actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient); + // Legacy + if (downloadInfo == null) + { + actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient); + } + // FF DistributedTask.NewActionMetadata + else + { + httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(downloadInfo.Authorization); + } httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); using (var response = await httpClient.GetAsync(link)) @@ -741,6 +980,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } } + // todo: Remove when feature flag DistributedTask.NewActionMetadata is removed private void ConfigureAuthorizationFromContext(IExecutionContext executionContext, HttpClient httpClient) { var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); @@ -872,6 +1112,48 @@ private ActionContainer PrepareRepositoryActionAsync(IExecutionContext execution } } + private static string GetDownloadInfoLookupKey(Pipelines.ActionStep action) + { + if (action.Reference.Type != Pipelines.ActionSourceType.Repository) + { + return null; + } + + var repositoryReference = action.Reference as Pipelines.RepositoryPathReference; + ArgUtil.NotNull(repositoryReference, nameof(repositoryReference)); + + if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase)) + { + throw new NotSupportedException(repositoryReference.RepositoryType); + } + + ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name)); + ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref)); + return $"{repositoryReference.Name}@{repositoryReference.Ref}"; + } + + private static AuthenticationHeaderValue CreateAuthHeader(string authorization) + { + if (string.IsNullOrEmpty(authorization)) + { + return null; + } + + var split = authorization.Split(new char[] { ' ' }, 2); + if (split.Length != 2 || string.IsNullOrWhiteSpace(split[0]) || string.IsNullOrWhiteSpace(split[1])) + { + throw new Exception("Unexpected authorization header format"); + } + + return new AuthenticationHeaderValue(split[0].Trim(), split[1].Trim()); + } + + // todo: Remove when feature flag DistributedTask.NewActionMetadata is removed private class ActionDownloadDetails { public string ArchiveLink { get; } @@ -884,6 +1166,17 @@ public ActionDownloadDetails(string archiveLink, Action { @@ -74,12 +74,12 @@ public async void PrepareActions_PullImageFromDockerHub() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_DownloadActionFromGraph() + public async void PrepareActions_DownloadActionFromGraph_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -116,12 +116,12 @@ public async void PrepareActions_DownloadActionFromGraph() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_DownloadBuiltInActionFromGraph_OnPremises() + public async void PrepareActions_DownloadBuiltInActionFromGraph_OnPremises_Legacy() { try { // Arrange - Setup(); + Setup(newActionMetadata: false); const string ActionName = "actions/sample-action"; var actions = new List { @@ -174,12 +174,12 @@ public async void PrepareActions_DownloadBuiltInActionFromGraph_OnPremises() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_DownloadActionFromDotCom_OnPremises() + public async void PrepareActions_DownloadActionFromDotCom_OnPremises_Legacy() { try { // Arrange - Setup(); + Setup(newActionMetadata: false); const string ActionName = "ownerName/sample-action"; var actions = new List { @@ -235,12 +235,12 @@ public async void PrepareActions_DownloadActionFromDotCom_OnPremises() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_DownloadUnknownActionFromGraph_OnPremises() + public async void PrepareActions_DownloadUnknownActionFromGraph_OnPremises_Legacy() { try { // Arrange - Setup(); + Setup(newActionMetadata: false); const string ActionName = "ownerName/sample-action"; var actions = new List { @@ -294,12 +294,12 @@ public async void PrepareActions_DownloadUnknownActionFromGraph_OnPremises() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_AlwaysClearActionsCache() + public async void PrepareActions_AlwaysClearActionsCache_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List(); @@ -324,12 +324,12 @@ public async void PrepareActions_AlwaysClearActionsCache() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_SkipDownloadActionForSelfRepo() + public async void PrepareActions_SkipDownloadActionForSelfRepo_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -360,12 +360,12 @@ public async void PrepareActions_SkipDownloadActionForSelfRepo() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithDockerfile() + public async void PrepareActions_RepositoryActionWithDockerfile_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -399,12 +399,12 @@ public async void PrepareActions_RepositoryActionWithDockerfile() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithDockerfileInRelativePath() + public async void PrepareActions_RepositoryActionWithDockerfileInRelativePath_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -440,12 +440,12 @@ public async void PrepareActions_RepositoryActionWithDockerfileInRelativePath() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithActionfile_Dockerfile() + public async void PrepareActions_RepositoryActionWithActionfile_Dockerfile_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -479,12 +479,12 @@ public async void PrepareActions_RepositoryActionWithActionfile_Dockerfile() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithActionfile_DockerfileRelativePath() + public async void PrepareActions_RepositoryActionWithActionfile_DockerfileRelativePath_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -519,12 +519,12 @@ public async void PrepareActions_RepositoryActionWithActionfile_DockerfileRelati [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithActionfile_DockerHubImage() + public async void PrepareActions_RepositoryActionWithActionfile_DockerHubImage_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -558,12 +558,12 @@ public async void PrepareActions_RepositoryActionWithActionfile_DockerHubImage() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithActionYamlFile_DockerHubImage() + public async void PrepareActions_RepositoryActionWithActionYamlFile_DockerHubImage_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -597,12 +597,12 @@ public async void PrepareActions_RepositoryActionWithActionYamlFile_DockerHubIma [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithActionfileAndDockerfile() + public async void PrepareActions_RepositoryActionWithActionfileAndDockerfile_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -637,12 +637,12 @@ public async void PrepareActions_RepositoryActionWithActionfileAndDockerfile() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_NotPullOrBuildImagesMultipleTimes() + public async void PrepareActions_NotPullOrBuildImagesMultipleTimes_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId1 = Guid.NewGuid(); var actionId2 = Guid.NewGuid(); var actionId3 = Guid.NewGuid(); @@ -777,12 +777,12 @@ public async void PrepareActions_NotPullOrBuildImagesMultipleTimes() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithActionfile_Node() + public async void PrepareActions_RepositoryActionWithActionfile_Node_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -814,12 +814,12 @@ public async void PrepareActions_RepositoryActionWithActionfile_Node() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithInvalidWrapperActionfile_Node() + public async void PrepareActions_RepositoryActionWithInvalidWrapperActionfile_Node_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); var actionId = Guid.NewGuid(); var actions = new List { @@ -857,12 +857,12 @@ public async void PrepareActions_RepositoryActionWithInvalidWrapperActionfile_No [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public async void PrepareActions_RepositoryActionWithWrapperActionfile_PreSteps() + public async void PrepareActions_RepositoryActionWithWrapperActionfile_PreSteps_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); _hc.EnqueueInstance(new Mock().Object); _hc.EnqueueInstance(new Mock().Object); @@ -912,12 +912,12 @@ public async void PrepareActions_RepositoryActionWithWrapperActionfile_PreSteps( [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsContainerRegistryActionDefinition() + public void LoadsContainerRegistryActionDefinition_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); Pipelines.ActionStep instance = new Pipelines.ActionStep() { @@ -949,12 +949,12 @@ public void LoadsContainerRegistryActionDefinition() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsScriptActionDefinition() + public void LoadsScriptActionDefinition_Legacy() { try { //Arrange - Setup(); + Setup(newActionMetadata: false); Pipelines.ActionStep instance = new Pipelines.ActionStep() { @@ -979,12 +979,12 @@ public void LoadsScriptActionDefinition() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsContainerActionDefinitionDockerfile() + public void LoadsContainerActionDefinitionDockerfile_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); // Prepare the task.json content. const string Content = @" # Container action @@ -1079,12 +1079,12 @@ public void LoadsContainerActionDefinitionDockerfile() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsContainerActionDefinitionRegistry() + public void LoadsContainerActionDefinitionRegistry_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); // Prepare the task.json content. const string Content = @" # Container action @@ -1179,12 +1179,12 @@ public void LoadsContainerActionDefinitionRegistry() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsNodeActionDefinition() + public void LoadsNodeActionDefinition_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); const string Content = @" # Container action name: 'Hello World' @@ -1247,12 +1247,12 @@ public void LoadsNodeActionDefinition() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsNodeActionDefinitionYaml() + public void LoadsNodeActionDefinitionYaml_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); const string Content = @" # Container action name: 'Hello World' @@ -1328,12 +1328,12 @@ public void LoadsNodeActionDefinitionYaml() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsContainerActionDefinitionDockerfile_SelfRepo() + public void LoadsContainerActionDefinitionDockerfile_SelfRepo_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); // Prepare the task.json content. const string Content = @" # Container action @@ -1427,12 +1427,12 @@ public void LoadsContainerActionDefinitionDockerfile_SelfRepo() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsContainerActionDefinitionRegistry_SelfRepo() + public void LoadsContainerActionDefinitionRegistry_SelfRepo_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); // Prepare the task.json content. const string Content = @" # Container action @@ -1526,12 +1526,12 @@ public void LoadsContainerActionDefinitionRegistry_SelfRepo() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsNodeActionDefinition_SelfRepo() + public void LoadsNodeActionDefinition_SelfRepo_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); const string Content = @" # Container action name: 'Hello World' @@ -1594,12 +1594,12 @@ public void LoadsNodeActionDefinition_SelfRepo() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsNodeActionDefinition_Cleanup() + public void LoadsNodeActionDefinition_Cleanup_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); const string Content = @" # Container action name: 'Hello World' @@ -1664,12 +1664,12 @@ public void LoadsNodeActionDefinition_Cleanup() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsContainerActionDefinitionDockerfile_Cleanup() + public void LoadsContainerActionDefinitionDockerfile_Cleanup_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); // Prepare the task.json content. const string Content = @" # Container action @@ -1766,12 +1766,12 @@ public void LoadsContainerActionDefinitionDockerfile_Cleanup() [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] - public void LoadsPluginActionDefinition() + public void LoadsPluginActionDefinition_Legacy() { try { // Arrange. - Setup(); + Setup(newActionMetadata: false); const string Content = @" name: 'Hello World' description: 'Greet the world and record the time' @@ -1830,132 +1830,1757 @@ public void LoadsPluginActionDefinition() } } - private void CreateAction(string yamlContent, out Pipelines.ActionStep instance, out string directory) +#if OS_LINUX + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_PullImageFromDockerHub() { - directory = Path.Combine(_workFolder, Constants.Path.ActionsDirectory, "GitHub/actions".Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), "master"); - string file = Path.Combine(directory, Constants.Path.ActionManifestYmlFile); - Directory.CreateDirectory(Path.GetDirectoryName(file)); - File.WriteAllText(file, yamlContent); - instance = new Pipelines.ActionStep() + try { - Id = Guid.NewGuid(), - Reference = new Pipelines.RepositoryPathReference() + //Arrange + Setup(); + // _ec.Variables. + var actionId = Guid.NewGuid(); + var actions = new List { - Name = "GitHub/actions", - Ref = "master", - RepositoryType = Pipelines.RepositoryTypes.GitHub - } - }; - } + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.ContainerRegistryReference() + { + Image = "ubuntu:16.04" + } + } + }; - private void CreateSelfRepoAction(string yamlContent, out Pipelines.ActionStep instance, out string directory) - { - directory = Path.Combine(_workFolder, "actions", "actions"); - string file = Path.Combine(directory, Constants.Path.ActionManifestYmlFile); - Directory.CreateDirectory(Path.GetDirectoryName(file)); - File.WriteAllText(file, yamlContent); - instance = new Pipelines.ActionStep() + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + //Assert + Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal("ubuntu:16.04", (steps[0].Data as ContainerSetupInfo).Container.Image); + } + finally { - Id = Guid.NewGuid(), - Reference = new Pipelines.RepositoryPathReference() - { - Name = "GitHub/actions", - Ref = "master", - RepositoryType = Pipelines.PipelineConstants.SelfAlias - } - }; + Teardown(); + } } - - /// - /// Creates a sample action in an archive on disk, similar to the archive - /// retrieved from GitHub's or GHES' repository API. - /// - /// The path on disk to the archive. -#if OS_WINDOWS - private Task CreateRepoArchive() -#else - private async Task CreateRepoArchive() #endif - { - const string Content = @" -# Container action -name: 'Hello World' -description: 'Greet the world' -author: 'GitHub' -icon: 'hello.svg' # vector art to display in the GitHub Marketplace -color: 'green' # optional, decorates the entry in the GitHub Marketplace -runs: - using: 'node12' - main: 'task.js' -"; - CreateAction(yamlContent: Content, instance: out _, directory: out string directory); - - var tempDir = _hc.GetDirectory(WellKnownDirectory.Temp); - Directory.CreateDirectory(tempDir); - var archiveFile = Path.Combine(tempDir, Path.GetRandomFileName()); - var trace = _hc.GetTrace(); -#if OS_WINDOWS - ZipFile.CreateFromDirectory(directory, archiveFile, CompressionLevel.Fastest, includeBaseDirectory: true); - return Task.FromResult(archiveFile); -#else - string tar = WhichUtil.Which("tar", require: true, trace: trace); - - // tar -xzf - using (var processInvoker = new ProcessInvokerWrapper()) + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_DownloadActionFromGraph() + { + try { - processInvoker.Initialize(_hc); - processInvoker.OutputDataReceived += new EventHandler((sender, args) => + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List { - if (!string.IsNullOrEmpty(args.Data)) + new Pipelines.ActionStep() { - trace.Info(args.Data); + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "actions/download-artifact", + Ref = "master", + RepositoryType = "GitHub" + } } - }); + }; - processInvoker.ErrorDataReceived += new EventHandler((sender, args) => - { - if (!string.IsNullOrEmpty(args.Data)) - { - trace.Error(args.Data); - } - }); + //Act + await _actionManager.PrepareActionsAsync(_ec.Object, actions); - string cwd = Path.GetDirectoryName(directory); - string inputDirectory = Path.GetFileName(directory); - int exitCode = await processInvoker.ExecuteAsync(_hc.GetDirectory(WellKnownDirectory.Bin), tar, $"-czf \"{archiveFile}\" -C \"{cwd}\" \"{inputDirectory}\"", null, CancellationToken.None); - if (exitCode != 0) - { - throw new NotSupportedException($"Can't use 'tar -czf' to create archive file: {archiveFile}. return code: {exitCode}."); - } + //Assert + var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/download-artifact", "master.completed"); + Assert.True(File.Exists(watermarkFile)); + + var actionYamlFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "actions/download-artifact", "master", "action.yml"); + Assert.True(File.Exists(actionYamlFile)); + _hc.GetTrace().Info(File.ReadAllText(actionYamlFile)); + } + finally + { + Teardown(); } - return archiveFile; -#endif } - private static string GetLinkToActionArchive(string apiUrl, string repository, string @ref) + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_AlwaysClearActionsCache() { -#if OS_WINDOWS - return $"{apiUrl}/repos/{repository}/zipball/{@ref}"; -#else - return $"{apiUrl}/repos/{repository}/tarball/{@ref}"; -#endif - } + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List(); - private void Setup([CallerMemberName] string name = "") - { - _ecTokenSource?.Dispose(); - _ecTokenSource = new CancellationTokenSource(); + var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "notexist/no", "notexist.completed"); + Directory.CreateDirectory(Path.GetDirectoryName(watermarkFile)); + File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString()); + Directory.CreateDirectory(Path.Combine(Path.GetDirectoryName(watermarkFile), "notexist")); + File.Copy(Path.Combine(TestUtil.GetSrcPath(), "Test", TestDataFolderName, "dockerfileaction.yml"), Path.Combine(Path.GetDirectoryName(watermarkFile), "notexist", "action.yml")); - // Test host context. - _hc = new TestHostContext(this, name); + //Act + await _actionManager.PrepareActionsAsync(_ec.Object, actions); - // Random work folder. - _workFolder = _hc.GetDirectory(WellKnownDirectory.Work); + // Make sure _actions folder get deleted + Assert.False(Directory.Exists(_hc.GetDirectory(WellKnownDirectory.Actions))); + } + finally + { + Teardown(); + } + } - _ec = new Mock(); - _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); - _ec.Setup(x => x.Variables).Returns(new Variables(_hc, new Dictionary())); + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_SkipDownloadActionForSelfRepo() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Path = "action", + RepositoryType = Pipelines.PipelineConstants.SelfAlias + } + } + }; + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + Assert.True(steps.Count == 0); + } + finally + { + Teardown(); + } + } + +#if OS_LINUX + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithDockerfile() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "repositoryactionwithdockerfile", + RepositoryType = "GitHub" + } + } + }; + + var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithdockerfile"); + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); + Assert.Equal(Path.Combine(actionDir, "Dockerfile"), (steps[0].Data as ContainerSetupInfo).Container.Dockerfile); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithDockerfileInRelativePath() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "repositoryactionwithdockerfileinrelativepath", + Path = "images/cli", + RepositoryType = "GitHub" + } + } + }; + + var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithdockerfileinrelativepath"); + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); + Assert.Equal(Path.Combine(actionDir, "images/cli", "Dockerfile"), (steps[0].Data as ContainerSetupInfo).Container.Dockerfile); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithActionfile_Dockerfile() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "repositoryactionwithdockerfileinrelativepath", + RepositoryType = "GitHub" + } + } + }; + + var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithdockerfileinrelativepath"); + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); + Assert.Equal(Path.Combine(actionDir, "Dockerfile"), (steps[0].Data as ContainerSetupInfo).Container.Dockerfile); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithActionfile_DockerfileRelativePath() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithActionfile_DockerfileRelativePath", + RepositoryType = "GitHub" + } + } + }; + + var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "RepositoryActionWithActionfile_DockerfileRelativePath"); + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); + Assert.Equal(Path.Combine(actionDir, "images/Dockerfile"), (steps[0].Data as ContainerSetupInfo).Container.Dockerfile); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithActionfile_DockerHubImage() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithActionfile_DockerHubImage", + RepositoryType = "GitHub" + } + } + }; + + var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "RepositoryActionWithActionfile_DockerHubImage"); + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal("ubuntu:18.04", (steps[0].Data as ContainerSetupInfo).Container.Image); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithActionYamlFile_DockerHubImage() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithActionYamlFile_DockerHubImage", + RepositoryType = "GitHub" + } + } + }; + + var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "RepositoryActionWithActionYamlFile_DockerHubImage"); + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + Assert.Equal((steps[0].Data as ContainerSetupInfo).StepIds[0], actionId); + Assert.Equal("ubuntu:18.04", (steps[0].Data as ContainerSetupInfo).Container.Image); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithActionfileAndDockerfile() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "repositoryactionwithactionfileanddockerfile", + RepositoryType = "GitHub" + } + } + }; + + var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithactionfileanddockerfile"); + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + Assert.Equal(actionId, (steps[0].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal(actionDir, (steps[0].Data as ContainerSetupInfo).Container.WorkingDirectory); + Assert.Equal(Path.Combine(actionDir, "Dockerfile"), (steps[0].Data as ContainerSetupInfo).Container.Dockerfile); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_NotPullOrBuildImagesMultipleTimes() + { + try + { + //Arrange + Setup(); + var actionId1 = Guid.NewGuid(); + var actionId2 = Guid.NewGuid(); + var actionId3 = Guid.NewGuid(); + var actionId4 = Guid.NewGuid(); + var actionId5 = Guid.NewGuid(); + var actionId6 = Guid.NewGuid(); + var actionId7 = Guid.NewGuid(); + var actionId8 = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId1, + Reference = new Pipelines.ContainerRegistryReference() + { + Image = "ubuntu:16.04" + } + }, + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId2, + Reference = new Pipelines.ContainerRegistryReference() + { + Image = "ubuntu:18.04" + } + }, + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId3, + Reference = new Pipelines.ContainerRegistryReference() + { + Image = "ubuntu:18.04" + } + }, + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId4, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "notpullorbuildimagesmultipletimes1", + RepositoryType = "GitHub" + } + }, + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId5, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "repositoryactionwithdockerfile", + RepositoryType = "GitHub" + } + }, + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId6, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "repositoryactionwithdockerfileinrelativepath", + RepositoryType = "GitHub" + } + }, + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId7, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "repositoryactionwithdockerfileinrelativepath", + RepositoryType = "GitHub" + } + }, + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId8, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "repositoryactionwithdockerfileinrelativepath", + Path = "images/cli", + RepositoryType = "GitHub" + } + } + }; + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + //Assert + Assert.Equal(actionId1, (steps[0].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal("ubuntu:16.04", (steps[0].Data as ContainerSetupInfo).Container.Image); + + Assert.Contains(actionId2, (steps[1].Data as ContainerSetupInfo).StepIds); + Assert.Contains(actionId3, (steps[1].Data as ContainerSetupInfo).StepIds); + Assert.Contains(actionId4, (steps[1].Data as ContainerSetupInfo).StepIds); + Assert.Equal("ubuntu:18.04", (steps[1].Data as ContainerSetupInfo).Container.Image); + + var actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithdockerfile"); + + Assert.Equal(actionId5, (steps[2].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal(actionDir, (steps[2].Data as ContainerSetupInfo).Container.WorkingDirectory); + Assert.Equal(Path.Combine(actionDir, "Dockerfile"), (steps[2].Data as ContainerSetupInfo).Container.Dockerfile); + + actionDir = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang", "runner_L0", "repositoryactionwithdockerfileinrelativepath"); + + Assert.Contains(actionId6, (steps[3].Data as ContainerSetupInfo).StepIds); + Assert.Contains(actionId7, (steps[3].Data as ContainerSetupInfo).StepIds); + Assert.Equal(actionDir, (steps[3].Data as ContainerSetupInfo).Container.WorkingDirectory); + Assert.Equal(Path.Combine(actionDir, "Dockerfile"), (steps[3].Data as ContainerSetupInfo).Container.Dockerfile); + + Assert.Equal(actionId8, (steps[4].Data as ContainerSetupInfo).StepIds[0]); + Assert.Equal(actionDir, (steps[4].Data as ContainerSetupInfo).Container.WorkingDirectory); + Assert.Equal(Path.Combine(actionDir, "images/cli", "Dockerfile"), (steps[4].Data as ContainerSetupInfo).Container.Dockerfile); + } + finally + { + Teardown(); + } + } +#endif + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithActionfile_Node() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "actions/setup-node", + Ref = "v1", + RepositoryType = "GitHub" + } + } + }; + + //Act + var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; + + // node.js based action doesn't need any extra steps to build/pull containers. + Assert.True(steps.Count == 0); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithInvalidWrapperActionfile_Node() + { + try + { + //Arrange + Setup(); + var actionId = Guid.NewGuid(); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithInvalidWrapperActionfile_Node", + RepositoryType = "GitHub" + } + } + }; + + //Act + try + { + await _actionManager.PrepareActionsAsync(_ec.Object, actions); + } + catch (ArgumentException) + { + var traceFile = Path.GetTempFileName(); + File.Copy(_hc.TraceFileName, traceFile, true); + Assert.Contains("Entry javascript file is not provided.", File.ReadAllText(traceFile)); + } + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void PrepareActions_RepositoryActionWithWrapperActionfile_PreSteps() + { + try + { + //Arrange + Setup(); + + _hc.EnqueueInstance(new Mock().Object); + _hc.EnqueueInstance(new Mock().Object); + + var actionId1 = Guid.NewGuid(); + var actionId2 = Guid.NewGuid(); + _hc.GetTrace().Info(actionId1); + _hc.GetTrace().Info(actionId2); + var actions = new List + { + new Pipelines.ActionStep() + { + Name = "action1", + Id = actionId1, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithWrapperActionfile_Node", + RepositoryType = "GitHub" + } + }, + new Pipelines.ActionStep() + { + Name = "action2", + Id = actionId2, + Reference = new Pipelines.RepositoryPathReference() + { + Name = "TingluoHuang/runner_L0", + Ref = "RepositoryActionWithWrapperActionfile_Docker", + RepositoryType = "GitHub" + } + } + }; + + //Act + var preResult = await _actionManager.PrepareActionsAsync(_ec.Object, actions); + Assert.Equal(2, preResult.PreStepTracker.Count); + Assert.NotNull(preResult.PreStepTracker[actionId1]); + Assert.NotNull(preResult.PreStepTracker[actionId2]); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsContainerRegistryActionDefinition() + { + try + { + //Arrange + Setup(); + + Pipelines.ActionStep instance = new Pipelines.ActionStep() + { + Id = Guid.NewGuid(), + Reference = new Pipelines.ContainerRegistryReference() + { + Image = "ubuntu:16.04" + } + }; + + _actionManager.CachedActionContainers[instance.Id] = new ContainerInfo() { ContainerImage = "ubuntu:16.04" }; + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.NotNull(definition.Data); + Assert.Equal("ubuntu:16.04", (definition.Data.Execution as ContainerActionExecutionData).Image); + Assert.True(string.IsNullOrEmpty((definition.Data.Execution as ContainerActionExecutionData).EntryPoint)); + Assert.Null((definition.Data.Execution as ContainerActionExecutionData).Arguments); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsScriptActionDefinition() + { + try + { + //Arrange + Setup(); + + Pipelines.ActionStep instance = new Pipelines.ActionStep() + { + Id = Guid.NewGuid(), + Reference = new Pipelines.ScriptReference() + }; + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.NotNull(definition.Data); + Assert.True(definition.Data.Execution.ExecutionType == ActionExecutionType.Script); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsContainerActionDefinitionDockerfile() + { + try + { + // Arrange. + Setup(); + // Prepare the task.json content. + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'docker' + image: 'Dockerfile' + args: + - '${{ inputs.greeting }}' + entrypoint: 'main.sh' + env: + Token: foo + Url: bar +"; + Pipelines.ActionStep instance; + string directory; + CreateAction(yamlContent: Content, instance: out instance, directory: out directory); + _actionManager.CachedActionContainers[instance.Id] = new ContainerInfo() { ContainerImage = "image:1234" }; + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as ContainerActionExecutionData)); // execution.Node + Assert.Equal("image:1234", (definition.Data.Execution as ContainerActionExecutionData).Image); + Assert.Equal("main.sh", (definition.Data.Execution as ContainerActionExecutionData).EntryPoint); + + foreach (var arg in (definition.Data.Execution as ContainerActionExecutionData).Arguments) + { + Assert.Equal("${{ inputs.greeting }}", arg.AssertScalar("arg").ToString()); + } + + foreach (var env in (definition.Data.Execution as ContainerActionExecutionData).Environment) + { + var key = env.Key.AssertString("key").Value; + if (key == "Token") + { + Assert.Equal("foo", env.Value.AssertString("value").Value); + } + else if (key == "Url") + { + Assert.Equal("bar", env.Value.AssertString("value").Value); + } + else + { + throw new NotSupportedException(key); + } + } + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsContainerActionDefinitionRegistry() + { + try + { + // Arrange. + Setup(); + // Prepare the task.json content. + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'docker' + image: 'docker://ubuntu:16.04' + args: + - '${{ inputs.greeting }}' + entrypoint: 'main.sh' + env: + Token: foo + Url: ${{inputs.greeting}} +"; + Pipelines.ActionStep instance; + string directory; + CreateAction(yamlContent: Content, instance: out instance, directory: out directory); + + _actionManager.CachedActionContainers[instance.Id] = new ContainerInfo() { ContainerImage = "ubuntu:16.04" }; + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as ContainerActionExecutionData)); + Assert.Equal("ubuntu:16.04", (definition.Data.Execution as ContainerActionExecutionData).Image); + Assert.Equal("main.sh", (definition.Data.Execution as ContainerActionExecutionData).EntryPoint); + + foreach (var arg in (definition.Data.Execution as ContainerActionExecutionData).Arguments) + { + Assert.Equal("${{ inputs.greeting }}", arg.AssertScalar("arg").ToString()); + } + + foreach (var env in (definition.Data.Execution as ContainerActionExecutionData).Environment) + { + var key = env.Key.AssertString("key").Value; + if (key == "Token") + { + Assert.Equal("foo", env.Value.AssertString("value").Value); + } + else if (key == "Url") + { + Assert.Equal("${{ inputs.greeting }}", env.Value.AssertScalar("value").ToString()); + } + else + { + throw new NotSupportedException(key); + } + } + + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsNodeActionDefinition() + { + try + { + // Arrange. + Setup(); + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'task.js' +"; + Pipelines.ActionStep instance; + string directory; + CreateAction(yamlContent: Content, instance: out instance, directory: out directory); + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as NodeJSActionExecutionData)); + Assert.Equal("task.js", (definition.Data.Execution as NodeJSActionExecutionData).Script); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsNodeActionDefinitionYaml() + { + try + { + // Arrange. + Setup(); + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'task.js' +"; + Pipelines.ActionStep instance; + string directory; + directory = Path.Combine(_workFolder, Constants.Path.ActionsDirectory, "GitHub/actions".Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), "master"); + string file = Path.Combine(directory, Constants.Path.ActionManifestYamlFile); + Directory.CreateDirectory(Path.GetDirectoryName(file)); + File.WriteAllText(file, Content); + instance = new Pipelines.ActionStep() + { + Id = Guid.NewGuid(), + Reference = new Pipelines.RepositoryPathReference() + { + Name = "GitHub/actions", + Ref = "master", + RepositoryType = Pipelines.RepositoryTypes.GitHub + } + }; + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as NodeJSActionExecutionData)); + Assert.Equal("task.js", (definition.Data.Execution as NodeJSActionExecutionData).Script); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsContainerActionDefinitionDockerfile_SelfRepo() + { + try + { + // Arrange. + Setup(); + // Prepare the task.json content. + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'docker' + image: 'Dockerfile' + args: + - '${{ inputs.greeting }}' + entrypoint: 'main.sh' + env: + Token: foo + Url: bar +"; + Pipelines.ActionStep instance; + string directory; + CreateSelfRepoAction(yamlContent: Content, instance: out instance, directory: out directory); + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as ContainerActionExecutionData)); // execution.Node + Assert.Equal("Dockerfile", (definition.Data.Execution as ContainerActionExecutionData).Image); + Assert.Equal("main.sh", (definition.Data.Execution as ContainerActionExecutionData).EntryPoint); + + foreach (var arg in (definition.Data.Execution as ContainerActionExecutionData).Arguments) + { + Assert.Equal("${{ inputs.greeting }}", arg.AssertScalar("arg").ToString()); + } + + foreach (var env in (definition.Data.Execution as ContainerActionExecutionData).Environment) + { + var key = env.Key.AssertString("key").Value; + if (key == "Token") + { + Assert.Equal("foo", env.Value.AssertString("value").Value); + } + else if (key == "Url") + { + Assert.Equal("bar", env.Value.AssertString("value").Value); + } + else + { + throw new NotSupportedException(key); + } + } + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsContainerActionDefinitionRegistry_SelfRepo() + { + try + { + // Arrange. + Setup(); + // Prepare the task.json content. + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'docker' + image: 'docker://ubuntu:16.04' + args: + - '${{ inputs.greeting }}' + entrypoint: 'main.sh' + env: + Token: foo + Url: ${{inputs.greeting}} +"; + Pipelines.ActionStep instance; + string directory; + CreateSelfRepoAction(yamlContent: Content, instance: out instance, directory: out directory); + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as ContainerActionExecutionData)); + Assert.Equal("docker://ubuntu:16.04", (definition.Data.Execution as ContainerActionExecutionData).Image); + Assert.Equal("main.sh", (definition.Data.Execution as ContainerActionExecutionData).EntryPoint); + + foreach (var arg in (definition.Data.Execution as ContainerActionExecutionData).Arguments) + { + Assert.Equal("${{ inputs.greeting }}", arg.AssertScalar("arg").ToString()); + } + + foreach (var env in (definition.Data.Execution as ContainerActionExecutionData).Environment) + { + var key = env.Key.AssertString("key").Value; + if (key == "Token") + { + Assert.Equal("foo", env.Value.AssertString("value").Value); + } + else if (key == "Url") + { + Assert.Equal("${{ inputs.greeting }}", env.Value.AssertScalar("value").ToString()); + } + else + { + throw new NotSupportedException(key); + } + } + + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsNodeActionDefinition_SelfRepo() + { + try + { + // Arrange. + Setup(); + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'task.js' +"; + Pipelines.ActionStep instance; + string directory; + CreateSelfRepoAction(yamlContent: Content, instance: out instance, directory: out directory); + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as NodeJSActionExecutionData)); + Assert.Equal("task.js", (definition.Data.Execution as NodeJSActionExecutionData).Script); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsNodeActionDefinition_Cleanup() + { + try + { + // Arrange. + Setup(); + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'task.js' + post: 'cleanup.js' +"; + Pipelines.ActionStep instance; + string directory; + CreateAction(yamlContent: Content, instance: out instance, directory: out directory); + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as NodeJSActionExecutionData)); + Assert.Equal("task.js", (definition.Data.Execution as NodeJSActionExecutionData).Script); + Assert.Equal("cleanup.js", (definition.Data.Execution as NodeJSActionExecutionData).Post); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsContainerActionDefinitionDockerfile_Cleanup() + { + try + { + // Arrange. + Setup(); + // Prepare the task.json content. + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'GitHub' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'docker' + image: 'Dockerfile' + args: + - '${{ inputs.greeting }}' + entrypoint: 'main.sh' + env: + Token: foo + Url: bar + post-entrypoint: 'cleanup.sh' +"; + Pipelines.ActionStep instance; + string directory; + CreateAction(yamlContent: Content, instance: out instance, directory: out directory); + _actionManager.CachedActionContainers[instance.Id] = new ContainerInfo() { ContainerImage = "image:1234" }; + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as ContainerActionExecutionData)); // execution.Node + Assert.Equal("image:1234", (definition.Data.Execution as ContainerActionExecutionData).Image); + Assert.Equal("main.sh", (definition.Data.Execution as ContainerActionExecutionData).EntryPoint); + Assert.Equal("cleanup.sh", (definition.Data.Execution as ContainerActionExecutionData).Post); + + foreach (var arg in (definition.Data.Execution as ContainerActionExecutionData).Arguments) + { + Assert.Equal("${{ inputs.greeting }}", arg.AssertScalar("arg").ToString()); + } + + foreach (var env in (definition.Data.Execution as ContainerActionExecutionData).Environment) + { + var key = env.Key.AssertString("key").Value; + if (key == "Token") + { + Assert.Equal("foo", env.Value.AssertString("value").Value); + } + else if (key == "Url") + { + Assert.Equal("bar", env.Value.AssertString("value").Value); + } + else + { + throw new NotSupportedException(key); + } + } + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void LoadsPluginActionDefinition() + { + try + { + // Arrange. + Setup(); + const string Content = @" +name: 'Hello World' +description: 'Greet the world and record the time' +author: 'Test Corporation' +inputs: + greeting: # id of input + description: 'The greeting we choose - will print ""{greeting}, World!"" on stdout' + required: true + default: 'Hello' + entryPoint: # id of input + description: 'optional docker entrypoint overwrite.' + required: false +outputs: + time: # id of output + description: 'The time we did the greeting' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + plugin: 'someplugin' +"; + Pipelines.ActionStep instance; + string directory; + CreateAction(yamlContent: Content, instance: out instance, directory: out directory); + + // Act. + Definition definition = _actionManager.LoadAction(_ec.Object, instance); + + // Assert. + Assert.NotNull(definition); + Assert.Equal(directory, definition.Directory); + Assert.NotNull(definition.Data); + Assert.NotNull(definition.Data.Inputs); // inputs + Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var input in definition.Data.Inputs) + { + var name = input.Key.AssertString("key").Value; + var value = input.Value.AssertScalar("value").ToString(); + + _hc.GetTrace().Info($"Default: {name} = {value}"); + inputDefaults[name] = value; + } + + Assert.Equal(2, inputDefaults.Count); + Assert.True(inputDefaults.ContainsKey("greeting")); + Assert.Equal("Hello", inputDefaults["greeting"]); + Assert.True(string.IsNullOrEmpty(inputDefaults["entryPoint"])); + Assert.NotNull(definition.Data.Execution); // execution + + Assert.NotNull((definition.Data.Execution as PluginActionExecutionData)); + Assert.Equal("plugin.class, plugin", (definition.Data.Execution as PluginActionExecutionData).Plugin); + Assert.Equal("plugin.cleanup, plugin", (definition.Data.Execution as PluginActionExecutionData).Post); + } + finally + { + Teardown(); + } + } + + private void CreateAction(string yamlContent, out Pipelines.ActionStep instance, out string directory) + { + directory = Path.Combine(_workFolder, Constants.Path.ActionsDirectory, "GitHub/actions".Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), "master"); + string file = Path.Combine(directory, Constants.Path.ActionManifestYmlFile); + Directory.CreateDirectory(Path.GetDirectoryName(file)); + File.WriteAllText(file, yamlContent); + instance = new Pipelines.ActionStep() + { + Id = Guid.NewGuid(), + Reference = new Pipelines.RepositoryPathReference() + { + Name = "GitHub/actions", + Ref = "master", + RepositoryType = Pipelines.RepositoryTypes.GitHub + } + }; + } + + private void CreateSelfRepoAction(string yamlContent, out Pipelines.ActionStep instance, out string directory) + { + directory = Path.Combine(_workFolder, "actions", "actions"); + string file = Path.Combine(directory, Constants.Path.ActionManifestYmlFile); + Directory.CreateDirectory(Path.GetDirectoryName(file)); + File.WriteAllText(file, yamlContent); + instance = new Pipelines.ActionStep() + { + Id = Guid.NewGuid(), + Reference = new Pipelines.RepositoryPathReference() + { + Name = "GitHub/actions", + Ref = "master", + RepositoryType = Pipelines.PipelineConstants.SelfAlias + } + }; + } + + /// + /// Creates a sample action in an archive on disk, similar to the archive + /// retrieved from GitHub's or GHES' repository API. + /// + /// The path on disk to the archive. +#if OS_WINDOWS + private Task CreateRepoArchive() +#else + private async Task CreateRepoArchive() +#endif + { + const string Content = @" +# Container action +name: 'Hello World' +description: 'Greet the world' +author: 'GitHub' +icon: 'hello.svg' # vector art to display in the GitHub Marketplace +color: 'green' # optional, decorates the entry in the GitHub Marketplace +runs: + using: 'node12' + main: 'task.js' +"; + CreateAction(yamlContent: Content, instance: out _, directory: out string directory); + + var tempDir = _hc.GetDirectory(WellKnownDirectory.Temp); + Directory.CreateDirectory(tempDir); + var archiveFile = Path.Combine(tempDir, Path.GetRandomFileName()); + var trace = _hc.GetTrace(); + +#if OS_WINDOWS + ZipFile.CreateFromDirectory(directory, archiveFile, CompressionLevel.Fastest, includeBaseDirectory: true); + return Task.FromResult(archiveFile); +#else + string tar = WhichUtil.Which("tar", require: true, trace: trace); + + // tar -xzf + using (var processInvoker = new ProcessInvokerWrapper()) + { + processInvoker.Initialize(_hc); + processInvoker.OutputDataReceived += new EventHandler((sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + trace.Info(args.Data); + } + }); + + processInvoker.ErrorDataReceived += new EventHandler((sender, args) => + { + if (!string.IsNullOrEmpty(args.Data)) + { + trace.Error(args.Data); + } + }); + + string cwd = Path.GetDirectoryName(directory); + string inputDirectory = Path.GetFileName(directory); + int exitCode = await processInvoker.ExecuteAsync(_hc.GetDirectory(WellKnownDirectory.Bin), tar, $"-czf \"{archiveFile}\" -C \"{cwd}\" \"{inputDirectory}\"", null, CancellationToken.None); + if (exitCode != 0) + { + throw new NotSupportedException($"Can't use 'tar -czf' to create archive file: {archiveFile}. return code: {exitCode}."); + } + } + return archiveFile; +#endif + } + + private static string GetLinkToActionArchive(string apiUrl, string repository, string @ref) + { +#if OS_WINDOWS + return $"{apiUrl}/repos/{repository}/zipball/{@ref}"; +#else + return $"{apiUrl}/repos/{repository}/tarball/{@ref}"; +#endif + } + + private void Setup([CallerMemberName] string name = "", bool newActionMetadata = true) + { + _ecTokenSource?.Dispose(); + _ecTokenSource = new CancellationTokenSource(); + + // Test host context. + _hc = new TestHostContext(this, name); + + // Random work folder. + _workFolder = _hc.GetDirectory(WellKnownDirectory.Work); + + _ec = new Mock(); + _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); + var variables = new Dictionary(); + if (newActionMetadata) + { + variables["DistributedTask.NewActionMetadata"] = "true"; + } + _ec.Setup(x => x.Variables).Returns(new Variables(_hc, variables)); _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"[{tag}]{message}"); }); From c4626d0c3a871067dfd5bf43a8ceb344514f8cf6 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 3 Jun 2020 23:24:53 -0400 Subject: [PATCH 62/86] Remove SPS/Token migration code. Remove GHES url manipulate code. (#513) * Remove SPS/Token migration code. Remove GHES url manipulate code. * feedback. --- src/Runner.Common/ConfigurationStore.cs | 22 +- src/Runner.Common/RunnerServer.cs | 4 - src/Runner.Common/Terminal.cs | 10 +- .../Configuration/ConfigurationManager.cs | 43 +- .../Configuration/CredentialManager.cs | 20 +- src/Runner.Listener/MessageListener.cs | 224 +-- src/Runner.Listener/Program.cs | 4 +- src/Runner.Worker/JobRunner.cs | 15 - src/Test/L0/Listener/MessageListenerL0.cs | 1253 +---------------- 9 files changed, 50 insertions(+), 1545 deletions(-) diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs index fc32ad436b1..0ae270420d1 100644 --- a/src/Runner.Common/ConfigurationStore.cs +++ b/src/Runner.Common/ConfigurationStore.cs @@ -108,9 +108,9 @@ public interface IConfigurationStore : IRunnerService CredentialData GetMigratedCredentials(); RunnerSettings GetSettings(); void SaveCredential(CredentialData credential); - void SaveMigratedCredential(CredentialData credential); void SaveSettings(RunnerSettings settings); void DeleteCredential(); + void DeleteMigratedCredential(); void DeleteSettings(); } @@ -232,21 +232,6 @@ public void SaveCredential(CredentialData credential) File.SetAttributes(_credFilePath, File.GetAttributes(_credFilePath) | FileAttributes.Hidden); } - public void SaveMigratedCredential(CredentialData credential) - { - Trace.Info("Saving {0} migrated credential @ {1}", credential.Scheme, _migratedCredFilePath); - if (File.Exists(_migratedCredFilePath)) - { - // Delete existing credential file first, since the file is hidden and not able to overwrite. - Trace.Info("Delete exist runner migrated credential file."); - IOUtil.DeleteFile(_migratedCredFilePath); - } - - IOUtil.SaveObject(credential, _migratedCredFilePath); - Trace.Info("Migrated Credentials Saved."); - File.SetAttributes(_migratedCredFilePath, File.GetAttributes(_migratedCredFilePath) | FileAttributes.Hidden); - } - public void SaveSettings(RunnerSettings settings) { Trace.Info("Saving runner settings."); @@ -268,6 +253,11 @@ public void DeleteCredential() IOUtil.Delete(_migratedCredFilePath, default(CancellationToken)); } + public void DeleteMigratedCredential() + { + IOUtil.Delete(_migratedCredFilePath, default(CancellationToken)); + } + public void DeleteSettings() { IOUtil.Delete(_configFilePath, default(CancellationToken)); diff --git a/src/Runner.Common/RunnerServer.cs b/src/Runner.Common/RunnerServer.cs index cbdcb898c29..5e284a17574 100644 --- a/src/Runner.Common/RunnerServer.cs +++ b/src/Runner.Common/RunnerServer.cs @@ -50,10 +50,6 @@ public interface IRunnerServer : IRunnerService // agent update Task UpdateAgentUpdateStateAsync(int agentPoolId, int agentId, string currentState); - - // runner authorization url - Task GetRunnerAuthUrlAsync(int runnerPoolId, int runnerId); - Task ReportRunnerAuthUrlErrorAsync(int runnerPoolId, int runnerId, string error); } public sealed class RunnerServer : RunnerService, IRunnerServer diff --git a/src/Runner.Common/Terminal.cs b/src/Runner.Common/Terminal.cs index f35a2220d81..2ad873b98e0 100644 --- a/src/Runner.Common/Terminal.cs +++ b/src/Runner.Common/Terminal.cs @@ -96,13 +96,14 @@ public void Write(string message, ConsoleColor? colorCode = null) Trace.Info($"WRITE: {message}"); if (!Silent) { - if(colorCode != null) + if (colorCode != null) { Console.ForegroundColor = colorCode.Value; Console.Write(message); Console.ResetColor(); } - else { + else + { Console.Write(message); } } @@ -120,13 +121,14 @@ public void WriteLine(string line, ConsoleColor? colorCode = null) Trace.Info($"WRITE LINE: {line}"); if (!Silent) { - if(colorCode != null) + if (colorCode != null) { Console.ForegroundColor = colorCode.Value; Console.WriteLine(line); Console.ResetColor(); } - else { + else + { Console.WriteLine(line); } } diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 9a98c7d6505..7d634067f3b 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -119,6 +119,19 @@ public async Task ConfigureAsync(CommandSettings command) // Determine the service deployment type based on connection data. (Hosted/OnPremises) runnerSettings.IsHostedServer = runnerSettings.GitHubUrl == null || IsHostedServer(new UriBuilder(runnerSettings.GitHubUrl)); + // Warn if the Actions server url and GHES server url has different Host + if (!runnerSettings.IsHostedServer) + { + // Example actionsServerUrl is https://my-ghes/_services/pipelines/[...] + // Example githubServerUrl is https://my-ghes + var actionsServerUrl = new Uri(runnerSettings.ServerUrl); + var githubServerUrl = new Uri(runnerSettings.GitHubUrl); + if (!string.Equals(actionsServerUrl.Authority, githubServerUrl.Authority, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"GitHub Actions is not properly configured in GHES. GHES url: {runnerSettings.GitHubUrl}, Actions url: {runnerSettings.ServerUrl}."); + } + } + // Validate can connect. await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds); @@ -221,36 +234,11 @@ public async Task ConfigureAsync(CommandSettings command) // Add Agent Id to settings runnerSettings.AgentId = agent.Id; - // respect the serverUrl resolve by server. - // in case of agent configured using collection url instead of account url. - string agentServerUrl; - if (agent.Properties.TryGetValidatedValue("ServerUrl", out agentServerUrl) && - !string.IsNullOrEmpty(agentServerUrl)) - { - Trace.Info($"Agent server url resolve by server: '{agentServerUrl}'."); - - // we need make sure the Schema/Host/Port component of the url remain the same. - UriBuilder inputServerUrl = new UriBuilder(runnerSettings.ServerUrl); - UriBuilder serverReturnedServerUrl = new UriBuilder(agentServerUrl); - if (Uri.Compare(inputServerUrl.Uri, serverReturnedServerUrl.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0) - { - inputServerUrl.Path = serverReturnedServerUrl.Path; - Trace.Info($"Replace server returned url's scheme://host:port component with user input server url's scheme://host:port: '{inputServerUrl.Uri.AbsoluteUri}'."); - runnerSettings.ServerUrl = inputServerUrl.Uri.AbsoluteUri; - } - else - { - runnerSettings.ServerUrl = agentServerUrl; - } - } - // See if the server supports our OAuth key exchange for credentials if (agent.Authorization != null && agent.Authorization.ClientId != Guid.Empty && agent.Authorization.AuthorizationUrl != null) { - UriBuilder configServerUrl = new UriBuilder(runnerSettings.ServerUrl); - UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl); var credentialData = new CredentialData { Scheme = Constants.Configuration.OAuth, @@ -258,7 +246,6 @@ public async Task ConfigureAsync(CommandSettings command) { { "clientId", agent.Authorization.ClientId.ToString("D") }, { "authorizationUrl", agent.Authorization.AuthorizationUrl.AbsoluteUri }, - { "oauthEndpointUrl", oauthEndpointUrlBuilder.Uri.AbsoluteUri }, }, }; @@ -464,7 +451,7 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, // update should replace the existing labels agent.Version = BuildConstants.RunnerPackage.Version; agent.OSDescription = RuntimeInformation.OSDescription; - + agent.Labels.Clear(); agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); @@ -475,7 +462,7 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, { agent.Labels.Add(new AgentLabel(userLabel, LabelType.User)); } - + return agent; } diff --git a/src/Runner.Listener/Configuration/CredentialManager.cs b/src/Runner.Listener/Configuration/CredentialManager.cs index 871aae4de6d..ee459abe70d 100644 --- a/src/Runner.Listener/Configuration/CredentialManager.cs +++ b/src/Runner.Listener/Configuration/CredentialManager.cs @@ -13,7 +13,7 @@ namespace GitHub.Runner.Listener.Configuration public interface ICredentialManager : IRunnerService { ICredentialProvider GetCredentialProvider(string credType); - VssCredentials LoadCredentials(bool preferMigrated = true); + VssCredentials LoadCredentials(); } public class CredentialManager : RunnerService, ICredentialManager @@ -40,7 +40,7 @@ public ICredentialProvider GetCredentialProvider(string credType) return creds; } - public VssCredentials LoadCredentials(bool preferMigrated = true) + public VssCredentials LoadCredentials() { IConfigurationStore store = HostContext.GetService(); @@ -50,14 +50,16 @@ public VssCredentials LoadCredentials(bool preferMigrated = true) } CredentialData credData = store.GetCredentials(); - - if (preferMigrated) + var migratedCred = store.GetMigratedCredentials(); + if (migratedCred != null) { - var migratedCred = store.GetMigratedCredentials(); - if (migratedCred != null) - { - credData = migratedCred; - } + credData = migratedCred; + + // Re-write .credentials with Token URL + store.SaveCredential(credData); + + // Delete .credentials_migrated + store.DeleteMigratedCredential(); } ICredentialProvider credProv = GetCredentialProvider(credData.Scheme); diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 5718654b767..0ad22e87df7 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -13,10 +13,7 @@ using System.Runtime.InteropServices; using GitHub.Runner.Common; using GitHub.Runner.Sdk; -using GitHub.Services.WebApi; -using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("Test")] namespace GitHub.Runner.Listener { [ServiceLocator(Default = typeof(MessageListener))] @@ -35,30 +32,18 @@ public sealed class MessageListener : RunnerService, IMessageListener private ITerminal _term; private IRunnerServer _runnerServer; private TaskAgentSession _session; - private ICredentialManager _credMgr; - private IConfigurationStore _configStore; private TimeSpan _getNextMessageRetryInterval; private readonly TimeSpan _sessionCreationRetryInterval = TimeSpan.FromSeconds(30); private readonly TimeSpan _sessionConflictRetryLimit = TimeSpan.FromMinutes(4); private readonly TimeSpan _clockSkewRetryLimit = TimeSpan.FromMinutes(30); private readonly Dictionary _sessionCreationExceptionTracker = new Dictionary(); - // Whether load credentials from .credentials_migrated file - internal bool _useMigratedCredentials; - - // need to check auth url if there is only .credentials and auth schema is OAuth - internal bool _needToCheckAuthorizationUrlUpdate; - internal Task _authorizationUrlMigrationBackgroundTask; - internal Task _authorizationUrlRollbackReattemptDelayBackgroundTask; - public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); _term = HostContext.GetService(); _runnerServer = HostContext.GetService(); - _credMgr = HostContext.GetService(); - _configStore = HostContext.GetService(); } public async Task CreateSessionAsync(CancellationToken token) @@ -73,8 +58,8 @@ public async Task CreateSessionAsync(CancellationToken token) // Create connection. Trace.Info("Loading Credentials"); - _useMigratedCredentials = !StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_SPSAUTHURL")); - VssCredentials creds = _credMgr.LoadCredentials(_useMigratedCredentials); + var credMgr = HostContext.GetService(); + VssCredentials creds = credMgr.LoadCredentials(); var agent = new TaskAgentReference { @@ -89,17 +74,6 @@ public async Task CreateSessionAsync(CancellationToken token) string errorMessage = string.Empty; bool encounteringError = false; - var originalCreds = _configStore.GetCredentials(); - var migratedCreds = _configStore.GetMigratedCredentials(); - if (migratedCreds == null) - { - _useMigratedCredentials = false; - if (originalCreds.Scheme == Constants.Configuration.OAuth) - { - _needToCheckAuthorizationUrlUpdate = true; - } - } - while (true) { token.ThrowIfCancellationRequested(); @@ -127,12 +101,6 @@ public async Task CreateSessionAsync(CancellationToken token) encounteringError = false; } - if (_needToCheckAuthorizationUrlUpdate) - { - // start background task try to get new authorization url - _authorizationUrlMigrationBackgroundTask = GetNewOAuthAuthorizationSetting(token); - } - return true; } catch (OperationCanceledException) when (token.IsCancellationRequested) @@ -164,44 +132,10 @@ public async Task CreateSessionAsync(CancellationToken token) } } - if (ex is TaskAgentSessionConflictException) - { - try - { - var newCred = await GetNewOAuthAuthorizationSetting(token, true); - if (newCred != null) - { - await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), newCred); - Trace.Info("Updated connection to use migrated credential for next CreateSession call."); - _useMigratedCredentials = true; - _authorizationUrlMigrationBackgroundTask = null; - _needToCheckAuthorizationUrlUpdate = false; - } - } - catch (Exception e) - { - Trace.Error("Fail to refresh connection with new authorization url."); - Trace.Error(e); - } - } - if (!IsSessionCreationExceptionRetriable(ex)) { - if (_useMigratedCredentials && !(ex is TaskAgentSessionConflictException)) - { - // migrated credentials might cause lose permission during permission check, - // we will force to use original credential and try again - _useMigratedCredentials = false; - var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36)); - _authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours. - creds = _credMgr.LoadCredentials(false); - Trace.Error("Fallback to original credentials and try again."); - } - else - { - _term.WriteError($"Failed to create session. {ex.Message}"); - return false; - } + _term.WriteError($"Failed to create session. {ex.Message}"); + return false; } if (!encounteringError) //print the message only on the first error @@ -262,51 +196,6 @@ public async Task GetNextMessageAsync(CancellationToken token) encounteringError = false; continuousError = 0; } - - if (_needToCheckAuthorizationUrlUpdate && - _authorizationUrlMigrationBackgroundTask?.IsCompleted == true) - { - if (HostContext.GetService().Busy || - HostContext.GetService().Busy) - { - Trace.Info("Job or runner updates in progress, update credentials next time."); - } - else - { - try - { - var newCred = await _authorizationUrlMigrationBackgroundTask; - await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), newCred); - Trace.Info("Updated connection to use migrated credential for next GetMessage call."); - _useMigratedCredentials = true; - _authorizationUrlMigrationBackgroundTask = null; - _needToCheckAuthorizationUrlUpdate = false; - } - catch (Exception ex) - { - Trace.Error("Fail to refresh connection with new authorization url."); - Trace.Error(ex); - } - } - } - - if (_authorizationUrlRollbackReattemptDelayBackgroundTask?.IsCompleted == true) - { - try - { - // we rolled back to use original creds about 2 days before, now it's a good time to try migrated creds again. - Trace.Info("Re-attempt to use migrated credential"); - var migratedCreds = _credMgr.LoadCredentials(); - await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), migratedCreds); - _useMigratedCredentials = true; - _authorizationUrlRollbackReattemptDelayBackgroundTask = null; - } - catch (Exception ex) - { - Trace.Error("Fail to refresh connection with new authorization url on rollback reattempt."); - Trace.Error(ex); - } - } } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -330,21 +219,7 @@ public async Task GetNextMessageAsync(CancellationToken token) } else if (!IsGetNextMessageExceptionRetriable(ex)) { - if (_useMigratedCredentials) - { - // migrated credentials might cause lose permission during permission check, - // we will force to use original credential and try again - _useMigratedCredentials = false; - var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36)); - _authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours. - var originalCreds = _credMgr.LoadCredentials(false); - await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), originalCreds); - Trace.Error("Fallback to original credentials and try again."); - } - else - { - throw; - } + throw; } else { @@ -536,94 +411,5 @@ ex is AccessDeniedException || return true; } } - - private async Task GetNewOAuthAuthorizationSetting(CancellationToken token, bool adhoc = false) - { - Trace.Info("Start checking oauth authorization url update."); - while (true) - { - try - { - var migratedAuthorizationUrl = await _runnerServer.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId); - if (!string.IsNullOrEmpty(migratedAuthorizationUrl)) - { - var credData = _configStore.GetCredentials(); - var clientId = credData.Data.GetValueOrDefault("clientId", null); - var currentAuthorizationUrl = credData.Data.GetValueOrDefault("authorizationUrl", null); - Trace.Info($"Current authorization url: {currentAuthorizationUrl}, new authorization url: {migratedAuthorizationUrl}"); - - if (string.Equals(currentAuthorizationUrl, migratedAuthorizationUrl, StringComparison.OrdinalIgnoreCase)) - { - // We don't need to update credentials. - Trace.Info("No needs to update authorization url"); - if (adhoc) - { - return null; - } - else - { - await Task.Delay(TimeSpan.FromMilliseconds(-1), token); - } - } - - var keyManager = HostContext.GetService(); - var signingCredentials = VssSigningCredentials.Create(() => keyManager.GetKey()); - - var migratedClientCredential = new VssOAuthJwtBearerClientCredential(clientId, migratedAuthorizationUrl, signingCredentials); - var migratedRunnerCredential = new VssOAuthCredential(new Uri(migratedAuthorizationUrl, UriKind.Absolute), VssOAuthGrant.ClientCredentials, migratedClientCredential); - - Trace.Info("Try connect service with Token Service OAuth endpoint."); - var runnerServer = HostContext.CreateService(); - await runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), migratedRunnerCredential); - await runnerServer.GetAgentPoolsAsync(); - Trace.Info($"Successfully connected service with new authorization url."); - - var migratedCredData = new CredentialData - { - Scheme = Constants.Configuration.OAuth, - Data = - { - { "clientId", clientId }, - { "authorizationUrl", migratedAuthorizationUrl }, - { "oauthEndpointUrl", migratedAuthorizationUrl }, - }, - }; - - _configStore.SaveMigratedCredential(migratedCredData); - return migratedRunnerCredential; - } - else - { - Trace.Verbose("No authorization url updates"); - } - } - catch (Exception ex) when (!token.IsCancellationRequested) - { - Trace.Error("Fail to get/test new authorization url."); - Trace.Error(ex); - - try - { - await _runnerServer.ReportRunnerAuthUrlErrorAsync(_settings.PoolId, _settings.AgentId, ex.ToString()); - } - catch (Exception e) - { - // best effort - Trace.Error("Fail to report the migration error"); - Trace.Error(e); - } - } - - if (adhoc) - { - return null; - } - else - { - var backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromMinutes(30), TimeSpan.FromMinutes(45)); - await HostContext.Delay(backoff, token); - } - } - } } } diff --git a/src/Runner.Listener/Program.cs b/src/Runner.Listener/Program.cs index 3181680f0e4..a24224dad65 100644 --- a/src/Runner.Listener/Program.cs +++ b/src/Runner.Listener/Program.cs @@ -102,7 +102,9 @@ private async static Task MainAsync(IHostContext context, string[] args) IRunner runner = context.GetService(); try { - return await runner.ExecuteCommand(command); + var returnCode = await runner.ExecuteCommand(command); + trace.Info($"Runner execution has finished with return code {returnCode}"); + return returnCode; } catch (OperationCanceledException) when (context.RunnerShutdownToken.IsCancellationRequested) { diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 58cf26bf48a..31dfb17145b 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -5,21 +5,13 @@ using GitHub.Services.WebApi; using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Net.Http; -using System.Text; -using System.IO.Compression; -using System.Diagnostics; -using Newtonsoft.Json.Linq; -using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.Runner.Common; using GitHub.Runner.Sdk; -using GitHub.DistributedTask.Pipelines.ContextData; -using GitHub.DistributedTask.ObjectTemplating; namespace GitHub.Runner.Worker { @@ -122,13 +114,6 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, _tempDirectoryManager = HostContext.GetService(); _tempDirectoryManager.InitializeTempDirectory(jobContext); - // // Expand container properties - // jobContext.Container?.ExpandProperties(jobContext.Variables); - // foreach (var sidecar in jobContext.SidecarContainers) - // { - // sidecar.ExpandProperties(jobContext.Variables); - // } - // Get the job extension. Trace.Info("Getting job extension."); IJobExtension jobExtension = HostContext.CreateService(); diff --git a/src/Test/L0/Listener/MessageListenerL0.cs b/src/Test/L0/Listener/MessageListenerL0.cs index ba0f0ee7716..a830e9c9940 100644 --- a/src/Test/L0/Listener/MessageListenerL0.cs +++ b/src/Test/L0/Listener/MessageListenerL0.cs @@ -63,7 +63,7 @@ public async void CreatesSession() tokenSource.Token)) .Returns(Task.FromResult(expectedSession)); - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); + _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials()); _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken }); _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); @@ -107,7 +107,7 @@ public async void DeleteSession() tokenSource.Token)) .Returns(Task.FromResult(expectedSession)); - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); + _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials()); _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken }); _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); @@ -154,7 +154,7 @@ public async void GetNextMessage() tokenSource.Token)) .Returns(Task.FromResult(expectedSession)); - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); + _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials()); _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken }); _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); @@ -231,315 +231,7 @@ public async void CreateSessionWithOriginalCredential() tokenSource.Token)) .Returns(Task.FromResult(expectedSession)); - _runnerServer - .Setup(x => x.GetRunnerAuthUrlAsync( - _settings.PoolId, - _settings.AgentId)) - .Returns(async () => - { - await Task.Delay(10); - return ""; - }); - - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.False(listener._useMigratedCredentials); - Assert.True(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.NotNull(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithMigratedCredential() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - var migratedCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - migratedCred.Data["authorizationUrl"] = "https://t.server"; - migratedCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(migratedCred); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.True(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithHostedCredential() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); - - _store.Setup(x => x.GetCredentials()).Returns(new CredentialData() { Scheme = Constants.Configuration.OAuthAccessToken }); - _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.False(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithMigratedCredentialFallBackOriginalSucceed() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - 123, - It.Is(y => y != null), - tokenSource.Token)) - .Callback(() => { _settings.PoolId = 1234; }) - .Throws(new TaskAgentPoolNotFoundException("L0 Pool not found")); - - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - 1234, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - var originalVssCred = new VssCredentials(); - var migratedVssCred = new VssCredentials(); - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(migratedVssCred); - _credMgr.Setup(x => x.LoadCredentials(false)).Returns(originalVssCred); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - var migratedCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - migratedCred.Data["authorizationUrl"] = "https://t.server"; - migratedCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(migratedCred); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - It.IsAny(), - It.Is(y => y != null), - tokenSource.Token), Times.Exactly(2)); - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - originalVssCred), Times.Once); - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - migratedVssCred), Times.Once); - - Assert.False(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.NotNull(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithMigratedCredentialFallBackOriginalStillFailed() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Throws(new TaskAgentPoolNotFoundException("L0 Pool not found")); - - var originalVssCred = new VssCredentials(); - var migratedVssCred = new VssCredentials(); - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(migratedVssCred); - _credMgr.Setup(x => x.LoadCredentials(false)).Returns(originalVssCred); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - var migratedCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - migratedCred.Data["authorizationUrl"] = "https://t.server"; - migratedCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(migratedCred); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.False(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Exactly(2)); - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - originalVssCred), Times.Once); - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - migratedVssCred), Times.Once); - - Assert.False(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.NotNull(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithOriginalGetMessageWaitForMigtateToMigrated() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - _runnerServer - .Setup(x => x.GetRunnerAuthUrlAsync( - _settings.PoolId, - _settings.AgentId)) - .Returns(async () => - { - await Task.Delay(10); - return ""; - }); - - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); + _credMgr.Setup(x => x.LoadCredentials()).Returns(new VssCredentials()); var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; originalCred.Data["authorizationUrl"] = "https://s.server"; @@ -562,943 +254,6 @@ public async void CreateSessionWithOriginalGetMessageWaitForMigtateToMigrated() _settings.PoolId, It.Is(y => y != null), tokenSource.Token), Times.Once()); - - Assert.False(listener._useMigratedCredentials); - Assert.True(listener._needToCheckAuthorizationUrlUpdate); - - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.NotNull(listener._authorizationUrlMigrationBackgroundTask); - - var arMessages = new TaskAgentMessage[] - { - new TaskAgentMessage - { - Body = "somebody1", - MessageId = 4234, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - }, - new TaskAgentMessage - { - Body = "somebody2", - MessageId = 4235, - MessageType = JobCancelMessage.MessageType - }, - null, //should be skipped by GetNextMessageAsync implementation - null, - new TaskAgentMessage - { - Body = "somebody3", - MessageId = 4236, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - } - }; - var messages = new Queue(arMessages); - - _runnerServer - .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token)) - .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) => - { - await Task.Delay(200); - return messages.Dequeue(); - }); - - TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token); - Assert.Equal(arMessages[0], message1); - Assert.Equal(arMessages[1], message2); - Assert.Equal(arMessages[4], message3); - - //Assert - _runnerServer - .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token), Times.Exactly(arMessages.Length)); - - _runnerServer - .Verify(x => x.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId), Times.AtLeast(2)); - - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Once); - - var tempLog = Path.GetTempFileName(); - File.Copy(tc.TraceFileName, tempLog, true); - var traceContent = File.ReadAllLines(tempLog); - Assert.DoesNotContain(traceContent, x => x.Contains("Try connect service with migrated OAuth endpoint.")); - - Assert.False(listener._useMigratedCredentials); - Assert.True(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.NotNull(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithOriginalGetMessageMigtateToMigrated() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - _runnerServer - .Setup(x => x.GetRunnerAuthUrlAsync( - _settings.PoolId, - _settings.AgentId)) - .Returns(async () => - { - await Task.Delay(100); - return "https://t.server"; - }); - - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.False(listener._useMigratedCredentials); - Assert.True(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.NotNull(listener._authorizationUrlMigrationBackgroundTask); - - var arMessages = new TaskAgentMessage[] - { - new TaskAgentMessage - { - Body = "somebody1", - MessageId = 4234, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - }, - new TaskAgentMessage - { - Body = "somebody2", - MessageId = 4235, - MessageType = JobCancelMessage.MessageType - }, - null, //should be skipped by GetNextMessageAsync implementation - null, - new TaskAgentMessage - { - Body = "somebody3", - MessageId = 4236, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - } - }; - var messages = new Queue(arMessages); - - _runnerServer - .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token)) - .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) => - { - await Task.Delay(200); - return messages.Dequeue(); - }); - - var newRunnerServer = new Mock(); - tc.EnqueueInstance(newRunnerServer.Object); - - var keyManager = new Mock(); - keyManager.Setup(x => x.GetKey()).Returns(new RSACryptoServiceProvider(2048)); - tc.SetSingleton(keyManager.Object); - - tc.SetSingleton(new Mock().Object); - tc.SetSingleton(new Mock().Object); - - TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token); - Assert.Equal(arMessages[0], message1); - Assert.Equal(arMessages[1], message2); - Assert.Equal(arMessages[4], message3); - - //Assert - _runnerServer - .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token), Times.Exactly(arMessages.Length)); - - _runnerServer - .Verify(x => x.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId), Times.Once); - - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Exactly(2)); - - newRunnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Once); - - newRunnerServer - .Verify(x => x.GetAgentPoolsAsync(null, TaskAgentPoolType.Automation), Times.Once); - - var tempLog = Path.GetTempFileName(); - File.Copy(tc.TraceFileName, tempLog, true); - var traceContent = File.ReadAllLines(tempLog); - Assert.Contains(traceContent, x => x.Contains("Try connect service with Token Service OAuth endpoint.")); - - Assert.True(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithOriginalGetMessageMigtateToMigratedWaitForIdle() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - _runnerServer - .Setup(x => x.GetRunnerAuthUrlAsync( - _settings.PoolId, - _settings.AgentId)) - .Returns(async () => - { - await Task.Delay(10); - return "https://t.server"; - }); - - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.False(listener._useMigratedCredentials); - Assert.True(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.NotNull(listener._authorizationUrlMigrationBackgroundTask); - - var arMessages = new TaskAgentMessage[] - { - new TaskAgentMessage - { - Body = "somebody1", - MessageId = 4234, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - }, - new TaskAgentMessage - { - Body = "somebody2", - MessageId = 4235, - MessageType = JobCancelMessage.MessageType - }, - null, //should be skipped by GetNextMessageAsync implementation - null, - new TaskAgentMessage - { - Body = "somebody3", - MessageId = 4236, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - } - }; - var messages = new Queue(arMessages); - - var busy = true; - var counter = 0; - _runnerServer - .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token)) - .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) => - { - await Task.Delay(200); - if (++counter == 4) - { - busy = false; - } - return messages.Dequeue(); - }); - - var newRunnerServer = new Mock(); - tc.EnqueueInstance(newRunnerServer.Object); - - var keyManager = new Mock(); - keyManager.Setup(x => x.GetKey()).Returns(new RSACryptoServiceProvider(2048)); - tc.SetSingleton(keyManager.Object); - - var jobDispatcher = new Mock(); - - jobDispatcher.Setup(x => x.Busy).Returns(() => - { - return busy; - }); - tc.SetSingleton(jobDispatcher.Object); - tc.SetSingleton(new Mock().Object); - - TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token); - Assert.Equal(arMessages[0], message1); - Assert.Equal(arMessages[1], message2); - Assert.Equal(arMessages[4], message3); - - //Assert - _runnerServer - .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token), Times.Exactly(arMessages.Length)); - - _runnerServer - .Verify(x => x.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId), Times.Once); - - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Exactly(2)); - - newRunnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Once); - - newRunnerServer - .Verify(x => x.GetAgentPoolsAsync(null, TaskAgentPoolType.Automation), Times.Once); - - var tempLog = Path.GetTempFileName(); - File.Copy(tc.TraceFileName, tempLog, true); - var traceContent = File.ReadAllLines(tempLog); - Assert.Contains(traceContent, x => x.Contains("Job or runner updates in progress, update credentials next time.")); - Assert.Contains(traceContent, x => x.Contains("Try connect service with Token Service OAuth endpoint.")); - - Assert.True(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithMigratedGetMessageNotMigrateAgain() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - _runnerServer - .Setup(x => x.GetRunnerAuthUrlAsync( - _settings.PoolId, - _settings.AgentId)) - .Returns(async () => - { - await Task.Delay(10); - return "https://t.server"; - }); - - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(new VssCredentials()); - - var migratedCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - migratedCred.Data["authorizationUrl"] = "https://t.server"; - migratedCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(migratedCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.False(listener._useMigratedCredentials); - Assert.True(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.NotNull(listener._authorizationUrlMigrationBackgroundTask); - - var arMessages = new TaskAgentMessage[] - { - new TaskAgentMessage - { - Body = "somebody1", - MessageId = 4234, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - }, - new TaskAgentMessage - { - Body = "somebody2", - MessageId = 4235, - MessageType = JobCancelMessage.MessageType - }, - null, //should be skipped by GetNextMessageAsync implementation - null, - new TaskAgentMessage - { - Body = "somebody3", - MessageId = 4236, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - } - }; - var messages = new Queue(arMessages); - - _runnerServer - .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token)) - .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) => - { - await Task.Delay(200); - return messages.Dequeue(); - }); - - var newRunnerServer = new Mock(); - tc.EnqueueInstance(newRunnerServer.Object); - - var keyManager = new Mock(); - keyManager.Setup(x => x.GetKey()).Returns(new RSACryptoServiceProvider(2048)); - tc.SetSingleton(keyManager.Object); - - tc.SetSingleton(new Mock().Object); - tc.SetSingleton(new Mock().Object); - - TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token); - Assert.Equal(arMessages[0], message1); - Assert.Equal(arMessages[1], message2); - Assert.Equal(arMessages[4], message3); - - //Assert - _runnerServer - .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token), Times.Exactly(arMessages.Length)); - - _runnerServer - .Verify(x => x.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId), Times.Once); - - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Once); - - newRunnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Never); - - newRunnerServer - .Verify(x => x.GetAgentPoolsAsync(null, TaskAgentPoolType.Automation), Times.Never); - - var tempLog = Path.GetTempFileName(); - File.Copy(tc.TraceFileName, tempLog, true); - var traceContent = File.ReadAllLines(tempLog); - Assert.Contains(traceContent, x => x.Contains("No needs to update authorization url")); - - Assert.False(listener._useMigratedCredentials); - Assert.True(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.NotNull(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithOriginalGetMessageMigrateToMigratedFallbackToOriginal() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - var originalVssCred = new VssCredentials(); - var migratedVssCred = new VssCredentials(); - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(migratedVssCred); - _credMgr.Setup(x => x.LoadCredentials(false)).Returns(originalVssCred); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - var migratedCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - migratedCred.Data["authorizationUrl"] = "https://t.server"; - migratedCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(migratedCred); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.True(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - - var arMessages = new TaskAgentMessage[] - { - new TaskAgentMessage - { - Body = "somebody1", - MessageId = 4234, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - }, - new TaskAgentMessage - { - Body = "somebody2", - MessageId = 4235, - MessageType = JobCancelMessage.MessageType - }, - null, //should be skipped by GetNextMessageAsync implementation - null, - new TaskAgentMessage - { - Body = "somebody3", - MessageId = 4236, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - } - }; - var messages = new Queue(arMessages); - - var counter = 0; - _runnerServer - .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token)) - .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) => - { - await Task.Delay(200); - counter++; - - if (counter == 5) - { - throw new TaskAgentNotFoundException("L0 runner not found"); - } - - if (counter == 6) - { - Assert.NotNull(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.False(listener._useMigratedCredentials); - } - - return messages.Dequeue(); - }); - - var newRunnerServer = new Mock(); - tc.EnqueueInstance(newRunnerServer.Object); - - var keyManager = new Mock(); - keyManager.Setup(x => x.GetKey()).Returns(new RSACryptoServiceProvider(2048)); - tc.SetSingleton(keyManager.Object); - - tc.SetSingleton(new Mock().Object); - tc.SetSingleton(new Mock().Object); - - TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token); - Assert.Equal(arMessages[0], message1); - Assert.Equal(arMessages[1], message2); - Assert.Equal(arMessages[4], message3); - - //Assert - _runnerServer - .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token), Times.Exactly(arMessages.Length + 1)); - - _runnerServer - .Verify(x => x.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId), Times.Never); - - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.AtLeast(2)); - - newRunnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Never); - - newRunnerServer - .Verify(x => x.GetAgentPoolsAsync(null, TaskAgentPoolType.Automation), Times.Never); - - var tempLog = Path.GetTempFileName(); - File.Copy(tc.TraceFileName, tempLog, true); - var traceContent = File.ReadAllLines(tempLog); - Assert.Contains(traceContent, x => x.Contains("Fallback to original credentials and try again.")); - - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithOriginalGetMessageMigrateToMigratedFallbackToOriginalReattemptMigrated() - { - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - var originalVssCred = new VssCredentials(); - var migratedVssCred = new VssCredentials(); - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(migratedVssCred); - _credMgr.Setup(x => x.LoadCredentials(false)).Returns(originalVssCred); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - var migratedCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - migratedCred.Data["authorizationUrl"] = "https://t.server"; - migratedCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(migratedCred); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.True(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - - var arMessages = new TaskAgentMessage[] - { - new TaskAgentMessage - { - Body = "somebody1", - MessageId = 4234, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - }, - new TaskAgentMessage - { - Body = "somebody2", - MessageId = 4235, - MessageType = JobCancelMessage.MessageType - }, - null, //should be skipped by GetNextMessageAsync implementation - null, - new TaskAgentMessage - { - Body = "somebody3", - MessageId = 4236, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - } - }; - var messages = new Queue(arMessages); - - var counter = 0; - _runnerServer - .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token)) - .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) => - { - await Task.Delay(200); - counter++; - - if (counter == 2) - { - throw new TaskAgentNotFoundException("L0 runner not found"); - } - - if (counter == 3) - { - Assert.NotNull(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - } - - return messages.Dequeue(); - }); - - var newRunnerServer = new Mock(); - tc.EnqueueInstance(newRunnerServer.Object); - - var keyManager = new Mock(); - keyManager.Setup(x => x.GetKey()).Returns(new RSACryptoServiceProvider(2048)); - tc.SetSingleton(keyManager.Object); - - tc.SetSingleton(new Mock().Object); - tc.SetSingleton(new Mock().Object); - - TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token); - Assert.Equal(arMessages[0], message1); - Assert.Equal(arMessages[1], message2); - Assert.Equal(arMessages[4], message3); - - //Assert - _runnerServer - .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token), Times.Exactly(arMessages.Length + 1)); - - _runnerServer - .Verify(x => x.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId), Times.Never); - - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Exactly(3)); - - newRunnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Never); - - newRunnerServer - .Verify(x => x.GetAgentPoolsAsync(null, TaskAgentPoolType.Automation), Times.Never); - - var tempLog = Path.GetTempFileName(); - File.Copy(tc.TraceFileName, tempLog, true); - var traceContent = File.ReadAllLines(tempLog); - Assert.Contains(traceContent, x => x.Contains("Fallback to original credentials and try again.")); - Assert.Contains(traceContent, x => x.Contains("Re-attempt to use migrated credential")); - - Assert.True(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Runner")] - public async void CreateSessionWithOriginalGetMessageWithOriginalEnvOverwrite() - { - try - { - Environment.SetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_SPSAUTHURL", "1"); - using (TestHostContext tc = CreateTestContext()) - using (var tokenSource = new CancellationTokenSource()) - { - Tracing trace = tc.GetTrace(); - - // Arrange. - var expectedSession = new TaskAgentSession(); - _runnerServer - .Setup(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token)) - .Returns(Task.FromResult(expectedSession)); - - var originalVssCred = new VssCredentials(); - var migratedVssCred = new VssCredentials(); - _credMgr.Setup(x => x.LoadCredentials(false)).Returns(originalVssCred); - _credMgr.Setup(x => x.LoadCredentials(true)).Returns(migratedVssCred); - - var originalCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - originalCred.Data["authorizationUrl"] = "https://s.server"; - originalCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - var migratedCred = new CredentialData() { Scheme = Constants.Configuration.OAuth }; - migratedCred.Data["authorizationUrl"] = "https://t.server"; - migratedCred.Data["clientId"] = "d842fd7b-61b0-4a80-96b4-f2797c353897"; - - _store.Setup(x => x.GetCredentials()).Returns(originalCred); - _store.Setup(x => x.GetMigratedCredentials()).Returns(migratedCred); - - // Act. - MessageListener listener = new MessageListener(); - listener.Initialize(tc); - - bool result = await listener.CreateSessionAsync(tokenSource.Token); - trace.Info("result: {0}", result); - - // Assert. - Assert.True(result); - _runnerServer - .Verify(x => x.CreateAgentSessionAsync( - _settings.PoolId, - It.Is(y => y != null), - tokenSource.Token), Times.Once()); - - Assert.False(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - - var arMessages = new TaskAgentMessage[] - { - new TaskAgentMessage - { - Body = "somebody1", - MessageId = 4234, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - }, - new TaskAgentMessage - { - Body = "somebody2", - MessageId = 4235, - MessageType = JobCancelMessage.MessageType - }, - null, //should be skipped by GetNextMessageAsync implementation - null, - new TaskAgentMessage - { - Body = "somebody3", - MessageId = 4236, - MessageType = JobRequestMessageTypes.PipelineAgentJobRequest - } - }; - var messages = new Queue(arMessages); - - _runnerServer - .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token)) - .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken) => - { - await Task.Delay(1); - return messages.Dequeue(); - }); - - TaskAgentMessage message1 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message2 = await listener.GetNextMessageAsync(tokenSource.Token); - TaskAgentMessage message3 = await listener.GetNextMessageAsync(tokenSource.Token); - Assert.Equal(arMessages[0], message1); - Assert.Equal(arMessages[1], message2); - Assert.Equal(arMessages[4], message3); - - //Assert - _runnerServer - .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), tokenSource.Token), Times.Exactly(arMessages.Length)); - - _runnerServer - .Verify(x => x.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId), Times.Never); - - _runnerServer - .Verify(x => x.ConnectAsync( - It.IsAny(), - It.IsAny()), Times.Once); - - Assert.False(listener._useMigratedCredentials); - Assert.False(listener._needToCheckAuthorizationUrlUpdate); - Assert.Null(listener._authorizationUrlRollbackReattemptDelayBackgroundTask); - Assert.Null(listener._authorizationUrlMigrationBackgroundTask); - } - } - finally - { - Environment.SetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_SPSAUTHURL", null); } } } From 3c5aef791c59de6474f7857b892f9ba555138810 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 5 Jun 2020 23:02:10 -0400 Subject: [PATCH 63/86] Fix null ref exception in SecretMasker caused by hashfiles timeout. (#516) --- src/Misc/dotnet-install.ps1 | 193 ++++++++++++++++++ src/Runner.Sdk/ProcessInvoker.cs | 14 +- .../Expressions/HashFilesFunction.cs | 34 ++- 3 files changed, 229 insertions(+), 12 deletions(-) diff --git a/src/Misc/dotnet-install.ps1 b/src/Misc/dotnet-install.ps1 index 16e9be8fed4..c0122cdc3b7 100644 --- a/src/Misc/dotnet-install.ps1 +++ b/src/Misc/dotnet-install.ps1 @@ -684,3 +684,196 @@ Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath Say "Installation finished" exit 0 + +# SIG # Begin signature block +# MIIjkQYJKoZIhvcNAQcCoIIjgjCCI34CAQExDzANBglghkgBZQMEAgEFADB5Bgor +# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG +# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAwp4UsNdAkvwY3 +# VhbuN9D6NGOz+qNqW2+62YubWa4qJaCCDYEwggX/MIID56ADAgECAhMzAAABh3IX +# chVZQMcJAAAAAAGHMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD +# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy +# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p +# bmcgUENBIDIwMTEwHhcNMjAwMzA0MTgzOTQ3WhcNMjEwMzAzMTgzOTQ3WjB0MQsw +# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u +# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy +# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +# AQDOt8kLc7P3T7MKIhouYHewMFmnq8Ayu7FOhZCQabVwBp2VS4WyB2Qe4TQBT8aB +# znANDEPjHKNdPT8Xz5cNali6XHefS8i/WXtF0vSsP8NEv6mBHuA2p1fw2wB/F0dH +# sJ3GfZ5c0sPJjklsiYqPw59xJ54kM91IOgiO2OUzjNAljPibjCWfH7UzQ1TPHc4d +# weils8GEIrbBRb7IWwiObL12jWT4Yh71NQgvJ9Fn6+UhD9x2uk3dLj84vwt1NuFQ +# itKJxIV0fVsRNR3abQVOLqpDugbr0SzNL6o8xzOHL5OXiGGwg6ekiXA1/2XXY7yV +# Fc39tledDtZjSjNbex1zzwSXAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE +# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUhov4ZyO96axkJdMjpzu2zVXOJcsw +# UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 +# ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDU4Mzg1MB8GA1UdIwQYMBaAFEhu +# ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu +# bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w +# Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 +# Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx +# MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAixmy +# S6E6vprWD9KFNIB9G5zyMuIjZAOuUJ1EK/Vlg6Fb3ZHXjjUwATKIcXbFuFC6Wr4K +# NrU4DY/sBVqmab5AC/je3bpUpjtxpEyqUqtPc30wEg/rO9vmKmqKoLPT37svc2NV +# BmGNl+85qO4fV/w7Cx7J0Bbqk19KcRNdjt6eKoTnTPHBHlVHQIHZpMxacbFOAkJr +# qAVkYZdz7ikNXTxV+GRb36tC4ByMNxE2DF7vFdvaiZP0CVZ5ByJ2gAhXMdK9+usx +# zVk913qKde1OAuWdv+rndqkAIm8fUlRnr4saSCg7cIbUwCCf116wUJ7EuJDg0vHe +# yhnCeHnBbyH3RZkHEi2ofmfgnFISJZDdMAeVZGVOh20Jp50XBzqokpPzeZ6zc1/g +# yILNyiVgE+RPkjnUQshd1f1PMgn3tns2Cz7bJiVUaqEO3n9qRFgy5JuLae6UweGf +# AeOo3dgLZxikKzYs3hDMaEtJq8IP71cX7QXe6lnMmXU/Hdfz2p897Zd+kU+vZvKI +# 3cwLfuVQgK2RZ2z+Kc3K3dRPz2rXycK5XCuRZmvGab/WbrZiC7wJQapgBodltMI5 +# GMdFrBg9IeF7/rP4EqVQXeKtevTlZXjpuNhhjuR+2DMt/dWufjXpiW91bo3aH6Ea +# jOALXmoxgltCp1K7hrS6gmsvj94cLRf50QQ4U8Qwggd6MIIFYqADAgECAgphDpDS +# AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK +# V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 +# IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 +# ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla +# MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS +# ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT +# H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB +# AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG +# OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S +# 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz +# y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 +# 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u +# M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 +# X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl +# XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP +# 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB +# l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF +# RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM +# CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ +# BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud +# DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO +# 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 +# LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y +# Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p +# Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y +# Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB +# FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw +# cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA +# XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY +# 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj +# 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd +# d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ +# Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf +# wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ +# aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j +# NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B +# xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 +# eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 +# r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I +# RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIVZjCCFWICAQEwgZUwfjELMAkG +# A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx +# HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z +# b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAYdyF3IVWUDHCQAAAAABhzAN +# BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor +# BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQga11B1DE+ +# y9z0lmEO+MC+bhXPKfWALB7Snkn7G/wCUncwQgYKKwYBBAGCNwIBDDE0MDKgFIAS +# AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN +# BgkqhkiG9w0BAQEFAASCAQBIgx+sFXkLXf7Xbx7opCD3uhpQGEQ4x/LsqTax0bu1 +# GC/cxiI+dodUz+T4hKj1ZQyUH0Zlce32GutY048O9tkr7fQyuohoFUgChdIATEOY +# qAIESFbDT07i7khJfO2pewlhgM+A5ClvBa8HAvV0wOd+2IVgv3pgow1LEJm0/5NB +# E3IFA+hFrqiWALOY0uUep4H20EHMrbqw3YoV3EodIkTj3fC76q4K/bF84EZLUgjY +# e4rmXac8n7A9qR18QzGl8usEJej4OHU4nlUT1J734m+AWIFmfb/Zr2MyXED0V4q4 +# Vbmw3O7xD9STeNYrn5RjPmGPEN04akHxhNUSqLIc9vxQoYIS8DCCEuwGCisGAQQB +# gjcDAwExghLcMIIS2AYJKoZIhvcNAQcCoIISyTCCEsUCAQMxDzANBglghkgBZQME +# AgEFADCCAVQGCyqGSIb3DQEJEAEEoIIBQwSCAT8wggE7AgEBBgorBgEEAYRZCgMB +# MDEwDQYJYIZIAWUDBAIBBQAEIPPK1A0D1n7ZEdgTjKPY4sWiOMtohMqGpFvG55NY +# SFHeAgZepuJh/dEYEjIwMjAwNTI5MTYyNzE1LjMxWjAEgAIB9KCB1KSB0TCBzjEL +# MAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1v +# bmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEpMCcGA1UECxMgTWlj +# cm9zb2Z0IE9wZXJhdGlvbnMgUHVlcnRvIFJpY28xJjAkBgNVBAsTHVRoYWxlcyBU +# U1MgRVNOOjYwQkMtRTM4My0yNjM1MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1T +# dGFtcCBTZXJ2aWNloIIORDCCBPUwggPdoAMCAQICEzMAAAEm37pLIrmCggcAAAAA +# ASYwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp +# bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw +# b3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAw +# HhcNMTkxMjE5MDExNDU5WhcNMjEwMzE3MDExNDU5WjCBzjELMAkGA1UEBhMCVVMx +# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT +# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEpMCcGA1UECxMgTWljcm9zb2Z0IE9wZXJh +# dGlvbnMgUHVlcnRvIFJpY28xJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNOOjYwQkMt +# RTM4My0yNjM1MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNl +# MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnjC+hpxO8w2VdBO18X8L +# Hk6XdfR9yNQ0y+MuBOY7n5YdgkVunvbk/f6q8UoNFAdYQjVLPSAHbi6tUMiNeMGH +# k1U0lUxAkja2W2/szj/ghuFklvfHNBbsuiUShlhRlqcFNS7KXL2iwKDijmOhWJPY +# a2bLEr4W/mQLbSXail5p6m138Ttx4MAVEzzuGI0Kwr8ofIL7z6zCeWDiBM57LrNC +# qHOA2wboeuMsG4O0Oz2LMAzBLbJZPRPnZAD2HdD4HUL2mzZ8wox74Mekb7RzrUP3 +# hiHpxXZceJvhIEKfAgVkB5kTZQnio8A1JijMjw8f4TmsJPdJWpi8ei73sexe8/Yj +# cwIDAQABo4IBGzCCARcwHQYDVR0OBBYEFEmrrB8XsH6YQo3RWKZfxqM0DmFBMB8G +# A1UdIwQYMBaAFNVjOlyKMZDzQ3t8RhvFM2hahW1VMFYGA1UdHwRPME0wS6BJoEeG +# RWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Rp +# bVN0YVBDQV8yMDEwLTA3LTAxLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUH +# MAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljVGltU3Rh +# UENBXzIwMTAtMDctMDEuY3J0MAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYIKwYB +# BQUHAwgwDQYJKoZIhvcNAQELBQADggEBAECW+51o6W/0J/O/npudfjVzMXq0u0cs +# HjqXpdRyH6o03jlmY5MXAui3cmPBKufijJxD2pMRPVMUNh3VA0PQuJeYrP06oFdq +# LpLxd3IJARm98vzaMgCz2nCwBDpe9X2M3Js9K1GAX+w4Az8N7J+Z6P1OD0VxHBdq +# eTaqDN1lk1vwagTN7t/WitxMXRDz0hRdYiWbATBAVgXXCOfzs3hnEv1n/EDab9HX +# OLMXKVY/+alqYKdV9lkuRp8Us1Q1WZy9z72Azu9x4mzft3fJ1puTjBHo5tHfixZo +# ummbI+WwjVCrku7pskJahfNi5amSgrqgR6nWAwvpJELccpVLdSxxmG0wggZxMIIE +# WaADAgECAgphCYEqAAAAAAACMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJV +# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE +# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9v +# dCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0xMDA3MDEyMTM2NTVaFw0y +# NTA3MDEyMTQ2NTVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u +# MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp +# b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIIBIjAN +# BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqR0NvHcRijog7PwTl/X6f2mUa3RU +# ENWlCgCChfvtfGhLLF/Fw+Vhwna3PmYrW/AVUycEMR9BGxqVHc4JE458YTBZsTBE +# D/FgiIRUQwzXTbg4CLNC3ZOs1nMwVyaCo0UN0Or1R4HNvyRgMlhgRvJYR4YyhB50 +# YWeRX4FUsc+TTJLBxKZd0WETbijGGvmGgLvfYfxGwScdJGcSchohiq9LZIlQYrFd +# /XcfPfBXday9ikJNQFHRD5wGPmd/9WbAA5ZEfu/QS/1u5ZrKsajyeioKMfDaTgaR +# togINeh4HLDpmc085y9Euqf03GS9pAHBIAmTeM38vMDJRF1eFpwBBU8iTQIDAQAB +# o4IB5jCCAeIwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFNVjOlyKMZDzQ3t8 +# RhvFM2hahW1VMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1UdDwQEAwIB +# hjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2VsuP6KJcYmjRPZSQW9fO +# mhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9w +# a2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNybDBaBggr +# BgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNv +# bS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3J0MIGgBgNVHSAB +# Af8EgZUwgZIwgY8GCSsGAQQBgjcuAzCBgTA9BggrBgEFBQcCARYxaHR0cDovL3d3 +# dy5taWNyb3NvZnQuY29tL1BLSS9kb2NzL0NQUy9kZWZhdWx0Lmh0bTBABggrBgEF +# BQcCAjA0HjIgHQBMAGUAZwBhAGwAXwBQAG8AbABpAGMAeQBfAFMAdABhAHQAZQBt +# AGUAbgB0AC4gHTANBgkqhkiG9w0BAQsFAAOCAgEAB+aIUQ3ixuCYP4FxAz2do6Eh +# b7Prpsz1Mb7PBeKp/vpXbRkws8LFZslq3/Xn8Hi9x6ieJeP5vO1rVFcIK1GCRBL7 +# uVOMzPRgEop2zEBAQZvcXBf/XPleFzWYJFZLdO9CEMivv3/Gf/I3fVo/HPKZeUqR +# UgCvOA8X9S95gWXZqbVr5MfO9sp6AG9LMEQkIjzP7QOllo9ZKby2/QThcJ8ySif9 +# Va8v/rbljjO7Yl+a21dA6fHOmWaQjP9qYn/dxUoLkSbiOewZSnFjnXshbcOco6I8 +# +n99lmqQeKZt0uGc+R38ONiU9MalCpaGpL2eGq4EQoO4tYCbIjggtSXlZOz39L9+ +# Y1klD3ouOVd2onGqBooPiRa6YacRy5rYDkeagMXQzafQ732D8OE7cQnfXXSYIghh +# 2rBQHm+98eEA3+cxB6STOvdlR3jo+KhIq/fecn5ha293qYHLpwmsObvsxsvYgrRy +# zR30uIUBHoD7G4kqVDmyW9rIDVWZeodzOwjmmC3qjeAzLhIp9cAvVCch98isTtoo +# uLGp25ayp0Kiyc8ZQU3ghvkqmqMRZjDTu3QyS99je/WZii8bxyGvWbWu3EQ8l1Bx +# 16HSxVXjad5XwdHeMMD9zOZN+w2/XU/pnR4ZOC+8z1gFLu8NoFA12u8JJxzVs341 +# Hgi62jbb01+P3nSISRKhggLSMIICOwIBATCB/KGB1KSB0TCBzjELMAkGA1UEBhMC +# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV +# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEpMCcGA1UECxMgTWljcm9zb2Z0IE9w +# ZXJhdGlvbnMgUHVlcnRvIFJpY28xJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNOOjYw +# QkMtRTM4My0yNjM1MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2 +# aWNloiMKAQEwBwYFKw4DAhoDFQAKZzI5aZnESumrToHx3Lqgxnr//KCBgzCBgKR+ +# MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS +# ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMT +# HU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMA0GCSqGSIb3DQEBBQUAAgUA +# 4nuQTDAiGA8yMDIwMDUyOTE3NDQ0NFoYDzIwMjAwNTMwMTc0NDQ0WjB3MD0GCisG +# AQQBhFkKBAExLzAtMAoCBQDie5BMAgEAMAoCAQACAiZJAgH/MAcCAQACAhEjMAoC +# BQDifOHMAgEAMDYGCisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEA +# AgMHoSChCjAIAgEAAgMBhqAwDQYJKoZIhvcNAQEFBQADgYEAprmyJTXdH9FmQZ0I +# mRSJdjc/RrSqDm8DUEq/h3FL73G/xvg9MbQj1J/h3hdlSIPcQXjrhL8hud/vyF0j +# IFaTK5YOcixkX++9t7Vz3Mn0KkQo8F4DNSyZEPpz682AyKKwLMJDy52pFFFKNP5l +# NpOz6YY1Od1xvk4nyN1WwfLnGswxggMNMIIDCQIBATCBkzB8MQswCQYDVQQGEwJV +# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE +# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGlt +# ZS1TdGFtcCBQQ0EgMjAxMAITMwAAASbfuksiuYKCBwAAAAABJjANBglghkgBZQME +# AgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJ +# BDEiBCB0IE0Q6P23RQlh8TFyp57UQQUF/sbui7mOMStRgTFZxTCB+gYLKoZIhvcN +# AQkQAi8xgeowgecwgeQwgb0EIDb9z++evV5wDO9qk5ZnbEZ8CTOuR+kZyu8xbTsJ +# CXUPMIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x +# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv +# bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAEm +# 37pLIrmCggcAAAAAASYwIgQgtwi02bvsGAOdpAxEF607G6g9PlyS8vc2bAUSHovH +# /IIwDQYJKoZIhvcNAQELBQAEggEAEMCfsXNudrjztjI6JNyNDVpdF1axRVcGiNy6 +# 67pgb1EePsjA2EaBB+5ZjgO/73JxuiVgsoXgH7em8tKG5RQJtcm5obVDb+jKksK4 +# qcFLA1f7seQRGfE06UAPnSFh2GqMtTNJGCXWwqWLH2LduTjOqPt8Nupo16ABFIT2 +# akTzBSJ81EHBkEU0Et6CgeaZiBYrCCXUtD+ASvLDkPSrjweQGu3Zk1SSROEzxMY9 +# jdlGfMkK2krMd9ub9UZ13RcQDijJqo+h6mz76pAuiFFvuQl6wMoSGFaaUQwfd+WQ +# gXlVVX/A9JFBihrxnDVglEPlsIOxCHkTeIxLfnAkCbax+9pevA== +# SIG # End signature block diff --git a/src/Runner.Sdk/ProcessInvoker.cs b/src/Runner.Sdk/ProcessInvoker.cs index a56b1475ae7..4ed4ce3b0d3 100644 --- a/src/Runner.Sdk/ProcessInvoker.cs +++ b/src/Runner.Sdk/ProcessInvoker.cs @@ -318,7 +318,12 @@ public async Task ExecuteAsync( } } - using (var registration = cancellationToken.Register(async () => await CancelAndKillProcessTree(killProcessOnCancel))) + var cancellationFinished = new TaskCompletionSource(); + using (var registration = cancellationToken.Register(async () => + { + await CancelAndKillProcessTree(killProcessOnCancel); + cancellationFinished.TrySetResult(true); + })) { Trace.Info($"Process started with process id {_proc.Id}, waiting for process exit."); while (true) @@ -344,6 +349,13 @@ public async Task ExecuteAsync( Trace.Info($"Finished process {_proc.Id} with exit code {_proc.ExitCode}, and elapsed time {_stopWatch.Elapsed}."); } + if (cancellationToken.IsCancellationRequested) + { + // Ensure cancellation also finish on the cancellationToken.Register thread. + await cancellationFinished.Task; + Trace.Info($"Process Cancellation finished."); + } + cancellationToken.ThrowIfCancellationRequested(); // Wait for process to finish. diff --git a/src/Runner.Worker/Expressions/HashFilesFunction.cs b/src/Runner.Worker/Expressions/HashFilesFunction.cs index ecbe00ce2cd..a19f13e3954 100644 --- a/src/Runner.Worker/Expressions/HashFilesFunction.cs +++ b/src/Runner.Worker/Expressions/HashFilesFunction.cs @@ -12,6 +12,8 @@ namespace GitHub.Runner.Worker.Expressions { public sealed class HashFilesFunction : Function { + private const int _hashFileTimeoutSeconds = 120; + protected sealed override Object EvaluateCore( EvaluationContext context, out ResultMemory resultMemory) @@ -89,19 +91,29 @@ protected sealed override Object EvaluateCore( } env["patterns"] = string.Join(Environment.NewLine, patterns); - int exitCode = p.ExecuteAsync(workingDirectory: githubWorkspace, - fileName: node, - arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"", - environment: env, - requireExitCodeZero: false, - cancellationToken: new CancellationTokenSource(TimeSpan.FromSeconds(120)).Token).GetAwaiter().GetResult(); - - if (exitCode != 0) + using (var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(_hashFileTimeoutSeconds))) { - throw new InvalidOperationException($"hashFiles('{ExpressionUtility.StringEscape(string.Join(", ", patterns))}') failed. Fail to hash files under directory '{githubWorkspace}'"); - } + try + { + int exitCode = p.ExecuteAsync(workingDirectory: githubWorkspace, + fileName: node, + arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"", + environment: env, + requireExitCodeZero: false, + cancellationToken: tokenSource.Token).GetAwaiter().GetResult(); - return hashResult; + if (exitCode != 0) + { + throw new InvalidOperationException($"hashFiles('{ExpressionUtility.StringEscape(string.Join(", ", patterns))}') failed. Fail to hash files under directory '{githubWorkspace}'"); + } + } + catch (OperationCanceledException) when (tokenSource.IsCancellationRequested) + { + throw new TimeoutException($"hashFiles('{ExpressionUtility.StringEscape(string.Join(", ", patterns))}') couldn't finish within {_hashFileTimeoutSeconds} seconds."); + } + + return hashResult; + } } private sealed class HashFilesTrace : ITraceWriter From f994ae0542fd8016812dbbf10bdf712d87eb0766 Mon Sep 17 00:00:00 2001 From: Nick Fields <50085412+nick-invision@users.noreply.github.com> Date: Fri, 5 Jun 2020 23:09:14 -0400 Subject: [PATCH 64/86] Reduce input validation warnings (#506) * Only raise a single warning for unexpected inputs * Update invalid input test to raise single warning --- src/Runner.Worker/ActionRunner.cs | 8 +++++++- src/Test/L0/Worker/ActionRunnerL0.cs | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index 0c4cc76ce59..81ecf1a1493 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -187,13 +187,19 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && // Validate inputs only for actions with action.yml if (Action.Reference.Type == Pipelines.ActionSourceType.Repository) { + var unexpectedInputs = new List(); foreach (var input in userInputs) { if (!validInputs.Contains(input)) { - ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']"); + unexpectedInputs.Add(input); } } + + if (unexpectedInputs.Count > 0) + { + ExecutionContext.Warning($"Unexpected input(s) '{string.Join("', '", unexpectedInputs)}', valid inputs are ['{string.Join("', '", validInputs)}']"); + } } // Load the action environment. diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index f8185d26dd3..662ea307d61 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -328,8 +328,7 @@ public async void WarnInvalidInputs() Assert.Equal("invalid1", finialInputs["invalid1"]); Assert.Equal("invalid2", finialInputs["invalid2"]); - _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Unexpected input 'invalid1'")), It.IsAny()), Times.Once); - _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Unexpected input 'invalid2'")), It.IsAny()), Times.Once); + _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Unexpected input(s) 'invalid1', 'invalid2'")), It.IsAny()), Times.Once); } private void Setup([CallerMemberName] string name = "") From eda463601cee7b9cdf28ea6a5fb995b00d9aa86e Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Mon, 8 Jun 2020 10:19:17 -0400 Subject: [PATCH 65/86] Update Links and Language to Git + VSCode (#522) --- docs/contribute.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contribute.md b/docs/contribute.md index 8d68a446f15..d330140023d 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -23,7 +23,7 @@ An ADR is an Architectural Decision Record. This allows consensus on the direct ### Required Dev Dependencies -![Win](res/win_sm.png) Git for Windows [Install Here](https://git-scm.com/downloads) (needed for dev sh script) +![Win](res/win_sm.png) ![*nix](res/linux_sm.png) Git for Windows and Linux [Install Here](https://git-scm.com/downloads) (needed for dev sh script) ### To Build, Test, Layout @@ -53,7 +53,7 @@ cd ./src ### Editors [Using Visual Studio Code](https://code.visualstudio.com/) -[Using Visual Studio 2019](https://www.visualstudio.com/vs/) +[Using Visual Studio](https://code.visualstudio.com/docs) ### Styling From 1aea04693201250b5fa8e192fce500421d26156a Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Mon, 8 Jun 2020 10:47:58 -0400 Subject: [PATCH 66/86] Add substep for developer flow for clarity (#523) --- docs/contribute.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/contribute.md b/docs/contribute.md index d330140023d..9c094ad542a 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -43,6 +43,7 @@ Sample developer flow: ```bash git clone https://github.com/actions/runner +cd runner cd ./src ./dev.(sh/cmd) layout # the runner that built from source is in {root}/_layout From 5815819f248b4f9c3fc45f7adfdf5d6b5725195f Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 9 Jun 2020 08:53:28 -0400 Subject: [PATCH 67/86] Resolve action download info (#515) --- src/Runner.Common/JobServer.cs | 10 ++ src/Runner.Worker/ActionManager.cs | 167 +++++++++--------- src/Runner.Worker/ExecutionContext.cs | 6 +- .../Generated/TaskHttpClientBase.cs | 32 ++++ src/Sdk/DTWebApi/WebApi/ActionDownloadInfo.cs | 40 +++++ .../WebApi/ActionDownloadInfoCollection.cs | 16 ++ src/Sdk/DTWebApi/WebApi/ActionReference.cs | 22 +++ .../DTWebApi/WebApi/ActionReferenceList.cs | 16 ++ src/Test/L0/Worker/ActionManagerL0.cs | 22 +++ 9 files changed, 250 insertions(+), 81 deletions(-) create mode 100644 src/Sdk/DTWebApi/WebApi/ActionDownloadInfo.cs create mode 100644 src/Sdk/DTWebApi/WebApi/ActionDownloadInfoCollection.cs create mode 100644 src/Sdk/DTWebApi/WebApi/ActionReference.cs create mode 100644 src/Sdk/DTWebApi/WebApi/ActionReferenceList.cs diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 86055541926..e3e0f551b8d 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -22,6 +22,7 @@ public interface IJobServer : IRunnerService Task> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable records, CancellationToken cancellationToken); Task RaisePlanEventAsync(Guid scopeIdentifier, string hubName, Guid planId, T eventData, CancellationToken cancellationToken) where T : JobEvent; Task GetTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); + Task ResolveActionDownloadInfoAsync(Guid scopeIdentifier, string hubName, Guid planId, ActionReferenceList actions, CancellationToken cancellationToken); } public sealed class JobServer : RunnerService, IJobServer @@ -113,5 +114,14 @@ public Task GetTimelineAsync(Guid scopeIdentifier, string hubName, Gui CheckConnection(); return _taskClient.GetTimelineAsync(scopeIdentifier, hubName, planId, timelineId, includeRecords: true, cancellationToken: cancellationToken); } + + //----------------------------------------------------------------- + // Action download info + //----------------------------------------------------------------- + public Task ResolveActionDownloadInfoAsync(Guid scopeIdentifier, string hubName, Guid planId, ActionReferenceList actions, CancellationToken cancellationToken) + { + CheckConnection(); + return _taskClient.ResolveActionDownloadInfoAsync(scopeIdentifier, hubName, planId, actions, cancellationToken: cancellationToken); + } } } diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 3004e680290..347eb3d0d03 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -14,6 +14,7 @@ using GitHub.Runner.Sdk; using GitHub.Runner.Worker.Container; using GitHub.Services.Common; +using WebApi = GitHub.DistributedTask.WebApi; using Pipelines = GitHub.DistributedTask.Pipelines; using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; @@ -546,10 +547,10 @@ private async Task BuildActionContainerAsync(IExecutionContext executionContext, } // This implementation is temporary and will be removed when we switch to a REST API call to the service to resolve the download info - private async Task RepoExistsAsync(IExecutionContext executionContext, Pipelines.RepositoryPathReference repositoryReference, string authorization) + private async Task RepoExistsAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo actionDownloadInfo, string token) { var apiUrl = GetApiUrl(executionContext); - var repoUrl = $"{apiUrl}/repos/{repositoryReference.Name}"; + var repoUrl = $"{apiUrl}/repos/{actionDownloadInfo.NameWithOwner}"; for (var attempt = 1; attempt <= 3; attempt++) { executionContext.Debug($"Checking whether repo exists: {repoUrl}"); @@ -558,7 +559,7 @@ private async Task RepoExistsAsync(IExecutionContext executionContext, Pip using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { - httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(authorization); + httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(token); httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); using (var response = await httpClient.GetAsync(repoUrl)) { @@ -582,11 +583,11 @@ private async Task RepoExistsAsync(IExecutionContext executionContext, Pip { if (attempt < 3) { - executionContext.Debug($"Failed checking whether repo '{repositoryReference.Name}' exists: {ex.Message}"); + executionContext.Debug($"Failed checking whether repo '{actionDownloadInfo.NameWithOwner}' exists: {ex.Message}"); } else { - executionContext.Error($"Failed checking whether repo '{repositoryReference.Name}' exists: {ex.Message}"); + executionContext.Error($"Failed checking whether repo '{actionDownloadInfo.NameWithOwner}' exists: {ex.Message}"); throw; } } @@ -596,73 +597,89 @@ private async Task RepoExistsAsync(IExecutionContext executionContext, Pip } // This implementation is temporary and will be replaced with a REST API call to the service to resolve - private async Task> GetDownloadInfoAsync(IExecutionContext executionContext, List actions) + private async Task> GetDownloadInfoAsync(IExecutionContext executionContext, List actions) { - var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + executionContext.Output("Getting action download info"); - var configurationStore = HostContext.GetService(); - var runnerSettings = configurationStore.GetSettings(); - var apiUrl = GetApiUrl(executionContext); - var accessToken = executionContext.GetGitHubContext("token"); - var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); - var authorization = $"Basic {base64EncodingToken}"; + // Convert to action reference + var actionReferences = actions + .GroupBy(x => GetDownloadInfoLookupKey(x)) + .Where(x => !string.IsNullOrEmpty(x.Key)) + .Select(x => + { + var action = x.First(); + var repositoryReference = action.Reference as Pipelines.RepositoryPathReference; + ArgUtil.NotNull(repositoryReference, nameof(repositoryReference)); + return new WebApi.ActionReference + { + NameWithOwner = repositoryReference.Name, + Ref = repositoryReference.Ref, + }; + }) + .ToList(); - foreach (var action in actions) + // Nothing to resolve? + if (actionReferences.Count == 0) { - var lookupKey = GetDownloadInfoLookupKey(action); - if (string.IsNullOrEmpty(lookupKey) || result.ContainsKey(lookupKey)) + return new Dictionary(); + } + + // Resolve download info + var jobServer = HostContext.GetService(); + var actionDownloadInfos = default(WebApi.ActionDownloadInfoCollection); + for (var attempt = 1; attempt <= 3; attempt++) + { + try { - continue; + actionDownloadInfos = await jobServer.ResolveActionDownloadInfoAsync(executionContext.Plan.ScopeIdentifier, executionContext.Plan.PlanType, executionContext.Plan.PlanId, new WebApi.ActionReferenceList { Actions = actionReferences }, executionContext.CancellationToken); + break; } + catch (Exception ex) when (attempt < 3) + { + executionContext.Output($"Failed to resolve action download info. Error: {ex.Message}"); + executionContext.Debug(ex.ToString()); + if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF"))) + { + var backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); + executionContext.Output($"Retrying in {backoff.TotalSeconds} seconds"); + await Task.Delay(backoff); + } + } + } - var repositoryReference = action.Reference as Pipelines.RepositoryPathReference; - ArgUtil.NotNull(repositoryReference, nameof(repositoryReference)); + ArgUtil.NotNull(actionDownloadInfos, nameof(actionDownloadInfos)); + ArgUtil.NotNull(actionDownloadInfos.Actions, nameof(actionDownloadInfos.Actions)); + var apiUrl = GetApiUrl(executionContext); + var defaultAccessToken = executionContext.GetGitHubContext("token"); + var configurationStore = HostContext.GetService(); + var runnerSettings = configurationStore.GetSettings(); - var downloadInfo = default(ActionDownloadInfo); + foreach (var actionDownloadInfo in actionDownloadInfos.Actions.Values) + { + // Add secret + HostContext.SecretMasker.AddValue(actionDownloadInfo.Authentication?.Token); + // Temporary code: Fix token and download URL if (runnerSettings.IsHostedServer) { - downloadInfo = new ActionDownloadInfo - { - NameWithOwner = repositoryReference.Name, - Ref = repositoryReference.Ref, - ArchiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref), - Authorization = authorization, - }; + actionDownloadInfo.Authentication = new WebApi.ActionDownloadAuthentication { Token = defaultAccessToken }; + actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", apiUrl); + actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", apiUrl); } - // Test whether the repo exists in the instance - else if (await RepoExistsAsync(executionContext, repositoryReference, authorization)) + else if (await RepoExistsAsync(executionContext, actionDownloadInfo, defaultAccessToken)) { - downloadInfo = new ActionDownloadInfo - { - NameWithOwner = repositoryReference.Name, - Ref = repositoryReference.Ref, - ArchiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref), - Authorization = authorization, - }; + actionDownloadInfo.Authentication = new WebApi.ActionDownloadAuthentication { Token = defaultAccessToken }; + actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", apiUrl); + actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", apiUrl); } - // Fallback to dotcom else { - downloadInfo = new ActionDownloadInfo - { - NameWithOwner = repositoryReference.Name, - Ref = repositoryReference.Ref, - ArchiveLink = BuildLinkToActionArchive(_dotcomApiUrl, repositoryReference.Name, repositoryReference.Ref), - Authorization = null, - }; + actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", "https://api.github.com"); + actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", "https://api.github.com"); } - - result.Add(lookupKey, downloadInfo); - } - - // Register secrets - foreach (var downloadInfo in result.Values) - { - HostContext.SecretMasker.AddValue(downloadInfo.Authorization); } - return result; + return actionDownloadInfos.Actions; } // todo: Remove when feature flag DistributedTask.NewActionMetadata is removed @@ -709,7 +726,6 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont { string apiUrl = GetApiUrl(executionContext); string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref); - Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); var downloadDetails = new ActionDownloadDetails(archiveLink, ConfigureAuthorizationFromContext); await DownloadRepositoryActionAsync(executionContext, downloadDetails, null, destDirectory); return; @@ -735,7 +751,6 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont foreach (var downloadAttempt in downloadAttempts) { - Trace.Info($"Download archive '{downloadAttempt.ArchiveLink}' to '{destDirectory}'."); try { await DownloadRepositoryActionAsync(executionContext, downloadAttempt, null, destDirectory); @@ -751,7 +766,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont } } - private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadInfo downloadInfo) + private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo downloadInfo) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); @@ -774,7 +789,6 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}'"); } - Trace.Info($"Download archive '{downloadInfo.ArchiveLink}' to '{destDirectory}'."); await DownloadRepositoryActionAsync(executionContext, null, downloadInfo, destDirectory); } @@ -799,7 +813,7 @@ private static string BuildLinkToActionArchive(string apiUrl, string repository, } // todo: Remove the parameter "actionDownloadDetails" when feature flag DistributedTask.NewActionMetadata is removed - private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, ActionDownloadInfo downloadInfo, string destDirectory) + private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, WebApi.ActionDownloadInfo downloadInfo, string destDirectory) { //download and extract action in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); @@ -807,11 +821,12 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont #if OS_WINDOWS string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip"); + string link = downloadInfo?.ZipballUrl ?? actionDownloadDetails.ArchiveLink; #else string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz"); + string link = downloadInfo?.TarballUrl ?? actionDownloadDetails.ArchiveLink; #endif - string link = downloadInfo != null ? downloadInfo.ArchiveLink : actionDownloadDetails.ArchiveLink; Trace.Info($"Save archive '{link}' into {archiveFile}."); try { @@ -839,7 +854,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont // FF DistributedTask.NewActionMetadata else { - httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(downloadInfo.Authorization); + httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(downloadInfo.Authentication?.Token); } httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); @@ -1137,20 +1152,23 @@ private static string GetDownloadInfoLookupKey(Pipelines.ActionStep action) return $"{repositoryReference.Name}@{repositoryReference.Ref}"; } - private static AuthenticationHeaderValue CreateAuthHeader(string authorization) + private static string GetDownloadInfoLookupKey(WebApi.ActionDownloadInfo info) { - if (string.IsNullOrEmpty(authorization)) - { - return null; - } + ArgUtil.NotNullOrEmpty(info.NameWithOwner, nameof(info.NameWithOwner)); + ArgUtil.NotNullOrEmpty(info.Ref, nameof(info.Ref)); + return $"{info.NameWithOwner}@{info.Ref}"; + } - var split = authorization.Split(new char[] { ' ' }, 2); - if (split.Length != 2 || string.IsNullOrWhiteSpace(split[0]) || string.IsNullOrWhiteSpace(split[1])) + private AuthenticationHeaderValue CreateAuthHeader(string token) + { + if (string.IsNullOrEmpty(token)) { - throw new Exception("Unexpected authorization header format"); + return null; } - return new AuthenticationHeaderValue(split[0].Trim(), split[1].Trim()); + var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{token}")); + HostContext.SecretMasker.AddValue(base64EncodingToken); + return new AuthenticationHeaderValue("Basic", base64EncodingToken); } // todo: Remove when feature flag DistributedTask.NewActionMetadata is removed @@ -1166,17 +1184,6 @@ public ActionDownloadDetails(string archiveLink, Action Endpoints { get; } + TaskOrchestrationPlanReference Plan { get; } PlanFeatures Features { get; } Variables Variables { get; } @@ -141,6 +142,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public Task ForceCompleted => _forceCompleted.Task; public CancellationToken CancellationToken => _cancellationTokenSource.Token; public List Endpoints { get; private set; } + public TaskOrchestrationPlanReference Plan { get; private set; } public Variables Variables { get; private set; } public Dictionary IntraActionState { get; private set; } public IDictionary> JobDefaults { get; private set; } @@ -275,6 +277,7 @@ public IExecutionContext CreateChild(Guid recordId, string displayName, string r child.Features = Features; child.Variables = Variables; child.Endpoints = Endpoints; + child.Plan = Plan; if (intraActionState == null) { child.IntraActionState = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -576,7 +579,8 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); - // Features + // Plan + Plan = message.Plan; Features = PlanUtil.GetFeatures(message.Plan); // Endpoints diff --git a/src/Sdk/DTGenerated/Generated/TaskHttpClientBase.cs b/src/Sdk/DTGenerated/Generated/TaskHttpClientBase.cs index 867b4f927ea..91adde9256a 100644 --- a/src/Sdk/DTGenerated/Generated/TaskHttpClientBase.cs +++ b/src/Sdk/DTGenerated/Generated/TaskHttpClientBase.cs @@ -317,5 +317,37 @@ public virtual Task GetTimelineAsync( userState: userState, cancellationToken: cancellationToken); } + + /// + /// [Preview API] Resolves information required to download actions (URL, token) defined in an orchestration. + /// + /// The project GUID to scope the request + /// The name of the server hub: "build" for the Build server or "rm" for the Release Management server + /// + /// + /// + /// The cancellation token to cancel operation. + public virtual Task ResolveActionDownloadInfoAsync( + Guid scopeIdentifier, + string hubName, + Guid planId, + ActionReferenceList actionReferenceList, + object userState = null, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("POST"); + Guid locationId = new Guid("27d7f831-88c1-4719-8ca1-6a061dad90eb"); + object routeValues = new { scopeIdentifier = scopeIdentifier, hubName = hubName, planId = planId }; + HttpContent content = new ObjectContent(actionReferenceList, new VssJsonMediaTypeFormatter(true)); + + return SendAsync( + httpMethod, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(6.0, 1), + userState: userState, + cancellationToken: cancellationToken, + content: content); + } } } diff --git a/src/Sdk/DTWebApi/WebApi/ActionDownloadInfo.cs b/src/Sdk/DTWebApi/WebApi/ActionDownloadInfo.cs new file mode 100644 index 00000000000..a6b0749f65a --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/ActionDownloadInfo.cs @@ -0,0 +1,40 @@ +using System; +using System.Runtime.Serialization; + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public class ActionDownloadInfo + { + [DataMember(EmitDefaultValue = false)] + public ActionDownloadAuthentication Authentication { get; set; } + + [DataMember(EmitDefaultValue = false)] + public string NameWithOwner { get; set; } + + [DataMember(EmitDefaultValue = false)] + public string ResolvedNameWithOwner { get; set; } + + [DataMember(EmitDefaultValue = false)] + public string ResolvedSha { get; set; } + + [DataMember(EmitDefaultValue = false)] + public string TarballUrl { get; set; } + + [DataMember(EmitDefaultValue = false)] + public string Ref { get; set; } + + [DataMember(EmitDefaultValue = false)] + public string ZipballUrl { get; set; } + } + + [DataContract] + public class ActionDownloadAuthentication + { + [DataMember(EmitDefaultValue = false)] + public DateTime ExpiresAt { get; set; } + + [DataMember(EmitDefaultValue = false)] + public string Token { get; set; } + } +} diff --git a/src/Sdk/DTWebApi/WebApi/ActionDownloadInfoCollection.cs b/src/Sdk/DTWebApi/WebApi/ActionDownloadInfoCollection.cs new file mode 100644 index 00000000000..1367bf86028 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/ActionDownloadInfoCollection.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public class ActionDownloadInfoCollection + { + [DataMember] + public IDictionary Actions + { + get; + set; + } + } +} diff --git a/src/Sdk/DTWebApi/WebApi/ActionReference.cs b/src/Sdk/DTWebApi/WebApi/ActionReference.cs new file mode 100644 index 00000000000..c6ea8a9eda3 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/ActionReference.cs @@ -0,0 +1,22 @@ +using System.Runtime.Serialization; + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public class ActionReference + { + [DataMember] + public string NameWithOwner + { + get; + set; + } + + [DataMember] + public string Ref + { + get; + set; + } + } +} diff --git a/src/Sdk/DTWebApi/WebApi/ActionReferenceList.cs b/src/Sdk/DTWebApi/WebApi/ActionReferenceList.cs new file mode 100644 index 00000000000..b118b99040e --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/ActionReferenceList.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public class ActionReferenceList + { + [DataMember] + public IList Actions + { + get; + set; + } + } +} diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index fb20b23568d..85851a55ac1 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -28,6 +28,7 @@ public sealed class ActionManagerL0 private Mock _configurationStore; private Mock _dockerManager; private Mock _ec; + private Mock _jobServer; private Mock _pluginManager; private TestHostContext _hc; private ActionManager _actionManager; @@ -3583,6 +3584,7 @@ private void Setup([CallerMemberName] string name = "", bool newActionMetadata = _ec.Setup(x => x.Variables).Returns(new Variables(_hc, variables)); _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); + _ec.Setup(x => x.Plan).Returns(new TaskOrchestrationPlanReference()); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"[{tag}]{message}"); }); _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, string message) => { _hc.GetTrace().Info($"[{issue.Type}]{issue.Message ?? message}"); }); _ec.Setup(x => x.GetGitHubContext("workspace")).Returns(Path.Combine(_workFolder, "actions", "actions")); @@ -3593,6 +3595,25 @@ private void Setup([CallerMemberName] string name = "", bool newActionMetadata = _dockerManager.Setup(x => x.DockerBuild(_ec.Object, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(0)); + _jobServer = new Mock(); + _jobServer.Setup(x => x.ResolveActionDownloadInfoAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((Guid scopeIdentifier, string hubName, Guid planId, ActionReferenceList actions, CancellationToken cancellationToken) => + { + var result = new ActionDownloadInfoCollection { Actions = new Dictionary() }; + foreach (var action in actions.Actions) + { + var key = $"{action.NameWithOwner}@{action.Ref}"; + result.Actions[key] = new ActionDownloadInfo + { + NameWithOwner = action.NameWithOwner, + Ref = action.Ref, + TarballUrl = $"/repos/{action.NameWithOwner}/tarball/{action.Ref}", + ZipballUrl = $"/repos/{action.NameWithOwner}/zipball/{action.Ref}", + }; + } + return Task.FromResult(result); + }); + _pluginManager = new Mock(); _pluginManager.Setup(x => x.GetPluginAction(It.IsAny())).Returns(new RunnerPluginActionInfo() { PluginTypeName = "plugin.class, plugin", PostPluginTypeName = "plugin.cleanup, plugin" }); @@ -3600,6 +3621,7 @@ private void Setup([CallerMemberName] string name = "", bool newActionMetadata = actionManifest.Initialize(_hc); _hc.SetSingleton(_dockerManager.Object); + _hc.SetSingleton(_jobServer.Object); _hc.SetSingleton(_pluginManager.Object); _hc.SetSingleton(actionManifest); _hc.SetSingleton(new HttpClientHandlerFactory()); From eaf39bb05820924b4d49c99285ee3d18dcbd25c7 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 12:11:13 -0400 Subject: [PATCH 68/86] add libicu66 for Ubuntu 20.04 (#535) --- src/Misc/layoutbin/installdependencies.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Misc/layoutbin/installdependencies.sh b/src/Misc/layoutbin/installdependencies.sh index 50a19982e0c..78671492516 100755 --- a/src/Misc/layoutbin/installdependencies.sh +++ b/src/Misc/layoutbin/installdependencies.sh @@ -70,8 +70,8 @@ then exit 1 fi - # libicu version prefer: libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52 - apt install -y libicu63 || apt install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52 + # libicu version prefer: libicu66 -> libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52 + apt install -y libicu66 || apt install -y libicu63 || apt install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52 if [ $? -ne 0 ] then echo "'apt' failed with exit code '$?'" @@ -99,8 +99,8 @@ then exit 1 fi - # libicu version prefer: libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52 - apt-get install -y libicu63 || apt-get install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52 + # libicu version prefer: libicu66 -> libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52 + apt-get install -y libicu66 || apt-get install -y libicu63 || apt-get install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52 if [ $? -ne 0 ] then echo "'apt-get' failed with exit code '$?'" From 312c7668a88daec33adad05d4682b3ffb44e2bdd Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 12:11:35 -0400 Subject: [PATCH 69/86] Fix DataContract with Token service (#532) --- src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs index fab331726cb..8ecf34271bd 100644 --- a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs +++ b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthTokenResponse.cs @@ -39,7 +39,7 @@ public String Error /// /// Gets or sets the error description for the response. /// - [DataMember(Name = "errordescription", EmitDefaultValue = false)] + [DataMember(Name = "error_description", EmitDefaultValue = false)] public String ErrorDescription { get; From 2e800f857e6a49e9d3095c38f96d5c9fb475c5b3 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 13:52:42 -0400 Subject: [PATCH 70/86] Skip search $PATH on command with fully qualified path (#526) --- src/Runner.Sdk/Util/WhichUtil.cs | 5 +++++ src/Test/L0/Util/WhichUtilL0.cs | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/Runner.Sdk/Util/WhichUtil.cs b/src/Runner.Sdk/Util/WhichUtil.cs index 71acd92d9e8..b4d94e51354 100644 --- a/src/Runner.Sdk/Util/WhichUtil.cs +++ b/src/Runner.Sdk/Util/WhichUtil.cs @@ -11,6 +11,11 @@ public static string Which(string command, bool require = false, ITraceWriter tr { ArgUtil.NotNullOrEmpty(command, nameof(command)); trace?.Info($"Which: '{command}'"); + if (Path.IsPathFullyQualified(command) && File.Exists(command)) + { + trace?.Info($"Fully qualified path: '{command}'"); + return command; + } string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable); if (string.IsNullOrEmpty(path)) { diff --git a/src/Test/L0/Util/WhichUtilL0.cs b/src/Test/L0/Util/WhichUtilL0.cs index 99e4a92a5ee..7271bc283bd 100644 --- a/src/Test/L0/Util/WhichUtilL0.cs +++ b/src/Test/L0/Util/WhichUtilL0.cs @@ -70,5 +70,24 @@ public void WhichThrowsWhenRequireAndNotFound() } } } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void WhichHandleFullyQualifiedPath() + { + using (TestHostContext hc = new TestHostContext(this)) + { + //Arrange + Tracing trace = hc.GetTrace(); + + // Act. + var gitPath = WhichUtil.Which("git", require: true, trace: trace); + var gitPath2 = WhichUtil.Which(gitPath, require: true, trace: trace); + + // Assert. + Assert.Equal(gitPath, gitPath2); + } + } } } From de4490d06d5ef58a8bf0c24b41f1f84f61c8df21 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 15:40:09 -0400 Subject: [PATCH 71/86] Restore SELinux context on service file when SELinux is enabled (#525) --- src/Misc/layoutbin/systemd.svc.sh.template | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Misc/layoutbin/systemd.svc.sh.template b/src/Misc/layoutbin/systemd.svc.sh.template index cbec3319754..bdbc998f78c 100644 --- a/src/Misc/layoutbin/systemd.svc.sh.template +++ b/src/Misc/layoutbin/systemd.svc.sh.template @@ -63,12 +63,25 @@ function install() sed "s/{{User}}/${run_as_user}/g; s/{{Description}}/$(echo ${SVC_DESCRIPTION} | sed -e 's/[\/&]/\\&/g')/g; s/{{RunnerRoot}}/$(echo ${RUNNER_ROOT} | sed -e 's/[\/&]/\\&/g')/g;" "${TEMPLATE_PATH}" > "${TEMP_PATH}" || failed "failed to create replacement temp file" mv "${TEMP_PATH}" "${UNIT_PATH}" || failed "failed to copy unit file" + + # Recent Fedora based Linux (CentOS/Redhat) has SELinux enabled by default + # We need to restore security context on the unit file we added otherwise SystemD have no access to it. + command -v getenforce > /dev/null + if [ $? -eq 0 ] + then + selinuxEnabled=$(getenforce) + if [[ $selinuxEnabled == "Enforcing" ]] + then + # SELinux is enabled, we will need to Restore SELinux Context for the service file + restorecon -r -v "${UNIT_PATH}" || failed "failed to restore SELinux context on ${UNIT_PATH}" + fi + fi # unit file should not be executable and world writable - chmod 664 ${UNIT_PATH} || failed "failed to set permissions on ${UNIT_PATH}" + chmod 664 "${UNIT_PATH}" || failed "failed to set permissions on ${UNIT_PATH}" systemctl daemon-reload || failed "failed to reload daemons" - # Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user. + # Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user. cp ./bin/runsvc.sh ./runsvc.sh || failed "failed to copy runsvc.sh" chown ${run_as_uid}:${run_as_gid} ./runsvc.sh || failed "failed to set owner for runsvc.sh" chmod 755 ./runsvc.sh || failed "failed to set permission for runsvc.sh" From e728b8594d6fca9c2a0207a116eb819135655317 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 11 Jun 2020 16:17:24 -0400 Subject: [PATCH 72/86] fix race condition. (#538) --- src/Runner.Sdk/ProcessInvoker.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Runner.Sdk/ProcessInvoker.cs b/src/Runner.Sdk/ProcessInvoker.cs index 4ed4ce3b0d3..78a9f2dd27e 100644 --- a/src/Runner.Sdk/ProcessInvoker.cs +++ b/src/Runner.Sdk/ProcessInvoker.cs @@ -346,14 +346,14 @@ public async Task ExecuteAsync( // data buffers one last time before returning ProcessOutput(); - Trace.Info($"Finished process {_proc.Id} with exit code {_proc.ExitCode}, and elapsed time {_stopWatch.Elapsed}."); - } + if (cancellationToken.IsCancellationRequested) + { + // Ensure cancellation also finish on the cancellationToken.Register thread. + await cancellationFinished.Task; + Trace.Info($"Process Cancellation finished."); + } - if (cancellationToken.IsCancellationRequested) - { - // Ensure cancellation also finish on the cancellationToken.Register thread. - await cancellationFinished.Task; - Trace.Info($"Process Cancellation finished."); + Trace.Info($"Finished process {_proc.Id} with exit code {_proc.ExitCode}, and elapsed time {_stopWatch.Elapsed}."); } cancellationToken.ThrowIfCancellationRequested(); From 89d1418e4856c2378aab28f72ea15850c117911a Mon Sep 17 00:00:00 2001 From: Lokesh Gopu Date: Thu, 11 Jun 2020 17:25:50 -0400 Subject: [PATCH 73/86] Update exception message (#540) --- src/Runner.Listener/Configuration/ConfigurationManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 7d634067f3b..ce0863e1a87 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -210,7 +210,7 @@ public async Task ConfigureAsync(CommandSettings command) else if (command.Unattended) { // if not replace and it is unattended config. - throw new TaskAgentExistsException($"Pool {runnerSettings.PoolId} already contains a runner with name {runnerSettings.AgentName}."); + throw new TaskAgentExistsException($"A runner exists with the same name {runnerSettings.AgentName}."); } } else From 4e7d27a53c84466562d366256b960433bafc469b Mon Sep 17 00:00:00 2001 From: eric sciple Date: Mon, 15 Jun 2020 13:13:47 -0400 Subject: [PATCH 74/86] remove temporary logic when resolving action download info (#550) --- src/Runner.Worker/ActionManager.cs | 67 +-------------------------- src/Test/L0/Worker/ActionManagerL0.cs | 4 +- 2 files changed, 4 insertions(+), 67 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 347eb3d0d03..b9720610d92 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -546,56 +546,6 @@ private async Task BuildActionContainerAsync(IExecutionContext executionContext, } } - // This implementation is temporary and will be removed when we switch to a REST API call to the service to resolve the download info - private async Task RepoExistsAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo actionDownloadInfo, string token) - { - var apiUrl = GetApiUrl(executionContext); - var repoUrl = $"{apiUrl}/repos/{actionDownloadInfo.NameWithOwner}"; - for (var attempt = 1; attempt <= 3; attempt++) - { - executionContext.Debug($"Checking whether repo exists: {repoUrl}"); - try - { - using (var httpClientHandler = HostContext.CreateHttpClientHandler()) - using (var httpClient = new HttpClient(httpClientHandler)) - { - httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(token); - httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); - using (var response = await httpClient.GetAsync(repoUrl)) - { - if (response.IsSuccessStatusCode) - { - return true; - } - else if (response.StatusCode == HttpStatusCode.NotFound) - { - return false; - } - else - { - // Throw - response.EnsureSuccessStatusCode(); - } - } - } - } - catch (Exception ex) - { - if (attempt < 3) - { - executionContext.Debug($"Failed checking whether repo '{actionDownloadInfo.NameWithOwner}' exists: {ex.Message}"); - } - else - { - executionContext.Error($"Failed checking whether repo '{actionDownloadInfo.NameWithOwner}' exists: {ex.Message}"); - throw; - } - } - } - - return false; // Never reaches here - } - // This implementation is temporary and will be replaced with a REST API call to the service to resolve private async Task> GetDownloadInfoAsync(IExecutionContext executionContext, List actions) { @@ -659,23 +609,10 @@ private async Task RepoExistsAsync(IExecutionContext executionContext, Web // Add secret HostContext.SecretMasker.AddValue(actionDownloadInfo.Authentication?.Token); - // Temporary code: Fix token and download URL - if (runnerSettings.IsHostedServer) + // Default auth token + if (string.IsNullOrEmpty(actionDownloadInfo.Authentication?.Token)) { actionDownloadInfo.Authentication = new WebApi.ActionDownloadAuthentication { Token = defaultAccessToken }; - actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", apiUrl); - actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", apiUrl); - } - else if (await RepoExistsAsync(executionContext, actionDownloadInfo, defaultAccessToken)) - { - actionDownloadInfo.Authentication = new WebApi.ActionDownloadAuthentication { Token = defaultAccessToken }; - actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", apiUrl); - actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", apiUrl); - } - else - { - actionDownloadInfo.TarballUrl = actionDownloadInfo.TarballUrl.Replace("", "https://api.github.com"); - actionDownloadInfo.ZipballUrl = actionDownloadInfo.ZipballUrl.Replace("", "https://api.github.com"); } } diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 85851a55ac1..58ffea5aa9b 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -3607,8 +3607,8 @@ private void Setup([CallerMemberName] string name = "", bool newActionMetadata = { NameWithOwner = action.NameWithOwner, Ref = action.Ref, - TarballUrl = $"/repos/{action.NameWithOwner}/tarball/{action.Ref}", - ZipballUrl = $"/repos/{action.NameWithOwner}/zipball/{action.Ref}", + TarballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/tarball/{action.Ref}", + ZipballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/zipball/{action.Ref}", }; } return Task.FromResult(result); From df7e16954e9bbcab82b85be44cad9e624f02cbe0 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 23 Jun 2020 13:57:37 -0400 Subject: [PATCH 75/86] print runner and machine name to log. (#539) --- src/Runner.Worker/JobExtension.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 235cc90638e..61f080c71a1 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -64,6 +64,20 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.Debug($"Starting: Set up job"); context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'"); + var setting = HostContext.GetService().GetSettings(); + var credFile = HostContext.GetConfigFile(WellKnownConfigFile.Credentials); + if (File.Exists(credFile)) + { + var credData = IOUtil.LoadObject(credFile); + if (credData != null && + credData.Data.TryGetValue("clientId", out var clientId)) + { + // print out HostName for self-hosted runner + context.Output($"Runner name: '{setting.AgentName}'"); + context.Output($"Machine name: '{Environment.MachineName}'"); + } + } + var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo); if (File.Exists(setupInfoFile)) { From 7cef9a27ca514e72c10f3c7f2bf1418b52275a68 Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Tue, 23 Jun 2020 14:05:28 -0400 Subject: [PATCH 76/86] release 2.267.0 runner. --- releaseNote.md | 20 ++++++++++++-------- src/runnerversion | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index b75e056e3ba..48d725bc551 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,14 +1,18 @@ ## Features - - N/A + - Resolve action download info from server (#508, #515, #550) + - Print runner and machine name to log. (#539) ## Bugs - - Handle `jq` returns "null" if the field does not exist in create-latest-svc.sh (#478) - - Switch GITHUB_URL to GITHUB_SERVER_URL (#482) - - Fix problem matcher for GHES (#488) - - Fix container action inputs validation warning (#490) - - Fix post step display name (#490) - - Fix worker crash due to exception from evaluating step.env (#490) + - Reduce input validation warnings (#506) + - Fix null ref exception in SecretMasker caused by `hashfiles` timeout. (#516) + - Add libicu66 to `./installDependencies.sh` for Ubuntu 20.04 (#535) + - Fix DataContract with Token service (#532) + - Skip search $PATH on command with fully qualified path (#526) + - Restore SELinux context on service file when SELinux is enabled (#525) ## Misc - - N/A + - Remove SPS/Token migration code. Remove GHES url manipulate code. (#513) + - Add sub-step for developer flow for clarity (#523) + - Update Links and Language to Git + VSCode (#522) + - Update runner configuration exception message (#540) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/runnerversion b/src/runnerversion index f9d36d71d1d..58301aa109e 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.263.0 +2.267.0 From a0942ed3459f199a293cabbe6f1d584e4ad352a5 Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Tue, 23 Jun 2020 15:35:32 -0400 Subject: [PATCH 77/86] Composite Actions Support for Multiple Run Steps (#549) * Composite Action Run Steps * Clean up trace messages + add Trace debug in ActionManager * Change String to string * Add comma to Composite * Change JobSteps to a List, Change Register Step function name * Add TODO, remove unn. content * Remove unnecessary code * Fix unit tests * Add verbose trace logs which are only viewable by devs * Sort usings in Composite Action Handler * Change 0 to location * Update context variables in composite action yaml * Add helpful error message for null steps --- src/Runner.Worker/ActionManager.cs | 20 + src/Runner.Worker/ActionManifestManager.cs | 32 +- src/Runner.Worker/ExecutionContext.cs | 22 +- .../Handlers/CompositeActionHandler.cs | 98 +++++ src/Runner.Worker/Handlers/HandlerFactory.cs | 5 + src/Runner.Worker/JobRunner.cs | 2 +- src/Runner.Worker/StepsRunner.cs | 19 +- src/Runner.Worker/action_yaml.json | 33 +- .../PipelineTemplateConstants.cs | 1 + .../PipelineTemplateConverter.cs | 347 +++++++++++++++++- .../PipelineTemplateEvaluator.cs | 48 ++- .../Pipelines/PipelineConstants.cs | 7 + src/Test/L0/Worker/StepsRunnerL0.cs | 26 +- 13 files changed, 624 insertions(+), 36 deletions(-) create mode 100644 src/Runner.Worker/Handlers/CompositeActionHandler.cs diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index b9720610d92..242ab79e052 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -395,6 +395,12 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}."); } } + else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + var compositeAction = definition.Data.Execution as CompositeActionExecutionData; + Trace.Info($"Load {compositeAction.Steps.Count} action steps."); + Trace.Verbose($"Details: {StringUtil.ConvertToJson(compositeAction.Steps)}"); + } else { throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString()); @@ -1038,6 +1044,11 @@ private ActionContainer PrepareRepositoryActionAsync(IExecutionContext execution Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation."); return null; } + else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation."); + return null; + } else { throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString()); @@ -1148,6 +1159,7 @@ public enum ActionExecutionType NodeJS, Plugin, Script, + Composite, } public sealed class ContainerActionExecutionData : ActionExecutionData @@ -1204,6 +1216,14 @@ public sealed class ScriptActionExecutionData : ActionExecutionData public override bool HasPost => false; } + public sealed class CompositeActionExecutionData : ActionExecutionData + { + public override ActionExecutionType ExecutionType => ActionExecutionType.Composite; + public override bool HasPre => false; + public override bool HasPost => false; + public List Steps { get; set; } + } + public abstract class ActionExecutionData { private string _initCondition = $"{Constants.Expressions.Always}()"; diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 4e9149d26b6..9095f498ddd 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -14,6 +14,7 @@ using YamlDotNet.Core.Events; using System.Globalization; using System.Linq; +using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker { @@ -92,7 +93,7 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani break; case "runs": - actionDefinition.Execution = ConvertRuns(context, actionPair.Value); + actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value); break; default: Trace.Info($"Ignore action property {propertyName}."); @@ -284,7 +285,7 @@ private TemplateContext CreateContext( // Add the file table if (_fileTable?.Count > 0) { - for (var i = 0 ; i < _fileTable.Count ; i++) + for (var i = 0; i < _fileTable.Count; i++) { result.GetFileId(_fileTable[i]); } @@ -294,6 +295,7 @@ private TemplateContext CreateContext( } private ActionExecutionData ConvertRuns( + IExecutionContext executionContext, TemplateContext context, TemplateToken inputsToken) { @@ -311,6 +313,8 @@ private ActionExecutionData ConvertRuns( var postToken = default(StringToken); var postEntrypointToken = default(StringToken); var postIfToken = default(StringToken); + var stepsLoaded = default(List); + foreach (var run in runsMapping) { var runsKey = run.Key.AssertString("runs key").Value; @@ -355,6 +359,15 @@ private ActionExecutionData ConvertRuns( case "pre-if": preIfToken = run.Value.AssertString("pre-if"); break; + case "steps": + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + var steps = run.Value.AssertSequence("steps"); + var evaluator = executionContext.ToPipelineTemplateEvaluator(); + stepsLoaded = evaluator.LoadCompositeSteps(steps); + break; + } + throw new Exception("You aren't supposed to be using Composite Actions yet!"); default: Trace.Info($"Ignore run property {runsKey}."); break; @@ -402,6 +415,21 @@ private ActionExecutionData ConvertRuns( }; } } + else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + if (stepsLoaded == null) + { + // TODO: Add a more helpful error message + including file name, etc. to show user that it's because of their yaml file + throw new ArgumentNullException($"No steps provided."); + } + else + { + return new CompositeActionExecutionData() + { + Steps = stepsLoaded, + }; + } + } else { throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead."); diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 0318974b01d..cea1e2fd4d7 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -63,7 +63,7 @@ public interface IExecutionContext : IRunnerService JobContext JobContext { get; } // Only job level ExecutionContext has JobSteps - Queue JobSteps { get; } + List JobSteps { get; } // Only job level ExecutionContext has PostJobSteps Stack PostJobSteps { get; } @@ -105,6 +105,7 @@ public interface IExecutionContext : IRunnerService // others void ForceTaskComplete(); void RegisterPostJobStep(IStep step); + void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location); } public sealed class ExecutionContext : RunnerService, IExecutionContext @@ -159,7 +160,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public List ServiceContainers { get; private set; } // Only job level ExecutionContext has JobSteps - public Queue JobSteps { get; private set; } + public List JobSteps { get; private set; } // Only job level ExecutionContext has PostJobSteps public Stack PostJobSteps { get; private set; } @@ -169,7 +170,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public bool EchoOnActionCommand { get; set; } - public TaskResult? Result { get @@ -266,6 +266,20 @@ public void RegisterPostJobStep(IStep step) Root.PostJobSteps.Push(step); } + /// + /// Helper function used in CompositeActionHandler::RunAsync to + /// add a child node, aka a step, to the current job to the Root.JobSteps based on the location. + /// + public void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location) + { + // TODO: For UI purposes, look at figuring out how to condense steps in one node => maybe use the same previous GUID + var newGuid = Guid.NewGuid(); + step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null); + step.ExecutionContext.ExpressionValues["inputs"] = inputsData; + // TODO: confirm whether not copying message contexts is safe + Root.JobSteps.Insert(location, step); + } + public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null) { Trace.Entering(); @@ -660,7 +674,7 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation PrependPath = new List(); // JobSteps for job ExecutionContext - JobSteps = new Queue(); + JobSteps = new List(); // PostJobSteps for job ExecutionContext PostJobSteps = new Stack(); diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs new file mode 100644 index 00000000000..7c5b25ed5fb --- /dev/null +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; +using Pipelines = GitHub.DistributedTask.Pipelines; + + +namespace GitHub.Runner.Worker.Handlers +{ + [ServiceLocator(Default = typeof(CompositeActionHandler))] + public interface ICompositeActionHandler : IHandler + { + CompositeActionExecutionData Data { get; set; } + } + public sealed class CompositeActionHandler : Handler, ICompositeActionHandler + { + public CompositeActionExecutionData Data { get; set; } + + public Task RunAsync(ActionRunStage stage) + { + // Validate args. + Trace.Entering(); + ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); + ArgUtil.NotNull(Inputs, nameof(Inputs)); + + var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(githubContext, nameof(githubContext)); + + var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp); + + // Resolve action steps + var actionSteps = Data.Steps; + + // Create Context Data to reuse for each composite action step + var inputsData = new DictionaryContextData(); + foreach (var i in Inputs) + { + inputsData[i.Key] = new StringContextData(i.Value); + } + + // Add each composite action step to the front of the queue + int location = 0; + foreach (Pipelines.ActionStep aStep in actionSteps) + { + // Ex: + // runs: + // using: "composite" + // steps: + // - uses: example/test-composite@v2 (a) + // - run echo hello world (b) + // - run echo hello world 2 (c) + // + // ethanchewy/test-composite/action.yaml + // runs: + // using: "composite" + // steps: + // - run echo hello world 3 (d) + // - run echo hello world 4 (e) + // + // Steps processed as follow: + // | a | + // | a | => | d | + // (Run step d) + // | a | + // | a | => | e | + // (Run step e) + // | a | + // (Run step a) + // | b | + // (Run step b) + // | c | + // (Run step c) + // Done. + + var actionRunner = HostContext.CreateService(); + actionRunner.Action = aStep; + actionRunner.Stage = stage; + actionRunner.Condition = aStep.Condition; + actionRunner.DisplayName = aStep.DisplayName; + // TODO: Do we need to add any context data from the job message? + // (See JobExtension.cs ~line 236) + + ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location); + location++; + } + + return Task.CompletedTask; + } + + } +} diff --git a/src/Runner.Worker/Handlers/HandlerFactory.cs b/src/Runner.Worker/Handlers/HandlerFactory.cs index 0f2413ef5b7..db4d6559c88 100644 --- a/src/Runner.Worker/Handlers/HandlerFactory.cs +++ b/src/Runner.Worker/Handlers/HandlerFactory.cs @@ -66,6 +66,11 @@ public IHandler Create( handler = HostContext.CreateService(); (handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData; } + else if (data.ExecutionType == ActionExecutionType.Composite) + { + handler = HostContext.CreateService(); + (handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData; + } else { // This should never happen. diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 31dfb17145b..33b291adb6d 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -152,7 +152,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, { foreach (var step in jobSteps) { - jobContext.JobSteps.Enqueue(step); + jobContext.JobSteps.Add(step); } await stepsRunner.RunAsync(jobContext); diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 485a4cdf980..e75d2e106f8 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -59,14 +59,15 @@ public async Task RunAsync(IExecutionContext jobContext) checkPostJobActions = true; while (jobContext.PostJobSteps.TryPop(out var postStep)) { - jobContext.JobSteps.Enqueue(postStep); + jobContext.JobSteps.Add(postStep); } continue; } - var step = jobContext.JobSteps.Dequeue(); - var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null; + var step = jobContext.JobSteps[0]; + jobContext.JobSteps.RemoveAt(0); + var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps[0] : null; Trace.Info($"Processing step: DisplayName='{step.DisplayName}'"); ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext)); @@ -409,7 +410,11 @@ private bool InitializeScope(IStep step, Dictionary scope = scopesToInitialize.Pop(); executionContext.Debug($"Initializing scope '{scope.Name}'"); executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName); - executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; + // TODO: Fix this temporary workaround for Composite Actions + if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; + } var templateEvaluator = executionContext.ToPipelineTemplateEvaluator(); var inputs = default(DictionaryContextData); try @@ -432,7 +437,11 @@ private bool InitializeScope(IStep step, Dictionary // Setup expression values var scopeName = executionContext.ScopeName; executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName); - executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; + // TODO: Fix this temporary workaround for Composite Actions + if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; + } return true; } diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index 7a8b847d31f..cb1d90b2e0b 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -32,7 +32,8 @@ "one-of": [ "container-runs", "node12-runs", - "plugin-runs" + "plugin-runs", + "composite-runs" ] }, "container-runs": { @@ -83,6 +84,36 @@ } } }, + "composite-runs": { + "mapping": { + "properties": { + "using": "non-empty-string", + "steps": "composite-steps" + } + } + }, + "composite-steps": { + "context": [ + "github", + "needs", + "strategy", + "matrix", + "secrets", + "steps", + "inputs", + "job", + "runner", + "env", + "always(0,0)", + "failure(0,0)", + "cancelled(0,0)", + "success(0,0)", + "hashFiles(1,255)" + ], + "sequence": { + "item-type": "any" + } + }, "container-runs-context": { "context": [ "inputs" diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index d1c886dd891..f2609462b23 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -65,6 +65,7 @@ public sealed class PipelineTemplateConstants public const String StepEnv = "step-env"; public const String StepIfResult = "step-if-result"; public const String Steps = "steps"; + public const String StepsInTemplate = "steps-in-template"; public const String StepsScopeInputs = "steps-scope-inputs"; public const String StepsScopeOutputs = "steps-scope-outputs"; public const String StepsTemplateRoot = "steps-template-root"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 43be43d3375..a952f58fbb9 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -29,7 +29,6 @@ internal static Boolean ConvertToIfResult( var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult); return evaluationResult.IsTruthy; } - internal static Boolean? ConvertToStepContinueOnError( TemplateContext context, TemplateToken token, @@ -264,5 +263,351 @@ internal static List> ConvertToJobServiceCont return result; } + + //Note: originally was List but we need to change to List to use the "Inputs" attribute + internal static List ConvertToSteps( + TemplateContext context, + TemplateToken steps) + { + var stepsSequence = steps.AssertSequence($"job {PipelineTemplateConstants.Steps}"); + + var result = new List(); + foreach (var stepsItem in stepsSequence) + { + var step = ConvertToStep(context, stepsItem); + if (step != null) // step = null means we are hitting error during step conversion, there should be an error in context.errors + { + if (step.Enabled) + { + result.Add(step); + } + } + } + + return result; + } + + private static ActionStep ConvertToStep( + TemplateContext context, + TemplateToken stepsItem) + { + var step = stepsItem.AssertMapping($"{PipelineTemplateConstants.Steps} item"); + var continueOnError = default(ScalarToken); + var env = default(TemplateToken); + var id = default(StringToken); + var ifCondition = default(String); + var ifToken = default(ScalarToken); + var name = default(ScalarToken); + var run = default(ScalarToken); + var scope = default(StringToken); + var timeoutMinutes = default(ScalarToken); + var uses = default(StringToken); + var with = default(TemplateToken); + var workingDir = default(ScalarToken); + var path = default(ScalarToken); + var clean = default(ScalarToken); + var fetchDepth = default(ScalarToken); + var lfs = default(ScalarToken); + var submodules = default(ScalarToken); + var shell = default(ScalarToken); + + foreach (var stepProperty in step) + { + var propertyName = stepProperty.Key.AssertString($"{PipelineTemplateConstants.Steps} item key"); + + switch (propertyName.Value) + { + case PipelineTemplateConstants.Clean: + clean = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Clean}"); + break; + + case PipelineTemplateConstants.ContinueOnError: + ConvertToStepContinueOnError(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + continueOnError = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} {PipelineTemplateConstants.ContinueOnError}"); + break; + + case PipelineTemplateConstants.Env: + ConvertToStepEnvironment(context, stepProperty.Value, StringComparer.Ordinal, allowExpressions: true); // Validate early if possible + env = stepProperty.Value; + break; + + case PipelineTemplateConstants.FetchDepth: + fetchDepth = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.FetchDepth}"); + break; + + case PipelineTemplateConstants.Id: + id = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Id}"); + if (!NameValidation.IsValid(id.Value, true)) + { + context.Error(id, $"Step id {id.Value} is invalid. Ids must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'"); + } + break; + + case PipelineTemplateConstants.If: + ifToken = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.If}"); + break; + + case PipelineTemplateConstants.Lfs: + lfs = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Lfs}"); + break; + + case PipelineTemplateConstants.Name: + name = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Name}"); + break; + + case PipelineTemplateConstants.Path: + path = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Path}"); + break; + + case PipelineTemplateConstants.Run: + run = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Run}"); + break; + + case PipelineTemplateConstants.Shell: + shell = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Shell}"); + break; + + case PipelineTemplateConstants.Scope: + scope = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Scope}"); + break; + + case PipelineTemplateConstants.Submodules: + submodules = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Submodules}"); + break; + + case PipelineTemplateConstants.TimeoutMinutes: + ConvertToStepTimeout(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + timeoutMinutes = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.TimeoutMinutes}"); + break; + + case PipelineTemplateConstants.Uses: + uses = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + break; + + case PipelineTemplateConstants.With: + ConvertToStepInputs(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + with = stepProperty.Value; + break; + + case PipelineTemplateConstants.WorkingDirectory: + workingDir = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.WorkingDirectory}"); + break; + + default: + propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Steps} item key"); // throws + break; + } + } + + // Fixup the if-condition + var isDefaultScope = String.IsNullOrEmpty(scope?.Value); + ifCondition = ConvertToIfCondition(context, ifToken, false, isDefaultScope); + + if (run != null) + { + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Environment = env, + Reference = new ScriptReference(), + }; + + var inputs = new MappingToken(null, null, null); + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Script), run); + + if (workingDir != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.WorkingDirectory), workingDir); + } + + if (shell != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Shell), shell); + } + + result.Inputs = inputs; + + return result; + } + else + { + uses.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Inputs = with, + Environment = env, + }; + + if (uses.Value.StartsWith("docker://", StringComparison.Ordinal)) + { + var image = uses.Value.Substring("docker://".Length); + result.Reference = new ContainerRegistryReference { Image = image }; + } + else if (uses.Value.StartsWith("./") || uses.Value.StartsWith(".\\")) + { + result.Reference = new RepositoryPathReference + { + RepositoryType = PipelineConstants.SelfAlias, + Path = uses.Value + }; + } + else + { + var usesSegments = uses.Value.Split('@'); + var pathSegments = usesSegments[0].Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); + var gitRef = usesSegments.Length == 2 ? usesSegments[1] : String.Empty; + + if (usesSegments.Length != 2 || + pathSegments.Length < 2 || + String.IsNullOrEmpty(pathSegments[0]) || + String.IsNullOrEmpty(pathSegments[1]) || + String.IsNullOrEmpty(gitRef)) + { + // todo: loc + context.Error(uses, $"Expected format {{org}}/{{repo}}[/path]@ref. Actual '{uses.Value}'"); + } + else + { + var repositoryName = $"{pathSegments[0]}/{pathSegments[1]}"; + var directoryPath = pathSegments.Length > 2 ? String.Join("/", pathSegments.Skip(2)) : String.Empty; + + result.Reference = new RepositoryPathReference + { + RepositoryType = RepositoryTypes.GitHub, + Name = repositoryName, + Ref = gitRef, + Path = directoryPath, + }; + } + } + + return result; + } + } + + /// + /// When empty, default to "success()". + /// When a status function is not referenced, format as "success() && <CONDITION>". + /// + private static String ConvertToIfCondition( + TemplateContext context, + TemplateToken token, + Boolean isJob, + Boolean isDefaultScope) + { + String condition; + if (token is null) + { + condition = null; + } + else if (token is BasicExpressionToken expressionToken) + { + condition = expressionToken.Expression; + } + else + { + var stringToken = token.AssertString($"{(isJob ? "job" : "step")} {PipelineTemplateConstants.If}"); + condition = stringToken.Value; + } + + if (String.IsNullOrWhiteSpace(condition)) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var expressionParser = new ExpressionParser(); + var functions = default(IFunctionInfo[]); + var namedValues = default(INamedValueInfo[]); + if (isJob) + { + namedValues = s_jobIfNamedValues; + // TODO: refactor into seperate functions + // functions = PhaseCondition.FunctionInfo; + } + else + { + namedValues = isDefaultScope ? s_stepNamedValues : s_stepInTemplateNamedValues; + functions = s_stepConditionFunctions; + } + + var node = default(ExpressionNode); + try + { + node = expressionParser.CreateTree(condition, null, namedValues, functions) as ExpressionNode; + } + catch (Exception ex) + { + context.Error(token, ex); + return null; + } + + if (node == null) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var hasStatusFunction = node.Traverse().Any(x => + { + if (x is Function function) + { + return String.Equals(function.Name, PipelineTemplateConstants.Always, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Cancelled, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Failure, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Success, StringComparison.OrdinalIgnoreCase); + } + + return false; + }); + + return hasStatusFunction ? condition : $"{PipelineTemplateConstants.Success}() && ({condition})"; + } + + private static readonly INamedValueInfo[] s_jobIfNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepInTemplateNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.Inputs), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly IFunctionInfo[] s_stepConditionFunctions = new IFunctionInfo[] + { + new FunctionInfo(PipelineTemplateConstants.Always, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Cancelled, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Failure, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Success, 0, 0), + new FunctionInfo(PipelineTemplateConstants.HashFiles, 1, Byte.MaxValue), + }; } } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index a36f5b7e3aa..55076e670e5 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -159,6 +159,32 @@ public String EvaluateStepDisplayName( return result; } + public List LoadCompositeSteps( + TemplateToken token + ) + { + var result = default(List); + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(null, null, setMissingContext: false); + // TODO: we might want to to have a bool to prevent it from filling in with missing context w/ dummy variables + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsInTemplate, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = PipelineTemplateConverter.ConvertToSteps(context, token); + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + return result; + } + + public Dictionary EvaluateStepEnvironment( TemplateToken token, DictionaryContextData contextData, @@ -400,7 +426,8 @@ public IList> EvaluateJobServiceContainers( private TemplateContext CreateContext( DictionaryContextData contextData, IList expressionFunctions, - IEnumerable> expressionState = null) + IEnumerable> expressionState = null, + bool setMissingContext = true) { var result = new TemplateContext { @@ -449,18 +476,21 @@ private TemplateContext CreateContext( // - Evaluating early when all referenced contexts are available, even though all allowed // contexts may not yet be available. For example, evaluating step display name can often // be performed early. - foreach (var name in s_expressionValueNames) + if (setMissingContext) { - if (!result.ExpressionValues.ContainsKey(name)) + foreach (var name in s_expressionValueNames) { - result.ExpressionValues[name] = null; + if (!result.ExpressionValues.ContainsKey(name)) + { + result.ExpressionValues[name] = null; + } } - } - foreach (var name in s_expressionFunctionNames) - { - if (!functionNames.Contains(name)) + foreach (var name in s_expressionFunctionNames) { - result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + if (!functionNames.Contains(name)) + { + result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + } } } diff --git a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs index 2d599dd9c55..2e03671fbb2 100644 --- a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs @@ -94,5 +94,12 @@ public static class WorkspaceCleanOptions public static readonly String Resources = "resources"; public static readonly String All = "all"; } + + public static class ScriptStepInputs + { + public static readonly String Script = "script"; + public static readonly String WorkingDirectory = "workingDirectory"; + public static readonly String Shell = "shell"; + } } } diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index 2d7cb9fb0c4..1dfee2252ad 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -80,7 +80,7 @@ public async Task RunNormalStepsAllStepPass() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -115,7 +115,7 @@ public async Task RunNormalStepsContinueOnError() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -154,7 +154,7 @@ public async Task RunsAfterFailureBasedOnCondition() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -208,7 +208,7 @@ public async Task RunsAlwaysSteps() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -287,7 +287,7 @@ public async Task SetsJobResultCorrectly() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -330,7 +330,7 @@ public async Task SkipsAfterFailureOnlyBaseOnCondition() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Step.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Step.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -361,7 +361,7 @@ public async Task AlwaysMeansAlways() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -391,7 +391,7 @@ public async Task TreatsConditionErrorAsFailure() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -417,7 +417,7 @@ public async Task StepEnvOverrideJobEnvContext() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -455,7 +455,7 @@ public async Task PopulateEnvContextForEachStep() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -493,7 +493,7 @@ public async Task PopulateEnvContextAfterSetupStepsContext() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -524,7 +524,7 @@ public async Task StepContextOutcome() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object, step3.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object, step3.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -560,7 +560,7 @@ public async Task StepContextConclusion() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object, step3.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object, step3.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); From 121deedeb5861767b436c789d73a8faa746981d8 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 30 Jun 2020 17:25:47 -0400 Subject: [PATCH 78/86] Fix trailing '.0' for Int64 values (#572) --- .../DTPipelines/Pipelines/ContextData/NumberContextData.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Sdk/DTPipelines/Pipelines/ContextData/NumberContextData.cs b/src/Sdk/DTPipelines/Pipelines/ContextData/NumberContextData.cs index 07d2172bcdb..82ad590b1a9 100644 --- a/src/Sdk/DTPipelines/Pipelines/ContextData/NumberContextData.cs +++ b/src/Sdk/DTPipelines/Pipelines/ContextData/NumberContextData.cs @@ -42,7 +42,12 @@ public override JToken ToJToken() var floored = Math.Floor(m_value); if (m_value == floored && m_value <= (Double)Int32.MaxValue && m_value >= (Double)Int32.MinValue) { - Int32 flooredInt = (Int32)floored; + var flooredInt = (Int32)floored; + return (JToken)flooredInt; + } + else if (m_value == floored && m_value <= (Double)Int64.MaxValue && m_value >= (Double)Int64.MinValue) + { + var flooredInt = (Int64)floored; return (JToken)flooredInt; } else From d42c9da2d7ee28481a89934353b9b67b63315ef4 Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Wed, 8 Jul 2020 10:16:51 -0400 Subject: [PATCH 79/86] Composite Actions: Support Env Flow (#557) * Composite Action Run Steps * Env Flow => Able to get env variables and overwrite current env variables => but it doesn't 'stick' * clean up * Clean up trace messages + add Trace debug in ActionManager * Add debugging message * Optimize runtime of code * Change String to string * Add comma to Composite * Change JobSteps to a List, Change Register Step function name * Add TODO, remove unn. content * Remove unnecessary code * Fix unit tests * Fix env format * Remove comment * Remove TODO message for context * Add verbose trace logs which are only viewable by devs * Sort usings in Composite Action Handler * Change 0 to location * Update context variables in composite action yaml * Add helpful error message for null steps * Fix Workflow Step Env overiding Parent Env * Remove env in composite action scope * Clean up * Revert back * revert back * add back envToken * Remove unnecessary code * Figure out how to handle set-env edge cases * formatting * fix unit tests * Fix windows unit test syntax error --- src/Misc/dotnet-install.ps1 | 379 +++++++++--------- src/Runner.Worker/ExecutionContext.cs | 21 +- .../Handlers/CompositeActionHandler.cs | 4 +- src/Runner.Worker/StepsRunner.cs | 18 +- .../PipelineTemplateEvaluator.cs | 3 +- src/Test/L0/Worker/StepsRunnerL0.cs | 30 +- 6 files changed, 246 insertions(+), 209 deletions(-) diff --git a/src/Misc/dotnet-install.ps1 b/src/Misc/dotnet-install.ps1 index c0122cdc3b7..206d4676192 100644 --- a/src/Misc/dotnet-install.ps1 +++ b/src/Misc/dotnet-install.ps1 @@ -154,7 +154,16 @@ function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [in function Get-Machine-Architecture() { Say-Invocation $MyInvocation - # possible values: amd64, x64, x86, arm64, arm + # On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems. + # To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432. + # PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE. + # Possible values: amd64, x64, x86, arm64, arm + + if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null ) + { + return $ENV:PROCESSOR_ARCHITEW6432 + } + return $ENV:PROCESSOR_ARCHITECTURE } @@ -686,194 +695,194 @@ Say "Installation finished" exit 0 # SIG # Begin signature block -# MIIjkQYJKoZIhvcNAQcCoIIjgjCCI34CAQExDzANBglghkgBZQMEAgEFADB5Bgor +# MIIjhwYJKoZIhvcNAQcCoIIjeDCCI3QCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG -# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAwp4UsNdAkvwY3 -# VhbuN9D6NGOz+qNqW2+62YubWa4qJaCCDYEwggX/MIID56ADAgECAhMzAAABh3IX -# chVZQMcJAAAAAAGHMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD +# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAiKYSY4KtkeThH +# d5M1aXqv1K0/pff07QwfUbYZ/qX5LqCCDYUwggYDMIID66ADAgECAhMzAAABiK9S +# 1rmSbej5AAAAAAGIMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p -# bmcgUENBIDIwMTEwHhcNMjAwMzA0MTgzOTQ3WhcNMjEwMzAzMTgzOTQ3WjB0MQsw +# bmcgUENBIDIwMTEwHhcNMjAwMzA0MTgzOTQ4WhcNMjEwMzAzMTgzOTQ4WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB -# AQDOt8kLc7P3T7MKIhouYHewMFmnq8Ayu7FOhZCQabVwBp2VS4WyB2Qe4TQBT8aB -# znANDEPjHKNdPT8Xz5cNali6XHefS8i/WXtF0vSsP8NEv6mBHuA2p1fw2wB/F0dH -# sJ3GfZ5c0sPJjklsiYqPw59xJ54kM91IOgiO2OUzjNAljPibjCWfH7UzQ1TPHc4d -# weils8GEIrbBRb7IWwiObL12jWT4Yh71NQgvJ9Fn6+UhD9x2uk3dLj84vwt1NuFQ -# itKJxIV0fVsRNR3abQVOLqpDugbr0SzNL6o8xzOHL5OXiGGwg6ekiXA1/2XXY7yV -# Fc39tledDtZjSjNbex1zzwSXAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE -# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUhov4ZyO96axkJdMjpzu2zVXOJcsw -# UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 -# ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDU4Mzg1MB8GA1UdIwQYMBaAFEhu -# ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu -# bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w -# Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 -# Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx -# MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAixmy -# S6E6vprWD9KFNIB9G5zyMuIjZAOuUJ1EK/Vlg6Fb3ZHXjjUwATKIcXbFuFC6Wr4K -# NrU4DY/sBVqmab5AC/je3bpUpjtxpEyqUqtPc30wEg/rO9vmKmqKoLPT37svc2NV -# BmGNl+85qO4fV/w7Cx7J0Bbqk19KcRNdjt6eKoTnTPHBHlVHQIHZpMxacbFOAkJr -# qAVkYZdz7ikNXTxV+GRb36tC4ByMNxE2DF7vFdvaiZP0CVZ5ByJ2gAhXMdK9+usx -# zVk913qKde1OAuWdv+rndqkAIm8fUlRnr4saSCg7cIbUwCCf116wUJ7EuJDg0vHe -# yhnCeHnBbyH3RZkHEi2ofmfgnFISJZDdMAeVZGVOh20Jp50XBzqokpPzeZ6zc1/g -# yILNyiVgE+RPkjnUQshd1f1PMgn3tns2Cz7bJiVUaqEO3n9qRFgy5JuLae6UweGf -# AeOo3dgLZxikKzYs3hDMaEtJq8IP71cX7QXe6lnMmXU/Hdfz2p897Zd+kU+vZvKI -# 3cwLfuVQgK2RZ2z+Kc3K3dRPz2rXycK5XCuRZmvGab/WbrZiC7wJQapgBodltMI5 -# GMdFrBg9IeF7/rP4EqVQXeKtevTlZXjpuNhhjuR+2DMt/dWufjXpiW91bo3aH6Ea -# jOALXmoxgltCp1K7hrS6gmsvj94cLRf50QQ4U8Qwggd6MIIFYqADAgECAgphDpDS -# AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK -# V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 -# IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 -# ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla -# MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS -# ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT -# H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB -# AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG -# OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S -# 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz -# y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 -# 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u -# M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 -# X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl -# XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP -# 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB -# l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF -# RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM -# CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ -# BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud -# DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO -# 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 -# LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y -# Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p -# Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y -# Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB -# FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw -# cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA -# XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY -# 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj -# 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd -# d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ -# Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf -# wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ -# aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j -# NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B -# xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 -# eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 -# r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I -# RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIVZjCCFWICAQEwgZUwfjELMAkG +# AQCSCNryE+Cewy2m4t/a74wZ7C9YTwv1PyC4BvM/kSWPNs8n0RTe+FvYfU+E9uf0 +# t7nYlAzHjK+plif2BhD+NgdhIUQ8sVwWO39tjvQRHjP2//vSvIfmmkRoML1Ihnjs +# 9kQiZQzYRDYYRp9xSQYmRwQjk5hl8/U7RgOiQDitVHaU7BT1MI92lfZRuIIDDYBd +# vXtbclYJMVOwqZtv0O9zQCret6R+fRSGaDNfEEpcILL+D7RV3M4uaJE4Ta6KAOdv +# V+MVaJp1YXFTZPKtpjHO6d9pHQPZiG7NdC6QbnRGmsa48uNQrb6AfmLKDI1Lp31W +# MogTaX5tZf+CZT9PSuvjOCLNAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE +# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUj9RJL9zNrPcL10RZdMQIXZN7MG8w +# VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh +# dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzQ1ODM4NjAfBgNVHSMEGDAW +# gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v +# d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw +# MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov +# L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx +# XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB +# ACnXo8hjp7FeT+H6iQlV3CcGnkSbFvIpKYafgzYCFo3UHY1VHYJVb5jHEO8oG26Q +# qBELmak6MTI+ra3WKMTGhE1sEIlowTcp4IAs8a5wpCh6Vf4Z/bAtIppP3p3gXk2X +# 8UXTc+WxjQYsDkFiSzo/OBa5hkdW1g4EpO43l9mjToBdqEPtIXsZ7Hi1/6y4gK0P +# mMiwG8LMpSn0n/oSHGjrUNBgHJPxgs63Slf58QGBznuXiRaXmfTUDdrvhRocdxIM +# i8nXQwWACMiQzJSRzBP5S2wUq7nMAqjaTbeXhJqD2SFVHdUYlKruvtPSwbnqSRWT +# GI8s4FEXt+TL3w5JnwVZmZkUFoioQDMMjFyaKurdJ6pnzbr1h6QW0R97fWc8xEIz +# LIOiU2rjwWAtlQqFO8KNiykjYGyEf5LyAJKAO+rJd9fsYR+VBauIEQoYmjnUbTXM +# SY2Lf5KMluWlDOGVh8q6XjmBccpaT+8tCfxpaVYPi1ncnwTwaPQvVq8RjWDRB7Pa +# 8ruHgj2HJFi69+hcq7mWx5nTUtzzFa7RSZfE5a1a5AuBmGNRr7f8cNfa01+tiWjV +# Kk1a+gJUBSP0sIxecFbVSXTZ7bqeal45XSDIisZBkWb+83TbXdTGMDSUFKTAdtC+ +# r35GfsN8QVy59Hb5ZYzAXczhgRmk7NyE6jD0Ym5TKiW5MIIHejCCBWKgAwIBAgIK +# YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV +# BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv +# c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm +# aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw +# OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE +# BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD +# VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG +# 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la +# UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc +# 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D +# dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ +# lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk +# kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 +# A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd +# X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL +# 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd +# sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 +# T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS +# 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI +# bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL +# BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD +# uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv +# c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf +# MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 +# dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf +# MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF +# BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h +# cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA +# YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn +# 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 +# v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b +# pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ +# KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy +# CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp +# mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi +# hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb +# BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS +# oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL +# gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX +# cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCFVgwghVUAgEBMIGVMH4x +# CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt +# b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p +# Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAAGIr1LWuZJt6PkAAAAA +# AYgwDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw +# HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIFxZ +# Yezh3liQqiGQuXNa+zYfoSIbLqOpdEn2ZKskBkisMEIGCisGAQQBgjcCAQwxNDAy +# oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j +# b20wDQYJKoZIhvcNAQEBBQAEggEAjLUrwCXJCPHZulZuKAQSX+MfnIRFAhlN7ru2 +# 6H8rudvhkWgqMISkLb9gFDPR5FhR4sqdYgKW4P0ERao9ypCGi1FWDLqygC2XBbHj +# NEQHBxHJs5SMsMAXNSIcYHqVAvhF3nXoseaNBkhOTrkQ1FS/fW7AfDGRbsiiESzv +# lebf92shZylBFKOsKQLAL0mF/B7xrxHJIj5dgQoD1phATRNHOEQj3jgmkidFWowV +# 4r8MzbxRhAEORbnJexlUoDQJQH3YwxuUyXkTvrYMTKSbGJLlwRaZQbrcBU0k4gCH +# y8Sci+p9Rq+aOTzLCoNrZyh9E7OdwVDm1FJAtY30bV50T2WSFKGCEuIwghLeBgor +# BgEEAYI3AwMBMYISzjCCEsoGCSqGSIb3DQEHAqCCErswghK3AgEDMQ8wDQYJYIZI +# AWUDBAIBBQAwggFRBgsqhkiG9w0BCRABBKCCAUAEggE8MIIBOAIBAQYKKwYBBAGE +# WQoDATAxMA0GCWCGSAFlAwQCAQUABCD7JNcBBSfhlKPL1tN3CEKRKJuT/dZ8RO9K +# orYLXJeLTwIGXvN89YD7GBMyMDIwMDcwMTE0MTYyMC40MDVaMASAAgH0oIHQpIHN +# MIHKMQswCQYDVQQGEwJVUzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1JlZG1vbmQx +# HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9z +# b2Z0IElyZWxhbmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMg +# VFNTIEVTTjoxNzlFLTRCQjAtODI0NjElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUt +# U3RhbXAgU2VydmljZaCCDjkwggTxMIID2aADAgECAhMzAAABDKp4btzMQkzBAAAA +# AAEMMA0GCSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNo +# aW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29y +# cG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEw +# MB4XDTE5MTAyMzIzMTkxNloXDTIxMDEyMTIzMTkxNlowgcoxCzAJBgNVBAYTAlVT +# MQswCQYDVQQIEwJXQTEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z +# b2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVy +# YXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNOOjE3OUUtNEJC +# MC04MjQ2MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNlMIIB +# IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAq5011+XqVJmQKtiw39igeEMv +# CLcZ1forbmxsDkpnCN1SrThKI+n2Pr3zqTzJVgdJFCoKm1ks1gtRJ7HaL6tDkrOw +# 8XJmfJaxyQAluCQ+e40NI+A4w+u59Gy89AVY5lJNrmCva6gozfg1kxw6abV5WWr+ +# PjEpNCshO4hxv3UqgMcCKnT2YVSZzF1Gy7APub1fY0P1vNEuOFKrNCEEvWIKRrqs +# eyBB73G8KD2yw6jfz0VKxNSRAdhJV/ghOyrDt5a+L6C3m1rpr8sqiof3iohv3ANI +# gNqw6ex+4+G+B7JMbIHbGpPdebedL6ePbuBCnbgJoDn340k0aw6ij21GvvUnkQID +# AQABo4IBGzCCARcwHQYDVR0OBBYEFAlCOq9DDIa0A0oqgKtM5vjuZeK+MB8GA1Ud +# IwQYMBaAFNVjOlyKMZDzQ3t8RhvFM2hahW1VMFYGA1UdHwRPME0wS6BJoEeGRWh0 +# dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1RpbVN0 +# YVBDQV8yMDEwLTA3LTAxLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKG +# Pmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljVGltU3RhUENB +# XzIwMTAtMDctMDEuY3J0MAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYIKwYBBQUH +# AwgwDQYJKoZIhvcNAQELBQADggEBAET3xBg/IZ9zdOfwbDGK7cK3qKYt/qUOlbRB +# zgeNjb32K86nGeRGkBee10dVOEGWUw6KtBeWh1LQ70b64/tLtiLcsf9JzaAyDYb1 +# sRmMi5fjRZ753TquaT8V7NJ7RfEuYfvZlubfQD0MVbU4tzsdZdYuxE37V2J9pN89 +# j7GoFNtAnSnCn1MRxENAILgt9XzeQzTEDhFYW0N2DNphTkRPXGjpDmwi6WtkJ5fv +# 0iTyB4dwEC+/ed0lGbFLcytJoMwfTNMdH6gcnHlMzsniornGFZa5PPiV78XoZ9Fe +# upKo8ZKNGhLLLB5GTtqfHex5no3ioVSq+NthvhX0I/V+iXJsopowggZxMIIEWaAD +# AgECAgphCYEqAAAAAAACMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzET +# MBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMV +# TWljcm9zb2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBD +# ZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0xMDA3MDEyMTM2NTVaFw0yNTA3 +# MDEyMTQ2NTVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw +# DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x +# JjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIIBIjANBgkq +# hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqR0NvHcRijog7PwTl/X6f2mUa3RUENWl +# CgCChfvtfGhLLF/Fw+Vhwna3PmYrW/AVUycEMR9BGxqVHc4JE458YTBZsTBED/Fg +# iIRUQwzXTbg4CLNC3ZOs1nMwVyaCo0UN0Or1R4HNvyRgMlhgRvJYR4YyhB50YWeR +# X4FUsc+TTJLBxKZd0WETbijGGvmGgLvfYfxGwScdJGcSchohiq9LZIlQYrFd/Xcf +# PfBXday9ikJNQFHRD5wGPmd/9WbAA5ZEfu/QS/1u5ZrKsajyeioKMfDaTgaRtogI +# Neh4HLDpmc085y9Euqf03GS9pAHBIAmTeM38vMDJRF1eFpwBBU8iTQIDAQABo4IB +# 5jCCAeIwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFNVjOlyKMZDzQ3t8RhvF +# M2hahW1VMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1UdDwQEAwIBhjAP +# BgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2VsuP6KJcYmjRPZSQW9fOmhjE +# MFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kv +# Y3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNybDBaBggrBgEF +# BQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9w +# a2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3J0MIGgBgNVHSABAf8E +# gZUwgZIwgY8GCSsGAQQBgjcuAzCBgTA9BggrBgEFBQcCARYxaHR0cDovL3d3dy5t +# aWNyb3NvZnQuY29tL1BLSS9kb2NzL0NQUy9kZWZhdWx0Lmh0bTBABggrBgEFBQcC +# AjA0HjIgHQBMAGUAZwBhAGwAXwBQAG8AbABpAGMAeQBfAFMAdABhAHQAZQBtAGUA +# bgB0AC4gHTANBgkqhkiG9w0BAQsFAAOCAgEAB+aIUQ3ixuCYP4FxAz2do6Ehb7Pr +# psz1Mb7PBeKp/vpXbRkws8LFZslq3/Xn8Hi9x6ieJeP5vO1rVFcIK1GCRBL7uVOM +# zPRgEop2zEBAQZvcXBf/XPleFzWYJFZLdO9CEMivv3/Gf/I3fVo/HPKZeUqRUgCv +# OA8X9S95gWXZqbVr5MfO9sp6AG9LMEQkIjzP7QOllo9ZKby2/QThcJ8ySif9Va8v +# /rbljjO7Yl+a21dA6fHOmWaQjP9qYn/dxUoLkSbiOewZSnFjnXshbcOco6I8+n99 +# lmqQeKZt0uGc+R38ONiU9MalCpaGpL2eGq4EQoO4tYCbIjggtSXlZOz39L9+Y1kl +# D3ouOVd2onGqBooPiRa6YacRy5rYDkeagMXQzafQ732D8OE7cQnfXXSYIghh2rBQ +# Hm+98eEA3+cxB6STOvdlR3jo+KhIq/fecn5ha293qYHLpwmsObvsxsvYgrRyzR30 +# uIUBHoD7G4kqVDmyW9rIDVWZeodzOwjmmC3qjeAzLhIp9cAvVCch98isTtoouLGp +# 25ayp0Kiyc8ZQU3ghvkqmqMRZjDTu3QyS99je/WZii8bxyGvWbWu3EQ8l1Bx16HS +# xVXjad5XwdHeMMD9zOZN+w2/XU/pnR4ZOC+8z1gFLu8NoFA12u8JJxzVs341Hgi6 +# 2jbb01+P3nSISRKhggLLMIICNAIBATCB+KGB0KSBzTCByjELMAkGA1UEBhMCVVMx +# CzAJBgNVBAgTAldBMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3Nv +# ZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh +# dGlvbnMgTGltaXRlZDEmMCQGA1UECxMdVGhhbGVzIFRTUyBFU046MTc5RS00QkIw +# LTgyNDYxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2WiIwoB +# ATAHBgUrDgMCGgMVAMsg9FQ9pgPLXI2Ld5z7xDS0QAZ9oIGDMIGApH4wfDELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx -# HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z -# b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAYdyF3IVWUDHCQAAAAABhzAN -# BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor -# BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQga11B1DE+ -# y9z0lmEO+MC+bhXPKfWALB7Snkn7G/wCUncwQgYKKwYBBAGCNwIBDDE0MDKgFIAS -# AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN -# BgkqhkiG9w0BAQEFAASCAQBIgx+sFXkLXf7Xbx7opCD3uhpQGEQ4x/LsqTax0bu1 -# GC/cxiI+dodUz+T4hKj1ZQyUH0Zlce32GutY048O9tkr7fQyuohoFUgChdIATEOY -# qAIESFbDT07i7khJfO2pewlhgM+A5ClvBa8HAvV0wOd+2IVgv3pgow1LEJm0/5NB -# E3IFA+hFrqiWALOY0uUep4H20EHMrbqw3YoV3EodIkTj3fC76q4K/bF84EZLUgjY -# e4rmXac8n7A9qR18QzGl8usEJej4OHU4nlUT1J734m+AWIFmfb/Zr2MyXED0V4q4 -# Vbmw3O7xD9STeNYrn5RjPmGPEN04akHxhNUSqLIc9vxQoYIS8DCCEuwGCisGAQQB -# gjcDAwExghLcMIIS2AYJKoZIhvcNAQcCoIISyTCCEsUCAQMxDzANBglghkgBZQME -# AgEFADCCAVQGCyqGSIb3DQEJEAEEoIIBQwSCAT8wggE7AgEBBgorBgEEAYRZCgMB -# MDEwDQYJYIZIAWUDBAIBBQAEIPPK1A0D1n7ZEdgTjKPY4sWiOMtohMqGpFvG55NY -# SFHeAgZepuJh/dEYEjIwMjAwNTI5MTYyNzE1LjMxWjAEgAIB9KCB1KSB0TCBzjEL -# MAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1v -# bmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEpMCcGA1UECxMgTWlj -# cm9zb2Z0IE9wZXJhdGlvbnMgUHVlcnRvIFJpY28xJjAkBgNVBAsTHVRoYWxlcyBU -# U1MgRVNOOjYwQkMtRTM4My0yNjM1MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1T -# dGFtcCBTZXJ2aWNloIIORDCCBPUwggPdoAMCAQICEzMAAAEm37pLIrmCggcAAAAA -# ASYwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp -# bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw -# b3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAw -# HhcNMTkxMjE5MDExNDU5WhcNMjEwMzE3MDExNDU5WjCBzjELMAkGA1UEBhMCVVMx -# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT -# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEpMCcGA1UECxMgTWljcm9zb2Z0IE9wZXJh -# dGlvbnMgUHVlcnRvIFJpY28xJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNOOjYwQkMt -# RTM4My0yNjM1MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNl -# MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnjC+hpxO8w2VdBO18X8L -# Hk6XdfR9yNQ0y+MuBOY7n5YdgkVunvbk/f6q8UoNFAdYQjVLPSAHbi6tUMiNeMGH -# k1U0lUxAkja2W2/szj/ghuFklvfHNBbsuiUShlhRlqcFNS7KXL2iwKDijmOhWJPY -# a2bLEr4W/mQLbSXail5p6m138Ttx4MAVEzzuGI0Kwr8ofIL7z6zCeWDiBM57LrNC -# qHOA2wboeuMsG4O0Oz2LMAzBLbJZPRPnZAD2HdD4HUL2mzZ8wox74Mekb7RzrUP3 -# hiHpxXZceJvhIEKfAgVkB5kTZQnio8A1JijMjw8f4TmsJPdJWpi8ei73sexe8/Yj -# cwIDAQABo4IBGzCCARcwHQYDVR0OBBYEFEmrrB8XsH6YQo3RWKZfxqM0DmFBMB8G -# A1UdIwQYMBaAFNVjOlyKMZDzQ3t8RhvFM2hahW1VMFYGA1UdHwRPME0wS6BJoEeG -# RWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Rp -# bVN0YVBDQV8yMDEwLTA3LTAxLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUH -# MAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljVGltU3Rh -# UENBXzIwMTAtMDctMDEuY3J0MAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYIKwYB -# BQUHAwgwDQYJKoZIhvcNAQELBQADggEBAECW+51o6W/0J/O/npudfjVzMXq0u0cs -# HjqXpdRyH6o03jlmY5MXAui3cmPBKufijJxD2pMRPVMUNh3VA0PQuJeYrP06oFdq -# LpLxd3IJARm98vzaMgCz2nCwBDpe9X2M3Js9K1GAX+w4Az8N7J+Z6P1OD0VxHBdq -# eTaqDN1lk1vwagTN7t/WitxMXRDz0hRdYiWbATBAVgXXCOfzs3hnEv1n/EDab9HX -# OLMXKVY/+alqYKdV9lkuRp8Us1Q1WZy9z72Azu9x4mzft3fJ1puTjBHo5tHfixZo -# ummbI+WwjVCrku7pskJahfNi5amSgrqgR6nWAwvpJELccpVLdSxxmG0wggZxMIIE -# WaADAgECAgphCYEqAAAAAAACMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJV -# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE -# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9v -# dCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0xMDA3MDEyMTM2NTVaFw0y -# NTA3MDEyMTQ2NTVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u -# MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp -# b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIIBIjAN -# BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqR0NvHcRijog7PwTl/X6f2mUa3RU -# ENWlCgCChfvtfGhLLF/Fw+Vhwna3PmYrW/AVUycEMR9BGxqVHc4JE458YTBZsTBE -# D/FgiIRUQwzXTbg4CLNC3ZOs1nMwVyaCo0UN0Or1R4HNvyRgMlhgRvJYR4YyhB50 -# YWeRX4FUsc+TTJLBxKZd0WETbijGGvmGgLvfYfxGwScdJGcSchohiq9LZIlQYrFd -# /XcfPfBXday9ikJNQFHRD5wGPmd/9WbAA5ZEfu/QS/1u5ZrKsajyeioKMfDaTgaR -# togINeh4HLDpmc085y9Euqf03GS9pAHBIAmTeM38vMDJRF1eFpwBBU8iTQIDAQAB -# o4IB5jCCAeIwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFNVjOlyKMZDzQ3t8 -# RhvFM2hahW1VMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1UdDwQEAwIB -# hjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2VsuP6KJcYmjRPZSQW9fO -# mhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9w -# a2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNybDBaBggr -# BgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNv -# bS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3J0MIGgBgNVHSAB -# Af8EgZUwgZIwgY8GCSsGAQQBgjcuAzCBgTA9BggrBgEFBQcCARYxaHR0cDovL3d3 -# dy5taWNyb3NvZnQuY29tL1BLSS9kb2NzL0NQUy9kZWZhdWx0Lmh0bTBABggrBgEF -# BQcCAjA0HjIgHQBMAGUAZwBhAGwAXwBQAG8AbABpAGMAeQBfAFMAdABhAHQAZQBt -# AGUAbgB0AC4gHTANBgkqhkiG9w0BAQsFAAOCAgEAB+aIUQ3ixuCYP4FxAz2do6Eh -# b7Prpsz1Mb7PBeKp/vpXbRkws8LFZslq3/Xn8Hi9x6ieJeP5vO1rVFcIK1GCRBL7 -# uVOMzPRgEop2zEBAQZvcXBf/XPleFzWYJFZLdO9CEMivv3/Gf/I3fVo/HPKZeUqR -# UgCvOA8X9S95gWXZqbVr5MfO9sp6AG9LMEQkIjzP7QOllo9ZKby2/QThcJ8ySif9 -# Va8v/rbljjO7Yl+a21dA6fHOmWaQjP9qYn/dxUoLkSbiOewZSnFjnXshbcOco6I8 -# +n99lmqQeKZt0uGc+R38ONiU9MalCpaGpL2eGq4EQoO4tYCbIjggtSXlZOz39L9+ -# Y1klD3ouOVd2onGqBooPiRa6YacRy5rYDkeagMXQzafQ732D8OE7cQnfXXSYIghh -# 2rBQHm+98eEA3+cxB6STOvdlR3jo+KhIq/fecn5ha293qYHLpwmsObvsxsvYgrRy -# zR30uIUBHoD7G4kqVDmyW9rIDVWZeodzOwjmmC3qjeAzLhIp9cAvVCch98isTtoo -# uLGp25ayp0Kiyc8ZQU3ghvkqmqMRZjDTu3QyS99je/WZii8bxyGvWbWu3EQ8l1Bx -# 16HSxVXjad5XwdHeMMD9zOZN+w2/XU/pnR4ZOC+8z1gFLu8NoFA12u8JJxzVs341 -# Hgi62jbb01+P3nSISRKhggLSMIICOwIBATCB/KGB1KSB0TCBzjELMAkGA1UEBhMC -# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV -# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEpMCcGA1UECxMgTWljcm9zb2Z0IE9w -# ZXJhdGlvbnMgUHVlcnRvIFJpY28xJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNOOjYw -# QkMtRTM4My0yNjM1MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2 -# aWNloiMKAQEwBwYFKw4DAhoDFQAKZzI5aZnESumrToHx3Lqgxnr//KCBgzCBgKR+ -# MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS -# ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMT -# HU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMA0GCSqGSIb3DQEBBQUAAgUA -# 4nuQTDAiGA8yMDIwMDUyOTE3NDQ0NFoYDzIwMjAwNTMwMTc0NDQ0WjB3MD0GCisG -# AQQBhFkKBAExLzAtMAoCBQDie5BMAgEAMAoCAQACAiZJAgH/MAcCAQACAhEjMAoC -# BQDifOHMAgEAMDYGCisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEA -# AgMHoSChCjAIAgEAAgMBhqAwDQYJKoZIhvcNAQEFBQADgYEAprmyJTXdH9FmQZ0I -# mRSJdjc/RrSqDm8DUEq/h3FL73G/xvg9MbQj1J/h3hdlSIPcQXjrhL8hud/vyF0j -# IFaTK5YOcixkX++9t7Vz3Mn0KkQo8F4DNSyZEPpz682AyKKwLMJDy52pFFFKNP5l -# NpOz6YY1Od1xvk4nyN1WwfLnGswxggMNMIIDCQIBATCBkzB8MQswCQYDVQQGEwJV -# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE -# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGlt -# ZS1TdGFtcCBQQ0EgMjAxMAITMwAAASbfuksiuYKCBwAAAAABJjANBglghkgBZQME -# AgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJ -# BDEiBCB0IE0Q6P23RQlh8TFyp57UQQUF/sbui7mOMStRgTFZxTCB+gYLKoZIhvcN -# AQkQAi8xgeowgecwgeQwgb0EIDb9z++evV5wDO9qk5ZnbEZ8CTOuR+kZyu8xbTsJ -# CXUPMIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x -# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv -# bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAEm -# 37pLIrmCggcAAAAAASYwIgQgtwi02bvsGAOdpAxEF607G6g9PlyS8vc2bAUSHovH -# /IIwDQYJKoZIhvcNAQELBQAEggEAEMCfsXNudrjztjI6JNyNDVpdF1axRVcGiNy6 -# 67pgb1EePsjA2EaBB+5ZjgO/73JxuiVgsoXgH7em8tKG5RQJtcm5obVDb+jKksK4 -# qcFLA1f7seQRGfE06UAPnSFh2GqMtTNJGCXWwqWLH2LduTjOqPt8Nupo16ABFIT2 -# akTzBSJ81EHBkEU0Et6CgeaZiBYrCCXUtD+ASvLDkPSrjweQGu3Zk1SSROEzxMY9 -# jdlGfMkK2krMd9ub9UZ13RcQDijJqo+h6mz76pAuiFFvuQl6wMoSGFaaUQwfd+WQ -# gXlVVX/A9JFBihrxnDVglEPlsIOxCHkTeIxLfnAkCbax+9pevA== -# SIG # End signature block +# HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9z +# b2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwDQYJKoZIhvcNAQEFBQACBQDipo0MMCIY +# DzIwMjAwNzAxMTIxODIwWhgPMjAyMDA3MDIxMjE4MjBaMHQwOgYKKwYBBAGEWQoE +# ATEsMCowCgIFAOKmjQwCAQAwBwIBAAICE70wBwIBAAICEeIwCgIFAOKn3owCAQAw +# NgYKKwYBBAGEWQoEAjEoMCYwDAYKKwYBBAGEWQoDAqAKMAgCAQACAwehIKEKMAgC +# AQACAwGGoDANBgkqhkiG9w0BAQUFAAOBgQCOPjlHOH8nYtgt2XnpKXenxPUR03ED +# xPBm8XR5Z1vIq53RU9jG6yYcYNTdK+q38SGZtu0W/SgagTfKCQhjhRakuv7rGSs2 +# dlhx9LGCoc/q1vqmZpRSjkqWVcc/NzmldUWIWnLlV6rmLGoDmfCH5BcsiU6Eo6wU +# iUVwnnXoqsCaBzGCAw0wggMJAgEBMIGTMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +# EwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3Nv +# ZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBD +# QSAyMDEwAhMzAAABDKp4btzMQkzBAAAAAAEMMA0GCWCGSAFlAwQCAQUAoIIBSjAa +# BgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwLwYJKoZIhvcNAQkEMSIEIDpwhjyu +# zgu3Kmxpnpz86ZlthBqEzG5vaEMOkYRyuFCaMIH6BgsqhkiG9w0BCRACLzGB6jCB +# 5zCB5DCBvQQgg5AWKX7M1+m2//+V7qmRvt1K/ww5Muu8XzGJBqygVCkwgZgwgYCk +# fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH +# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD +# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAQyqeG7czEJMwQAA +# AAABDDAiBCD11urvv5vgo4gFVQ2NMVrzgxT87Yuiq16YdswYbaYeITANBgkqhkiG +# 9w0BAQsFAASCAQAi3q8hwcT2ft4b2EleaiyZxOImV/cKusmth1dtCh5/Jb0GbOld +# f5cSalrjf42MNPodWAtgmWozkYrQF6HxnsOiYiamfRA8E3E7xyRMy7AFfAhjcwMi +# xaW4Iye6E1Ec6LtULANxfDtG/KIdCWdZxKqOezL3nzFNQWmm1mXPV+UnKpnJkA3E +# DsQOUWk8J6ojDurhrP536WI+3arg8PcnppHBLd/xNKYdlsTb+6qndgzKXkDDt1CV +# 4zCyuZ7bO8eyZAmNoSZz22k7vus9UjBz/CDhXylo20N43nr29rWPItUgH4uvOGQn +# t26Y/yjBaQImz32psrfJEMbQ7cl789s8WOx8 +# SIG # End signature block \ No newline at end of file diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index cea1e2fd4d7..83e47c74c5f 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -105,7 +105,7 @@ public interface IExecutionContext : IRunnerService // others void ForceTaskComplete(); void RegisterPostJobStep(IStep step); - void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location); + void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location, Dictionary envData); } public sealed class ExecutionContext : RunnerService, IExecutionContext @@ -270,13 +270,28 @@ public void RegisterPostJobStep(IStep step) /// Helper function used in CompositeActionHandler::RunAsync to /// add a child node, aka a step, to the current job to the Root.JobSteps based on the location. /// - public void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location) + public void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location, Dictionary envData) { // TODO: For UI purposes, look at figuring out how to condense steps in one node => maybe use the same previous GUID var newGuid = Guid.NewGuid(); step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null); step.ExecutionContext.ExpressionValues["inputs"] = inputsData; - // TODO: confirm whether not copying message contexts is safe + + // Add the composite action environment variables to each step. + // If the key already exists, we override it since the composite action env variables will have higher precedence + // Note that for each composite action step, it's environment variables will be set in the StepRunner automatically + // step.ExecutionContext.SetEnvironmentVariables(envData); +#if OS_WINDOWS + var envContext = new DictionaryContextData(); +#else + var envContext = new CaseSensitiveDictionaryContextData(); +#endif + foreach (var pair in envData) + { + envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty); + } + step.ExecutionContext.ExpressionValues["env"] = envContext; + Root.JobSteps.Insert(location, step); } diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index 7c5b25ed5fb..ddf5821ad39 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -84,10 +84,8 @@ public Task RunAsync(ActionRunStage stage) actionRunner.Stage = stage; actionRunner.Condition = aStep.Condition; actionRunner.DisplayName = aStep.DisplayName; - // TODO: Do we need to add any context data from the job message? - // (See JobExtension.cs ~line 236) - ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location); + ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location, Environment); location++; } diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index e75d2e106f8..966f73dc1a4 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -93,12 +93,28 @@ public async Task RunAsync(IExecutionContext jobContext) #else var envContext = new CaseSensitiveDictionaryContextData(); #endif - step.ExecutionContext.ExpressionValues["env"] = envContext; + // Global env foreach (var pair in step.ExecutionContext.EnvironmentVariables) { envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty); } + // Stomps over with outside step env + if (step.ExecutionContext.ExpressionValues.TryGetValue("env", out var envContextData)) + { +#if OS_WINDOWS + var dict = envContextData as DictionaryContextData; +#else + var dict = envContextData as CaseSensitiveDictionaryContextData; +#endif + foreach (var pair in dict) + { + envContext[pair.Key] = pair.Value; + } + } + + step.ExecutionContext.ExpressionValues["env"] = envContext; + bool evaluateStepEnvFailed = false; if (step is IActionRunner actionStep) { diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index 55076e670e5..f09a905bbe8 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -160,8 +160,7 @@ public String EvaluateStepDisplayName( } public List LoadCompositeSteps( - TemplateToken token - ) + TemplateToken token) { var result = default(List); if (token != null && token.Type != TokenType.Null) diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index 1dfee2252ad..2fde1bcb675 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -426,11 +426,11 @@ public async Task StepEnvOverrideJobEnvContext() Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); #if OS_WINDOWS - Assert.Equal("100", _ec.Object.ExpressionValues["env"].AssertDictionary("env")["env1"].AssertString("100")); - Assert.Equal("github_actions", _ec.Object.ExpressionValues["env"].AssertDictionary("env")["env2"].AssertString("github_actions")); + Assert.Equal("100", step1.Object.ExecutionContext.ExpressionValues["env"].AssertDictionary("env")["env1"].AssertString("100")); + Assert.Equal("github_actions", step1.Object.ExecutionContext.ExpressionValues["env"].AssertDictionary("env")["env2"].AssertString("github_actions")); #else - Assert.Equal("100", _ec.Object.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env1"].AssertString("100")); - Assert.Equal("github_actions", _ec.Object.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env2"].AssertString("github_actions")); + Assert.Equal("100", step1.Object.ExecutionContext.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env1"].AssertString("100")); + Assert.Equal("github_actions", step1.Object.ExecutionContext.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env2"].AssertString("github_actions")); #endif } } @@ -463,13 +463,13 @@ public async Task PopulateEnvContextForEachStep() // Assert. Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); #if OS_WINDOWS - Assert.Equal("1000", _ec.Object.ExpressionValues["env"].AssertDictionary("env")["env1"].AssertString("1000")); - Assert.Equal("github_actions", _ec.Object.ExpressionValues["env"].AssertDictionary("env")["env3"].AssertString("github_actions")); - Assert.False(_ec.Object.ExpressionValues["env"].AssertDictionary("env").ContainsKey("env2")); + Assert.Equal("1000", step2.Object.ExecutionContext.ExpressionValues["env"].AssertDictionary("env")["env1"].AssertString("1000")); + Assert.Equal("github_actions", step2.Object.ExecutionContext.ExpressionValues["env"].AssertDictionary("env")["env3"].AssertString("github_actions")); + Assert.False(step2.Object.ExecutionContext.ExpressionValues["env"].AssertDictionary("env").ContainsKey("env2")); #else - Assert.Equal("1000", _ec.Object.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env1"].AssertString("1000")); - Assert.Equal("github_actions", _ec.Object.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env3"].AssertString("github_actions")); - Assert.False(_ec.Object.ExpressionValues["env"].AssertCaseSensitiveDictionary("env").ContainsKey("env2")); + Assert.Equal("1000", step2.Object.ExecutionContext.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env1"].AssertString("1000")); + Assert.Equal("github_actions", step2.Object.ExecutionContext.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env3"].AssertString("github_actions")); + Assert.False(step2.Object.ExecutionContext.ExpressionValues["env"].AssertCaseSensitiveDictionary("env").ContainsKey("env2")); #endif } } @@ -501,11 +501,11 @@ public async Task PopulateEnvContextAfterSetupStepsContext() // Assert. Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); #if OS_WINDOWS - Assert.Equal("1000", _ec.Object.ExpressionValues["env"].AssertDictionary("env")["env1"].AssertString("1000")); - Assert.Equal("something", _ec.Object.ExpressionValues["env"].AssertDictionary("env")["env2"].AssertString("something")); + Assert.Equal("1000", step2.Object.ExecutionContext.ExpressionValues["env"].AssertDictionary("env")["env1"].AssertString("1000")); + Assert.Equal("something", step2.Object.ExecutionContext.ExpressionValues["env"].AssertDictionary("env")["env2"].AssertString("something")); #else - Assert.Equal("1000", _ec.Object.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env1"].AssertString("1000")); - Assert.Equal("something", _ec.Object.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env2"].AssertString("something")); + Assert.Equal("1000", step2.Object.ExecutionContext.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env1"].AssertString("1000")); + Assert.Equal("something", step2.Object.ExecutionContext.ExpressionValues["env"].AssertCaseSensitiveDictionary("env")["env2"].AssertString("something")); #endif } } @@ -602,7 +602,7 @@ private Mock CreateStep(TestHostContext hc, TaskResult result, st stepContext.Setup(x => x.WriteDebug).Returns(true); stepContext.Setup(x => x.Variables).Returns(_variables); stepContext.Setup(x => x.EnvironmentVariables).Returns(_env); - stepContext.Setup(x => x.ExpressionValues).Returns(_contexts); + stepContext.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); stepContext.Setup(x => x.ExpressionFunctions).Returns(new List()); stepContext.Setup(x => x.JobContext).Returns(_jobContext); stepContext.Setup(x => x.StepsContext).Returns(_stepContext); From 5822a38c39eddc2dbbfe902466c0d3eace9f1259 Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Wed, 8 Jul 2020 11:20:38 -0400 Subject: [PATCH 80/86] Add bash command for running custom runner (#569) --- docs/contribute.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/contribute.md b/docs/contribute.md index 9c094ad542a..7f9b6d3230f 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -51,6 +51,19 @@ cd ./src ./dev.(sh/cmd) test # run all unit tests before git commit/push ``` +View logs: +```bash +cd runner/_layout/_diag +ls +cat (Runner/Worker)_TIMESTAMP.log # view your log file +``` + +Run Runner: +```bash +cd runner/_layout +./run.sh # run your custom runner +``` + ### Editors [Using Visual Studio Code](https://code.visualstudio.com/) From 9d7bd4706b5e96108f09f2b9fd32c5d170d5cec2 Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Wed, 8 Jul 2020 17:15:16 -0400 Subject: [PATCH 81/86] Improve Error Messaging for Actions by Using ExecutionContext's FileTable as Single Source of Truth and by Passing FileID to All Children Tokens. (#564) * Composite Action Run Steps * Env Flow => Able to get env variables and overwrite current env variables => but it doesn't 'stick' * clean up * Clean up trace messages + add Trace debug in ActionManager * Add debugging message * Optimize runtime of code * Change String to string * Add comma to Composite * Change JobSteps to a List, Change Register Step function name * Add TODO, remove unn. content * Remove unnecessary code * Fix unit tests * Fix env format * Remove comment * Remove TODO message for context * Add verbose trace logs which are only viewable by devs * Initial Start for FileTable stuff * Progress towards passing FileTable or FileID or FileName * Sort usings in Composite Action Handler * Change 0 to location * Update context variables in composite action yaml * Add helpful error message for null steps * Pass fileID to all children token of root action token * Change confusing term context => templateContext, Eliminate _fileTable and only use ExecutionContext.FileTable + update this table when need be * Remove unnessary FileID attribute from CompositeActionExecutionData * Clean up file path for error message * Remove todo * Fix Workflow Step Env overiding Parent Env * Remove env in composite action scope * Clean up * Revert back * revert back * add back envToken * Remove unnecessary code * Add file length check * Clean up * Figure out how to handle set-env edge cases * formatting * fix unit tests * Fix windows unit test syntax error * Fix period * Sanity check for fileTable add + remove unn. code * revert back * Add back line break * Fix null errors * Address situation if FileTable is null + add sanity check for adding file to fileTable * add line * Revert * Fix unit tests to instantiate a FileTable * Fix logic for trimming manifestfile path * Add null check * Add filetable to testing file, remove ? since we know filetable should never be non null --- src/Runner.Worker/ActionManifestManager.cs | 52 ++++++++++++------- src/Test/L0/Worker/ActionManagerL0.cs | 1 + src/Test/L0/Worker/ActionManifestManagerL0.cs | 1 + src/Test/L0/Worker/ActionRunnerL0.cs | 1 + 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 9095f498ddd..ae5ca73d7c0 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -33,8 +33,6 @@ public interface IActionManifestManager : IRunnerService public sealed class ActionManifestManager : RunnerService, IActionManifestManager { private TemplateSchema _actionManifestSchema; - private IReadOnlyList _fileTable; - public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); @@ -55,22 +53,39 @@ public override void Initialize(IHostContext hostContext) public ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile) { - var context = CreateContext(executionContext); + var templateContext = CreateContext(executionContext); ActionDefinitionData actionDefinition = new ActionDefinitionData(); + + // Clean up file name real quick + // Instead of using Regex which can be computationally expensive, + // we can just remove the # of characters from the fileName according to the length of the basePath + string basePath = HostContext.GetDirectory(WellKnownDirectory.Actions); + string fileRelativePath = manifestFile; + if (manifestFile.Contains(basePath)) + { + fileRelativePath = manifestFile.Remove(0, basePath.Length + 1); + } + try { var token = default(TemplateToken); // Get the file ID - var fileId = context.GetFileId(manifestFile); - _fileTable = context.GetFileTable(); + var fileId = templateContext.GetFileId(fileRelativePath); + + // Add this file to the FileTable in executionContext if it hasn't been added already + // we use > since fileID is 1 indexed + if (fileId > executionContext.FileTable.Count) + { + executionContext.FileTable.Add(fileRelativePath); + } // Read the file var fileContent = File.ReadAllText(manifestFile); using (var stringReader = new StringReader(fileContent)) { - var yamlObjectReader = new YamlObjectReader(null, stringReader); - token = TemplateReader.Read(context, "action-root", yamlObjectReader, fileId, out _); + var yamlObjectReader = new YamlObjectReader(fileId, stringReader); + token = TemplateReader.Read(templateContext, "action-root", yamlObjectReader, fileId, out _); } var actionMapping = token.AssertMapping("action manifest root"); @@ -89,11 +104,11 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani break; case "inputs": - ConvertInputs(context, actionPair.Value, actionDefinition); + ConvertInputs(templateContext, actionPair.Value, actionDefinition); break; case "runs": - actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value); + actionDefinition.Execution = ConvertRuns(executionContext, templateContext, actionPair.Value); break; default: Trace.Info($"Ignore action property {propertyName}."); @@ -104,24 +119,24 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani catch (Exception ex) { Trace.Error(ex); - context.Errors.Add(ex); + templateContext.Errors.Add(ex); } - if (context.Errors.Count > 0) + if (templateContext.Errors.Count > 0) { - foreach (var error in context.Errors) + foreach (var error in templateContext.Errors) { Trace.Error($"Action.yml load error: {error.Message}"); executionContext.Error(error.Message); } - throw new ArgumentException($"Fail to load {manifestFile}"); + throw new ArgumentException($"Fail to load {fileRelativePath}"); } if (actionDefinition.Execution == null) { executionContext.Debug($"Loaded action.yml file: {StringUtil.ConvertToJson(actionDefinition)}"); - throw new ArgumentException($"Top level 'runs:' section is required for {manifestFile}"); + throw new ArgumentException($"Top level 'runs:' section is required for {fileRelativePath}"); } else { @@ -282,13 +297,10 @@ private TemplateContext CreateContext( result.ExpressionFunctions.Add(item); } - // Add the file table - if (_fileTable?.Count > 0) + // Add the file table from the Execution Context + for (var i = 0; i < executionContext.FileTable.Count; i++) { - for (var i = 0; i < _fileTable.Count; i++) - { - result.GetFileId(_fileTable[i]); - } + result.GetFileId(executionContext.FileTable[i]); } return result; diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 58ffea5aa9b..b1ccb284b89 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -3584,6 +3584,7 @@ private void Setup([CallerMemberName] string name = "", bool newActionMetadata = _ec.Setup(x => x.Variables).Returns(new Variables(_hc, variables)); _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); + _ec.Setup(x => x.FileTable).Returns(new List()); _ec.Setup(x => x.Plan).Returns(new TaskOrchestrationPlanReference()); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"[{tag}]{message}"); }); _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, string message) => { _hc.GetTrace().Info($"[{issue.Type}]{issue.Message ?? message}"); }); diff --git a/src/Test/L0/Worker/ActionManifestManagerL0.cs b/src/Test/L0/Worker/ActionManifestManagerL0.cs index 07f99a0aec8..73734192260 100644 --- a/src/Test/L0/Worker/ActionManifestManagerL0.cs +++ b/src/Test/L0/Worker/ActionManifestManagerL0.cs @@ -759,6 +759,7 @@ private void Setup([CallerMemberName] string name = "") _ec.Setup(x => x.Variables).Returns(new Variables(_hc, new Dictionary())); _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); + _ec.Setup(x => x.FileTable).Returns(new List()); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"{tag}{message}"); }); _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, string message) => { _hc.GetTrace().Info($"[{issue.Type}]{issue.Message ?? message}"); }); } diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index 662ea307d61..1851f47d750 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -379,6 +379,7 @@ private void Setup([CallerMemberName] string name = "") _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); _ec.Setup(x => x.IntraActionState).Returns(new Dictionary()); _ec.Setup(x => x.EnvironmentVariables).Returns(new Dictionary()); + _ec.Setup(x => x.FileTable).Returns(new List()); _ec.Setup(x => x.SetGitHubContext(It.IsAny(), It.IsAny())); _ec.Setup(x => x.GetGitHubContext(It.IsAny())).Returns("{\"foo\":\"bar\"}"); _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); From 6c3958f365c3cbefec28e5b71933831e5ac6496f Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Mon, 13 Jul 2020 12:30:31 -0400 Subject: [PATCH 82/86] Composite Run Steps ADR (#554) * start * Inputs + Outputs * Clarify docs * Finish Environment * Add if condition * Clarify language * Update 0549-composite-run-steps.md * timeout-minutes * Finish * add relevant example * Fix syntax * fix env example * fix yaml syntax * Update 0549-composite-run-steps.md * Update file names, add more relevant example if condition * Add note to continue-on-error * Apply changes to If Condition * bolding * Update 0549-composite-run-steps.md * Update 0549-composite-run-steps.md * Update 0549-composite-run-steps.md * Update 0549-composite-run-steps.md * Syntax support + spacing * Add guiding principles. * Update 0549-composite-run-steps.md * Reverse order. * Update 0549-composite-run-steps.md * change from job to step * Update 0549-composite-run-steps.md * Update 0549-composite-run-steps.md * Update 0549-composite-run-steps.md * Add Secrets * Update 0549-composite-run-steps.md * Update 0549-composite-run-steps.md * Fix output example * Fix output example * Fix action examples to use using. * fix output variable name * update workingDir + env * Defaults + continue-on-error * Update Outputs Section * Eliminate Env * Secrets * Update timeout-minutes * Update 0549-composite-run-steps.md * Update 0549-composite-run-steps.md * Fix example. * Remove TODOs * Update 0549-composite-run-steps.md * Update 0549-composite-run-steps.md --- docs/adrs/0549-composite-run-steps.md | 275 ++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 docs/adrs/0549-composite-run-steps.md diff --git a/docs/adrs/0549-composite-run-steps.md b/docs/adrs/0549-composite-run-steps.md new file mode 100644 index 00000000000..fef72cd92de --- /dev/null +++ b/docs/adrs/0549-composite-run-steps.md @@ -0,0 +1,275 @@ +# ADR 054x: Composite Run Steps + +**Date**: 2020-06-17 + +**Status**: Proposed + +**Relevant PR**: https://github.com/actions/runner/pull/549 + +## Context + +Customers want to be able to compose actions from actions (ex: https://github.com/actions/runner/issues/438) + +An important step towards meeting this goal is to build in functionality for actions where users can simply execute any number of steps. + +## Guiding Principles + +We don't want the workflow author to need to know how the internal workings of the action work. Users shouldn't know the internal workings of the composite action (for example, `default.shell` and `default.workingDir` should not be inherited from the workflow file to the action file). When deciding how to design certain parts of composite run steps, we want to think one logical step from the consumer. + +A composite action is treated as **one** individual job step (aka encapsulation). + + +## Decision + +**In this ADR, we only support running multiple run steps in an Action.** In doing so, we build in support for mapping and flowing the inputs, outputs, and env variables (ex: All nested steps should have access to its parents' input variables and nested steps can overwrite the input variables). + +## Steps + +Example `workflow.yml` + +```yaml +jobs: + build: + runs-on: self-hosted + steps: + - id: step1 + uses: actions/setup-python@v1 + - id: step2 + uses: actions/setup-node@v2 + - uses: actions/checkout@v2 + - uses: user/composite@v1 + - name: workflow step 1 + run: echo hello world 3 + - name: workflow step 2 + run: echo hello world 4 +``` + +Example `user/composite/action.yml` + +```yaml +runs: + using: "composite" + steps: + - run: pip install -r requirements.txt + - run: npm install +``` + +Example Output + +```yaml +[npm installation output] +[pip requirements output] +echo hello world 3 +echo hello world 4 +``` + +We add a token called "composite" which allows our Runner code to process composite actions. By invoking "using: composite", our Runner code then processes the "steps" attribute, converts this template code to a list of steps, and finally runs each run step sequentially. If any step fails and there are no `if` conditions defined, the whole composite action job fails. + +## Inputs + +Example `workflow.yml`: + +```yaml +steps: + - id: foo + uses: user/composite@v1 + with: + your_name: "Octocat" +``` + +Example `user/composite/action.yml`: + +```yaml +inputs: + your_name: + description: 'Your name' + default: 'Ethan' +runs: + using: "composite" + steps: + - run: echo hello ${{ inputs.your_name }} +``` + +Example Output: + +``` +hello Octocat +``` + +Each input variable in the composite action is only viewable in its own scope. + +## Outputs + +Example `workflow.yml`: + +```yaml +... +steps: + - id: foo + uses: user/composite@v1 + - run: echo random-number ${{ steps.foo.outputs.random-number }} +``` + +Example `user/composite/action.yml`: + +```yaml +outputs: + random-number: + description: "Random number" + value: ${{ steps.random-number-generator.outputs.random-id }} +runs: + using: "composite" + steps: + - id: random-number-generator + run: echo "::set-output name=random-number::$(echo $RANDOM)" +``` + +Example Output: + +``` +::set-output name=my-output::43243 +random-number 43243 +``` + +Each of the output variables from the composite action is viewable from the workflow file that uses the composite action. In other words, every child action output(s) is viewable only by its parent using dot notation (ex `steps.foo.outputs.random-number`). + +Moreover, the output ids are only accessible within the scope where it was defined. Note that in the example above, in our `workflow.yml` file, it should not have access to output id (i.e. `random-id`). The reason why we are doing this is because we don't want to require the workflow author to know the internal workings of the composite action. + +## Context + +Similar to the workflow file, the composite action has access to the [same context objects](https://help.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#contexts) (ex: `github`, `env`, `strategy`). + +## Environment + +In the Composite Action, you'll only be able to use `::set-env::` to set environment variables just like you could with other actions. + +## Secrets + +**Note** : This feature will be focused on in a future ADR. + +We'll pass the secrets from the composite action's parents (ex: the workflow file) to the composite action. Secrets can be created in the composite action with the secrets context. In the actions yaml, we'll automatically mask the secret. + + +## If Condition + +Example `workflow.yml`: + +```yaml +steps: + - run: exit 1 + - uses: user/composite@v1 # <--- this will run, as it's marked as always runing + if: always() +``` + +Example `user/composite/action.yml`: + +```yaml +runs: + using: "composite" + steps: + - run: echo "just succeeding" + - run: echo "I will run, as my current scope is succeeding" + if: success() + - run: exit 1 + - run: echo "I will not run, as my current scope is now failing" +``` + +See the paragraph below for a rudimentary approach (thank you to @cybojenix for the idea, example, and explanation for this approach): + +The `if` statement in the parent (in the example above, this is the `workflow.yml`) shows whether or not we should run the composite action. So, our composite action will run since the `if` condition for running the composite action is `always()`. + +**Note that the if condition on the parent does not propogate to the rest of its children though.** + +In the child action (in this example, this is the `action.yml`), it starts with a clean slate (in other words, no imposing if conditions). Similar to the logic in the paragraph above, `echo "I will run, as my current scope is succeeding"` will run since the `if` condition checks if the previous steps **within this composite action** has not failed. `run: echo "I will not run, as my current scope is now failing"` will not run since the previous step resulted in an error and by default, the if expression is set to `success()` if the if condition is not set for a step. + + +What if a step has `cancelled()`? We do the opposite of our approach above if `cancelled()` is used for any of our composite run steps. We will cancel any step that has this condition if the workflow is cancelled at all. + +## Timeout-minutes + +Example `workflow.yml`: + +```yaml +steps: + - id: bar + uses: user/test@v1 + timeout-minutes: 50 +``` + +Example `user/composite/action.yml`: + +```yaml +runs: + using: "composite" + steps: + - id: foo1 + run: echo test 1 + timeout-minutes: 10 + - id: foo2 + run: echo test 2 + - id: foo3 + run: echo test 3 + timeout-minutes: 10 +``` + +A composite action in its entirety is a job. You can set both timeout-minutes for the whole composite action or its steps as long as the the sum of the `timeout-minutes` for each composite action step that has the attribute `timeout-minutes` is less than or equals to `timeout-minutes` for the composite action. There is no default timeout-minutes for each composite action step. + +If the time taken for any of the steps in combination or individually exceed the whole composite action `timeout-minutes` attribute, the whole job will fail (1). If an individual step exceeds its own `timeout-minutes` attribute but the total time that has been used including this step is below the overall composite action `timeout-minutes`, the individual step will fail but the rest of the steps will run based on their own `timeout-minutes` attribute (they will still abide by condition (1) though). + +For reference, in the example above, if the composite step `foo1` takes 11 minutes to run, that step will fail but the rest of the steps, `foo1` and `foo2`, will proceed as long as their total runtime with the previous failed `foo1` action is less than the composite action's `timeout-minutes` (50 minutes). If the composite step `foo2` takes 51 minutes to run, it will cause the whole composite action job to fail. I + +The rationale behind this is that users can configure their steps with the `if` condition to conditionally set how steps rely on each other. Due to the additional capabilities that are offered with combining `timeout-minutes` and/or `if`, we wanted the `timeout-minutes` condition to be as dumb as possible and not effect other steps. + +[Usage limits still apply](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions?query=if%28%29#usage-limits) + + +## Continue-on-error + +Example `workflow.yml`: + +```yaml +steps: + - run: exit 1 + - id: bar + uses: user/test@v1 + continue-on-error: false + - id: foo + run: echo "Hello World" <------- This step will not run +``` + +Example `user/composite/action.yml`: + +```yaml +runs: + using: "composite" + steps: + - run: exit 1 + continue-on-error: true + - run: echo "Hello World 2" <----- This step will run +``` + +If any of the steps fail in the composite action and the `continue-on-error` is set to `false` for the whole composite action step in the workflow file, then the steps below it will run. On the flip side, if `continue-on-error` is set to `true` for the whole composite action step in the workflow file, the next job step will run. + +For the composite action steps, it follows the same logic as above. In this example, `"Hello World 2"` will be outputted because the previous step has `continue-on-error` set to `true` although that previous step errored. + +## Defaults + +The composite action author will be required to set the `shell` and `workingDir` of the composite action. Moreover, the composite action author will be able to explicitly set the shell for each composite run step. The workflow author will not have the ability to change these attributes. + +## Visualizing Composite Action in the GitHub Actions UI +We want all the composite action's steps to be condensed into the original composite action node. + +Here is a visual represenation of the [first example](#Steps) + +```yaml +| composite_action_node | + | echo hello world 1 | + | echo hello world 2 | +| echo hello world 3 | +| echo hello world 4 | + +``` + + +## Conclusion +This ADR lays the framework for eventually supporting nested Composite Actions within Composite Actions. This ADR allows for users to run multiple run steps within a GitHub Composite Action with the support of inputs, outputs, environment, and context for use in any steps as well as the if, timeout-minutes, and the continue-on-error attributes for each Composite Action step. From cb2b32378157f07a7f386b3a9a5e54f5bafbcb7f Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Mon, 13 Jul 2020 17:23:19 -0400 Subject: [PATCH 83/86] Composite Run Steps Outputs (#568) * Composite Action Run Steps * Env Flow => Able to get env variables and overwrite current env variables => but it doesn't 'stick' * clean up * Clean up trace messages + add Trace debug in ActionManager * Add debugging message * Optimize runtime of code * Change String to string * Add comma to Composite * Change JobSteps to a List, Change Register Step function name * Add TODO, remove unn. content * Remove unnecessary code * Fix unit tests * Fix env format * Remove comment * Remove TODO message for context * Add verbose trace logs which are only viewable by devs * Initial Start for FileTable stuff * Progress towards passing FileTable or FileID or FileName * Sort usings in Composite Action Handler * Change 0 to location * Update context variables in composite action yaml * Add helpful error message for null steps * Pass fileID to all children token of root action token * Change confusing term context => templateContext, Eliminate _fileTable and only use ExecutionContext.FileTable + update this table when need be * Remove unnessary FileID attribute from CompositeActionExecutionData * Clean up file path for error message * Remove todo * Initial start/framework for output handling * Outline different class vs Handler approach * Remove InitializeScope * Remove InitializeScope * Fix Workflow Step Env overiding Parent Env * First Approach for Attaching ID + Group ID to each Composite Action Step * Add GroupID to the ActionDefinitionData * starting foundation for handling clean up outputs step * Pass outputs data to each composite action step to enable set-output functionality * Create ScopeName for whole composite action. This will enable us to add to the StepsContext[ScopeName] for the composite action which will allow us to use all these outputs in the cleanup step * Hook up composite output step to handler => tmmrw implement composite output handler * Add post composite action step to cleanup outputs => triggers composite output cleanup handler * Fix Outputs Token handling start. Add individual step scope names. * Set up Scope Name and Context Name naming system{ * Figured out how to pass Parent Execution Context to clean up step * Figured out how to pass Parent Execution Context and scope names to clean up step * Add GetOutput function for StepsContext * Generate child scope name correctly if parent scope name is null * Simplify InitializeScope() * Outputs are set correctly and able to get all final outputs in handler * Parse through Action Outputs * Fix null ScopeName + ContextName in CompositeOutputHandler * Shift over handling of Action Outputs to output handler * First attempt to fix null retrievals for output variables * Basic Support for Outputs Done. * Clean up pt.1 * Refactor outputs to avoid using Action Reference * Clean up code * Clean up part 2 * Add clarifying comments for the output handler * Remove TODO * Remove env in composite action scope * Clean up * Revert back * revert back * add back envToken * Remove unnecessary code * Add file length check * Clean up * Figure out how to handle set-env edge cases * formatting * fix unit tests * Fix windows unit test syntax error * Fix period * Sanity check for fileTable add + remove unn. code * revert back * Add back line break * Fix null errors * Address situation if FileTable is null + add sanity check for adding file to fileTable * add line * Revert * Fix unit tests to instantiate a FileTable * Fix logic for trimming manifestfile path * Add null check * Revert * Revert * revert * spacing * Add filetable to testing file, remove ? since we know filetable should never be non null * Fix Throw logic * Clarify template outputs token * Add another type support for outputs to avoid container unit tests errors * Add mapping for parity * Build support for new outputs format * Refactor to avoid duplication of action yaml for workflow yaml * Move SDK work in ActionManifestManager, Condense Code * Defer runs evaluation till after for loop to ensure order doesn't matter * Fix logic error in setting scope and context names * Add Regex + Add Child Context name null resolution * move private function to bottom of class --- src/Runner.Worker/ActionManager.cs | 7 +- src/Runner.Worker/ActionManifestManager.cs | 85 +++- src/Runner.Worker/ExecutionContext.cs | 57 ++- .../Handlers/CompositeActionHandler.cs | 26 +- .../Handlers/CompositeActionOutputHandler.cs | 53 +++ src/Runner.Worker/Handlers/HandlerFactory.cs | 12 +- src/Runner.Worker/StepsRunner.cs | 378 ++++++------------ src/Runner.Worker/action_yaml.json | 36 +- 8 files changed, 372 insertions(+), 282 deletions(-) create mode 100644 src/Runner.Worker/Handlers/CompositeActionOutputHandler.cs diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 242ab79e052..65c3d3593b7 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -398,8 +398,10 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) { var compositeAction = definition.Data.Execution as CompositeActionExecutionData; - Trace.Info($"Load {compositeAction.Steps.Count} action steps."); - Trace.Verbose($"Details: {StringUtil.ConvertToJson(compositeAction.Steps)}"); + Trace.Info($"Load {compositeAction.Steps?.Count ?? 0} action steps."); + Trace.Verbose($"Details: {StringUtil.ConvertToJson(compositeAction?.Steps)}"); + Trace.Info($"Load: {compositeAction.Outputs?.Count ?? 0} number of outputs"); + Trace.Info($"Details: {StringUtil.ConvertToJson(compositeAction?.Outputs)}"); } else { @@ -1222,6 +1224,7 @@ public sealed class CompositeActionExecutionData : ActionExecutionData public override bool HasPre => false; public override bool HasPost => false; public List Steps { get; set; } + public MappingToken Outputs { get; set; } } public abstract class ActionExecutionData diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index ae5ca73d7c0..a7be7cb17cc 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -23,11 +23,15 @@ public interface IActionManifestManager : IRunnerService { ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile); + DictionaryContextData EvaluateCompositeOutputs(IExecutionContext executionContext, TemplateToken token, IDictionary extraExpressionValues); + List EvaluateContainerArguments(IExecutionContext executionContext, SequenceToken token, IDictionary extraExpressionValues); Dictionary EvaluateContainerEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary extraExpressionValues); string EvaluateDefaultInput(IExecutionContext executionContext, string inputName, TemplateToken token); + + void SetAllCompositeOutputs(IExecutionContext parentExecutionContext, DictionaryContextData actionOutputs); } public sealed class ActionManifestManager : RunnerService, IActionManifestManager @@ -89,6 +93,9 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani } var actionMapping = token.AssertMapping("action manifest root"); + var actionOutputs = default(MappingToken); + var actionRunValueToken = default(TemplateToken); + foreach (var actionPair in actionMapping) { var propertyName = actionPair.Key.AssertString($"action.yml property key"); @@ -99,6 +106,15 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani actionDefinition.Name = actionPair.Value.AssertString("name").Value; break; + case "outputs": + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + actionOutputs = actionPair.Value.AssertMapping("outputs"); + break; + } + Trace.Info($"Ignore action property outputs. Outputs for a whole action is not supported yet."); + break; + case "description": actionDefinition.Description = actionPair.Value.AssertString("description").Value; break; @@ -108,13 +124,21 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani break; case "runs": - actionDefinition.Execution = ConvertRuns(executionContext, templateContext, actionPair.Value); + // Defer runs token evaluation to after for loop to ensure that order of outputs doesn't matter. + actionRunValueToken = actionPair.Value; break; + default: Trace.Info($"Ignore action property {propertyName}."); break; } } + + // Evaluate Runs Last + if (actionRunValueToken != null) + { + actionDefinition.Execution = ConvertRuns(executionContext, templateContext, actionRunValueToken, actionOutputs); + } } catch (Exception ex) { @@ -146,6 +170,61 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani return actionDefinition; } + public void SetAllCompositeOutputs( + IExecutionContext parentExecutionContext, + DictionaryContextData actionOutputs) + { + // Each pair is structured like this + // We ignore "description" for now + // { + // "the-output-name": { + // "description": "", + // "value": "the value" + // }, + // ... + // } + foreach (var pair in actionOutputs) + { + var outputsName = pair.Key; + var outputsAttributes = pair.Value as DictionaryContextData; + outputsAttributes.TryGetValue("value", out var val); + var outputsValue = val as StringContextData; + + // Set output in the whole composite scope. + if (!String.IsNullOrEmpty(outputsName) && !String.IsNullOrEmpty(outputsValue)) + { + parentExecutionContext.SetOutput(outputsName, outputsValue, out _); + } + } + } + + public DictionaryContextData EvaluateCompositeOutputs( + IExecutionContext executionContext, + TemplateToken token, + IDictionary extraExpressionValues) + { + var result = default(DictionaryContextData); + + if (token != null) + { + var context = CreateContext(executionContext, extraExpressionValues); + try + { + token = TemplateEvaluator.Evaluate(context, "outputs", token, 0, null, omitHeader: true); + context.Errors.Check(); + result = token.ToContextData().AssertDictionary("composite outputs"); + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + + return result ?? new DictionaryContextData(); + } + public List EvaluateContainerArguments( IExecutionContext executionContext, SequenceToken token, @@ -309,7 +388,8 @@ private TemplateContext CreateContext( private ActionExecutionData ConvertRuns( IExecutionContext executionContext, TemplateContext context, - TemplateToken inputsToken) + TemplateToken inputsToken, + MappingToken outputs = null) { var runsMapping = inputsToken.AssertMapping("runs"); var usingToken = default(StringToken); @@ -439,6 +519,7 @@ private ActionExecutionData ConvertRuns( return new CompositeActionExecutionData() { Steps = stepsLoaded, + Outputs = outputs }; } } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 83e47c74c5f..23b26aee1d5 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Web; @@ -52,7 +53,6 @@ public interface IExecutionContext : IRunnerService IDictionary> JobDefaults { get; } Dictionary JobOutputs { get; } IDictionary EnvironmentVariables { get; } - IDictionary Scopes { get; } IList FileTable { get; } StepsContext StepsContext { get; } DictionaryContextData ExpressionValues { get; } @@ -70,6 +70,8 @@ public interface IExecutionContext : IRunnerService bool EchoOnActionCommand { get; set; } + IExecutionContext FinalizeContext { get; set; } + // Initialize void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token); void CancelToken(); @@ -105,7 +107,7 @@ public interface IExecutionContext : IRunnerService // others void ForceTaskComplete(); void RegisterPostJobStep(IStep step); - void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location, Dictionary envData); + IStep RegisterNestedStep(IActionRunner step, DictionaryContextData inputsData, int location, Dictionary envData, bool cleanUp = false); } public sealed class ExecutionContext : RunnerService, IExecutionContext @@ -120,6 +122,9 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private event OnMatcherChanged _onMatcherChanged; + // Regex used for checking if ScopeName meets the condition that shows that its id is null. + private readonly static Regex _generatedContextNamePattern = new Regex("^__[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + private IssueMatcherConfig[] _matchers; private IPagingLogger _logger; @@ -149,7 +154,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public IDictionary> JobDefaults { get; private set; } public Dictionary JobOutputs { get; private set; } public IDictionary EnvironmentVariables { get; private set; } - public IDictionary Scopes { get; private set; } public IList FileTable { get; private set; } public StepsContext StepsContext { get; private set; } public DictionaryContextData ExpressionValues { get; } = new DictionaryContextData(); @@ -170,6 +174,8 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public bool EchoOnActionCommand { get; set; } + public IExecutionContext FinalizeContext { get; set; } + public TaskResult? Result { get @@ -270,17 +276,36 @@ public void RegisterPostJobStep(IStep step) /// Helper function used in CompositeActionHandler::RunAsync to /// add a child node, aka a step, to the current job to the Root.JobSteps based on the location. /// - public void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location, Dictionary envData) + public IStep RegisterNestedStep( + IActionRunner step, + DictionaryContextData inputsData, + int location, + Dictionary envData, + bool cleanUp = false) { // TODO: For UI purposes, look at figuring out how to condense steps in one node => maybe use the same previous GUID var newGuid = Guid.NewGuid(); - step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null); + + // If the context name is empty and the scope name is empty, we would generate a unique scope name for this child in the following format: + // "__" + var safeContextName = !string.IsNullOrEmpty(ContextName) ? ContextName : $"__{newGuid}"; + + // Set Scope Name. Note, for our design, we consider each step in a composite action to have the same scope + // This makes it much simpler to handle their outputs at the end of the Composite Action + var childScopeName = !string.IsNullOrEmpty(ScopeName) ? $"{ScopeName}.{safeContextName}" : safeContextName; + + var childContextName = !string.IsNullOrEmpty(step.Action.ContextName) ? step.Action.ContextName : $"__{Guid.NewGuid()}"; + + step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), childScopeName, childContextName); step.ExecutionContext.ExpressionValues["inputs"] = inputsData; + // Set Parent Attribute for Clean Up Step + if (cleanUp) + { + step.ExecutionContext.FinalizeContext = this; + } + // Add the composite action environment variables to each step. - // If the key already exists, we override it since the composite action env variables will have higher precedence - // Note that for each composite action step, it's environment variables will be set in the StepRunner automatically - // step.ExecutionContext.SetEnvironmentVariables(envData); #if OS_WINDOWS var envContext = new DictionaryContextData(); #else @@ -293,6 +318,8 @@ public void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int step.ExecutionContext.ExpressionValues["env"] = envContext; Root.JobSteps.Insert(location, step); + + return step; } public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null) @@ -317,7 +344,6 @@ public IExecutionContext CreateChild(Guid recordId, string displayName, string r } child.EnvironmentVariables = EnvironmentVariables; child.JobDefaults = JobDefaults; - child.Scopes = Scopes; child.FileTable = FileTable; child.StepsContext = StepsContext; foreach (var pair in ExpressionValues) @@ -466,7 +492,8 @@ public void SetOutput(string name, string value, out string reference) { ArgUtil.NotNullOrEmpty(name, nameof(name)); - if (String.IsNullOrEmpty(ContextName)) + // if the ContextName follows the __GUID format which is set as the default value for ContextName if null for Composite Actions. + if (String.IsNullOrEmpty(ContextName) || _generatedContextNamePattern.IsMatch(ContextName)) { reference = null; return; @@ -633,16 +660,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // Steps context (StepsRunner manages adding the scoped steps context) StepsContext = new StepsContext(); - // Scopes - Scopes = new Dictionary(StringComparer.OrdinalIgnoreCase); - if (message.Scopes?.Count > 0) - { - foreach (var scope in message.Scopes) - { - Scopes[scope.Name] = scope; - } - } - // File table FileTable = new List(message.FileTable ?? new string[0]); diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index ddf5821ad39..4b6ebdd3bb0 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -47,6 +47,7 @@ public Task RunAsync(ActionRunStage stage) // Add each composite action step to the front of the queue int location = 0; + foreach (Pipelines.ActionStep aStep in actionSteps) { // Ex: @@ -85,12 +86,35 @@ public Task RunAsync(ActionRunStage stage) actionRunner.Condition = aStep.Condition; actionRunner.DisplayName = aStep.DisplayName; - ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location, Environment); + var step = ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location, Environment); + + InitializeScope(step); + location++; } + // Create a step that handles all the composite action steps' outputs + Pipelines.ActionStep cleanOutputsStep = new Pipelines.ActionStep(); + cleanOutputsStep.ContextName = ExecutionContext.ContextName; + cleanOutputsStep.DisplayName = "Composite Action Steps Cleanup"; + // Use the same reference type as our composite steps. + cleanOutputsStep.Reference = Action; + + var actionRunner2 = HostContext.CreateService(); + actionRunner2.Action = cleanOutputsStep; + actionRunner2.Stage = ActionRunStage.Main; + actionRunner2.Condition = "always()"; + actionRunner2.DisplayName = "Composite Action Steps Cleanup"; + ExecutionContext.RegisterNestedStep(actionRunner2, inputsData, location, Environment, true); + return Task.CompletedTask; } + private void InitializeScope(IStep step) + { + var stepsContext = step.ExecutionContext.StepsContext; + var scopeName = step.ExecutionContext.ScopeName; + step.ExecutionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName); + } } } diff --git a/src/Runner.Worker/Handlers/CompositeActionOutputHandler.cs b/src/Runner.Worker/Handlers/CompositeActionOutputHandler.cs new file mode 100644 index 00000000000..ea52412003e --- /dev/null +++ b/src/Runner.Worker/Handlers/CompositeActionOutputHandler.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using GitHub.DistributedTask.ObjectTemplating.Schema; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; +using Pipelines = GitHub.DistributedTask.Pipelines; + +namespace GitHub.Runner.Worker.Handlers +{ + [ServiceLocator(Default = typeof(CompositeActionOutputHandler))] + public interface ICompositeActionOutputHandler : IHandler + { + CompositeActionExecutionData Data { get; set; } + } + + public sealed class CompositeActionOutputHandler : Handler, ICompositeActionOutputHandler + { + public CompositeActionExecutionData Data { get; set; } + + + public Task RunAsync(ActionRunStage stage) + { + // Evaluate the mapped outputs value + if (Data.Outputs != null) + { + // Evaluate the outputs in the steps context to easily retrieve the values + var actionManifestManager = HostContext.GetService(); + + // Format ExpressionValues to Dictionary + var evaluateContext = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in ExecutionContext.ExpressionValues) + { + evaluateContext[pair.Key] = pair.Value; + } + + // Get the evluated composite outputs' values mapped to the outputs named + DictionaryContextData actionOutputs = actionManifestManager.EvaluateCompositeOutputs(ExecutionContext, Data.Outputs, evaluateContext); + + // Set the outputs for the outputs object in the whole composite action + actionManifestManager.SetAllCompositeOutputs(ExecutionContext.FinalizeContext, actionOutputs); + } + + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/src/Runner.Worker/Handlers/HandlerFactory.cs b/src/Runner.Worker/Handlers/HandlerFactory.cs index db4d6559c88..4591ccab21d 100644 --- a/src/Runner.Worker/Handlers/HandlerFactory.cs +++ b/src/Runner.Worker/Handlers/HandlerFactory.cs @@ -68,8 +68,16 @@ public IHandler Create( } else if (data.ExecutionType == ActionExecutionType.Composite) { - handler = HostContext.CreateService(); - (handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData; + if (executionContext.FinalizeContext == null) + { + handler = HostContext.CreateService(); + (handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData; + } + else + { + handler = HostContext.CreateService(); + (handler as ICompositeActionOutputHandler).Data = data as CompositeActionExecutionData; + } } else { diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 966f73dc1a4..553f792482a 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -67,7 +67,6 @@ public async Task RunAsync(IExecutionContext jobContext) var step = jobContext.JobSteps[0]; jobContext.JobSteps.RemoveAt(0); - var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps[0] : null; Trace.Info($"Processing step: DisplayName='{step.DisplayName}'"); ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext)); @@ -83,171 +82,170 @@ public async Task RunAsync(IExecutionContext jobContext) step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo(PipelineTemplateConstants.Success, 0, 0)); step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo(PipelineTemplateConstants.HashFiles, 1, byte.MaxValue)); - // Initialize scope - if (InitializeScope(step, scopeInputs)) - { - // Populate env context for each step - Trace.Info("Initialize Env context for step"); + step.ExecutionContext.ExpressionValues["steps"] = step.ExecutionContext.StepsContext.GetScope(step.ExecutionContext.ScopeName); + + // Populate env context for each step + Trace.Info("Initialize Env context for step"); #if OS_WINDOWS - var envContext = new DictionaryContextData(); + var envContext = new DictionaryContextData(); #else - var envContext = new CaseSensitiveDictionaryContextData(); + var envContext = new CaseSensitiveDictionaryContextData(); #endif - // Global env - foreach (var pair in step.ExecutionContext.EnvironmentVariables) - { - envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty); - } - // Stomps over with outside step env - if (step.ExecutionContext.ExpressionValues.TryGetValue("env", out var envContextData)) - { + // Global env + foreach (var pair in step.ExecutionContext.EnvironmentVariables) + { + envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty); + } + + // Stomps over with outside step env + if (step.ExecutionContext.ExpressionValues.TryGetValue("env", out var envContextData)) + { #if OS_WINDOWS - var dict = envContextData as DictionaryContextData; + var dict = envContextData as DictionaryContextData; #else - var dict = envContextData as CaseSensitiveDictionaryContextData; + var dict = envContextData as CaseSensitiveDictionaryContextData; #endif - foreach (var pair in dict) - { - envContext[pair.Key] = pair.Value; - } + foreach (var pair in dict) + { + envContext[pair.Key] = pair.Value; } + } - step.ExecutionContext.ExpressionValues["env"] = envContext; + step.ExecutionContext.ExpressionValues["env"] = envContext; - bool evaluateStepEnvFailed = false; - if (step is IActionRunner actionStep) - { - // Set GITHUB_ACTION - step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name); + bool evaluateStepEnvFailed = false; + if (step is IActionRunner actionStep) + { + // Set GITHUB_ACTION + step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name); - try - { - // Evaluate and merge action's env block to env context - var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); - var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer); - foreach (var env in actionEnvironment) - { - envContext[env.Key] = new StringContextData(env.Value ?? string.Empty); - } - } - catch (Exception ex) + try + { + // Evaluate and merge action's env block to env context + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); + var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer); + foreach (var env in actionEnvironment) { - // fail the step since there is an evaluate error. - Trace.Info("Caught exception from expression for step.env"); - evaluateStepEnvFailed = true; - step.ExecutionContext.Error(ex); - CompleteStep(step, nextStep, TaskResult.Failed); + envContext[env.Key] = new StringContextData(env.Value ?? string.Empty); } } + catch (Exception ex) + { + // fail the step since there is an evaluate error. + Trace.Info("Caught exception from expression for step.env"); + evaluateStepEnvFailed = true; + step.ExecutionContext.Error(ex); + CompleteStep(step, TaskResult.Failed); + } + } - if (!evaluateStepEnvFailed) + if (!evaluateStepEnvFailed) + { + try { - try + // Register job cancellation call back only if job cancellation token not been fire before each step run + if (!jobContext.CancellationToken.IsCancellationRequested) { - // Register job cancellation call back only if job cancellation token not been fire before each step run - if (!jobContext.CancellationToken.IsCancellationRequested) + // Test the condition again. The job was canceled after the condition was originally evaluated. + jobCancelRegister = jobContext.CancellationToken.Register(() => { - // Test the condition again. The job was canceled after the condition was originally evaluated. - jobCancelRegister = jobContext.CancellationToken.Register(() => + // mark job as cancelled + jobContext.Result = TaskResult.Canceled; + jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); + + step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'."); + var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only + var conditionReTestResult = false; + if (HostContext.RunnerShutdownToken.IsCancellationRequested) { - // mark job as cancelled - jobContext.Result = TaskResult.Canceled; - jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); - - step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'."); - var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only - var conditionReTestResult = false; - if (HostContext.RunnerShutdownToken.IsCancellationRequested) + step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown."); + } + else + { + try { - step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown."); + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter); + var condition = new BasicExpressionToken(null, null, null, step.Condition); + conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); } - else + catch (Exception ex) { - try - { - var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter); - var condition = new BasicExpressionToken(null, null, null, step.Condition); - conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); - } - catch (Exception ex) - { - // Cancel the step since we get exception while re-evaluate step condition. - Trace.Info("Caught exception from expression when re-test condition on job cancellation."); - step.ExecutionContext.Error(ex); - } + // Cancel the step since we get exception while re-evaluate step condition. + Trace.Info("Caught exception from expression when re-test condition on job cancellation."); + step.ExecutionContext.Error(ex); } - - if (!conditionReTestResult) - { - // Cancel the step. - Trace.Info("Cancel current running step."); - step.ExecutionContext.CancelToken(); - } - }); - } - else - { - if (jobContext.Result != TaskResult.Canceled) - { - // mark job as cancelled - jobContext.Result = TaskResult.Canceled; - jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); } - } - // Evaluate condition. - step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'"); - var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext); - var conditionResult = false; - var conditionEvaluateError = default(Exception); - if (HostContext.RunnerShutdownToken.IsCancellationRequested) - { - step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown."); - } - else - { - try - { - var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter); - var condition = new BasicExpressionToken(null, null, null, step.Condition); - conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); - } - catch (Exception ex) + if (!conditionReTestResult) { - Trace.Info("Caught exception from expression."); - Trace.Error(ex); - conditionEvaluateError = ex; + // Cancel the step. + Trace.Info("Cancel current running step."); + step.ExecutionContext.CancelToken(); } - } - - // no evaluate error but condition is false - if (!conditionResult && conditionEvaluateError == null) + }); + } + else + { + if (jobContext.Result != TaskResult.Canceled) { - // Condition == false - Trace.Info("Skipping step due to condition evaluation."); - CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace); + // mark job as cancelled + jobContext.Result = TaskResult.Canceled; + jobContext.JobContext.Status = jobContext.Result?.ToActionResult(); } - else if (conditionEvaluateError != null) + } + + // Evaluate condition. + step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'"); + var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext); + var conditionResult = false; + var conditionEvaluateError = default(Exception); + if (HostContext.RunnerShutdownToken.IsCancellationRequested) + { + step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown."); + } + else + { + try { - // fail the step since there is an evaluate error. - step.ExecutionContext.Error(conditionEvaluateError); - CompleteStep(step, nextStep, TaskResult.Failed); + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter); + var condition = new BasicExpressionToken(null, null, null, step.Condition); + conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState()); } - else + catch (Exception ex) { - // Run the step. - await RunStepAsync(step, jobContext.CancellationToken); - CompleteStep(step, nextStep); + Trace.Info("Caught exception from expression."); + Trace.Error(ex); + conditionEvaluateError = ex; } } - finally + + // no evaluate error but condition is false + if (!conditionResult && conditionEvaluateError == null) { - if (jobCancelRegister != null) - { - jobCancelRegister?.Dispose(); - jobCancelRegister = null; - } + // Condition == false + Trace.Info("Skipping step due to condition evaluation."); + CompleteStep(step, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace); + } + else if (conditionEvaluateError != null) + { + // fail the step since there is an evaluate error. + step.ExecutionContext.Error(conditionEvaluateError); + CompleteStep(step, TaskResult.Failed); + } + else + { + // Run the step. + await RunStepAsync(step, jobContext.CancellationToken); + CompleteStep(step); + } + } + finally + { + if (jobCancelRegister != null) + { + jobCancelRegister?.Dispose(); + jobCancelRegister = null; } } } @@ -401,125 +399,9 @@ private async Task RunStepAsync(IStep step, CancellationToken jobCancellationTok step.ExecutionContext.Debug($"Finishing: {step.DisplayName}"); } - private bool InitializeScope(IStep step, Dictionary scopeInputs) - { - var executionContext = step.ExecutionContext; - var stepsContext = executionContext.StepsContext; - if (!string.IsNullOrEmpty(executionContext.ScopeName)) - { - // Gather uninitialized current and ancestor scopes - var scope = executionContext.Scopes[executionContext.ScopeName]; - var scopesToInitialize = default(Stack); - while (scope != null && !scopeInputs.ContainsKey(scope.Name)) - { - if (scopesToInitialize == null) - { - scopesToInitialize = new Stack(); - } - scopesToInitialize.Push(scope); - scope = string.IsNullOrEmpty(scope.ParentName) ? null : executionContext.Scopes[scope.ParentName]; - } - - // Initialize current and ancestor scopes - while (scopesToInitialize?.Count > 0) - { - scope = scopesToInitialize.Pop(); - executionContext.Debug($"Initializing scope '{scope.Name}'"); - executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName); - // TODO: Fix this temporary workaround for Composite Actions - if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) - { - executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; - } - var templateEvaluator = executionContext.ToPipelineTemplateEvaluator(); - var inputs = default(DictionaryContextData); - try - { - inputs = templateEvaluator.EvaluateStepScopeInputs(scope.Inputs, executionContext.ExpressionValues, executionContext.ExpressionFunctions); - } - catch (Exception ex) - { - Trace.Info($"Caught exception from initialize scope '{scope.Name}'"); - Trace.Error(ex); - executionContext.Error(ex); - executionContext.Complete(TaskResult.Failed); - return false; - } - - scopeInputs[scope.Name] = inputs; - } - } - - // Setup expression values - var scopeName = executionContext.ScopeName; - executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName); - // TODO: Fix this temporary workaround for Composite Actions - if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) - { - executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; - } - - return true; - } - - private void CompleteStep(IStep step, IStep nextStep, TaskResult? result = null, string resultCode = null) + private void CompleteStep(IStep step, TaskResult? result = null, string resultCode = null) { var executionContext = step.ExecutionContext; - if (!string.IsNullOrEmpty(executionContext.ScopeName)) - { - // Gather current and ancestor scopes to finalize - var scope = executionContext.Scopes[executionContext.ScopeName]; - var scopesToFinalize = default(Queue); - var nextStepScopeName = nextStep?.ExecutionContext.ScopeName; - while (scope != null && - !string.Equals(nextStepScopeName, scope.Name, StringComparison.OrdinalIgnoreCase) && - !(nextStepScopeName ?? string.Empty).StartsWith($"{scope.Name}.", StringComparison.OrdinalIgnoreCase)) - { - if (scopesToFinalize == null) - { - scopesToFinalize = new Queue(); - } - scopesToFinalize.Enqueue(scope); - scope = string.IsNullOrEmpty(scope.ParentName) ? null : executionContext.Scopes[scope.ParentName]; - } - - // Finalize current and ancestor scopes - var stepsContext = step.ExecutionContext.StepsContext; - while (scopesToFinalize?.Count > 0) - { - scope = scopesToFinalize.Dequeue(); - executionContext.Debug($"Finalizing scope '{scope.Name}'"); - executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.Name); - executionContext.ExpressionValues["inputs"] = null; - var templateEvaluator = executionContext.ToPipelineTemplateEvaluator(); - var outputs = default(DictionaryContextData); - try - { - outputs = templateEvaluator.EvaluateStepScopeOutputs(scope.Outputs, executionContext.ExpressionValues, executionContext.ExpressionFunctions); - } - catch (Exception ex) - { - Trace.Info($"Caught exception from finalize scope '{scope.Name}'"); - Trace.Error(ex); - executionContext.Error(ex); - executionContext.Complete(TaskResult.Failed); - return; - } - - if (outputs?.Count > 0) - { - var parentScopeName = scope.ParentName; - var contextName = scope.ContextName; - foreach (var pair in outputs) - { - var outputName = pair.Key; - var outputValue = pair.Value.ToString(); - stepsContext.SetOutput(parentScopeName, contextName, outputName, outputValue, out var reference); - executionContext.Debug($"{reference}='{outputValue}'"); - } - } - } - } executionContext.Complete(result, resultCode: resultCode); } diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index cb1d90b2e0b..82b24a6951f 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -7,7 +7,8 @@ "name": "string", "description": "string", "inputs": "inputs", - "runs": "runs" + "runs": "runs", + "outputs": "outputs" }, "loose-key-type": "non-empty-string", "loose-value-type": "any" @@ -28,6 +29,20 @@ "loose-value-type": "any" } }, + "outputs": { + "mapping": { + "loose-key-type": "non-empty-string", + "loose-value-type": "outputs-attributes" + } + }, + "outputs-attributes": { + "mapping": { + "properties": { + "description": "string", + "value": "output-value" + } + } + }, "runs": { "one-of": [ "container-runs", @@ -95,19 +110,13 @@ "composite-steps": { "context": [ "github", - "needs", "strategy", "matrix", - "secrets", "steps", "inputs", "job", "runner", "env", - "always(0,0)", - "failure(0,0)", - "cancelled(0,0)", - "success(0,0)", "hashFiles(1,255)" ], "sequence": { @@ -120,6 +129,19 @@ ], "string": {} }, + "output-value": { + "context": [ + "github", + "strategy", + "matrix", + "steps", + "inputs", + "job", + "runner", + "env" + ], + "string": {} + }, "input-default-context": { "context": [ "github", From 5e0cde8649bdd72fd9659b10253a7fbf5dbc43c1 Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Mon, 13 Jul 2020 17:55:15 -0400 Subject: [PATCH 84/86] Composite Actions UI (#578) * Composite Action Run Steps * Env Flow => Able to get env variables and overwrite current env variables => but it doesn't 'stick' * clean up * Clean up trace messages + add Trace debug in ActionManager * Add debugging message * Optimize runtime of code * Change String to string * Add comma to Composite * Change JobSteps to a List, Change Register Step function name * Add TODO, remove unn. content * Remove unnecessary code * Fix unit tests * Fix env format * Remove comment * Remove TODO message for context * Add verbose trace logs which are only viewable by devs * Initial Start for FileTable stuff * Progress towards passing FileTable or FileID or FileName * Sort usings in Composite Action Handler * Change 0 to location * Update context variables in composite action yaml * Add helpful error message for null steps * Pass fileID to all children token of root action token * Change confusing term context => templateContext, Eliminate _fileTable and only use ExecutionContext.FileTable + update this table when need be * Remove unnessary FileID attribute from CompositeActionExecutionData * Clean up file path for error message * Remove todo * Initial start/framework for output handling * Outline different class vs Handler approach * Remove InitializeScope * Remove InitializeScope * Fix Workflow Step Env overiding Parent Env * First Approach for Attaching ID + Group ID to each Composite Action Step * Add GroupID to the ActionDefinitionData * starting foundation for handling clean up outputs step * Pass outputs data to each composite action step to enable set-output functionality * Create ScopeName for whole composite action. This will enable us to add to the StepsContext[ScopeName] for the composite action which will allow us to use all these outputs in the cleanup step * Hook up composite output step to handler => tmmrw implement composite output handler * Add post composite action step to cleanup outputs => triggers composite output cleanup handler * Fix Outputs Token handling start. Add individual step scope names. * Set up Scope Name and Context Name naming system{ * Figured out how to pass Parent Execution Context to clean up step * Figured out how to pass Parent Execution Context and scope names to clean up step * Add GetOutput function for StepsContext * Generate child scope name correctly if parent scope name is null * Simplify InitializeScope() * Outputs are set correctly and able to get all final outputs in handler * Parse through Action Outputs * Fix null ScopeName + ContextName in CompositeOutputHandler * Shift over handling of Action Outputs to output handler * First attempt to fix null retrievals for output variables * Basic Support for Outputs Done. * Clean up pt.1 * Refactor outputs to avoid using Action Reference * Clean up code * Clean up part 2 * Add clarifying comments for the output handler * Remove TODO * Remove env in composite action scope * Clean up * Revert back * revert back * add back envToken * Remove unnecessary code * Add file length check * Clean up * Fix logging issue * Figure out how to handle set-env edge cases * formatting * fix unit tests * Fix windows unit test syntax error * Fix period * Sanity check for fileTable add + remove unn. code * revert back * Add back line break * Fix null errors * Address situation if FileTable is null + add sanity check for adding file to fileTable * add line * Revert * Fix unit tests to instantiate a FileTable * Fix logic for trimming manifestfile path * Add null check * Revert * Revert * revert * spacing * Add filetable to testing file, remove ? since we know filetable should never be non null * Fix Throw logic * Clarify template outputs token * Add another type support for outputs to avoid container unit tests errors * Add mapping for parity * Build support for new outputs format * Build support for new outputs format * Refactor to avoid duplication of action yaml for workflow yaml * revert * revert * revert * spacing --- src/Runner.Worker/ExecutionContext.cs | 24 +++++++++++-------- .../Handlers/CompositeActionHandler.cs | 3 --- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 23b26aee1d5..b31dc97e4c3 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -75,7 +75,7 @@ public interface IExecutionContext : IRunnerService // Initialize void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token); void CancelToken(); - IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null); + IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null, IPagingLogger logger = null); // logging bool WriteDebug { get; } @@ -283,12 +283,9 @@ public IStep RegisterNestedStep( Dictionary envData, bool cleanUp = false) { - // TODO: For UI purposes, look at figuring out how to condense steps in one node => maybe use the same previous GUID - var newGuid = Guid.NewGuid(); - // If the context name is empty and the scope name is empty, we would generate a unique scope name for this child in the following format: // "__" - var safeContextName = !string.IsNullOrEmpty(ContextName) ? ContextName : $"__{newGuid}"; + var safeContextName = !string.IsNullOrEmpty(ContextName) ? ContextName : $"__{Guid.NewGuid()}"; // Set Scope Name. Note, for our design, we consider each step in a composite action to have the same scope // This makes it much simpler to handle their outputs at the end of the Composite Action @@ -296,7 +293,8 @@ public IStep RegisterNestedStep( var childContextName = !string.IsNullOrEmpty(step.Action.ContextName) ? step.Action.ContextName : $"__{Guid.NewGuid()}"; - step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), childScopeName, childContextName); + step.ExecutionContext = Root.CreateChild(_record.Id, step.DisplayName, _record.Id.ToString("N"), childScopeName, childContextName, logger: _logger); + step.ExecutionContext.ExpressionValues["inputs"] = inputsData; // Set Parent Attribute for Clean Up Step @@ -322,7 +320,7 @@ public IStep RegisterNestedStep( return step; } - public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null) + public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null, IPagingLogger logger = null) { Trace.Entering(); @@ -370,9 +368,15 @@ public IExecutionContext CreateChild(Guid recordId, string displayName, string r { child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder); } - - child._logger = HostContext.CreateService(); - child._logger.Setup(_mainTimelineId, recordId); + if (logger != null) + { + child._logger = logger; + } + else + { + child._logger = HostContext.CreateService(); + child._logger.Setup(_mainTimelineId, recordId); + } return child; } diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index 4b6ebdd3bb0..d48f922d177 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -84,7 +84,6 @@ public Task RunAsync(ActionRunStage stage) actionRunner.Action = aStep; actionRunner.Stage = stage; actionRunner.Condition = aStep.Condition; - actionRunner.DisplayName = aStep.DisplayName; var step = ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location, Environment); @@ -96,7 +95,6 @@ public Task RunAsync(ActionRunStage stage) // Create a step that handles all the composite action steps' outputs Pipelines.ActionStep cleanOutputsStep = new Pipelines.ActionStep(); cleanOutputsStep.ContextName = ExecutionContext.ContextName; - cleanOutputsStep.DisplayName = "Composite Action Steps Cleanup"; // Use the same reference type as our composite steps. cleanOutputsStep.Reference = Action; @@ -104,7 +102,6 @@ public Task RunAsync(ActionRunStage stage) actionRunner2.Action = cleanOutputsStep; actionRunner2.Stage = ActionRunStage.Main; actionRunner2.Condition = "always()"; - actionRunner2.DisplayName = "Composite Action Steps Cleanup"; ExecutionContext.RegisterNestedStep(actionRunner2, inputsData, location, Environment, true); return Task.CompletedTask; From a711bd9494e2a3e00fcac3f8a685cf1f30572026 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 28 Jul 2020 14:52:38 -0400 Subject: [PATCH 85/86] add `workflow_dispatch` --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 89355e3167a..5320b3797d8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,7 @@ name: Runner CI on: + workflow_dispatch: push: branches: - master From dcda342eccd152fe69521d59de016ef09b70400c Mon Sep 17 00:00:00 2001 From: Marek Mahut Date: Thu, 22 Apr 2021 23:45:33 +0200 Subject: [PATCH 86/86] use /usr/bin/env to find bash in scripts (#314) --- src/Misc/externals.sh | 2 +- src/Misc/layoutbin/darwin.svc.sh.template | 2 +- src/Misc/layoutbin/installdependencies.sh | 2 +- src/Misc/layoutbin/runsvc.sh | 2 +- src/Misc/layoutbin/systemd.svc.sh.template | 2 +- src/Misc/layoutbin/update.sh.template | 2 +- src/Misc/layoutroot/config.sh | 2 +- src/Misc/layoutroot/env.sh | 2 +- src/Misc/layoutroot/run.sh | 2 +- src/dev.sh | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index bc90fea1c63..55e05f28ab7 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash PACKAGERUNTIME=$1 PRECACHE=$2 diff --git a/src/Misc/layoutbin/darwin.svc.sh.template b/src/Misc/layoutbin/darwin.svc.sh.template index 8d2f96512f8..4986b20ab6a 100644 --- a/src/Misc/layoutbin/darwin.svc.sh.template +++ b/src/Misc/layoutbin/darwin.svc.sh.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash SVC_NAME="{{SvcNameVar}}" SVC_NAME=${SVC_NAME// /_} diff --git a/src/Misc/layoutbin/installdependencies.sh b/src/Misc/layoutbin/installdependencies.sh index 78671492516..e58f5004a26 100755 --- a/src/Misc/layoutbin/installdependencies.sh +++ b/src/Misc/layoutbin/installdependencies.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash user_id=`id -u` diff --git a/src/Misc/layoutbin/runsvc.sh b/src/Misc/layoutbin/runsvc.sh index 1919b9c21f4..695968deb93 100755 --- a/src/Misc/layoutbin/runsvc.sh +++ b/src/Misc/layoutbin/runsvc.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # convert SIGTERM signal to SIGINT # for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html diff --git a/src/Misc/layoutbin/systemd.svc.sh.template b/src/Misc/layoutbin/systemd.svc.sh.template index bdbc998f78c..df00e75e63c 100644 --- a/src/Misc/layoutbin/systemd.svc.sh.template +++ b/src/Misc/layoutbin/systemd.svc.sh.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash SVC_NAME="{{SvcNameVar}}" SVC_NAME=${SVC_NAME// /_} diff --git a/src/Misc/layoutbin/update.sh.template b/src/Misc/layoutbin/update.sh.template index c09cc1d5b4c..d7eeacac621 100644 --- a/src/Misc/layoutbin/update.sh.template +++ b/src/Misc/layoutbin/update.sh.template @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # runner will replace key words in the template and generate a batch script to run. # Keywords: diff --git a/src/Misc/layoutroot/config.sh b/src/Misc/layoutroot/config.sh index 11602459644..025f414f72d 100755 --- a/src/Misc/layoutroot/config.sh +++ b/src/Misc/layoutroot/config.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash user_id=`id -u` diff --git a/src/Misc/layoutroot/env.sh b/src/Misc/layoutroot/env.sh index cfe1a2cafc1..51544f35005 100755 --- a/src/Misc/layoutroot/env.sh +++ b/src/Misc/layoutroot/env.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash varCheckList=( 'LANG' diff --git a/src/Misc/layoutroot/run.sh b/src/Misc/layoutroot/run.sh index 827290ec4cb..f4c756ab69f 100755 --- a/src/Misc/layoutroot/run.sh +++ b/src/Misc/layoutroot/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Validate not sudo user_id=`id -u` diff --git a/src/dev.sh b/src/dev.sh index 43474c68c92..66ee5616a5d 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ############################################################################### #