From 6653e7d9cb708c49c9dc75bf672ea3f24f68a360 Mon Sep 17 00:00:00 2001 From: Colin Kinloch Date: Tue, 4 Aug 2026 19:20:55 +0100 Subject: [PATCH] Add `expand_vars` to expand variables in passed strings This code is adapted from the `obs-gitlab-runner` project. It adds `shellexpand` for extracting variables from a given string and `shell-words` for quoting and escaping variables. Also adds `Variable::masked_value` to return masked variables without the overhead from `fmt::Display`. --- Cargo.lock | 64 ++++++++++++++++++ gitlab-runner/Cargo.toml | 2 + gitlab-runner/examples/demo-runner.rs | 10 +++ gitlab-runner/src/job.rs | 56 +++++++++++++++- gitlab-runner/tests/integration.rs | 95 +++++++++++++++++++++++++-- 5 files changed, 221 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a13e87d..7d418a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -521,6 +521,27 @@ dependencies = [ "ctutils", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -780,6 +801,8 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", + "shell-words", + "shellexpand", "tempfile", "thiserror 2.0.18", "tokio", @@ -1247,6 +1270,15 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "libredox" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" +dependencies = [ + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1416,6 +1448,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1684,6 +1722,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + [[package]] name = "regex" version = "1.12.3" @@ -2009,6 +2058,21 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "dirs", +] + [[package]] name = "shlex" version = "1.3.0" diff --git a/gitlab-runner/Cargo.toml b/gitlab-runner/Cargo.toml index fbf8f0a..fd74ad6 100644 --- a/gitlab-runner/Cargo.toml +++ b/gitlab-runner/Cargo.toml @@ -37,6 +37,8 @@ rand = "0.10.1" tokio-util = { version = "0.7.18", features = [ "io" ] } tokio-retry2 = { version = "0.9.1", features = ["jitter"] } normalize-path = "0.2.1" +shellexpand = "3.1.2" +shell-words = "1.1.1" [dev-dependencies] tokio = { version = "1.50.0", features = [ "full", "test-util" ] } diff --git a/gitlab-runner/examples/demo-runner.rs b/gitlab-runner/examples/demo-runner.rs index d4050b0..e01686e 100644 --- a/gitlab-runner/examples/demo-runner.rs +++ b/gitlab-runner/examples/demo-runner.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::collections::HashSet; use std::io::{IsTerminal, Read}; use anyhow::{Context, Result}; @@ -123,6 +124,15 @@ impl Run { } Ok(()) } + "echo" => { + let mut expanding = HashSet::new(); + let expanded = p + .map(move |l| self.job.expand_vars_inner(l, true, true, &mut expanding)) + .collect::>() + .join(" "); + outputln!("{}", expanded); + Ok(()) + } _ => { outputln!("Unknown command\n"); Err(()) diff --git a/gitlab-runner/src/job.rs b/gitlab-runner/src/job.rs index 67114f4..587bee7 100644 --- a/gitlab-runner/src/job.rs +++ b/gitlab-runner/src/job.rs @@ -3,7 +3,8 @@ use crate::artifact::Artifact; use crate::client::{Client, JobArtifactFile, JobDependency, JobResponse, JobVariable}; use crate::outputln; use bytes::{Bytes, BytesMut}; -use std::collections::HashMap; +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use tokio::io::AsyncWrite; @@ -34,6 +35,15 @@ impl<'a> Variable<'a> { &self.v.value } + /// Return the value of the variable or "" + pub fn masked_value(&self) -> &'a str { + if self.masked() { + "" + } else { + &self.v.value + } + } + /// Whether or not the variable is masked pub fn masked(&self) -> bool { self.v.masked @@ -333,6 +343,50 @@ impl Job { }) } + /// Returns `line` with variables expanded. + /// + /// When `quote` is `true` variables are quoted and special characters are escaped to retain their literal meaning in Unix shell syntax. + /// When `mask` is `true` the masked variables are replaced making the output suitable for logging. + /// + /// Recursive variable expand to empty strings + pub fn expand_vars<'s>(&self, line: &'s str, quote: bool, mask: bool) -> Cow<'s, str> { + self.expand_vars_inner(line, quote, mask, &mut HashSet::new()) + } + + /// See [`Job::expand_vars`] + /// + /// This function must be supplied with an empty `HashMap` that is used to track variables to avoid infinite loops. + pub fn expand_vars_inner<'s>( + &self, + line: &'s str, + quote: bool, + mask: bool, + expanding: &mut HashSet, + ) -> Cow<'s, str> { + shellexpand::env_with_context_no_errors(line, |var| { + if !expanding.insert(var.to_owned()) { + return Some("".into()); + } + + let value = + self.variable(var).map_or( + "", + |v: Variable<'_>| if mask { v.masked_value() } else { v.value() }, + ); + + let expanded = self.expand_vars_inner(value, false, mask, expanding); + expanding.remove(var); + Some( + if quote { + shell_words::quote(expanded.as_ref()) + } else { + expanded + } + .into_owned(), + ) + }) + } + /// Get a reference to the jobs build dir. pub fn build_dir(&self) -> &Path { &self.build_dir diff --git a/gitlab-runner/tests/integration.rs b/gitlab-runner/tests/integration.rs index d160de1..c12a1d6 100644 --- a/gitlab-runner/tests/integration.rs +++ b/gitlab-runner/tests/integration.rs @@ -4,6 +4,7 @@ use gitlab_runner::{GitlabLayer, JobHandler, JobResult, Phase, Runner, RunnerBui use gitlab_runner_mock::{ GitlabRunnerMock, MockJob, MockJobState, MockJobStepName, MockJobStepWhen, }; +use std::collections::HashSet; use std::fmt::Debug; use std::future::Future; use std::sync::{Arc, Mutex}; @@ -670,8 +671,12 @@ async fn runner_delay() { #[tokio::test] async fn job_variables() { - const TEST_VARIABLE: &str = "TEST_VARIABLE"; - const TEST_VALUE: &str = "a testing value"; + const TEST_VARIABLE: (&str, &str) = ("TEST_VARIABLE", "a testing value"); + const TEST_EXPAND_VARIABLE: (&str, &str) = ("TEST_EXPAND_VARIABLE", "$TEST_VARIABLE"); + const TEST_EXPAND_TWICE_VARIABLE: (&str, &str) = + ("TEST_EXPAND_TWICE_VARIABLE", "$TEST_EXPAND_VARIABLE"); + const TEST_RECURSIVE_VARIABLE: (&str, &str) = + ("TEST_RECURSIVE_VARIABLE", "$TEST_RECURSIVE_VARIABLE"); let mock = GitlabRunnerMock::start().await; let mut builder = mock.job_builder("variables".to_string()); @@ -683,7 +688,30 @@ async fn job_variables() { MockJobStepWhen::OnSuccess, false, ); - builder.add_variable(TEST_VARIABLE.to_owned(), TEST_VALUE.to_owned(), false, true); + builder.add_variable( + TEST_VARIABLE.0.to_owned(), + TEST_VARIABLE.1.to_owned(), + false, + true, + ); + builder.add_variable( + TEST_EXPAND_VARIABLE.0.to_owned(), + TEST_EXPAND_VARIABLE.1.to_owned(), + false, + true, + ); + builder.add_variable( + TEST_EXPAND_TWICE_VARIABLE.0.to_owned(), + TEST_EXPAND_TWICE_VARIABLE.1.to_owned(), + false, + false, + ); + builder.add_variable( + TEST_RECURSIVE_VARIABLE.0.to_owned(), + TEST_RECURSIVE_VARIABLE.1.to_owned(), + false, + false, + ); let job = builder.build(); mock.enqueue_job(job.clone()); @@ -697,11 +725,68 @@ async fn job_variables() { assert!(id.public()); assert!(!id.masked()); - let test = job.variable(TEST_VARIABLE).unwrap(); - assert_eq!(test.value(), TEST_VALUE); + let test = job.variable(TEST_VARIABLE.0).unwrap(); + assert_eq!(test.value(), TEST_VARIABLE.1); assert!(!test.public()); assert!(test.masked()); + assert_eq!( + job.expand_vars( + format!("${}", TEST_EXPAND_VARIABLE.0).as_str(), + false, + false + ), + TEST_VARIABLE.1, + ); + assert_eq!( + job.expand_vars(format!("${}", TEST_EXPAND_VARIABLE.0).as_str(), true, false), + format!("'{}'", TEST_VARIABLE.1), + ); + assert_eq!( + job.expand_vars( + format!("${{{}}}", TEST_EXPAND_VARIABLE.0).as_str(), + false, + false + ), + TEST_VARIABLE.1, + ); + assert_eq!( + job.expand_vars( + format!("${}", TEST_EXPAND_TWICE_VARIABLE.0).as_str(), + false, + false, + ), + TEST_VARIABLE.1, + ); + assert_eq!( + job.expand_vars( + format!("prefix-${}-suffix", TEST_EXPAND_TWICE_VARIABLE.0).as_str(), + false, + true, + ), + "prefix--suffix", + ); + assert_eq!( + job.expand_vars( + format!("${}", TEST_RECURSIVE_VARIABLE.0).as_str(), + false, + true, + ), + "", + ); + let mut expanding = HashSet::new(); + assert_eq!( + format!( + "${} ${}", + TEST_EXPAND_TWICE_VARIABLE.0, TEST_EXPAND_VARIABLE.0 + ) + .split_whitespace() + .map(|l| job.expand_vars_inner(l, true, false, &mut expanding,)) + .collect::>() + .join(" "), + "'a testing value' 'a testing value'", + ); + SimpleRun::dummy(Ok(())).await }) .await