diff --git a/Cargo.lock b/Cargo.lock index a13e87d..91285d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "aws-lc-rs" version = "1.16.2" @@ -371,6 +377,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.1.7" @@ -553,6 +574,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "equivalent" version = "1.0.2" @@ -569,6 +596,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -604,6 +642,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -707,6 +751,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -772,6 +831,9 @@ dependencies = [ "gitlab-runner-mock", "glob", "hmac 0.13.0", + "metrics", + "metrics-exporter-prometheus", + "metrics-util", "normalize-path", "parking_lot", "pin-project", @@ -831,13 +893,19 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -845,6 +913,9 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -963,6 +1034,7 @@ dependencies = [ "hyper", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1235,6 +1307,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "left-right" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "libbz2-rs-sys" version = "0.2.2" @@ -1274,6 +1357,19 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1304,6 +1400,60 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64", + "evmap", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "indexmap", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "rustls", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "indexmap", + "metrics", + "ordered-float", + "quanta", + "radix_trie", + "rand 0.9.2", + "rand_xoshiro", + "rapidhash", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -1358,6 +1508,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "normalize-path" version = "0.2.1" @@ -1379,6 +1538,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.17.0" @@ -1416,6 +1584,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1493,6 +1670,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + [[package]] name = "potential_utf" version = "0.1.4" @@ -1542,6 +1725,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1620,6 +1818,16 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.9.2" @@ -1675,6 +1883,33 @@ dependencies = [ "rand_core 0.10.0", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1889,6 +2124,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -2031,6 +2272,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -2720,6 +2967,22 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -2729,6 +2992,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/gitlab-runner/Cargo.toml b/gitlab-runner/Cargo.toml index fbf8f0a..ff7a22c 100644 --- a/gitlab-runner/Cargo.toml +++ b/gitlab-runner/Cargo.toml @@ -37,6 +37,7 @@ rand = "0.10.1" tokio-util = { version = "0.7.18", features = [ "io" ] } tokio-retry2 = { version = "0.9.1", features = ["jitter"] } normalize-path = "0.2.1" +metrics = "0.24.6" [dev-dependencies] tokio = { version = "1.50.0", features = [ "full", "test-util" ] } @@ -45,6 +46,8 @@ gitlab-runner-mock = { path = "../gitlab-runner-mock" } futures = "0.3.32" anyhow = "1.0.102" clap = { version = "4.6.0", features = ["derive"] } +metrics-util = "0.20.4" +metrics-exporter-prometheus = "0.18.3" [build-dependencies] vergen-gitcl = "9.1.0" diff --git a/gitlab-runner/examples/demo-runner.rs b/gitlab-runner/examples/demo-runner.rs index d4050b0..b796ff2 100644 --- a/gitlab-runner/examples/demo-runner.rs +++ b/gitlab-runner/examples/demo-runner.rs @@ -9,6 +9,7 @@ use gitlab_runner::job::Job; use gitlab_runner::{ GitlabLayer, JobHandler, JobResult, Phase, RunnerBuilder, UploadableFile, outputln, }; +use metrics_exporter_prometheus::PrometheusBuilder; use serde::Deserialize; use tokio::signal::unix::{SignalKind, signal}; use tracing::{debug, info}; @@ -215,6 +216,8 @@ async fn main() -> Result<()> { .with(layer) .init(); + PrometheusBuilder::new().install().unwrap(); + info!("Using {} as build storage prefix", dir.path().display()); let mut runner = RunnerBuilder::new(opts.server, opts.token, dir.path(), jobs) diff --git a/gitlab-runner/src/run.rs b/gitlab-runner/src/run.rs index 4cbdb2a..f630675 100644 --- a/gitlab-runner/src/run.rs +++ b/gitlab-runner/src/run.rs @@ -16,6 +16,9 @@ use crate::uploader::Uploader; use crate::{CancellableJobHandler, UploadableFile}; use crate::{JobResult, Phase}; +const METRICS_COUNTER_JOBS_STARTED: &str = "gitlab-runner-rs.jobs-started"; +const METRICS_COUNTER_JOBS_FINISHED: &str = "gitlab-runner-rs.jobs-finished"; + async fn run( job: Job, client: Client, @@ -300,6 +303,8 @@ impl Run { U: UploadableFile + Send + 'static, Ret: Future> + Send + 'static, { + metrics::counter!(METRICS_COUNTER_JOBS_STARTED).increment(1); + let cancel_token = CancellationToken::new(); let job = Job::new( @@ -359,5 +364,7 @@ impl Run { }; self.update(state, &cancel_token).await; } + + metrics::counter!(METRICS_COUNTER_JOBS_FINISHED).increment(1); } } diff --git a/gitlab-runner/src/runlist.rs b/gitlab-runner/src/runlist.rs index 9c91151..04a883b 100644 --- a/gitlab-runner/src/runlist.rs +++ b/gitlab-runner/src/runlist.rs @@ -5,6 +5,8 @@ use tokio::sync::watch; use crate::job::JobLog; +const METRICS_GAUGE_JOBS_RUNNING: &str = "gitlab-runner-rs.jobs-running"; + #[derive(Debug)] pub(crate) struct RunListEntry where @@ -45,10 +47,15 @@ impl Inner where K: Eq + std::hash::Hash, { + fn update_running_count(&self, n: usize) { + metrics::gauge!(METRICS_GAUGE_JOBS_RUNNING).set(n as u32); + self.changed.send_replace(n); + } + fn remove(&self, key: &K) { let mut r = self.rundata.write(); r.remove(key); - self.changed.send_replace(r.len()); + self.update_running_count(r.len()); } } @@ -91,7 +98,7 @@ where pub(crate) fn insert(&mut self, id: K, data: V) -> RunListEntry { let mut r = self.inner.rundata.write(); r.insert(id.clone(), data.clone()); - self.inner.changed.send_replace(r.len()); + self.inner.update_running_count(r.len()); RunListEntry { owner: self.inner.clone(), key: id, diff --git a/gitlab-runner/tests/integration.rs b/gitlab-runner/tests/integration.rs index d160de1..4bb7b25 100644 --- a/gitlab-runner/tests/integration.rs +++ b/gitlab-runner/tests/integration.rs @@ -4,6 +4,9 @@ use gitlab_runner::{GitlabLayer, JobHandler, JobResult, Phase, Runner, RunnerBui use gitlab_runner_mock::{ GitlabRunnerMock, MockJob, MockJobState, MockJobStepName, MockJobStepWhen, }; +use metrics_util::debugging::{DebugValue, DebuggingRecorder}; +use metrics_util::{CompositeKey, MetricKind}; +use std::collections::HashMap; use std::fmt::Debug; use std::future::Future; use std::sync::{Arc, Mutex}; @@ -749,3 +752,79 @@ async fn job_drain() { .with_subscriber(subscriber) .await; } + +// set_default_local_recorder needs to be run on a single-thread async runtime. +#[tokio::test(flavor = "current_thread")] +async fn job_metrics() { + type SnapshotMap = HashMap< + CompositeKey, + ( + Option, + Option, + DebugValue, + ), + >; + + fn get_counter(snapshot: &SnapshotMap, key: &str) -> u64 { + let Some((_, _, value)) = snapshot.get(&CompositeKey::new( + MetricKind::Counter, + metrics::Key::from_name(key.to_owned()), + )) else { + return 0; + }; + let DebugValue::Counter(value) = value else { + panic!("unexpected value for {key}: {value:?}"); + }; + *value + } + + fn get_gauge(snapshot: &SnapshotMap, key: &str) -> f64 { + let Some((_, _, value)) = snapshot.get(&CompositeKey::new( + MetricKind::Gauge, + metrics::Key::from_name(key.to_owned()), + )) else { + return 0.0; + }; + let DebugValue::Gauge(value) = value else { + panic!("unexpected value for {key}: {value:?}"); + }; + value.into_inner() + } + + let rec = DebuggingRecorder::new(); + let _default_rec = metrics::set_default_local_recorder(&rec); + let snapshotter = rec.snapshotter(); + + let mock = GitlabRunnerMock::start().await; + let job = mock.add_dummy_job("job success".to_string()); + + let (mut runner, subscriber, _dir) = setup_runner(&mock).await; + async { + let snapshot = snapshotter.snapshot().into_hashmap(); + assert_eq!(get_counter(&snapshot, "gitlab-runner-rs.jobs-started"), 0); + assert_eq!(get_counter(&snapshot, "gitlab-runner-rs.jobs-finished"), 0); + assert_eq!(get_gauge(&snapshot, "gitlab-runner-rs.jobs-running"), 0.0); + + let snapshotter2 = snapshotter.clone(); + let got_job = runner + .request_job(move |_| { + let snapshot = snapshotter2.snapshot().into_hashmap(); + assert_eq!(get_counter(&snapshot, "gitlab-runner-rs.jobs-started"), 1); + assert_eq!(get_counter(&snapshot, "gitlab-runner-rs.jobs-finished"), 0); + assert_eq!(get_gauge(&snapshot, "gitlab-runner-rs.jobs-running"), 1.0); + SimpleRun::dummy(Ok(())) + }) + .await + .unwrap(); + assert!(got_job); + runner.wait_for_space(1).await; + assert_eq!(MockJobState::Success, job.state()); + + let snapshot = snapshotter.snapshot().into_hashmap(); + assert_eq!(get_counter(&snapshot, "gitlab-runner-rs.jobs-started"), 0); + assert_eq!(get_counter(&snapshot, "gitlab-runner-rs.jobs-finished"), 1); + assert_eq!(get_gauge(&snapshot, "gitlab-runner-rs.jobs-running"), 0.0); + } + .with_subscriber(subscriber) + .await; +}