diff --git a/config/grafana/dashboard-configmap.yaml b/config/grafana/dashboard-configmap.yaml new file mode 100644 index 0000000..c46c7d9 --- /dev/null +++ b/config/grafana/dashboard-configmap.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: agentrax-grafana-dashboard + namespace: agentrax-system + labels: + app.kubernetes.io/name: agentrax + app.kubernetes.io/component: grafana-dashboard + # kube-prometheus-stack's Grafana sidecar watches for this label and + # auto-imports the JSON payload below as a Grafana dashboard. + grafana_dashboard: "1" +data: + agentrax-dashboard.json: | + { + "title": "Agentrax Operator — RED Dashboard", + "uid": "agentrax-red", + "tags": ["agentrax", "operator", "kubernetes"], + "timezone": "browser", + "schemaVersion": 38, + "refresh": "30s", + "templating": { + "list": [ + { + "name": "tenant", + "label": "Tenant", + "type": "query", + "datasource": "Prometheus", + "query": "label_values(agentrax_tenant_quota_usage_ratio, tenant)", + "includeAll": true, + "allValue": ".*", + "refresh": 1, + "multi": true, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Rate — Reconcile Operations / sec", + "type": "timeseries", + "gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(agentrax_reconcile_duration_seconds_count{tenant=~\"$tenant\"}[5m])) by (tenant)", + "legendFormat": "{{ tenant }}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "lineWidth": 2 } + } + } + }, + { + "id": 2, + "title": "Errors — Reconcile Error Rate / sec", + "type": "timeseries", + "gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"agentdeployment\"}[5m]))", + "legendFormat": "errors/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { "fixedColor": "red", "mode": "fixed" }, + "custom": { "lineWidth": 2 } + } + } + }, + { + "id": 3, + "title": "Duration — Reconcile P99 Latency (seconds)", + "type": "timeseries", + "gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(agentrax_reconcile_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])) by (le, tenant))", + "legendFormat": "P99 {{ tenant }}" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(agentrax_reconcile_duration_seconds_bucket{tenant=~\"$tenant\"}[5m])) by (le, tenant))", + "legendFormat": "P50 {{ tenant }}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "lineWidth": 2 }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 2 } + ] + } + } + } + }, + { + "id": 4, + "title": "Quota — Tenant Replica Usage Ratio", + "type": "gauge", + "gridPos": { "x": 12, "y": 8, "w": 12, "h": 8 }, + "datasource": "Prometheus", + "targets": [ + { + "expr": "agentrax_tenant_quota_usage_ratio{tenant=~\"$tenant\"}", + "legendFormat": "{{ tenant }}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.7 }, + { "color": "red", "value": 0.9 } + ] + } + } + } + } + ] + } diff --git a/config/prometheus/alerting-rules.yaml b/config/prometheus/alerting-rules.yaml new file mode 100644 index 0000000..ae80dbb --- /dev/null +++ b/config/prometheus/alerting-rules.yaml @@ -0,0 +1,59 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: agentrax-alerts + namespace: agentrax-system + labels: + app.kubernetes.io/name: agentrax + app.kubernetes.io/component: alerting + # kube-prometheus-stack discovers PrometheusRule resources by this label. + release: kube-prometheus-stack +spec: + groups: + + # ------------------------------------------------------------------------- + # Reconcile latency — fires when the P99 reconcile time sustained > 2s. + # This indicates the operator is struggling to converge: slow API server, + # contention on the TenantQuota lock, or an unhealthy admission webhook. + # ------------------------------------------------------------------------- + - name: agentrax.reconcile + rules: + - alert: AgentraxReconcileLatencyHigh + expr: | + histogram_quantile(0.99, + sum(rate(agentrax_reconcile_duration_seconds_bucket[5m])) + by (le, controller, tenant) + ) > 2 + for: 5m + labels: + severity: critical + annotations: + summary: >- + Reconcile P99 > 2s for controller={{ $labels.controller }} + tenant={{ $labels.tenant }} + description: >- + The AgentDeployment reconciler P99 latency has exceeded 2 seconds + for 5 minutes. Check kube-apiserver latency, webhook response times, + and operator logs for slow database or registry calls. + runbook_url: "https://github.com/gitcommitankit/agentrax/docs/ARCHITECTURE.md" + + # ------------------------------------------------------------------------- + # Quota saturation — fires when a tenant's replica usage ratio exceeds 90%. + # Early warning before the hard cap blocks new AgentDeployment creates. + # ------------------------------------------------------------------------- + - name: agentrax.quota + rules: + - alert: AgentraxTenantQuotaHigh + expr: agentrax_tenant_quota_usage_ratio > 0.9 + for: 2m + labels: + severity: warning + annotations: + summary: >- + Tenant {{ $labels.tenant }} replica quota usage > 90% + description: >- + Tenant {{ $labels.tenant }} is using more than 90% of its + maxTotalReplicas quota. New AgentDeployment creates will be + rejected by the admission webhook when the limit is reached. + Consider raising maxTotalReplicas in the TenantQuota spec. + runbook_url: "https://github.com/gitcommitankit/agentrax/docs/ARCHITECTURE.md" diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed13716..d97fe52 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,3 @@ resources: - monitor.yaml +- alerting-rules.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index 9a6d7d0..1f13c33 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -6,6 +6,7 @@ metadata: control-plane: controller-manager app.kubernetes.io/name: agentrax app.kubernetes.io/managed-by: kustomize + release: kube-prometheus-stack name: controller-manager-metrics-monitor namespace: system spec: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 839166f..4fb37ed 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -435,6 +435,40 @@ The `.github/workflows/terraform-lint.yml` workflow runs on every PR touching `i --- +### 4.10 Custom Prometheus Metrics, Alerting, & Grafana Dashboard + +Agentrax exports two custom metrics via `internal/observability/metrics.go`, both registered against the controller-runtime shared Prometheus registry (exposed on `/metrics`): + +| Metric | Type | Labels | What it measures | +| :--- | :--- | :--- | :--- | +| `agentrax_reconcile_duration_seconds` | Histogram | `controller`, `tenant` | Wall-clock duration of every `AgentDeployment` reconcile loop | +| `agentrax_tenant_quota_usage_ratio` | Gauge | `tenant` | `usedTotalReplicas / maxTotalReplicas` — approaches 1.0 at the quota ceiling | + +#### Instrumentation Points + +- **`internal/controller/agentdeployment_controller.go`**: A deferred closure at the top of `Reconcile()` records the histogram observation on every exit path — including early returns, errors, and requeues — so no reconcile path is missed. +- **`internal/controller/tenantquota_controller.go`**: The quota gauge is updated after `ComputeUsage()` runs, guarded against divide-by-zero when `MaxTotalReplicas` is zero. + +#### Alerting Rules (`config/prometheus/alerting-rules.yaml`) + +A `PrometheusRule` CRD with two alert groups, auto-discovered by the Prometheus Operator: + +| Alert | Expression | Duration | Severity | +| :--- | :--- | :--- | :--- | +| `AgentraxReconcileLatencyHigh` | P99 reconcile latency > 2s | 5 min | critical | +| `AgentraxTenantQuotaHigh` | Quota usage ratio > 0.9 | 2 min | warning | + +#### Grafana RED Dashboard (`config/grafana/dashboard-configmap.yaml`) + +A `ConfigMap` labelled `grafana_dashboard: "1"` auto-imported by the `kube-prometheus-stack` Grafana sidecar. Contains 4 panels: + +- **Rate**: `rate(agentrax_reconcile_duration_seconds_count[5m])` per tenant +- **Errors**: `rate(controller_runtime_reconcile_errors_total[5m])` for the `agentdeployment` controller +- **Duration**: P99 and P50 latency time-series with red/yellow/green threshold colouring at 2s/1s +- **Quota**: Gauge panel per tenant with a `$tenant` template variable; turns red above 90% + +--- + ## 5. Architectural Decision Records (ADRs) & Trade-Offs | Decision | Alternative Considered | Trade-Off & Rationale for Agentrax | diff --git a/go.mod b/go.mod index 550c94d..70ad0f1 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,9 @@ require ( github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.0 + github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_model v0.6.1 + github.com/stretchr/testify v1.10.0 k8s.io/api v0.31.0 k8s.io/apiextensions-apiserver v0.31.0 k8s.io/apimachinery v0.31.0 @@ -53,8 +56,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect - github.com/prometheus/client_model v0.6.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/cobra v1.8.1 // indirect diff --git a/internal/controller/agentdeployment_controller.go b/internal/controller/agentdeployment_controller.go index d64b90a..daaf848 100644 --- a/internal/controller/agentdeployment_controller.go +++ b/internal/controller/agentdeployment_controller.go @@ -43,6 +43,7 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1" + "github.com/gitcommitankit/agentrax/internal/observability" "github.com/gitcommitankit/agentrax/internal/registry" "github.com/gitcommitankit/agentrax/internal/rollout" "github.com/gitcommitankit/agentrax/internal/scaling" @@ -126,6 +127,16 @@ type AgentDeploymentReconciler struct { func (r *AgentDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) + // Observe reconcile wall-clock duration on every exit path, including early + // returns, errors, and requeues. The tenant label uses the request namespace + // because each namespace maps 1:1 to a tenant in Agentrax. + start := time.Now() + defer func() { + observability.ReconcileDuration. + WithLabelValues("agentdeployment", req.Namespace). + Observe(time.Since(start).Seconds()) + }() + // 1. Fetch the AgentDeployment; return immediately if it has been deleted. ad := &agentraxv1alpha1.AgentDeployment{} if err := r.Get(ctx, req.NamespacedName, ad); err != nil { diff --git a/internal/controller/tenantquota_controller.go b/internal/controller/tenantquota_controller.go index 4b0ce7e..77ecc56 100644 --- a/internal/controller/tenantquota_controller.go +++ b/internal/controller/tenantquota_controller.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1" + "github.com/gitcommitankit/agentrax/internal/observability" "github.com/gitcommitankit/agentrax/internal/quota" ) @@ -57,6 +58,7 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request) tq := &agentraxv1alpha1.TenantQuota{} if err := r.Get(ctx, req.NamespacedName, tq); err != nil { if apierrors.IsNotFound(err) { + observability.QuotaUsageRatio.DeleteLabelValues(req.Namespace) return ctrl.Result{}, nil } return ctrl.Result{}, fmt.Errorf("fetching TenantQuota: %w", err) @@ -89,6 +91,7 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request) latest := &agentraxv1alpha1.TenantQuota{} if err := r.Get(ctx, req.NamespacedName, latest); err != nil { if apierrors.IsNotFound(err) { + observability.QuotaUsageRatio.DeleteLabelValues(req.Namespace) return ctrl.Result{}, nil } return ctrl.Result{}, fmt.Errorf("re-fetching TenantQuota for status update: %w", err) @@ -100,6 +103,16 @@ func (r *TenantQuotaReconciler) Reconcile(ctx context.Context, req ctrl.Request) latest.Status.UsedGPUs = usage.UsedGPUs latest.Status.UsedTotalReplicas = usage.UsedTotalReplicas + // Emit quota usage ratio metric so Grafana and alerting rules can track + // how close this tenant is to its replica ceiling in real time. + // Guard against divide-by-zero: if MaxTotalReplicas is zero the ratio is 0. + if latest.Spec.MaxTotalReplicas > 0 { + ratio := float64(usage.UsedTotalReplicas) / float64(latest.Spec.MaxTotalReplicas) + observability.QuotaUsageRatio.WithLabelValues(latest.Namespace).Set(ratio) + } else { + observability.QuotaUsageRatio.WithLabelValues(latest.Namespace).Set(0) + } + // 5. Set or clear the OverQuota condition based on whether usage exceeds spec. // Use latest.Spec (re-fetched) rather than tq.Spec (first fetch) to avoid // evaluating against a ceiling that may have changed between the two Gets. diff --git a/internal/controller/tenantquota_controller_test.go b/internal/controller/tenantquota_controller_test.go index 33db6e0..cb84474 100644 --- a/internal/controller/tenantquota_controller_test.go +++ b/internal/controller/tenantquota_controller_test.go @@ -30,6 +30,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" agentraxv1alpha1 "github.com/gitcommitankit/agentrax/api/v1alpha1" + "github.com/gitcommitankit/agentrax/internal/observability" + "github.com/prometheus/client_golang/prometheus/testutil" + crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ) var _ = Describe("TenantQuota Controller", func() { @@ -155,6 +158,45 @@ var _ = Describe("TenantQuota Controller", func() { g.Expect(tqFetched.Status.UsedAgents).To(BeNumerically("==", 0)) }, timeout, interval).Should(Succeed()) }) + + It("updates and cleans up QuotaUsageRatio metric keyed by tenant namespace", func() { + tq := makeTQ("tq-metric", tqNS, 6, 4, 10, 6) + Expect(k8sClient.Create(ctx, tq)).To(Succeed()) + + ad := makeBasicAD("ad-metric-1", tqNS, "tq-metric", 2) + Expect(k8sClient.Create(ctx, ad)).To(Succeed()) + + Eventually(func(g Gomega) { + val := testutil.ToFloat64(observability.QuotaUsageRatio.WithLabelValues(tqNS)) + g.Expect(val).To(BeNumerically("~", 0.2, 1e-4)) + }, timeout, interval).Should(Succeed()) + + // Deleting the TenantQuota should trigger reconcile NotFound path and remove the series. + Expect(k8sClient.Delete(ctx, tq)).To(Succeed()) + Eventually(func() bool { + err := k8sClient.Get(ctx, namespacedName("tq-metric", tqNS), &agentraxv1alpha1.TenantQuota{}) + return apierrors.IsNotFound(err) + }, timeout, interval).Should(BeTrue()) + + Eventually(func(g Gomega) { + mfs, err := crmetrics.Registry.Gather() + g.Expect(err).NotTo(HaveOccurred()) + found := false + for _, mf := range mfs { + if mf.GetName() == "agentrax_tenant_quota_usage_ratio" { + for _, m := range mf.GetMetric() { + for _, lbl := range m.GetLabel() { + if lbl.GetName() == "tenant" && lbl.GetValue() == tqNS { + found = true + break + } + } + } + } + } + g.Expect(found).To(BeFalse(), "expected no metric sample labeled tenant=%q to remain", tqNS) + }, timeout, interval).Should(Succeed()) + }) }) // ── OverQuota condition ─────────────────────────────────────────────────── diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go new file mode 100644 index 0000000..3f0fd27 --- /dev/null +++ b/internal/observability/metrics.go @@ -0,0 +1,41 @@ +// Package observability provides custom Prometheus metrics for the Agentrax operator. +// Metrics are registered against the controller-runtime shared registry so they are +// automatically exposed on the /metrics endpoint that cmd/main.go wires up. +package observability + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// ReconcileDuration tracks the wall-clock duration of each AgentDeployment reconcile loop. +// Labels: +// - controller: always "agentdeployment" for the main reconciler +// - tenant: the Kubernetes namespace (which maps 1:1 to a tenant in Agentrax) +var ReconcileDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "agentrax_reconcile_duration_seconds", + Help: "Duration of AgentDeployment reconcile loops in seconds.", + // DefBuckets covers sub-millisecond to 10s which spans expected reconcile times. + Buckets: prometheus.DefBuckets, + }, + []string{"controller", "tenant"}, +) + +// QuotaUsageRatio tracks the current replica usage as a fraction of maxTotalReplicas. +// A value of 1.0 means the tenant is at the hard replica cap. +// Labels: +// - tenant: the Kubernetes namespace (which maps 1:1 to a tenant in Agentrax) +var QuotaUsageRatio = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "agentrax_tenant_quota_usage_ratio", + Help: "Current replica usage as a fraction of maxTotalReplicas (0.0–1.0) per tenant.", + }, + []string{"tenant"}, +) + +func init() { + // MustRegister panics on duplicate registration — safe here because init + // runs exactly once per process. Both metrics are owned by this package. + metrics.Registry.MustRegister(ReconcileDuration, QuotaUsageRatio) +} diff --git a/internal/observability/metrics_test.go b/internal/observability/metrics_test.go new file mode 100644 index 0000000..c3db85c --- /dev/null +++ b/internal/observability/metrics_test.go @@ -0,0 +1,85 @@ +package observability_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/gitcommitankit/agentrax/internal/observability" +) + +// TestReconcileDurationObserve verifies that ReconcileDuration records +// observations and that the histogram is correctly exported via text format. +func TestReconcileDurationObserve(t *testing.T) { + reg := prometheus.NewRegistry() + observability.ReconcileDuration.Reset() + reg.MustRegister(observability.ReconcileDuration) + + observability.ReconcileDuration.WithLabelValues("agentdeployment", "tenant-test").Observe(0.5) + observability.ReconcileDuration.WithLabelValues("agentdeployment", "tenant-test").Observe(1.5) + + expected := ` + # HELP agentrax_reconcile_duration_seconds Duration of AgentDeployment reconcile loops in seconds. + # TYPE agentrax_reconcile_duration_seconds histogram + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.005"} 0 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.01"} 0 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.025"} 0 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.05"} 0 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.1"} 0 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.25"} 0 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="0.5"} 1 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="1"} 1 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="2.5"} 2 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="5"} 2 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="10"} 2 + agentrax_reconcile_duration_seconds_bucket{controller="agentdeployment",tenant="tenant-test",le="+Inf"} 2 + agentrax_reconcile_duration_seconds_sum{controller="agentdeployment",tenant="tenant-test"} 2 + agentrax_reconcile_duration_seconds_count{controller="agentdeployment",tenant="tenant-test"} 2 + ` + err := testutil.GatherAndCompare(reg, strings.NewReader(expected), "agentrax_reconcile_duration_seconds") + require.NoError(t, err) + + count, err := testutil.GatherAndCount(reg, "agentrax_reconcile_duration_seconds") + require.NoError(t, err) + assert.Greater(t, count, 0, "expected histogram to produce at least one metric series") +} + +// TestQuotaUsageRatioSet verifies that QuotaUsageRatio records the expected +// gauge value for each test case. +func TestQuotaUsageRatioSet(t *testing.T) { + tests := []struct { + name string + tenant string + value float64 + expected float64 + }{ + {"normal usage below ceiling", "tenant-alpha", 0.75, 0.75}, + {"at ceiling", "tenant-beta", 1.0, 1.0}, + {"zero usage", "tenant-gamma", 0.0, 0.0}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + reg := prometheus.NewRegistry() + observability.QuotaUsageRatio.Reset() + reg.MustRegister(observability.QuotaUsageRatio) + + observability.QuotaUsageRatio.WithLabelValues(tc.tenant).Set(tc.value) + + expected := fmt.Sprintf(` + # HELP agentrax_tenant_quota_usage_ratio Current replica usage as a fraction of maxTotalReplicas (0.0–1.0) per tenant. + # TYPE agentrax_tenant_quota_usage_ratio gauge + agentrax_tenant_quota_usage_ratio{tenant="%s"} %g + `, tc.tenant, tc.value) + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(expected), "agentrax_tenant_quota_usage_ratio")) + + got := testutil.ToFloat64(observability.QuotaUsageRatio.WithLabelValues(tc.tenant)) + assert.InDelta(t, tc.expected, got, 1e-9) + }) + } +}