From c91cc535e1b5508e8b2e4a06db0d41c0ef923004 Mon Sep 17 00:00:00 2001 From: dobbi84 Date: Thu, 4 Jun 2026 11:29:22 +0200 Subject: [PATCH 1/6] fix: correct OCP/SMART raw metric metadata - data_units_read/written help now states the values are in thousands of 512-byte units (bytes = value * 512000), per the NVMe spec. - physical_media_units_* help corrected from "1000h sector size" to bytes (128-bit byte counter split lo/hi), per the OCP spec. - bad_user/system_nand_blocks_normalized changed from counter to gauge: normalized 0-100 values decrease over time, so rate()/increase() on a counter would misread them as resets. --- cmd/collector.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/cmd/collector.go b/cmd/collector.go index 85ca44a..832df27 100644 --- a/cmd/collector.go +++ b/cmd/collector.go @@ -155,13 +155,15 @@ func newNvmeCollector(collectorStates map[string]bool) prometheus.Collector { counterValueFactory.CreateLogMetricProvider( "nvme_data_units_read", - "Total number of 512-byte data units read from the NVMe device by the host", + "Number of 512-byte data units read by the host. Per the NVMe spec this is "+ + "reported in thousands and rounded up, so bytes read = value * 512000 (1000 * 512)", "data_units_read", ), counterValueFactory.CreateLogMetricProvider( "nvme_data_units_written", - "Total number of 512-byte data units written to the NVMe device by the host", + "Number of 512-byte data units written by the host. Per the NVMe spec this is "+ + "reported in thousands and rounded up, so bytes written = value * 512000 (1000 * 512)", "data_units_written", ), @@ -254,22 +256,26 @@ func newNvmeCollector(collectorStates map[string]bool) prometheus.Collector { ocpLogMetricProviders := []pkg.MetricProvider{ counterValueFactory.CreateLogMetricProvider( "nvme_physical_media_units_written_hi", - "Physical media units written to the device (high 64 bits). Unit size is 1000h sector size", + "Bytes written to the physical media, high 64 bits of a 128-bit byte counter "+ + "(per OCP spec the value is in bytes; hi is ~always 0 below ~18 EB)", "Physical media units written.hi", ), counterValueFactory.CreateLogMetricProvider( "nvme_physical_media_units_written_lo", - "Physical media units written to the device (low 64 bits). Unit size is 1000h sector size", + "Bytes written to the physical media, low 64 bits of a 128-bit byte counter "+ + "(per OCP spec the value is in bytes)", "Physical media units written.lo", ), counterValueFactory.CreateLogMetricProvider( "nvme_physical_media_units_read_hi", - "Physical media units read from the device (high 64 bits). Unit size is 1000h sector size", + "Bytes read from the physical media, high 64 bits of a 128-bit byte counter "+ + "(per OCP spec the value is in bytes; hi is ~always 0 below ~18 EB)", "Physical media units read.hi", ), counterValueFactory.CreateLogMetricProvider( "nvme_physical_media_units_read_lo", - "Physical media units read from the device (low 64 bits). Unit size is 1000h sector size", + "Bytes read from the physical media, low 64 bits of a 128-bit byte counter "+ + "(per OCP spec the value is in bytes)", "Physical media units read.lo", ), counterValueFactory.CreateLogMetricProvider( @@ -277,9 +283,10 @@ func newNvmeCollector(collectorStates map[string]bool) prometheus.Collector { "Raw count of user NAND blocks that have been retired due to errors", "Bad user nand blocks - Raw", ), - counterValueFactory.CreateLogMetricProvider( + gaugeValueFactory.CreateLogMetricProvider( "nvme_bad_user_nand_blocks_normalized", - "Normalized value (0-100) of bad user NAND blocks relative to the maximum allowed", + "Normalized value (0-100) of bad user NAND blocks relative to the maximum allowed. "+ + "Gauge: this value decreases as blocks are retired, so it is not a counter", "Bad user nand blocks - Normalized", ), counterValueFactory.CreateLogMetricProvider( @@ -287,9 +294,10 @@ func newNvmeCollector(collectorStates map[string]bool) prometheus.Collector { "Raw count of system area NAND blocks that have been retired due to errors", "Bad system nand blocks - Raw", ), - counterValueFactory.CreateLogMetricProvider( + gaugeValueFactory.CreateLogMetricProvider( "nvme_bad_system_nand_blocks_normalized", - "Normalized value (0-100) of bad system NAND blocks relative to the maximum allowed", + "Normalized value (0-100) of bad system NAND blocks relative to the maximum allowed. "+ + "Gauge: this value decreases as blocks are retired, so it is not a counter", "Bad system nand blocks - Normalized", ), counterValueFactory.CreateLogMetricProvider( From 3793d3df58347c9a890d1e925c78f43fcdcc775b Mon Sep 17 00:00:00 2001 From: dobbi84 Date: Thu, 4 Jun 2026 11:30:10 +0200 Subject: [PATCH 2/6] docs(prom): annotate OCP rules and add observed DWPD Document every magic constant and unit assumption (512000 = 1000*512 data-unit-to-byte factor, 1e12 decimal TB, 3000 P/E cycles, 365*5 warranty window, 273.15 Kelvin offset) and note which rules are stateful (rate/predict_linear) and must stay in Prometheus. Add device:DWPD_observed, the actual drive-writes-per-day computed from the host write rate, complementing the existing rated DWPD_calculated. --- resources/prom/nvme_ocp.yml | 47 ++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/resources/prom/nvme_ocp.yml b/resources/prom/nvme_ocp.yml index ba24439..85b251f 100644 --- a/resources/prom/nvme_ocp.yml +++ b/resources/prom/nvme_ocp.yml @@ -1,39 +1,80 @@ +# Recording rules deriving OCP/SMART metrics from the raw values exposed by +# nvme_exporter. These stay in Prometheus (not the exporter) on purpose: +# - rate()/predict_linear() rules are stateful and need stored history. +# - the rest embed deployment-/vendor-specific constants (P/E cycles, +# warranty years) that are far easier to tune here than in a recompiled +# binary deployed across hosts. +# +# Unit reference (verified against the specs): +# * nvme_data_units_written / _read: reported in THOUSANDS of 512-byte units +# and rounded up (NVMe Base Spec). Bytes = value * 512000 (= 1000 * 512). +# * nvme_physical_media_units_written_lo / _hi: the value is in BYTES, split +# across the low/high 64 bits of a 128-bit counter (OCP Datacenter NVMe SSD +# Spec). The low word alone covers ~18 EB, so _hi is effectively always 0 +# and using _lo by itself is correct in practice. +# * nvme_physical_size: bytes. +# * nvme_temperature: Kelvin. groups: - name: Nvme device Write Amplification rules: + # Lifetime write-amplification factor: NAND bytes written / host bytes written. + # Both sides are in bytes (see unit reference above), so the ratio is dimensionless. + # Uses _lo only, which is correct until physical writes exceed ~18 EB. - record: device:nvme_device_waf:total expr: nvme_physical_media_units_written_lo / (nvme_data_units_written * 512000) + # NAND bytes written per second. - record: device:nvme_physical_media_units_written_lo:rate expr: rate(nvme_physical_media_units_written_lo[1m]) + # Host bytes written per second (512000 converts data units -> bytes). - record: device:nvme_data_units_written:rate_bytes expr: rate(nvme_data_units_written[5m])*512000 + # Instantaneous (windowed) write amplification = NAND write rate / host write rate. - record: device:nvme_device_writeamp:rate expr: device:nvme_physical_media_units_written_lo:rate / device:nvme_data_units_written:rate_bytes - name: NVMe device Temperature from K to Celsius rules: + # 273.15 = absolute zero offset (Kelvin -> Celsius). - record: device:nvme_temperature:celsius expr: (nvme_temperature - 273.15) - - name: NVMe Physical size of SSDs (in TB) + - name: NVMe Physical size of SSDs (in TB) rules: + # 1e12 = decimal terabyte (TB, not TiB). round to 2 decimal places. - record: device:nvme_physical_size:TB expr: round(nvme_physical_size / 1000000000000, 0.01) - name: Calculation of TBW (capacity times flash cycles over waf) rules: + # ASSUMPTION: 3000 = rated NAND program/erase cycles. This is TLC-class; + # QLC is ~1000 and SLC ~100k. Tune per drive model/fleet — a wrong value + # here skews TBW, DRL and DWPD below. - record: device:TBW_numerator expr: device:nvme_physical_size:TB * 3000 + # Total Bytes Written the drive is rated for = (capacity * P/E cycles) / WAF. - record: device:TBW_calculated expr: device:TBW_numerator / on(device) device:nvme_device_waf:total - name: Estimation of the device remaining life rules: + # Device Remaining Life % = 100 * (1 - host_TB_written / rated_TBW). + # 512000 converts data units -> bytes; 1e12 converts bytes -> TB. - record: device:DRL_calculated expr: 100 * (1 - ((nvme_data_units_written*512000)/1e12) / on(device) device:TBW_calculated) - + - name: Prediction of DRL in 4 hours rules: + # Linear extrapolation of DRL over the next 4h (4*3600 s) from the last 10m. + # predict_linear is stateful (needs stored history) -> Prometheus only. - record: device:projected_DRL_4h expr: predict_linear(device:DRL_calculated[10m], 4*3600) - name: Calculation of dynamic Drives Writes per Day (DWPD) rules: + # Rated/endurance DWPD = rated_TBW / (capacity * warranty_days_in_TB-terms). + # ASSUMPTION: 365*5 = a 5-year warranty window. Adjust to the drive's warranty. - record: device:DWPD_calculated - expr: device:TBW_calculated / on(device) (365*5*device:nvme_physical_size:TB) + expr: device:TBW_calculated / on(device) (365*5*device:nvme_physical_size:TB) + # Observed (actual) drive writes per day = host bytes/day / capacity_bytes. + # 512000: data units -> bytes; 86400: per-second rate -> per-day. + # rate() is stateful, so observed DWPD can only be computed here, not by the + # (stateless) exporter. Window [1d] is a rolling daily average; shorten it + # (e.g. [3h]) for a more responsive recent-DWPD value. + - record: device:DWPD_observed + expr: (rate(nvme_data_units_written[1d]) * 512000 * 86400) / on(device) nvme_physical_size From cb28e14014ea329058f294e5a6121da78b599a21 Mon Sep 17 00:00:00 2001 From: dobbi84 Date: Thu, 4 Jun 2026 11:51:02 +0200 Subject: [PATCH 3/6] fix: expose nvme_log_page_guid as an info metric The GUID is a 128-bit identifier; read via gjson .Float() it was always 0/garbage. Add a label-metric provider that emits a constant-1 gauge nvme_log_page_guid_info{guid="..."} carrying the GUID as a label, the standard Prometheus pattern for string identifiers. --- cmd/collector.go | 31 ++++++++++++++++++++++++++++--- pkg/provider.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/cmd/collector.go b/cmd/collector.go index 832df27..d8ebb69 100644 --- a/cmd/collector.go +++ b/cmd/collector.go @@ -53,6 +53,29 @@ func (f *ProviderFactory) CreateLogMetricProvider( ) } +// CreateLabelMetricProvider builds an info-metric provider that emits a +// constant 1 and exposes the string at jsonKey as labelName (e.g. a GUID). +func (f *ProviderFactory) CreateLabelMetricProvider( + fqName string, + help string, + jsonKey string, + labelName string, +) pkg.MetricProvider { + labels := make([]string, 0, len(f.defaultLabels)+1) + labels = append(labels, f.defaultLabels...) + labels = append(labels, labelName) + + return pkg.NewLabelMetricProvider( + prometheus.NewDesc( + fqName, + help, + labels, + nil, + ), + jsonKey, + ) +} + func (f *ProviderFactory) CreateInfoMetricProvider( fqName string, help string, @@ -405,10 +428,12 @@ func newNvmeCollector(collectorStates map[string]bool) prometheus.Collector { "Version number of the OCP SMART log page specification", "Log page version", ), - gaugeValueFactory.CreateLogMetricProvider( - "nvme_log_page_guid", - "GUID (Globally Unique Identifier) of the OCP SMART log page", + gaugeValueFactory.CreateLabelMetricProvider( + "nvme_log_page_guid_info", + "OCP SMART log page GUID, exposed as the 'guid' label on a constant-1 info "+ + "metric (it is a 128-bit identifier, not a meaningful number)", "Log page GUID", + "guid", ), gaugeValueFactory.CreateLogMetricProvider( "nvme_errata_version_field", diff --git a/pkg/provider.go b/pkg/provider.go index 61875b5..4970081 100644 --- a/pkg/provider.go +++ b/pkg/provider.go @@ -17,6 +17,13 @@ type MetricProvider struct { // jsonKey is the string key that the object needs to access // in the device JSON to fetch the metric float64 value jsonKey string + + // valueAsLabel, when true, makes GetMetric emit a constant gauge of 1 + // and attach the string found at jsonKey as the last label instead of + // parsing it as a float. This is the Prometheus "info metric" pattern, + // used for identifiers/strings (e.g. a 128-bit GUID) that have no + // meaningful numeric value. + valueAsLabel bool } // NewMetricProvider is the constructor for MetricProvider objects. @@ -33,6 +40,21 @@ func NewMetricProvider( } } +// NewLabelMetricProvider builds an "info metric" provider: it emits a constant +// gauge of 1 and attaches the string value at jsonKey as an extra label. Use it +// for identifiers/strings that cannot be represented as a float. +func NewLabelMetricProvider( + desc *prometheus.Desc, + jsonKey string, +) MetricProvider { + return MetricProvider{ + Desc: desc, + ValueType: prometheus.GaugeValue, + jsonKey: jsonKey, + valueAsLabel: true, + } +} + // GetMetric computes the metric from the // data in JSON form. func (ip MetricProvider) GetMetric( @@ -53,6 +75,15 @@ func (ip MetricProvider) GetMetric( return nil } + // Info-metric mode: emit a constant 1 with the string value as an extra label. + if ip.valueAsLabel { + labelValues := make([]string, 0, len(labels)+1) + labelValues = append(labelValues, labels...) + labelValues = append(labelValues, result.String()) + + return prometheus.MustNewConstMetric(ip.Desc, ip.ValueType, 1, labelValues...) + } + // Handle both scalar values (v2.8) and object values (v2.11+) // In v2.11+, some fields like critical_warning are objects with a "value" field var value float64 From 1350ea453d3a8540112059a79d5db396b9daa000 Mon Sep 17 00:00:00 2001 From: dobbi84 Date: Thu, 4 Jun 2026 11:51:02 +0200 Subject: [PATCH 4/6] refactor(prom): derive endurance from percent_used; add DWPD_lifetime Replace the hard-coded 3000 P/E-cycle guess with the device's own nvme_percent_used (the vendor wear estimate, which already accounts for real endurance and write amplification): - DRL_calculated is now clamp_min(100 - nvme_percent_used, 0) - TBW_calculated is derived from percent_used (guarded to >0) - removed TBW_numerator (the *3000 rule) Keep 365*5 (warranty window) as it is contractual, not telemetry; keep the spec unit constants (512000, 1e12, 273.15). Add device:DWPD_lifetime (total host writes / capacity / powered-on days), a constant-free, single-scrape DWPD. --- resources/prom/nvme_ocp.yml | 89 +++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/resources/prom/nvme_ocp.yml b/resources/prom/nvme_ocp.yml index 85b251f..2fef631 100644 --- a/resources/prom/nvme_ocp.yml +++ b/resources/prom/nvme_ocp.yml @@ -1,24 +1,34 @@ # Recording rules deriving OCP/SMART metrics from the raw values exposed by # nvme_exporter. These stay in Prometheus (not the exporter) on purpose: # - rate()/predict_linear() rules are stateful and need stored history. -# - the rest embed deployment-/vendor-specific constants (P/E cycles, -# warranty years) that are far easier to tune here than in a recompiled -# binary deployed across hosts. +# - the endurance rules embed a warranty window that is contractual, not +# telemetry, and is easier to tune here than in a recompiled binary. +# +# Constants used below, and why each is or isn't a tunable assumption: +# * 512000 = 1000 * 512: NVMe reports data units in THOUSANDS of 512-byte +# units (rounded up), so bytes = value * 512000. Spec-defined, not tunable. +# * 1e12: decimal terabyte (TB, not TiB). Unit conversion, not tunable. +# * 273.15: Kelvin -> Celsius offset. Not tunable. +# * 365*5: assumed 5-year warranty window for RATED DWPD. This is contractual +# and cannot be derived from telemetry — adjust it to the drive's warranty. +# +# Endurance (remaining life / rated TBW) is derived from the device's own +# nvme_percent_used (the vendor's estimate of life consumed, which already +# accounts for real cell endurance and write amplification). This replaces the +# previous hard-coded "3000 P/E cycles" guess. # # Unit reference (verified against the specs): -# * nvme_data_units_written / _read: reported in THOUSANDS of 512-byte units -# and rounded up (NVMe Base Spec). Bytes = value * 512000 (= 1000 * 512). -# * nvme_physical_media_units_written_lo / _hi: the value is in BYTES, split -# across the low/high 64 bits of a 128-bit counter (OCP Datacenter NVMe SSD -# Spec). The low word alone covers ~18 EB, so _hi is effectively always 0 -# and using _lo by itself is correct in practice. -# * nvme_physical_size: bytes. -# * nvme_temperature: Kelvin. +# * nvme_physical_media_units_written_lo/_hi: BYTES, split across the low/high +# 64 bits of a 128-bit counter (OCP spec). _lo alone covers ~18 EB, so _hi +# is effectively always 0 and using _lo by itself is correct in practice. +# * nvme_physical_size: bytes. nvme_temperature: Kelvin. +# * nvme_percent_used: vendor estimate of % life consumed (0 = new; can exceed +# 100 once the rated endurance is passed). groups: - name: Nvme device Write Amplification rules: # Lifetime write-amplification factor: NAND bytes written / host bytes written. - # Both sides are in bytes (see unit reference above), so the ratio is dimensionless. + # Both sides are in bytes, so the ratio is dimensionless. # Uses _lo only, which is correct until physical writes exceed ~18 EB. - record: device:nvme_device_waf:total expr: nvme_physical_media_units_written_lo / (nvme_data_units_written * 512000) @@ -41,40 +51,43 @@ groups: # 1e12 = decimal terabyte (TB, not TiB). round to 2 decimal places. - record: device:nvme_physical_size:TB expr: round(nvme_physical_size / 1000000000000, 0.01) - - name: Calculation of TBW (capacity times flash cycles over waf) - rules: - # ASSUMPTION: 3000 = rated NAND program/erase cycles. This is TLC-class; - # QLC is ~1000 and SLC ~100k. Tune per drive model/fleet — a wrong value - # here skews TBW, DRL and DWPD below. - - record: device:TBW_numerator - expr: device:nvme_physical_size:TB * 3000 - # Total Bytes Written the drive is rated for = (capacity * P/E cycles) / WAF. - - record: device:TBW_calculated - expr: device:TBW_numerator / on(device) device:nvme_device_waf:total - name: Estimation of the device remaining life rules: - # Device Remaining Life % = 100 * (1 - host_TB_written / rated_TBW). - # 512000 converts data units -> bytes; 1e12 converts bytes -> TB. + # Device Remaining Life % straight from the vendor's own wear estimate. + # No P/E-cycle assumption: nvme_percent_used already reflects real endurance + # and write amplification. clamp_min floors it at 0 once life is exceeded. - record: device:DRL_calculated - expr: 100 * (1 - ((nvme_data_units_written*512000)/1e12) / on(device) device:TBW_calculated) - + expr: clamp_min(100 - nvme_percent_used, 0) + - name: Rated TBW implied by the vendor wear estimate (in TB) + rules: + # Total host TB the drive implies it can take = host_TB_written / fraction_used. + # 512000: data units -> bytes; 1e12: bytes -> TB. Guarded to percent_used > 0 + # so it is not emitted (avoids div-by-zero) until the drive has measurable wear. + - record: device:TBW_calculated + expr: (((nvme_data_units_written * 512000) / 1e12) / (nvme_percent_used / 100)) and (nvme_percent_used > 0) - name: Prediction of DRL in 4 hours rules: - # Linear extrapolation of DRL over the next 4h (4*3600 s) from the last 10m. - # predict_linear is stateful (needs stored history) -> Prometheus only. + # Linear extrapolation of DRL over the next 4h (4*3600 s). + # NOTE: nvme_percent_used is coarse (integer %, updated infrequently), so a + # short lookback often yields a flat slope; [6h] gives predict_linear enough + # history to detect movement. predict_linear is stateful -> Prometheus only. - record: device:projected_DRL_4h - expr: predict_linear(device:DRL_calculated[10m], 4*3600) - - - name: Calculation of dynamic Drives Writes per Day (DWPD) + expr: predict_linear(device:DRL_calculated[6h], 4*3600) + - name: Calculation of Drives Writes per Day (DWPD) rules: - # Rated/endurance DWPD = rated_TBW / (capacity * warranty_days_in_TB-terms). - # ASSUMPTION: 365*5 = a 5-year warranty window. Adjust to the drive's warranty. + # RATED DWPD = rated_TBW / (warranty_days * capacity_TB). + # 365*5 = assumed 5-year warranty window — adjust to the drive's warranty. - record: device:DWPD_calculated expr: device:TBW_calculated / on(device) (365*5*device:nvme_physical_size:TB) - # Observed (actual) drive writes per day = host bytes/day / capacity_bytes. - # 512000: data units -> bytes; 86400: per-second rate -> per-day. - # rate() is stateful, so observed DWPD can only be computed here, not by the - # (stateless) exporter. Window [1d] is a rolling daily average; shorten it - # (e.g. [3h]) for a more responsive recent-DWPD value. + # OBSERVED (recent) DWPD = host bytes/day / capacity_bytes. + # 512000: data units -> bytes; 86400: per-second rate -> per-day. rate() is + # stateful, so this can only be computed here, not by the stateless exporter. + # Window [1d] is a rolling daily average; shorten (e.g. [3h]) for responsiveness. - record: device:DWPD_observed expr: (rate(nvme_data_units_written[1d]) * 512000 * 86400) / on(device) nvme_physical_size + # LIFETIME-AVERAGE DWPD = total host bytes / capacity / powered-on days. + # Constant-free and computable from a single scrape. NOTE: nvme_power_on_hours + # is powered-on time, not calendar time, so a drive that is often off will + # show a higher lifetime DWPD than its true per-calendar-day rate. + - record: device:DWPD_lifetime + expr: ((nvme_data_units_written * 512000) / on(device) nvme_physical_size) / (nvme_power_on_hours / 24) From f5ab911a54af4830b260f361cc28842b3f8d8033 Mon Sep 17 00:00:00 2001 From: dobbi84 Date: Thu, 4 Jun 2026 11:57:58 +0200 Subject: [PATCH 5/6] refactor(prom): split remaining-life into accurate value + rate-based projection Two distinct rules for two distinct questions: - device:DRL_calculated: accurate remaining-life % from the vendor's own percent_used (how much life is left now). - device:DRL_days_remaining: estimated days to end-of-life at the current write rate. The wear rate comes from the continuous data-units counter (smooth), the endurance size from percent_used. Replaces device:projected_DRL_4h, whose predict_linear over the coarse integer percent_used produced a flat/jumpy slope regardless of window. --- resources/prom/nvme_ocp.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/resources/prom/nvme_ocp.yml b/resources/prom/nvme_ocp.yml index 2fef631..b048a4d 100644 --- a/resources/prom/nvme_ocp.yml +++ b/resources/prom/nvme_ocp.yml @@ -1,6 +1,6 @@ # Recording rules deriving OCP/SMART metrics from the raw values exposed by # nvme_exporter. These stay in Prometheus (not the exporter) on purpose: -# - rate()/predict_linear() rules are stateful and need stored history. +# - rate()-based rules are stateful and need stored history. # - the endurance rules embed a warranty window that is contractual, not # telemetry, and is easier to tune here than in a recompiled binary. # @@ -53,11 +53,27 @@ groups: expr: round(nvme_physical_size / 1000000000000, 0.01) - name: Estimation of the device remaining life rules: - # Device Remaining Life % straight from the vendor's own wear estimate. + # ACCURATE remaining life %, straight from the vendor's own wear estimate. # No P/E-cycle assumption: nvme_percent_used already reflects real endurance # and write amplification. clamp_min floors it at 0 once life is exceeded. + # Answers "how much life is left right now". - record: device:DRL_calculated expr: clamp_min(100 - nvme_percent_used, 0) + # SMOOTH projection companion: estimated days until nvme_percent_used reaches + # 100% at the current write rate. Answers "when will it run out". + # The wear RATE comes from the continuous data-units counter (smooth), while + # the total endurance is calibrated from percent_used — so this avoids both + # the 3000 P/E guess and predict_linear over the coarse percent_used step. + # remaining_units = data_units_written * (100 - used)/used + # daily_units = rate(data_units_written[1d]) * 86400 + # days = remaining_units / daily_units (the 512000 factor cancels) + # Guarded against new (used=0) and idle (rate=0) drives to avoid div-by-zero. + - record: device:DRL_days_remaining + expr: | + (nvme_data_units_written * (100 - nvme_percent_used) / nvme_percent_used) + / (rate(nvme_data_units_written[1d]) * 86400) + and (nvme_percent_used > 0) + and (rate(nvme_data_units_written[1d]) > 0) - name: Rated TBW implied by the vendor wear estimate (in TB) rules: # Total host TB the drive implies it can take = host_TB_written / fraction_used. @@ -65,14 +81,6 @@ groups: # so it is not emitted (avoids div-by-zero) until the drive has measurable wear. - record: device:TBW_calculated expr: (((nvme_data_units_written * 512000) / 1e12) / (nvme_percent_used / 100)) and (nvme_percent_used > 0) - - name: Prediction of DRL in 4 hours - rules: - # Linear extrapolation of DRL over the next 4h (4*3600 s). - # NOTE: nvme_percent_used is coarse (integer %, updated infrequently), so a - # short lookback often yields a flat slope; [6h] gives predict_linear enough - # history to detect movement. predict_linear is stateful -> Prometheus only. - - record: device:projected_DRL_4h - expr: predict_linear(device:DRL_calculated[6h], 4*3600) - name: Calculation of Drives Writes per Day (DWPD) rules: # RATED DWPD = rated_TBW / (warranty_days * capacity_TB). From 4f9192254d48ff0340b000a7b5680929e8bcc18a Mon Sep 17 00:00:00 2001 From: dobbi84 Date: Thu, 4 Jun 2026 12:46:01 +0200 Subject: [PATCH 6/6] fix(grafana): repair dashboard JSON and sync with metric changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add the missing root '{' — the JSON was invalid and could not be imported into Grafana at all. - Fix broken metric references: - device:projected_DRL_8h (rule never existed) -> device:DRL_days_remaining, retitled to estimated days-to-EOL with day units and proper thresholds. - node_nvme_info (not exported) -> nvme_physical_size, with the Disk Info table regrouped on the real info labels (model_number, serial_number, firmware, generic_path) plus capacity. - Remove a stray 'nvme_num' target. - Add panels for the new recording rules: device:DWPD_observed and device:DWPD_lifetime. - Correct stale/copy-paste descriptions and titles (DRL gauge now reflects percent_used basis, DWPD reflects telemetry-derived TBW, unsafe-shutdowns description, truncated 'Host Read/Write Commands' titles, data-units-written node scoping). - Drop the hard-coded node IP from the OCP row title and the baked-in node selection; clean dashboard title and null the id for clean import. --- resources/grafana/dashboard_SMART_OCP.json | 257 ++++++++++++++++----- 1 file changed, 193 insertions(+), 64 deletions(-) diff --git a/resources/grafana/dashboard_SMART_OCP.json b/resources/grafana/dashboard_SMART_OCP.json index e78e0df..9cea22e 100644 --- a/resources/grafana/dashboard_SMART_OCP.json +++ b/resources/grafana/dashboard_SMART_OCP.json @@ -1,4 +1,5 @@ - "annotations": { +{ + "annotations": { "list": [ { "builtIn": 1, @@ -24,7 +25,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 9, + "id": null, "links": [ { "asDropdown": false, @@ -658,19 +659,6 @@ "legendFormat": "{{device}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "nvme_num", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" } ], "title": "Number of Error Information Log Entries Over Time", @@ -1035,7 +1023,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Per-device hardware inventory (model, serial, firmware, generic path) with the physical capacity reported by the exporter info metrics.", "fieldConfig": { "defaults": { "color": { @@ -1049,27 +1037,7 @@ }, "inspect": false }, - "mappings": [ - { - "options": { - "live": { - "color": "green", - "index": 0 - } - }, - "type": "value" - }, - { - "options": { - "pattern": "!live", - "result": { - "color": "red", - "index": 1 - } - }, - "type": "regex" - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -1078,7 +1046,7 @@ } ] }, - "unit": "none" + "unit": "bytes" }, "overrides": [] }, @@ -1113,7 +1081,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "node_nvme_info", + "expr": "nvme_physical_size{instance=~\"[[node]]\",device=~\"[[device]]\"}", "format": "table", "fullMetaSearch": false, "includeNullMetadata": true, @@ -1136,21 +1104,27 @@ "aggregations": [], "operation": "groupby" }, - "firmware_revision": { + "model_number": { "aggregations": [], "operation": "groupby" }, - "model": { + "serial_number": { "aggregations": [], "operation": "groupby" }, - "serial": { + "firmware": { "aggregations": [], "operation": "groupby" }, - "state": { + "generic_path": { "aggregations": [], "operation": "groupby" + }, + "Value": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" } } } @@ -1501,7 +1475,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "NVME metrics are collected via \"text file Node exporter\", this metric reports how long ago the letrics have been collected by the local cronjob.", + "description": "Contains the number of unsafe shutdowns. This count is incremented when a Clean Shutdown Notification is not received prior to loss of power.", "fieldConfig": { "defaults": { "color": { @@ -2401,7 +2375,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "nvme_data_units_written{device=~\"[[device]]\"}", + "expr": "nvme_data_units_written{instance=~\"[[node]]\",device=~\"[[device]]\"}", "instant": true, "legendFormat": "{{device}}", "range": false, @@ -2478,7 +2452,7 @@ "refId": "A" } ], - "title": "Host Read Commands per", + "title": "Host Read Commands", "transparent": true, "type": "stat" }, @@ -2487,7 +2461,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "NVME SMART metric data_units_read_total", + "description": "Number of SMART Host Write Commands completed by the controller (lifetime count).", "fieldConfig": { "defaults": { "color": { @@ -2541,14 +2515,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "nvme_host_write_commands{device=~\"[[device]]\"}", + "expr": "nvme_host_write_commands{instance=~\"[[node]]\",device=~\"[[device]]\"}", "instant": true, "legendFormat": "{{device}}", "range": false, "refId": "A" } ], - "title": "Host Write Commands per Second for $node", + "title": "Host Write Commands", "transparent": true, "type": "stat" } @@ -2701,7 +2675,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Calculated as (1 - Hostwrites/TBW) following SNIA specifications on Endurance.\nThis visualization shows the following 3 thresholds:\nFrom 100 -> 50 = The device can be considered healthy (blue)\nFrom 50 - 25 -> The disk is Worn and performance might already be degraded (yellow)\nBelow 25 -> The disk is potentially ready for EOL", + "description": "Device Remaining Life % = 100 - nvme_percent_used, the vendor's own estimate of consumed endurance (already accounts for real cell endurance and write amplification). Thresholds: 100->50 healthy (blue), 50->25 worn (yellow), below 25 near EOL (pink).", "fieldConfig": { "defaults": { "color": { @@ -2817,15 +2791,21 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null }, { - "color": "red", - "value": 80 + "color": "#EAB839", + "value": 30 + }, + { + "color": "green", + "value": 90 } ] - } + }, + "unit": "d", + "min": 0 }, "overrides": [ { @@ -2882,22 +2862,23 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "device:projected_DRL_8h", + "expr": "device:DRL_days_remaining{device=~\"[[device]]\"}", "instant": false, "legendFormat": "{{device}}", "range": true, "refId": "A" } ], - "title": "Estimated DRL in 8 hours", - "type": "timeseries" + "title": "Estimated days until End-of-Life (at current write rate)", + "type": "timeseries", + "description": "Estimated days until the vendor wear estimate (nvme_percent_used) reaches 100%, at the current write rate. The wear rate comes from the continuous data-units counter; total endurance is calibrated from percent_used (no fixed P/E assumption). Lower is worse: red below 30 days, yellow below 90, green above." }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "DWPD is calculated as TBW/(365 X 5 X disk_capacity)\nfollowing SNIA specifications on Endurance.\nThe blue threshold indicates a DWPD compatible with the value provided by the vendor. Orange indicates a value above the recommended DWPD. In this latter case, the disk might meet its EOL sooner than expected.", + "description": "Rated DWPD = implied_TBW / (5 years x capacity). implied_TBW is derived from the vendor's nvme_percent_used wear estimate (no fixed P/E assumption). Blue = within the vendor-rated DWPD; orange (>1) = above it, so the disk may reach EOL sooner than expected. Adjust the 5-year window in the recording rule to match the drive warranty.", "fieldConfig": { "defaults": { "color": { @@ -2969,9 +2950,159 @@ ], "title": "DWPD (Given Current Usage) ", "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Observed drive-writes-per-day over the last 24h (host bytes written per day / capacity). Reflects the actual recent workload, independent of any endurance assumption.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 78, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "device:DWPD_observed{device=~\"[[device]]\"}", + "format": "time_series", + "instant": true, + "legendFormat": "{{device}}", + "range": false, + "refId": "A" + } + ], + "title": "DWPD (Observed, last 24h)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Lifetime-average drive-writes-per-day = total host writes / capacity / powered-on days. NOTE: based on power-on hours, not calendar time, so a frequently-powered-off drive reads high.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 79, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "device:DWPD_lifetime{device=~\"[[device]]\"}", + "format": "time_series", + "instant": true, + "legendFormat": "{{device}}", + "range": false, + "refId": "A" + } + ], + "title": "DWPD (Lifetime average)", + "type": "stat" } ], - "title": "Node 100.127.205.166:9188 OCP Metrics", + "title": "Disk Endurance & Efficiency (OCP)", "type": "row" } ], @@ -2997,11 +3128,9 @@ }, { "current": { - "text": [ - "100.127.205.166:9188" - ], + "text": "All", "value": [ - "100.127.205.166:9188" + "$__all" ] }, "datasource": { @@ -3067,7 +3196,7 @@ ] }, "timezone": "", - "title": "NVME_OCP_Dashboard Copy", + "title": "NVMe SMART & OCP Dashboard", "uid": "dee59s62kbchsb", "version": 29, "weekStart": ""