diff --git a/cmd/http.go b/cmd/http.go index db6c34b5..3fe739b9 100644 --- a/cmd/http.go +++ b/cmd/http.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "log" "net/http" "os" "strings" @@ -85,17 +86,17 @@ var httpCmd = &cobra.Command{ case "aws": solver, err = aws.NewAwsPositionEstimateClient(ctx, logger.Logger) if err != nil { - logger.Logger.Error("error while creating AWS position estimate client", zap.Error(err)) + logger.Logger.Error("error while creating AWS position estimate client", zap.String("category", "client_init_failed")) os.Exit(1) } case "loracloud": if LoracloudAccessToken == "" { - logger.Logger.Error("loracloud access token is required for loracloud solver") + logger.Logger.Error("loracloud access token is required for loracloud solver", zap.String("category", "config_invalid")) os.Exit(1) } solver, err = loracloud.NewLoracloudClient(ctx, LoracloudAccessToken, logger.Logger) if err != nil { - logger.Logger.Error("error while creating LoRa Cloud position estimate client", zap.Error(err)) + logger.Logger.Error("error while creating LoRa Cloud position estimate client", zap.String("category", "client_init_failed")) os.Exit(1) } } @@ -133,18 +134,23 @@ var httpCmd = &cobra.Command{ // middleware handler := loggingMiddleware(logger.Logger, router) - logger.Logger.Info("starting HTTP server", zap.String("host", host), zap.Uint64("port", uint64(port))) - err = http.ListenAndServe(fmt.Sprintf("%v:%v", host, port), handler) + logger.Logger.Info("starting HTTP server", zap.Uint64("port", uint64(port))) + server := &http.Server{ + Addr: fmt.Sprintf("%v:%v", host, port), + Handler: handler, + ErrorLog: log.New(safeHTTPErrorLog{}, "", 0), + } + err = server.ListenAndServe() if err != nil { - logger.Logger.Error("error while starting HTTP server", zap.Error(err)) + logger.Logger.Error("error while starting HTTP server", zap.String("category", "server_start_failed")) os.Exit(1) } }, } func addDecoder(ctx context.Context, router *http.ServeMux, path string, decoder decoder.Decoder) { - logger.Logger.Debug("adding decoder", zap.String("path", path)) + logger.Logger.Debug("adding decoder") router.HandleFunc("POST /"+path, getHandler(ctx, decoder)) } @@ -162,7 +168,7 @@ func getHandler(ctx context.Context, targetDecoder decoder.Decoder) func(http.Re logger.Logger.Debug("decoding request") err := json.NewDecoder(r.Body).Decode(&req) if err != nil { - logger.Logger.Error("error while decoding request", zap.Error(err)) + logger.Logger.Error("error while decoding request", zap.String("category", "request_decode_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": err.Error(), @@ -172,7 +178,7 @@ func getHandler(ctx context.Context, targetDecoder decoder.Decoder) func(http.Re } if err := validator.New().Struct(req); err != nil { - logger.Logger.Error("request validation failed", zap.Error(err)) + logger.Logger.Error("request validation failed", zap.String("category", "request_validation_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": "request validation failed", "docs": "https://docs.truvami.com", @@ -180,29 +186,22 @@ func getHandler(ctx context.Context, targetDecoder decoder.Decoder) func(http.Re return } - logger.Logger.Debug("set context values", - zap.String("devEui", req.DevEUI), - zap.Uint8("port", req.Port), - zap.String("payload", req.Payload), - ) ctx = context.WithValue(ctx, decoder.DEVEUI_CONTEXT_KEY, req.DevEUI) ctx = context.WithValue(ctx, decoder.PORT_CONTEXT_KEY, req.Port) ctx = context.WithValue(ctx, decoder.FCNT_CONTEXT_KEY, 1) // Default frame count, can be adjusted as needed - logger.Logger.Debug("decoding payload") - var warnings []string = nil data, err := targetDecoder.Decode(ctx, req.Payload, req.Port) if err != nil { if errors.Is(err, helpers.ErrValidationFailed) { warnings = []string{} for _, err := range helpers.UnwrapError(err) { - logger.Logger.Warn("validation error", zap.Error(err), zap.String("devEui", req.DevEUI), zap.Uint8("port", req.Port)) + logger.Logger.Warn("validation error", zap.String("category", "validation_failed")) warnings = append(warnings, err.Error()) } logger.Logger.Warn("validation for some fields failed - are you using the correct port?") } else { - logger.Logger.Error("error while decoding payload", zap.Error(err), zap.String("devEui", req.DevEUI), zap.Uint8("port", req.Port)) + logger.Logger.Error("error while decoding payload", zap.String("category", "payload_decode_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": err.Error(), @@ -212,7 +211,6 @@ func getHandler(ctx context.Context, targetDecoder decoder.Decoder) func(http.Re } } - logger.Logger.Info("payload decoded successfully", zap.String("devEui", req.DevEUI), zap.Uint8("port", req.Port)) setBody(w, http.StatusOK, map[string]any{ "data": data.Data, "warnings": warnings, @@ -221,7 +219,7 @@ func getHandler(ctx context.Context, targetDecoder decoder.Decoder) func(http.Re } func addEncoder(router *http.ServeMux, path string, encoder encoder.Encoder) { - logger.Logger.Debug("adding encoder", zap.String("path", path)) + logger.Logger.Debug("adding encoder") router.HandleFunc("POST /"+path, getEncoderHandler(encoder)) } @@ -237,7 +235,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. logger.Logger.Debug("decoding request") err := json.NewDecoder(r.Body).Decode(&rawReq) if err != nil { - logger.Logger.Error("error while decoding request", zap.Error(err)) + logger.Logger.Error("error while decoding request", zap.String("category", "request_decode_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": err.Error(), @@ -247,7 +245,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. } if err := validator.New().Struct(rawReq); err != nil { - logger.Logger.Error("request validation failed", zap.Error(err)) + logger.Logger.Error("request validation failed", zap.String("category", "request_validation_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": "request validation failed", "docs": "https://docs.truvami.com", @@ -267,7 +265,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. case 128: var payload smartlabelEncoder.Port128Payload if err := json.Unmarshal(rawReq.Payload, &payload); err != nil { - logger.Logger.Error("error unmarshaling payload", zap.Error(err)) + logger.Logger.Error("error unmarshaling payload", zap.String("category", "payload_unmarshal_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": fmt.Sprintf("Error unmarshaling payload: %v", err), "docs": "https://docs.truvami.com", @@ -276,7 +274,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. } structPayload = payload default: - logger.Logger.Error("unsupported port", zap.Uint8("port", rawReq.Port)) + logger.Logger.Error("unsupported port", zap.String("category", "unsupported_port")) setBody(w, http.StatusBadRequest, map[string]any{ "error": fmt.Sprintf("Unsupported port: %d", rawReq.Port), "docs": "https://docs.truvami.com", @@ -288,7 +286,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. case 128: var payload tagslEncoder.Port128Payload if err := json.Unmarshal(rawReq.Payload, &payload); err != nil { - logger.Logger.Error("error unmarshaling payload", zap.Error(err)) + logger.Logger.Error("error unmarshaling payload", zap.String("category", "payload_unmarshal_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": fmt.Sprintf("Error unmarshaling payload: %v", err), "docs": "https://docs.truvami.com/docs/payloads/tag%20S/v3.2.0/", @@ -299,7 +297,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. case 129: var payload tagslEncoder.Port129Payload if err := json.Unmarshal(rawReq.Payload, &payload); err != nil { - logger.Logger.Error("error unmarshaling payload", zap.Error(err)) + logger.Logger.Error("error unmarshaling payload", zap.String("category", "payload_unmarshal_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": fmt.Sprintf("Error unmarshaling payload: %v", err), "docs": "https://docs.truvami.com/docs/payloads/tag%20S/v3.2.0/", @@ -310,7 +308,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. case 131: var payload tagslEncoder.Port131Payload if err := json.Unmarshal(rawReq.Payload, &payload); err != nil { - logger.Logger.Error("error unmarshaling payload", zap.Error(err)) + logger.Logger.Error("error unmarshaling payload", zap.String("category", "payload_unmarshal_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": fmt.Sprintf("Error unmarshaling payload: %v", err), "docs": "https://docs.truvami.com/docs/payloads/tag%20S/v3.2.0/", @@ -321,7 +319,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. case 134: var payload tagslEncoder.Port134Payload if err := json.Unmarshal(rawReq.Payload, &payload); err != nil { - logger.Logger.Error("error unmarshaling payload", zap.Error(err)) + logger.Logger.Error("error unmarshaling payload", zap.String("category", "payload_unmarshal_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": fmt.Sprintf("Error unmarshaling payload: %v", err), "docs": "https://docs.truvami.com/docs/payloads/tag%20S/v3.2.0/", @@ -330,7 +328,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. } structPayload = payload default: - logger.Logger.Error("unsupported port", zap.Uint8("port", rawReq.Port)) + logger.Logger.Error("unsupported port", zap.String("category", "unsupported_port")) setBody(w, http.StatusBadRequest, map[string]any{ "error": fmt.Sprintf("Unsupported port: %d", rawReq.Port), "docs": "https://docs.truvami.com/docs/payloads/tag%20S/v3.2.0/", @@ -339,7 +337,7 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. } default: // For other device types, you would add similar switch statements - logger.Logger.Error("unsupported device type", zap.String("path", r.URL.Path)) + logger.Logger.Error("unsupported device type", zap.String("category", "unsupported_device")) setBody(w, http.StatusBadRequest, map[string]any{ "error": "Unsupported device type", "docs": "https://docs.truvami.com", @@ -347,20 +345,18 @@ func getEncoderHandler(encoder encoder.Encoder) func(http.ResponseWriter, *http. return } - logger.Logger.Debug("encoding payload", zap.Any("payload", structPayload), zap.Uint8("port", rawReq.Port)) - var warnings []string = nil encoded, err := encoder.Encode(structPayload, rawReq.Port) if err != nil { if errors.Is(err, helpers.ErrValidationFailed) { warnings = []string{} for _, err := range helpers.UnwrapError(err) { - logger.Logger.Warn("validation error", zap.Error(err)) + logger.Logger.Warn("validation error", zap.String("category", "validation_failed")) warnings = append(warnings, err.Error()) } logger.Logger.Warn("validation for some fields failed - are you using the correct port?") } else { - logger.Logger.Error("error while encoding payload", zap.Error(err)) + logger.Logger.Error("error while encoding payload", zap.String("category", "payload_encode_failed")) setBody(w, http.StatusBadRequest, map[string]any{ "error": err.Error(), @@ -394,20 +390,18 @@ func setHeaders(w http.ResponseWriter, status int) { } func setBody(w http.ResponseWriter, status int, body map[string]any) { - logger.Logger.Debug("encoding response") - // add traceId traceId := uuid.New().String() body["traceId"] = traceId data, err := json.Marshal(body) if err != nil { - logger.Logger.Error("error while encoding response", zap.Error(err)) + logger.Logger.Error("error while encoding response", zap.String("category", "response_encode_failed")) setHeaders(w, http.StatusInternalServerError) - _, err = w.Write([]byte(err.Error())) + _, err = w.Write([]byte("internal server error")) if err != nil { - logger.Logger.Error("error while sending response", zap.Error(err)) + logger.Logger.Error("error while sending response", zap.String("category", "response_send_failed")) } return } @@ -415,45 +409,40 @@ func setBody(w http.ResponseWriter, status int, body map[string]any) { setHeaders(w, status) _, err = w.Write(data) if err != nil { - logger.Logger.Error("error while sending response", zap.Error(err)) + logger.Logger.Error("error while sending response", zap.String("category", "response_send_failed")) return } - - logger.Logger.Debug("response sent", zap.Any("response", string(data))) } func healthHandler(w http.ResponseWriter, r *http.Request) { setHeaders(w, http.StatusOK) _, err := w.Write([]byte("OK")) if err != nil { - logger.Logger.Error("error while sending response", zap.Error(err)) + logger.Logger.Error("error while sending response", zap.String("category", "response_send_failed")) } } +type safeHTTPErrorLog struct{} + +func (safeHTTPErrorLog) Write(p []byte) (int, error) { + if logger.Logger != nil { + logger.Logger.Error("http server error", zap.String("category", "internal")) + } + return len(p), nil +} + func loggingMiddleware(logger *zap.Logger, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // generate a unique request ID - requestID := uuid.New().String() - w.Header().Set("X-Request-ID", requestID) - - // start timer start := time.Now() - // use a ResponseWriter wrapper to capture the status code rw := &responseWriter{ResponseWriter: w, statusCode: http.StatusOK} - // process the request next.ServeHTTP(rw, r) - // log the details logger.Info("HTTP request", - zap.String("requestId", requestID), zap.String("method", r.Method), - zap.String("url", r.URL.String()), zap.Int("status", rw.statusCode), - zap.String("remoteAddress", r.RemoteAddr), - zap.String("userAgent", r.UserAgent()), - zap.Duration("latency", time.Since(start)), + zap.Duration("duration", time.Since(start)), ) }) } diff --git a/cmd/http_privacy_test.go b/cmd/http_privacy_test.go new file mode 100644 index 00000000..61af7122 --- /dev/null +++ b/cmd/http_privacy_test.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/truvami/decoder/internal/logger" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestLoggingMiddlewareEmitsOnlySafeAccessFields(t *testing.T) { + observed, logs := observer.New(zapcore.InfoLevel) + safeLogger := zap.New(logger.WrapCore(observed)) + + handler := loggingMiddleware(safeLogger, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + req := httptest.NewRequest(http.MethodPost, "/tagsl/v1?devEui=0123456789ABCDEF&payload=deadbeef", nil) + req.RemoteAddr = "203.0.113.10:1234" + req.Header.Set("User-Agent", "secret-agent/1.0") + rec := httptest.NewRecorder() + + handler.ServeHTTP(rec, req) + + if logs.Len() != 1 { + t.Fatalf("expected 1 access log entry, got %d", logs.Len()) + } + + entry := logs.All()[0] + if entry.Message != "HTTP request" { + t.Fatalf("unexpected log message %q", entry.Message) + } + + fields := entry.ContextMap() + for _, forbidden := range []string{"devEui", "payload", "url", "route", "requestId", "remoteAddress", "userAgent", "response"} { + if _, ok := fields[forbidden]; ok { + t.Fatalf("forbidden field %q present in access log", forbidden) + } + } + + for _, required := range []string{"method", "status", "duration"} { + if _, ok := fields[required]; !ok { + t.Fatalf("required field %q missing from access log", required) + } + } + + if fields["method"] != http.MethodPost { + t.Fatalf("expected method POST, got %v", fields["method"]) + } +} + +func TestSafeHTTPErrorLogDoesNotEmitRawErrorText(t *testing.T) { + observed, logs := observer.New(zapcore.ErrorLevel) + logger.Logger = zap.New(logger.WrapCore(observed)) + + writer := safeHTTPErrorLog{} + raw := []byte("listen tcp: lookup evil.example: no such host") + if _, err := writer.Write(raw); err != nil { + t.Fatalf("write failed: %v", err) + } + + if logs.Len() != 1 { + t.Fatalf("expected 1 log entry, got %d", logs.Len()) + } + + entry := logs.All()[0] + if entry.Message != "http server error" { + t.Fatalf("unexpected message %q", entry.Message) + } + if entry.ContextMap()["category"] != "internal" { + t.Fatalf("expected category=internal, got %v", entry.ContextMap()["category"]) + } + for _, field := range entry.Context { + if field.Key == "error" || field.String == string(raw) { + t.Fatal("raw server error text must not appear in logs") + } + } +} + +func TestPrintJSONWritesToStdoutNotLogger(t *testing.T) { + observed, logs := observer.New(zapcore.InfoLevel) + logger.Logger = zap.New(logger.WrapCore(observed)) + + originalJSON := Json + Json = true + defer func() { Json = originalJSON }() + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + + printJSON(map[string]string{"latitude": "47.0"}) + + _ = w.Close() + os.Stdout = oldStdout + + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("read stdout: %v", err) + } + _ = r.Close() + + if logs.Len() != 0 { + t.Fatalf("expected no zap output for JSON CLI mode, got %d entries", logs.Len()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"latitude":"47.0"`)) && !bytes.Contains(buf.Bytes(), []byte(`"latitude": "47.0"`)) { + t.Fatalf("expected JSON on stdout, got %q", buf.String()) + } +} diff --git a/cmd/root.go b/cmd/root.go index d93a691b..6c3f52d1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,7 +13,6 @@ import ( "github.com/truvami/decoder/internal/logger" "github.com/truvami/decoder/internal/selfupdate" "go.uber.org/zap" - "go.uber.org/zap/zapcore" ) var banner = []string{ @@ -64,9 +63,11 @@ func init() { } var rootCmd = &cobra.Command{ - Use: "decoder", - Short: "truvami payload decoder cli helper", - Version: Version, + Use: "decoder", + Short: "truvami payload decoder cli helper", + Version: Version, + SilenceErrors: true, + SilenceUsage: true, Long: getBanner() + ` A CLI tool to help decode @truvami payloads.`, @@ -77,24 +78,6 @@ A CLI tool to help decode @truvami payloads.`, options = append(options, logger.WithDebug()) } - if Json { - // create a custom encoder - encoderConfig := zapcore.EncoderConfig{ - TimeKey: "time", - LevelKey: "level", - NameKey: "logger", - CallerKey: "caller", - MessageKey: "msg", - StacktraceKey: "", // disable stack traces - LineEnding: zapcore.DefaultLineEnding, - EncodeLevel: zapcore.CapitalLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, - EncodeDuration: zapcore.StringDurationEncoder, - } - - options = append(options, logger.WithEncoder(zapcore.NewJSONEncoder(encoderConfig))) - } - logger.NewLogger(options...) // Non-blocking update check (ignore network errors). @@ -130,24 +113,24 @@ func Execute() { } func printJSON(data any) { + payload := map[string]any{"data": data} + if Json { - logger.Logger.Info("successfully decoded payload", zap.Any("data", data)) + marshaled, err := json.Marshal(payload) + if err != nil { + fmt.Fprintln(os.Stderr, "marshaling error") + os.Exit(1) + } + fmt.Println(string(marshaled)) return } - logger.Logger.Info("successfully decoded payload") - - // print data beautifully and formatted - marshaled, err := json.MarshalIndent(map[string]any{ - "data": data, - }, "", " ") - - // handle marshaling error + marshaled, err := json.MarshalIndent(payload, "", " ") if err != nil { - logger.Logger.Fatal("marshaling error", zap.Error(err)) + fmt.Fprintln(os.Stderr, "marshaling error") + os.Exit(1) } - // print the marshaled data fmt.Println() fmt.Println(string(marshaled)) fmt.Println() diff --git a/examples/advanced_tag_s_l/main.go b/examples/advanced_tag_s_l/main.go index 1d08554b..a73fe102 100644 --- a/examples/advanced_tag_s_l/main.go +++ b/examples/advanced_tag_s_l/main.go @@ -2,36 +2,32 @@ package main import ( "context" - "log" + "fmt" "github.com/truvami/decoder/pkg/decoder" "github.com/truvami/decoder/pkg/decoder/tagsl/v1" ) func main() { - log.Println("initializing tag S / L decoder...") + fmt.Println("initializing tag S / L decoder...") d := tagsl.NewTagSLv1Decoder() - // decode data - log.Println("decoding data...") + fmt.Println("decoding data...") data, err := d.Decode(context.Background(), "0002c420ff005ed85a12b4180719142607", 1) if err != nil { panic(err) } - // check if decoded payload has the GNSS feature if !data.Is(decoder.FeatureGNSS) { panic("decoded payload does not have GNSS feature") } - // cast to GNSS data gnssData, ok := data.Data.(decoder.UplinkFeatureGNSS) if !ok { panic("failed to cast to GNSS data") } - // print GNSS data - log.Printf("Latitude: %f\n", gnssData.GetLatitude()) - log.Printf("Longitude: %f\n", gnssData.GetLongitude()) - log.Printf("Altitude: %f\n", gnssData.GetAltitude()) + fmt.Printf("Latitude: %f\n", gnssData.GetLatitude()) + fmt.Printf("Longitude: %f\n", gnssData.GetLongitude()) + fmt.Printf("Altitude: %f\n", gnssData.GetAltitude()) } diff --git a/examples/advanced_tag_s_l/main_test.go b/examples/advanced_tag_s_l/main_test.go index 11d18788..1df6b703 100644 --- a/examples/advanced_tag_s_l/main_test.go +++ b/examples/advanced_tag_s_l/main_test.go @@ -2,22 +2,40 @@ package main import ( "bytes" - "log" + "io" + "os" "strings" "testing" ) -func TestMain(t *testing.T) { - // Create a buffer to capture the output +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + + fn() + + _ = w.Close() + os.Stdout = oldStdout + var buf bytes.Buffer - log.SetOutput(&buf) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("read stdout: %v", err) + } + _ = r.Close() - // Run the main function - main() + return buf.String() +} + +func TestMain(t *testing.T) { + out := captureStdout(t, main) - // Check if the expected output is present in the buffer - expectedOutput := `46.407935` - if !strings.Contains(buf.String(), expectedOutput) { - t.Errorf("expected output %q not found", expectedOutput) + if !strings.Contains(out, "46.407935") { + t.Errorf("expected output %q not found in %q", "46.407935", out) } } diff --git a/examples/basic_nomadxl/main.go b/examples/basic_nomadxl/main.go index 6dbe0794..729a40b4 100644 --- a/examples/basic_nomadxl/main.go +++ b/examples/basic_nomadxl/main.go @@ -3,28 +3,25 @@ package main import ( "context" "encoding/json" - "log" + "fmt" "github.com/truvami/decoder/pkg/decoder/nomadxl/v1" ) func main() { - log.Println("initializing nomad XL decoder...") + fmt.Println("initializing nomad XL decoder...") d := nomadxl.NewNomadXLv1Decoder() - // decode data - log.Println("decoding data...") + fmt.Println("decoding data...") data, err := d.Decode(context.Background(), "0000793000020152004b6076000c838c00003994", 103) if err != nil { panic(err) } - // data to json j, err := json.Marshal(data) if err != nil { panic(err) } - // print json - log.Printf("result: %s\n", j) + fmt.Println(string(j)) } diff --git a/examples/basic_nomadxl/main_test.go b/examples/basic_nomadxl/main_test.go index c2ab1f96..6d07500f 100644 --- a/examples/basic_nomadxl/main_test.go +++ b/examples/basic_nomadxl/main_test.go @@ -2,22 +2,40 @@ package main import ( "bytes" - "log" + "io" + "os" "strings" "testing" ) -func TestMain(t *testing.T) { - // Create a buffer to capture the output +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + + fn() + + _ = w.Close() + os.Stdout = oldStdout + var buf bytes.Buffer - log.SetOutput(&buf) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("read stdout: %v", err) + } + _ = r.Close() - // Run the main function - main() + return buf.String() +} + +func TestMain(t *testing.T) { + out := captureStdout(t, main) - // Check if the expected output is present in the buffer - expectedOutput := `49.39894` - if !strings.Contains(buf.String(), expectedOutput) { - t.Errorf("expected output %q not found", expectedOutput) + if !strings.Contains(out, "49.39894") { + t.Errorf("expected output %q not found in %q", "49.39894", out) } } diff --git a/examples/basic_nomadxs/main.go b/examples/basic_nomadxs/main.go index 8acbcc3c..16c987df 100644 --- a/examples/basic_nomadxs/main.go +++ b/examples/basic_nomadxs/main.go @@ -3,28 +3,25 @@ package main import ( "context" "encoding/json" - "log" + "fmt" "github.com/truvami/decoder/pkg/decoder/nomadxs/v1" ) func main() { - log.Println("initializing nomad XS decoder...") + fmt.Println("initializing nomad XS decoder...") d := nomadxs.NewNomadXSv1Decoder() - // decode data - log.Println("decoding data...") + fmt.Println("decoding data...") data, err := d.Decode(context.Background(), "0002c420ff005ed85a12b4180719142607240001ffbaffc2fc6f00d71d2e", 1) if err != nil { panic(err) } - // data to json j, err := json.Marshal(data) if err != nil { panic(err) } - // print json - log.Printf("result: %s\n", j) + fmt.Println(string(j)) } diff --git a/examples/basic_nomadxs/main_test.go b/examples/basic_nomadxs/main_test.go index 11d18788..1df6b703 100644 --- a/examples/basic_nomadxs/main_test.go +++ b/examples/basic_nomadxs/main_test.go @@ -2,22 +2,40 @@ package main import ( "bytes" - "log" + "io" + "os" "strings" "testing" ) -func TestMain(t *testing.T) { - // Create a buffer to capture the output +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + + fn() + + _ = w.Close() + os.Stdout = oldStdout + var buf bytes.Buffer - log.SetOutput(&buf) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("read stdout: %v", err) + } + _ = r.Close() - // Run the main function - main() + return buf.String() +} + +func TestMain(t *testing.T) { + out := captureStdout(t, main) - // Check if the expected output is present in the buffer - expectedOutput := `46.407935` - if !strings.Contains(buf.String(), expectedOutput) { - t.Errorf("expected output %q not found", expectedOutput) + if !strings.Contains(out, "46.407935") { + t.Errorf("expected output %q not found in %q", "46.407935", out) } } diff --git a/examples/basic_tag_s_l/main.go b/examples/basic_tag_s_l/main.go index c87a8c3d..d5363cff 100644 --- a/examples/basic_tag_s_l/main.go +++ b/examples/basic_tag_s_l/main.go @@ -3,28 +3,25 @@ package main import ( "context" "encoding/json" - "log" + "fmt" "github.com/truvami/decoder/pkg/decoder/tagsl/v1" ) func main() { - log.Println("initializing tag S / L decoder...") + fmt.Println("initializing tag S / L decoder...") d := tagsl.NewTagSLv1Decoder() - // decode data - log.Println("decoding data...") + fmt.Println("decoding data...") data, err := d.Decode(context.Background(), "0002c420ff005ed85a12b4180719142607", 1) if err != nil { panic(err) } - // data to json j, err := json.Marshal(data) if err != nil { panic(err) } - // print json - log.Printf("result: %s\n", j) + fmt.Println(string(j)) } diff --git a/examples/basic_tag_s_l/main_test.go b/examples/basic_tag_s_l/main_test.go index 11d18788..1df6b703 100644 --- a/examples/basic_tag_s_l/main_test.go +++ b/examples/basic_tag_s_l/main_test.go @@ -2,22 +2,40 @@ package main import ( "bytes" - "log" + "io" + "os" "strings" "testing" ) -func TestMain(t *testing.T) { - // Create a buffer to capture the output +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + + fn() + + _ = w.Close() + os.Stdout = oldStdout + var buf bytes.Buffer - log.SetOutput(&buf) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("read stdout: %v", err) + } + _ = r.Close() - // Run the main function - main() + return buf.String() +} + +func TestMain(t *testing.T) { + out := captureStdout(t, main) - // Check if the expected output is present in the buffer - expectedOutput := `46.407935` - if !strings.Contains(buf.String(), expectedOutput) { - t.Errorf("expected output %q not found", expectedOutput) + if !strings.Contains(out, "46.407935") { + t.Errorf("expected output %q not found in %q", "46.407935", out) } } diff --git a/examples/basic_tagxl/main.go b/examples/basic_tagxl/main.go index 2d3f5ff0..ba7f29c3 100644 --- a/examples/basic_tagxl/main.go +++ b/examples/basic_tagxl/main.go @@ -3,7 +3,7 @@ package main import ( "context" "encoding/json" - "log" + "fmt" "github.com/truvami/decoder/pkg/decoder/tagxl/v1" "github.com/truvami/decoder/pkg/solver/aws" @@ -26,19 +26,16 @@ func main() { logger.Info("initializing tag XL decoder...") d := tagxl.NewTagXLv1Decoder(ctx, solver, logger) - // decode data logger.Info("decoding data...") data, err := d.Decode(ctx, "05ab859590e78d0cc1805a9428b2de73d80cc9c9a3329a01a5e3cba3546b7454395747a1cd6effd2fdeebefe8fac39a60e", 192) if err != nil { panic(err) } - // data to json j, err := json.Marshal(data) if err != nil { panic(err) } - // print json - log.Printf("result: %s\n", j) + fmt.Println(string(j)) } diff --git a/examples/basic_tagxl/main_test.go b/examples/basic_tagxl/main_test.go index 34dc61c6..9a18d398 100644 --- a/examples/basic_tagxl/main_test.go +++ b/examples/basic_tagxl/main_test.go @@ -2,29 +2,46 @@ package main import ( "bytes" - "log" + "io" + "os" "strings" "testing" ) -func TestMain(t *testing.T) { - // Create a buffer to capture the output +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + + fn() + + _ = w.Close() + os.Stdout = oldStdout + var buf bytes.Buffer - log.SetOutput(&buf) + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("read stdout: %v", err) + } + _ = r.Close() - // main() panics on AWS API errors; recover and skip when that happens. + return buf.String() +} + +func TestMain(t *testing.T) { defer func() { if r := recover(); r != nil { t.Skipf("skipping: main() panicked (likely AWS API unavailable): %v", r) } }() - // Run the main function - main() + out := captureStdout(t, main) - // Check if the expected output is present in the buffer - expectedOutput := `longitude` - if !strings.Contains(buf.String(), expectedOutput) { - t.Errorf("expected output %q not found", expectedOutput) + if !strings.Contains(out, "longitude") { + t.Errorf("expected output %q not found in %q", "longitude", out) } } diff --git a/internal/logger/filter.go b/internal/logger/filter.go new file mode 100644 index 00000000..6dc33003 --- /dev/null +++ b/internal/logger/filter.go @@ -0,0 +1,89 @@ +package logger + +import "go.uber.org/zap/zapcore" + +var allowedFieldKeys = map[string]struct{}{ + "category": {}, + "current": {}, + "decoder": {}, + "duration": {}, + "from": {}, + "hint": {}, + "latency": {}, + "latest": {}, + "method": {}, + "note": {}, + "outcome": {}, + "port": {}, + "status": {}, + "threshold": {}, + "to": {}, + "type": {}, + "error_type": {}, +} + +type filteringCore struct { + zapcore.Core +} + +func newFilteringCore(core zapcore.Core) zapcore.Core { + return &filteringCore{Core: core} +} + +// WrapCore applies the fail-closed field allowlist to an existing zap core. +func WrapCore(core zapcore.Core) zapcore.Core { + return newFilteringCore(core) +} + +func (c *filteringCore) With(fields []zapcore.Field) zapcore.Core { + return &filteringCore{Core: c.Core.With(filterFields(fields))} +} + +func (c *filteringCore) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if !c.Enabled(ent.Level) { + return ce + } + return ce.AddCore(ent, c) +} + +func (c *filteringCore) Write(ent zapcore.Entry, fields []zapcore.Field) error { + return c.Core.Write(ent, filterFields(fields)) +} + +func filterFields(fields []zapcore.Field) []zapcore.Field { + if len(fields) == 0 { + return fields + } + filtered := make([]zapcore.Field, 0, len(fields)) + for _, field := range fields { + if allowedField(field) { + filtered = append(filtered, field) + } + } + return filtered +} + +func allowedField(field zapcore.Field) bool { + if _, ok := allowedFieldKeys[field.Key]; !ok { + return false + } + switch field.Type { //nolint:exhaustive // fail-closed allowlist accepts only primitive field types + case zapcore.BoolType, + zapcore.Float32Type, + zapcore.Float64Type, + zapcore.Int16Type, + zapcore.Int32Type, + zapcore.Int64Type, + zapcore.Int8Type, + zapcore.StringType, + zapcore.Uint16Type, + zapcore.Uint32Type, + zapcore.Uint64Type, + zapcore.Uint8Type, + zapcore.DurationType, + zapcore.TimeType: + return true + default: + return false + } +} diff --git a/internal/logger/filter_test.go b/internal/logger/filter_test.go new file mode 100644 index 00000000..1d502581 --- /dev/null +++ b/internal/logger/filter_test.go @@ -0,0 +1,87 @@ +package logger + +import ( + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestFilterFieldsStripsSensitiveKeys(t *testing.T) { + filtered := filterFields([]zapcore.Field{ + {Key: "devEui", Type: zapcore.StringType, String: "0123456789ABCDEF"}, + {Key: "payload", Type: zapcore.StringType, String: "deadbeef"}, + {Key: "route", Type: zapcore.StringType, String: "/tagsl/v1"}, + {Key: "method", Type: zapcore.StringType, String: "POST"}, + zap.Any("method", map[string]string{"secret": "payload"}), + }) + + if len(filtered) != 1 { + t.Fatalf("expected 1 allowed field, got %d", len(filtered)) + } + for _, field := range filtered { + if field.Key != "method" { + t.Fatalf("unexpected allowed field %q", field.Key) + } + } +} + +func TestFilteringCoreDoesNotEmitSensitiveFields(t *testing.T) { + observed, logs := observer.New(zapcore.InfoLevel) + Logger = zap.New(newFilteringCore(observed)) + + Logger.Info("HTTP request", + zap.String("route", "/tagsl/v1"), + zap.String("method", "POST"), + zap.Int("status", 200), + zap.String("devEui", "0123456789ABCDEF"), + zap.String("payload", "deadbeef"), + zap.String("url", "http://localhost/tagsl/v1?secret=1"), + ) + + if logs.Len() != 1 { + t.Fatalf("expected 1 log entry, got %d", logs.Len()) + } + + encoded := logs.All()[0] + for _, field := range encoded.Context { + if _, allowed := allowedFieldKeys[field.Key]; !allowed { + t.Fatalf("unexpected field key in log output: %q", field.Key) + } + } + + out := encoded.ContextMap() + if _, ok := out["devEui"]; ok { + t.Fatal("devEui must not appear in log output") + } + if _, ok := out["payload"]; ok { + t.Fatal("payload must not appear in log output") + } + if _, ok := out["url"]; ok { + t.Fatal("url must not appear in log output") + } + if _, ok := out["route"]; ok { + t.Fatal("route must not appear in log output") + } +} + +func TestFilteringCoreBlocksRawErrorField(t *testing.T) { + observed, logs := observer.New(zapcore.ErrorLevel) + Logger = zap.New(newFilteringCore(observed)) + + Logger.Error("decode failed", zap.Error(stringsErr("payload=deadbeef devEui=0123456789ABCDEF"))) + + if logs.Len() != 1 { + t.Fatalf("expected 1 log entry, got %d", logs.Len()) + } + for _, field := range logs.All()[0].Context { + if field.Key == "error" { + t.Fatal("raw error field must not appear in log output") + } + } +} + +type stringsErr string + +func (e stringsErr) Error() string { return string(e) } diff --git a/internal/logger/logger.go b/internal/logger/logger.go index bdf8e7df..77c33b85 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -41,11 +41,11 @@ func NewLogger(options ...Option) { opt(config) } - core := zapcore.NewCore( + core := newFilteringCore(zapcore.NewCore( config.Encoder, zapcore.AddSync(os.Stdout), config.Level, - ) + )) Logger = zap.New(core) } diff --git a/pkg/common/helpers.go b/pkg/common/helpers.go index cb159457..d2fe4b38 100644 --- a/pkg/common/helpers.go +++ b/pkg/common/helpers.go @@ -12,7 +12,6 @@ import ( "github.com/go-playground/validator" "github.com/truvami/decoder/internal/logger" - "go.uber.org/zap" ) func HexStringToBytes(hexString string) ([]byte, error) { @@ -94,7 +93,7 @@ func convertFieldValue(rawValue any, fieldType reflect.Type, transform func(v an var value any = nil var err error = nil - if fieldType.Kind() == reflect.Ptr { + if fieldType.Kind() == reflect.Pointer { ptr = true fieldType = fieldType.Elem() } @@ -208,10 +207,9 @@ func Decode(payloadHex *string, config *PayloadConfig) (any, error) { } } if !found { - tagHex := fmt.Sprintf("0x%02x", tag) - unknownTLVTagsTotal.WithLabelValues(tagHex).Inc() + unknownTLVTagsTotal.Inc() if logger.Logger != nil { - logger.Logger.Warn("skipping unknown tag", zap.String("tag", tagHex), zap.Int("length", length)) + logger.Logger.Warn("skipping unknown tag") } } index += length @@ -260,7 +258,7 @@ func insertFieldBytes(fieldValue reflect.Value, length int, transform func(v any var bytes []byte var err error = nil - if fieldValue.Kind() == reflect.Ptr { + if fieldValue.Kind() == reflect.Pointer { if fieldValue.IsNil() { null = true bytes = make([]byte, length) @@ -487,7 +485,7 @@ func TimePointerCompare(alpha *time.Time, bravo *time.Time) bool { } func DerefValue(value reflect.Value) any { - if value.Kind() == reflect.Ptr && !value.IsNil() { + if value.Kind() == reflect.Pointer && !value.IsNil() { return value.Elem().Interface() } diff --git a/pkg/common/metrics.go b/pkg/common/metrics.go index fee44256..fdca3040 100644 --- a/pkg/common/metrics.go +++ b/pkg/common/metrics.go @@ -6,8 +6,8 @@ import ( ) var ( - unknownTLVTagsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + unknownTLVTagsTotal = promauto.NewCounter(prometheus.CounterOpts{ Name: "truvami_common_unknown_tlv_tags_total", Help: "The total number of unknown TLV tags encountered during decoding", - }, []string{"tag"}) + }) ) diff --git a/pkg/common/metrics_privacy_test.go b/pkg/common/metrics_privacy_test.go new file mode 100644 index 00000000..280b5b5b --- /dev/null +++ b/pkg/common/metrics_privacy_test.go @@ -0,0 +1,24 @@ +package common + +import ( + "strings" + "testing" + + dto "github.com/prometheus/client_model/go" +) + +func TestUnknownTLVMetricHasNoSensitiveLabels(t *testing.T) { + metric := &dto.Metric{} + if err := unknownTLVTagsTotal.Write(metric); err != nil { + t.Fatalf("write metric: %v", err) + } + + for _, label := range metric.GetLabel() { + name := label.GetName() + if strings.Contains(strings.ToLower(name), "tag") || + strings.Contains(strings.ToLower(name), "payload") || + strings.Contains(strings.ToLower(name), "deveui") { + t.Fatalf("metric must not expose sensitive label %q", name) + } + } +} diff --git a/pkg/solver/aws/aws.go b/pkg/solver/aws/aws.go index 6cb61af7..1b8055e5 100644 --- a/pkg/solver/aws/aws.go +++ b/pkg/solver/aws/aws.go @@ -61,10 +61,6 @@ func (c PositionEstimateClient) Solve(ctx context.Context, payload string) (*dec start := time.Now() awsPositionEstimatesTotalCounter.Inc() - c.logger.Debug("Starting position estimate request", - zap.String("payload", payload), - ) - // remove first 2 characters from the payload if len(payload) > 2 { payload = payload[2:] @@ -93,15 +89,10 @@ func (c PositionEstimateClient) Solve(ctx context.Context, payload string) (*dec output, err := c.client.GetPositionEstimate(ctx, input) if err != nil { awsPositionEstimatesFailureCounter.Inc() + c.logger.Error("position estimate request failed", zap.String("category", "request_failed")) return nil, fmt.Errorf("failed to get position estimate: %w", err) } - c.logger.Debug("Position estimate received", - zap.String("payload", payload), - zap.ByteString("geoJson", output.GeoJsonPayload), - zap.Any("metadata", output.ResultMetadata), - ) - var position *GeoJsonResponse err = json.Unmarshal(output.GeoJsonPayload, &position) if err != nil { diff --git a/pkg/solver/loracloud/loracloud.go b/pkg/solver/loracloud/loracloud.go index 47084def..a8dabbfa 100644 --- a/pkg/solver/loracloud/loracloud.go +++ b/pkg/solver/loracloud/loracloud.go @@ -39,7 +39,7 @@ func (m LoracloudClient) isSemtechLoRaCloudShutdown() error { if m.timeNow().After(time.Date(2025, 7, 31, 0, 0, 0, 0, time.UTC)) { return ErrSemtechLoRaCloudShutdown } - m.logger.Warn("LoRa Cloud is Sunsetting on 31.07.2025", zap.String("url", "https://www.semtech.com/loracloud-shutdown")) + m.logger.Warn("LoRa Cloud is Sunsetting on 31.07.2025") return nil } @@ -321,24 +321,22 @@ func (m LoracloudClient) DeliverUplinkMessage(devEui string, uplinkMsg UplinkMsg // We sent a position with the EndOfGroup GNSS-NG header flag set - we expect a position resolution if header.EndOfGroup { - metricDevEui := uplinkResponse.Result.Deveui - if uplinkResponse.GetTimestamp() == nil { - loracloudPositionEstimateNoCapturedAtSetCounter.WithLabelValues(metricDevEui).Inc() + PositionEstimateNoCapturedAtSetCounter.Inc() } if !uplinkResponse.HasValidCoordinates() { - loracloudPositionEstimateZeroCoordinatesSetCounter.WithLabelValues(metricDevEui).Inc() + PositionEstimateZeroCoordinatesSetCounter.Inc() } if uplinkResponse.GetTimestamp() == nil && uplinkResponse.HasValidCoordinates() { - loracloudPositionEstimateNoCapturedAtSetWithValidCoordinatesCounter.WithLabelValues(metricDevEui).Inc() + PositionEstimateNoCapturedAtSetWithValidCoordinatesCounter.Inc() } if uplinkResponse.HasValidPositionResolution() { - loracloudPositionEstimateValidCounter.WithLabelValues(metricDevEui).Inc() + PositionEstimateValidCounter.Inc() } else { - loracloudPositionEstimateInvalidCounter.WithLabelValues(metricDevEui).Inc() - m.logger.Error("position resolution is invalid", zap.Any("uplinkResponse", uplinkResponse)) + PositionEstimateInvalidCounter.Inc() + m.logger.Error("position resolution is invalid", zap.String("category", "position_invalid")) return nil, ErrPositionResolutionIsEmpty } } diff --git a/pkg/solver/loracloud/metrics.go b/pkg/solver/loracloud/metrics.go index 4e4dbc60..2fe3045a 100644 --- a/pkg/solver/loracloud/metrics.go +++ b/pkg/solver/loracloud/metrics.go @@ -6,24 +6,24 @@ import ( ) var ( - loracloudPositionEstimateNoCapturedAtSetCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + PositionEstimateNoCapturedAtSetCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "truvami_loracloud_position_estimate_no_captured_at_set_total", Help: "The total number of position estimate responses where the captured at (UTC) timestamp is not set", - }, []string{"devEui"}) - loracloudPositionEstimateZeroCoordinatesSetCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + }) + PositionEstimateZeroCoordinatesSetCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "truvami_loracloud_position_estimate_zero_coordinates_set_total", Help: "The total number of position estimate responses where the coordinates are set to 0", - }, []string{"devEui"}) - loracloudPositionEstimateNoCapturedAtSetWithValidCoordinatesCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + }) + PositionEstimateNoCapturedAtSetWithValidCoordinatesCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "truvami_loracloud_position_estimate_no_captured_at_set_with_valid_coordinates_total", Help: "The total number of position estimate responses where the captured at (UTC) timestamp is not set and the coordinates are valid", - }, []string{"devEui"}) - loracloudPositionEstimateValidCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + }) + PositionEstimateValidCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "truvami_loracloud_position_estimate_valid_total", Help: "The total number of position estimate responses where the captured at (UTC) timestamp is set and the coordinates are valid", - }, []string{"devEui"}) - loracloudPositionEstimateInvalidCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + }) + PositionEstimateInvalidCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "truvami_loracloud_position_estimate_invalid_total", Help: "The total number of position estimate responses where the position resolution is invalid", - }, []string{"devEui"}) + }) ) diff --git a/pkg/solver/loracloud/v2/loracloud.go b/pkg/solver/loracloud/v2/loracloud.go index 98c93de2..8a5c36f2 100644 --- a/pkg/solver/loracloud/v2/loracloud.go +++ b/pkg/solver/loracloud/v2/loracloud.go @@ -59,22 +59,21 @@ func NewLoracloudClient(ctx context.Context, accessToken string, logger *zap.Log } // Warn for Semtech LoRaCloud shutdown (defensive) if client.BaseUrl == SemtechLoRaCloudBaseUrl && time.Now().After(time.Date(2025, 7, 31, 0, 0, 0, 0, time.UTC)) { - logger.Warn("LoRa Cloud is Sunsetting on 31.07.2025", zap.String("url", "https://www.semtech.com/loracloud-shutdown")) + logger.Warn("LoRa Cloud is Sunsetting on 31.07.2025") } return client, nil } func (l LoracloudClient) Solve(ctx context.Context, payload string, options solver.SolverV2Options) (*decoder.DecodedUplink, error) { start := time.Now() - baseURLLabel := l.BaseUrl defer func() { - loracloudV2RequestDurationSeconds.WithLabelValues(baseURLLabel).Observe(time.Since(start).Seconds()) + RequestDurationSeconds.Observe(time.Since(start).Seconds()) }() // Validate options (do NOT read from context) if err := l.validateOptions(payload, options); err != nil { - loracloudV2RequestsTotal.WithLabelValues(baseURLLabel, "error").Inc() - loracloudV2ErrorsTotal.WithLabelValues(baseURLLabel, "invalid_options").Inc() + RequestsTotal.WithLabelValues("error").Inc() + ErrorsTotal.WithLabelValues("invalid_options").Inc() return nil, common.WrapError(ErrInvalidOptions, err) } @@ -92,8 +91,8 @@ func (l LoracloudClient) Solve(ctx context.Context, payload string, options solv // Reuse v1 client for actual HTTP and response shaping, to keep behavior aligned v1Client, err := v1.NewLoracloudClient(ctx, l.accessToken, l.logger, v1.WithBaseUrl(l.BaseUrl)) if err != nil { - loracloudV2RequestsTotal.WithLabelValues(baseURLLabel, "error").Inc() - loracloudV2ErrorsTotal.WithLabelValues(baseURLLabel, "build_request").Inc() + RequestsTotal.WithLabelValues("error").Inc() + ErrorsTotal.WithLabelValues("build_request").Inc() return nil, common.WrapError(ErrBuildRequest, err) } @@ -107,38 +106,36 @@ func (l LoracloudClient) Solve(ctx context.Context, payload string, options solv resp, err := v1Client.DeliverUplinkMessage(options.DevEui, uplink) if err != nil { - loracloudV2RequestsTotal.WithLabelValues(baseURLLabel, "error").Inc() + RequestsTotal.WithLabelValues("error").Inc() switch { case errors.Is(err, v1.ErrUnexpectedStatusCode): - loracloudV2ErrorsTotal.WithLabelValues(baseURLLabel, "unexpected_status").Inc() + ErrorsTotal.WithLabelValues("unexpected_status").Inc() return nil, common.WrapError(ErrUnexpectedStatus, err) case errors.Is(err, v1.ErrDecodingResponse): - loracloudV2ErrorsTotal.WithLabelValues(baseURLLabel, "decode_failed").Inc() + ErrorsTotal.WithLabelValues("decode_failed").Inc() return nil, common.WrapError(ErrDecodeFailed, err) case errors.Is(err, v1.ErrPositionResolutionIsEmpty): - loracloudV2ErrorsTotal.WithLabelValues(baseURLLabel, "position_invalid").Inc() + ErrorsTotal.WithLabelValues("position_invalid").Inc() return nil, common.WrapError(ErrPositionInvalid, err) default: - loracloudV2ErrorsTotal.WithLabelValues(baseURLLabel, "request_failed").Inc() + ErrorsTotal.WithLabelValues("request_failed").Inc() return nil, common.WrapError(ErrRequestFailed, err) } } // Defensive validation of response if resp == nil { - loracloudV2RequestsTotal.WithLabelValues(baseURLLabel, "error").Inc() - loracloudV2ResponseInvalidTotal.WithLabelValues(baseURLLabel).Inc() + RequestsTotal.WithLabelValues("error").Inc() + ResponseInvalidTotal.Inc() return nil, common.WrapError(ErrResponseInvalid, fmt.Errorf("nil response")) } - devEui := resp.Result.Deveui // v1 client already normalized and removed hyphens - // Visibility counters similar to v1 (best effort) if resp.GetTimestamp() == nil { - loracloudV2PositionInvalidTotal.WithLabelValues(devEui).Inc() + PositionInvalidTotal.Inc() } if !resp.HasValidCoordinates() { - loracloudV2PositionInvalidTotal.WithLabelValues(devEui).Inc() + PositionInvalidTotal.Inc() } validPosition := resp.HasValidPositionResolution() @@ -148,8 +145,8 @@ func (l LoracloudClient) Solve(ctx context.Context, payload string, options solv if validPosition { features = append(features, decoder.FeatureGNSS) } else { - loracloudV2ErrorsTotal.WithLabelValues(baseURLLabel, "position_invalid").Inc() - l.logger.Debug("position resolution invalid (no GNSS feature set)", zap.Any("uplinkResponse", resp)) + ErrorsTotal.WithLabelValues("position_invalid").Inc() + l.logger.Debug("position resolution invalid (no GNSS feature set)", zap.String("category", "position_invalid")) } withTimestamp := options.Timestamp != nil @@ -163,7 +160,7 @@ func (l LoracloudClient) Solve(ctx context.Context, payload string, options solv features = append(features, decoder.FeatureTimestamp) timestampForBufferedCheck = options.Timestamp } else if resp.GetTimestamp() != nil { - l.logger.Info("no timestamp provided, but LoRaCloud / Traxmate returned one", zap.String("devEui", devEui), zap.Time("timestamp", *resp.GetTimestamp())) + l.logger.Info("no timestamp provided, but LoRaCloud / Traxmate returned one", zap.String("category", "timestamp_from_response")) features = append(features, decoder.FeatureTimestamp) timestampForBufferedCheck = resp.GetTimestamp() } @@ -174,7 +171,7 @@ func (l LoracloudClient) Solve(ctx context.Context, payload string, options solv if timestampForBufferedCheck.Before(thresholdAgo) { buffered = true features = append(features, decoder.FeatureBuffered) - loracloudV2BufferedDetectedTotal.WithLabelValues(devEui, l.bufferedThreshold.String()).Inc() + BufferedDetectedTotal.WithLabelValues(l.bufferedThreshold.String()).Inc() } } @@ -224,7 +221,7 @@ func (l LoracloudClient) Solve(ctx context.Context, payload string, options solv data = &dataBase{resp: resp} } - loracloudV2RequestsTotal.WithLabelValues(baseURLLabel, "success").Inc() + RequestsTotal.WithLabelValues("success").Inc() return decoder.NewDecodedUplink(features, data), nil } diff --git a/pkg/solver/loracloud/v2/metrics.go b/pkg/solver/loracloud/v2/metrics.go index 82793bdb..35616f46 100644 --- a/pkg/solver/loracloud/v2/metrics.go +++ b/pkg/solver/loracloud/v2/metrics.go @@ -6,34 +6,34 @@ import ( ) var ( - loracloudV2RequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + RequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "truvami_loracloud_v2_requests_total", Help: "Total number of LoRaCloud v2 solver requests", - }, []string{"base_url", "outcome"}) // outcome: success|error + }, []string{"outcome"}) // outcome: success|error - loracloudV2RequestDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ + RequestDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "truvami_loracloud_v2_request_duration_seconds", Help: "Duration of LoRaCloud v2 solver requests in seconds", Buckets: prometheus.DefBuckets, - }, []string{"base_url"}) + }) - loracloudV2ResponseInvalidTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + ResponseInvalidTotal = promauto.NewCounter(prometheus.CounterOpts{ Name: "truvami_loracloud_v2_response_invalid_total", Help: "Total number of invalid responses from LoRaCloud v2", - }, []string{"base_url"}) + }) - loracloudV2PositionInvalidTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + PositionInvalidTotal = promauto.NewCounter(prometheus.CounterOpts{ Name: "truvami_loracloud_v2_position_invalid_total", Help: "Total number of invalid position resolutions (missing timestamp or zero coordinates)", - }, []string{"devEui"}) + }) - loracloudV2BufferedDetectedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + BufferedDetectedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "truvami_loracloud_v2_timestamp_buffered_detected_total", Help: "Total number of uplinks considered buffered due to past timestamp", - }, []string{"devEui", "threshold"}) + }, []string{"threshold"}) - loracloudV2ErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + ErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "truvami_loracloud_v2_errors_total", Help: "Total number of errors in LoRaCloud v2 solver", - }, []string{"base_url", "type"}) // type: build_request|request_failed|unexpected_status|decode_failed|response_invalid|position_invalid|invalid_options + }, []string{"type"}) // type: build_request|request_failed|unexpected_status|decode_failed|response_invalid|position_invalid|invalid_options ) diff --git a/pkg/solver/metrics_privacy_test.go b/pkg/solver/metrics_privacy_test.go new file mode 100644 index 00000000..486b494d --- /dev/null +++ b/pkg/solver/metrics_privacy_test.go @@ -0,0 +1,60 @@ +package solver_test + +import ( + "strings" + "testing" + + dto "github.com/prometheus/client_model/go" + loracloud "github.com/truvami/decoder/pkg/solver/loracloud" + loracloudv2 "github.com/truvami/decoder/pkg/solver/loracloud/v2" +) + +func TestLoracloudMetricsHaveNoSensitiveLabels(t *testing.T) { + unlabeled := []interface { + Write(*dto.Metric) error + }{ + loracloud.PositionEstimateNoCapturedAtSetCounter, + loracloud.PositionEstimateZeroCoordinatesSetCounter, + loracloud.PositionEstimateNoCapturedAtSetWithValidCoordinatesCounter, + loracloud.PositionEstimateValidCounter, + loracloud.PositionEstimateInvalidCounter, + loracloudv2.ResponseInvalidTotal, + loracloudv2.PositionInvalidTotal, + loracloudv2.RequestDurationSeconds, + } + + for _, collector := range unlabeled { + metric := &dto.Metric{} + if err := collector.Write(metric); err != nil { + t.Fatalf("write metric: %v", err) + } + assertNoSensitiveLabels(t, metric.GetLabel()) + } + + labeled := []struct { + write func(*dto.Metric) error + }{ + {func(m *dto.Metric) error { return loracloudv2.RequestsTotal.WithLabelValues("success").Write(m) }}, + {func(m *dto.Metric) error { return loracloudv2.ErrorsTotal.WithLabelValues("request_failed").Write(m) }}, + {func(m *dto.Metric) error { return loracloudv2.BufferedDetectedTotal.WithLabelValues("5m0s").Write(m) }}, + } + + for _, tc := range labeled { + metric := &dto.Metric{} + if err := tc.write(metric); err != nil { + t.Fatalf("write metric: %v", err) + } + assertNoSensitiveLabels(t, metric.GetLabel()) + } +} + +func assertNoSensitiveLabels(t *testing.T, labels []*dto.LabelPair) { + t.Helper() + for _, label := range labels { + name := strings.ToLower(label.GetName()) + switch name { + case "deveui", "base_url", "tag", "payload", "url": + t.Fatalf("metric must not expose sensitive label %q", label.GetName()) + } + } +}