From 2aaa62558cb39296162756f5b9f5848fec27eb8d Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Tue, 5 May 2026 19:00:36 +0200 Subject: [PATCH 1/8] Add SOT target control and state telemetry messages Add SetSotTargetCtrl to control.proto for setting/clearing SOT targets. Add SotState message format with State enum (IDLE/TRACKING/LOST). Add SotStateTel to telemetry.proto for reporting tracking state. Zero-size bounding box (width=0, height=0) convention clears the target. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- protobuf_definitions/control.proto | 10 ++++++++++ protobuf_definitions/message_formats.proto | 13 +++++++++++++ protobuf_definitions/telemetry.proto | 8 ++++++++ 3 files changed, 31 insertions(+) diff --git a/protobuf_definitions/control.proto b/protobuf_definitions/control.proto index 42301321..c2747a36 100644 --- a/protobuf_definitions/control.proto +++ b/protobuf_definitions/control.proto @@ -247,3 +247,13 @@ message StartLogStreamingCtrl { // Message sent to stop streaming log entries to the client. message StopLogStreamingCtrl { } + +// Issue a command to set or clear the single-object tracking (SOT) target. +// +// Send a bounding box to start tracking a new target. The CV pipeline will +// initialize the SOT tracker on the specified region of the video frame. +// To clear the current target (stop tracking), send a bounding box with +// width and height set to zero. +message SetSotTargetCtrl { + BoundingBox bounding_box = 1; // Target bounding box. Zero size (width=0, height=0) clears the target. +} diff --git a/protobuf_definitions/message_formats.proto b/protobuf_definitions/message_formats.proto index c093ec86..0fe909c1 100644 --- a/protobuf_definitions/message_formats.proto +++ b/protobuf_definitions/message_formats.proto @@ -1470,3 +1470,16 @@ message OperatorInfo { string name = 1; // Full name of the operator. string email = 2; // E-mail address of the operator. } + +// Single-object tracking (SOT) state reported by the computer vision pipeline. +message SotState { + // Current state of the SOT tracker. + enum State { + STATE_UNSPECIFIED = 0; // Unspecified state. + STATE_IDLE = 1; // No target selected, waiting for a target bounding box. + STATE_TRACKING = 2; // Actively tracking a target. + STATE_LOST = 3; // Target was lost (tracker failed to follow the object). + } + State state = 1; // Current tracking state. + BoundingBox bounding_box = 2; // Current tracked bounding box (valid when TRACKING). +} diff --git a/protobuf_definitions/telemetry.proto b/protobuf_definitions/telemetry.proto index caf938ae..3c684c12 100644 --- a/protobuf_definitions/telemetry.proto +++ b/protobuf_definitions/telemetry.proto @@ -341,3 +341,11 @@ message TurbidityFilterTel { message CameraPanTiltZoomTel { CameraPanTiltZoom camera_pan_tilt_zoom = 1; // Current pan, tilt, and zoom state. } + +// Single-object tracking (SOT) state telemetry from the computer vision pipeline. +// +// Reports the current tracking state and the tracked bounding box when active. +// Published periodically while the SOT pipeline is running. +message SotStateTel { + SotState sot_state = 1; // Current SOT state and bounding box. +} From da66693fd82dd98548f82f556c32a6c245e4767b Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Thu, 7 May 2026 19:31:25 +0200 Subject: [PATCH 2/8] feat: add image dimensions to SotState Add image_width and image_height fields to SotState so receivers know the frame resolution the bounding box coordinates are relative to. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- protobuf_definitions/message_formats.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protobuf_definitions/message_formats.proto b/protobuf_definitions/message_formats.proto index 0fe909c1..f432089a 100644 --- a/protobuf_definitions/message_formats.proto +++ b/protobuf_definitions/message_formats.proto @@ -1482,4 +1482,6 @@ message SotState { } State state = 1; // Current tracking state. BoundingBox bounding_box = 2; // Current tracked bounding box (valid when TRACKING). + uint32 image_width = 3; // Width of the source frame in pixels. + uint32 image_height = 4; // Height of the source frame in pixels. } From 2b6382bbd084fbe277d5a7f8cde2d0ecd00d99b3 Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Thu, 7 May 2026 19:48:27 +0200 Subject: [PATCH 3/8] feat: add image dimensions to SetSotTargetCtrl Add image_width and image_height so the receiver can rescale bounding box coordinates from the app's frame to the model's actual source resolution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- protobuf_definitions/control.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protobuf_definitions/control.proto b/protobuf_definitions/control.proto index c2747a36..2c834091 100644 --- a/protobuf_definitions/control.proto +++ b/protobuf_definitions/control.proto @@ -256,4 +256,6 @@ message StopLogStreamingCtrl { // width and height set to zero. message SetSotTargetCtrl { BoundingBox bounding_box = 1; // Target bounding box. Zero size (width=0, height=0) clears the target. + uint32 image_width = 2; // Width of the frame the bounding box was drawn on. + uint32 image_height = 3; // Height of the frame the bounding box was drawn on. } From 8ed27e220cac978c16bbfbfcc7c69900fb7ad0bc Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Tue, 12 May 2026 17:05:44 +0200 Subject: [PATCH 4/8] Add CvModelInfo message and cv_models field to DroneInfo Add CvModelInfo protobuf message with name, type, running, device, and package_id fields. Add repeated cv_models field to DroneInfo for reporting loaded computer vision models in telemetry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- protobuf_definitions/message_formats.proto | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/protobuf_definitions/message_formats.proto b/protobuf_definitions/message_formats.proto index e68f6731..d2062b4a 100644 --- a/protobuf_definitions/message_formats.proto +++ b/protobuf_definitions/message_formats.proto @@ -757,6 +757,20 @@ enum PressureSensorType { // Information about the drone. // +// Information about a loaded computer vision model. +message CvModelInfo { + enum ModelType { + MODEL_TYPE_UNSPECIFIED = 0; + MODEL_TYPE_DETECTION = 1; + MODEL_TYPE_SOT = 2; + } + string name = 1; // Human-readable model name (from model_meta.json). + ModelType type = 2; // Type of CV model. + bool running = 3; // Whether the model is currently running. + string device = 4; // Execution provider (e.g. "cuda", "tensorrt"). + string package_id = 5; // Package directory name (e.g. "tinyyolov2_package"). Stable identifier. +} + // This message contains serial numbers and version information for // internal components in the drone. Primarily used for diagnostics, or to // determine the origin of a logfile. @@ -772,6 +786,7 @@ message DroneInfo { bytes bb_uid = 8; // Backbone unique identifier. GuestPortInfo gp = 9; // Guest port information. PressureSensorType depth_sensor = 11; // Type of depth sensor that is connected to the drone. + repeated CvModelInfo cv_models = 12; // List of loaded computer vision models. } // Known error states for the drone. From a018b2a1ecab87158cfd05c329ce936044490180 Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Wed, 13 May 2026 18:02:49 +0200 Subject: [PATCH 5/8] Add labels field to CvModelInfo Include the full list of class names the model can detect, indexed by class_id, in the DroneInfoTel telemetry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- protobuf_definitions/message_formats.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/protobuf_definitions/message_formats.proto b/protobuf_definitions/message_formats.proto index d2062b4a..d5be1521 100644 --- a/protobuf_definitions/message_formats.proto +++ b/protobuf_definitions/message_formats.proto @@ -769,6 +769,7 @@ message CvModelInfo { bool running = 3; // Whether the model is currently running. string device = 4; // Execution provider (e.g. "cuda", "tensorrt"). string package_id = 5; // Package directory name (e.g. "tinyyolov2_package"). Stable identifier. + repeated string labels = 6; // Class names the model can detect (indexed by class_id). } // This message contains serial numbers and version information for From 7ebed786c5c798e0e3dd40e26ae8b1b4c70fe7f6 Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Mon, 18 May 2026 17:32:36 +0200 Subject: [PATCH 6/8] feat: add segmentation mask message for object detection --- protobuf_definitions/message_formats.proto | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/protobuf_definitions/message_formats.proto b/protobuf_definitions/message_formats.proto index d5be1521..26a3fa45 100644 --- a/protobuf_definitions/message_formats.proto +++ b/protobuf_definitions/message_formats.proto @@ -1446,6 +1446,18 @@ message BoundingBox { uint32 height = 4; // Height of the bounding box (px). } +// RLE-encoded binary segmentation mask relative to the bounding box. +// +// The mask bitmap has dimensions (mask_width x mask_height) covering the +// detection's bounding box area. The counts field stores run-length encoded +// data as packed uint16 little-endian: alternating background/foreground +// pixel runs starting with background. +message SegmentationMask { + uint32 mask_width = 1; // Width of the RLE bitmap. + uint32 mask_height = 2; // Height of the RLE bitmap. + bytes counts = 3; // RLE counts as packed uint16 little-endian. +} + // A single object detection from a computer vision model. message ObjectDetection { BoundingBox bounding_box = 1; // Bounding box of the detected object. @@ -1453,6 +1465,7 @@ message ObjectDetection { uint32 class_id = 3; // Numeric class identifier from the model. string class_name = 4; // Human-readable class name. uint32 tracking_id = 5; // Unique ID for tracking the same object across frames. + SegmentationMask mask = 6; // Instance segmentation mask (absent if model has no segmentation). } // A list of object detections from a single model for a single video frame. From 409f808bed6ecc9986e65c7194d09517b3dcc59e Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Tue, 19 May 2026 16:22:31 +0200 Subject: [PATCH 7/8] Add explicit ClearSotTargetCtrl message Preferred way to clear the SOT target (stop tracking). The zero-size bounding box in SetSotTargetCtrl still works for backwards compatibility. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- protobuf_definitions/control.proto | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/protobuf_definitions/control.proto b/protobuf_definitions/control.proto index 2c834091..86362461 100644 --- a/protobuf_definitions/control.proto +++ b/protobuf_definitions/control.proto @@ -259,3 +259,10 @@ message SetSotTargetCtrl { uint32 image_width = 2; // Width of the frame the bounding box was drawn on. uint32 image_height = 3; // Height of the frame the bounding box was drawn on. } + +// Explicitly clear the single-object tracking (SOT) target (stop tracking). +// +// Preferred over sending a zero-size bounding box in SetSotTargetCtrl. +// Both methods are supported for backwards compatibility. +message ClearSotTargetCtrl { +} From 652b0c98546fded7e286e684fe6ea1547544f81d Mon Sep 17 00:00:00 2001 From: Juan Pablo Pino Bravo Date: Tue, 19 May 2026 16:26:13 +0200 Subject: [PATCH 8/8] refactor: clarify SetSotTargetCtrl and ClearSotTargetCtrl descriptions --- protobuf_definitions/control.proto | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/protobuf_definitions/control.proto b/protobuf_definitions/control.proto index 86362461..a9c02ef4 100644 --- a/protobuf_definitions/control.proto +++ b/protobuf_definitions/control.proto @@ -248,21 +248,16 @@ message StartLogStreamingCtrl { message StopLogStreamingCtrl { } -// Issue a command to set or clear the single-object tracking (SOT) target. +// Issue a command to set the single-object tracking (SOT) target. // // Send a bounding box to start tracking a new target. The CV pipeline will // initialize the SOT tracker on the specified region of the video frame. -// To clear the current target (stop tracking), send a bounding box with -// width and height set to zero. message SetSotTargetCtrl { - BoundingBox bounding_box = 1; // Target bounding box. Zero size (width=0, height=0) clears the target. + BoundingBox bounding_box = 1; // Target bounding box. uint32 image_width = 2; // Width of the frame the bounding box was drawn on. uint32 image_height = 3; // Height of the frame the bounding box was drawn on. } -// Explicitly clear the single-object tracking (SOT) target (stop tracking). -// -// Preferred over sending a zero-size bounding box in SetSotTargetCtrl. -// Both methods are supported for backwards compatibility. +// Issue a command to clear the single-object tracking (SOT) target (stop tracking). message ClearSotTargetCtrl { }