diff --git a/protobuf_definitions/control.proto b/protobuf_definitions/control.proto index 42301321..a9c02ef4 100644 --- a/protobuf_definitions/control.proto +++ b/protobuf_definitions/control.proto @@ -247,3 +247,17 @@ message StartLogStreamingCtrl { // Message sent to stop streaming log entries to the client. message StopLogStreamingCtrl { } + +// Issue a command to set the single-object tracking (SOT) target. +// +// Send a bounding box to start tracking a new target. The CV pipeline will +// initialize the SOT tracker on the specified region of the video frame. +message SetSotTargetCtrl { + BoundingBox bounding_box = 1; // Target bounding box. + uint32 image_width = 2; // Width of the frame the bounding box was drawn on. + uint32 image_height = 3; // Height of the frame the bounding box was drawn on. +} + +// Issue a command to clear the single-object tracking (SOT) target (stop tracking). +message ClearSotTargetCtrl { +} diff --git a/protobuf_definitions/message_formats.proto b/protobuf_definitions/message_formats.proto index a503cfa4..26a3fa45 100644 --- a/protobuf_definitions/message_formats.proto +++ b/protobuf_definitions/message_formats.proto @@ -757,6 +757,21 @@ enum PressureSensorType { // Information about the drone. // +// Information about a loaded computer vision model. +message CvModelInfo { + enum ModelType { + MODEL_TYPE_UNSPECIFIED = 0; + MODEL_TYPE_DETECTION = 1; + MODEL_TYPE_SOT = 2; + } + string name = 1; // Human-readable model name (from model_meta.json). + ModelType type = 2; // Type of CV model. + bool running = 3; // Whether the model is currently running. + string device = 4; // Execution provider (e.g. "cuda", "tensorrt"). + string package_id = 5; // Package directory name (e.g. "tinyyolov2_package"). Stable identifier. + repeated string labels = 6; // Class names the model can detect (indexed by class_id). +} + // This message contains serial numbers and version information for // internal components in the drone. Primarily used for diagnostics, or to // determine the origin of a logfile. @@ -772,6 +787,7 @@ message DroneInfo { bytes bb_uid = 8; // Backbone unique identifier. GuestPortInfo gp = 9; // Guest port information. PressureSensorType depth_sensor = 11; // Type of depth sensor that is connected to the drone. + repeated CvModelInfo cv_models = 12; // List of loaded computer vision models. } // Known error states for the drone. @@ -1430,6 +1446,18 @@ message BoundingBox { uint32 height = 4; // Height of the bounding box (px). } +// RLE-encoded binary segmentation mask relative to the bounding box. +// +// The mask bitmap has dimensions (mask_width x mask_height) covering the +// detection's bounding box area. The counts field stores run-length encoded +// data as packed uint16 little-endian: alternating background/foreground +// pixel runs starting with background. +message SegmentationMask { + uint32 mask_width = 1; // Width of the RLE bitmap. + uint32 mask_height = 2; // Height of the RLE bitmap. + bytes counts = 3; // RLE counts as packed uint16 little-endian. +} + // A single object detection from a computer vision model. message ObjectDetection { BoundingBox bounding_box = 1; // Bounding box of the detected object. @@ -1437,6 +1465,7 @@ message ObjectDetection { uint32 class_id = 3; // Numeric class identifier from the model. string class_name = 4; // Human-readable class name. uint32 tracking_id = 5; // Unique ID for tracking the same object across frames. + SegmentationMask mask = 6; // Instance segmentation mask (absent if model has no segmentation). } // A list of object detections from a single model for a single video frame. @@ -1471,3 +1500,18 @@ message OperatorInfo { string name = 1; // Full name of the operator. string email = 2; // E-mail address of the operator. } + +// Single-object tracking (SOT) state reported by the computer vision pipeline. +message SotState { + // Current state of the SOT tracker. + enum State { + STATE_UNSPECIFIED = 0; // Unspecified state. + STATE_IDLE = 1; // No target selected, waiting for a target bounding box. + STATE_TRACKING = 2; // Actively tracking a target. + STATE_LOST = 3; // Target was lost (tracker failed to follow the object). + } + State state = 1; // Current tracking state. + BoundingBox bounding_box = 2; // Current tracked bounding box (valid when TRACKING). + uint32 image_width = 3; // Width of the source frame in pixels. + uint32 image_height = 4; // Height of the source frame in pixels. +} diff --git a/protobuf_definitions/telemetry.proto b/protobuf_definitions/telemetry.proto index caf938ae..3c684c12 100644 --- a/protobuf_definitions/telemetry.proto +++ b/protobuf_definitions/telemetry.proto @@ -341,3 +341,11 @@ message TurbidityFilterTel { message CameraPanTiltZoomTel { CameraPanTiltZoom camera_pan_tilt_zoom = 1; // Current pan, tilt, and zoom state. } + +// Single-object tracking (SOT) state telemetry from the computer vision pipeline. +// +// Reports the current tracking state and the tracked bounding box when active. +// Published periodically while the SOT pipeline is running. +message SotStateTel { + SotState sot_state = 1; // Current SOT state and bounding box. +}