Harden telemetry delivery and refine recipe metadata

docs/Privacy.md

@@ -6,11 +6,15 @@ The software may collect information about you and your use of the software and
 ***
 
 ## Technical Details
-Olive uses the [OpenTelemetry](https://opentelemetry.io/) API for its implementation. Telemetry is turned ON by default. Based on user consent, this data may be periodically sent to Microsoft servers following GDPR and privacy regulations for anonymity and data access controls. Application, device, and version information is collected automatically.
+Telemetry is turned ON by default. Based on user consent, this data may be periodically sent to Microsoft servers following GDPR and privacy regulations for anonymity and data access controls. Application, device, and version information is collected automatically.
 
 In addition, Olive may collect additional telemetry data such as:
 - Invoked commands
 - Performance data
 - Exception information
 
-Collection of this additional telemetry can be disabled by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. Telemetry is also automatically disabled when a CI/CD environment is detected (e.g., GitHub Actions, Azure Pipelines, Jenkins). If telemetry is enabled, but cannot be sent to Microsoft, it will be stored locally and sent when a connection is available. You can override the default cache location by setting the `OLIVE_TELEMETRY_CACHE_DIR` environment variable to a valid directory path.
+You can disable telemetry by adding the `--disable_telemetry` flag to any Olive CLI command, or by setting the `OLIVE_DISABLE_TELEMETRY` environment variable to `1` before running. When telemetry is disabled this way, the additional telemetry above (commands, performance, exceptions) is not sent. A minimal device-id heartbeat — a non-reversible hashed device identifier plus basic operating-system name, version, release, and architecture — is still sent outside CI/CD environments so Microsoft can count active devices; it contains no command, performance, or exception data.
+
+In CI/CD environments (e.g., GitHub Actions, Azure Pipelines, Jenkins), Olive suppresses the device-id heartbeat and the action/error events and only emits the `OliveRecipe` event. The `OliveRecipe` event may include recipe metadata such as pass types, explicitly configured target settings, the host system type (including the default `LocalSystem` host) and any explicitly configured host accelerator settings, whether a custom package config was provided, a redacted snapshot of custom package-config overrides, and a redacted snapshot of explicitly supplied config overrides. Setting `OLIVE_DISABLE_TELEMETRY=1` in a CI/CD environment sends nothing at all.
+
+Telemetry is implemented using only the Python standard library. Events are written to a local per-user SQLite queue and uploaded in the background to Microsoft over HTTPS. If telemetry is enabled but cannot be sent (for example, while offline), events remain in the local queue and are uploaded on a later run when a connection is available.

olive/cli/base.py

@@ -115,7 +115,7 @@ def _run_workflow(self):
                     mark_test_output_path(self.args.output_path)
                 print("Dry run mode enabled. Configuration file is generated but no optimization is performed.")
                 return None
-            workflow_output = olive_run(run_config)
+            workflow_output = olive_run(run_config, recipe_telemetry_metadata=self._get_recipe_telemetry_metadata())
             if getattr(self.args, "test", None) not in (None, False):
                 mark_test_output_path(self.args.output_path)
             if not workflow_output.has_output_model():
@@ -124,6 +124,17 @@ def _run_workflow(self):
                 print(f"Model is saved at {self.args.output_path}")
             return workflow_output
 
+    def _get_recipe_telemetry_metadata(self) -> dict[str, str]:
+        recipe_name = self.__class__.__name__
+        if recipe_name.endswith("Command"):
+            recipe_name = recipe_name[: -len("Command")]
+        return {
+            "recipe_name": recipe_name,
+            "recipe_command": recipe_name,
+            "recipe_source": "generated_cli",
+            "recipe_format": "generated",
+        }
+
     @staticmethod
     def _parse_extra_options(kv_items):
         from onnxruntime_genai import __version__ as OrtGenaiVersion

olive/cli/launcher.py

@@ -2,6 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 # --------------------------------------------------------------------------
+import os
 import sys
 from argparse import ArgumentParser
 from warnings import warn
@@ -66,9 +67,11 @@ def main(raw_args=None, called_as_console_script: bool = True):
 
     args, unknown_args = parser.parse_known_args(raw_args)
 
-    telemetry = Telemetry()
+    # Honor --disable-telemetry BEFORE constructing Telemetry, so a disabled run
+    # never starts the uploader or drains/uploads the durable store.
     if args.disable_telemetry:
-        telemetry.disable_telemetry()
+        os.environ["OLIVE_DISABLE_TELEMETRY"] = "1"
+    telemetry = Telemetry()
 
     if not hasattr(args, "func"):
         parser.print_help()

olive/cli/run.py

@@ -53,16 +53,22 @@ def register_subcommand(parser: ArgumentParser):
 
     @action
     def run(self):
+        from copy import deepcopy
+        from pathlib import Path
+
         from olive.common.config_utils import load_config_file
         from olive.workflows import run as olive_run
 
         # allow the run_config to be a dict already (for api use)
-        run_config = self.args.run_config
-        if not isinstance(run_config, dict):
-            run_config = load_config_file(run_config)
+        run_config_input = self.args.run_config
+        run_config = (
+            deepcopy(run_config_input) if isinstance(run_config_input, dict) else load_config_file(run_config_input)
+        )
+        config_overrides = {}
         if input_model_config := get_input_model_config(self.args, required=False):
             print("Replacing input model config in run config")
             run_config["input_model"] = input_model_config
+            config_overrides["input_model"] = input_model_config
         elif self.args.test not in (None, False):
             input_model = run_config.get("input_model")
             if not isinstance(input_model, dict) or input_model.get("type", "").lower() != "hfmodel":
@@ -79,6 +85,19 @@ def run(self):
                 run_config.get("engine", {}).pop(rc_key, None)
                 # add value to run config directly
                 run_config[rc_key] = arg_value
+                config_overrides[rc_key] = arg_value
+
+        recipe_telemetry_metadata = {
+            "recipe_command": "WorkflowRun",
+            "recipe_source": "config_dict" if isinstance(run_config_input, dict) else "config_file",
+            "recipe_format": "dict"
+            if isinstance(run_config_input, dict)
+            else Path(run_config_input).suffix.lstrip(".").lower() or "unknown",
+            "execution_mode": "list_required_packages" if self.args.list_required_packages else "run",
+            "package_config_provided": bool(self.args.package_config),
+        }
+        if config_overrides:
+            recipe_telemetry_metadata["config_overrides"] = config_overrides
 
         output_path = run_config.get("output_dir") or run_config.get("engine", {}).get("output_dir")
         validate_test_output_path(output_path, self.args.test)
@@ -89,6 +108,7 @@ def run(self):
             list_required_packages=self.args.list_required_packages,
             tempdir=self.args.tempdir,
             package_config=self.args.package_config,
+            recipe_telemetry_metadata=recipe_telemetry_metadata,
         )
         if self.args.test not in (None, False):
             mark_test_output_path(output_path)

olive/systems/docker/docker_system.py

@@ -232,6 +232,8 @@ def _prepare_run_params(self) -> dict:
 
     def _prepare_environment(self, base_env) -> dict:
         """Prepare environment variables for container."""
+        from olive.telemetry.telemetry import is_ci_environment
+
         # Convert list to dict if needed
         if isinstance(base_env, list):
             environment = {env.split("=")[0]: env.split("=")[1] for env in base_env}
@@ -241,6 +243,8 @@ def _prepare_environment(self, base_env) -> dict:
         # Add default environment variables
         environment.setdefault("PYTHONPYCACHEPREFIX", "/tmp")
         environment["OLIVE_LOG_LEVEL"] = logging.getLevelName(logger.getEffectiveLevel())
+        if is_ci_environment():
+            environment["CI"] = "1"
 
         # Add HuggingFace token if needed
         if self.hf_token:

olive/systems/docker/workflow_runner.py

@@ -20,7 +20,7 @@ def runner_entry(config):
         config = json.load(f)
 
     logger.info("Running workflow with config: %s", config)
-    olive_run(config)
+    olive_run(config, emit_recipe_telemetry=False)
 
 
 if __name__ == "__main__":

olive/telemetry/constants.py

@@ -3,6 +3,6 @@
 # Licensed under the MIT License.
 # --------------------------------------------------------------------------
 
-"""OneCollector connection string."""
+"""Telemetry constants."""
 
-CONNECTION_STRING = "SW5zdHJ1bWVudGF0aW9uS2V5PTlkNWRkYWVjNjFlMjQ1NjdiNzg4YTIwYWVhMzI0NjMxLTcyMzdkN2M2LWVlNjEtNGNmZC1iYjdiLTU5MDNhOTcyYzJlNC03MDQ3"
+CONNECTION_STRING = "SW5zdHJ1bWVudGF0aW9uS2V5PTYyMTUwOTExZGMwMDRmYzliYjY3YmE5NjA2NDI3ZTU2LWVjNjFmOWFmLTVkN2EtNGQxOS1hZjMxLWI5Y2Q2OWU5ODdmMS02OTE1"

olive/telemetry/library/__init__.py

@@ -3,62 +3,25 @@
 # Licensed under the MIT License.
 # --------------------------------------------------------------------------
 
-"""OneCollector Exporter for OpenTelemetry Python.
+"""OneCollector building blocks (standard library only).
 
-This package provides an OpenTelemetry exporter that sends telemetry data
-to Microsoft OneCollector using the Common Schema JSON format.
-
-Example usage:
-
-    from onecollector_exporter import (
-        OneCollectorLogExporter,
-        OneCollectorExporterOptions,
-        get_telemetry_logger,
-    )
-
-    # Option 1: Use with OpenTelemetry SDK directly
-    options = OneCollectorExporterOptions(
-        connection_string="InstrumentationKey=your-key-here"
-    )
-    exporter = OneCollectorLogExporter(options=options)
-
-    # Add to logger provider
-    from opentelemetry.sdk._logs import LoggerProvider
-    from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
-
-    provider = LoggerProvider()
-    provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
-
-    # Option 2: Use the simplified telemetry logger
-    logger = get_telemetry_logger(
-        connection_string="InstrumentationKey=your-key-here"
-    )
-    logger.log("MyEvent", {"key": "value"})
-    logger.shutdown()
+Helpers for serializing telemetry to Common Schema JSON and posting it to the
+Microsoft OneCollector endpoint. These modules have no third-party dependency
+and are driven directly by the SQLite-backed uploader.
 """
 
-from olive.telemetry.library.callback_manager import CallbackManager, PayloadTransmittedCallbackArgs
-from olive.telemetry.library.connection_string_parser import ConnectionStringParser
-from olive.telemetry.library.event_source import OneCollectorEventId, OneCollectorEventSource, event_source
-from olive.telemetry.library.exporter import OneCollectorLogExporter
-from olive.telemetry.library.options import (
+from .callback_manager import CallbackManager, PayloadTransmittedCallbackArgs
+from .connection_string_parser import ConnectionStringParser
+from .event_source import OneCollectorEventId, OneCollectorEventSource, event_source
+from .options import (
     CompressionType,
     OneCollectorExporterOptions,
     OneCollectorExporterValidationError,
     OneCollectorTransportOptions,
 )
-from olive.telemetry.library.payload_builder import PayloadBuilder
-from olive.telemetry.library.retry import RetryHandler
-from olive.telemetry.library.serialization import CommonSchemaJsonSerializationHelper
-from olive.telemetry.library.telemetry_logger import (
-    TelemetryLogger,
-    get_telemetry_logger,
-    log_event,
-    shutdown_telemetry,
-)
-from olive.telemetry.library.transport import HttpJsonPostTransport, ITransport
-
-__version__ = "0.0.1"
+from .payload_builder import PayloadBuilder
+from .serialization import CommonSchemaJsonSerializationHelper
+from .transport import HttpJsonPostTransport, ITransport
 
 __all__ = [
     "CallbackManager",
@@ -71,14 +34,8 @@
     "OneCollectorEventSource",
     "OneCollectorExporterOptions",
     "OneCollectorExporterValidationError",
-    "OneCollectorLogExporter",
     "OneCollectorTransportOptions",
     "PayloadBuilder",
     "PayloadTransmittedCallbackArgs",
-    "RetryHandler",
-    "TelemetryLogger",
     "event_source",
-    "get_telemetry_logger",
-    "log_event",
-    "shutdown_telemetry",
 ]