chore: dev to main merge

.devcontainer/README.md

@@ -52,6 +52,9 @@ A development container (or dev container for short) lets you use a container as
 
 > Note: The Fabric installer notebook ([`fabric_solution_installer.ipynb`](../infra/fabric/deploy/fabric_solution_installer.ipynb)) runs **inside the Fabric workspace**, not in this container. It manages its own dependencies via `%pip install`; the container's Python packages do not affect it.
 
+### Package Feed Proxy (Microsoft-Managed Devices)
+All `pip install` commands above route through the Microsoft Package Feed Proxy (`https://packagefeedproxy.microsoft.io/pypi/simple/`) by default, via the `PIP_INDEX_URL` environment variable set in [`devcontainer.json`](./devcontainer.json)'s `containerEnv` (with a host `PIP_INDEX_URL` taking precedence if already set). This keeps `pip install` working once direct access to `pypi.org` is blocked on Microsoft-managed devices. To use public PyPI instead (e.g. outside a Microsoft-managed network), set `PIP_INDEX_URL=https://pypi.org/simple/` on the host before the container starts.
+
 ## How to use this dev container
 
 This README is a **reference for the dev container itself** — what's installed, how it's configured, and how to customize it. For end-to-end deployment instructions (prerequisites, `azd up`, optional configuration variables, expected results, cleanup), see the single source of truth: [`docs/DeploymentGuide.md`](../docs/DeploymentGuide.md).

.devcontainer/devcontainer.json

@@ -3,6 +3,9 @@
     "build": {
         "dockerfile": "Dockerfile"
     },
+    "containerEnv": {
+        "PIP_INDEX_URL": "${localEnv:PIP_INDEX_URL:https://packagefeedproxy.microsoft.io/pypi/simple/}"
+    },
     "features": {
         "ghcr.io/azure/azure-dev/azd:latest": {
             "version": "latest"

.devcontainer/post-create.sh

@@ -7,6 +7,14 @@ set -e
 
 echo "🚀 Setting up Microsoft IQ Solution Accelerator development environment..."
 
+# Microsoft Package Feed Proxy (CFS) configuration.
+# All pip installs in this script route through the CFS-protected proxy instead of
+# pypi.org / files.pythonhosted.org directly. PIP_INDEX_URL is normally already set via
+# devcontainer.json's containerEnv, but is (re)exported here as a safety net for anyone
+# invoking this script outside of the devcontainer.
+export PIP_INDEX_URL="${PIP_INDEX_URL:-https://packagefeedproxy.microsoft.io/pypi/simple/}"
+echo "📦 Using pip index URL: $PIP_INDEX_URL"
+
 # Note: Core tools already provided by devcontainer.json:
 # - Python 3.x (base image) with pip and venv
 # - Azure CLI + Bicep (azure-cli feature)
@@ -31,7 +39,7 @@ python3 -m pip --version
 
 # Upgrade pip
 echo "🐍 Upgrading pip..."
-python3 -m pip install --upgrade pip
+python3 -m pip install --index-url "$PIP_INDEX_URL" --upgrade pip
 
 # Install Python requirements for the project
 echo "📋 Installing Python dependencies globally..."
@@ -40,7 +48,7 @@ echo "📋 Installing Python dependencies globally..."
 # This improves deployment script performance by avoiding repeated installations
 if [ -f "./requirements.txt" ]; then
     echo "📦 Installing main Fabric requirements globally..."
-    python3 -m pip install -r "./requirements.txt"
+    python3 -m pip install --index-url "$PIP_INDEX_URL" -r "./requirements.txt"
     echo "✅ Main Fabric requirements installed successfully"
 else
     echo "⚠️ Warning: ./requirements.txt not found"
@@ -49,7 +57,7 @@ fi
 # Install data generation requirements (optional)
 if [ -f "./src/fabric/datagen/requirements.txt" ]; then
     echo "📦 Installing data generation requirements..."
-    python3 -m pip install -r "./src/fabric/datagen/requirements.txt"
+    python3 -m pip install --index-url "$PIP_INDEX_URL" -r "./src/fabric/datagen/requirements.txt"
     echo "✅ Data generation requirements installed successfully"
 else
     echo "ℹ️ Info: ./src/fabric/datagen/requirements.txt not found (optional)"
@@ -58,15 +66,15 @@ fi
 # Install fabric-launcher if in multi-root workspace
 if [ -d "../fabric-launcher" ]; then
     echo "📦 Installing fabric-launcher in editable mode..."
-    python3 -m pip install -e "../fabric-launcher[dev]"
+    python3 -m pip install --index-url "$PIP_INDEX_URL" -e "../fabric-launcher[dev]"
     echo "✅ fabric-launcher installed successfully"
 else
     echo "ℹ️ Info: fabric-launcher not found in workspace (optional)"
 fi
 
 # Install additional development tools
 echo "🛠️ Installing development tools..."
-if ! python3 -m pip install --user \
+if ! python3 -m pip install --index-url "$PIP_INDEX_URL" --user \
     black \
     flake8 \
     pytest \

.github/workflows/azure-dev.yml

@@ -66,6 +66,12 @@ env:
   # --- Azure region ---
   AZURE_LOCATION: ${{ vars.AZURE_LOCATION || 'eastus' }}
 
+  # --- Package feed proxy ---
+  # Routes pip installs through the Microsoft Package Feed Proxy so CI keeps working
+  # once direct access to pypi.org is blocked on Microsoft-managed devices/runners.
+  # Override via a repo/environment Variable if a different index is needed.
+  PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL || 'https://packagefeedproxy.microsoft.io/pypi/simple/' }}
+
   # --- Common ---
   # ENABLE_TELEMETRY: ${{ vars.ENABLE_TELEMETRY }}
   DEPLOYING_USER_PRINCIPAL_TYPE: ${{ vars.DEPLOYING_USER_PRINCIPAL_TYPE || 'ServicePrincipal' }}
@@ -109,10 +115,10 @@ jobs:
       resource_group_name: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
 
       - name: Setup Azure CLI
-        uses: azure/login@v3
+        uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1
         with:
           client-id: ${{ env.AZURE_CLIENT_ID }}
           tenant-id: ${{ env.AZURE_TENANT_ID }}
@@ -135,7 +141,7 @@ jobs:
           echo "✅ Bicep linting completed"
 
       - name: Install azd
-        uses: Azure/setup-azd@v2
+        uses: Azure/setup-azd@0b7e3a35ab00f2eee7080c845eb39c3f0ebfa553 # v2.4.0
 
       - name: Azure Developer CLI login
         shell: bash
@@ -222,17 +228,17 @@ jobs:
       solution_suffix: ${{ steps.get_output_linux.outputs.solution_suffix }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
 
       - name: Azure CLI login
-        uses: azure/login@v3
+        uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1
         with:
           client-id: ${{ env.AZURE_CLIENT_ID }}
           tenant-id: ${{ env.AZURE_TENANT_ID }}
           subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}
 
       - name: Install azd
-        uses: Azure/setup-azd@v2
+        uses: Azure/setup-azd@0b7e3a35ab00f2eee7080c845eb39c3f0ebfa553 # v2.4.0
 
       - name: Azure Developer CLI login
         shell: bash
@@ -390,17 +396,17 @@ jobs:
       SOLUTION_SUFFIX: ${{ needs.deploy.outputs.solution_suffix }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
 
       - name: Azure CLI login
-        uses: azure/login@v3
+        uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1
         with:
           client-id: ${{ env.AZURE_CLIENT_ID }}
           tenant-id: ${{ env.AZURE_TENANT_ID }}
           subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}
 
       - name: Install azd
-        uses: Azure/setup-azd@v2
+        uses: Azure/setup-azd@0b7e3a35ab00f2eee7080c845eb39c3f0ebfa553 # v2.4.0
 
       - name: Azure Developer CLI login
         shell: bash

.github/workflows/template-validation.yml

@@ -25,10 +25,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Validate Azure Template
-        uses: microsoft/template-validation-action@v0.4.3
+        uses: microsoft/template-validation-action@9f56864b1ddbfb56dd27f3cbc71a00750cdb9a18 # v0.4.3
         id: validation
         env:
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
@@ -38,6 +38,7 @@ jobs:
           AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DEPLOYING_USER_PRINCIPAL_TYPE: ServicePrincipal
+          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL || 'https://packagefeedproxy.microsoft.io/pypi/simple/' }}
 
       - name: Print result
         run: cat ${{ steps.validation.outputs.resultFile }}

infra/fabric/deploy/fabric_solution_installer.ipynb

@@ -202,18 +202,18 @@
     "\n",
     "    # Resolve ontology directory and detect folder structure\n",
     "    ontology_folder = f\"{ontology_name}.Ontology\"\n",
-    "    \n",
+    "\n",
     "    # Search for the ontology directory in repository root and all subfolders\n",
     "    ontology_dir = None\n",
     "    folder_path = None\n",
-    "    \n",
+    "\n",
     "    for root, dirs, files in os.walk(repository_directory):\n",
     "        if ontology_folder in dirs:\n",
     "            ontology_dir = os.path.join(root, ontology_folder)\n",
     "            # Extract folder path - look for workspace, fabric_workspace, or definitions patterns\n",
     "            # to determine the target folder structure in Fabric\n",
     "            rel_path = os.path.relpath(root, repository_directory)\n",
-    "            \n",
+    "\n",
     "            # Check if this is in a recognized workspace structure\n",
     "            path_parts = rel_path.split(os.sep)\n",
     "            if 'workspace' in path_parts:\n",
@@ -227,10 +227,10 @@
     "                if len(path_parts) > workspace_idx + 1:\n",
     "                    folder_path = '/'.join(path_parts[workspace_idx + 1:])\n",
     "            break\n",
-    "    \n",
+    "\n",
     "    if not ontology_dir:\n",
     "        raise FileNotFoundError(f\"Ontology directory '{ontology_folder}' not found in repository\")\n",
-    "    \n",
+    "\n",
     "    print(f\"   Ontology directory: {ontology_dir}\")\n",
     "    if folder_path:\n",
     "        print(f\"   Target folder: {folder_path}\")\n",
@@ -248,27 +248,27 @@
     "    print(\"2. Building lakehouse item ID mapping...\")\n",
     "    lakehouse_id_map = {}\n",
     "    list_url = f\"v1/workspaces/{workspace_id}/lakehouses\"\n",
-    "    \n",
+    "\n",
     "    while list_url:\n",
     "        list_response = client.get(list_url)\n",
     "        if list_response.status_code == 200:\n",
     "            response_data = list_response.json()\n",
     "            lakehouses = response_data.get(\"value\", [])\n",
-    "            \n",
+    "\n",
     "            for lakehouse in lakehouses:\n",
     "                if lakehouse.get(\"displayName\") == LAKEHOUSE_NAME:\n",
     "                    lakehouse_id_map[LAKEHOUSE_NAME] = lakehouse.get(\"id\")\n",
     "                    print(f\"   Found lakehouse '{LAKEHOUSE_NAME}': {lakehouse.get('id')}\")\n",
     "                    list_url = None  # Stop pagination once found\n",
     "                    break\n",
-    "            \n",
+    "\n",
     "            # Get continuation token for next page (if lakehouse not found yet)\n",
     "            if list_url and not lakehouse_id_map:\n",
     "                list_url = response_data.get(\"continuationUri\")\n",
     "        else:\n",
     "            print(f\"   ⚠️ Failed to list lakehouses: HTTP {list_response.status_code}\")\n",
     "            break\n",
-    "    \n",
+    "\n",
     "    if not lakehouse_id_map:\n",
     "        print(f\"   ⚠️ Warning: Could not find lakehouse '{LAKEHOUSE_NAME}' in workspace\")\n",
     "\n",
@@ -373,13 +373,28 @@
     "                    print(f\"   Status: {status}, retrying in {retry_after}s...\")\n",
     "            else:\n",
     "                raise Exception(f\"Failed to poll operation: {poll_response.status_code} {poll_response.text}\")\n",
+    "\n",
+    "        # A 202 create doesn't return the item body directly - fetch it from the operation result\n",
+    "        if not existing_ontology_id:\n",
+    "            result_response = client.get(f\"v1/operations/{operation_id}/result\")\n",
+    "            if result_response.status_code == 200:\n",
+    "                existing_ontology_id = result_response.json().get(\"id\")\n",
     "    elif response.status_code in (200, 201):\n",
     "        if not existing_ontology_id:\n",
     "            existing_ontology_id = response.json().get(\"id\")\n",
     "        print(f\"   Ontology {'updated' if existing_ontology_id else 'created'} successfully.\")\n",
     "    else:\n",
     "        raise Exception(f\"API call failed: {response.status_code} {response.text}\")\n",
     "\n",
+    "    # Fallback: look up the item by name if the id still wasn't resolved above\n",
+    "    if not existing_ontology_id:\n",
+    "        lookup_response = client.get(f\"v1/workspaces/{workspace_id}/items?type=Ontology\")\n",
+    "        if lookup_response.status_code == 200:\n",
+    "            for item in lookup_response.json().get(\"value\", []):\n",
+    "                if item[\"displayName\"] == ontology_name:\n",
+    "                    existing_ontology_id = item[\"id\"]\n",
+    "                    break\n",
+    "\n",
     "    # Step 7: Move ontology to the appropriate folder if needed\n",
     "    if folder_path:\n",
     "        print(f\"7. Moving ontology to folder '{folder_path}'...\")\n",
@@ -433,7 +448,7 @@
     "\n",
     "def _replace_sql_endpoints(obj, lakehouse_sql_endpoint_map):\n",
     "    \"\"\"Recursively replace SQL endpoint values based on the sibling itemId → lakehouse endpoint mapping.\n",
-    "    \n",
+    "\n",
     "    This function can replace various endpoint-related fields in the ontology definition:\n",
     "    - sqlEndpoint: SQL endpoint connection string\n",
     "    - connectionString: Alternative field name for SQL endpoints\n",
@@ -449,7 +464,7 @@
     "                # Replace connectionString if present\n",
     "                if \"connectionString\" in obj:\n",
     "                    obj[\"connectionString\"] = lakehouse_sql_endpoint_map[item_id]\n",
-    "        \n",
+    "\n",
     "        # Recurse into nested dictionaries\n",
     "        for value in obj.values():\n",
     "            _replace_sql_endpoints(value, lakehouse_sql_endpoint_map)\n",
@@ -469,7 +484,70 @@
     "    deployed_ontology_ids.append((ontology_name, ontology_id))\n",
     "    print(f\"✅ Ontology '{ontology_name}' deployed successfully (ID: {ontology_id})\")\n",
     "\n",
-    "print(f\"\\n✅ All {len(deployed_ontology_ids)} ontologies deployed successfully\")"
+    "print(f\"\\n✅ All {len(deployed_ontology_ids)} ontologies deployed successfully\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69e7e7b2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def check_graph_model_ready(ontology_name,\n",
+    "                             client=client,\n",
+    "                             workspace_id=workspace_id,\n",
+    "                             initial_wait_seconds=360,\n",
+    "                             attempts=3,\n",
+    "                             wait_seconds=30):\n",
+    "    \"\"\"Check whether the ontology's graph model has finished its automatic initial load,\n",
+    "    and actively retry the RefreshGraph job if Fabric's own background refresh failed\n",
+    "    (e.g. error code GraphNotRefreshable, which often clears once the lakehouse SQL\n",
+    "    endpoint has finished syncing).\n",
+    "    \"\"\"\n",
+    "    graph_models_response = client.get(f\"v1/workspaces/{workspace_id}/graphModels\")\n",
+    "    if graph_models_response.status_code != 200:\n",
+    "        print(f\"  ⚠️ Failed to list graph models for '{ontology_name}': HTTP {graph_models_response.status_code} {graph_models_response.text}\")\n",
+    "        return\n",
+    "    graph_models = graph_models_response.json().get(\"value\", [])\n",
+    "    match = next((g for g in graph_models if g.get(\"displayName\", \"\").startswith(f\"{ontology_name}_graph_\")), None)\n",
+    "\n",
+    "    if not match:\n",
+    "        print(f\"   ⚠️ Could not find a graph model for '{ontology_name}'; skipping readiness check\")\n",
+    "        return\n",
+    "\n",
+    "    print(f\"⏳ Waiting {initial_wait_seconds}s for '{ontology_name}' graph model to finish its initial load...\")\n",
+    "    time.sleep(initial_wait_seconds)\n",
+    "\n",
+    "    for attempt in range(1, attempts + 1):\n",
+    "        response = client.post(f\"v1/workspaces/{workspace_id}/graphModels/{match['id']}/jobs/instances?jobType=RefreshGraph\")\n",
+    "\n",
+    "        job_status = None\n",
+    "        if response.status_code == 202:\n",
+    "            job_url = response.headers.get(\"Location\")\n",
+    "            elapsed = 0\n",
+    "            job_status = \"NotStarted\"\n",
+    "            while job_status in (\"NotStarted\", \"InProgress\") and elapsed < 300:\n",
+    "                time.sleep(wait_seconds)\n",
+    "                elapsed += wait_seconds\n",
+    "                job_status = client.get(job_url).json().get(\"status\")\n",
+    "        elif response.status_code == 200:\n",
+    "            job_status = \"Completed\"\n",
+    "\n",
+    "        if job_status == \"Completed\":\n",
+    "            print(f\"   ✅ Graph model refreshed successfully (attempt {attempt})\")\n",
+    "            return\n",
+    "\n",
+    "        print(f\"   Attempt {attempt} refresh not successful yet (status: {job_status or response.status_code})\")\n",
+    "        if attempt < attempts:\n",
+    "            time.sleep(wait_seconds)\n",
+    "\n",
+    "    print(f\"   ⚠️ Graph model refresh did not succeed after {attempts} attempts.\")\n",
+    "    print(f\"      Verify the ontology's mapped lakehouse tables contain data, then retry the refresh from the portal.\")\n",
+    "\n",
+    "\n",
+    "for ontology_name, _ in deployed_ontology_ids:\n",
+    "    check_graph_model_ready(ontology_name)"
    ]
   },
   {
@@ -493,6 +571,30 @@
     "print(\"✅ Data Agent items deployed successfully\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0c6ad359",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Publish the deployed Data Agent so it's live and usable outside the workspace\n",
+    "\n",
+    "print(\"📣 Publishing Data Agent...\")\n",
+    "\n",
+    "data_agents = client.get(f\"v1/workspaces/{workspace_id}/dataAgents\").json().get(\"value\", [])\n",
+    "agent = next((a for a in data_agents if a.get(\"displayName\") == \"RetailSC Ontology Agent\"), None)\n",
+    "\n",
+    "if not agent:\n",
+    "    print(\"   ⚠️ Could not find the Data Agent to publish; skipping\")\n",
+    "else:\n",
+    "    response = client.post(\n",
+    "        f\"v1/workspaces/{workspace_id}/dataAgents/{agent['id']}/staging/publish\",\n",
+    "        json={\"publishedDescription\": \"Published by the Microsoft IQ solution installer\"},\n",
+    "    )\n",
+    "    print(f\"   ✅ Data Agent '{agent['displayName']}' published successfully\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,