From 2a5494c882c9e79563213471b2b934de454be47f Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 11:05:02 -0700 Subject: [PATCH 01/10] Disable field_caps to avoid large responses - Disabled getFields() method that calls _field_caps API - Returns empty array to skip autocomplete feature - Fixes performance issues with indices containing many dynamic fields - Related to https://github.com/quickwit-oss/quickwit-datasource/issues/172 --- src/datasource/base.ts | 121 +++++++++++++++++++++-------------------- 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/src/datasource/base.ts b/src/datasource/base.ts index b0abba7..09ccf9d 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -332,65 +332,68 @@ export class BaseQuickwitDataSource } getFields(spec: FieldCapsSpec = {}): Observable { - const range = spec.range || getDefaultTimeRange(); - return from( - this.getResource('_elastic/' + this.index + '/_field_caps', { - start_timestamp: Math.floor(range.from.valueOf() / SECOND), - end_timestamp: Math.ceil(range.to.valueOf() / SECOND), - }) - ).pipe( - map((field_capabilities_response: FieldCapabilitiesResponse) => { - // Cache field → type on the datasource for modifyQuery to consult. - // Quickwit routes phrase queries to the text variant first on multi-indexed - // fields (text+keyword), and text fields don't index positions by default. - // So prefer 'text' when present — it drives safer operator choices downstream. - for (const [name, caps] of Object.entries(field_capabilities_response.fields)) { - const typeKeys = Object.keys(caps); - const chosen = typeKeys.includes('text') ? 'text' : typeKeys[0]; - if (chosen) { - this.fieldTypes[name] = chosen; - } - } - - const shouldAddField = (field: any) => { - if (spec.aggregatable !== undefined && field.aggregatable !== spec.aggregatable) { - return false; - } - if (spec.searchable !== undefined && field.searchable !== spec.searchable) { - return false; - } - if ( - spec.type && - spec.type.length !== 0 && - !(spec.type.includes(field.type) || spec.type.includes(fieldTypeMap[field.type])) - ) { - return false; - } - return true; - }; - const fieldCapabilities = Object.entries(field_capabilities_response.fields) - .flatMap(([field_name, field_capabilities]) => { - return Object.values(field_capabilities).map((field_capability) => { - field_capability.field_name = field_name; - return field_capability; - }); - }) - .filter(shouldAddField) - .map((field_capability) => { - return { - text: field_capability.field_name, - type: fieldTypeMap[field_capability.type], - }; - }); - const uniquefieldCapabilities = fieldCapabilities - .filter( - (field_capability, index, self) => - index === self.findIndex((t) => t.text === field_capability.text && t.type === field_capability.type) - ) - .sort((a, b) => a.text.localeCompare(b.text)); - return uniquefieldCapabilities; - }) - ); + // PATCH: Disable field_caps to avoid large responses on indices with many dynamic fields + return of([]); + + // DISABLED: const range = spec.range || getDefaultTimeRange(); + // DISABLED: return from( + // DISABLED: this.getResource('_elastic/' + this.index + '/_field_caps', { + // DISABLED: start_timestamp: Math.floor(range.from.valueOf() / SECOND), + // DISABLED: end_timestamp: Math.ceil(range.to.valueOf() / SECOND), + // DISABLED: }) + // DISABLED: ).pipe( + // DISABLED: map((field_capabilities_response: FieldCapabilitiesResponse) => { + // DISABLED: // Cache field → type on the datasource for modifyQuery to consult. + // DISABLED: // Quickwit routes phrase queries to the text variant first on multi-indexed + // DISABLED: // fields (text+keyword), and text fields don't index positions by default. + // DISABLED: // So prefer 'text' when present — it drives safer operator choices downstream. + // DISABLED: for (const [name, caps] of Object.entries(field_capabilities_response.fields)) { + // DISABLED: const typeKeys = Object.keys(caps); + // DISABLED: const chosen = typeKeys.includes('text') ? 'text' : typeKeys[0]; + // DISABLED: if (chosen) { + // DISABLED: this.fieldTypes[name] = chosen; + // DISABLED: } + // DISABLED: } + // DISABLED: + // DISABLED: const shouldAddField = (field: any) => { + // DISABLED: if (spec.aggregatable !== undefined && field.aggregatable !== spec.aggregatable) { + // DISABLED: return false; + // DISABLED: } + // DISABLED: if (spec.searchable !== undefined && field.searchable !== spec.searchable) { + // DISABLED: return false; + // DISABLED: } + // DISABLED: if ( + // DISABLED: spec.type && + // DISABLED: spec.type.length !== 0 && + // DISABLED: !(spec.type.includes(field.type) || spec.type.includes(fieldTypeMap[field.type])) + // DISABLED: ) { + // DISABLED: return false; + // DISABLED: } + // DISABLED: return true; + // DISABLED: }; + // DISABLED: const fieldCapabilities = Object.entries(field_capabilities_response.fields) + // DISABLED: .flatMap(([field_name, field_capabilities]) => { + // DISABLED: return Object.values(field_capabilities).map((field_capability) => { + // DISABLED: field_capability.field_name = field_name; + // DISABLED: return field_capability; + // DISABLED: }); + // DISABLED: }) + // DISABLED: .filter(shouldAddField) + // DISABLED: .map((field_capability) => { + // DISABLED: return { + // DISABLED: text: field_capability.field_name, + // DISABLED: type: fieldTypeMap[field_capability.type], + // DISABLED: }; + // DISABLED: }); + // DISABLED: const uniquefieldCapabilities = fieldCapabilities + // DISABLED: .filter( + // DISABLED: (field_capability, index, self) => + // DISABLED: index === self.findIndex((t) => t.text === field_capability.text && t.type === field_capability.type) + // DISABLED: ) + // DISABLED: .sort((a, b) => a.text.localeCompare(b.text)); + // DISABLED: return uniquefieldCapabilities; + // DISABLED: }) + // DISABLED: ); } /** From 366b26a90cde04365d7976e06bc7331a90b2dea5 Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 11:08:26 -0700 Subject: [PATCH 02/10] Add CI/CD workflow to build and push to ECR - Builds patched Quickwit datasource plugin - Removes signature files for patched version - Builds Grafana Docker image with patched plugin - Pushes to ECR repository: grafana-quickwit - Triggers on push to disable-field-caps-all-fields branch or tags --- .github/workflows/build-and-push.yml | 178 +++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .github/workflows/build-and-push.yml diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml new file mode 100644 index 0000000..8411e7f --- /dev/null +++ b/.github/workflows/build-and-push.yml @@ -0,0 +1,178 @@ +name: Build and Push Grafana with Quickwit Plugin + +on: + push: + branches: + - disable-field-caps-all-fields + - main + tags: + - 'v*' + workflow_dispatch: + +env: + GRAFANA_VERSION: 12.4.0 + AWS_REGION: us-east-1 + ECR_REPOSITORY: grafana-quickwit + +jobs: + build-plugin: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.21' + + - name: Install dependencies + run: npm ci + + - name: Build frontend + run: npm run build + + - name: Build backend binaries + run: | + mage -v buildAll || echo "Mage not available, using go build" + if [ ! -f dist/gpx_quickwit_linux_amd64 ]; then + cd pkg + GOOS=linux GOARCH=amd64 go build -o ../dist/gpx_quickwit_linux_amd64 . + GOOS=linux GOARCH=arm64 go build -o ../dist/gpx_quickwit_linux_arm64 . + fi + + - name: Remove signature files for patched plugin + run: | + cd dist + rm -f MANIFEST.txt + jq 'del(.signature)' plugin.json > plugin.json.tmp && mv plugin.json.tmp plugin.json + + - name: Package plugin + run: | + cd dist + zip -r quickwit-quickwit-datasource-patched.zip . -x "*.zip" + + - name: Upload plugin artifact + uses: actions/upload-artifact@v4 + with: + name: quickwit-plugin-patched + path: dist/quickwit-quickwit-datasource-patched.zip + retention-days: 30 + + build-and-push-image: + needs: build-plugin + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download plugin artifact + uses: actions/download-artifact@v4 + with: + name: quickwit-plugin-patched + path: ./plugin + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Generate image tags + id: meta + run: | + BRANCH_NAME=${GITHUB_REF#refs/heads/} + SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7) + + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + TAG_VERSION=${GITHUB_REF#refs/tags/v} + IMAGE_TAG="${GRAFANA_VERSION}-quickwit-${TAG_VERSION}" + else + IMAGE_TAG="${GRAFANA_VERSION}-quickwit-0.6.0-patched-${SHORT_SHA}" + fi + + echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + + if [[ "${{ github.ref }}" == "refs/heads/main" || "${{ github.ref }}" == refs/tags/* ]]; then + TAGS="${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${IMAGE_TAG}" + TAGS="${TAGS},${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:latest" + else + TAGS="${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${IMAGE_TAG}" + fi + + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + + - name: Create Dockerfile + run: | + cat > Dockerfile <<'EOF' + FROM grafana/grafana:${{ env.GRAFANA_VERSION }} + + USER root + + # Install patched Quickwit plugin + COPY plugin/quickwit-quickwit-datasource-patched.zip /tmp/plugin.zip + RUN set -ex && \ + mkdir -p /var/lib/grafana/plugins && \ + cd /var/lib/grafana/plugins && \ + unzip -q /tmp/plugin.zip -d quickwit-quickwit-datasource && \ + rm /tmp/plugin.zip && \ + chown -R 472:0 /var/lib/grafana/plugins + + USER grafana + + ENV GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=quickwit-quickwit-datasource + + LABEL org.opencontainers.image.source="https://github.com/Iterable/quickwit-datasource" + LABEL org.opencontainers.image.description="Grafana with patched Quickwit datasource plugin (field_caps disabled)" + LABEL grafana.version="${{ env.GRAFANA_VERSION }}" + LABEL quickwit.plugin.version="0.6.0-patched" + + EXPOSE 3000 + + HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1 + EOF + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: false + + - name: Output image details + run: | + echo "### Docker Image Pushed 🚀" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Registry**: ${{ steps.login-ecr.outputs.registry }}" >> $GITHUB_STEP_SUMMARY + echo "**Repository**: ${{ env.ECR_REPOSITORY }}" >> $GITHUB_STEP_SUMMARY + echo "**Tags**:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Grafana Version**: ${{ env.GRAFANA_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "**Quickwit Plugin**: 0.6.0-patched (field_caps disabled)" >> $GITHUB_STEP_SUMMARY From 4160f635d5c4f4ddc630505fa4b21ddd6aef1c46 Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 11:08:56 -0700 Subject: [PATCH 03/10] Add CI/CD setup documentation --- CI-CD-SETUP.md | 126 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 CI-CD-SETUP.md diff --git a/CI-CD-SETUP.md b/CI-CD-SETUP.md new file mode 100644 index 0000000..6ba9360 --- /dev/null +++ b/CI-CD-SETUP.md @@ -0,0 +1,126 @@ +# CI/CD Setup for Grafana Quickwit Image + +This repository includes a GitHub Actions workflow that automatically builds and pushes a Grafana Docker image with the patched Quickwit datasource plugin to ECR. + +## Required GitHub Secrets + +The workflow requires the following secret to be configured in the repository: + +### `AWS_ROLE_ARN` +AWS IAM Role ARN with permissions to push to ECR. + +**Example format**: `arn:aws:iam::337909757619:role/github-actions-ecr-push` + +**Required Permissions**: +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload" + ], + "Resource": [ + "arn:aws:ecr:us-east-1:337909757619:repository/grafana-quickwit" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken" + ], + "Resource": "*" + } + ] +} +``` + +## Setting up the Secret + +1. Go to the repository on GitHub: https://github.com/Iterable/quickwit-datasource +2. Navigate to **Settings** → **Secrets and variables** → **Actions** +3. Click **New repository secret** +4. Name: `AWS_ROLE_ARN` +5. Value: The ARN of your IAM role (e.g., `arn:aws:iam::337909757619:role/github-actions-ecr-push`) +6. Click **Add secret** + +## IAM Role Trust Policy + +The IAM role must trust GitHub Actions from the Iterable organization: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::337909757619:oidc-provider/token.actions.githubusercontent.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" + }, + "StringLike": { + "token.actions.githubusercontent.com:sub": "repo:Iterable/quickwit-datasource:*" + } + } + } + ] +} +``` + +## Workflow Triggers + +The workflow runs on: +- **Push** to `disable-field-caps-all-fields` branch +- **Push** to `main` branch +- **Tags** matching `v*` pattern +- **Manual** trigger via workflow_dispatch + +## Image Tags + +Images are tagged as: +- `-quickwit--` for branch builds +- `-quickwit-` for tag builds +- `latest` for main branch or tag builds + +**Example**: `12.4.0-quickwit-0.6.0-patched-a1b2c3d` + +## Target ECR Repository + +- **Repository**: `grafana-quickwit` +- **Region**: `us-east-1` +- **Registry**: `337909757619.dkr.ecr.us-east-1.amazonaws.com` + +## Verifying the Workflow + +After setting up the secret, the workflow will run automatically on the next push. You can also trigger it manually: + +1. Go to **Actions** tab +2. Select **Build and Push Grafana with Quickwit Plugin** +3. Click **Run workflow** +4. Select the branch and click **Run workflow** + +## Troubleshooting + +**Error: Unable to locate credentials** +- Verify the `AWS_ROLE_ARN` secret is set correctly +- Check that the IAM role exists and the ARN is correct + +**Error: AccessDenied** +- Verify the IAM role has the correct permissions policy +- Verify the IAM role's trust policy allows GitHub Actions from this repository + +**Error: Repository does not exist** +- Verify the ECR repository `grafana-quickwit` exists in `us-east-1` +- Check the repository name in the workflow matches exactly From 43058adc4620b6df0ad82a89d712fb0b9913c2ff Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 11:11:52 -0700 Subject: [PATCH 04/10] Update CI/CD to use gha-runner-ecr-publish like Backstage - Uses gha-runner-ecr-publish runner with existing AWS credentials - Follows Backstage pattern for ECR authentication - No need for AWS_ROLE_ARN secret - Builds on push to branches or PR merge - Manual workflow dispatch available --- .github/workflows/build-and-push.yml | 182 +++++++++++++-------------- 1 file changed, 89 insertions(+), 93 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 8411e7f..1aac9cc 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -5,28 +5,52 @@ on: branches: - disable-field-caps-all-fields - main - tags: - - 'v*' + pull_request: + types: [closed] workflow_dispatch: + inputs: + force_publish: + description: 'Force publish image' + required: false + type: boolean + default: false env: GRAFANA_VERSION: 12.4.0 - AWS_REGION: us-east-1 + AWS_REGION_MGT: us-east-1 + DOCKER_REGISTRY_MGT: 337909757619.dkr.ecr.us-east-1.amazonaws.com ECR_REPOSITORY: grafana-quickwit jobs: - build-plugin: - runs-on: ubuntu-latest + build-and-publish: + name: Build and Publish Grafana Quickwit Image + runs-on: gha-runner-ecr-publish + outputs: + githash: ${{ steps.metadata.outputs.githash }} + image_tag: ${{ steps.metadata.outputs.image_tag }} steps: - name: Checkout code uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Generate metadata + id: metadata + run: | + SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7) + IMAGE_TAG="${GRAFANA_VERSION}-quickwit-0.6.0-patched-${SHORT_SHA}" + + echo "githash=${{ github.sha }}" >> $GITHUB_OUTPUT + echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + + echo "Image will be tagged as: ${IMAGE_TAG}" - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: '20' - cache: 'npm' - name: Setup Go uses: actions/setup-go@v5 @@ -41,82 +65,34 @@ jobs: - name: Build backend binaries run: | - mage -v buildAll || echo "Mage not available, using go build" - if [ ! -f dist/gpx_quickwit_linux_amd64 ]; then + # Try using mage if available, otherwise use go build directly + if command -v mage &> /dev/null; then + mage -v buildAll + else + echo "Mage not available, building with go directly" cd pkg GOOS=linux GOARCH=amd64 go build -o ../dist/gpx_quickwit_linux_amd64 . GOOS=linux GOARCH=arm64 go build -o ../dist/gpx_quickwit_linux_arm64 . + cd .. fi - name: Remove signature files for patched plugin run: | cd dist rm -f MANIFEST.txt - jq 'del(.signature)' plugin.json > plugin.json.tmp && mv plugin.json.tmp plugin.json + if [ -f plugin.json ]; then + # Remove signature field from plugin.json + jq 'del(.signature)' plugin.json > plugin.json.tmp && mv plugin.json.tmp plugin.json + fi - name: Package plugin run: | cd dist zip -r quickwit-quickwit-datasource-patched.zip . -x "*.zip" + ls -lh quickwit-quickwit-datasource-patched.zip - - name: Upload plugin artifact - uses: actions/upload-artifact@v4 - with: - name: quickwit-plugin-patched - path: dist/quickwit-quickwit-datasource-patched.zip - retention-days: 30 - - build-and-push-image: - needs: build-plugin - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Download plugin artifact - uses: actions/download-artifact@v4 - with: - name: quickwit-plugin-patched - path: ./plugin - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Generate image tags - id: meta - run: | - BRANCH_NAME=${GITHUB_REF#refs/heads/} - SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7) - - if [[ "${{ github.ref }}" == refs/tags/* ]]; then - TAG_VERSION=${GITHUB_REF#refs/tags/v} - IMAGE_TAG="${GRAFANA_VERSION}-quickwit-${TAG_VERSION}" - else - IMAGE_TAG="${GRAFANA_VERSION}-quickwit-0.6.0-patched-${SHORT_SHA}" - fi - - echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT - echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT - - if [[ "${{ github.ref }}" == "refs/heads/main" || "${{ github.ref }}" == refs/tags/* ]]; then - TAGS="${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${IMAGE_TAG}" - TAGS="${TAGS},${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:latest" - else - TAGS="${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${IMAGE_TAG}" - fi - - echo "tags=${TAGS}" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Create Dockerfile run: | @@ -126,7 +102,7 @@ jobs: USER root # Install patched Quickwit plugin - COPY plugin/quickwit-quickwit-datasource-patched.zip /tmp/plugin.zip + COPY dist/quickwit-quickwit-datasource-patched.zip /tmp/plugin.zip RUN set -ex && \ mkdir -p /var/lib/grafana/plugins && \ cd /var/lib/grafana/plugins && \ @@ -142,6 +118,7 @@ jobs: LABEL org.opencontainers.image.description="Grafana with patched Quickwit datasource plugin (field_caps disabled)" LABEL grafana.version="${{ env.GRAFANA_VERSION }}" LABEL quickwit.plugin.version="0.6.0-patched" + LABEL githash="${{ steps.metadata.outputs.githash }}" EXPOSE 3000 @@ -149,30 +126,49 @@ jobs: CMD wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1 EOF - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Build Docker image + run: | + docker buildx build \ + --platform linux/amd64 \ + --load \ + --tag ${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} \ + --tag ${{ env.ECR_REPOSITORY }}:latest \ + -f Dockerfile . + + - name: Publish to ECR + id: publish + if: | + github.event_name == 'workflow_dispatch' && github.event.inputs.force_publish == 'true' || + github.event.action == 'closed' && github.event.pull_request.merged == true || + github.ref == 'refs/heads/main' || + github.ref == 'refs/heads/disable-field-caps-all-fields' + run: | + aws ecr get-login-password --region $AWS_REGION_MGT | docker login --username AWS --password-stdin $DOCKER_REGISTRY_MGT - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max - provenance: false - - - name: Output image details + docker tag ${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} \ + $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} + + docker tag ${{ env.ECR_REPOSITORY }}:latest \ + $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:latest + + docker push $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} + docker push $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:latest + + SUMMARY=$'# Published Grafana Quickwit Image to ECR\n' + SUMMARY+=$'## Images\n' + SUMMARY+=$'* '"$DOCKER_REGISTRY_MGT"'/${{ env.ECR_REPOSITORY }}:'"${{ steps.metadata.outputs.image_tag }}"$'\n' + SUMMARY+=$'* '"$DOCKER_REGISTRY_MGT"'/${{ env.ECR_REPOSITORY }}:latest'$'\n' + SUMMARY+=$'\n## Details\n' + SUMMARY+=$'* **Grafana Version**: ${{ env.GRAFANA_VERSION }}\n' + SUMMARY+=$'* **Quickwit Plugin**: 0.6.0-patched (field_caps disabled)\n' + SUMMARY+=$'* **Git Hash**: ${{ steps.metadata.outputs.githash }}\n' + echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY + + - name: Build Summary (No Publish) + if: steps.publish.outcome == 'skipped' run: | - echo "### Docker Image Pushed 🚀" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Registry**: ${{ steps.login-ecr.outputs.registry }}" >> $GITHUB_STEP_SUMMARY - echo "**Repository**: ${{ env.ECR_REPOSITORY }}" >> $GITHUB_STEP_SUMMARY - echo "**Tags**:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Grafana Version**: ${{ env.GRAFANA_VERSION }}" >> $GITHUB_STEP_SUMMARY - echo "**Quickwit Plugin**: 0.6.0-patched (field_caps disabled)" >> $GITHUB_STEP_SUMMARY + SUMMARY=$'# Built Grafana Quickwit Image (Not Published)\n' + SUMMARY+=$'## Image Tag\n' + SUMMARY+=$'* ${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }}\n' + SUMMARY+=$'\n_Image was built but not published to ECR. Publish occurs on PR merge or manual workflow dispatch._\n' + echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY From c2cf8c11a7e0063510788d73d46a2de5c16f4196 Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 11:12:21 -0700 Subject: [PATCH 05/10] Update CI/CD docs - no secrets needed with gha-runner --- CI-CD-SETUP.md | 209 +++++++++++++++++++++++++------------------------ 1 file changed, 107 insertions(+), 102 deletions(-) diff --git a/CI-CD-SETUP.md b/CI-CD-SETUP.md index 6ba9360..a5cf6cf 100644 --- a/CI-CD-SETUP.md +++ b/CI-CD-SETUP.md @@ -2,125 +2,130 @@ This repository includes a GitHub Actions workflow that automatically builds and pushes a Grafana Docker image with the patched Quickwit datasource plugin to ECR. -## Required GitHub Secrets - -The workflow requires the following secret to be configured in the repository: - -### `AWS_ROLE_ARN` -AWS IAM Role ARN with permissions to push to ECR. - -**Example format**: `arn:aws:iam::337909757619:role/github-actions-ecr-push` - -**Required Permissions**: -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:PutImage", - "ecr:InitiateLayerUpload", - "ecr:UploadLayerPart", - "ecr:CompleteLayerUpload" - ], - "Resource": [ - "arn:aws:ecr:us-east-1:337909757619:repository/grafana-quickwit" - ] - }, - { - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken" - ], - "Resource": "*" - } - ] -} -``` +## Overview -## Setting up the Secret - -1. Go to the repository on GitHub: https://github.com/Iterable/quickwit-datasource -2. Navigate to **Settings** → **Secrets and variables** → **Actions** -3. Click **New repository secret** -4. Name: `AWS_ROLE_ARN` -5. Value: The ARN of your IAM role (e.g., `arn:aws:iam::337909757619:role/github-actions-ecr-push`) -6. Click **Add secret** - -## IAM Role Trust Policy - -The IAM role must trust GitHub Actions from the Iterable organization: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::337909757619:oidc-provider/token.actions.githubusercontent.com" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" - }, - "StringLike": { - "token.actions.githubusercontent.com:sub": "repo:Iterable/quickwit-datasource:*" - } - } - } - ] -} -``` +The workflow uses the **`gha-runner-ecr-publish`** self-hosted runner which already has AWS credentials configured. **No additional secrets are required.** ## Workflow Triggers -The workflow runs on: -- **Push** to `disable-field-caps-all-fields` branch -- **Push** to `main` branch -- **Tags** matching `v*` pattern -- **Manual** trigger via workflow_dispatch - -## Image Tags - -Images are tagged as: -- `-quickwit--` for branch builds -- `-quickwit-` for tag builds -- `latest` for main branch or tag builds +The workflow runs and **publishes to ECR** on: +- **Push** to `disable-field-caps-all-fields` or `main` branches +- **PR merge** to these branches +- **Manual** workflow dispatch with `force_publish` option -**Example**: `12.4.0-quickwit-0.6.0-patched-a1b2c3d` +The workflow **builds but does not publish** on: +- Pull request events (for testing) +- Other branch pushes -## Target ECR Repository +## Image Details +### Target ECR Repository - **Repository**: `grafana-quickwit` - **Region**: `us-east-1` - **Registry**: `337909757619.dkr.ecr.us-east-1.amazonaws.com` -## Verifying the Workflow +### Image Tags +Images are tagged as: +- `-quickwit--` + - Example: `12.4.0-quickwit-0.6.0-patched-a1b2c3d` +- `latest` (always points to the most recent build) + +### Image Contents +- **Base**: Grafana 12.4.0 +- **Plugin**: Quickwit datasource v0.6.0 (patched to disable field_caps) +- **Platform**: linux/amd64 + +## What the Workflow Does + +1. **Build Plugin** + - Installs Node.js and Go dependencies + - Builds frontend (TypeScript → JavaScript) + - Builds backend (Go binaries for Linux) + - Removes signature files (since plugin is patched) + - Packages as ZIP + +2. **Build Docker Image** + - Creates Dockerfile dynamically + - Copies patched plugin into Grafana base image + - Configures unsigned plugin loading + - Adds metadata labels + +3. **Publish to ECR** (conditional) + - Authenticates to ECR using runner's AWS credentials + - Tags image with git hash and `latest` + - Pushes both tags to ECR + - Generates build summary + +## Running the Workflow + +### Automatic (Recommended) +Just push commits to `disable-field-caps-all-fields` branch: +```bash +git push origin disable-field-caps-all-fields +``` -After setting up the secret, the workflow will run automatically on the next push. You can also trigger it manually: +The workflow will automatically build and push to ECR. -1. Go to **Actions** tab +### Manual Trigger +1. Go to the **Actions** tab in GitHub 2. Select **Build and Push Grafana with Quickwit Plugin** 3. Click **Run workflow** -4. Select the branch and click **Run workflow** +4. Select branch: `disable-field-caps-all-fields` +5. Check **force_publish** if you want to publish to ECR +6. Click **Run workflow** + +## Verifying the Build + +After the workflow completes: + +1. **Check GitHub Actions**: The workflow summary will show the published image tags +2. **Check ECR**: + ```bash + aws ecr describe-images \ + --repository-name grafana-quickwit \ + --region us-east-1 \ + --query 'sort_by(imageDetails,& imagePushedAt)[-5:]' \ + --output table + ``` + +## Using the Image + +Once published, reference the image in your deployments: + +```yaml +# Using specific version +image: 337909757619.dkr.ecr.us-east-1.amazonaws.com/grafana-quickwit:12.4.0-quickwit-0.6.0-patched-a1b2c3d + +# Or using latest +image: 337909757619.dkr.ecr.us-east-1.amazonaws.com/grafana-quickwit:latest +``` ## Troubleshooting -**Error: Unable to locate credentials** -- Verify the `AWS_ROLE_ARN` secret is set correctly -- Check that the IAM role exists and the ARN is correct +### Build Fails on Plugin Build +- Check Node.js and Go versions in the workflow match requirements +- Review build logs for npm or go errors + +### Docker Build Fails +- Verify the Grafana base image version exists +- Check that plugin ZIP was created successfully + +### ECR Push Fails +- Verify the `gha-runner-ecr-publish` runner has ECR write permissions +- Check that the ECR repository `grafana-quickwit` exists +- Verify AWS credentials on the runner are valid + +### Workflow Doesn't Trigger +- Ensure you're pushing to the correct branch +- Check workflow file syntax in `.github/workflows/build-and-push.yml` +- Verify GitHub Actions are enabled for the repository + +## Comparing with Backstage Setup -**Error: AccessDenied** -- Verify the IAM role has the correct permissions policy -- Verify the IAM role's trust policy allows GitHub Actions from this repository +This workflow follows the same pattern as `Iterable/backstage`: +- Uses `gha-runner-ecr-publish` runner +- Authenticates with `aws ecr get-login-password` +- Conditionally publishes based on event type +- Generates summary with published tags -**Error: Repository does not exist** -- Verify the ECR repository `grafana-quickwit` exists in `us-east-1` -- Check the repository name in the workflow matches exactly +No IAM roles or GitHub secrets are required because the self-hosted runner already has the necessary AWS permissions. From 13e25c697c16ced33bc074690b0dc65df775cabb Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 11:18:35 -0700 Subject: [PATCH 06/10] Remove 'latest' tag, use immutable version tags only - Removed 'latest' tag for production safety - Uses git tag if available (e.g., v0.6.0-patched-1) - Falls back to SHA-based tag for untagged commits - Matches best practice for prod/preprod deployments --- .github/workflows/build-and-push.yml | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 1aac9cc..b3631d8 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -39,7 +39,16 @@ jobs: id: metadata run: | SHORT_SHA=$(echo ${{ github.sha }} | cut -c1-7) - IMAGE_TAG="${GRAFANA_VERSION}-quickwit-0.6.0-patched-${SHORT_SHA}" + + # Use git tag if available, otherwise use short SHA + if git describe --exact-match --tags HEAD 2>/dev/null; then + GIT_TAG=$(git describe --exact-match --tags HEAD) + # Strip 'v' prefix if present + VERSION=${GIT_TAG#v} + IMAGE_TAG="${GRAFANA_VERSION}-quickwit-${VERSION}" + else + IMAGE_TAG="${GRAFANA_VERSION}-quickwit-0.6.0-patched-${SHORT_SHA}" + fi echo "githash=${{ github.sha }}" >> $GITHUB_OUTPUT echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT @@ -141,23 +150,24 @@ jobs: github.event_name == 'workflow_dispatch' && github.event.inputs.force_publish == 'true' || github.event.action == 'closed' && github.event.pull_request.merged == true || github.ref == 'refs/heads/main' || - github.ref == 'refs/heads/disable-field-caps-all-fields' + github.ref == 'refs/heads/disable-field-caps-all-fields' || + startsWith(github.ref, 'refs/tags/') run: | aws ecr get-login-password --region $AWS_REGION_MGT | docker login --username AWS --password-stdin $DOCKER_REGISTRY_MGT docker tag ${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} \ $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} - docker tag ${{ env.ECR_REPOSITORY }}:latest \ - $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:latest - docker push $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }} - docker push $DOCKER_REGISTRY_MGT/${{ env.ECR_REPOSITORY }}:latest SUMMARY=$'# Published Grafana Quickwit Image to ECR\n' - SUMMARY+=$'## Images\n' - SUMMARY+=$'* '"$DOCKER_REGISTRY_MGT"'/${{ env.ECR_REPOSITORY }}:'"${{ steps.metadata.outputs.image_tag }}"$'\n' - SUMMARY+=$'* '"$DOCKER_REGISTRY_MGT"'/${{ env.ECR_REPOSITORY }}:latest'$'\n' + SUMMARY+=$'## Image\n' + SUMMARY+=$'```\n' + SUMMARY+=$''$DOCKER_REGISTRY_MGT'/${{ env.ECR_REPOSITORY }}:${{ steps.metadata.outputs.image_tag }}\n' + SUMMARY+=$'```\n' + SUMMARY+=$'\n## Usage in Deployments\n' + SUMMARY+=$'**Preprod**: Update gitops to use this tag for testing\n' + SUMMARY+=$'**Prod**: Promote this tag after preprod validation\n' SUMMARY+=$'\n## Details\n' SUMMARY+=$'* **Grafana Version**: ${{ env.GRAFANA_VERSION }}\n' SUMMARY+=$'* **Quickwit Plugin**: 0.6.0-patched (field_caps disabled)\n' From b68b48151a69b48de55994fb502690e45bf7433e Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 11:18:52 -0700 Subject: [PATCH 07/10] Document immutable tagging strategy --- CI-CD-SETUP.md | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/CI-CD-SETUP.md b/CI-CD-SETUP.md index a5cf6cf..4233dbc 100644 --- a/CI-CD-SETUP.md +++ b/CI-CD-SETUP.md @@ -25,10 +25,23 @@ The workflow **builds but does not publish** on: - **Registry**: `337909757619.dkr.ecr.us-east-1.amazonaws.com` ### Image Tags -Images are tagged as: -- `-quickwit--` - - Example: `12.4.0-quickwit-0.6.0-patched-a1b2c3d` -- `latest` (always points to the most recent build) + +Images use **immutable tags only** (no `latest` tag for production safety): + +**For Git Tags** (e.g., `v0.6.0-patched-1`): +``` +12.4.0-quickwit-0.6.0-patched-1 +``` + +**For Untagged Commits**: +``` +12.4.0-quickwit-0.6.0-patched-a1b2c3d +``` + +Where: +- `12.4.0` = Grafana version +- `0.6.0-patched` = Quickwit plugin version (patched) +- `a1b2c3d` = Short git SHA (7 chars) ### Image Contents - **Base**: Grafana 12.4.0 @@ -90,14 +103,30 @@ After the workflow completes: ## Using the Image -Once published, reference the image in your deployments: +### Deployment Strategy + +1. **Find the latest tag** from the workflow output or ECR +2. **Deploy to preprod** for testing +3. **Promote to prod** after validation ```yaml -# Using specific version +# Preprod - test new builds +image: 337909757619.dkr.ecr.us-east-1.amazonaws.com/grafana-quickwit:12.4.0-quickwit-0.6.0-patched-a1b2c3d + +# Prod - promote after preprod validation image: 337909757619.dkr.ecr.us-east-1.amazonaws.com/grafana-quickwit:12.4.0-quickwit-0.6.0-patched-a1b2c3d +``` + +### Creating Release Tags + +To create a versioned release: + +```bash +# Create and push a version tag +git tag -a v0.6.0-patched-1 -m "Release v0.6.0-patched-1" +git push origin v0.6.0-patched-1 -# Or using latest -image: 337909757619.dkr.ecr.us-east-1.amazonaws.com/grafana-quickwit:latest +# This will create image tag: 12.4.0-quickwit-0.6.0-patched-1 ``` ## Troubleshooting From 9aaff4e34efdb830886b3e983fc9b15073207547 Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 11:23:01 -0700 Subject: [PATCH 08/10] Fix TypeScript errors: remove unused imports - Removed unused FieldCapabilitiesResponse import - Removed unused fieldTypeMap import - Both were only needed for field_caps functionality --- src/datasource/base.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datasource/base.ts b/src/datasource/base.ts index 09ccf9d..224c791 100644 --- a/src/datasource/base.ts +++ b/src/datasource/base.ts @@ -16,7 +16,7 @@ import { TimeRange, ToggleFilterAction, } from '@grafana/data'; -import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery, FieldCapabilitiesResponse } from '@/types'; +import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, TermsQuery } from '@/types'; import { DataSourceWithBackend, getTemplateSrv, @@ -29,7 +29,7 @@ import { isMetricAggregationWithField } from 'components/QueryEditor/MetricAggre import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregationsEditor/utils'; import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations'; import ElasticsearchLanguageProvider from 'LanguageProvider'; -import { fieldTypeMap, hasWhiteSpace, isSimpleToken } from 'utils'; +import { hasWhiteSpace, isSimpleToken } from 'utils'; import { addAddHocFilter } from 'modifyQuery'; import { getQueryResponseProcessor } from 'datasource/processResponse'; import { normalizeInternalLinkQuery } from '@/queryModel'; From 5fe7b04f20f90362ab7b9168c6f171a64120f1ce Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 12:04:44 -0700 Subject: [PATCH 09/10] Switch to GitHub-hosted runner with OIDC authentication - Change from gha-runner-ecr-publish to ubuntu-latest - Add AWS OIDC authentication with role assumption - Requires AWS_ROLE_ARN secret to be configured Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/build-and-push.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index b3631d8..4c69f19 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -24,7 +24,10 @@ env: jobs: build-and-publish: name: Build and Publish Grafana Quickwit Image - runs-on: gha-runner-ecr-publish + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read outputs: githash: ${{ steps.metadata.outputs.githash }} image_tag: ${{ steps.metadata.outputs.image_tag }} @@ -35,6 +38,12 @@ jobs: with: fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION_MGT }} + - name: Generate metadata id: metadata run: | From e6dc3686ff42904d645d2029a30c14c26fd44625 Mon Sep 17 00:00:00 2001 From: Prarthana Basgod Date: Tue, 12 May 2026 12:05:24 -0700 Subject: [PATCH 10/10] Revert "Switch to GitHub-hosted runner with OIDC authentication" This reverts commit 5fe7b04f20f90362ab7b9168c6f171a64120f1ce. --- .github/workflows/build-and-push.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml index 4c69f19..b3631d8 100644 --- a/.github/workflows/build-and-push.yml +++ b/.github/workflows/build-and-push.yml @@ -24,10 +24,7 @@ env: jobs: build-and-publish: name: Build and Publish Grafana Quickwit Image - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read + runs-on: gha-runner-ecr-publish outputs: githash: ${{ steps.metadata.outputs.githash }} image_tag: ${{ steps.metadata.outputs.image_tag }} @@ -38,12 +35,6 @@ jobs: with: fetch-depth: 0 - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: ${{ env.AWS_REGION_MGT }} - - name: Generate metadata id: metadata run: |