From 0784dafaf4f052d99364ec5e1512fe4e5287b303 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Wed, 15 Jul 2026 17:13:12 +0530 Subject: [PATCH 01/18] adding flavor, test-container and workflow --- .github/workflows/slc_ci.yml | 61 +++++++++ .github/workflows/slc_ci_build_slc.yml | 126 ++++++++++++++++++ .github/workflows/slc_ci_check_for_build.yml | 53 ++++++++ .github/workflows/slc_ci_flavor.yml | 49 +++++++ .../slc_ci_prepare_test_container.yml | 49 +++++++ .github/workflows/slc_ci_test_slc.yml | 118 ++++++++++++++++ ext/01_nodoc | 9 ++ ext/scripts/security_scan/exec_oyster.R | 68 ++++++++++ ext/scripts/security_scan/run.sh | 44 ++++++ ext/scripts/security_scan/run_oyster.sh | 20 +++ ext/scripts/security_scan/run_trivy.sh | 24 ++++ .../FLAVOR_DESCRIPTION.md | 7 + flavors/test-udf-runner-cpp-v1/ci.json | 15 +++ .../base_test_build_run/Dockerfile | 19 +++ .../flavor_base/base_test_deps/Dockerfile | 6 + .../flavor_base/build_deps/Dockerfile | 13 ++ .../flavor_base/build_run/Dockerfile | 34 +++++ .../flavor_base/build_steps.py | 83 ++++++++++++ .../flavor_base/language_definitions.json | 23 ++++ .../flavor_base/packages.yml | 59 ++++++++ .../flavor_base/release/Dockerfile | 6 + .../flavor_base/security_scan/.trivyignore | 1 + .../flavor_base/security_scan/Dockerfile | 13 ++ .../flavor_base/security_scan/trivy.rego | 5 + .../flavor_base/security_scan/trivy.yaml | 4 + .../flavor_base/udfclient_deps/Dockerfile | 18 +++ flavors/test-udf-runner-cpp-v1/packages.yml | 26 ++++ test_container/__init__.py | 1 + test_container/build/Dockerfile | 31 +++++ test_container/build/deps/01_nodoc | 9 ++ test_container/build/deps/requirements.txt | 6 + .../test_data/enginedb_small/import.sql | 3 + test_container/tests/__init__.py | 1 + test_container/tests/test/__init__.py | 1 + test_container/tests/test/smoke/__init__.py | 1 + test_container/tests/test/smoke/smoke_test.py | 22 +++ 36 files changed, 1028 insertions(+) create mode 100644 .github/workflows/slc_ci.yml create mode 100644 .github/workflows/slc_ci_build_slc.yml create mode 100644 .github/workflows/slc_ci_check_for_build.yml create mode 100644 .github/workflows/slc_ci_flavor.yml create mode 100644 .github/workflows/slc_ci_prepare_test_container.yml create mode 100644 .github/workflows/slc_ci_test_slc.yml create mode 100644 ext/01_nodoc create mode 100644 ext/scripts/security_scan/exec_oyster.R create mode 100755 ext/scripts/security_scan/run.sh create mode 100755 ext/scripts/security_scan/run_oyster.sh create mode 100755 ext/scripts/security_scan/run_trivy.sh create mode 100644 flavors/test-udf-runner-cpp-v1/FLAVOR_DESCRIPTION.md create mode 100644 flavors/test-udf-runner-cpp-v1/ci.json create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/base_test_build_run/Dockerfile create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/base_test_deps/Dockerfile create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/language_definitions.json create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/release/Dockerfile create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/.trivyignore create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.rego create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.yaml create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile create mode 100644 flavors/test-udf-runner-cpp-v1/packages.yml create mode 100644 test_container/__init__.py create mode 100644 test_container/build/Dockerfile create mode 100644 test_container/build/deps/01_nodoc create mode 100644 test_container/build/deps/requirements.txt create mode 100644 test_container/test_data/enginedb_small/import.sql create mode 100644 test_container/tests/__init__.py create mode 100644 test_container/tests/test/__init__.py create mode 100644 test_container/tests/test/smoke/__init__.py create mode 100644 test_container/tests/test/smoke/smoke_test.py diff --git a/.github/workflows/slc_ci.yml b/.github/workflows/slc_ci.yml new file mode 100644 index 0000000..1eab094 --- /dev/null +++ b/.github/workflows/slc_ci.yml @@ -0,0 +1,61 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI + +on: + push: + branches: + - master + - main + - develop + - 'rebuild/**' + pull_request: + +jobs: + prepare-test-container: + uses: ./.github/workflows/slc_ci_prepare_test_container.yml + secrets: inherit + + get-flavors: + needs: prepare-test-container + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - id: set-matrix + run: poetry run -- exaslc-ci get-flavors --github-output-var matrix + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + ci: + needs: get-flavors + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.flavor }} + cancel-in-progress: true + strategy: + fail-fast: true + matrix: + flavor: ${{fromJson(needs.get-flavors.outputs.matrix)}} + uses: ./.github/workflows/slc_ci_flavor.yml + secrets: inherit + with: + flavor: ${{ matrix.flavor }} + + slc-ready: + needs: ci + name: SLC Allow Merge + runs-on: ubuntu-24.04 + steps: + - name: Print message + run: echo SLC CI finished diff --git a/.github/workflows/slc_ci_build_slc.yml b/.github/workflows/slc_ci_build_slc.yml new file mode 100644 index 0000000..48bc495 --- /dev/null +++ b/.github/workflows/slc_ci_build_slc.yml @@ -0,0 +1,126 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Build-Flavor + +on: + workflow_call: + inputs: + flavor: + description: 'A flavor' + required: true + type: string + build-mode: + description: 'Build mode: possible values: "normal", "rebuild", "release".' + required: true + type: string + + secrets: + CI4_DOCKERHUB_USERNAME: + required: true + CI4_DOCKERHUB_TOKEN: + required: true + +jobs: + detect-runner: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Detect runner + id: detect-runner + run: poetry run -- exaslc-ci get-build-runners --flavor "$FLAVOR" --github-output-var runner-matrix + env: + FLAVOR: ${{ inputs.flavor }} + + outputs: + runner-matrix: ${{ steps.detect-runner.outputs.runner-matrix }} + + build: + needs: detect-runner + strategy: + matrix: + runner: ${{ fromJson(needs.detect-runner.outputs.runner-matrix) }} + runs-on: + labels: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + large-packages: false + + - name: Free disk space by removing large directories + run: | + sudo rm -rf /usr/local/graalvm/ + sudo rm -rf /usr/local/.ghcup/ + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /opt/ghc + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Detect Platform + id: detect-platform + run: poetry run -- exaslc-ci get-platform --github-output-var platform_machine + + - name: Build SLC ${{ inputs.flavor }} + id: build-slc + run: | + echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + poetry run -- exaslc-ci export-and-scan-vulnerabilities --flavor "$FLAVOR" --branch-name "$BRANCH_NAME" --commit-sha "$COMMIT_SHA" --docker-user "$DOCKER_USERNAME" --docker-password "$DOCKER_PASSWORD" --github-output-var slc_paths --build-mode "$BUILD_MODE" + env: + DOCKER_USERNAME: ${{ secrets.CI4_DOCKERHUB_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.CI4_DOCKERHUB_TOKEN }} + BRANCH_NAME: ${{ github.head_ref || github.ref }} + COMMIT_SHA: ${{ github.sha }} + FLAVOR: ${{ inputs.flavor }} + BUILD_MODE: ${{ inputs.build-mode }} + + - name: Check for password + run: grep -rq ${{ secrets.CI4_DOCKERHUB_TOKEN }} .build_output && exit 1 || exit 0 + + - uses: actions/upload-artifact@v6 + if: ${{ always() }} + name: Upload log artifacts + with: + name: logs_build_${{ inputs.flavor }}_${{ matrix.runner }} + path: | + .build_output/jobs/**/* + .build_output/security_scan/**/* + .build_output/meta_data/**/* + retention-days: 30 + + - uses: actions/upload-artifact@v6 + name: Upload release SLC for flavor ${{ inputs.flavor }} + with: + name: slc-${{ inputs.flavor }}-${{ fromJson(steps.build-slc.outputs.slc_paths).slc_release.goal }}-${{ steps.detect-platform.outputs.platform_machine }} + path: "${{ fromJson(steps.build-slc.outputs.slc_paths).slc_release.path }}" + compression-level: 0 + retention-days: 1 + + - uses: actions/upload-artifact@v6 + name: Upload test SLC for flavor ${{ inputs.flavor }} + with: + name: slc-${{ inputs.flavor }}-${{ fromJson(steps.build-slc.outputs.slc_paths).slc_test.goal }}-${{ steps.detect-platform.outputs.platform_machine }} + path: "${{ fromJson(steps.build-slc.outputs.slc_paths).slc_test.path }}" + compression-level: 0 + retention-days: 1 diff --git a/.github/workflows/slc_ci_check_for_build.yml b/.github/workflows/slc_ci_check_for_build.yml new file mode 100644 index 0000000..e8041c0 --- /dev/null +++ b/.github/workflows/slc_ci_check_for_build.yml @@ -0,0 +1,53 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Check-For-Build + +on: + workflow_call: + inputs: + flavor: + description: 'A flavor' + required: true + type: string + outputs: + result: + description: "Build-Mode: Possible value 'no_build_needed', 'normal', 'rebuild'" + value: ${{ jobs.check_for_build.outputs.build_mode }} + +jobs: + check_for_build: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v6 + if: ${{ github.event_name == 'pull_request' }} + with: + fetch-depth: 0 + submodules: 'recursive' + ref: ${{ github.event.pull_request.head.sha }} + + - uses: actions/checkout@v6 + if: ${{ github.event_name != 'pull_request' }} + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Run check + id: run-check + run: | + poetry run -- exaslc-ci check-if-build-needed --flavor "$FLAVOR" --base-ref "$BASE_REF" --github-event "$GITHUB_EVENT" --github-output-var build_mode + env: + BASE_REF: ${{ github.base_ref }} + FLAVOR: ${{ inputs.flavor }} + GITHUB_EVENT: ${{ github.event_name }} + + outputs: + build_mode: ${{ steps.run-check.outputs.build_mode }} diff --git a/.github/workflows/slc_ci_flavor.yml b/.github/workflows/slc_ci_flavor.yml new file mode 100644 index 0000000..aed710f --- /dev/null +++ b/.github/workflows/slc_ci_flavor.yml @@ -0,0 +1,49 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Flavor + +on: + workflow_call: + inputs: + flavor: + description: 'A flavor' + required: true + type: string + secrets: + CI4_DOCKERHUB_USERNAME: + required: true + CI4_DOCKERHUB_TOKEN: + required: true + +jobs: + check_for_build: + uses: ./.github/workflows/slc_ci_check_for_build.yml + with: + flavor: ${{ inputs.flavor }} + + do_nothing: + if: ${{ needs.check_for_build.outputs.result == 'no_build_needed' }} + needs: [check_for_build] + runs-on: ubuntu-24.04 + steps: + - name: Print message + run: echo skipping build and test for flavor ${{ inputs.flavor }} + + build: + needs: check_for_build + uses: ./.github/workflows/slc_ci_build_slc.yml + if: ${{ needs.check_for_build.outputs.result != 'no_build_needed' }} + with: + flavor: ${{ inputs.flavor }} + build-mode: ${{ needs.check_for_build.outputs.result }} + secrets: inherit + + test: + needs: [check_for_build, build] + if: ${{ needs.check_for_build.outputs.result != 'no_build_needed' }} + uses: ./.github/workflows/slc_ci_test_slc.yml + with: + flavor: ${{ inputs.flavor }} + secrets: inherit diff --git a/.github/workflows/slc_ci_prepare_test_container.yml b/.github/workflows/slc_ci_prepare_test_container.yml new file mode 100644 index 0000000..a4e1b01 --- /dev/null +++ b/.github/workflows/slc_ci_prepare_test_container.yml @@ -0,0 +1,49 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Prepare-Test-Container + +on: + workflow_call: + secrets: + CI4_DOCKERHUB_USERNAME: + required: true + CI4_DOCKERHUB_TOKEN: + required: true + +jobs: + prepare-test-container: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Prepare test container + run: | + echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + poetry run -- exaslc-ci prepare-test-container --commit-sha "$COMMIT_SHA" --docker-user "$DOCKER_USERNAME" --docker-password "$DOCKER_PASSWORD" + env: + DOCKER_USERNAME: ${{ secrets.CI4_DOCKERHUB_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.CI4_DOCKERHUB_TOKEN }} + COMMIT_SHA: ${{ github.sha }} + + - name: Check for password + run: grep -rq ${{ secrets.CI4_DOCKERHUB_TOKEN }} .build_output && exit 1 || exit 0 + + - uses: actions/upload-artifact@v6 + if: ${{ always() }} + name: Upload log artifacts + with: + name: logs_prepare_test_container + path: | + .build_output/jobs/**/* + .build_output/meta_data/**/* + retention-days: 30 diff --git a/.github/workflows/slc_ci_test_slc.yml b/.github/workflows/slc_ci_test_slc.yml new file mode 100644 index 0000000..ebfcad4 --- /dev/null +++ b/.github/workflows/slc_ci_test_slc.yml @@ -0,0 +1,118 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Test + +on: + workflow_call: + inputs: + flavor: + description: 'A flavor' + required: true + type: string + secrets: + CI4_DOCKERHUB_USERNAME: + required: true + CI4_DOCKERHUB_TOKEN: + required: true + +jobs: + detect-test-matrix: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Detect Test Matrix + id: detect-test-matrix + run: | + poetry run -- exaslc-ci get-test-matrix --flavor "$FLAVOR" --github-output-var test_matrix + env: + FLAVOR: ${{ inputs.flavor }} + + outputs: + test-matrix: ${{ steps.detect-test-matrix.outputs.test_matrix }} + + run-tests: + needs: [detect-test-matrix] + runs-on: + labels: ${{ matrix.test-runner }} + strategy: + fail-fast: true + matrix: ${{ fromJson(needs.detect-test-matrix.outputs.test-matrix) }} + + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + large-packages: false + + - name: Free disk space by removing large directories + run: | + sudo rm -rf /usr/local/graalvm/ + sudo rm -rf /usr/local/.ghcup/ + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /opt/ghc + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Detect Platform + id: detect-platform + run: poetry run -- exaslc-ci get-platform --github-output-var platform_machine + + - name: Download SLC ${{ matrix.goal }} + id: download-slc + uses: actions/download-artifact@v7 + with: + name: slc-${{ inputs.flavor }}-${{ matrix.goal }}-${{ steps.detect-platform.outputs.platform_machine }} + path: "slc" + + - name: Print SLCs + run: ls -la slc + + - name: Allow unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Run Test-Set ${{ matrix.test-set-name }} + run: | + echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + poetry run -- exaslc-ci run-tests --flavor "$FLAVOR" --docker-user "$DOCKER_USERNAME" --docker-password "$DOCKER_PASSWORD" --test-set-name "$TEST_SET_NAME" --slc-directory "$SLC_DIRECTORY" --commit-sha "$COMMIT_SHA" + env: + DOCKER_USERNAME: ${{ secrets.CI4_DOCKERHUB_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.CI4_DOCKERHUB_TOKEN }} + FLAVOR: ${{ inputs.flavor }} + TEST_SET_NAME: ${{ matrix.test-set-name }} + SLC_DIRECTORY: ${{ steps.download-slc.outputs.download-path }} + COMMIT_SHA: ${{ github.sha }} + + - name: Check for password + run: grep -rq ${{ secrets.CI4_DOCKERHUB_TOKEN }} .build_output && exit 1 || exit 0 + + - uses: actions/upload-artifact@v6 + if: ${{ always() }} + name: Upload log artifacts + with: + name: logs_tests_${{ inputs.flavor }}_${{ matrix.test-set-name }}_${{ matrix.test-runner }} + path: | + .build_output/jobs/**/* + .build_output/meta_data/**/* + retention-days: 30 diff --git a/ext/01_nodoc b/ext/01_nodoc new file mode 100644 index 0000000..7320020 --- /dev/null +++ b/ext/01_nodoc @@ -0,0 +1,9 @@ +path-exclude /usr/share/doc/* +# we need to keep copyright files for legal reasons +path-include /usr/share/doc/*/copyright +path-exclude /usr/share/man/* +path-exclude /usr/share/groff/* +path-exclude /usr/share/info/* +# lintian stuff is small, but really unnecessary +path-exclude /usr/share/lintian/* +path-exclude /usr/share/linda/* diff --git a/ext/scripts/security_scan/exec_oyster.R b/ext/scripts/security_scan/exec_oyster.R new file mode 100644 index 0000000..4ccfb8a --- /dev/null +++ b/ext/scripts/security_scan/exec_oyster.R @@ -0,0 +1,68 @@ +is_not_in_ignore_list <- function(vulnerabilities, ignore_list) { + for (vulner in vulnerabilities) { + for (vuln in vulner) { + for (vul in vuln) { + for (vul_id in vul$id) { + if (!vul_id %in% ignore_list) { + return(TRUE) + } + } + } + } + } + FALSE +} + +filter_for_ignore_list <- function(df, ignore_list) { + res <- data.frame() + for (row in nrow(df)) { + if (is_not_in_ignore_list(df[row,"vulnerabilities"], ignore_list)) { + res <- rbind(res, df[row,]) + } + } + res +} + +read_ignore_list <- function() { + oyster_ignore_list_file <- "/.oysterignore" + res <- c() + if (file.exists(oyster_ignore_list_file)) { + res <- read.table(oyster_ignore_list_file, header=FALSE) + } + res +} + +filter_out_ignored_cves <- function(df) { + ignore_list <- read_ignore_list() + filter_for_ignore_list(df, ignore_list) +} + +library("oysteR") +audit <- audit_installed_r_pkgs() + +install.packages("knitr") + +audit_vuln <- audit[audit$no_of_vulnerabilities > 0,] + +filtered_vuln <- filter_out_ignored_cves(audit_vuln) + +library(jsonlite) +args = commandArgs(trailingOnly=TRUE) + +n_vulnerabilities = nrow(filtered_vuln) + +report <- toJSON(filtered_vuln) +write(report, paste0(args[1], "/oyster.json")) + +exit_code <- 0 +if(n_vulnerabilities > 0) { + exit_code <- 1 + library(knitr) + write(kable(filtered_vuln), paste0(args[1], "/oyster.md")) + print(kable(filtered_vuln, format="simple")) + print(paste(n_vulnerabilities, "vulnerabilities found!")) +} else { + write("No vulnerabilities found!", paste0(args[1], "/oyster.md")) +} + +quit(status=exit_code) diff --git a/ext/scripts/security_scan/run.sh b/ext/scripts/security_scan/run.sh new file mode 100755 index 0000000..f022e24 --- /dev/null +++ b/ext/scripts/security_scan/run.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "You must provide output path as argument" + exit 1 +fi + +output_path=$1 +mkdir -p "$output_path" + +#Ignore shellcheck rule. SECURITY_SCANNERS is an environment variable +#shellcheck disable=SC2153 +echo "$SECURITY_SCANNERS" + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" + +if [ -z "${SECURITY_SCANNERS-}" ]; then + echo "No security scanners set. Please set SECURITY_SCANNERS." + exit 1 +fi + +function _run_security_scanners() { + for current_scanner in "$@"; do + case "$current_scanner" in + trivy) + "$SCRIPT_DIR/run_trivy.sh" "$output_path" + ;; + oyster) + "$SCRIPT_DIR/run_oyster.sh" "$output_path" + ;; + *) + echo "Unknown scanner: $current_scanner" + exit 1 + ;; + esac + done +} + +security_scanners=$SECURITY_SCANNERS +#Ignore shellcheck rule. As security_scanners we can't convert it to an array other than passing it without quotes to _run_security_scanners +#shellcheck disable=SC2086 +_run_security_scanners $security_scanners diff --git a/ext/scripts/security_scan/run_oyster.sh b/ext/scripts/security_scan/run_oyster.sh new file mode 100755 index 0000000..6eff2c3 --- /dev/null +++ b/ext/scripts/security_scan/run_oyster.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "You must provide output path as argument" + exit 1 +fi + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" + +output_path=$1 + +echo Start oysteR... +if Rscript "$SCRIPT_DIR/exec_oyster.R" "$output_path" &> "$output_path/oyster_debug.log" ; then + echo "No vulnerabilities found!" +else + cat "$output_path/oyster_debug.log" + exit 1 +fi diff --git a/ext/scripts/security_scan/run_trivy.sh b/ext/scripts/security_scan/run_trivy.sh new file mode 100755 index 0000000..9900c05 --- /dev/null +++ b/ext/scripts/security_scan/run_trivy.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "You must provide output path as argument" + exit 1 +fi + +TRIVY_CACHE_LOCATION="https://dli4ip9yror05.cloudfront.net" + +mkdir -p "$HOME/.cache/" +pushd "$HOME/.cache/" + +curl -s -o trivy_cache.tar.gz "${TRIVY_CACHE_LOCATION}/trivy_cache.tar.gz" +tar xf trivy_cache.tar.gz + +popd + +output_path=$1 + +trivy rootfs --no-progress --offline-scan --format json --timeout 15m0s --skip-java-db-update --skip-db-update --config /trivy.yaml --output "$output_path/trivy_report.json" / > /dev/null +trivy rootfs --no-progress --offline-scan --format table --timeout 15m0s --skip-java-db-update --skip-db-update --config /trivy.yaml --output "$output_path/trivy_report.txt" / > /dev/null +trivy rootfs --no-progress --offline-scan --timeout 15m0s --skip-db-update --skip-java-db-update --config /trivy.yaml --show-suppressed --severity "HIGH,CRITICAL" --exit-code 1 / diff --git a/flavors/test-udf-runner-cpp-v1/FLAVOR_DESCRIPTION.md b/flavors/test-udf-runner-cpp-v1/FLAVOR_DESCRIPTION.md new file mode 100644 index 0000000..768bbc4 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/FLAVOR_DESCRIPTION.md @@ -0,0 +1,7 @@ +This flavor builds and smoke-tests the retained `udf-runner-cpp/v1` launcher surface. + +It is intentionally minimal: + +- build the reduced C++ runner bundle +- publish `language_definitions.json` for benchmark and streaming modes +- run one lightweight smoke test from `test_container` diff --git a/flavors/test-udf-runner-cpp-v1/ci.json b/flavors/test-udf-runner-cpp-v1/ci.json new file mode 100644 index 0000000..e88555a --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/ci.json @@ -0,0 +1,15 @@ +{ + "build_runners": ["ubuntu-24.04"], + "test_config": { + "default_test_runners": ["ubuntu-24.04"], + "test_sets": [ + { + "name": "smoke", + "files": [], + "folders": ["smoke"], + "goal": "release", + "generic_language_tests": [] + } + ] + } +} diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_build_run/Dockerfile new file mode 100644 index 0000000..90dc0af --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_build_run/Dockerfile @@ -0,0 +1,19 @@ +FROM {{base_test_deps}} + +COPY --from={{build_run}} /usr /usr +RUN true +COPY --from={{build_run}} /lib /lib +RUN true +COPY --from={{build_run}} /bin /bin +RUN true +COPY --from={{build_run}} /opt /opt +RUN true +COPY --from={{build_run}} /etc /etc +RUN true +COPY --from={{build_run}} /build_info /build_info +RUN true +COPY --from={{build_run}} /exaudf /exaudf +RUN true + +RUN ldconfig +RUN mkdir -p /conf /buckets diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_deps/Dockerfile new file mode 100644 index 0000000..dd98448 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_deps/Dockerfile @@ -0,0 +1,6 @@ +FROM {{build_deps}} + +RUN mkdir -p /build_info/packages + +COPY base_test_deps_packages.yml /build_info/packages/base_test_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/base_test_deps_packages.yml --build-step base_test_deps diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile new file mode 100644 index 0000000..7c695a9 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile @@ -0,0 +1,13 @@ +FROM exasol/script-language-container:exaslpm-1.3.1-ubuntu-24.04 +ENV DEBIAN_FRONTEND=noninteractive + +COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc + +RUN mkdir -p /build_info/packages + +COPY build_deps_packages.yml /build_info/packages/build_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/build_deps_packages.yml --build-step build_deps + +RUN locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + ldconfig diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile new file mode 100644 index 0000000..cbce02e --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile @@ -0,0 +1,34 @@ +FROM {{udfclient_deps}} + +COPY --from={{build_deps}} /usr /usr +RUN true +COPY --from={{build_deps}} /lib /lib +RUN true +COPY --from={{build_deps}} /bin /bin +RUN true +COPY --from={{build_deps}} /opt /opt +RUN true +COPY --from={{build_deps}} /etc /etc +RUN true +COPY --from={{build_deps}} /build_info /build_info +RUN true + +RUN exaslpm export-variables --out-file /env +RUN ldconfig + +RUN mkdir /exaudfclient /exaudf +COPY exaudfclient/ /exaudfclient/ + +WORKDIR /exaudfclient/ +RUN ["/bin/bash", "-c", "source /env && bash build.sh --lockfile_mode=off --config no-tty --config fast-binary --define benchmark=true --define bash=true //:udf_runner_cpp_v1_gen //:udf_runner_cpp_v1_static_gen"] + +RUN cp -r -L bazel-bin/* /exaudf +RUN mv /exaudf/udf_runner_cpp_v1 /exaudf/exaudfclient +RUN mv /exaudf/udf_runner_cpp_v1_static /exaudf/exaudfclient_static + +WORKDIR /exaudfclient/base +RUN ./test_udfclient.sh /exaudf/exaudfclient +RUN ./test_udfclient.sh /exaudf/exaudfclient_static + +WORKDIR /exaudfclient/base +RUN ["/bin/bash", "-c", "source /env && bazel test --lockfile_mode=off --config no-tty //exaudflib/test:exaudflib-test //script_options_parser/legacy/test:script-options-legacy-parser-test //script_options_parser/ctpg/test:script-options-ctpg-parser-test"] diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py new file mode 100644 index 0000000..957bfd5 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -0,0 +1,83 @@ +from typing import Dict + +from exasol.slc.internal.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask + + +class AnalyzeUDFClientDeps(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "udfclient_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildDeps(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "build_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"01_nodoc": "ext/01_nodoc"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildRun(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "build_run" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps, "udfclient_deps": AnalyzeUDFClientDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"exaudfclient": "udf-runner-cpp/v1"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestDeps(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "base_test_deps" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "base_test_build_run" + + def requires_tasks(self): + return {"base_test_deps": AnalyzeBaseTestDeps, "build_run": AnalyzeBuildRun} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeRelease(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "release" + + def requires_tasks(self): + return {"build_run": AnalyzeBuildRun} + + def get_path_in_flavor(self): + return "flavor_base" + + +class SecurityScan(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "security_scan" + + def requires_tasks(self): + return {"release": AnalyzeRelease} + + def get_path_in_flavor(self): + return "flavor_base" diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/language_definitions.json b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definitions.json new file mode 100644 index 0000000..e7cd83c --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definitions.json @@ -0,0 +1,23 @@ +{ + "schema_version": 2, + "language_definitions": [ + { + "protocol": "localzmq+protobuf", + "aliases": ["CPP_BENCHMARK"], + "parameters": [{"key": "lang", "value": "benchmark"}], + "udf_client_path": { + "executable": "/exaudf/exaudfclient" + }, + "deprecation": null + }, + { + "protocol": "localzmq+protobuf", + "aliases": ["CPP_STREAMING"], + "parameters": [{"key": "lang", "value": "streaming"}], + "udf_client_path": { + "executable": "/exaudf/exaudfclient" + }, + "deprecation": null + } + ] +} diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml b/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml new file mode 100644 index 0000000..e6a1aae --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml @@ -0,0 +1,59 @@ +build_steps: +- name: build_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: coreutils + version: 9.4-3ubuntu* + - name: tar + version: 1.35+dfsg-3build1 + - name: curl + version: 8.5.0-2ubuntu* + - name: build-essential + version: 12.10ubuntu* + - name: protobuf-compiler + version: 3.21.12-8.2ubuntu* + - name: chrpath + version: 0.16-2build1 + - name: locales + version: 2.39-0ubuntu* + variables: + PROTOBUF_BIN: /usr/bin/protoc + - name: install_bazel + tools: + bazel: + version: "8.3.1" + validation_cfg: + version_mandatory: true +- name: base_test_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: gdb + version: 15.0.50.20240403-0ubuntu* + - name: valgrind + version: 1:3.22.0-0ubuntu* + - name: strace + version: 6.8-0ubuntu* + validation_cfg: + version_mandatory: true +- name: security_scan + phases: + - name: install_deps + apt: + packages: + - name: gnupg + - name: install_trivy + apt: + repos: + trivy: + key_url: https://aquasecurity.github.io/trivy-repo/deb/public.key + entry: deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main + out_file: trivy.list + packages: + - name: trivy + validation_cfg: + version_mandatory: false +version: 1.0.0 diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/release/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/release/Dockerfile new file mode 100644 index 0000000..c707577 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/release/Dockerfile @@ -0,0 +1,6 @@ +FROM {{build_run}} + +COPY language_definitions.json /exaudf/language_definitions.json + +RUN mkdir -p /build_info/actual_installed_packages/release && \ + exaslpm list-all-installed-packages --out-file /build_info/actual_installed_packages/release/packages.yml diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/.trivyignore b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/.trivyignore new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/.trivyignore @@ -0,0 +1 @@ + diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile new file mode 100644 index 0000000..50b4070 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile @@ -0,0 +1,13 @@ +FROM {{release}} +ENV DEBIAN_FRONTEND=noninteractive + +RUN mkdir -p /build_info/packages +COPY security_scan_packages.yml /build_info/packages/security_scan_packages.yml +RUN exaslpm install --package-file /build_info/packages/security_scan_packages.yml --build-step security_scan + +ENV SECURITY_SCANNERS="trivy" +COPY /security_scan/.trivyignore /.trivyignore +COPY /security_scan/trivy.rego /trivy.rego +COPY /security_scan/trivy.yaml /trivy.yaml + +ENTRYPOINT ["/scripts/security_scan/run.sh"] diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.rego b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.rego new file mode 100644 index 0000000..3d5f3ea --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.rego @@ -0,0 +1,5 @@ +package trivy + +import data.lib.trivy + +default ignore = false diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.yaml b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.yaml new file mode 100644 index 0000000..5578d2e --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.yaml @@ -0,0 +1,4 @@ +scan: + skip-files: + - /usr/bin/trivy +ignore-policy: /trivy.rego diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile new file mode 100644 index 0000000..6bfae4c --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile @@ -0,0 +1,18 @@ +FROM exasol/script-language-container:exaslpm-1.3.1-ubuntu-24.04 +ENV DEBIAN_FRONTEND=noninteractive + +COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc +COPY scripts /scripts + +RUN mkdir -p /build_info/packages +COPY udfclient_deps_packages.yml /build_info/packages/udfclient_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/udfclient_deps_packages.yml --build-step udfclient_deps + +RUN usermod -l exasolution ubuntu +RUN groupmod -n exasolution ubuntu +RUN addgroup --gid 500 exausers +RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 diff --git a/flavors/test-udf-runner-cpp-v1/packages.yml b/flavors/test-udf-runner-cpp-v1/packages.yml new file mode 100644 index 0000000..3dc426b --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/packages.yml @@ -0,0 +1,26 @@ +build_steps: +- name: udfclient_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: coreutils + version: 9.4-3ubuntu* + - name: locales + version: 2.39-0ubuntu* + - name: libnss-db + version: 2.2.5-3build2 + - name: libzmq3-dev + version: 4.3.5-1build2 + - name: libprotobuf-dev + version: 3.21.12-8.2ubuntu* + - name: libssl-dev + version: 3.0.13-0ubuntu* + variables: + PROTOBUF_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu + PROTOBUF_INCLUDE_PREFIX: /usr/include + ZMQ_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu + ZMQ_INCLUDE_PREFIX: /usr/include + validation_cfg: + version_mandatory: true +version: 1.0.0 diff --git a/test_container/__init__.py b/test_container/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test_container/__init__.py @@ -0,0 +1 @@ + diff --git a/test_container/build/Dockerfile b/test_container/build/Dockerfile new file mode 100644 index 0000000..662fe8e --- /dev/null +++ b/test_container/build/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:22.04 +ENV DEBIAN_FRONTEND=noninteractive + +COPY deps/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc + +RUN apt-get -y update && \ + apt-get -y install \ + openjdk-17-jdk \ + locales \ + python3.10-venv \ + python3-pip \ + unixodbc \ + unixodbc-dev \ + curl \ + git && \ + locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + apt-get -y clean && \ + apt-get -y autoremove && \ + ldconfig + +RUN curl -fsSL https://get.docker.com | bash + +RUN mkdir -p downloads/ODBC downloads/JDBC downloads/EXAplus +RUN curl -s https://x-up.s3.amazonaws.com/7.x/24.1.1/Exasol_ODBC-24.1.1-Linux_x86_64.tar.gz | tar -C downloads/ODBC --strip-components 2 -zxf - +RUN curl -s https://x-up.s3.amazonaws.com/7.x/24.1.1/Exasol_JDBC-24.1.1.tar.gz | tar -C downloads/JDBC --strip-components 2 -zxf - +RUN curl -s https://x-up.s3.amazonaws.com/7.x/24.1.1/EXAplus-24.1.1.tar.gz | tar -C downloads/EXAplus --strip-components 2 -zxf - +ENV EXAPLUS=/downloads/EXAplus/exaplus + +COPY deps/requirements.txt requirements.txt +RUN pip3 install -r "requirements.txt" diff --git a/test_container/build/deps/01_nodoc b/test_container/build/deps/01_nodoc new file mode 100644 index 0000000..7320020 --- /dev/null +++ b/test_container/build/deps/01_nodoc @@ -0,0 +1,9 @@ +path-exclude /usr/share/doc/* +# we need to keep copyright files for legal reasons +path-include /usr/share/doc/*/copyright +path-exclude /usr/share/man/* +path-exclude /usr/share/groff/* +path-exclude /usr/share/info/* +# lintian stuff is small, but really unnecessary +path-exclude /usr/share/lintian/* +path-exclude /usr/share/linda/* diff --git a/test_container/build/deps/requirements.txt b/test_container/build/deps/requirements.txt new file mode 100644 index 0000000..bc2f1da --- /dev/null +++ b/test_container/build/deps/requirements.txt @@ -0,0 +1,6 @@ +pyodbc +pytz +lxml +docker +scipy +exasol-python-test-framework==0.6.0 diff --git a/test_container/test_data/enginedb_small/import.sql b/test_container/test_data/enginedb_small/import.sql new file mode 100644 index 0000000..d851311 --- /dev/null +++ b/test_container/test_data/enginedb_small/import.sql @@ -0,0 +1,3 @@ +-- Intentionally empty for the initial smoke setup. +-- The first flavor-CI milestone only checks container bootstrap, +-- database connectivity, and that the built release payload is mounted. diff --git a/test_container/tests/__init__.py b/test_container/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test_container/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/test_container/tests/test/__init__.py b/test_container/tests/test/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test_container/tests/test/__init__.py @@ -0,0 +1 @@ + diff --git a/test_container/tests/test/smoke/__init__.py b/test_container/tests/test/smoke/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test_container/tests/test/smoke/__init__.py @@ -0,0 +1 @@ + diff --git a/test_container/tests/test/smoke/smoke_test.py b/test_container/tests/test/smoke/smoke_test.py new file mode 100644 index 0000000..1deb0e8 --- /dev/null +++ b/test_container/tests/test/smoke/smoke_test.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +from pathlib import Path + +from exasol_python_test_framework import udf + + +class SmokeTest(udf.TestCase): + def test_db_connection(self): + rows = self.query("SELECT 1 FROM dual") + self.assertRowsEqual([(1,)], rows) + + def test_release_payload_present(self): + bucket_root = Path("/buckets/bfsdefault/myudfs") + language_definitions = list(bucket_root.glob("**/language_definitions.json")) + udf_clients = list(bucket_root.glob("**/exaudf/exaudfclient")) + self.assertGreaterEqual(len(language_definitions), 1) + self.assertGreaterEqual(len(udf_clients), 1) + + +if __name__ == "__main__": + udf.main() From 0296a68f829ac1c7942e0b4538d28e442fef925c Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Wed, 15 Jul 2026 18:22:26 +0530 Subject: [PATCH 02/18] adding language definition --- flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py | 3 +++ flavors/test-udf-runner-cpp-v1/flavor_base/language_definition | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/language_definition diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py index 957bfd5..246eb52 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -68,6 +68,9 @@ def get_build_step(self) -> str: def requires_tasks(self): return {"build_run": AnalyzeBuildRun} + def get_language_definition(self) -> str: + return "language_definitions.json" + def get_path_in_flavor(self): return "flavor_base" diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition new file mode 100644 index 0000000..d984598 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition @@ -0,0 +1,2 @@ +CPP_BENCHMARK=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=benchmark#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient +CPP_STREAMING=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=streaming#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient From e3b2b15fa9bbf6edf84121621274d32db49b3ab0 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Wed, 15 Jul 2026 18:42:00 +0530 Subject: [PATCH 03/18] target base_test_build_run --- flavors/test-udf-runner-cpp-v1/ci.json | 2 +- test_container/tests/test/smoke/smoke_test.py | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/flavors/test-udf-runner-cpp-v1/ci.json b/flavors/test-udf-runner-cpp-v1/ci.json index e88555a..c2c6a86 100644 --- a/flavors/test-udf-runner-cpp-v1/ci.json +++ b/flavors/test-udf-runner-cpp-v1/ci.json @@ -7,7 +7,7 @@ "name": "smoke", "files": [], "folders": ["smoke"], - "goal": "release", + "goal": "base_test_build_run", "generic_language_tests": [] } ] diff --git a/test_container/tests/test/smoke/smoke_test.py b/test_container/tests/test/smoke/smoke_test.py index 1deb0e8..ff4ed81 100644 --- a/test_container/tests/test/smoke/smoke_test.py +++ b/test_container/tests/test/smoke/smoke_test.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -from pathlib import Path - from exasol_python_test_framework import udf @@ -10,13 +8,6 @@ def test_db_connection(self): rows = self.query("SELECT 1 FROM dual") self.assertRowsEqual([(1,)], rows) - def test_release_payload_present(self): - bucket_root = Path("/buckets/bfsdefault/myudfs") - language_definitions = list(bucket_root.glob("**/language_definitions.json")) - udf_clients = list(bucket_root.glob("**/exaudf/exaudfclient")) - self.assertGreaterEqual(len(language_definitions), 1) - self.assertGreaterEqual(len(udf_clients), 1) - if __name__ == "__main__": udf.main() From c11e42dde261f5557aea8e3ab6bc34c67be5733b Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Thu, 16 Jul 2026 12:50:48 +0530 Subject: [PATCH 04/18] cleanup and rewiring --- ext/01_nodoc | 9 --- ext/scripts/security_scan/exec_oyster.R | 68 ------------------- ext/scripts/security_scan/run.sh | 43 ------------ ext/scripts/security_scan/run_oyster.sh | 20 ------ ext/scripts/security_scan/run_trivy.sh | 24 ------- .../flavor_base/build_run/Dockerfile | 4 -- .../flavor_base/packages.yml | 20 +++--- .../flavor_base/security_scan/.trivyignore | 1 - .../flavor_base/security_scan/Dockerfile | 6 -- .../flavor_base/security_scan/trivy.rego | 5 -- .../flavor_base/security_scan/trivy.yaml | 4 -- .../flavor_base/udfclient_deps/Dockerfile | 3 - flavors/test-udf-runner-cpp-v1/packages.yml | 12 ++-- 13 files changed, 16 insertions(+), 203 deletions(-) delete mode 100644 ext/01_nodoc delete mode 100644 ext/scripts/security_scan/exec_oyster.R delete mode 100755 ext/scripts/security_scan/run_oyster.sh delete mode 100755 ext/scripts/security_scan/run_trivy.sh delete mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/.trivyignore delete mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.rego delete mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.yaml diff --git a/ext/01_nodoc b/ext/01_nodoc deleted file mode 100644 index 7320020..0000000 --- a/ext/01_nodoc +++ /dev/null @@ -1,9 +0,0 @@ -path-exclude /usr/share/doc/* -# we need to keep copyright files for legal reasons -path-include /usr/share/doc/*/copyright -path-exclude /usr/share/man/* -path-exclude /usr/share/groff/* -path-exclude /usr/share/info/* -# lintian stuff is small, but really unnecessary -path-exclude /usr/share/lintian/* -path-exclude /usr/share/linda/* diff --git a/ext/scripts/security_scan/exec_oyster.R b/ext/scripts/security_scan/exec_oyster.R deleted file mode 100644 index 4ccfb8a..0000000 --- a/ext/scripts/security_scan/exec_oyster.R +++ /dev/null @@ -1,68 +0,0 @@ -is_not_in_ignore_list <- function(vulnerabilities, ignore_list) { - for (vulner in vulnerabilities) { - for (vuln in vulner) { - for (vul in vuln) { - for (vul_id in vul$id) { - if (!vul_id %in% ignore_list) { - return(TRUE) - } - } - } - } - } - FALSE -} - -filter_for_ignore_list <- function(df, ignore_list) { - res <- data.frame() - for (row in nrow(df)) { - if (is_not_in_ignore_list(df[row,"vulnerabilities"], ignore_list)) { - res <- rbind(res, df[row,]) - } - } - res -} - -read_ignore_list <- function() { - oyster_ignore_list_file <- "/.oysterignore" - res <- c() - if (file.exists(oyster_ignore_list_file)) { - res <- read.table(oyster_ignore_list_file, header=FALSE) - } - res -} - -filter_out_ignored_cves <- function(df) { - ignore_list <- read_ignore_list() - filter_for_ignore_list(df, ignore_list) -} - -library("oysteR") -audit <- audit_installed_r_pkgs() - -install.packages("knitr") - -audit_vuln <- audit[audit$no_of_vulnerabilities > 0,] - -filtered_vuln <- filter_out_ignored_cves(audit_vuln) - -library(jsonlite) -args = commandArgs(trailingOnly=TRUE) - -n_vulnerabilities = nrow(filtered_vuln) - -report <- toJSON(filtered_vuln) -write(report, paste0(args[1], "/oyster.json")) - -exit_code <- 0 -if(n_vulnerabilities > 0) { - exit_code <- 1 - library(knitr) - write(kable(filtered_vuln), paste0(args[1], "/oyster.md")) - print(kable(filtered_vuln, format="simple")) - print(paste(n_vulnerabilities, "vulnerabilities found!")) -} else { - write("No vulnerabilities found!", paste0(args[1], "/oyster.md")) -} - -quit(status=exit_code) diff --git a/ext/scripts/security_scan/run.sh b/ext/scripts/security_scan/run.sh index f022e24..a9bf588 100755 --- a/ext/scripts/security_scan/run.sh +++ b/ext/scripts/security_scan/run.sh @@ -1,44 +1 @@ #!/bin/bash - -set -euo pipefail - -if [ $# -lt 1 ]; then - echo "You must provide output path as argument" - exit 1 -fi - -output_path=$1 -mkdir -p "$output_path" - -#Ignore shellcheck rule. SECURITY_SCANNERS is an environment variable -#shellcheck disable=SC2153 -echo "$SECURITY_SCANNERS" - -SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" - -if [ -z "${SECURITY_SCANNERS-}" ]; then - echo "No security scanners set. Please set SECURITY_SCANNERS." - exit 1 -fi - -function _run_security_scanners() { - for current_scanner in "$@"; do - case "$current_scanner" in - trivy) - "$SCRIPT_DIR/run_trivy.sh" "$output_path" - ;; - oyster) - "$SCRIPT_DIR/run_oyster.sh" "$output_path" - ;; - *) - echo "Unknown scanner: $current_scanner" - exit 1 - ;; - esac - done -} - -security_scanners=$SECURITY_SCANNERS -#Ignore shellcheck rule. As security_scanners we can't convert it to an array other than passing it without quotes to _run_security_scanners -#shellcheck disable=SC2086 -_run_security_scanners $security_scanners diff --git a/ext/scripts/security_scan/run_oyster.sh b/ext/scripts/security_scan/run_oyster.sh deleted file mode 100755 index 6eff2c3..0000000 --- a/ext/scripts/security_scan/run_oyster.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -if [ $# -lt 1 ]; then - echo "You must provide output path as argument" - exit 1 -fi - -SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" - -output_path=$1 - -echo Start oysteR... -if Rscript "$SCRIPT_DIR/exec_oyster.R" "$output_path" &> "$output_path/oyster_debug.log" ; then - echo "No vulnerabilities found!" -else - cat "$output_path/oyster_debug.log" - exit 1 -fi diff --git a/ext/scripts/security_scan/run_trivy.sh b/ext/scripts/security_scan/run_trivy.sh deleted file mode 100755 index 9900c05..0000000 --- a/ext/scripts/security_scan/run_trivy.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -if [ $# -lt 1 ]; then - echo "You must provide output path as argument" - exit 1 -fi - -TRIVY_CACHE_LOCATION="https://dli4ip9yror05.cloudfront.net" - -mkdir -p "$HOME/.cache/" -pushd "$HOME/.cache/" - -curl -s -o trivy_cache.tar.gz "${TRIVY_CACHE_LOCATION}/trivy_cache.tar.gz" -tar xf trivy_cache.tar.gz - -popd - -output_path=$1 - -trivy rootfs --no-progress --offline-scan --format json --timeout 15m0s --skip-java-db-update --skip-db-update --config /trivy.yaml --output "$output_path/trivy_report.json" / > /dev/null -trivy rootfs --no-progress --offline-scan --format table --timeout 15m0s --skip-java-db-update --skip-db-update --config /trivy.yaml --output "$output_path/trivy_report.txt" / > /dev/null -trivy rootfs --no-progress --offline-scan --timeout 15m0s --skip-db-update --skip-java-db-update --config /trivy.yaml --show-suppressed --severity "HIGH,CRITICAL" --exit-code 1 / diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile index cbce02e..bba9e83 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile @@ -28,7 +28,3 @@ RUN mv /exaudf/udf_runner_cpp_v1_static /exaudf/exaudfclient_static WORKDIR /exaudfclient/base RUN ./test_udfclient.sh /exaudf/exaudfclient -RUN ./test_udfclient.sh /exaudf/exaudfclient_static - -WORKDIR /exaudfclient/base -RUN ["/bin/bash", "-c", "source /env && bazel test --lockfile_mode=off --config no-tty //exaudflib/test:exaudflib-test //script_options_parser/legacy/test:script-options-legacy-parser-test //script_options_parser/ctpg/test:script-options-ctpg-parser-test"] diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml b/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml index e6a1aae..156a02a 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml @@ -5,19 +5,19 @@ build_steps: apt: packages: - name: coreutils - version: 9.4-3ubuntu* + version: "*" - name: tar - version: 1.35+dfsg-3build1 + version: "*" - name: curl - version: 8.5.0-2ubuntu* + version: "*" - name: build-essential - version: 12.10ubuntu* + version: "*" - name: protobuf-compiler - version: 3.21.12-8.2ubuntu* + version: "*" - name: chrpath - version: 0.16-2build1 + version: "*" - name: locales - version: 2.39-0ubuntu* + version: "*" variables: PROTOBUF_BIN: /usr/bin/protoc - name: install_bazel @@ -32,11 +32,11 @@ build_steps: apt: packages: - name: gdb - version: 15.0.50.20240403-0ubuntu* + version: "*" - name: valgrind - version: 1:3.22.0-0ubuntu* + version: "*" - name: strace - version: 6.8-0ubuntu* + version: "*" validation_cfg: version_mandatory: true - name: security_scan diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/.trivyignore b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/.trivyignore deleted file mode 100644 index 8b13789..0000000 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/.trivyignore +++ /dev/null @@ -1 +0,0 @@ - diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile index 50b4070..2d366b8 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile @@ -3,11 +3,5 @@ ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /build_info/packages COPY security_scan_packages.yml /build_info/packages/security_scan_packages.yml -RUN exaslpm install --package-file /build_info/packages/security_scan_packages.yml --build-step security_scan - -ENV SECURITY_SCANNERS="trivy" -COPY /security_scan/.trivyignore /.trivyignore -COPY /security_scan/trivy.rego /trivy.rego -COPY /security_scan/trivy.yaml /trivy.yaml ENTRYPOINT ["/scripts/security_scan/run.sh"] diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.rego b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.rego deleted file mode 100644 index 3d5f3ea..0000000 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.rego +++ /dev/null @@ -1,5 +0,0 @@ -package trivy - -import data.lib.trivy - -default ignore = false diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.yaml b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.yaml deleted file mode 100644 index 5578d2e..0000000 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/trivy.yaml +++ /dev/null @@ -1,4 +0,0 @@ -scan: - skip-files: - - /usr/bin/trivy -ignore-policy: /trivy.rego diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile index 6bfae4c..766a9e4 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile @@ -1,9 +1,6 @@ FROM exasol/script-language-container:exaslpm-1.3.1-ubuntu-24.04 ENV DEBIAN_FRONTEND=noninteractive -COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc -COPY scripts /scripts - RUN mkdir -p /build_info/packages COPY udfclient_deps_packages.yml /build_info/packages/udfclient_deps_packages.yml RUN exaslpm install --package-file /build_info/packages/udfclient_deps_packages.yml --build-step udfclient_deps diff --git a/flavors/test-udf-runner-cpp-v1/packages.yml b/flavors/test-udf-runner-cpp-v1/packages.yml index 3dc426b..4075951 100644 --- a/flavors/test-udf-runner-cpp-v1/packages.yml +++ b/flavors/test-udf-runner-cpp-v1/packages.yml @@ -5,17 +5,17 @@ build_steps: apt: packages: - name: coreutils - version: 9.4-3ubuntu* + version: "*" - name: locales - version: 2.39-0ubuntu* + version: "*" - name: libnss-db - version: 2.2.5-3build2 + version: "*" - name: libzmq3-dev - version: 4.3.5-1build2 + version: "*" - name: libprotobuf-dev - version: 3.21.12-8.2ubuntu* + version: "*" - name: libssl-dev - version: 3.0.13-0ubuntu* + version: "*" variables: PROTOBUF_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu PROTOBUF_INCLUDE_PREFIX: /usr/include From ffeb551b89285ba3860d1b8c759869ebece37d07 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Thu, 16 Jul 2026 15:57:07 +0530 Subject: [PATCH 05/18] adding flavor_test_build_run --- .../test-udf-runner-cpp-v1/flavor_base/build_steps.py | 11 +++++++++++ .../flavor_base/flavor_test_build_run/Dockerfile | 1 + 2 files changed, 12 insertions(+) create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/flavor_test_build_run/Dockerfile diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py index 246eb52..87634a2 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -61,6 +61,17 @@ def get_path_in_flavor(self): return "flavor_base" +class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "flavor_test_build_run" + + def requires_tasks(self): + return {"base_test_build_run": AnalyzeBaseTestBuildRun} + + def get_path_in_flavor(self): + return "flavor_base" + + class AnalyzeRelease(DockerFlavorAnalyzeImageTask): def get_build_step(self) -> str: return "release" diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/flavor_test_build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/flavor_test_build_run/Dockerfile new file mode 100644 index 0000000..5f5c00e --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/flavor_test_build_run/Dockerfile @@ -0,0 +1 @@ +FROM {{base_test_build_run}} From 94173f3a10c917035a97fd45c1574a0e8f831c6c Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Thu, 16 Jul 2026 16:12:27 +0530 Subject: [PATCH 06/18] removed all ref to nodoc --- .../test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile | 2 -- flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile index 7c695a9..6a3ce10 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile @@ -1,8 +1,6 @@ FROM exasol/script-language-container:exaslpm-1.3.1-ubuntu-24.04 ENV DEBIAN_FRONTEND=noninteractive -COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc - RUN mkdir -p /build_info/packages COPY build_deps_packages.yml /build_info/packages/build_deps_packages.yml diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py index 87634a2..d9f04d7 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -8,7 +8,7 @@ def get_build_step(self) -> str: return "udfclient_deps" def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + return {"scripts": "ext/scripts"} def get_path_in_flavor(self): return "flavor_base" @@ -19,7 +19,7 @@ def get_build_step(self) -> str: return "build_deps" def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"01_nodoc": "ext/01_nodoc"} + return {} def get_path_in_flavor(self): return "flavor_base" From 5fa72a627cb4f9515039e658daf2f16d13a98250 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Thu, 16 Jul 2026 17:34:36 +0530 Subject: [PATCH 07/18] Fix test flavor build steps and user/group setup for CI --- .../flavor_base/udfclient_deps/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile index 766a9e4..cc29dda 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile @@ -7,8 +7,8 @@ RUN exaslpm install --package-file /build_info/packages/udfclient_deps_packages. RUN usermod -l exasolution ubuntu RUN groupmod -n exasolution ubuntu -RUN addgroup --gid 500 exausers -RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN groupadd --gid 500 exausers +RUN useradd --uid 500 --gid 500 --no-create-home --shell /usr/sbin/nologin --comment "First Last,RoomNumber,WorkPhone,HomePhone" exadefusr ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en From 26a56c25ce7dd64bb7f2f6338e3126d2c4d52a49 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Thu, 16 Jul 2026 18:09:56 +0530 Subject: [PATCH 08/18] fixing docker copy --- .../test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile index bba9e83..dacc405 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile @@ -1,15 +1,10 @@ FROM {{udfclient_deps}} COPY --from={{build_deps}} /usr /usr -RUN true COPY --from={{build_deps}} /lib /lib -RUN true COPY --from={{build_deps}} /bin /bin -RUN true COPY --from={{build_deps}} /opt /opt -RUN true COPY --from={{build_deps}} /etc /etc -RUN true COPY --from={{build_deps}} /build_info /build_info RUN true From f2c6a37d9ffbb4260b37604bb96b991b464a7b4c Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 10:08:08 +0530 Subject: [PATCH 09/18] rewiring builds --- .../flavor_base/build_run/Dockerfile | 8 -------- flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py | 5 ++++- .../flavor_base/udfclient_deps/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile index dacc405..c058d8a 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile @@ -1,13 +1,5 @@ FROM {{udfclient_deps}} -COPY --from={{build_deps}} /usr /usr -COPY --from={{build_deps}} /lib /lib -COPY --from={{build_deps}} /bin /bin -COPY --from={{build_deps}} /opt /opt -COPY --from={{build_deps}} /etc /etc -COPY --from={{build_deps}} /build_info /build_info -RUN true - RUN exaslpm export-variables --out-file /env RUN ldconfig diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py index d9f04d7..3293ea6 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -7,6 +7,9 @@ class AnalyzeUDFClientDeps(DockerFlavorAnalyzeImageTask): def get_build_step(self) -> str: return "udfclient_deps" + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps} + def get_additional_build_directories_mapping(self) -> Dict[str, str]: return {"scripts": "ext/scripts"} @@ -30,7 +33,7 @@ def get_build_step(self) -> str: return "build_run" def requires_tasks(self): - return {"build_deps": AnalyzeBuildDeps, "udfclient_deps": AnalyzeUDFClientDeps} + return {"udfclient_deps": AnalyzeUDFClientDeps} def get_additional_build_directories_mapping(self) -> Dict[str, str]: return {"exaudfclient": "udf-runner-cpp/v1"} diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile index cc29dda..428601e 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile @@ -1,4 +1,4 @@ -FROM exasol/script-language-container:exaslpm-1.3.1-ubuntu-24.04 +FROM {{build_deps}} ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /build_info/packages From 0b3b3b9b11870818facee5c88deb2b02d05cccad Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 10:26:52 +0530 Subject: [PATCH 10/18] CI build graph wiring and including swig headers --- .../v1/streaming_container/streamingcontainer.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/udf-runner-cpp/v1/streaming_container/streamingcontainer.h b/udf-runner-cpp/v1/streaming_container/streamingcontainer.h index b45faac..295fd3e 100644 --- a/udf-runner-cpp/v1/streaming_container/streamingcontainer.h +++ b/udf-runner-cpp/v1/streaming_container/streamingcontainer.h @@ -1,4 +1,15 @@ +#include "exaudflib/vm/swig_vm.h" +#include "exaudflib/swig/swig_meta_data.h" +#include "exaudflib/swig/swig_result_handler.h" +#include "exaudflib/swig/swig_table_iterator.h" + +#include +#include +#include +#include +#include + using namespace SWIGVMContainers; using namespace std; From 42a903858c029a4f58ddc10bc84de99be290113b Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 11:38:26 +0530 Subject: [PATCH 11/18] fixing compilation errors --- udf-runner-cpp/v1/benchmark_container/benchmark_container.cc | 1 + udf-runner-cpp/v1/exa_vm_factory.cc | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/udf-runner-cpp/v1/benchmark_container/benchmark_container.cc b/udf-runner-cpp/v1/benchmark_container/benchmark_container.cc index 92de436..7cd910d 100644 --- a/udf-runner-cpp/v1/benchmark_container/benchmark_container.cc +++ b/udf-runner-cpp/v1/benchmark_container/benchmark_container.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include using namespace SWIGVMContainers; diff --git a/udf-runner-cpp/v1/exa_vm_factory.cc b/udf-runner-cpp/v1/exa_vm_factory.cc index c510fc0..3f4ef73 100644 --- a/udf-runner-cpp/v1/exa_vm_factory.cc +++ b/udf-runner-cpp/v1/exa_vm_factory.cc @@ -13,14 +13,14 @@ std::function create_vm(const std::string& argv_lang, bool use_ctpg_options_parser) { if(argv_lang.compare("lang=streaming") == 0) { #ifdef ENABLE_STREAMING_VM - return []() { return new SWIGVMContainers::StreamingVM(false); }; + return []() { return new StreamingVM(false); }; #else throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Streaming support"); #endif } else if(argv_lang.compare("lang=benchmark") == 0) { #ifdef ENABLE_BENCHMARK_VM - return []() { return new SWIGVMContainers::BenchmarkVM(false); }; + return []() { return new BenchmarkVM(false); }; #else throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Benchmark support"); #endif From 4f399323f66cf7dfdb745d35b12d891d2166c909 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 12:18:20 +0530 Subject: [PATCH 12/18] seems workflow picking wrong sha? --- .github/workflows/slc_ci_build_slc.yml | 2 +- .github/workflows/slc_ci_prepare_test_container.yml | 2 +- .github/workflows/slc_ci_test_slc.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/slc_ci_build_slc.yml b/.github/workflows/slc_ci_build_slc.yml index 48bc495..9c63219 100644 --- a/.github/workflows/slc_ci_build_slc.yml +++ b/.github/workflows/slc_ci_build_slc.yml @@ -91,7 +91,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.CI4_DOCKERHUB_USERNAME }} DOCKER_PASSWORD: ${{ secrets.CI4_DOCKERHUB_TOKEN }} BRANCH_NAME: ${{ github.head_ref || github.ref }} - COMMIT_SHA: ${{ github.sha }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} FLAVOR: ${{ inputs.flavor }} BUILD_MODE: ${{ inputs.build-mode }} diff --git a/.github/workflows/slc_ci_prepare_test_container.yml b/.github/workflows/slc_ci_prepare_test_container.yml index a4e1b01..6ed56ad 100644 --- a/.github/workflows/slc_ci_prepare_test_container.yml +++ b/.github/workflows/slc_ci_prepare_test_container.yml @@ -33,7 +33,7 @@ jobs: env: DOCKER_USERNAME: ${{ secrets.CI4_DOCKERHUB_USERNAME }} DOCKER_PASSWORD: ${{ secrets.CI4_DOCKERHUB_TOKEN }} - COMMIT_SHA: ${{ github.sha }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check for password run: grep -rq ${{ secrets.CI4_DOCKERHUB_TOKEN }} .build_output && exit 1 || exit 0 diff --git a/.github/workflows/slc_ci_test_slc.yml b/.github/workflows/slc_ci_test_slc.yml index ebfcad4..965b017 100644 --- a/.github/workflows/slc_ci_test_slc.yml +++ b/.github/workflows/slc_ci_test_slc.yml @@ -102,7 +102,7 @@ jobs: FLAVOR: ${{ inputs.flavor }} TEST_SET_NAME: ${{ matrix.test-set-name }} SLC_DIRECTORY: ${{ steps.download-slc.outputs.download-path }} - COMMIT_SHA: ${{ github.sha }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check for password run: grep -rq ${{ secrets.CI4_DOCKERHUB_TOKEN }} .build_output && exit 1 || exit 0 From 2bd818838d4b4dc06ba1244cf0db14c5bb17cee2 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 12:36:37 +0530 Subject: [PATCH 13/18] add zmq dependency --- flavors/test-udf-runner-cpp-v1/packages.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flavors/test-udf-runner-cpp-v1/packages.yml b/flavors/test-udf-runner-cpp-v1/packages.yml index 4075951..8d1eb9b 100644 --- a/flavors/test-udf-runner-cpp-v1/packages.yml +++ b/flavors/test-udf-runner-cpp-v1/packages.yml @@ -12,6 +12,8 @@ build_steps: version: "*" - name: libzmq3-dev version: "*" + - name: cppzmq-dev + version: "*" - name: libprotobuf-dev version: "*" - name: libssl-dev From 9c365a893ef0808a9e83cd993cfab7fce607c61a Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 13:10:37 +0530 Subject: [PATCH 14/18] make sure scripts is available in the image --- flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py | 3 +++ .../flavor_base/security_scan/Dockerfile | 1 + 2 files changed, 4 insertions(+) diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py index 3293ea6..7a77072 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -96,5 +96,8 @@ def get_build_step(self) -> str: def requires_tasks(self): return {"release": AnalyzeRelease} + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"scripts": "ext/scripts"} + def get_path_in_flavor(self): return "flavor_base" diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile index 2d366b8..00e5715 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile @@ -3,5 +3,6 @@ ENV DEBIAN_FRONTEND=noninteractive RUN mkdir -p /build_info/packages COPY security_scan_packages.yml /build_info/packages/security_scan_packages.yml +COPY scripts /scripts ENTRYPOINT ["/scripts/security_scan/run.sh"] From d3fa770fb16d54a0aa786a88365a45c65df5844d Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 13:33:07 +0530 Subject: [PATCH 15/18] removing the security scan completely --- ext/scripts/security_scan/run.sh | 1 - .../flavor_base/build_steps.py | 14 -------------- .../flavor_base/packages.yml | 17 ----------------- .../flavor_base/security_scan/Dockerfile | 8 -------- 4 files changed, 40 deletions(-) delete mode 100755 ext/scripts/security_scan/run.sh delete mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile diff --git a/ext/scripts/security_scan/run.sh b/ext/scripts/security_scan/run.sh deleted file mode 100755 index a9bf588..0000000 --- a/ext/scripts/security_scan/run.sh +++ /dev/null @@ -1 +0,0 @@ -#!/bin/bash diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py index 7a77072..e940f2a 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -87,17 +87,3 @@ def get_language_definition(self) -> str: def get_path_in_flavor(self): return "flavor_base" - - -class SecurityScan(DockerFlavorAnalyzeImageTask): - def get_build_step(self) -> str: - return "security_scan" - - def requires_tasks(self): - return {"release": AnalyzeRelease} - - def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"scripts": "ext/scripts"} - - def get_path_in_flavor(self): - return "flavor_base" diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml b/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml index 156a02a..bf25d3a 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml @@ -39,21 +39,4 @@ build_steps: version: "*" validation_cfg: version_mandatory: true -- name: security_scan - phases: - - name: install_deps - apt: - packages: - - name: gnupg - - name: install_trivy - apt: - repos: - trivy: - key_url: https://aquasecurity.github.io/trivy-repo/deb/public.key - entry: deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main - out_file: trivy.list - packages: - - name: trivy - validation_cfg: - version_mandatory: false version: 1.0.0 diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile deleted file mode 100644 index 00e5715..0000000 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM {{release}} -ENV DEBIAN_FRONTEND=noninteractive - -RUN mkdir -p /build_info/packages -COPY security_scan_packages.yml /build_info/packages/security_scan_packages.yml -COPY scripts /scripts - -ENTRYPOINT ["/scripts/security_scan/run.sh"] From 457364442cfc33096f4925ecdee02118fd51ec55 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 13:53:14 +0530 Subject: [PATCH 16/18] fix stale ext scripts --- flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py index e940f2a..4a7b612 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -11,7 +11,7 @@ def requires_tasks(self): return {"build_deps": AnalyzeBuildDeps} def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"scripts": "ext/scripts"} + return {} def get_path_in_flavor(self): return "flavor_base" From 2bec4d111b1e16074a4d40592898443f6f7e6885 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 14:34:20 +0530 Subject: [PATCH 17/18] re-introduce a min no-op security scan --- ext/scripts/security_scan/run.sh | 12 ++++++++++++ .../flavor_base/build_steps.py | 14 ++++++++++++++ .../flavor_base/security_scan/Dockerfile | 6 ++++++ 3 files changed, 32 insertions(+) create mode 100755 ext/scripts/security_scan/run.sh create mode 100644 flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile diff --git a/ext/scripts/security_scan/run.sh b/ext/scripts/security_scan/run.sh new file mode 100755 index 0000000..52434f5 --- /dev/null +++ b/ext/scripts/security_scan/run.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "You must provide output path as argument" + exit 1 +fi + +output_path=$1 +mkdir -p "$output_path" +echo "Security scan skipped for bootstrap flavor." > "$output_path/noop_security_scan.txt" diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py index 4a7b612..81917d4 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -87,3 +87,17 @@ def get_language_definition(self) -> str: def get_path_in_flavor(self): return "flavor_base" + + +class SecurityScan(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "security_scan" + + def requires_tasks(self): + return {"release": AnalyzeRelease} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile new file mode 100644 index 0000000..31862b7 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile @@ -0,0 +1,6 @@ +FROM {{release}} +ENV DEBIAN_FRONTEND=noninteractive + +COPY scripts /scripts + +ENTRYPOINT ["/scripts/security_scan/run.sh"] From 6fe8d996d31630945415547a0e0ab8657e352267 Mon Sep 17 00:00:00 2001 From: "Shankar.KV" Date: Fri, 17 Jul 2026 15:06:59 +0530 Subject: [PATCH 18/18] fix multi line issue --- flavors/test-udf-runner-cpp-v1/flavor_base/language_definition | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition index d984598..6148554 100644 --- a/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition @@ -1,2 +1 @@ -CPP_BENCHMARK=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=benchmark#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient -CPP_STREAMING=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=streaming#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient +CPP_BENCHMARK=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=benchmark#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient CPP_STREAMING=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=streaming#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient