diff --git a/.github/workflows/slc_ci.yml b/.github/workflows/slc_ci.yml new file mode 100644 index 0000000..1eab094 --- /dev/null +++ b/.github/workflows/slc_ci.yml @@ -0,0 +1,61 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI + +on: + push: + branches: + - master + - main + - develop + - 'rebuild/**' + pull_request: + +jobs: + prepare-test-container: + uses: ./.github/workflows/slc_ci_prepare_test_container.yml + secrets: inherit + + get-flavors: + needs: prepare-test-container + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - id: set-matrix + run: poetry run -- exaslc-ci get-flavors --github-output-var matrix + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + ci: + needs: get-flavors + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.flavor }} + cancel-in-progress: true + strategy: + fail-fast: true + matrix: + flavor: ${{fromJson(needs.get-flavors.outputs.matrix)}} + uses: ./.github/workflows/slc_ci_flavor.yml + secrets: inherit + with: + flavor: ${{ matrix.flavor }} + + slc-ready: + needs: ci + name: SLC Allow Merge + runs-on: ubuntu-24.04 + steps: + - name: Print message + run: echo SLC CI finished diff --git a/.github/workflows/slc_ci_build_slc.yml b/.github/workflows/slc_ci_build_slc.yml new file mode 100644 index 0000000..9c63219 --- /dev/null +++ b/.github/workflows/slc_ci_build_slc.yml @@ -0,0 +1,126 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Build-Flavor + +on: + workflow_call: + inputs: + flavor: + description: 'A flavor' + required: true + type: string + build-mode: + description: 'Build mode: possible values: "normal", "rebuild", "release".' + required: true + type: string + + secrets: + CI4_DOCKERHUB_USERNAME: + required: true + CI4_DOCKERHUB_TOKEN: + required: true + +jobs: + detect-runner: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Detect runner + id: detect-runner + run: poetry run -- exaslc-ci get-build-runners --flavor "$FLAVOR" --github-output-var runner-matrix + env: + FLAVOR: ${{ inputs.flavor }} + + outputs: + runner-matrix: ${{ steps.detect-runner.outputs.runner-matrix }} + + build: + needs: detect-runner + strategy: + matrix: + runner: ${{ fromJson(needs.detect-runner.outputs.runner-matrix) }} + runs-on: + labels: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + large-packages: false + + - name: Free disk space by removing large directories + run: | + sudo rm -rf /usr/local/graalvm/ + sudo rm -rf /usr/local/.ghcup/ + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /opt/ghc + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Detect Platform + id: detect-platform + run: poetry run -- exaslc-ci get-platform --github-output-var platform_machine + + - name: Build SLC ${{ inputs.flavor }} + id: build-slc + run: | + echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + poetry run -- exaslc-ci export-and-scan-vulnerabilities --flavor "$FLAVOR" --branch-name "$BRANCH_NAME" --commit-sha "$COMMIT_SHA" --docker-user "$DOCKER_USERNAME" --docker-password "$DOCKER_PASSWORD" --github-output-var slc_paths --build-mode "$BUILD_MODE" + env: + DOCKER_USERNAME: ${{ secrets.CI4_DOCKERHUB_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.CI4_DOCKERHUB_TOKEN }} + BRANCH_NAME: ${{ github.head_ref || github.ref }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + FLAVOR: ${{ inputs.flavor }} + BUILD_MODE: ${{ inputs.build-mode }} + + - name: Check for password + run: grep -rq ${{ secrets.CI4_DOCKERHUB_TOKEN }} .build_output && exit 1 || exit 0 + + - uses: actions/upload-artifact@v6 + if: ${{ always() }} + name: Upload log artifacts + with: + name: logs_build_${{ inputs.flavor }}_${{ matrix.runner }} + path: | + .build_output/jobs/**/* + .build_output/security_scan/**/* + .build_output/meta_data/**/* + retention-days: 30 + + - uses: actions/upload-artifact@v6 + name: Upload release SLC for flavor ${{ inputs.flavor }} + with: + name: slc-${{ inputs.flavor }}-${{ fromJson(steps.build-slc.outputs.slc_paths).slc_release.goal }}-${{ steps.detect-platform.outputs.platform_machine }} + path: "${{ fromJson(steps.build-slc.outputs.slc_paths).slc_release.path }}" + compression-level: 0 + retention-days: 1 + + - uses: actions/upload-artifact@v6 + name: Upload test SLC for flavor ${{ inputs.flavor }} + with: + name: slc-${{ inputs.flavor }}-${{ fromJson(steps.build-slc.outputs.slc_paths).slc_test.goal }}-${{ steps.detect-platform.outputs.platform_machine }} + path: "${{ fromJson(steps.build-slc.outputs.slc_paths).slc_test.path }}" + compression-level: 0 + retention-days: 1 diff --git a/.github/workflows/slc_ci_check_for_build.yml b/.github/workflows/slc_ci_check_for_build.yml new file mode 100644 index 0000000..e8041c0 --- /dev/null +++ b/.github/workflows/slc_ci_check_for_build.yml @@ -0,0 +1,53 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Check-For-Build + +on: + workflow_call: + inputs: + flavor: + description: 'A flavor' + required: true + type: string + outputs: + result: + description: "Build-Mode: Possible value 'no_build_needed', 'normal', 'rebuild'" + value: ${{ jobs.check_for_build.outputs.build_mode }} + +jobs: + check_for_build: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v6 + if: ${{ github.event_name == 'pull_request' }} + with: + fetch-depth: 0 + submodules: 'recursive' + ref: ${{ github.event.pull_request.head.sha }} + + - uses: actions/checkout@v6 + if: ${{ github.event_name != 'pull_request' }} + with: + fetch-depth: 0 + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Run check + id: run-check + run: | + poetry run -- exaslc-ci check-if-build-needed --flavor "$FLAVOR" --base-ref "$BASE_REF" --github-event "$GITHUB_EVENT" --github-output-var build_mode + env: + BASE_REF: ${{ github.base_ref }} + FLAVOR: ${{ inputs.flavor }} + GITHUB_EVENT: ${{ github.event_name }} + + outputs: + build_mode: ${{ steps.run-check.outputs.build_mode }} diff --git a/.github/workflows/slc_ci_flavor.yml b/.github/workflows/slc_ci_flavor.yml new file mode 100644 index 0000000..aed710f --- /dev/null +++ b/.github/workflows/slc_ci_flavor.yml @@ -0,0 +1,49 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Flavor + +on: + workflow_call: + inputs: + flavor: + description: 'A flavor' + required: true + type: string + secrets: + CI4_DOCKERHUB_USERNAME: + required: true + CI4_DOCKERHUB_TOKEN: + required: true + +jobs: + check_for_build: + uses: ./.github/workflows/slc_ci_check_for_build.yml + with: + flavor: ${{ inputs.flavor }} + + do_nothing: + if: ${{ needs.check_for_build.outputs.result == 'no_build_needed' }} + needs: [check_for_build] + runs-on: ubuntu-24.04 + steps: + - name: Print message + run: echo skipping build and test for flavor ${{ inputs.flavor }} + + build: + needs: check_for_build + uses: ./.github/workflows/slc_ci_build_slc.yml + if: ${{ needs.check_for_build.outputs.result != 'no_build_needed' }} + with: + flavor: ${{ inputs.flavor }} + build-mode: ${{ needs.check_for_build.outputs.result }} + secrets: inherit + + test: + needs: [check_for_build, build] + if: ${{ needs.check_for_build.outputs.result != 'no_build_needed' }} + uses: ./.github/workflows/slc_ci_test_slc.yml + with: + flavor: ${{ inputs.flavor }} + secrets: inherit diff --git a/.github/workflows/slc_ci_prepare_test_container.yml b/.github/workflows/slc_ci_prepare_test_container.yml new file mode 100644 index 0000000..6ed56ad --- /dev/null +++ b/.github/workflows/slc_ci_prepare_test_container.yml @@ -0,0 +1,49 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Prepare-Test-Container + +on: + workflow_call: + secrets: + CI4_DOCKERHUB_USERNAME: + required: true + CI4_DOCKERHUB_TOKEN: + required: true + +jobs: + prepare-test-container: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Prepare test container + run: | + echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + poetry run -- exaslc-ci prepare-test-container --commit-sha "$COMMIT_SHA" --docker-user "$DOCKER_USERNAME" --docker-password "$DOCKER_PASSWORD" + env: + DOCKER_USERNAME: ${{ secrets.CI4_DOCKERHUB_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.CI4_DOCKERHUB_TOKEN }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Check for password + run: grep -rq ${{ secrets.CI4_DOCKERHUB_TOKEN }} .build_output && exit 1 || exit 0 + + - uses: actions/upload-artifact@v6 + if: ${{ always() }} + name: Upload log artifacts + with: + name: logs_prepare_test_container + path: | + .build_output/jobs/**/* + .build_output/meta_data/**/* + retention-days: 30 diff --git a/.github/workflows/slc_ci_test_slc.yml b/.github/workflows/slc_ci_test_slc.yml new file mode 100644 index 0000000..965b017 --- /dev/null +++ b/.github/workflows/slc_ci_test_slc.yml @@ -0,0 +1,118 @@ +# ---- AUTOMATICALLY GENERATED FILE ---- +# ---- DO NOT EDIT MANUALLY, BUT USE PYTHON MODULE "exasol.slc_ci_setup" ---- +# ---- from https://github.com/exasol/script-languages-container-ci-setup TO UPDATE ---- + +name: SLC-CI-Test + +on: + workflow_call: + inputs: + flavor: + description: 'A flavor' + required: true + type: string + secrets: + CI4_DOCKERHUB_USERNAME: + required: true + CI4_DOCKERHUB_TOKEN: + required: true + +jobs: + detect-test-matrix: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Detect Test Matrix + id: detect-test-matrix + run: | + poetry run -- exaslc-ci get-test-matrix --flavor "$FLAVOR" --github-output-var test_matrix + env: + FLAVOR: ${{ inputs.flavor }} + + outputs: + test-matrix: ${{ steps.detect-test-matrix.outputs.test_matrix }} + + run-tests: + needs: [detect-test-matrix] + runs-on: + labels: ${{ matrix.test-runner }} + strategy: + fail-fast: true + matrix: ${{ fromJson(needs.detect-test-matrix.outputs.test-matrix) }} + + steps: + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + large-packages: false + + - name: Free disk space by removing large directories + run: | + sudo rm -rf /usr/local/graalvm/ + sudo rm -rf /usr/local/.ghcup/ + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /opt/ghc + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Detect Platform + id: detect-platform + run: poetry run -- exaslc-ci get-platform --github-output-var platform_machine + + - name: Download SLC ${{ matrix.goal }} + id: download-slc + uses: actions/download-artifact@v7 + with: + name: slc-${{ inputs.flavor }}-${{ matrix.goal }}-${{ steps.detect-platform.outputs.platform_machine }} + path: "slc" + + - name: Print SLCs + run: ls -la slc + + - name: Allow unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Run Test-Set ${{ matrix.test-set-name }} + run: | + echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + poetry run -- exaslc-ci run-tests --flavor "$FLAVOR" --docker-user "$DOCKER_USERNAME" --docker-password "$DOCKER_PASSWORD" --test-set-name "$TEST_SET_NAME" --slc-directory "$SLC_DIRECTORY" --commit-sha "$COMMIT_SHA" + env: + DOCKER_USERNAME: ${{ secrets.CI4_DOCKERHUB_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.CI4_DOCKERHUB_TOKEN }} + FLAVOR: ${{ inputs.flavor }} + TEST_SET_NAME: ${{ matrix.test-set-name }} + SLC_DIRECTORY: ${{ steps.download-slc.outputs.download-path }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Check for password + run: grep -rq ${{ secrets.CI4_DOCKERHUB_TOKEN }} .build_output && exit 1 || exit 0 + + - uses: actions/upload-artifact@v6 + if: ${{ always() }} + name: Upload log artifacts + with: + name: logs_tests_${{ inputs.flavor }}_${{ matrix.test-set-name }}_${{ matrix.test-runner }} + path: | + .build_output/jobs/**/* + .build_output/meta_data/**/* + retention-days: 30 diff --git a/ext/scripts/security_scan/run.sh b/ext/scripts/security_scan/run.sh new file mode 100755 index 0000000..52434f5 --- /dev/null +++ b/ext/scripts/security_scan/run.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "You must provide output path as argument" + exit 1 +fi + +output_path=$1 +mkdir -p "$output_path" +echo "Security scan skipped for bootstrap flavor." > "$output_path/noop_security_scan.txt" diff --git a/flavors/test-udf-runner-cpp-v1/FLAVOR_DESCRIPTION.md b/flavors/test-udf-runner-cpp-v1/FLAVOR_DESCRIPTION.md new file mode 100644 index 0000000..768bbc4 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/FLAVOR_DESCRIPTION.md @@ -0,0 +1,7 @@ +This flavor builds and smoke-tests the retained `udf-runner-cpp/v1` launcher surface. + +It is intentionally minimal: + +- build the reduced C++ runner bundle +- publish `language_definitions.json` for benchmark and streaming modes +- run one lightweight smoke test from `test_container` diff --git a/flavors/test-udf-runner-cpp-v1/ci.json b/flavors/test-udf-runner-cpp-v1/ci.json new file mode 100644 index 0000000..c2c6a86 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/ci.json @@ -0,0 +1,15 @@ +{ + "build_runners": ["ubuntu-24.04"], + "test_config": { + "default_test_runners": ["ubuntu-24.04"], + "test_sets": [ + { + "name": "smoke", + "files": [], + "folders": ["smoke"], + "goal": "base_test_build_run", + "generic_language_tests": [] + } + ] + } +} diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_build_run/Dockerfile new file mode 100644 index 0000000..90dc0af --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_build_run/Dockerfile @@ -0,0 +1,19 @@ +FROM {{base_test_deps}} + +COPY --from={{build_run}} /usr /usr +RUN true +COPY --from={{build_run}} /lib /lib +RUN true +COPY --from={{build_run}} /bin /bin +RUN true +COPY --from={{build_run}} /opt /opt +RUN true +COPY --from={{build_run}} /etc /etc +RUN true +COPY --from={{build_run}} /build_info /build_info +RUN true +COPY --from={{build_run}} /exaudf /exaudf +RUN true + +RUN ldconfig +RUN mkdir -p /conf /buckets diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_deps/Dockerfile new file mode 100644 index 0000000..dd98448 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/base_test_deps/Dockerfile @@ -0,0 +1,6 @@ +FROM {{build_deps}} + +RUN mkdir -p /build_info/packages + +COPY base_test_deps_packages.yml /build_info/packages/base_test_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/base_test_deps_packages.yml --build-step base_test_deps diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile new file mode 100644 index 0000000..6a3ce10 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_deps/Dockerfile @@ -0,0 +1,11 @@ +FROM exasol/script-language-container:exaslpm-1.3.1-ubuntu-24.04 +ENV DEBIAN_FRONTEND=noninteractive + +RUN mkdir -p /build_info/packages + +COPY build_deps_packages.yml /build_info/packages/build_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/build_deps_packages.yml --build-step build_deps + +RUN locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + ldconfig diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile new file mode 100644 index 0000000..c058d8a --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_run/Dockerfile @@ -0,0 +1,17 @@ +FROM {{udfclient_deps}} + +RUN exaslpm export-variables --out-file /env +RUN ldconfig + +RUN mkdir /exaudfclient /exaudf +COPY exaudfclient/ /exaudfclient/ + +WORKDIR /exaudfclient/ +RUN ["/bin/bash", "-c", "source /env && bash build.sh --lockfile_mode=off --config no-tty --config fast-binary --define benchmark=true --define bash=true //:udf_runner_cpp_v1_gen //:udf_runner_cpp_v1_static_gen"] + +RUN cp -r -L bazel-bin/* /exaudf +RUN mv /exaudf/udf_runner_cpp_v1 /exaudf/exaudfclient +RUN mv /exaudf/udf_runner_cpp_v1_static /exaudf/exaudfclient_static + +WORKDIR /exaudfclient/base +RUN ./test_udfclient.sh /exaudf/exaudfclient diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py new file mode 100644 index 0000000..81917d4 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/build_steps.py @@ -0,0 +1,103 @@ +from typing import Dict + +from exasol.slc.internal.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask + + +class AnalyzeUDFClientDeps(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "udfclient_deps" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildDeps(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "build_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildRun(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "build_run" + + def requires_tasks(self): + return {"udfclient_deps": AnalyzeUDFClientDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"exaudfclient": "udf-runner-cpp/v1"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestDeps(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "base_test_deps" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "base_test_build_run" + + def requires_tasks(self): + return {"base_test_deps": AnalyzeBaseTestDeps, "build_run": AnalyzeBuildRun} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "flavor_test_build_run" + + def requires_tasks(self): + return {"base_test_build_run": AnalyzeBaseTestBuildRun} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeRelease(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "release" + + def requires_tasks(self): + return {"build_run": AnalyzeBuildRun} + + def get_language_definition(self) -> str: + return "language_definitions.json" + + def get_path_in_flavor(self): + return "flavor_base" + + +class SecurityScan(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "security_scan" + + def requires_tasks(self): + return {"release": AnalyzeRelease} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/flavor_test_build_run/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/flavor_test_build_run/Dockerfile new file mode 100644 index 0000000..5f5c00e --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/flavor_test_build_run/Dockerfile @@ -0,0 +1 @@ +FROM {{base_test_build_run}} diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition new file mode 100644 index 0000000..6148554 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definition @@ -0,0 +1 @@ +CPP_BENCHMARK=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=benchmark#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient CPP_STREAMING=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=streaming#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/language_definitions.json b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definitions.json new file mode 100644 index 0000000..e7cd83c --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/language_definitions.json @@ -0,0 +1,23 @@ +{ + "schema_version": 2, + "language_definitions": [ + { + "protocol": "localzmq+protobuf", + "aliases": ["CPP_BENCHMARK"], + "parameters": [{"key": "lang", "value": "benchmark"}], + "udf_client_path": { + "executable": "/exaudf/exaudfclient" + }, + "deprecation": null + }, + { + "protocol": "localzmq+protobuf", + "aliases": ["CPP_STREAMING"], + "parameters": [{"key": "lang", "value": "streaming"}], + "udf_client_path": { + "executable": "/exaudf/exaudfclient" + }, + "deprecation": null + } + ] +} diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml b/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml new file mode 100644 index 0000000..bf25d3a --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/packages.yml @@ -0,0 +1,42 @@ +build_steps: +- name: build_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: coreutils + version: "*" + - name: tar + version: "*" + - name: curl + version: "*" + - name: build-essential + version: "*" + - name: protobuf-compiler + version: "*" + - name: chrpath + version: "*" + - name: locales + version: "*" + variables: + PROTOBUF_BIN: /usr/bin/protoc + - name: install_bazel + tools: + bazel: + version: "8.3.1" + validation_cfg: + version_mandatory: true +- name: base_test_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: gdb + version: "*" + - name: valgrind + version: "*" + - name: strace + version: "*" + validation_cfg: + version_mandatory: true +version: 1.0.0 diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/release/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/release/Dockerfile new file mode 100644 index 0000000..c707577 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/release/Dockerfile @@ -0,0 +1,6 @@ +FROM {{build_run}} + +COPY language_definitions.json /exaudf/language_definitions.json + +RUN mkdir -p /build_info/actual_installed_packages/release && \ + exaslpm list-all-installed-packages --out-file /build_info/actual_installed_packages/release/packages.yml diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile new file mode 100644 index 0000000..31862b7 --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/security_scan/Dockerfile @@ -0,0 +1,6 @@ +FROM {{release}} +ENV DEBIAN_FRONTEND=noninteractive + +COPY scripts /scripts + +ENTRYPOINT ["/scripts/security_scan/run.sh"] diff --git a/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile new file mode 100644 index 0000000..428601e --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/flavor_base/udfclient_deps/Dockerfile @@ -0,0 +1,15 @@ +FROM {{build_deps}} +ENV DEBIAN_FRONTEND=noninteractive + +RUN mkdir -p /build_info/packages +COPY udfclient_deps_packages.yml /build_info/packages/udfclient_deps_packages.yml +RUN exaslpm install --package-file /build_info/packages/udfclient_deps_packages.yml --build-step udfclient_deps + +RUN usermod -l exasolution ubuntu +RUN groupmod -n exasolution ubuntu +RUN groupadd --gid 500 exausers +RUN useradd --uid 500 --gid 500 --no-create-home --shell /usr/sbin/nologin --comment "First Last,RoomNumber,WorkPhone,HomePhone" exadefusr + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 diff --git a/flavors/test-udf-runner-cpp-v1/packages.yml b/flavors/test-udf-runner-cpp-v1/packages.yml new file mode 100644 index 0000000..8d1eb9b --- /dev/null +++ b/flavors/test-udf-runner-cpp-v1/packages.yml @@ -0,0 +1,28 @@ +build_steps: +- name: udfclient_deps + phases: + - name: install_apt_packages + apt: + packages: + - name: coreutils + version: "*" + - name: locales + version: "*" + - name: libnss-db + version: "*" + - name: libzmq3-dev + version: "*" + - name: cppzmq-dev + version: "*" + - name: libprotobuf-dev + version: "*" + - name: libssl-dev + version: "*" + variables: + PROTOBUF_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu + PROTOBUF_INCLUDE_PREFIX: /usr/include + ZMQ_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu + ZMQ_INCLUDE_PREFIX: /usr/include + validation_cfg: + version_mandatory: true +version: 1.0.0 diff --git a/test_container/__init__.py b/test_container/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test_container/__init__.py @@ -0,0 +1 @@ + diff --git a/test_container/build/Dockerfile b/test_container/build/Dockerfile new file mode 100644 index 0000000..662fe8e --- /dev/null +++ b/test_container/build/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:22.04 +ENV DEBIAN_FRONTEND=noninteractive + +COPY deps/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc + +RUN apt-get -y update && \ + apt-get -y install \ + openjdk-17-jdk \ + locales \ + python3.10-venv \ + python3-pip \ + unixodbc \ + unixodbc-dev \ + curl \ + git && \ + locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + apt-get -y clean && \ + apt-get -y autoremove && \ + ldconfig + +RUN curl -fsSL https://get.docker.com | bash + +RUN mkdir -p downloads/ODBC downloads/JDBC downloads/EXAplus +RUN curl -s https://x-up.s3.amazonaws.com/7.x/24.1.1/Exasol_ODBC-24.1.1-Linux_x86_64.tar.gz | tar -C downloads/ODBC --strip-components 2 -zxf - +RUN curl -s https://x-up.s3.amazonaws.com/7.x/24.1.1/Exasol_JDBC-24.1.1.tar.gz | tar -C downloads/JDBC --strip-components 2 -zxf - +RUN curl -s https://x-up.s3.amazonaws.com/7.x/24.1.1/EXAplus-24.1.1.tar.gz | tar -C downloads/EXAplus --strip-components 2 -zxf - +ENV EXAPLUS=/downloads/EXAplus/exaplus + +COPY deps/requirements.txt requirements.txt +RUN pip3 install -r "requirements.txt" diff --git a/test_container/build/deps/01_nodoc b/test_container/build/deps/01_nodoc new file mode 100644 index 0000000..7320020 --- /dev/null +++ b/test_container/build/deps/01_nodoc @@ -0,0 +1,9 @@ +path-exclude /usr/share/doc/* +# we need to keep copyright files for legal reasons +path-include /usr/share/doc/*/copyright +path-exclude /usr/share/man/* +path-exclude /usr/share/groff/* +path-exclude /usr/share/info/* +# lintian stuff is small, but really unnecessary +path-exclude /usr/share/lintian/* +path-exclude /usr/share/linda/* diff --git a/test_container/build/deps/requirements.txt b/test_container/build/deps/requirements.txt new file mode 100644 index 0000000..bc2f1da --- /dev/null +++ b/test_container/build/deps/requirements.txt @@ -0,0 +1,6 @@ +pyodbc +pytz +lxml +docker +scipy +exasol-python-test-framework==0.6.0 diff --git a/test_container/test_data/enginedb_small/import.sql b/test_container/test_data/enginedb_small/import.sql new file mode 100644 index 0000000..d851311 --- /dev/null +++ b/test_container/test_data/enginedb_small/import.sql @@ -0,0 +1,3 @@ +-- Intentionally empty for the initial smoke setup. +-- The first flavor-CI milestone only checks container bootstrap, +-- database connectivity, and that the built release payload is mounted. diff --git a/test_container/tests/__init__.py b/test_container/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test_container/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/test_container/tests/test/__init__.py b/test_container/tests/test/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test_container/tests/test/__init__.py @@ -0,0 +1 @@ + diff --git a/test_container/tests/test/smoke/__init__.py b/test_container/tests/test/smoke/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test_container/tests/test/smoke/__init__.py @@ -0,0 +1 @@ + diff --git a/test_container/tests/test/smoke/smoke_test.py b/test_container/tests/test/smoke/smoke_test.py new file mode 100644 index 0000000..ff4ed81 --- /dev/null +++ b/test_container/tests/test/smoke/smoke_test.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +from exasol_python_test_framework import udf + + +class SmokeTest(udf.TestCase): + def test_db_connection(self): + rows = self.query("SELECT 1 FROM dual") + self.assertRowsEqual([(1,)], rows) + + +if __name__ == "__main__": + udf.main() diff --git a/udf-runner-cpp/v1/benchmark_container/benchmark_container.cc b/udf-runner-cpp/v1/benchmark_container/benchmark_container.cc index 92de436..7cd910d 100644 --- a/udf-runner-cpp/v1/benchmark_container/benchmark_container.cc +++ b/udf-runner-cpp/v1/benchmark_container/benchmark_container.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include using namespace SWIGVMContainers; diff --git a/udf-runner-cpp/v1/exa_vm_factory.cc b/udf-runner-cpp/v1/exa_vm_factory.cc index c510fc0..3f4ef73 100644 --- a/udf-runner-cpp/v1/exa_vm_factory.cc +++ b/udf-runner-cpp/v1/exa_vm_factory.cc @@ -13,14 +13,14 @@ std::function create_vm(const std::string& argv_lang, bool use_ctpg_options_parser) { if(argv_lang.compare("lang=streaming") == 0) { #ifdef ENABLE_STREAMING_VM - return []() { return new SWIGVMContainers::StreamingVM(false); }; + return []() { return new StreamingVM(false); }; #else throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Streaming support"); #endif } else if(argv_lang.compare("lang=benchmark") == 0) { #ifdef ENABLE_BENCHMARK_VM - return []() { return new SWIGVMContainers::BenchmarkVM(false); }; + return []() { return new BenchmarkVM(false); }; #else throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Benchmark support"); #endif diff --git a/udf-runner-cpp/v1/streaming_container/streamingcontainer.h b/udf-runner-cpp/v1/streaming_container/streamingcontainer.h index b45faac..295fd3e 100644 --- a/udf-runner-cpp/v1/streaming_container/streamingcontainer.h +++ b/udf-runner-cpp/v1/streaming_container/streamingcontainer.h @@ -1,4 +1,15 @@ +#include "exaudflib/vm/swig_vm.h" +#include "exaudflib/swig/swig_meta_data.h" +#include "exaudflib/swig/swig_result_handler.h" +#include "exaudflib/swig/swig_table_iterator.h" + +#include +#include +#include +#include +#include + using namespace SWIGVMContainers; using namespace std;