diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 4749f95..a24aa99 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -9,9 +9,6 @@ on: env: USE_BAZEL_VERSION: 8.3.1 BAZEL_PACKAGE_VERSION: 8.3.1 - JAVA_PREFIX: /usr/lib/jvm/java-17-openjdk-amd64 - PYTHON3_PREFIX: /usr - PYTHON3_VERSION: python3 PROTOBUF_BIN: protoc PROTOBUF_LIBRARY_PREFIX: /usr/lib/x86_64-linux-gnu PROTOBUF_INCLUDE_PREFIX: /usr/include @@ -28,7 +25,7 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y curl libprotobuf-dev libzmq3-dev openjdk-17-jdk protobuf-compiler python3-dev python3-numpy valgrind + sudo apt-get install -y curl libprotobuf-dev libzmq3-dev protobuf-compiler valgrind - name: Install bazel run: | BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" @@ -70,7 +67,7 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y curl libprotobuf-dev libzmq3-dev openjdk-17-jdk protobuf-compiler python3-dev python3-numpy valgrind + sudo apt-get install -y curl libprotobuf-dev libzmq3-dev protobuf-compiler valgrind - name: Install bazel run: | BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" diff --git a/udf-runner-cpp/v1/.bazelrc b/udf-runner-cpp/v1/.bazelrc index 2492aa6..99cdc8f 100644 --- a/udf-runner-cpp/v1/.bazelrc +++ b/udf-runner-cpp/v1/.bazelrc @@ -1,8 +1,6 @@ build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX # TODO add environment variables for R libraries build:benchmark --define benchmark=true -build:java --define java=true --action_env=JAVA_PREFIX -build:python --define python=true --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION build:fast-binary --define binary_type=fast_binary //:udf_runner_cpp_v1_gen build:slow-wrapper --define binary_type=slow_wrapper //:udf_runner_cpp_v1_gen build:static-binary //:udf_runner_cpp_v1_static_gen @@ -26,5 +24,4 @@ build:valgrind -c dbg build:valgrind --copt -g build:valgrind --strip=never build:valgrind --copt -DVALGRIND_ACTIVE -build:fix_conda_ar_tool --features=-archive_param_file build:plugin_client --define plugin_client=true diff --git a/udf-runner-cpp/v1/.gitignore b/udf-runner-cpp/v1/.gitignore index 3876f67..a28bde6 100644 --- a/udf-runner-cpp/v1/.gitignore +++ b/udf-runner-cpp/v1/.gitignore @@ -3,6 +3,7 @@ bazel-genfiles bazel-out bazel-src bazel-testlogs +bazel-v1 bazel-exaudfclient graph.in graph.png diff --git a/udf-runner-cpp/v1/BUILD b/udf-runner-cpp/v1/BUILD index b69a9fc..e45cab1 100644 --- a/udf-runner-cpp/v1/BUILD +++ b/udf-runner-cpp/v1/BUILD @@ -6,19 +6,11 @@ load("//:variables.bzl", "VM_ENABLED_DEFINES") VM_ENABLED_DEPS=select({ "@exaudfclient_base//:benchmark": ["@exaudfclient_base//benchmark_container:benchmark_container"], "//conditions:default": [] - }) + select({ - "@exaudfclient_base//:java": ["@exaudfclient_base//javacontainer:javacontainer"], - "//conditions:default": [] }) + select({ "@exaudfclient_base//:bash": ["@exaudfclient_base//streaming_container:streamingcontainer"], "//conditions:default": [] }) -VM_PYTHON3_DEPS=select({ - "@exaudfclient_base//:python": ["@exaudfclient_base//python/python3:pythoncontainer"], - "//conditions:default": [] - }) - genrule( name = "exaudflib_output_path_header", @@ -55,7 +47,7 @@ cc_library( "@exaudfclient_base//exaudflib:header", "@exaudfclient_base//utils:utils", "@exaudfclient_base//exaudflib:exaudflib-deps", - ] + VM_ENABLED_DEPS + VM_PYTHON3_DEPS, + ] + VM_ENABLED_DEPS, defines = VM_ENABLED_DEFINES, data = ["@exaudfclient_base//:libexaudflib_complete.so"], ) @@ -77,8 +69,7 @@ cc_binary( ## With this binary we simulate an error in our build system, that is a direct depedency to protobuf/zmq, ## which then must be detected with the linker namespace tests: ## test/linker_namespace_sanity/linker_namespace_sanity.py checks the wrong configuration -## Besides this the test under test/python3/all/linker_namespace.py checks the normal build, which expects -## not to find any occurence of the dependencies (protobuf/zmq) in the primary linker namespace. +## The normal build should not expose protobuf/zmq in the primary linker namespace. ## ## We need to explicitly declare the dependency of protobuf/zmq here, as the exaudflib is a static lib (//exaudflib:exaudflib) ## and hence does not contain dependency information. We cannot declare the shared lib (:exaudflib_complete.so) @@ -99,8 +90,8 @@ cc_binary( data = ["@exaudfclient_base//:libexaudflib_complete.so"], ) -# Workarround for the hardcoded paths in exaudfclient for libexaudflib_complete.so and python_ext_dataframe.cc -# - libexaudflib_complete.so and python_ext_dataframe.cc get dynamically loaded, therefore the exaudfclient needs to know their paths +# Workarround for the hardcoded paths in udf-runner-cpp for libexaudflib_complete.so +# - libexaudflib_complete.so gets dynamically loaded, therefore the udf-runner-cpp needs to know their path # - Most flexible way to provides these paths would environment variables # - The exasol database can't provide these paths, because they depend on the container # - A workarround to provide these paths would be wrapper bash script which set these environment variables diff --git a/udf-runner-cpp/v1/MODULE.bazel b/udf-runner-cpp/v1/MODULE.bazel index 1f99449..6b30947 100644 --- a/udf-runner-cpp/v1/MODULE.bazel +++ b/udf-runner-cpp/v1/MODULE.bazel @@ -4,31 +4,9 @@ bazel_dep(name = "exaudfclient_base", version = "1.0.0") #For now, the Bazel Modul "exaudfclient_base" is stored in subdirectory "base". #Later we might fetch it via Http or Git (after splitting up script-languages repository) local_path_override(module_name="exaudfclient_base", path="base") -bazel_dep(name = "rules_jvm_external", version = "6.2") - -numpy_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "numpy_local_repository") -numpy_local_repository(name = "numpy") - -python_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "python_local_repository") -python_local_repository(name = "python3", python_version="python3") zmq_local_repository = use_repo_rule("@exaudfclient_base//:zmq_repository.bzl", "zmq_local_repository") zmq_local_repository(name = "zmq") protobuf_local_repository = use_repo_rule("@exaudfclient_base//:protobuf_repository.bzl", "protobuf_local_repository") protobuf_local_repository(name = "protobuf") - -java_local_repository = use_repo_rule("@exaudfclient_base//:java_repository.bzl", "java_local_repository") -java_local_repository(name = "java") - - -maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") -maven.install( - artifacts = [ - "com.exasol:udf-api-java:1.0.5", - ], - repositories = [ - "https://repo1.maven.org/maven2", - ], -) -use_repo(maven, "maven") diff --git a/udf-runner-cpp/v1/README.md b/udf-runner-cpp/v1/README.md index 3614922..6019b4f 100644 --- a/udf-runner-cpp/v1/README.md +++ b/udf-runner-cpp/v1/README.md @@ -1,17 +1,16 @@ -# What is the exaudfclient? +# What is the udf-runner-cpp? -The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in the language Python, Java and R. Further languages can be integrated via language binding between C/C++ and the desired langauge. Python 3, Java and R use [SWIG](http://www.swig.org/) for the language binding. +The udf-runner-cpp connects to the database via [ZeroMQ](https://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). This repository contains the C++ runner, the benchmark and streaming VM support. -# How to build the exaudfclient? +# How to build the udf-runner-cpp? ## Prerequisites For the build system: -- Open JDK 11 - bazel-7.2.1 for more details see [Bazel documentation](https://docs.bazel.build/versions/master/install.html) -The exaudfclient was tested with the following versions of its dependencies: +The udf-runner-cpp is tested with the following versions of its dependencies: - swig-2.0.4 or swig-3.0.12 - protobuf 3.12.4 @@ -21,14 +20,11 @@ The exaudfclient was tested with the following versions of its dependencies: For the language support: -- Python 3.10 for pythoncontainer - - for Python 3 the build requires [Numpy](https://www.numpy.org/) and [Pandas](https://pandas.pydata.org/) in addition for the Pandas Dataframe Support -- OpenJDK 11 for javacontainer -- R 4.4 for the rcontainer +- no additional language-specific tool-chains are required for the retained runner modes in this repository. ## Start a build -The exaudfclient is a multi-language projects. Therefore, we are using [Bazel](https://docs.bazel.build/versions/master/bazel-overview.html) as build system, because it provide build support for many languages and allows to mix these languages. Because the exaudfclient has language bindings to Python 2/3, Java and R, we need specify where Bazel can find the correponding library- and header-files. This is done by Environment Variables. +The udf-runner-cpp is built with [Bazel](https://docs.bazel.build/versions/master/bazel-overview.html). In this repository, the remaining build-time environment variables are used for protobuf and zeromq discovery. For executing the build locally, you can use the script @@ -44,11 +40,9 @@ and set the Environment Variables via Docker. Both build script can receive parameters, such as Bazel commandline parameter, Bazel define (--define {key}={value}) and targets. -With Bazel defines you can specify which language support is actually compiled into you exaudfclient executbale. The currently supported defines are +With Bazel defines you can specify which retained VM support is compiled into your udf-runner-cpp executable. The currently supported defines are - --define java=true - --define r=true - --define python=true + --define bash=true --define benchmark=true # This language is only for test and development purpose and benchmarks the performance of the C++ Implementation @@ -60,11 +54,11 @@ Bazel allows to query the dependencies of a target. Furthermore, it can export t visualize_deps.sh -# How is the exaudfclient structured? +# How is the udf-runner-cpp structured? -The exaudfclient consists mainly out of three parts the main function in [exa_udfclient.cc](exa_udfclient.cc), the libexaudf and the language implementations. The first part the main function actually only loads the two other parts. However, in this case it is important how it loads the two other parts, because we need the libexaudf in a different linker namespace than the language implementation to prevent library conflicts. The libexaudf uses [ZeroMQ](http://zeromq.org/) and [Protobuf](https://developers.google.com/protocol-buffers/) to communicate with the Exsol Database, but UDFs could be use the same libraries in a different version which would lead to library conflict. The following figure shows the dependencies between the components. +The udf-runner-cpp consists mainly of three parts: the main function in [exa_udfclient.cc](exa_udfclient.cc), the libexaudf, and the retained language implementations. The first part actually only loads the two other parts. However, in this case it is important how it loads the two other parts, because we need the libexaudf in a different linker namespace than the language implementation to prevent library conflicts. The libexaudf uses [ZeroMQ](https://zeromq.org/) and [Protobuf](https://developers.google.com/protocol-buffers/) to communicate with the Exasol database, but UDFs could use the same libraries in a different version which would lead to library conflicts. The following figure shows the dependencies between the components. -![exaudfclient dependencies](docs/exaudfclient.png) +![udf-runner-cpp dependencies](docs/exaudfclient.png) The usage of multiple linker namespace requires some precautions in the build process and in the implementation. diff --git a/udf-runner-cpp/v1/base/.bazelrc b/udf-runner-cpp/v1/base/.bazelrc index 4a15be7..2316c08 100644 --- a/udf-runner-cpp/v1/base/.bazelrc +++ b/udf-runner-cpp/v1/base/.bazelrc @@ -1,7 +1,5 @@ build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX build:benchmark --define benchmark=true -build:java --define java=true --action_env=JAVA_PREFIX -build:python --define python=true --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION build:debug-build --sandbox_debug --config=verbose build:no-symlinks --symlink_prefix=/ build:asan --strip=never diff --git a/udf-runner-cpp/v1/base/BUILD b/udf-runner-cpp/v1/base/BUILD index 98ce24c..49d2eaf 100644 --- a/udf-runner-cpp/v1/base/BUILD +++ b/udf-runner-cpp/v1/base/BUILD @@ -12,16 +12,6 @@ config_setting( define_values = {"benchmark": "true"}, ) -config_setting( - name = "python", - define_values = {"python": "true"}, -) - -config_setting( - name = "java", - define_values = {"java": "true"}, -) - config_setting( name = "bash", define_values = {"bash": "true"}, diff --git a/udf-runner-cpp/v1/base/MODULE.bazel b/udf-runner-cpp/v1/base/MODULE.bazel index 6031111..1e2a4ab 100644 --- a/udf-runner-cpp/v1/base/MODULE.bazel +++ b/udf-runner-cpp/v1/base/MODULE.bazel @@ -2,41 +2,9 @@ module(name="exaudfclient_base", version = "1.0") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "googletest", version = "1.15.0") -bazel_dep(name = "rules_jvm_external", version = "6.2") - -# The following is necessary to build the UDF client for Java -# Since Bazel >8 the Python hermetic build rules must not run as root -# However, since the UDF client is built as root in a docker container -# we must disable this behavior. The only way is to configure the Python hermetic toolchain -bazel_dep(name = "rules_python", version = "1.7.0") - -python = use_extension("@rules_python//python/extensions:python.bzl", "python") -python.defaults(python_version = "3.11") -python.toolchain(python_version = "3.11", ignore_root_user_error = True,) - - -numpy_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "numpy_local_repository") -numpy_local_repository(name = "numpy") - -python_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "python_local_repository") -python_local_repository(name = "python3", python_version="python3") zmq_local_repository = use_repo_rule("@exaudfclient_base//:zmq_repository.bzl", "zmq_local_repository") zmq_local_repository(name = "zmq") protobuf_local_repository = use_repo_rule("@exaudfclient_base//:protobuf_repository.bzl", "protobuf_local_repository") protobuf_local_repository(name = "protobuf") - -java_local_repository = use_repo_rule("@exaudfclient_base//:java_repository.bzl", "java_local_repository") -java_local_repository(name = "java") - -maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") -maven.install( - artifacts = [ - "com.exasol:udf-api-java:1.0.5", - ], - repositories = [ - "https://repo1.maven.org/maven2", - ], -) -use_repo(maven, "maven") diff --git a/udf-runner-cpp/v1/base/exaudfclient.template.sh b/udf-runner-cpp/v1/base/exaudfclient.template.sh index ca8d30d..ddf1c90 100644 --- a/udf-runner-cpp/v1/base/exaudfclient.template.sh +++ b/udf-runner-cpp/v1/base/exaudfclient.template.sh @@ -3,5 +3,4 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" echo "Changing to script directory $SCRIPT_DIR" cd "$SCRIPT_DIR" || return 1 -export LIBPYEXADATAFRAME_DIR="$SCRIPT_DIR/external/exaudfclient_base+/python/python3" export LD_LIBRARY_PATH="/opt/conda/cuda-compat/:$LD_LIBRARY_PATH" #Temporary hack for the Cuda ML flavor(s) diff --git a/udf-runner-cpp/v1/base/java_repository.bzl b/udf-runner-cpp/v1/base/java_repository.bzl deleted file mode 100644 index 76cdb74..0000000 --- a/udf-runner-cpp/v1/base/java_repository.bzl +++ /dev/null @@ -1,53 +0,0 @@ -load("@bazel_skylib//lib:paths.bzl", "paths") - -# Workaround for the problems of JNI/JVM with rpath's used by Bazel. -# Problem Description: -# - Bazel creates for all external local repositories symlinks into its build directory -# - Bazel than writes during compilation these paths to the rpath of the binary -# - JNI/JVM uses the rpath for loading additonal shared libraries, -# but for a unknown reason it seems to truncate these pathes. -# - We assume two possible reason, the symlink paths of bazel contain a @, or -# the paths are to long and get for that reason truncated -# - In the end, it we wasn't able to convinve JNI/JVM from other pathes -# for its libraries, as the original pathes where apt installed the files. -# Solution: -# - We solved the problem by adding the original pahtes for the libraries into the rpath via the linkopts - -def _find_shared_libraries(prefix,library,p_repository_ctx): - command_result = p_repository_ctx.execute(["find", '%s'%prefix,'-name','%s'%library]) #TODO only one result - if command_result.return_code != 0: - fail("Could not acquire path of libjvm.so, got return code %s stderr: \n %s" - % (command_result.return_code, command_result.stderr)) - path_to_library = command_result.stdout.strip("\n") - print("path to %s: %s"%(library,path_to_library)) - return path_to_library - -def _java_local_repository_impl(repository_ctx): - prefix = "/usr/lib/jvm/java-11-openjdk-amd64" - if 'JAVA_PREFIX' in repository_ctx.os.environ: - prefix = repository_ctx.os.environ['JAVA_PREFIX'] - print("java prefix in environment specified; %s"%prefix) - - path_to_libjvm = paths.dirname(_find_shared_libraries(prefix,"libjvm.so",repository_ctx)) - - defines = '"ENABLE_JAVA_VM"' - build_file_content = """ -cc_library( - name = "java", - srcs = glob(["{prefix}/include/*.h"], allow_empty=False), - hdrs = glob(["{prefix}/include/*.h","{prefix}/include/linux/*.h"], allow_empty=False), - includes = ["{prefix}/include","{prefix}/include/linux"], - defines = [{defines}], - linkopts = ["-ljvm","-L{rpath_libjvm}",'-Wl,-rpath','{rpath_libjvm}'], - visibility = ["//visibility:public"] -)""".format( defines=defines, prefix="java", rpath_libjvm=path_to_libjvm) - print(build_file_content) - - repository_ctx.symlink(prefix, "./java") - repository_ctx.file("BUILD", build_file_content) - -java_local_repository = repository_rule( - implementation=_java_local_repository_impl, - local = True, - environ = ["JAVA_PREFIX"]) - diff --git a/udf-runner-cpp/v1/base/javacontainer/BUILD b/udf-runner-cpp/v1/base/javacontainer/BUILD deleted file mode 100644 index b4df439..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/BUILD +++ /dev/null @@ -1,151 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -genrule( - name = "exascript_java_tmp_cc", - cmd = """ - mkdir -p java_src/com/exasol/swig - mkdir -p build_exascript_java_tmp_cc/exaudflib - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_java_tmp_cc/exaudflib - cd build_exascript_java_tmp_cc - swig -v -O -DEXTERNAL_PROCESS -Wall -c++ -java -addextern -module exascript_java -package com.exasol.swig -outdir "../java_src/com/exasol/swig" -o "../exascript_java_tmp.cc" exaudflib/exascript.i - cd .. - cp java_src/com/exasol/swig/ConnectionInformationWrapper.java $(location ConnectionInformationWrapper.java) - cp java_src/com/exasol/swig/exascript_java.java $(location exascript_java.java) - cp java_src/com/exasol/swig/exascript_javaJNI.java $(location raw/exascript_javaJNI.java) - cp java_src/com/exasol/swig/ExportSpecificationWrapper.java $(location ExportSpecificationWrapper.java) - cp java_src/com/exasol/swig/ImportSpecificationWrapper.java $(location ImportSpecificationWrapper.java) - cp java_src/com/exasol/swig/Metadata.java $(location Metadata.java) - cp java_src/com/exasol/swig/ResultHandler.java $(location raw/ResultHandler.java) - cp java_src/com/exasol/swig/SWIGTYPE_p_ExecutionGraph__ExportSpecification.java $(location SWIGTYPE_p_ExecutionGraph__ExportSpecification.java) - cp java_src/com/exasol/swig/SWIGTYPE_p_ExecutionGraph__ImportSpecification.java $(location SWIGTYPE_p_ExecutionGraph__ImportSpecification.java) - cp java_src/com/exasol/swig/SWIGVM_datatype_e.java $(location SWIGVM_datatype_e.java) - cp java_src/com/exasol/swig/SWIGVM_itertype_e.java $(location SWIGVM_itertype_e.java) - cp java_src/com/exasol/swig/TableIterator.java $(location raw/TableIterator.java) - cp exascript_java_tmp.cc $(location exascript_java_tmp.cc) - """, - outs = ["ConnectionInformationWrapper.java", - "exascript_java.java", - "ExportSpecificationWrapper.java", - "ImportSpecificationWrapper.java", - "Metadata.java", - "SWIGTYPE_p_ExecutionGraph__ExportSpecification.java", - "SWIGTYPE_p_ExecutionGraph__ImportSpecification.java", - "SWIGVM_datatype_e.java", - "SWIGVM_itertype_e.java", - "raw/exascript_javaJNI.java", - "raw/ResultHandler.java", - "raw/TableIterator.java", - "exascript_java_tmp.cc" - ], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h"] -) - -genrule( - name = "exascript_java_tmp_h", - cmd = """ - mkdir build_exascript_java_tmp_h - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_java_tmp_h - cp -r "$(location exascript_java_tmp.cc)" build_exascript_java_tmp_h - cd build_exascript_java_tmp_h - swig -v -DEXTERNAL_PROCESS -c++ -java -external-runtime "../$(location exascript_java_tmp.h)" - """, - outs = ["exascript_java_tmp.h"], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h", ":exascript_java_tmp.cc"] -) - - -genrule( - name = "filter_swig_code_exascript_java_h", - cmd = 'python3 $(location //:filter_swig_code.py) "$@" "$<"', - outs = ["exascript_java.h"], - srcs = [":exascript_java_tmp_h"], - tools = ["//:filter_swig_code.py"] -) - -genrule( - name = "filter_swig_code_exascript_java_cc", - cmd = """ - TMPDIR=`mktemp -d` - mkdir -p "$$TMPDIR"/java_src/com/exasol/swig - cp $(location :raw/TableIterator.java) "$$TMPDIR"/java_src/com/exasol/swig/ - cp $(location :raw/ResultHandler.java) "$$TMPDIR"/java_src/com/exasol/swig/ - cp $(location :raw/exascript_javaJNI.java) "$$TMPDIR"/java_src/com/exasol/swig/ - find "$$TMPDIR"/java_src/com/exasol/swig/ -name *.java -type f -exec chmod 644 {} \\; - cp -r -L $(location //:filter_swig_code.py) $(location :exascript_java_tmp.cc) "$$TMPDIR" - (cd "$$TMPDIR" - python3 filter_swig_code.py "exascript_java.cc" "exascript_java_tmp.cc") - cp "$$TMPDIR"/exascript_java.cc $(location :exascript_java.cc) - cp "$$TMPDIR"/java_src/com/exasol/swig/TableIterator.java $(location TableIterator.java) - cp "$$TMPDIR"/java_src/com/exasol/swig/ResultHandler.java $(location ResultHandler.java) - cp "$$TMPDIR"/java_src/com/exasol/swig/exascript_javaJNI.java $(location exascript_javaJNI.java) - rm -rf "$$TMPDIR" - """, - outs = [ - "exascript_java.cc", - "TableIterator.java", - "ResultHandler.java", - "exascript_javaJNI.java" - ], - srcs = [":exascript_java_tmp.cc", ":raw/TableIterator.java", - ":raw/ResultHandler.java", ":raw/exascript_javaJNI.java"], - tools = ["//:filter_swig_code.py"] -) - -cc_library( - name = "exascript_java", - srcs = [":exascript_java.cc",], - deps = ["@java//:java","//exaudflib:exaudflib-deps","//exaudflib:header"], - copts= ["-O0","-fno-lto"], - # We limit this target to -O0 (no optimization) and -fno-lto. because otherwise we get compiler warnings of the sort - # note: "code may be misoptimized unless -fno-strict-aliasing is used." - # Normally, this indicates that compiling with higher optimizations levels may break your progrom. Unitl now, we don't have enough tests - # to verify if higher levels of optimizations might work. We needed to deactivate link time optimization, too, because this delay the compilation - # until the exaudfclient binary is build and causes than there the same problem. - alwayslink=True, -) - -cc_library( - name = "javacontainer", - srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", - ":javacontainer_builder.h", ":javacontainer_builder.cc", ":dummy"], - hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], - deps = ["@java//:java", ":exascript_java", "//exaudflib:header", - "//utils:utils","//javacontainer/script_options:java_script_option_lines", - "//swig_factory:swig_factory"], -# copts= ["-O0","-fno-lto"], - alwayslink=True, -) - -#workaround to build jars together with javacontainer c++ library -genrule( - name = "dummy", - cmd = 'touch "$@"', - outs = ["dummy.h"], - srcs = [":exaudf_deploy.jar"], -) - -java_binary( - name = "exaudf", - srcs = glob(["*.java"])+ - #The following are generated classes - [ - ":ConnectionInformationWrapper.java", - ":exascript_java.java", - ":exascript_javaJNI.java", - ":ExportSpecificationWrapper.java", - ":ImportSpecificationWrapper.java", - ":Metadata.java", - ":ResultHandler.java", - ":SWIGTYPE_p_ExecutionGraph__ExportSpecification.java", - ":SWIGTYPE_p_ExecutionGraph__ImportSpecification.java", - ":SWIGVM_datatype_e.java", - ":SWIGVM_itertype_e.java", - ":TableIterator.java" - ], - deps = ["@maven//:com_exasol_udf_api_java"], - create_executable=False, -) - -exports_files([ - "ExaStackTraceCleaner.java", - ]) diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaCompiler.java b/udf-runner-cpp/v1/base/javacontainer/ExaCompiler.java deleted file mode 100644 index 947fa0e..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaCompiler.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.exasol; - -import java.util.List; -import java.util.Arrays; -import java.util.ArrayList; -import java.io.StringWriter; -import javax.tools.SimpleJavaFileObject; -import javax.tools.JavaFileObject; -import javax.tools.ToolProvider; -import javax.tools.JavaCompiler; -import javax.tools.JavaCompiler.CompilationTask; -import java.net.URI; - -class ExaCompiler { - static void compile(String classname, String code, String classpath) throws ExaCompilationException { - temporaryClasspath = classpath; - JavaFileObject file = new JavaSource(classname, code); - Iterable compilationUnits = Arrays.asList(file); - List optionList = new ArrayList(Arrays.asList("-d", temporaryClasspath)); - - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - StringWriter compilationOutput = new StringWriter(); - CompilationTask task = compiler.getTask(compilationOutput, null, null, optionList, null, compilationUnits); - - boolean success = false; - try { - success = task.call(); - } catch (Exception e) { - // ignore - } - if (!success) { - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1158: "+compilationOutput.toString()); - } - } - - static void compile(String classname, String code) throws ExaCompilationException { - compile(classname, code, temporaryClasspath); - } - - private static class JavaSource extends SimpleJavaFileObject { - final String code; - - JavaSource(String classname, String code) { - super(URI.create("string:///" + classname.replace(".","/") + Kind.SOURCE.extension), Kind.SOURCE); - this.code = code; - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return this.code; - } - } - - private static String temporaryClasspath; -} diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaConnectionInformationImpl.java b/udf-runner-cpp/v1/base/javacontainer/ExaConnectionInformationImpl.java deleted file mode 100644 index 1d17c78..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaConnectionInformationImpl.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.exasol; - - -public class ExaConnectionInformationImpl implements ExaConnectionInformation { - - private ConnectionType type; - private String address; - private String user; - private String password; - - public ExaConnectionInformationImpl(String type, String address, String user, String password) - { - if (type.equals("password")) { - this.type = ConnectionType.PASSWORD; - } else { - throw new IllegalStateException("F-UDF-CL-SL-JAVA-1157: ExaConnectionInformationImpl: received unknown connection type: "+type); - } - this.address = address; - this.user = user; - this.password = password; - } - - @Override - public ConnectionType getType() {return type;} - - @Override - public String getAddress() {return address;} - - @Override - public String getUser() {return user;} - - @Override - public String getPassword() {return password;} - -} diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaExportSpecificationImpl.java b/udf-runner-cpp/v1/base/javacontainer/ExaExportSpecificationImpl.java deleted file mode 100644 index fe3f1d5..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaExportSpecificationImpl.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.exasol; - - -import java.util.List; -import java.util.Map; - - -public class ExaExportSpecificationImpl implements ExaExportSpecification { - - - private String connectionName; - private ExaConnectionInformation connectionInformation; - private Map parameters; - private boolean hasTruncate; - private boolean hasReplace; - private String createdBy; - private List sourceColumnNames; - - public ExaExportSpecificationImpl(String connectionName, ExaConnectionInformation connectionInformation, Map parameters, - boolean hasTruncate, boolean hasReplace, String createdBy, - List sourceColumnNames) { - this.connectionName = connectionName; - this.connectionInformation = connectionInformation; - this.parameters = parameters; - this.hasTruncate = hasTruncate; - this.hasReplace = hasReplace; - this.createdBy = createdBy; - this.sourceColumnNames = sourceColumnNames; - } - - @Override - public boolean hasConnectionName() { - return connectionName != null; - } - - @Override - public String getConnectionName() { - return connectionName; - } - - @Override - public boolean hasConnectionInformation() { - return connectionInformation != null; - } - - // same class as used for getConnection() - @Override - public ExaConnectionInformation getConnectionInformation() { - return connectionInformation; - } - - @Override - public Map getParameters() { - return parameters; - } - - @Override - public boolean hasTruncate() { - return hasTruncate; - } - - @Override - public boolean hasReplace() { - return hasReplace; - } - - @Override - public boolean hasCreatedBy() { - return createdBy != null; - } - - @Override - public String getCreatedBy() { - return createdBy; - } - - @Override - public List getSourceColumnNames() { - return sourceColumnNames; - } - -} diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaImportSpecificationImpl.java b/udf-runner-cpp/v1/base/javacontainer/ExaImportSpecificationImpl.java deleted file mode 100644 index 11e9c73..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaImportSpecificationImpl.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.exasol; - - -import java.util.List; -import java.util.Map; - - -public class ExaImportSpecificationImpl implements ExaImportSpecification { - - - private boolean isSubselect; - private List subselectColumnNames; - private List subselectColumnTypes; - private String connectionName; - private ExaConnectionInformation connectionInformation; - private Map parameters; - - public ExaImportSpecificationImpl(boolean isSubselect, List subselectColumnNames, List subselectColumnTypes, String connectionName, ExaConnectionInformation connectionInformation, Map parameters) { - this.isSubselect = isSubselect; - this.subselectColumnNames = subselectColumnNames; - this.subselectColumnTypes = subselectColumnTypes; - this.connectionName = connectionName; - this.connectionInformation = connectionInformation; - this.parameters = parameters; - } - - @Override - public boolean isSubselect() { - return isSubselect; - } - - @Override - public List getSubselectColumnNames() { - return subselectColumnNames; - } - - // returns a string like "VARCHAR(100)" - @Override - public List getSubselectColumnSqlTypes() { - return subselectColumnTypes; - } - - @Override - public boolean hasConnectionName() { - return connectionName != null; - } - - @Override - public String getConnectionName() { - return connectionName; - } - - @Override - public boolean hasConnectionInformation() { - return connectionInformation != null; - } - - // same class as used for getConnection() - @Override - public ExaConnectionInformation getConnectionInformation() { - return connectionInformation; - } - - @Override - public Map getParameters() { - return parameters; - } -} diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaIteratorImpl.java b/udf-runner-cpp/v1/base/javacontainer/ExaIteratorImpl.java deleted file mode 100644 index 567c10d..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaIteratorImpl.java +++ /dev/null @@ -1,679 +0,0 @@ -package com.exasol; - -import java.util.ArrayList; -import java.io.PrintWriter; -import java.lang.reflect.Method; -import java.lang.reflect.InvocationTargetException; -import java.math.BigDecimal; -import java.sql.Date; -import java.sql.Timestamp; -import java.io.IOError; -import com.exasol.swig.ResultHandler; -import com.exasol.swig.TableIterator; -import com.exasol.swig.Metadata; - -class ExaIteratorImpl implements ExaIterator { - private ExaMetadata exaMetadata; - private TableIterator tableIterator; - private ResultHandler resultHandler; - private boolean finished; - private boolean singleInput; - private boolean singleOutput; - private boolean insideRun; - private Object[] cache; - private String[] inputColumnTypes; - private String[] outputColumnTypes; - private ArrayList columnNames; - - @Override - public long size() throws ExaIterationException { - long size = tableIterator.rowsInGroup(); - String exMsg = tableIterator.checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaIterationException("F-UDF-CL-SL-JAVA-1101: "+exMsg); - } - return size; - } - - @Override - public boolean next() throws ExaIterationException { - if (insideRun && singleInput) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1102: next() function is not allowed in scalar context"); - clearCache(); - if (finished) - return false; - boolean next = tableIterator.next(); - String exMsg = tableIterator.checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaIterationException("F-UDF-CL-SL-JAVA-1103: "+exMsg); - } - if (!next) - finished = true; - return next; - } - - @Override - public void reset() throws ExaIterationException { - if (singleInput) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1104: reset() function is not allowed in scalar context"); - clearCache(); - tableIterator.reset(); - String exMsg = tableIterator.checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaIterationException("F-UDF-CL-SL-JAVA-1105: "+exMsg); - } - finished = false; - } - - @Override - public void emit(Object... values) throws ExaIterationException, ExaDataTypeException { - if (insideRun && singleOutput) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1106: emit() function is not allowed in scalar context"); - - if(values != null){ - if (values.length != exaMetadata.getOutputColumnCount()) { - String errorText = "E-UDF-CL-SL-JAVA-1107: emit() takes exactly " + exaMetadata.getOutputColumnCount(); - errorText += (exaMetadata.getOutputColumnCount() > 1) ? " arguments" : " argument"; - errorText += " (" + values.length + " given)"; - throw new ExaIterationException(errorText); - } - - for (int i = 0; i < values.length; i++) { - if (values[i] == null) { - resultHandler.setNull(i); - } - else if (values[i] instanceof Byte || values[i] instanceof Short || values[i] instanceof Integer) { - Number val = (Number) values[i]; - if (outputColumnTypes[i].equals("INT32")) - resultHandler.setInt32(i, val.intValue()); - else if (outputColumnTypes[i].equals("INT64")) - resultHandler.setInt64(i, val.longValue()); - else if (outputColumnTypes[i].equals("NUMERIC")) - resultHandler.setNumeric(i, val.toString()); - else if (outputColumnTypes[i].equals("DOUBLE")) - resultHandler.setDouble(i, val.doubleValue()); - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1108: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of type " + - outputColumnTypes[i] + - " but data given have type " + - values[i].getClass().getCanonicalName() - ); - } - else if (values[i] instanceof Long) { - Long val = (Long) values[i]; - if (outputColumnTypes[i].equals("INT32")) { - if (isConversionToIntegerSafe(val, exaMetadata.getOutputColumnName(i))) - resultHandler.setInt32(i, val.intValue()); - } - else if (outputColumnTypes[i].equals("INT64")) - resultHandler.setInt64(i, val.longValue()); - else if (outputColumnTypes[i].equals("NUMERIC")) - resultHandler.setNumeric(i, val.toString()); - else if (outputColumnTypes[i].equals("DOUBLE")) - resultHandler.setDouble(i, val.doubleValue()); - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1109: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of type " + - outputColumnTypes[i] + - " but data given have type " + - values[i].getClass().getCanonicalName() - ); - } - else if (values[i] instanceof Float || values[i] instanceof Double) { - Number val = (Number) values[i]; - if (outputColumnTypes[i].equals("INT32")) { - if (isConversionToIntegerSafe(val, exaMetadata.getOutputColumnName(i))) - resultHandler.setInt32(i, val.intValue()); - } - else if (outputColumnTypes[i].equals("INT64")) { - if (isConversionToLongSafe(val, exaMetadata.getOutputColumnName(i))) - resultHandler.setInt64(i, val.longValue()); - } - else if (outputColumnTypes[i].equals("NUMERIC")) - resultHandler.setNumeric(i, val.toString()); - else if (outputColumnTypes[i].equals("DOUBLE")) - resultHandler.setDouble(i, val.doubleValue()); - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1110: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of type "+ - outputColumnTypes[i] + - " but data given have type " + - values[i].getClass().getCanonicalName() - ); - } - else if (values[i] instanceof BigDecimal) { - BigDecimal val = (BigDecimal) values[i]; - if (outputColumnTypes[i].equals("INT32")) - resultHandler.setInt32(i, val.intValueExact()); - else if (outputColumnTypes[i].equals("INT64")) - resultHandler.setInt64(i, val.longValueExact()); - else if (outputColumnTypes[i].equals("NUMERIC")) - resultHandler.setNumeric(i, val.toString()); - else if (outputColumnTypes[i].equals("DOUBLE")) - resultHandler.setDouble(i, val.doubleValue()); - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1111: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of type " + - outputColumnTypes[i] + - " but data given have type " + - values[i].getClass().getCanonicalName() - ); - } - else if (values[i] instanceof Boolean) { - Boolean val = (Boolean) values[i]; - if (outputColumnTypes[i].equals("BOOLEAN")) - resultHandler.setBoolean(i, val.booleanValue()); - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1112: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of type " + - outputColumnTypes[i] + - " but data given have type " + - values[i].getClass().getCanonicalName() - ); - } - else if (values[i] instanceof String) { - String val = (String) values[i]; - if (outputColumnTypes[i].equals("STRING")) { - byte[] utf8Bytes = null; - try { - utf8Bytes = val.getBytes("UTF-8"); - } catch (java.io.UnsupportedEncodingException ex) { - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1113: Column with name '" + - exaMetadata.getOutputColumnName(i) + - "' contains invalid UTF-8 data" - ); - } - resultHandler.setString(i, utf8Bytes); - } - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1114: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of type " + - outputColumnTypes[i] + - " but data given have type " + - values[i].getClass().getCanonicalName() - ); - } - else if (values[i] instanceof Date) { - Date val = (Date) values[i]; - if (outputColumnTypes[i].equals("DATE")) - resultHandler.setDate(i, val.toString()); - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1115: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of type " + - outputColumnTypes[i] + - " but data given have type " + - values[i].getClass().getCanonicalName() - ); - } - else if (values[i] instanceof Timestamp) { - Timestamp val = (Timestamp) values[i]; - if (outputColumnTypes[i].equals("TIMESTAMP")) - resultHandler.setTimestamp(i, val.toString()); - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1116: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of type " + - outputColumnTypes[i] + - " but data given have type " + - values[i].getClass().getCanonicalName() - ); - } - else { - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1117: emit column '" + - exaMetadata.getOutputColumnName(i) + - "' is of unsupported type " + - values[i].getClass().getCanonicalName() - ); - } - - String exMsg = resultHandler.checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1118: "+exMsg); - } - } - }else{ - if(exaMetadata.getOutputColumnCount()==1){ - resultHandler.setNull(0); - }else{ - String errorText = "E-UDF-CL-SL-JAVA-1119: emit() takes exactly " + exaMetadata.getOutputColumnCount(); - errorText += (exaMetadata.getOutputColumnCount() > 1) ? " arguments" : " argument"; - errorText += " (" + 1 + " given)"; - throw new ExaIterationException(errorText); - } - } - - boolean next = resultHandler.next(); - String exMsg = resultHandler.checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaIterationException("F-UDF-CL-SL-JAVA-1120: "+exMsg); - } - if (!next) { - throw new ExaIterationException("F-UDF-CL-SL-JAVA-1121: Internal error while emiting row"); - } - } - - @Override - public Integer getInteger(int column) throws ExaIterationException, ExaDataTypeException { - Object object = getObject(column); - if (object == null) - return null; - if (!isConversionToIntegerSafe(object, columnNames.get(column))) - return null; - if (object instanceof Integer) - return (Integer) object; - else if (object instanceof Long) - return new Integer(((Long) object).intValue()); - else if (object instanceof BigDecimal) - return new Integer(((BigDecimal) object).intValueExact()); - else if (object instanceof Double) - return new Integer(((Double) object).intValue()); - else - throw - new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1122: getInteger cannot convert column '" + - columnNames.get(column) + - "' of type " + - exaMetadata.getInputColumnSqlType(column) + - " to an Integer" - ); - } - - @Override - public Integer getInteger(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1123: Column with name '" + name + "' does not exist"); - return getInteger(col); - } - - @Override - public Long getLong(int column) throws ExaIterationException, ExaDataTypeException { - Object object = getObject(column); - if (object == null) - return null; - if (!isConversionToLongSafe(object, columnNames.get(column))) - return null; - if (object instanceof Integer) - return new Long(((Integer) object).longValue()); - else if (object instanceof Long) - return (Long) object; - else if (object instanceof BigDecimal) - return new Long(((BigDecimal) object).longValueExact()); - else if (object instanceof Double) - return new Long(((Double) object).longValue()); - else - throw - new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1124: getLong cannot convert column '" + - columnNames.get(column) + - "' of type " + - exaMetadata.getInputColumnSqlType(column) + - " to a Long" - ); - } - - @Override - public Long getLong(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1125: Column with name '" + name + "' does not exist"); - return getLong(col); - } - - @Override - public BigDecimal getBigDecimal(int column) throws ExaIterationException, ExaDataTypeException { - Object object = getObject(column); - if (object == null) - return null; - if (object instanceof Integer) - return new BigDecimal(((Integer) object).intValue()); - else if (object instanceof Long) - return new BigDecimal(((Long) object).longValue()); - else if (object instanceof BigDecimal) - return (BigDecimal) object; - else if (object instanceof Double) - return new BigDecimal(((Double) object).doubleValue()); - else - throw - new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1126: getBigDecimal cannot convert column '" + - columnNames.get(column) + - "' of type " + - exaMetadata.getInputColumnSqlType(column) + - " to a BigDecimal" - ); - } - - @Override - public BigDecimal getBigDecimal(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1127: Column with name '" + name + "' does not exist"); - return getBigDecimal(col); - } - - @Override - public Double getDouble(int column) throws ExaIterationException, ExaDataTypeException { - Object object = getObject(column); - if (object == null) - return null; - if (object instanceof Integer) - return new Double(((Integer)object).doubleValue()); - else if (object instanceof Long) - return new Double(((Long)object).doubleValue()); - else if (object instanceof BigDecimal) - return new Double(((BigDecimal)object).doubleValue()); - else if (object instanceof Double) - return (Double) object; - else - throw - new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1128: getDouble cannot convert column '" + - columnNames.get(column) + - "' of type " + - exaMetadata.getInputColumnSqlType(column) + - " to a Double" - ); - } - - @Override - public Double getDouble(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1129: Column with name '" + name + "' does not exist"); - return getDouble(col); - } - - @Override - public String getString(int column) throws ExaIterationException, ExaDataTypeException { - Object object = getObject(column); - if (object == null) - return null; - if (object instanceof String) - return (String) object; - else if (object instanceof Integer || object instanceof Long || object instanceof BigDecimal || object instanceof Double || - object instanceof Boolean || object instanceof Date || object instanceof Timestamp) - return object.toString(); - else - throw - new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1130: getString cannot convert column '" + - columnNames.get(column) + - "' of type " + - exaMetadata.getInputColumnSqlType(column) + - " to a String" - ); - } - - @Override - public String getString(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1131: Column with name '" + name + "' does not exist"); - return getString(col); - } - - @Override - public Boolean getBoolean(int column) throws ExaIterationException, ExaDataTypeException { - Object object = getObject(column); - if (object == null) - return null; - if (object instanceof Boolean) - return (Boolean) object; - else - throw new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1132: getBoolean cannot convert column '" + - columnNames.get(column) + - "' of type " + - exaMetadata.getInputColumnSqlType(column) + - " to a Boolean" - ); - } - - @Override - public Boolean getBoolean(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1133: Column with name '" + name + "' does not exist"); - return getBoolean(col); - } - - @Override - public Date getDate(int column) throws ExaIterationException, ExaDataTypeException { - Object object = getObject(column); - if (object == null) - return null; - if (object instanceof Date) - return (Date) object; - else - throw - new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1134: getDate cannot convert column '" + - columnNames.get(column) + - "' of type " + - exaMetadata.getInputColumnSqlType(column) + - " to a Date"); - } - - @Override - public Date getDate(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1135: Column with name '" + name + "' does not exist"); - return getDate(col); - } - - @Override - public Timestamp getTimestamp(int column) throws ExaIterationException, ExaDataTypeException { - Object object = getObject(column); - if (object == null) - return null; - if (object instanceof Timestamp) - return (Timestamp) object; - else - throw - new ExaDataTypeException( - "E-UDF-CL-SL-JAVA-1136: getTimestamp cannot convert column '" + - columnNames.get(column) + - "' of type " + - exaMetadata.getInputColumnSqlType(column) + - " to a Timestamp" - ); - } - - @Override - public Timestamp getTimestamp(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1137: Column with name '" + name + "' does not exist"); - return getTimestamp(col); - } - - @Override - public Object getObject(int column) throws ExaIterationException, ExaDataTypeException { - if (column < 0 || column >= exaMetadata.getInputColumnCount()) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1138: Column number " + column + " does not exist"); - - if (finished) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1139: Iteration finished"); - - if (cache[column] != null) - return cache[column]; - - Object val = null; - switch (inputColumnTypes[column]) { - case "INT32": - val = new Integer(tableIterator.getInt32(column)); - break; - case "INT64": - val = new Long(tableIterator.getInt64(column)); - break; - case "NUMERIC": - String numeric = tableIterator.getNumeric(column); - if (!tableIterator.wasNull()) - val = new BigDecimal(numeric); - break; - case "DOUBLE": - val = new Double(tableIterator.getDouble(column)); - break; - case "STRING": - byte[] utf8Bytes = tableIterator.getString(column); - try { - val = new String(utf8Bytes, "UTF-8"); - } catch (java.io.UnsupportedEncodingException ex) { - throw new ExaDataTypeException("F-UDF-CL-SL-JAVA-1140: Column with name '" + columnNames.get(column) + "' contains invalid UTF-8 data"); - } - break; - case "BOOLEAN": - val = new Boolean(tableIterator.getBoolean(column)); - break; - case "DATE": - String date = tableIterator.getDate(column); - if (!tableIterator.wasNull()) - val = Date.valueOf(date); - break; - case "TIMESTAMP": - String timestamp = tableIterator.getTimestamp(column); - if (!tableIterator.wasNull()) - val = Timestamp.valueOf(timestamp); - break; - default: - throw new ExaDataTypeException("F-UDF-CL-SL-JAVA-1141: Column with name '" + columnNames.get(column) + "' has an invalid data type"); - } - if (tableIterator.wasNull()) - val = null; - cache[column] = val; - return val; - } - - @Override - public Object getObject(String name) throws ExaIterationException, ExaDataTypeException { - int col = columnNames.indexOf(name); - if (col == -1) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1142: Column with name '" + name + "' does not exist"); - return getObject(col); - } - - private boolean isConversionToIntegerSafe(Object from, String name) throws ExaDataTypeException { - if (from == null) - return false; - - if (from instanceof Long) { - long val = (Long) from; - if (val > Integer.MAX_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1143: emit column '" + name + "' has value of " - + val + " but column can only have maximum value of " + Integer.MAX_VALUE); - if (val < Integer.MIN_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1144: emit column '" + name + "' has value of " - + val + " but column can only have minimum value of " + Integer.MIN_VALUE); - } - else if (from instanceof Float) { - float val = (Float) from; - if (val > Integer.MAX_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1145: emit column '" + name + "' has value of " - + val + " but column can only have maximum value of " + Integer.MAX_VALUE); - if (val < Integer.MIN_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1146: emit column '" + name + "' has value of " - + val + " but column can only have minimum value of " + Integer.MIN_VALUE); - if (val != Math.floor(val)) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1147: emit column '" + name + "' has a non-integer value of " + val); - } - else if (from instanceof Double) { - double val = (Double) from; - if (val > Integer.MAX_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1148: emit column '" + name + "' has value of " - + val + " but column can only have maximum value of " + Integer.MAX_VALUE); - if (val < Integer.MIN_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1149: emit column '" + name + "' has value of " - + val + " but column can only have minimum value of " + Integer.MIN_VALUE); - if (val != Math.floor(val)) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1150: emit column '" + name + "' has a non-integer value of " + val); - } - - return true; - } - - private boolean isConversionToLongSafe(Object from, String name) throws ExaDataTypeException { - if (from == null) - return false; - - if (from instanceof Float) { - float val = (Float) from; - if (val > Long.MAX_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1151: emit column '" + name + "' has value of " - + val + " but column can only have maximum value of " + Long.MAX_VALUE); - if (val < Long.MIN_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1152: emit column '" + name + "' has value of " - + val + " but column can only have minimum value of " + Long.MIN_VALUE); - if (val != Math.floor(val)) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1153: emit column '" + name + "' has a non-integer value of " + val); - } - else if (from instanceof Double) { - double val = (Double) from; - if (val > Long.MAX_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1154: emit column '" + name + "' has value of " - + val + " but column can only have maximum value of " + Long.MAX_VALUE); - if (val < Long.MIN_VALUE) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1155: emit column '" + name + "' has value of " - + val + " but column can only have minimum value of " + Long.MIN_VALUE); - if (val != Math.floor(val)) - throw new ExaDataTypeException("E-UDF-CL-SL-JAVA-1156: emit column '" + name + "' has a non-integer value of " + val); - } - - return true; - } - - private void clearCache() { - for (int i = 0; i < cache.length; i++) - cache[i] = null; - } - - void setInsideRun(boolean inside) { - insideRun = inside; - } - - ExaIteratorImpl(ExaMetadata exaMetadata, TableIterator tableIterator, ResultHandler resultHandler) throws ExaIterationException { - this.exaMetadata = exaMetadata; - this.tableIterator = tableIterator; - this.resultHandler = resultHandler; - - finished = false; - singleInput = this.exaMetadata.getInputType().equals("SCALAR") ? true : false; - singleOutput = this.exaMetadata.getOutputType().equals("RETURN") ? true : false; - insideRun = false; - cache = new Object[(int) this.exaMetadata.getInputColumnCount()]; - - Metadata tmp = new Metadata(); - inputColumnTypes = new String[(int) this.exaMetadata.getInputColumnCount()]; - for (int i = 0; i < inputColumnTypes.length; i++) { - inputColumnTypes[i] = tmp.inputColumnType(i).toString(); - } - outputColumnTypes = new String[(int) this.exaMetadata.getOutputColumnCount()]; - for (int i = 0; i < outputColumnTypes.length; i++) { - outputColumnTypes[i] = tmp.outputColumnType(i).toString(); - } - - columnNames = new ArrayList(); - for(int i = 0; i < this.exaMetadata.getInputColumnCount(); i++) { - columnNames.add(this.exaMetadata.getInputColumnName(i)); - } - } -} diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaMetadataImpl.java b/udf-runner-cpp/v1/base/javacontainer/ExaMetadataImpl.java deleted file mode 100644 index 0681112..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaMetadataImpl.java +++ /dev/null @@ -1,305 +0,0 @@ -package com.exasol; - -import java.math.BigInteger; -import java.util.ArrayList; -import com.exasol.swig.Metadata; -import com.exasol.swig.ConnectionInformationWrapper; - -class ExaMetadataImpl implements ExaMetadata { - private Metadata metadata; - - private String databaseName; - private String databaseVersion; - private String scriptLanguage; - private String scriptName; - private String currentUser; - private String scopeUser; - private String currentSchema; - private String scriptSchema; - private String scriptCode; - private String sessionId; - private long statementId; - private long nodeCount; - private long nodeId; - private String vmId; - private BigInteger memoryLimit; - private String inputType; - private long inputColumnCount; - private ColumnInfo[] inputColumns; - private String outputType; - private long outputColumnCount; - private ColumnInfo[] outputColumns; - private ArrayList importedScripts; - - @Override - public String getDatabaseName() { return databaseName; } - @Override - public String getDatabaseVersion() { return databaseVersion; } - @Override - public String getScriptLanguage() { return scriptLanguage; } - @Override - public String getScriptName() { return scriptName; } - @Override - public String getCurrentUser() { return currentUser; } - @Override - public String getScopeUser() { return scopeUser; } - @Override - public String getCurrentSchema() { return currentSchema; } - @Override - public String getScriptSchema() { return scriptSchema; } - @Override - public String getScriptCode() { return scriptCode; } - @Override - public String getSessionId() { return sessionId; } - @Override - public long getStatementId() { return statementId; } - @Override - public long getNodeCount() { return nodeCount; } - @Override - public long getNodeId() { return nodeId; } - @Override - public String getVmId() { return vmId; } - @Override - public BigInteger getMemoryLimit() { return memoryLimit; } - @Override - public String getInputType() { return inputType; } - @Override - public long getInputColumnCount() {return inputColumnCount; } - @Override - public String getInputColumnName(int column) throws ExaIterationException { - if (column < 0 || column >= inputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1083: Column number " + column + " does not exist"); - return inputColumns[column].name; - } - @Override - public Class getInputColumnType(int column) throws ExaIterationException { - if (column < 0 || column >= inputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1084: Column number " + column + " does not exist"); - return inputColumns[column].type; - } - @Override - public String getInputColumnSqlType(int column) throws ExaIterationException { - if (column < 0 || column >= inputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1085: Column number " + column + " does not exist"); - return inputColumns[column].sqlType; - } - @Override - public long getInputColumnPrecision(int column) throws ExaIterationException { - if (column < 0 || column >= inputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1086: Column number " + column + " does not exist"); - return inputColumns[column].precision; - } - @Override - public long getInputColumnScale(int column) throws ExaIterationException { - if (column < 0 || column >= inputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1087: Column number " + column + " does not exist"); - return inputColumns[column].scale; - } - @Override - public long getInputColumnLength(int column) throws ExaIterationException { - if (column < 0 || column >= inputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1088: Column number " + column + " does not exist"); - return inputColumns[column].length; - } - @Override - public String getOutputType() { return outputType; } - @Override - public long getOutputColumnCount() { return outputColumnCount; } - @Override - public String getOutputColumnName(int column) throws ExaIterationException { - if (column < 0 || column >= outputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1089: Column number " + column + " does not exist"); - return outputColumns[column].name; - } - @Override - public Class getOutputColumnType(int column) throws ExaIterationException { - if (column < 0 || column >= outputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1090: Column number " + column + " does not exist"); - return outputColumns[column].type; - } - @Override - public String getOutputColumnSqlType(int column) throws ExaIterationException { - if (column < 0 || column >= outputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1091: Column number " + column + " does not exist"); - return outputColumns[column].sqlType; - } - @Override - public long getOutputColumnPrecision(int column) throws ExaIterationException { - if (column < 0 || column >= outputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1092: Column number " + column + " does not exist"); - return outputColumns[column].precision; - } - @Override - public long getOutputColumnScale(int column) throws ExaIterationException { - if (column < 0 || column >= outputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1093: Column number " + column + " does not exist"); - return outputColumns[column].scale; - } - @Override - public long getOutputColumnLength(int column) throws ExaIterationException { - if (column < 0 || column >= outputColumns.length) - throw new ExaIterationException("E-UDF-CL-SL-JAVA-1094: Column number " + column + " does not exist"); - return outputColumns[column].length; - } - - @Override - public Class importScript(String name) throws ExaCompilationException, ClassNotFoundException { - if (name == null) - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1095: Script name is null"); - boolean isQuoted = (name.charAt(0) == '"' && name.charAt(name.length() - 1) == '"'); - String scriptName = isQuoted ? name.substring(1, name.length() - 1) : name.toUpperCase(); - if (!importedScripts.contains(scriptName)) { - String code = metadata.moduleContent(name); - code = "package com.exasol;\r\n" + code; - String exMsg = checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1096: "+exMsg); - } - try{ - ExaCompiler.compile("com.exasol." + scriptName, code); - }catch(ExaCompilationException ex){ - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1097: "+ex.toString()); - } - importedScripts.add(scriptName); - } - return Class.forName("com.exasol." + scriptName); - } - - @Override - public ExaConnectionInformation getConnection(String name) throws ExaConnectionAccessException { - if (name == null) { - throw new ExaConnectionAccessException("E-UDF-CL-SL-JAVA-1098: Connection name is null"); - } - boolean isQuoted = (name.charAt(0) == '"' && name.charAt(name.length() - 1) == '"'); - String connectionName = isQuoted ? name.substring(1, name.length() - 1) : name.toUpperCase(); - ConnectionInformationWrapper w = metadata.connectionInformation(connectionName); - String exMsg = checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaConnectionAccessException("E-UDF-CL-SL-JAVA-1099: "+exMsg); - } - return new ExaConnectionInformationImpl(w.copyKind(), w.copyAddress(), w.copyUser(), w.copyPassword()); - } - - public String checkException() { - return metadata.checkException(); - } - - ExaMetadataImpl() throws ClassNotFoundException, ExaDataTypeException { - metadata = new Metadata(); - - databaseName = metadata.databaseName(); - databaseVersion = metadata.databaseVersion(); - scriptLanguage = "Java " + System.getProperty("java.version"); - scriptName = metadata.scriptName(); - scriptSchema = metadata.scriptSchema(); - currentUser = metadata.currentUser(); - scopeUser = metadata.scopeUser(); - currentSchema = metadata.currentSchema(); - scriptCode = metadata.scriptCode(); - sessionId = metadata.sessionID_S(); - statementId = metadata.statementID(); - nodeCount = metadata.nodeCount(); - nodeId = metadata.nodeID(); - vmId = metadata.vmID_S(); - memoryLimit = metadata.memoryLimit(); - - inputType = (metadata.inputType().toString().equals("EXACTLY_ONCE")) ? "SCALAR" : "SET"; - inputColumnCount = metadata.inputColumnCount(); - inputColumns = new ColumnInfo[(int) inputColumnCount]; - for (int i = 0; i < inputColumns.length; i++) { - inputColumns[i] = new ColumnInfo(); - inputColumns[i].initInputColumnInfo(i); - } - - outputType = (metadata.outputType().toString().equals("EXACTLY_ONCE")) ? "RETURN" : "EMIT"; - outputColumnCount = metadata.outputColumnCount(); - outputColumns = new ColumnInfo[(int) outputColumnCount]; - for (int i = 0; i < outputColumns.length; i++) { - outputColumns[i] = new ColumnInfo(); - outputColumns[i].initOutputColumnInfo(i); - } - - importedScripts = new ArrayList(); - } - - private class ColumnInfo { - private String name; - private String exaType; - private Class type; - private String sqlType; - private long precision; - private long scale; - private long length; - - private void initInputColumnInfo(long column) throws ClassNotFoundException, ExaDataTypeException { - name = metadata.inputColumnName(column); - exaType = metadata.inputColumnType(column).toString(); - sqlType = metadata.inputColumnTypeName(column); - precision = metadata.inputColumnPrecision(column); - scale = metadata.inputColumnScale(column); - length = metadata.inputColumnSize(column); - setColumnInfo(); - } - - private void initOutputColumnInfo(long column) throws ClassNotFoundException, ExaDataTypeException { - name = metadata.outputColumnName(column); - exaType = metadata.outputColumnType(column).toString(); - sqlType = metadata.outputColumnTypeName(column); - precision = metadata.outputColumnPrecision(column); - scale = metadata.outputColumnScale(column); - length = metadata.outputColumnSize(column); - setColumnInfo(); - - } - - private void setColumnInfo() throws ClassNotFoundException, ExaDataTypeException { - switch(exaType) { - case "INT32": - type = Class.forName("java.lang.Integer"); - scale = 0; - length = 0; - break; - case "INT64": - type = Class.forName("java.lang.Long"); - scale = 0; - length = 0; - break; - case "NUMERIC": - type = Class.forName("java.math.BigDecimal"); - length = 0; - break; - case "DOUBLE": - type = Class.forName("java.lang.Double"); - precision = 0; - scale = 0; - length = 0; - break; - case "STRING": - type = Class.forName("java.lang.String"); - precision = 0; - scale = 0; - break; - case "BOOLEAN": - type = Class.forName("java.lang.Boolean"); - precision = 0; - scale = 0; - length = 0; - break; - case "DATE": - type = Class.forName("java.sql.Date"); - precision = 0; - scale = 0; - length = 0; - break; - case "TIMESTAMP": - type = Class.forName("java.sql.Timestamp"); - precision = 0; - scale = 0; - length = 0; - break; - default: - throw new ExaDataTypeException("F-UDF-CL-SL-JAVA-1100: data type " + exaType + " is not supported"); - } - } - } -} diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaStackTraceCleaner.java b/udf-runner-cpp/v1/base/javacontainer/ExaStackTraceCleaner.java deleted file mode 100644 index 58559c3..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaStackTraceCleaner.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.exasol; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.MalformedParameterizedTypeException; -import java.lang.reflect.Method; -import java.lang.reflect.UndeclaredThrowableException; -import java.io.StringWriter; -import java.io.PrintWriter; -import java.util.LinkedList; -import java.util.ListIterator; -import java.util.Arrays; - -class ExaStackTraceCleaner { - - public ExaStackTraceCleaner(final String triggerClassName) { - mTriggerClassName = triggerClassName; - } - - public String cleanStackTrace(final Throwable src) { - final Throwable th = unpack(src); - cleanExceptionChain(th); - return format(th); - } - - private Throwable unpack(final Throwable src) { - Throwable exc = src; - while (exc != null && (exc instanceof InvocationTargetException || - exc instanceof MalformedParameterizedTypeException || - exc instanceof UndeclaredThrowableException)) { - Throwable cause = exc.getCause(); - if (cause == null) - break; - else - exc = cause; - } - return exc; - } - - private String format(final Throwable src) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - src.printStackTrace(pw); - String stacktrace = sw.toString(); - LinkedList stacktrace_lines = new LinkedList(Arrays.asList(stacktrace.split("\\r?\\n"))); - - ListIterator list_Iter = stacktrace_lines.listIterator(0); - StringWriter stringWriter = new StringWriter(); - PrintWriter writer = new PrintWriter(stringWriter, true); - while (list_Iter.hasNext()) { - String line = (String) list_Iter.next(); - writer.println(line.replaceFirst("^\tat ", "")); - } - String cleanedStacktrace = stringWriter.toString(); - return cleanedStacktrace; - } - - private void cleanExceptionChain(final Throwable src) { - StackTraceElement[] stackTraceElements = src.getStackTrace(); - Integer start_index = null; - LinkedList newStackTrace = new LinkedList<>(); - - if (stackTraceElements.length > 0) { - for (int idxStackTraceElement = (stackTraceElements.length - 1); idxStackTraceElement >= 0; idxStackTraceElement--) { - StackTraceElement stackTraceElement = stackTraceElements[idxStackTraceElement]; - boolean addStackTrace = true; - if (stackTraceElement.getClassName().equals(mTriggerClassName)) { - if (start_index == null) { - start_index = idxStackTraceElement; - } - } else if ("java.base".equals(stackTraceElement.getModuleName())) { - if (start_index != null && - (stackTraceElement.getClassName().startsWith("jdk.internal.reflect") || - stackTraceElement.getClassName().startsWith("java.lang.reflect"))) { - addStackTrace = false; - } - } else { - start_index = null; - } - if (addStackTrace) { - newStackTrace.add(0, stackTraceElement); - } - } - StackTraceElement[] newArr = new StackTraceElement[newStackTrace.size()]; - newArr = newStackTrace.toArray(newArr); - src.setStackTrace(newArr); - } - if (src.getCause() != null) { - cleanExceptionChain(src.getCause()); - } - } - - private final String mTriggerClassName; -} \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaUndefinedSingleCallException.java b/udf-runner-cpp/v1/base/javacontainer/ExaUndefinedSingleCallException.java deleted file mode 100644 index 5bdab65..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaUndefinedSingleCallException.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.exasol; - -/** - * This class is not part of the public Java UDF API - */ -public class ExaUndefinedSingleCallException extends Exception { - private static final long serialVersionUID = 1L; - public ExaUndefinedSingleCallException(String undefinedRemoteFn) { - super("Undefined single call fn: "+undefinedRemoteFn); - this.undefinedRemoteFn = undefinedRemoteFn; - } - public String getUndefinedRemoteFn() { - return undefinedRemoteFn; - } - private String undefinedRemoteFn; -} diff --git a/udf-runner-cpp/v1/base/javacontainer/ExaWrapper.java b/udf-runner-cpp/v1/base/javacontainer/ExaWrapper.java deleted file mode 100644 index c881fdd..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/ExaWrapper.java +++ /dev/null @@ -1,273 +0,0 @@ -package com.exasol; - -import java.io.IOError; -import java.io.StringWriter; -import java.io.PrintWriter; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.MalformedParameterizedTypeException; -import java.lang.reflect.Method; -import java.lang.reflect.UndeclaredThrowableException; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.ListIterator; -import java.util.List; -import java.util.Map; -import java.util.ArrayList; -import java.util.HashMap; -import com.exasol.swig.ResultHandler; -import com.exasol.swig.TableIterator; -import com.exasol.swig.ImportSpecificationWrapper; -import com.exasol.swig.ExportSpecificationWrapper; -import com.exasol.swig.ConnectionInformationWrapper; -import com.exasol.ExaStackTraceCleaner; - -class ExaWrapper { - static byte[] runSingleCall(String fn, Object args) throws Throwable { - Class argClass = Object.class; - if (args != null) { - if (args instanceof ImportSpecificationWrapper) { - ImportSpecificationWrapper is = (ImportSpecificationWrapper)args; - List columnNames = new ArrayList(); - List columnTypes = new ArrayList(); - if (is.isSubselect()) { - for (int i=0; i parameters = new HashMap(); - for (int i=0; i parameters = new HashMap(); - for (int i=0; i sourceColumnNames = new ArrayList(); - for (int i=0; i 0) { - throw new ExaIterationException("UDF.CL.SL.JAVA-1161: "+exMsg); - } - // Take the scriptClass name specified by user in the script, or the name of the script as fallback - boolean userDefinedScriptName = true; - String scriptClassName = System.getProperty("exasol.scriptclass", ""); - if (scriptClassName.trim().isEmpty()) { - userDefinedScriptName = false; - scriptClassName = exaMetadata.getScriptName(); - scriptClassName = scriptClassName.replace('.', '_'); // ** see comment in run() method - scriptClassName = "com.exasol." + scriptClassName; - } - try { - Class scriptClass = Class.forName(scriptClassName); - if (args == null) { - Class[] params = {ExaMetadata.class}; - Method method = scriptClass.getDeclaredMethod(fn, params); - String resS = String.valueOf(method.invoke(null, exaMetadata)); - return resS.getBytes("UTF-8"); - } else { - Class[] params = {ExaMetadata.class,argClass}; - Method method = scriptClass.getDeclaredMethod(fn, params); - String resS = String.valueOf(method.invoke(null, exaMetadata, args)); - return resS.getBytes("UTF-8"); - } - } catch (java.lang.ClassNotFoundException ex) { - if (userDefinedScriptName) { - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1066: The main script class defined via %scriptclass cannot be found: " + scriptClassName); - } else { - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1067: The main script class (same name as the script) cannot be found: " + scriptClassName + ". Please create the class or specify the class via %scriptclass."); - } - } catch (InvocationTargetException ex) { - throw convertReflectiveExceptionToCause("F-UDF-CL-SL-JAVA-1068","Exception during singleCall "+fn,ex); - } catch (NoSuchMethodException ex) { - throw new ExaUndefinedSingleCallException(fn); - } - } - - static Class getScriptClass(ExaMetadataImpl exaMetadata) throws Throwable { - boolean userDefinedScriptName = true; - String scriptClassName = System.getProperty("exasol.scriptclass", ""); - if (scriptClassName.trim().isEmpty()) { - userDefinedScriptName = false; - scriptClassName = exaMetadata.getScriptName(); - // Only for test simulator (e.g., script.java -> script_java) - scriptClassName = scriptClassName.replace('.', '_'); - scriptClassName = "com.exasol." + scriptClassName; - } - try{ - // Take the scriptClass name specified by user in the script, or the name of the script as fallback - Class scriptClass = Class.forName(scriptClassName); - return scriptClass; - } catch (java.lang.ClassNotFoundException ex) { - if (userDefinedScriptName) { - throw new ExaCompilationException("F-UDF.CL.SL.JAVA-1072: The main script class defined via %scriptclass cannot be found: " + scriptClassName); - } else { - throw new ExaCompilationException("F-UDF.CL.SL.JAVA-1073: The main script class (same name as the script) cannot be found: " + scriptClassName + ". Please create the class or specify the class via %scriptclass."); - } - } - } - - - static ExaMetadataImpl getMetaData() throws Throwable { - ExaMetadataImpl exaMetadata = new ExaMetadataImpl(); - String exMsg = exaMetadata.checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaIterationException("UDF.CL.SL.JAVA-1165: "+exMsg); - } - return exaMetadata; - } - - static void run() throws Throwable { - ExaMetadataImpl exaMetadata = null; - try{ - exaMetadata = getMetaData(); - }catch(ExaIterationException ex){ - throw new ExaIterationException("F-UDF.CL.SL.JAVA-1069: "+ex.getMessage()); - } - TableIterator tableIterator = new TableIterator(); - String exMsg = tableIterator.checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaIterationException("F-UDF-CL-SL-JAVA-1070: "+exMsg); - } - ResultHandler resultHandler = new ResultHandler(tableIterator); - exMsg = resultHandler.checkException(); - if (exMsg != null && exMsg.length() > 0) { - throw new ExaIterationException("F-UDF-CL-SL-JAVA-1071: "+exMsg); - } - - ExaIteratorImpl exaIter = new ExaIteratorImpl(exaMetadata, tableIterator, resultHandler); - - Class scriptClass = null; - try { - scriptClass = getScriptClass(exaMetadata); - } catch (ExaCompilationException ex){ - throw new ExaCompilationException("F-UDF.CL.SL.JAVA-1165: "+ex.getMessage()); - } - // init() - try { - Class[] initParams = {ExaMetadata.class}; - Method initMethod = scriptClass.getDeclaredMethod("init", initParams); - initMethod.invoke(null, exaMetadata); - } catch (InvocationTargetException ex) { - throw convertReflectiveExceptionToCause("F-UDF-CL-SL-JAVA-1074","Exception during init",ex); - } catch (NoSuchMethodException ex) { - System.err.println("W-UDF-CL-SL-JAVA-1075: Skipping init, because init method cannot be found."); - } - - // run() - Class[] runParams = {ExaMetadata.class, ExaIterator.class}; - Method runMethod = scriptClass.getDeclaredMethod("run", runParams); - - try { - if (exaMetadata.getInputType().equals("SET")) { // MULTIPLE INPUT - if (exaMetadata.getOutputType().equals("EMIT")) { // MULTIPLE OUTPUT - if (!runMethod.getReturnType().equals(Void.TYPE)) - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1076: EMITS requires a void return type for run()"); - exaIter.setInsideRun(true); - Object returnValue = runMethod.invoke(null, exaMetadata, exaIter); - exaIter.setInsideRun(false); - } - else { // EXACTLY_ONCE OUTPUT - if (runMethod.getReturnType().equals(Void.TYPE)) - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1077: RETURNS requires a non-void return type for run()"); - exaIter.setInsideRun(true); - Object returnValue = runMethod.invoke(null, exaMetadata, exaIter); - exaIter.setInsideRun(false); - exaIter.emit(returnValue); - } - } - else { // EXACTLY_ONCE INPUT - if (exaMetadata.getOutputType().equals("EMIT")) { // MULTIPLE OUTPUT - if (!runMethod.getReturnType().equals(Void.TYPE)) - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1078: EMITS requires a void return type for run()"); - do { - exaIter.setInsideRun(true); - Object returnValue = runMethod.invoke(null, exaMetadata, exaIter); - exaIter.setInsideRun(false); - } while (exaIter.next()); - } - else { // EXACTLY_ONCE OUTPUT - if (runMethod.getReturnType().equals(Void.TYPE)) - throw new ExaCompilationException("F-UDF-CL-SL-JAVA-1079: RETURNS requires a non-void return type for run()"); - do { - exaIter.setInsideRun(true); - Object returnValue = runMethod.invoke(null, exaMetadata, exaIter); - exaIter.setInsideRun(false); - exaIter.emit(returnValue); - } while (exaIter.next()); - } - } - } - catch (InvocationTargetException ex) { - throw convertReflectiveExceptionToCause("F-UDF-CL-SL-JAVA-1080","Exception during run",ex); - } - - resultHandler.flush(); - } - - static void cleanup() throws Throwable { - // FIXME cleanup gets only called if run is successful - // cleanup() - ExaMetadataImpl exaMetadata = null; - try{ - exaMetadata = getMetaData(); - }catch(ExaIterationException ex){ - throw new ExaIterationException("F-UDF.CL.SL.JAVA-1166: "+ex.getMessage()); - } - Class scriptClass = null; - try { - scriptClass = getScriptClass(exaMetadata); - } catch (ExaCompilationException ex){ - throw new ExaCompilationException("F-UDF.CL.SL.JAVA-1167: "+ex.getMessage()); - } - try { - Class[] cleanupParams = {ExaMetadata.class}; - Method cleanupMethod = scriptClass.getDeclaredMethod("cleanup", cleanupParams); - cleanupMethod.invoke(null, exaMetadata); - } - catch (InvocationTargetException ex) { - throw convertReflectiveExceptionToCause("F-UDF-CL-SL-JAVA-1081","Exception during cleanup",ex); - } - catch (NoSuchMethodException ex) { - System.err.println("W-UDF.CL.SL.JAVA-1082: Skipping cleanup, because cleanup method cannot be found."); - } - } - - private static Throwable convertReflectiveExceptionToCause(String error_code, String errorMessage, Throwable ex) { - ExaStackTraceCleaner exaStackTraceCleaner = new ExaStackTraceCleaner(ExaWrapper.class.getName()); - String cleanedStacktrace = exaStackTraceCleaner.cleanStackTrace(ex); - String error_message=error_code+": "+errorMessage+" \n"+cleanedStacktrace; - System.out.println(error_message); - return new ExaUDFException(error_message); - } -} diff --git a/udf-runner-cpp/v1/base/javacontainer/exascript_java_jni_decl.h b/udf-runner-cpp/v1/base/javacontainer/exascript_java_jni_decl.h deleted file mode 100644 index 022e4c9..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/exascript_java_jni_decl.h +++ /dev/null @@ -1,261 +0,0 @@ -#ifndef EXASCRIPT_JAVA_JNI_DECL_H -#define EXASCRIPT_JAVA_JNI_DECL_H - -#ifdef __cplusplus -extern "C" { -#endif - - -// the signature definitions can be printed with 'javah -classpath . com.exasol.swig.exascript_javaJNI' -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ConnectionInformationWrapper_1copyKind(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ConnectionInformationWrapper_1copyAddress(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ConnectionInformationWrapper_1copyUser(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ConnectionInformationWrapper_1copyPassword(JNIEnv *, jclass, jlong, jobject); - -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_new_1ImportSpecificationWrapper(JNIEnv *jenv, jclass, jlong); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1isSubselect(JNIEnv *, jclass, jlong, jobject); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1numSubselectColumns(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copySubselectColumnName(JNIEnv *, jclass, jlong, jobject, jlong); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copySubselectColumnType(JNIEnv *, jclass, jlong, jobject, jlong); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1hasConnectionName(JNIEnv *, jclass, jlong, jobject); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1hasConnectionInformation(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copyConnectionName(JNIEnv *, jclass, jlong, jobject); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1getConnectionInformation(JNIEnv *, jclass, jlong, jobject); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1getNumberOfParameters(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copyKey(JNIEnv *, jclass, jlong, jobject, jlong); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copyValue(JNIEnv *, jclass, jlong, jobject, jlong); - -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_new_1ExportSpecificationWrapper(JNIEnv *, jclass, jlong); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasConnectionName(JNIEnv *, jclass, jlong, jobject); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasConnectionInformation(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copyConnectionName(JNIEnv *, jclass, jlong, jobject); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1getConnectionInformation(JNIEnv *, jclass, jlong, jobject); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1getNumberOfParameters(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copyKey(JNIEnv *, jclass, jlong, jobject, jlong); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copyValue(JNIEnv *, jclass, jlong, jobject, jlong); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasTruncate(JNIEnv *, jclass, jlong, jobject); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasReplace(JNIEnv *, jclass, jlong, jobject); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasCreatedBy(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copyCreatedBy(JNIEnv *, jclass, jlong, jobject); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1numSourceColumns(JNIEnv *, jclass, jlong, jobject); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copySourceColumnName(JNIEnv *, jclass, jlong, jobject, jlong); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_delete_1ExportSpecificationWrapper(JNIEnv *, jclass, jlong); - -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_UNSUPPORTED_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_DOUBLE_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_INT32_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_INT64_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_NUMERIC_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_TIMESTAMP_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_DATE_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_STRING_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_BOOLEAN_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_EXACTLY_1ONCE_1get(JNIEnv *jenv, jclass jcls); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_MULTIPLE_1get(JNIEnv *jenv, jclass jcls); - -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_new_1Metadata(JNIEnv *jenv, jclass jcls); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1databaseName(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1databaseVersion(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1scriptName(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1scriptSchema(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1currentUser(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1scopeUser(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1currentSchema(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1scriptCode(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1moduleContent(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2); -jobject JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1connectionInformation(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jstring jarg2); -jobject JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1sessionID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1sessionID_1S(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1statementID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1nodeCount(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1nodeID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jobject JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1vmID(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1vmID_1S(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jobject JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1memoryLimit(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnCount(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnName(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnTypeName(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnPrecision(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnScale(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnCount(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnName(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnTypeName(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnPrecision(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnScale(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputType(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_Metadata_1checkException(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_delete_1Metadata(JNIEnv *jenv, jclass jcls, jlong jarg1); - -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_new_1TableIterator(JNIEnv *jenv, jclass jcls); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1checkException(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1reinitialize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1next(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1eot(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1reset(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1restBufferSize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1rowsInGroup(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1rowsCompleted(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jdouble JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getDouble(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jbyteArray JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getStringByteArray(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jint JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getInt32(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getInt64(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getNumeric(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getDate(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getTimestamp(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getBoolean(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1wasNull(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_delete_1TableIterator(JNIEnv *jenv, jclass jcls, jlong jarg1); - -jlong JNICALL Java_com_exasol_swig_exascript_1javaJNI_new_1ResultHandler(JNIEnv *jenv, jclass jcls); -jstring JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1checkException(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1reinitialize(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -jboolean JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1next(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1flush(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setDouble(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jdouble jarg3); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setStringByteArray(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jbyteArray jarg3, jlong jarg4); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setInt32(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jint jarg3); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setInt64(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jlong jarg3); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setNumeric(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jstring jarg3); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setDate(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jstring jarg3); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setTimestamp(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jstring jarg3); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setBoolean(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2, jboolean jarg3); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setNull(JNIEnv *jenv, jclass jcls, jlong jarg1, jobject jarg1_, jlong jarg2); -void JNICALL Java_com_exasol_swig_exascript_1javaJNI_delete_1ResultHandler(JNIEnv *jenv, jclass jcls, jlong jarg1); - -// This array is a mapping from the native java methods in exascript_javaJNI class to -// The signature syntax can be printed via "javap -s -p exascript_javaJNI" in the folder containing the exascript_javaJNI.class -JNINativeMethod methods[] = { - {(char *)"UNSUPPORTED_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_UNSUPPORTED_1get}, - {(char *)"DOUBLE_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_DOUBLE_1get}, - {(char *)"INT32_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_INT32_1get}, - {(char *)"INT64_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_INT64_1get}, - {(char *)"NUMERIC_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_NUMERIC_1get}, - {(char *)"TIMESTAMP_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TIMESTAMP_1get}, - {(char *)"DATE_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_DATE_1get}, - {(char *)"STRING_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_STRING_1get}, - {(char *)"BOOLEAN_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_BOOLEAN_1get}, - {(char *)"EXACTLY_ONCE_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_EXACTLY_1ONCE_1get}, - {(char *)"MULTIPLE_get", (char *)"()I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_MULTIPLE_1get}, - {(char *)"new_Metadata", (char *)"()J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_new_1Metadata}, - {(char *)"Metadata_databaseName", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1databaseName}, - {(char *)"Metadata_databaseVersion", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1databaseVersion}, - {(char *)"Metadata_scriptName", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1scriptName}, - {(char *)"Metadata_scriptSchema", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1scriptSchema}, - {(char *)"Metadata_currentUser", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1currentUser}, - {(char *)"Metadata_scopeUser", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1scopeUser}, - {(char *)"Metadata_currentSchema", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1currentSchema}, - {(char *)"Metadata_scriptCode", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1scriptCode}, - {(char *)"Metadata_moduleContent", (char *)"(JLcom/exasol/swig/Metadata;Ljava/lang/String;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1moduleContent}, - {(char *)"Metadata_connectionInformation", (char *)"(JLcom/exasol/swig/Metadata;Ljava/lang/String;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1connectionInformation}, - {(char *)"Metadata_sessionID", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/math/BigInteger;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1sessionID}, - {(char *)"Metadata_sessionID_S", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1sessionID_1S}, - {(char *)"Metadata_statementID", (char *)"(JLcom/exasol/swig/Metadata;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1statementID}, - {(char *)"Metadata_nodeCount", (char *)"(JLcom/exasol/swig/Metadata;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1nodeCount}, - {(char *)"Metadata_nodeID", (char *)"(JLcom/exasol/swig/Metadata;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1nodeID}, - {(char *)"Metadata_vmID", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/math/BigInteger;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1vmID}, - {(char *)"Metadata_vmID_S", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1vmID_1S}, - {(char *)"Metadata_memoryLimit", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/math/BigInteger;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1memoryLimit}, - {(char *)"Metadata_inputColumnCount", (char *)"(JLcom/exasol/swig/Metadata;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnCount}, - {(char *)"Metadata_inputColumnName", (char *)"(JLcom/exasol/swig/Metadata;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnName}, - {(char *)"Metadata_inputColumnType", (char *)"(JLcom/exasol/swig/Metadata;J)I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnType}, - {(char *)"Metadata_inputColumnTypeName", (char *)"(JLcom/exasol/swig/Metadata;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnTypeName}, - {(char *)"Metadata_inputColumnSize", (char *)"(JLcom/exasol/swig/Metadata;J)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnSize}, - {(char *)"Metadata_inputColumnPrecision", (char *)"(JLcom/exasol/swig/Metadata;J)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnPrecision}, - {(char *)"Metadata_inputColumnScale", (char *)"(JLcom/exasol/swig/Metadata;J)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputColumnScale}, - {(char *)"Metadata_inputType", (char *)"(JLcom/exasol/swig/Metadata;)I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1inputType}, - {(char *)"Metadata_outputColumnCount", (char *)"(JLcom/exasol/swig/Metadata;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnCount}, - {(char *)"Metadata_outputColumnName", (char *)"(JLcom/exasol/swig/Metadata;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnName}, - {(char *)"Metadata_outputColumnType", (char *)"(JLcom/exasol/swig/Metadata;J)I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnType}, - {(char *)"Metadata_outputColumnTypeName", (char *)"(JLcom/exasol/swig/Metadata;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnTypeName}, - {(char *)"Metadata_outputColumnSize", (char *)"(JLcom/exasol/swig/Metadata;J)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnSize}, - {(char *)"Metadata_outputColumnPrecision", (char *)"(JLcom/exasol/swig/Metadata;J)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnPrecision}, - {(char *)"Metadata_outputColumnScale", (char *)"(JLcom/exasol/swig/Metadata;J)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputColumnScale}, - {(char *)"Metadata_outputType", (char *)"(JLcom/exasol/swig/Metadata;)I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1outputType}, - {(char *)"Metadata_checkException", (char *)"(JLcom/exasol/swig/Metadata;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_Metadata_1checkException}, - {(char *)"delete_Metadata", (char *)"(J)V", (void *)&JNICALL Java_com_exasol_swig_exascript_1javaJNI_delete_1Metadata}, - - {(char *)"new_TableIterator", (char *)"()J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_new_1TableIterator}, - {(char *)"TableIterator_checkException", (char *)"(JLcom/exasol/swig/TableIterator;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1checkException}, - {(char *)"TableIterator_reinitialize", (char *)"(JLcom/exasol/swig/TableIterator;)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1reinitialize}, - {(char *)"TableIterator_next", (char *)"(JLcom/exasol/swig/TableIterator;)Z", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1next}, - {(char *)"TableIterator_eot", (char *)"(JLcom/exasol/swig/TableIterator;)Z", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1eot}, - {(char *)"TableIterator_reset", (char *)"(JLcom/exasol/swig/TableIterator;)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1reset}, - {(char *)"TableIterator_restBufferSize", (char *)"(JLcom/exasol/swig/TableIterator;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1restBufferSize}, - {(char *)"TableIterator_rowsInGroup", (char *)"(JLcom/exasol/swig/TableIterator;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1rowsInGroup}, - {(char *)"TableIterator_rowsCompleted", (char *)"(JLcom/exasol/swig/TableIterator;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1rowsCompleted}, - {(char *)"TableIterator_getDouble", (char *)"(JLcom/exasol/swig/TableIterator;J)D", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getDouble}, - {(char *)"TableIterator_getStringByteArray", (char *)"(JLcom/exasol/swig/TableIterator;J)[B", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getStringByteArray}, - {(char *)"TableIterator_getInt32", (char *)"(JLcom/exasol/swig/TableIterator;J)I", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getInt32}, - {(char *)"TableIterator_getInt64", (char *)"(JLcom/exasol/swig/TableIterator;J)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getInt64}, - {(char *)"TableIterator_getNumeric", (char *)"(JLcom/exasol/swig/TableIterator;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getNumeric}, - {(char *)"TableIterator_getDate", (char *)"(JLcom/exasol/swig/TableIterator;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getDate}, - {(char *)"TableIterator_getTimestamp", (char *)"(JLcom/exasol/swig/TableIterator;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getTimestamp}, - {(char *)"TableIterator_getBoolean", (char *)"(JLcom/exasol/swig/TableIterator;J)Z", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1getBoolean}, - {(char *)"TableIterator_wasNull", (char *)"(JLcom/exasol/swig/TableIterator;)Z", (void *)&Java_com_exasol_swig_exascript_1javaJNI_TableIterator_1wasNull}, - {(char *)"delete_TableIterator", (char *)"(J)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_delete_1TableIterator}, - - {(char *)"new_ResultHandler", (char *)"(JLcom/exasol/swig/TableIterator;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_new_1ResultHandler}, - {(char *)"ResultHandler_checkException", (char *)"(JLcom/exasol/swig/ResultHandler;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1checkException}, - {(char *)"ResultHandler_reinitialize", (char *)"(JLcom/exasol/swig/ResultHandler;)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1reinitialize}, - {(char *)"ResultHandler_next", (char *)"(JLcom/exasol/swig/ResultHandler;)Z", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1next}, - {(char *)"ResultHandler_flush", (char *)"(JLcom/exasol/swig/ResultHandler;)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1flush}, - {(char *)"ResultHandler_setDouble", (char *)"(JLcom/exasol/swig/ResultHandler;JD)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setDouble}, - {(char *)"ResultHandler_setStringByteArray", (char *)"(JLcom/exasol/swig/ResultHandler;J[BJ)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setStringByteArray}, - {(char *)"ResultHandler_setInt32", (char *)"(JLcom/exasol/swig/ResultHandler;JI)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setInt32}, - {(char *)"ResultHandler_setInt64", (char *)"(JLcom/exasol/swig/ResultHandler;JJ)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setInt64}, - {(char *)"ResultHandler_setNumeric", (char *)"(JLcom/exasol/swig/ResultHandler;JLjava/lang/String;)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setNumeric}, - {(char *)"ResultHandler_setDate", (char *)"(JLcom/exasol/swig/ResultHandler;JLjava/lang/String;)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setDate}, - {(char *)"ResultHandler_setTimestamp", (char *)"(JLcom/exasol/swig/ResultHandler;JLjava/lang/String;)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setTimestamp}, - {(char *)"ResultHandler_setBoolean", (char *)"(JLcom/exasol/swig/ResultHandler;JZ)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setBoolean}, - {(char *)"ResultHandler_setNull", (char *)"(JLcom/exasol/swig/ResultHandler;J)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ResultHandler_1setNull}, - {(char *)"delete_ResultHandler", (char *)"(J)V", (void *)&Java_com_exasol_swig_exascript_1javaJNI_delete_1ResultHandler}, - - - {(char *)"ConnectionInformationWrapper_copyKind",(char *)"(JLcom/exasol/swig/ConnectionInformationWrapper;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ConnectionInformationWrapper_1copyKind}, - {(char *)"ConnectionInformationWrapper_copyAddress",(char *)"(JLcom/exasol/swig/ConnectionInformationWrapper;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ConnectionInformationWrapper_1copyAddress}, - {(char *)"ConnectionInformationWrapper_copyUser",(char *)"(JLcom/exasol/swig/ConnectionInformationWrapper;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ConnectionInformationWrapper_1copyUser}, - {(char *)"ConnectionInformationWrapper_copyPassword",(char *)"(JLcom/exasol/swig/ConnectionInformationWrapper;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ConnectionInformationWrapper_1copyPassword}, - - - {(char *)"new_ImportSpecificationWrapper",(char*)"(J)J",(void*)&Java_com_exasol_swig_exascript_1javaJNI_new_1ImportSpecificationWrapper}, - {(char *)"ImportSpecificationWrapper_isSubselect",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;)Z", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1isSubselect}, - {(char *)"ImportSpecificationWrapper_numSubselectColumns",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1numSubselectColumns}, - {(char *)"ImportSpecificationWrapper_copySubselectColumnName",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copySubselectColumnName}, - {(char *)"ImportSpecificationWrapper_copySubselectColumnType",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copySubselectColumnType}, - {(char *)"ImportSpecificationWrapper_hasConnectionName",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;)Z", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1hasConnectionName}, - {(char *)"ImportSpecificationWrapper_hasConnectionInformation",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;)Z", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1hasConnectionInformation}, - {(char *)"ImportSpecificationWrapper_copyConnectionName",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copyConnectionName}, - {(char *)"ImportSpecificationWrapper_getConnectionInformation",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1getConnectionInformation}, - {(char *)"ImportSpecificationWrapper_getNumberOfParameters",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;)J", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1getNumberOfParameters}, - {(char *)"ImportSpecificationWrapper_copyKey",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copyKey}, - {(char *)"ImportSpecificationWrapper_copyValue",(char *)"(JLcom/exasol/swig/ImportSpecificationWrapper;J)Ljava/lang/String;", (void *)&Java_com_exasol_swig_exascript_1javaJNI_ImportSpecificationWrapper_1copyValue}, - - - {(char *)"new_ExportSpecificationWrapper",(char *)"(J)J",(void *)&Java_com_exasol_swig_exascript_1javaJNI_new_1ExportSpecificationWrapper}, - {(char *)"ExportSpecificationWrapper_hasConnectionName",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)Z",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasConnectionName}, - {(char *)"ExportSpecificationWrapper_hasConnectionInformation",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)Z",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasConnectionInformation}, - {(char *)"ExportSpecificationWrapper_copyConnectionName",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)Ljava/lang/String;",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copyConnectionName}, - {(char *)"ExportSpecificationWrapper_getConnectionInformation",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)J",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1getConnectionInformation}, - {(char *)"ExportSpecificationWrapper_getNumberOfParameters",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)J",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1getNumberOfParameters}, - {(char *)"ExportSpecificationWrapper_copyKey",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;J)Ljava/lang/String;",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copyKey}, - {(char *)"ExportSpecificationWrapper_copyValue",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;J)Ljava/lang/String;",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copyValue}, - {(char *)"ExportSpecificationWrapper_hasTruncate",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)Z",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasTruncate}, - {(char *)"ExportSpecificationWrapper_hasReplace",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)Z",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasReplace}, - {(char *)"ExportSpecificationWrapper_hasCreatedBy",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)Z",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1hasCreatedBy}, - {(char *)"ExportSpecificationWrapper_copyCreatedBy",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)Ljava/lang/String;",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copyCreatedBy}, - {(char *)"ExportSpecificationWrapper_numSourceColumns",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;)J",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1numSourceColumns}, - {(char *)"ExportSpecificationWrapper_copySourceColumnName",(char *)"(JLcom/exasol/swig/ExportSpecificationWrapper;J)Ljava/lang/String;",(void *)&Java_com_exasol_swig_exascript_1javaJNI_ExportSpecificationWrapper_1copySourceColumnName}, - {(char *)"delete_ExportSpecificationWrapper",(char *)"(J)V",(void *)&Java_com_exasol_swig_exascript_1javaJNI_delete_1ExportSpecificationWrapper}, - -}; - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/udf-runner-cpp/v1/base/javacontainer/javacontainer.cc b/udf-runner-cpp/v1/base/javacontainer/javacontainer.cc deleted file mode 100644 index d0e8556..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/javacontainer.cc +++ /dev/null @@ -1,57 +0,0 @@ -#include "javacontainer/javacontainer.h" -#include "javacontainer/javacontainer_impl.h" -#include "javacontainer/script_options/extractor.h" - -using namespace SWIGVMContainers; -using namespace std; - -JavaVMach::JavaVMach(bool checkOnly,std::unique_ptr extractor) { - try { - m_impl = new JavaVMImpl(checkOnly, false, std::move(extractor)); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-JAVA-1000: "+std::string(err.what()); - } catch (...) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-JAVA-1001: JVM crashed for unknown reason"; - } -} - - -bool JavaVMach::run() { - try { - return m_impl->run(); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-JAVA-1002: "+std::string(err.what()); - } catch (...) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-JAVA-1003: JVM crashed for unknown reason"; - } - return false; -} - -void JavaVMach::shutdown() { - try { - m_impl->shutdown(); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-JAVA-1004: "+std::string(err.what()); - } catch (...) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-JAVA-1005: JVM crashed for unknown reason"; - } -} - -const char* JavaVMach::singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args) { - try { - return m_impl->singleCall(fn, args, calledUndefinedSingleCall); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-JAVA-1006: "+std::string(err.what()); - } catch (...) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-JAVA-1007: JVM crashed for unknown reason"; - } - return strdup(""); -} diff --git a/udf-runner-cpp/v1/base/javacontainer/javacontainer.h b/udf-runner-cpp/v1/base/javacontainer/javacontainer.h deleted file mode 100644 index fca3d33..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/javacontainer.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef JAVACONTAINER_H -#define JAVACONTAINER_H - - -#include "exaudflib/vm/swig_vm.h" -#include -#include - -#ifdef ENABLE_JAVA_VM - -namespace SWIGVMContainers { - -class JavaVMImpl; - -namespace JavaScriptOptions { - -struct Extractor; - -} - -class JavaVMach: public SWIGVM { - public: - /* - * scriptOptionsParser: JavaVMach takes ownership of ScriptOptionsParser pointer. - */ - JavaVMach(bool checkOnly, std::unique_ptr extractor); - virtual ~JavaVMach() {} - virtual void shutdown(); - virtual bool run(); - virtual const char* singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args); - private: - JavaVMImpl *m_impl; -}; - -} //namespace SWIGVMContainers - - -#endif //ENABLE_JAVA_VM - - -#endif //JAVACONTAINER_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/javacontainer_builder.cc b/udf-runner-cpp/v1/base/javacontainer/javacontainer_builder.cc deleted file mode 100644 index f44b9ef..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/javacontainer_builder.cc +++ /dev/null @@ -1,31 +0,0 @@ -#include "javacontainer/javacontainer_builder.h" -#include "javacontainer/script_options/extractor_impl.h" -#include "swig_factory/swig_factory_impl.h" - -#ifdef ENABLE_JAVA_VM - -namespace SWIGVMContainers { - -JavaContainerBuilder::JavaContainerBuilder() -: m_useCtpgParser(false) {} - -JavaContainerBuilder& JavaContainerBuilder::useCtpgParser() { - m_useCtpgParser = true; - return *this; -} - -JavaVMach* JavaContainerBuilder::build() { - std::unique_ptr extractor; - if (m_useCtpgParser) { - extractor = std::make_unique(std::make_unique()); - } else { - extractor = std::make_unique(std::make_unique()); - } - return new JavaVMach(false, std::move(extractor)); -} - - -} //namespace SWIGVMContainers - - -#endif //ENABLE_JAVA_VM diff --git a/udf-runner-cpp/v1/base/javacontainer/javacontainer_builder.h b/udf-runner-cpp/v1/base/javacontainer/javacontainer_builder.h deleted file mode 100644 index 9a7113d..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/javacontainer_builder.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef JAVACONTAINER_BUILDER_H -#define JAVACONTAINER_BUILDER_H - -#include -#include "javacontainer/javacontainer.h" - -#ifdef ENABLE_JAVA_VM - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -struct ScriptOptionsParser; - -} - -class JavaContainerBuilder { - public: - JavaContainerBuilder(); - - JavaContainerBuilder& useCtpgParser(); - - JavaVMach* build(); - - private: - bool m_useCtpgParser; -}; - -} //namespace SWIGVMContainers - - -#endif //ENABLE_JAVA_VM - - -#endif //JAVACONTAINER_BUILDER_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/javacontainer_impl.cc b/udf-runner-cpp/v1/base/javacontainer/javacontainer_impl.cc deleted file mode 100644 index 5820bb2..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/javacontainer_impl.cc +++ /dev/null @@ -1,454 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "exascript_java_jni_decl.h" - -#include "utils/debug_message.h" -#include "javacontainer/javacontainer.h" -#include "javacontainer/javacontainer_impl.h" -#include "javacontainer/script_options/extractor.h" - - -using namespace SWIGVMContainers; -using namespace std; - -JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI, - std::unique_ptr extractor) -: m_checkOnly(checkOnly) -, m_exaJavaPath("") -, m_localClasspath("/tmp") // **IMPORTANT**: /tmp needs to be in the classpath, otherwise ExaCompiler crashe with com.exasol.ExaCompilationException: /DATE_STRING.java:3: error: error while writing DATE_STRING: could not create parent directories -, m_scriptCode(SWIGVM_params->script_code) -, m_jvm(NULL) -, m_env(NULL) -, m_needsCompilation(true) -{ - - stringstream ss; - //Path "external/exaudfclient_base+/" comes from external module "exaudfclient_base" - m_exaJavaPath = "/exaudf/external/exaudfclient_base+/javacontainer"; // TODO hardcoded path - - parseScriptOptions(std::move(extractor)); - - m_needsCompilation = checkNeedsCompilation(); - if (m_needsCompilation) { - DBG_FUNC_CALL(cerr,addPackageToScript()); - DBG_FUNC_CALL(cerr,addLocalClasspath()); - } - DBG_FUNC_CALL(cerr,setJvmOptions()); - if(false == noJNI) { - DBG_FUNC_CALL(cerr,createJvm()); - DBG_FUNC_CALL(cerr,registerFunctions()); - if (m_needsCompilation) { - DBG_FUNC_CALL(cerr,compileScript()); - } - } -} - -void JavaVMImpl::parseScriptOptions(std::unique_ptr extractor) { - - DBG_FUNC_CALL(cerr,extractor->extract(m_scriptCode)); - - DBG_FUNC_CALL(cerr,setClasspath()); - - m_jvmOptions = std::move(extractor->moveJvmOptions()); - - extractor->iterateJarPaths([&](const std::string& s) { addJarToClasspath(s);}); -} - -void JavaVMImpl::shutdown() { - if (m_checkOnly) - throw JavaVMach::exception("F-UDF.CL.SL.JAVA-1159: Java VM in check only mode"); - jclass cls = m_env->FindClass("com/exasol/ExaWrapper"); - string calledUndefinedSingleCall; - check("F-UDF.CL.SL.JAVA-1160",calledUndefinedSingleCall); - if (!cls) - throw JavaVMach::exception("F-UDF.CL.SL.JAVA-1161: FindClass for ExaWrapper failed"); - jmethodID mid = m_env->GetStaticMethodID(cls, "cleanup", "()V"); - check("F-UDF.CL.SL.JAVA-1162",calledUndefinedSingleCall); - if (!mid) - throw JavaVMach::exception("F-UDF.CL.SL.JAVA-1163: GetStaticMethodID for run failed"); - m_env->CallStaticVoidMethod(cls, mid); - check("F-UDF.CL.SL.JAVA-1164",calledUndefinedSingleCall); - try { - m_jvm->DestroyJavaVM(); - } catch(...) { - - } -} - -bool JavaVMImpl::run() { - if (m_checkOnly) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1008: Java VM in check only mode"); - jclass cls = m_env->FindClass("com/exasol/ExaWrapper"); - string calledUndefinedSingleCall; - check("F-UDF-CL-SL-JAVA-1009",calledUndefinedSingleCall); - if (!cls) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1010: FindClass for ExaWrapper failed"); - jmethodID mid = m_env->GetStaticMethodID(cls, "run", "()V"); - check("F-UDF-CL-SL-JAVA-1011",calledUndefinedSingleCall); - if (!mid) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1012: GetStaticMethodID for run failed"); - m_env->CallStaticVoidMethod(cls, mid); - check("F-UDF-CL-SL-JAVA-1013",calledUndefinedSingleCall); - return true; -} - -static string singleCallResult; - -const char* JavaVMImpl::singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args, string& calledUndefinedSingleCall) { - - if (m_checkOnly) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1014: Java VM in check only mode"); - - const char* func = NULL; - switch (fn) { - case SC_FN_NIL: break; - case SC_FN_DEFAULT_OUTPUT_COLUMNS: func = "getDefaultOutputColumns"; break; - case SC_FN_VIRTUAL_SCHEMA_ADAPTER_CALL: func = "adapterCall"; break; - case SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC: func = "generateSqlForImportSpec"; break; - case SC_FN_GENERATE_SQL_FOR_EXPORT_SPEC: func = "generateSqlForExportSpec"; break; - } - if (func == NULL) { - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1015: Unknown single call "+std::to_string(fn)); - } - jclass cls = m_env->FindClass("com/exasol/ExaWrapper"); - check("F-UDF-CL-SL-JAVA-1016",calledUndefinedSingleCall); - if (!cls) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1017: FindClass for ExaWrapper failed"); - jmethodID mid = m_env->GetStaticMethodID(cls, "runSingleCall", "(Ljava/lang/String;Ljava/lang/Object;)[B"); - check("F-UDF-CL-SL-JAVA-1018",calledUndefinedSingleCall); - if (!mid) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1019: GetStaticMethodID for run failed"); - jstring fn_js = m_env->NewStringUTF(func); - check("F-UDF-CL-SL-JAVA-1020",calledUndefinedSingleCall); - - - // Prepare arg - // TODO VS This will be refactored completely - // We intentionally define these variables outside so that they are not destroyed too early - jobject args_js = NULL; - ExecutionGraph::ImportSpecification* imp_spec; - ExecutionGraph::ImportSpecificationWrapper imp_spec_wrapper(NULL); - ExecutionGraph::ExportSpecification* exp_spec; - ExecutionGraph::ExportSpecificationWrapper exp_spec_wrapper(NULL); - if (fn == SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC) { - imp_spec = const_cast(dynamic_cast(&args)); - imp_spec_wrapper = ExecutionGraph::ImportSpecificationWrapper(imp_spec); - if (imp_spec) - { - jclass import_spec_wrapper_cls = m_env->FindClass("com/exasol/swig/ImportSpecificationWrapper"); - check("F-UDF-CL-SL-JAVA-1021",calledUndefinedSingleCall); - jmethodID import_spec_wrapper_constructor = m_env->GetMethodID(import_spec_wrapper_cls, "", "(JZ)V"); - check("F-UDF-CL-SL-JAVA-1022",calledUndefinedSingleCall); - args_js = m_env->NewObject(import_spec_wrapper_cls, import_spec_wrapper_constructor, &imp_spec_wrapper, false); - } - } else if (fn == SC_FN_GENERATE_SQL_FOR_EXPORT_SPEC) { - exp_spec = const_cast(dynamic_cast(&args)); - exp_spec_wrapper = ExecutionGraph::ExportSpecificationWrapper(exp_spec); - if (exp_spec) - { - jclass export_spec_wrapper_cls = m_env->FindClass("com/exasol/swig/ExportSpecificationWrapper"); - check("F-UDF-CL-SL-JAVA-1023",calledUndefinedSingleCall); - jmethodID export_spec_wrapper_constructor = m_env->GetMethodID(export_spec_wrapper_cls, "", "(JZ)V"); - check("F-UDF-CL-SL-JAVA-1024",calledUndefinedSingleCall); - args_js = m_env->NewObject(export_spec_wrapper_cls, export_spec_wrapper_constructor, &exp_spec_wrapper, false); - } - } else if (fn == SC_FN_VIRTUAL_SCHEMA_ADAPTER_CALL) { - const ExecutionGraph::StringDTO* argDto = dynamic_cast(&args); - string string_arg = argDto->getArg(); - args_js = m_env->NewStringUTF(string_arg.c_str()); - } - - check("F-UDF-CL-SL-JAVA-1025",calledUndefinedSingleCall); - jbyteArray resJ = (jbyteArray)m_env->CallStaticObjectMethod(cls, mid, fn_js, args_js); - if (check("F-UDF-CL-SL-JAVA-1026",calledUndefinedSingleCall) == 0) return strdup(""); - jsize resLen = m_env->GetArrayLength(resJ); - check("F-UDF-CL-SL-JAVA-1027",calledUndefinedSingleCall); - char* buffer = new char[resLen + 1]; - m_env->GetByteArrayRegion(resJ, 0, resLen, reinterpret_cast(buffer)); - buffer[resLen] = '\0'; - singleCallResult = string(buffer); - delete buffer; - m_env->DeleteLocalRef(args_js); - m_env->DeleteLocalRef(resJ); - - - return singleCallResult.c_str(); -} - -void JavaVMImpl::addPackageToScript() { - // Each script is generated in the com.exasol package, not in the default - // package. Scripts classes may not be in the default package because - // com.exasol.ExaWrapper requires to call run() on them (accessing default - // package from a non-default package is not allowed). Furthermore, only if - // the script is in the same package as ExaWrapper the script can be - // defined as package-private. - m_scriptCode = "package com.exasol;\r\n" + m_scriptCode; -} - -void JavaVMImpl::createJvm() { - unsigned int numJvmOptions = m_jvmOptions.size(); - JavaVMOption *options = new JavaVMOption[numJvmOptions]; - for (size_t i = 0; i < numJvmOptions; ++i) { - options[i].optionString = strdup((char*)(m_jvmOptions[i].c_str())); - DBGVAR(cerr,options[i].optionString); - options[i].extraInfo = NULL; - } - - JavaVMInitArgs vm_args; - vm_args.version = JNI_VERSION_1_2; - vm_args.nOptions = numJvmOptions; - vm_args.options = options; - vm_args.ignoreUnrecognized = JNI_FALSE; - //vm_args.ignoreUnrecognized = JNI_TRUE; - DBGVAR(cerr,m_env); - - DBG_FUNC_CALL(cerr,int rc = JNI_CreateJavaVM(&m_jvm, (void**)&m_env, &vm_args)); - if (rc != JNI_OK) { - stringstream ss; - ss << "F-UDF-CL-SL-JAVA-1028: Cannot start the JVM: "; - switch (rc) { - case JNI_ERR: ss << "unknown error"; break; - case JNI_EDETACHED: ss << "thread is detached from VM"; break; - case JNI_EVERSION: ss << "version error"; break; - case JNI_ENOMEM: ss << "out of memory"; break; - case JNI_EEXIST: ss << "VM already exists"; break; - case JNI_EINVAL: ss << "invalid arguments"; break; - default: ss << "unknown"; break; - } - ss << " (" << rc << ")"; - delete [] options; - throw JavaVMach::exception(ss.str()); - } - delete [] options; -} - -void JavaVMImpl::compileScript() { - string calledUndefinedSingleCall; - jstring classnameStr = m_env->NewStringUTF(SWIGVM_params->script_name); - check("F-UDF-CL-SL-JAVA-1029",calledUndefinedSingleCall); - jstring codeStr = m_env->NewStringUTF(m_scriptCode.c_str()); - check("F-UDF-CL-SL-JAVA-1030",calledUndefinedSingleCall); - jstring classpathStr = m_env->NewStringUTF(m_localClasspath.c_str()); - check("F-UDF-CL-SL-JAVA-1031",calledUndefinedSingleCall); - if (!classnameStr || !codeStr || !classpathStr) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1032: NewStringUTF for compile failed"); - jclass cls = m_env->FindClass("com/exasol/ExaCompiler"); - check("F-UDF-CL-SL-JAVA-1033",calledUndefinedSingleCall); - if (!cls) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1034: FindClass for ExaCompiler failed"); - jmethodID mid = m_env->GetStaticMethodID(cls, "compile", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V"); - check("F-UDF-CL-SL-JAVA-1035",calledUndefinedSingleCall); - if (!mid) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1036: GetStaticMethodID for compile failed"); - m_env->CallStaticVoidMethod(cls, mid, classnameStr, codeStr, classpathStr); - check("F-UDF-CL-SL-JAVA-1037",calledUndefinedSingleCall); -} - - -bool JavaVMImpl::check(const string& errorCode, string& calledUndefinedSingleCall) { - jthrowable ex = m_env->ExceptionOccurred(); - if (ex) { - m_env->ExceptionClear(); - - jclass undefinedSingleCallExceptionClass = m_env->FindClass("com/exasol/ExaUndefinedSingleCallException"); - if (!undefinedSingleCallExceptionClass) { - throw JavaVMach::exception(errorCode+": F-UDF-CL-SL-JAVA-1042: FindClass for com.exasol.ExaUndefinedSingleCallException failed"); - } - if (m_env->IsInstanceOf(ex, undefinedSingleCallExceptionClass)) { - jmethodID undefinedRemoteFn = m_env->GetMethodID(undefinedSingleCallExceptionClass, "getUndefinedRemoteFn", "()Ljava/lang/String;"); - check("F-UDF-CL-SL-JAVA-1043",calledUndefinedSingleCall); - if (!undefinedRemoteFn) - throw JavaVMach::exception(errorCode+": F-UDF-CL-SL-JAVA-1044: com.exasol.ExaUndefinedSingleCallException.getUndefinedRemoteFn() could not be found"); - jobject undefinedRemoteFnString = m_env->CallObjectMethod(ex,undefinedRemoteFn); - if (undefinedRemoteFnString) { - jstring fn = static_cast(undefinedRemoteFnString); - const char *fn_str = m_env->GetStringUTFChars(fn,0); - std::string fn_string = fn_str; - m_env->ReleaseStringUTFChars(fn,fn_str); - calledUndefinedSingleCall = fn_string; - return 0; - //swig_undefined_single_call_exception ex(fn_string); - //throwException(ex); - } else { - throw JavaVMach::exception(errorCode+": F-UDF-CL-SL-JAVA-1045: Internal error: getUndefinedRemoteFn() returned no result"); - } - } - - string exceptionMessage = ""; - jclass exClass = m_env->GetObjectClass(ex); - if (!exClass) - throw JavaVMach::exception(errorCode+": F-UDF-CL-SL-JAVA-1046: FindClass for Throwable failed"); - // Throwable.toString() - jmethodID toString = m_env->GetMethodID(exClass, "toString", "()Ljava/lang/String;"); - check("F-UDF-CL-SL-JAVA-1047",calledUndefinedSingleCall); - if (!toString) - throw JavaVMach::exception(errorCode+": F-UDF-CL-SL-JAVA-1048: Throwable.toString() could not be found"); - jobject object = m_env->CallObjectMethod(ex, toString); - if (object) { - jstring message = static_cast(object); - char const *utfMessage = m_env->GetStringUTFChars(message, 0); - exceptionMessage.append("\n"); - exceptionMessage.append(utfMessage); - m_env->ReleaseStringUTFChars(message, utfMessage); - } - else { - exceptionMessage.append(errorCode+": F-UDF-CL-SL-JAVA-1049: Throwable.toString(): result is null"); - } - -// // Build Stacktrace -// -// // Throwable.getStackTrace() -// jmethodID getStackTrace = m_env->GetMethodID(exClass, "getStackTrace", "()[Ljava/lang/StackTraceElement;"); -// check("F-UDF-CL-SL-JAVA-1050",calledUndefinedSingleCall); -// if (!getStackTrace) -// throwException(errorCode+": F-UDF-CL-SL-JAVA-1051: Throwable.getStackTrace() could not be found"); -// jobjectArray frames = (jobjectArray)m_env->CallObjectMethod(ex, getStackTrace); -// if (frames) { -// jclass frameClass = m_env->FindClass("java/lang/StackTraceElement"); -// check("F-UDF-CL-SL-JAVA-1052",calledUndefinedSingleCall); -// if (!frameClass) -// throwException(errorCode+": F-UDF-CL-SL-JAVA-1053: FindClass for StackTraceElement failed"); -// jmethodID frameToString = m_env->GetMethodID(frameClass, "toString", "()Ljava/lang/String;"); -// check("F-UDF-CL-SL-JAVA-1054",calledUndefinedSingleCall); -// if (!frameToString) -// throwException(errorCode+": F-UDF-CL-SL-JAVA-1055: StackTraceElement.toString() could not be found"); -// jsize framesLength = m_env->GetArrayLength(frames); -// for (int i = 0; i < framesLength; i++) { -// jobject frame = m_env->GetObjectArrayElement(frames, i); -// check("F-UDF-CL-SL-JAVA-1056",calledUndefinedSingleCall); -// jobject frameMsgObj = m_env->CallObjectMethod(frame, frameToString); -// if (frameMsgObj) { -// jstring message = static_cast(frameMsgObj); -// const char *utfMessage = m_env->GetStringUTFChars(message, 0); -// string line = utfMessage; -// // If stack trace lines are wrapper or reflection entries, stop -// if ((line.find("ExaWrapper.") == 0) || (line.find("ExaCompiler.") == 0) || (line.find("sun.reflect.") == 0)) { -// if (i != 0) -// exceptionMessage.append("\n"); -// m_env->ReleaseStringUTFChars(message, utfMessage); -// break; -// } -// else { -// if (i == 0) -// exceptionMessage.append("\nStack trace:"); -// exceptionMessage.append("\n"); -// exceptionMessage.append(utfMessage); -// m_env->ReleaseStringUTFChars(message, utfMessage); -// } -// } -// } -// } - throw JavaVMach::exception(errorCode+": "+exceptionMessage); - } - return 1; -} - -void JavaVMImpl::registerFunctions() { - string calledUndefinedSingleCall; - jclass cls = m_env->FindClass("com/exasol/swig/exascript_javaJNI"); - check("F-UDF-CL-SL-JAVA-1057",calledUndefinedSingleCall); - if (!cls) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1058: FindClass for exascript_javaJNI failed"); - int rc = m_env->RegisterNatives(cls, methods, sizeof(methods) / sizeof(methods[0])); - check("F-UDF-CL-SL-JAVA-1059",calledUndefinedSingleCall); - if (rc) - throw JavaVMach::exception("F-UDF-CL-SL-JAVA-1060: RegisterNatives failed"); -} - -void JavaVMImpl::setClasspath() { - m_exaJarPath = m_exaJavaPath + "/exaudf_deploy.jar"; - m_classpath = m_exaJarPath; -} - -void JavaVMImpl::addLocalClasspath() { - m_classpath = m_localClasspath + ":" + m_classpath; -} - -bool JavaVMImpl::checkNeedsCompilation() { - std::string trimmedScriptCode = m_scriptCode; - trimmedScriptCode.erase(0, trimmedScriptCode.find_first_not_of("\t\n\r ")); // left trim - trimmedScriptCode.erase(trimmedScriptCode.find_last_not_of("\t\n\r ") + 1); // right trim - return false == trimmedScriptCode.empty(); -} - -void JavaVMImpl::addJarToClasspath(const string& path) { - string jarPath = path; // m_exaJavaPath + "/jars/" + path; - - struct stat st; - int rc = stat(jarPath.c_str(), &st); - if (rc && errno == ENOENT) { - // Not found in HOME, try path directly - jarPath = path; - rc = stat(jarPath.c_str(), &st); - if (rc) { - stringstream errorMsg; - errorMsg << "F-UDF-CL-SL-JAVA-1061: Java VM cannot find '" << jarPath.c_str() << "': " << strerror(errno); - throw JavaVMach::exception(errorMsg.str()); - } - } - else if (rc) { - stringstream errorMsg; - errorMsg << "F-UDF-CL-SL-JAVA-1062: Java VM cannot find '" << jarPath.c_str() << "': " << strerror(errno); - throw JavaVMach::exception(errorMsg.str()); - } - - if (!S_ISREG(st.st_mode)) { - stringstream errorMsg; - errorMsg << "F-UDF-CL-SL-JAVA-1063: '" << jarPath.c_str() << "' is not a regular file"; - throw JavaVMach::exception(errorMsg.str()); - } - - // Add file to classpath - m_classpath += ":" + jarPath; -} - - -void JavaVMImpl::setJvmOptions() { - bool minHeap = false; - bool maxHeap = false; - bool maxStack = false; - for (vector::iterator it = m_jvmOptions.begin(); it != m_jvmOptions.end(); ++it) { - if ((*it).find("-Xms") != string::npos) { - minHeap = true; - } - else if ((*it).find("-Xmx") != string::npos) { - maxHeap = true; - } - else if ((*it).find("-Xss") != string::npos) { - maxStack = true; - } - } - - // Initial heap size - unsigned long minHeapSize = 128; - unsigned long maxHeapSize = 1024; - if (!minHeap) { - stringstream ss; - ss << "-Xms" << minHeapSize << "m"; - m_jvmOptions.push_back(ss.str()); - } - // Max heap size - if (!maxHeap) { - unsigned long maxHeapSizeMb = static_cast(0.625 * (SWIGVM_params->maximal_memory_limit / (1024 * 1024))); - maxHeapSizeMb = (maxHeapSizeMb < minHeapSize) ? minHeapSize : maxHeapSizeMb; - maxHeapSizeMb = (maxHeapSizeMb > maxHeapSize) ? maxHeapSize : maxHeapSizeMb; - stringstream ss; - ss << "-Xmx" << maxHeapSizeMb << "m"; - m_jvmOptions.push_back(ss.str()); - } - // Max thread stack size - if (!maxStack) - m_jvmOptions.push_back("-Xss512k"); - // Error file path - m_jvmOptions.push_back("-XX:ErrorFile=" + m_localClasspath + "/hs_err_pid%p.log"); - // Classpath - m_jvmOptions.push_back("-Djava.class.path=" + m_classpath); - // Serial garbage collection - m_jvmOptions.push_back("-XX:+UseSerialGC"); // TODO allow different Garbage Collectors, multiple options are not allowed, so we need to check if options was specified by the user or otherwise use -XX:+UseSerialGC as default -} diff --git a/udf-runner-cpp/v1/base/javacontainer/javacontainer_impl.h b/udf-runner-cpp/v1/base/javacontainer/javacontainer_impl.h deleted file mode 100644 index 0423b21..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/javacontainer_impl.h +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef JAVACONTAINER_IMPL_H -#define JAVACONTAINER_IMPL_H - -#include -#include -#include -#include - -#include "exaudflib/vm/swig_vm.h" -#include - -#ifdef ENABLE_JAVA_VM - -class JavaVMTest; - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - struct Extractor; -} - -class JavaVMImpl { - public: - friend class ::JavaVMTest; - /* - * scriptOptionsParser: JavaVMImpl takes ownership of ScriptOptionsParser pointer. - */ - JavaVMImpl(bool checkOnly, bool noJNI, std::unique_ptr extractor); - ~JavaVMImpl() {} - void shutdown(); - bool run(); - const char* singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args, - std::string& calledUndefinedSingleCall); - private: - void createJvm(); - void addPackageToScript(); - void compileScript(); - bool check(const std::string& errorCode, std::string& calledUndefinedSingleCall); // returns 0 if the check failed - void registerFunctions(); - void addLocalClasspath(); - bool checkNeedsCompilation(); - void setClasspath(); - void setJvmOptions(); - void addJarToClasspath(const std::string& path); - void parseScriptOptions(std::unique_ptr extractor); - bool m_checkOnly; - std::string m_exaJavaPath; - std::string m_localClasspath; - std::string m_scriptCode; - std::string m_exaJarPath; - std::string m_classpath; - std::vector m_jvmOptions; - JavaVM *m_jvm; - JNIEnv *m_env; - bool m_needsCompilation; -}; - -} //namespace SWIGVMContainers - - -#endif //ENABLE_JAVA_VM - - -#endif //JAVACONTAINER_IMPL_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/BUILD b/udf-runner-cpp/v1/base/javacontainer/script_options/BUILD deleted file mode 100644 index 01b8b72..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - - -cc_library( - name = "java_script_option_lines", - hdrs = [":extractor.h", ":extractor_impl.h", ":parser_legacy.h", ":parser_ctpg.h", - ":converter_v2.h", ":converter_legacy.h"], - srcs = [":parser.h", ":converter.h", ":converter.cc", ":parser_legacy.cc", ":extractor_impl.h", - ":extractor_impl.cc", ":converter_legacy.cc", ":converter_legacy.h", - ":converter_v2.cc", ":converter_v2.h", - ":keywords.h", ":keywords.cc", ":distinct_script_set.h", ":distinct_script_set.cc", ":parser_ctpg.cc", - ":parser_ctpg_script_importer.cc", ":parser_ctpg_script_importer.h", - ":parser_ctpg_jvm_options_parser.cc", ":parser_ctpg_jvm_options_parser.h", ":string_ops.h", ":string_ops.cc"], - deps = ["//script_options_parser/legacy:script_option_lines_parser_legacy", - "//script_options_parser/ctpg:script_option_lines_parser_ctpg", "//utils:utils", - "//exaudflib:header", "//exaudflib:exaudflib-deps", "//swig_factory:swig_factory_if", - "//script_options_parser:exception"], -) diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/converter.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/converter.cc deleted file mode 100644 index d2f7103..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/converter.cc +++ /dev/null @@ -1,25 +0,0 @@ -#include "javacontainer/script_options/converter.h" - -#include - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -Converter::Converter() -: m_jvmOptions() -, m_whitespace(" \t\f\v") {} - -void Converter::convertScriptClassName(const std::string & value) { - - if (value.empty()) { - return; - } - m_jvmOptions.push_back("-Dexasol.scriptclass=" + value); -} - - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/converter.h b/udf-runner-cpp/v1/base/javacontainer/script_options/converter.h deleted file mode 100644 index ec7516e..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/converter.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERCONVERTER_H -#define SCRIPTOPTIONLINEPARSERCONVERTER_H 1 - -#include -#include -#include - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -/** - * This class retrieves the raw Java script option values (scriptclass, jvmoption, jar) - * and converts them to the proper format expected by the JvmContainerImpl class. - * Besides the converter functions it provides methods to access the converted values. - */ -class Converter { - - public: - typedef std::function tJarIteratorCallback; - - Converter(); - - void convertScriptClassName(const std::string & value); - - virtual void convertJvmOption(const std::string & value) = 0; - - std::vector&& moveJvmOptions() { - return std::move(m_jvmOptions); - } - - virtual void convertExternalJar(const std::string & value) = 0; - - virtual void iterateJarPaths(tJarIteratorCallback callback) const = 0; - - protected: - - std::vector m_jvmOptions; - - const std::string m_whitespace; -}; - - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPARSERCONVERTER_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/converter_legacy.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/converter_legacy.cc deleted file mode 100644 index ddf4276..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/converter_legacy.cc +++ /dev/null @@ -1,42 +0,0 @@ -#include "javacontainer/script_options/converter_legacy.h" -#include "javacontainer/script_options/string_ops.h" -#include -#include -#include - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -ConverterLegacy::ConverterLegacy() -: Converter() -, m_jarPaths() {} - -void ConverterLegacy::convertExternalJar(const std::string& value) { - std::istringstream stream(value); - std::string jar; - - while (std::getline(stream, jar, ':')) { - m_jarPaths.insert(jar); - } -} - -void ConverterLegacy::convertJvmOption(const std::string& value) { - std::string::size_type start = 0, delim = 0; - while ((start = value.find_first_not_of(m_whitespace, delim)) != std::string::npos) { - delim = value.find_first_of(m_whitespace, start); - const std::string::size_type len = (std::string::npos == delim) ? std::string::npos : delim - start; - m_jvmOptions.push_back(value.substr(start, len)); - } -} - - -void ConverterLegacy::iterateJarPaths(Converter::tJarIteratorCallback callback) const { - std::for_each(m_jarPaths.begin(), m_jarPaths.end(), callback); -} - - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/converter_legacy.h b/udf-runner-cpp/v1/base/javacontainer/script_options/converter_legacy.h deleted file mode 100644 index 6fb8abd..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/converter_legacy.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERCONVERTERLEGACY_H -#define SCRIPTOPTIONLINEPARSERCONVERTERLEGACY_H 1 - -#include -#include -#include -#include - -#include "javacontainer/script_options/converter.h" - - - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -/** - * This class is a specialization for the generic converter class. - * It implements conversion of the jar option according to the requirements in the old - * parser implementation. - */ -class ConverterLegacy : public Converter { - - public: - ConverterLegacy(); - - void convertExternalJar(const std::string & value) override; - - void convertJvmOption(const std::string & value) override; - - void iterateJarPaths(tJarIteratorCallback callback) const override; - - private: - - std::set m_jarPaths; - -}; - - - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPARSERCONVERTER_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/converter_v2.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/converter_v2.cc deleted file mode 100644 index 9f279f4..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/converter_v2.cc +++ /dev/null @@ -1,37 +0,0 @@ -#include "javacontainer/script_options/converter_v2.h" -#include "javacontainer/script_options/string_ops.h" -#include "javacontainer/script_options/parser_ctpg_jvm_options_parser.h" -#include -#include -#include - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -ConverterV2::ConverterV2() -: Converter() -, m_jarPaths() {} - -void ConverterV2::convertExternalJar(const std::string & value) { - std::istringstream stream(value); - std::string jar; - - while (std::getline(stream, jar, ':')) { - m_jarPaths.push_back(jar); - } -} - - -void ConverterV2::convertJvmOption(const std::string & value) { - JvmOptionsCTPG::parseJvmOptions(value, m_jvmOptions); -} - -void ConverterV2::iterateJarPaths(Converter::tJarIteratorCallback callback) const { - std::for_each(m_jarPaths.begin(), m_jarPaths.end(), callback); -} - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/converter_v2.h b/udf-runner-cpp/v1/base/javacontainer/script_options/converter_v2.h deleted file mode 100644 index 8807d90..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/converter_v2.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERCONVERTERV2_H -#define SCRIPTOPTIONLINEPARSERCONVERTERV2_H 1 - -#include -#include -#include - -#include "javacontainer/script_options/converter.h" - - - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -/** - * This class is a specialization for the generic converter class. - * It implements conversion of the jar option according to the requirements in the new - * parser implementation. - */ -class ConverterV2 : public Converter { - - public: - ConverterV2(); - - void convertExternalJar(const std::string & value) override; - - void convertJvmOption(const std::string & value) override; - - void iterateJarPaths(tJarIteratorCallback callback) const override; - - private: - - std::vector m_jarPaths; - -}; - - - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPARSERCONVERTER_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/distinct_script_set.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/distinct_script_set.cc deleted file mode 100644 index 34898ec..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/distinct_script_set.cc +++ /dev/null @@ -1,17 +0,0 @@ -#include "javacontainer/script_options/distinct_script_set.h" -#include - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - - -bool DistinctScriptSet::addScript(const char *script) { - std::string strScript = std::string(script); - return m_importedScripts.insert(strScript).second; -} - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/distinct_script_set.h b/udf-runner-cpp/v1/base/javacontainer/script_options/distinct_script_set.h deleted file mode 100644 index 081681b..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/distinct_script_set.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERCHECKSUM_H -#define SCRIPTOPTIONLINEPARSERCHECKSUM_H 1 - -#include -#include -#include - - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -class DistinctScriptSet { - -public: - DistinctScriptSet() = default; - - bool addScript(const char *script); - -private: - std::unordered_set m_importedScripts; -}; - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPARSERCHECKSUM_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/extractor.h b/udf-runner-cpp/v1/base/javacontainer/script_options/extractor.h deleted file mode 100644 index 8d1f669..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/extractor.h +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPEXTRACTOR_H -#define SCRIPTOPTIONLINEPEXTRACTOR_H 1 - -#include -#include -#include - - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -/** - * Abstract interface for the Extractor class. - * Defines methods to extract the Java options from the script code. - * The extraction process searches for the known Java Options and handles them appropriatly. - * The script code is then modified, where the found options are removed. - * The interface defines methods to access the found Jar- and JvmOption options. - * The scriptclass and import options are processed internally by the respective extractor implementation. - */ -struct Extractor { - - typedef std::function tJarIteratorCallback; - - virtual ~Extractor() {} - - /** - * Access the found Jar paths. For each found jar path the given callback function is called with the jar option as argument. - */ - virtual void iterateJarPaths(tJarIteratorCallback callback) const = 0; - - /** - * Access the Jvm option options. The extractor implementations must store the found Jvm Options in a std::vector. - * The vector is returned as rvalue. - */ - virtual std::vector&& moveJvmOptions() = 0; - - /** - * Run the extraction. This will: - * 1. Add the first `scriptclass` option to the list of Jvm options. - * 2. Replace and (nested) reference of an `import` script - * 3. Find and store all Jar options - * 4. Find and store all `jvmoption` options - * 5. Remove `scriptclass`, `jar`, `import` and `jvmoption` from the script code. The behavior is implementation specific. - */ - virtual void extract(std::string & scriptCode) = 0; -}; - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPEXTRACTOR_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/extractor_impl.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/extractor_impl.cc deleted file mode 100644 index b156d44..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/extractor_impl.cc +++ /dev/null @@ -1,54 +0,0 @@ -#include "javacontainer/script_options/extractor_impl.h" -#include "javacontainer/script_options/parser.h" - -#include "utils/debug_message.h" -#include - -#define EXTR_DBG_FUNC_CALL(f) DBG_FUNC_CALL(std::cerr, f) - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -template -ExtractorImpl::ExtractorImpl(std::unique_ptr swigFactory) -: m_parser(std::move(swigFactory)) -, m_converter() {} - -template -void ExtractorImpl::iterateJarPaths(Extractor::tJarIteratorCallback callback) const { - return m_converter.iterateJarPaths(callback); -} - -template -std::vector&& ExtractorImpl::moveJvmOptions() { - return std::move(m_converter.moveJvmOptions()); -} - -template -void ExtractorImpl::extract(std::string & scriptCode) { - m_parser.prepareScriptCode(scriptCode); - EXTR_DBG_FUNC_CALL(m_parser.parseForScriptClass( [&](const std::string& value){ - EXTR_DBG_FUNC_CALL(m_converter.convertScriptClassName(value)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used - })); - EXTR_DBG_FUNC_CALL(m_parser.extractImportScripts()); - EXTR_DBG_FUNC_CALL(m_parser.parseForJvmOptions( [&](const std::string& value){ - EXTR_DBG_FUNC_CALL(m_converter.convertJvmOption(value)); - })); - - EXTR_DBG_FUNC_CALL(m_parser.parseForExternalJars( [&](const std::string& value){ - EXTR_DBG_FUNC_CALL(m_converter.convertExternalJar(value)); - })); - - scriptCode = std::move(m_parser.getScriptCode()); -} - - -// Explict class template instantiations -template class ExtractorImpl; -template class ExtractorImpl; - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/extractor_impl.h b/udf-runner-cpp/v1/base/javacontainer/script_options/extractor_impl.h deleted file mode 100644 index e65fe5a..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/extractor_impl.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPEXTRACTORIMPL_H -#define SCRIPTOPTIONLINEPEXTRACTORIMPL_H 1 - -#include -#include -#include - - -#include "javacontainer/script_options/extractor.h" -#include "javacontainer/script_options/converter_legacy.h" -#include "javacontainer/script_options/converter_v2.h" -#include "javacontainer/script_options/parser_ctpg.h" -#include "javacontainer/script_options/parser_legacy.h" -#include "swig_factory/swig_factory.h" - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -/** - * Concrete implementation for the Extractor class. - * Given template parameter TParser and TConverter define concrete behavior. - */ -template -class ExtractorImpl : public Extractor { - - public: - - ExtractorImpl(std::unique_ptr swigFactory); - - virtual void iterateJarPaths(tJarIteratorCallback callback) const override; - std::vector&& moveJvmOptions() override; - - void extract(std::string & scriptCode); - - private: - TParser m_parser; - TConverter m_converter; -}; - -typedef ExtractorImpl tExtractorLegacy; -typedef ExtractorImpl tExtractorV2; - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPEXTRACTORIMPL_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/keywords.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/keywords.cc deleted file mode 100644 index 4ac2aff..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/keywords.cc +++ /dev/null @@ -1,33 +0,0 @@ -#include "javacontainer/script_options/keywords.h" -#include - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -Keywords::Keywords(bool withScriptOptionsPrefix) -: m_jarKeyword() -, m_scriptClassKeyword() -, m_importKeyword() -, m_jvmKeyword() { - const std::string_view jar{"%jar"}; - const std::string_view scriptClass{"%scriptclass"}; - const std::string_view import{"%import"}; - const std::string_view jvm{"%jvmoption"}; - if (withScriptOptionsPrefix) { - m_jarKeyword = jar; - m_scriptClassKeyword = scriptClass; - m_importKeyword = import; - m_jvmKeyword = jvm; - } else { - m_jarKeyword.assign(jar.substr(1)); - m_scriptClassKeyword.assign(scriptClass.substr(1)); - m_importKeyword.assign(import.substr(1)); - m_jvmKeyword.assign(jvm.substr(1)); - } -} - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/keywords.h b/udf-runner-cpp/v1/base/javacontainer/script_options/keywords.h deleted file mode 100644 index 314bf5b..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/keywords.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef SCRIPTOPTIONLINEKEYWORDS_H -#define SCRIPTOPTIONLINEKEYWORDS_H 1 - -#include - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -class Keywords { - public: - Keywords(bool withScriptOptionsPrefix); - const std::string & scriptClassKeyword() { return m_scriptClassKeyword; } - const std::string & importKeyword() { return m_importKeyword; } - const std::string & jvmKeyword() { return m_jvmKeyword; } - const std::string & jarKeyword() { return m_jarKeyword; } - private: - std::string m_jarKeyword; - std::string m_scriptClassKeyword; - std::string m_importKeyword; - std::string m_jvmKeyword; -}; - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - - -#endif // SCRIPTOPTIONLINEKEYWORDS_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser.h b/udf-runner-cpp/v1/base/javacontainer/script_options/parser.h deleted file mode 100644 index abb497f..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSER_H -#define SCRIPTOPTIONLINEPARSER_H 1 - -#include -#include -#include - - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -struct ScriptOptionsParser { - virtual ~ScriptOptionsParser() {}; - /* - Passes the script code for parsing to the parser. The parser might modify the script code, because it will remove - known options. - */ - virtual void prepareScriptCode(const std::string & scriptCode) = 0; - /* - Searches for script class option. - If the option is found, the function removes the option from "scriptCode" and calls "callback" with the option value and position - within "scriptCode". - */ - virtual void parseForScriptClass(std::function callback) = 0; - /* - Searches for JVM options. - If an option is found, the function removes the option from "scriptCode" and calls "callback" with the option value and position - within "scriptCode". - */ - virtual void parseForJvmOptions(std::function callback) = 0; - - /* - Searches for External Jar. - If an option is found, the function removes the option from "scriptCode" and calls "callback" with the option value and position - within "scriptCode". - */ - virtual void parseForExternalJars(std::function callback) = 0; - - /* - Searches for the "%import" options and embeds the respective imported script code at the same location as - the option in the script code. - */ - virtual void extractImportScripts() = 0; - - /* - Returns the (eventually modified) script code. - */ - virtual std::string && getScriptCode() = 0; -}; - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - - -#endif //SCRIPTOPTIONLINEPARSER_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg.cc deleted file mode 100644 index 6209a78..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg.cc +++ /dev/null @@ -1,162 +0,0 @@ -#include "javacontainer/script_options/parser_ctpg.h" -#include "javacontainer/script_options/parser_ctpg_script_importer.h" -#include "utils/exceptions.h" -#include "script_options_parser/exception.h" -#include "swig_factory/swig_factory.h" -#include "javacontainer/script_options/string_ops.h" -#include -#include - - -namespace ctpg_parser = ExecutionGraph::OptionsLineParser::CTPG; - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -ScriptOptionLinesParserCTPG::ScriptOptionLinesParserCTPG(std::unique_ptr swigFactory) -: m_scriptCode() -, m_keywords(false) -, m_needParsing(true) -, m_swigFactory(std::move(swigFactory)) {} - -void ScriptOptionLinesParserCTPG::prepareScriptCode(const std::string & scriptCode) { - m_scriptCode = scriptCode; -} - -void ScriptOptionLinesParserCTPG::parseForScriptClass(std::function callback) { - try { - parseForSingleOption(m_keywords.scriptClassKeyword(), [&](const std::string &option) { - std::string trimmedValue(option); - StringOps::trim(trimmedValue); - callback(trimmedValue); - }); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1623"); - } -} - -void ScriptOptionLinesParserCTPG::parseForJvmOptions(std::function callback) { - try { - parseForMultipleOption(m_keywords.jvmKeyword(), callback); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1624"); - } -} - -void ScriptOptionLinesParserCTPG::parseForExternalJars(std::function callback) { - try { - parseForMultipleOption(m_keywords.jarKeyword(), [callback] (const std::string &option) { - std::string formattedValue(option); - StringOps::replaceTrailingEscapeWhitespaces(formattedValue); - callback(formattedValue); - }); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1625"); - } -} - -void ScriptOptionLinesParserCTPG::extractImportScripts() { - - try { - parse(); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1626"); - } - - const auto optionIt = m_foundOptions.find(m_keywords.importKeyword()); - if (optionIt != m_foundOptions.end()) { - CTPG::ScriptImporter scriptImporter(*m_swigFactory, m_keywords); - scriptImporter.importScript(m_scriptCode, m_foundOptions); - //The imported scripts will change the location of the other options in m_foundOptions - //Also there might be new JVM / External Jar options - //=> We need to clear the option map and reset the parser. - m_foundOptions.clear(); - m_needParsing = true; - } -} - -std::string && ScriptOptionLinesParserCTPG::getScriptCode() { - try { - parse(); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1627"); - } - //Remove all options from script code in reverse order - struct option_location { - size_t pos; - size_t len; - }; - struct comp { - bool operator()(option_location a, option_location b) const { - return a.pos > b.pos; - } - }; - std::set option_locations; - for (const auto & option: m_foundOptions) { - for (const auto & option_loc: option.second) { - option_location loc = { .pos = option_loc.idx_in_source, .len = option_loc.size}; - option_locations.insert(loc); - } - } - for (const auto option_loc: option_locations) { - m_scriptCode.erase(option_loc.pos, option_loc.len); - } - return std::move(m_scriptCode); -} - -void ScriptOptionLinesParserCTPG::parse() { - if (m_needParsing) { - if(!m_foundOptions.empty()) { - throw std::logic_error( - "F-UDF-CL-SL-JAVA-1620 Internal error. Parser result is not empty. " - "Please open a bug ticket at https://github.com/exasol/script-languages-release/issues/new."); - } - try { - ExecutionGraph::OptionsLineParser::CTPG::parseOptions(m_scriptCode, m_foundOptions); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1621"); - } - - m_needParsing = false; - - //Check for unknown options - for (const auto & option: m_foundOptions) { - if (m_keywords.jarKeyword() != option.first && - m_keywords.scriptClassKeyword() != option.first && - m_keywords.importKeyword() != option.first && - m_keywords.jvmKeyword() != option.first) { - std::stringstream ss; - ss << "F-UDF-CL-SL-JAVA-1622 " << "Unexpected option: " << option.first; - throw std::invalid_argument(ss.str()); - } - } - } -} - -void ScriptOptionLinesParserCTPG::parseForSingleOption(const std::string key, std::function callback) { - parse(); - const auto optionIt = m_foundOptions.find(key); - if (optionIt != m_foundOptions.end()) { - if (optionIt->second.size() != 1) { - std::stringstream ss; - ss << "F-UDF-CL-SL-JAVA-1628 found " << optionIt->second.size() << " instances for script option key '" << key << "' but expected at most one." << std::endl; - throw std::invalid_argument(ss.str()); - } - callback(optionIt->second[0].value); - } -} - -void ScriptOptionLinesParserCTPG::parseForMultipleOption(const std::string key, std::function callback) { - parse(); - const auto optionIt = m_foundOptions.find(key); - if (optionIt != m_foundOptions.end()) { - for (const auto & option : optionIt->second) { - callback(option.value); - } - } -} - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg.h b/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg.h deleted file mode 100644 index 1f7bad8..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERCTPGY_H -#define SCRIPTOPTIONLINEPARSERCTPGY_H 1 - -#include "javacontainer/script_options/parser.h" -#include "javacontainer/script_options/keywords.h" -#include "script_options_parser/ctpg/script_option_lines_ctpg.h" -#include - -namespace SWIGVMContainers { - -struct SwigFactory; - -namespace JavaScriptOptions { - -class ScriptOptionLinesParserCTPG : public ScriptOptionsParser { - - public: - ScriptOptionLinesParserCTPG(std::unique_ptr swigFactory); - - virtual ~ScriptOptionLinesParserCTPG() {}; - - void prepareScriptCode(const std::string & scriptCode) override; - - void parseForScriptClass(std::function callback) override; - - void parseForJvmOptions(std::function callback) override; - - void parseForExternalJars(std::function callback) override; - - void extractImportScripts() override; - - std::string && getScriptCode() override; - - private: - void parse(); - - void parseForSingleOption(const std::string key, std::function callback); - void parseForMultipleOption(const std::string key, std::function callback); - - void importScripts(SwigFactory & swigFactory); - - private: - std::string m_scriptCode; - Keywords m_keywords; - ExecutionGraph::OptionsLineParser::CTPG::options_map_t m_foundOptions; - bool m_needParsing; - std::unique_ptr m_swigFactory; -}; - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPARSERCTPGY_H diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_jvm_options_parser.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_jvm_options_parser.cc deleted file mode 100644 index 826af38..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_jvm_options_parser.cc +++ /dev/null @@ -1,98 +0,0 @@ -#include "javacontainer/script_options/parser_ctpg_jvm_options_parser.h" -#include "script_options_parser/ctpg/ctpg.hpp" -#include - -using namespace exaudf_ctpg; -using namespace exaudf_ctpg::ftors; - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -namespace JvmOptionsCTPG { - - -const auto to_options(std::string&& e) -{ - tJvmOptions options{e}; - return options; -} - -auto&& add_option(std::string&& e, tJvmOptions&& ob) -{ - ob.push_back(std::move(e)); - return std::move(ob); -} - -auto convert_escape_sequence(std::string_view escape_seq) -{ - if (escape_seq == R"_(\ )_") { - return std::string_view(" "); - } else if (escape_seq == R"_(\t)_") { - return std::string_view("\t"); - } else if (escape_seq == R"_(\f)_") { - return std::string_view("\f"); - } else if (escape_seq == R"_(\v)_") { - return std::string_view("\v"); - } else if (escape_seq == R"_(\\)_") { - return std::string_view("\\"); - } else { - throw std::invalid_argument(std::string("Internal parser error: Unexpected escape sequence " + std::string(escape_seq))); - } -} - - -constexpr char not_separator_pattern[] = R"_([^ \x09\x0c\x0b])_"; -constexpr char whitespaces_pattern[] = R"_([ \x09\x0c\x0b]+)_"; -constexpr char escape_pattern[] = R"_(\\\\|\\t|\\ |\\f|\\v)_"; - -constexpr regex_term not_separator("not_separator"); -constexpr regex_term whitespaces("whitespace"); -constexpr regex_term escape_seq("escape_seq"); - -constexpr nterm text("text"); -constexpr nterm option_element("option_element"); - - -constexpr parser jvm_option_parser( - text, - terms(escape_seq, not_separator, whitespaces), - nterms(text, option_element), - rules( - text() - >= [] () {return tJvmOptions();}, - text(option_element) - >= [](auto&& e) { return to_options(std::move(e)); }, - text(text, whitespaces) - >= [](auto&& ob, skip) { return std::move(ob); }, - text(text, whitespaces, option_element) - >= [](auto&& ob, skip, auto&& e) { return add_option(std::move(e), std::move(ob)); }, - option_element(not_separator) - >= [](auto ns) { return std::string(ns); }, - option_element(option_element, not_separator) - >= [](auto &&ob, auto c) { return std::move(ob.append(c)); }, - option_element(option_element, escape_seq) - >= [](auto &&ob, auto es) { return std::move(ob.append(convert_escape_sequence(es))); } - ) -); - - -void parseJvmOptions(const std::string & jvmOptions, tJvmOptions& result) { - std::stringstream error_buffer; - auto && res = jvm_option_parser.parse( - parse_options{}.set_skip_whitespace(false), - buffers::string_buffer(jvmOptions.c_str()), - error_buffer); - if (res.has_value()) { - result.insert(result.end(), res.value().begin(), res.value().end()); - } - else { - throw std::invalid_argument(error_buffer.str()); - } -} - -} //namespace JvmOptionsCTPG - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_jvm_options_parser.h b/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_jvm_options_parser.h deleted file mode 100644 index 84ee8dd..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_jvm_options_parser.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERCTPGJVMOPTIONSPARSER_H -#define SCRIPTOPTIONLINEPARSERCTPGJVMOPTIONSPARSER_H 1 - -#include -#include - - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -namespace JvmOptionsCTPG { - -typedef std::vector tJvmOptions; - -void parseJvmOptions(const std::string & jvmOptions, tJvmOptions& result); - - - -} //namespace CTPG - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_script_importer.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_script_importer.cc deleted file mode 100644 index d5daeba..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_script_importer.cc +++ /dev/null @@ -1,111 +0,0 @@ -#include "javacontainer/script_options/parser_ctpg_script_importer.h" -#include "swig_factory/swig_factory.h" -#include "utils/debug_message.h" -#include "utils/exceptions.h" -#include "script_options_parser/exception.h" -#include "javacontainer/script_options/string_ops.h" - -#include -#include -#include - -namespace ctpg_parser = ExecutionGraph::OptionsLineParser::CTPG; - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -namespace CTPG { - -ScriptImporter::ScriptImporter(SwigFactory & swigFactory, Keywords & keywords) -: m_importedScripts() -, m_swigFactory(swigFactory) -, m_metaData() -, m_keywords(keywords) {} - -void ScriptImporter::importScript(std::string & scriptCode, - ctpg_parser::options_map_t & options) { - importScript(scriptCode, options, 0); -} - -void ScriptImporter::collectImportScripts(const ScriptImporter::OptionValues_t & option_values, - const size_t recursionDepth, - std::vector &result) { - for (const auto & option: option_values) { - const char *importScriptCode = findImportScript(option.value); - std::string importScriptCodeStr; - if (m_importedScripts.addScript(importScriptCode) ) { - // Script has not been imported yet - // If this imported script contains %import statements - // they will be resolved in the next recursion. - ctpg_parser::options_map_t newOptions; - try { - ExecutionGraph::OptionsLineParser::CTPG::parseOptions(importScriptCode, newOptions); - } catch(const ExecutionGraph::OptionParserException & ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1630"); - } - importScriptCodeStr.assign(importScriptCode); - importScript(importScriptCodeStr, newOptions, recursionDepth + 1); - } - CollectedScript replacedScript = {.script = std::move(importScriptCodeStr), .origPos = option.idx_in_source, .origLen = option.size }; - result.push_back(std::move(replacedScript)); - } -} - -void ScriptImporter::replaceImportScripts(std::string & scriptCode, - const std::vector &collectedImportScripts) { - //Replace the imported script bodies from end to start. - //Doing it in forward order would invalidate the offsets of later import scripts. - for (auto optionIt = collectedImportScripts.crbegin(); optionIt != collectedImportScripts.crend(); optionIt++) { - scriptCode.replace(optionIt->origPos, optionIt->origLen, optionIt->script); - } -} - - -void ScriptImporter::importScript(std::string & scriptCode, - ctpg_parser::options_map_t & options, - const size_t recursionDepth) { - const auto optionIt = options.find(std::string(m_keywords.importKeyword())); - - if (recursionDepth >= cMaxRecursionDepth) { - throw std::runtime_error("F-UDF-CL-SL-JAVA-1633: Maximal recursion depth for importing scripts reached."); - } - if (optionIt != options.end()) { - m_importedScripts.addScript(scriptCode.c_str()); - //Sort options from first in script to last in script - std::sort(optionIt->second.begin(), optionIt->second.end(), - [](const ctpg_parser::ScriptOption& first, const ctpg_parser::ScriptOption& second) - { - return first.idx_in_source < second.idx_in_source; - }); - std::vector collectedScripts; - collectedScripts.reserve(optionIt->second.size()); - //In order to continue compatibility with legacy implementation - //we must collect import scripts in forward direction but then replace in reverse direction. - collectImportScripts(optionIt->second, recursionDepth, collectedScripts); - replaceImportScripts(scriptCode, collectedScripts); - } -} - -const char* ScriptImporter::findImportScript(const std::string & scriptKey) { - std::string trimmedScriptKey(scriptKey); - StringOps::trim(trimmedScriptKey); - if (!m_metaData) { - m_metaData.reset(m_swigFactory.makeSwigMetadata()); - if (!m_metaData) { - throw std::runtime_error("F-UDF-CL-SL-JAVA-1631: Failure while importing scripts"); - } - } - const char *importScriptCode = m_metaData->moduleContent(trimmedScriptKey.c_str()); - const char *exception = m_metaData->checkException(); - if (exception) { - throw std::runtime_error("F-UDF-CL-SL-JAVA-1632: " + std::string(exception)); - } - return importScriptCode; -} - -} //namespace CTPG - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_script_importer.h b/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_script_importer.h deleted file mode 100644 index 284a406..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_ctpg_script_importer.h +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H -#define SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H 1 - -#include "javacontainer/script_options/distinct_script_set.h" -#include "javacontainer/script_options/keywords.h" -#include "exaudflib/swig/swig_meta_data.h" -#include "script_options_parser/ctpg/script_option_lines_ctpg.h" -#include - - -namespace SWIGVMContainers { - - struct SwigFactory; - -namespace JavaScriptOptions { - -namespace CTPG { - - - -class ScriptImporter { - - public: - ScriptImporter(SwigFactory & swigFactory, Keywords & keywords); - - void importScript(std::string & scriptCode, ExecutionGraph::OptionsLineParser::CTPG::options_map_t & options); - - private: - struct CollectedScript { - CollectedScript(CollectedScript&&) = default; - std::string script; - size_t origPos; - size_t origLen; - }; - - typedef ExecutionGraph::OptionsLineParser::CTPG::options_map_t::mapped_type OptionValues_t; - - void importScript(std::string & scriptCode, - ExecutionGraph::OptionsLineParser::CTPG::options_map_t & options, - const size_t recursionDepth); - const char* findImportScript(const std::string & scriptKey); - - void collectImportScripts(const OptionValues_t & option_values, - const size_t recursionDepth, - std::vector &result); - - void replaceImportScripts(std::string & scriptCode, - const std::vector &collectedImportScripts); - - DistinctScriptSet m_importedScripts; - SwigFactory & m_swigFactory; - std::unique_ptr m_metaData; - Keywords & m_keywords; - //The empirical maximal value for recursion depth is ~18000. So we add a little bit extra to have some buffer. - const size_t cMaxRecursionDepth = 10000U; -}; - -} //namespace CTPG - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPARSERCTPGSCRIPTIMPORTER_H diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_legacy.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/parser_legacy.cc deleted file mode 100644 index 3da5b93..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_legacy.cc +++ /dev/null @@ -1,198 +0,0 @@ -#include "javacontainer/script_options/parser_legacy.h" -#include "javacontainer/script_options/distinct_script_set.h" -#include "script_options_parser/legacy/script_option_lines.h" -#include "exaudflib/swig/swig_meta_data.h" -#include "swig_factory/swig_factory.h" -#include "utils/exceptions.h" -#include "script_options_parser/exception.h" - -#include - - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -ScriptOptionLinesParserLegacy::ScriptOptionLinesParserLegacy(std::unique_ptr swigFactory) -: m_whitespace(" \t\f\v") -, m_lineend(";") -, m_scriptCode() -, m_keywords(true) -, m_swigFactory(std::move(swigFactory)) {} - -void ScriptOptionLinesParserLegacy::prepareScriptCode(const std::string & scriptCode) { - m_scriptCode = scriptCode; -} - -void ScriptOptionLinesParserLegacy::extractImportScripts() { - std::unique_ptr metaData; - // Attention: We must hash the parent script before modifying it (adding the - // package definition). Otherwise we don't recognize if the script imports its self - DistinctScriptSet importedScripts; - importedScripts.addScript(m_scriptCode.c_str()); - /* - The following while loop iteratively replaces import scripts in the script code. Each replacement is done in two steps: - 1. Remove the "%import ..." option in the script code - 2. Insert the new script code at the same location - It can happen that the imported scripts have again an "%import ..." option. - Those cases will be handled in the next iteration of the while loop, because the parser searches the options in - each iteration from the beginning of the (then modified) script code. - For example, lets assume the following Jave UDF, stored in member variable 'm_scriptCode': - %import other_script_A; - class MyJavaUdf { - static void run(ExaMetadata exa, ExaIterator ctx) throws Exception { - ctx.emit(\"Success!\");\n" - } - }; - and 'other_script_A' is defined as (which will be retrieved over SWIGMetadata.moduleContent()): - %import other_script_B; - class OtherClassA { - static void doSomething() { - } - }; - and other_script_B as: - %import other_script_A; - class OtherClassB { - static void doSomething() { - } - }; - The first iteration of the while loop would modify the member variable m_scriptCode to: - %import other_script_B; - class OtherClassA { - static void doSomething() { - } - }; - class MyJavaUdf { - static void run(ExaMetadata exa, ExaIterator ctx) throws Exception { - ctx.emit(\"Success!\");\n" - } - }; - The second iteration of the while loop would modify the member variable m_scriptCode to: - The first iteration of the while loop would modify the member variable m_scriptCode to: - %import other_script_A; - class OtherClassB { - static void doSomething() { - } - }; - class OtherClassA { - static void doSomething() { - } - }; - class MyJavaUdf { - static void run(ExaMetadata exa, ExaIterator ctx) throws Exception { - ctx.emit(\"Success!\");\n" - } - }; - The third iteration of the while loop would modify the member variable m_scriptCode to: - class OtherClassB { - static void doSomething() { - } - }; - class OtherClassA { - static void doSomething() { - } - }; - class MyJavaUdf { - static void run(ExaMetadata exa, ExaIterator ctx) throws Exception { - ctx.emit(\"Success!\");\n" - } - }; - because the content of "other_script_A" is already stored in the DistinctScriptSet, - and the parser only removes the script options keyword, but does not insert again the code of other_script_A. - The fourth iteration of the while loop would detect no further import option keywords and would break the loop. - */ - while (true) { - std::string newScript; - size_t scriptPos; - try { - parseForSingleOption(m_keywords.importKeyword(), - [&](const std::string& value, size_t pos){scriptPos = pos; newScript = value;}); - } catch (const ExecutionGraph::OptionParserException & ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1614"); - } - if (!newScript.empty()) { - if (!metaData) { - metaData.reset(m_swigFactory->makeSwigMetadata()); - if (!metaData) - throw std::runtime_error("F-UDF-CL-SL-JAVA-1615: Failure while importing scripts"); - } - const char *importScriptCode = metaData->moduleContent(newScript.c_str()); - const char *exception = metaData->checkException(); - if (exception) - throw std::runtime_error("F-UDF-CL-SL-JAVA-1616: " + std::string(exception)); - if (importedScripts.addScript(importScriptCode)) { - // Script has not been imported yet - // If this imported script contains %import statements - // they will be resolved in the next iteration of the while loop. - m_scriptCode.insert(scriptPos, importScriptCode); - } - } else { - break; - } - } -} - -void ScriptOptionLinesParserLegacy::parseForScriptClass(std::function callback) { - try { - parseForSingleOption(m_keywords.scriptClassKeyword(), - [&](const std::string& value, size_t pos){callback(value);}); - } catch (const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1610"); - } -} - -void ScriptOptionLinesParserLegacy::parseForJvmOptions(std::function callback) { - try { - parseForMultipleOptions(m_keywords.jvmKeyword(), - [&](const std::string& value, size_t pos){callback(value);}); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1612"); - } -} - -void ScriptOptionLinesParserLegacy::parseForExternalJars(std::function callback) { - try { - parseForMultipleOptions(m_keywords.jarKeyword(), - [&](const std::string& value, size_t pos){callback(value);}); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1613"); - } -} - -std::string && ScriptOptionLinesParserLegacy::getScriptCode() { - return std::move(m_scriptCode); -} - -void ScriptOptionLinesParserLegacy::parseForSingleOption(const std::string & keyword, - std::function callback) { - size_t pos; - try { - const std::string option = - ExecutionGraph::extractOptionLine(m_scriptCode, keyword, m_whitespace, m_lineend, pos); - if (option != "") { - callback(option, pos); - } - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1606"); - } -} - -void ScriptOptionLinesParserLegacy::parseForMultipleOptions(const std::string & keyword, - std::function callback) { - size_t pos; - while (true) { - try { - const std::string option = - ExecutionGraph::extractOptionLine(m_scriptCode, keyword, m_whitespace, m_lineend, pos); - if (option == "") - break; - callback(option, pos); - } catch(const ExecutionGraph::OptionParserException& ex) { - Utils::rethrow(ex, "F-UDF-CL-SL-JAVA-1607"); - } - } -} - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_legacy.h b/udf-runner-cpp/v1/base/javacontainer/script_options/parser_legacy.h deleted file mode 100644 index 6159cff..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/parser_legacy.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERLEGACY_H -#define SCRIPTOPTIONLINEPARSERLEGACY_H 1 - - -#include "javacontainer/script_options/parser.h" -#include "javacontainer/script_options/keywords.h" - -#include - -namespace SWIGVMContainers { - -struct SwigFactory; - -namespace JavaScriptOptions { - -class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { - - public: - ScriptOptionLinesParserLegacy(std::unique_ptr swigFactory); - - virtual ~ScriptOptionLinesParserLegacy() {}; - - void prepareScriptCode(const std::string & scriptCode) override; - - void parseForScriptClass(std::function callback) override; - - void parseForJvmOptions(std::function callback) override; - - void parseForExternalJars(std::function callback) override; - - void extractImportScripts() override; - - std::string && getScriptCode() override; - - private: - void parseForSingleOption(const std::string& key, - std::function callback); - void parseForMultipleOptions(const std::string& key, - std::function callback); - - private: - const std::string m_whitespace; - const std::string m_lineend; - std::string m_scriptCode; - Keywords m_keywords; - std::unique_ptr m_swigFactory; -}; - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - - -#endif //SCRIPTOPTIONLINEPARSERLEGACY_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/string_ops.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/string_ops.cc deleted file mode 100644 index d6d6d89..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/string_ops.cc +++ /dev/null @@ -1,79 +0,0 @@ -#include "javacontainer/script_options/string_ops.h" -#include -#include -#include - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -namespace StringOps { - -inline uint32_t countBackslashesBackwards(const std::string & s, size_t pos) { - uint32_t retVal(0); - while (pos >= 0 && s.at(pos--) == '\\') retVal++; - return retVal; -} - -inline size_t replaceOnlyBackslashSequencesButKeepChar(std::string & s, size_t backslashStartIdx, size_t nBackslashes) { - const uint32_t nHalfBackslashes = (nBackslashes>>1); - if (nHalfBackslashes > 0) { - s = s.erase(backslashStartIdx, nHalfBackslashes ); - } - const size_t newBackslashEndIdx = backslashStartIdx + nHalfBackslashes; - return newBackslashEndIdx + 1; //+1 because of the last None-Whitespace character we need to keep -} - -inline size_t replaceBackslashSequencesAndWhitespaceSequence(std::string & s, size_t backslashStartIdx, - size_t nBackslashes, const char* replacement) { - const uint32_t nHalfBackslashes = (nBackslashes>>1); - s = s.erase(backslashStartIdx, nHalfBackslashes+1 ); //Delete also backslash of whitespace escape sequence - const size_t newBackslashEndIdx = backslashStartIdx + nHalfBackslashes; - s = s.replace(newBackslashEndIdx, 1, replacement); - return newBackslashEndIdx + 1; //+1 because of the replaced whitespace character -} - -inline size_t replaceCharAtPositionAndBackslashes(std::string & s, size_t pos, const char* replacement) { - const size_t nBackslashes = (pos > 0) ? countBackslashesBackwards(s, pos-1) : 0; - - const size_t backslashStartIdx = pos-nBackslashes; - if(nBackslashes % 2 == 0) { - return replaceOnlyBackslashSequencesButKeepChar(s, backslashStartIdx, nBackslashes); - } - else { - return replaceBackslashSequencesAndWhitespaceSequence(s, backslashStartIdx, nBackslashes, replacement); - } -} - -void replaceTrailingEscapeWhitespaces(std::string & s) { - if (s.size() > 0) { - const size_t lastNoneWhitespaceIdx = s.find_last_not_of(" \t\v\f"); - if (lastNoneWhitespaceIdx != std::string::npos) { - size_t firstWhitespaceAfterNoneWhitespaceIdx = lastNoneWhitespaceIdx + 1; - if (s.size() > 1) { - if(s[lastNoneWhitespaceIdx] == 't') { - firstWhitespaceAfterNoneWhitespaceIdx = replaceCharAtPositionAndBackslashes(s, lastNoneWhitespaceIdx, "\t"); - } else if (s[lastNoneWhitespaceIdx] == '\\' && lastNoneWhitespaceIdx < (s.size() - 1) && s.at(lastNoneWhitespaceIdx+1) == ' ') { - firstWhitespaceAfterNoneWhitespaceIdx = replaceCharAtPositionAndBackslashes(s, lastNoneWhitespaceIdx+1, " "); - } else if (s[lastNoneWhitespaceIdx] == 'f') { - firstWhitespaceAfterNoneWhitespaceIdx = replaceCharAtPositionAndBackslashes(s, lastNoneWhitespaceIdx, "\f"); - } else if (s[lastNoneWhitespaceIdx] == 'v') { - firstWhitespaceAfterNoneWhitespaceIdx = replaceCharAtPositionAndBackslashes(s, lastNoneWhitespaceIdx, "\v"); - } - } - if (firstWhitespaceAfterNoneWhitespaceIdx != std::string::npos && - firstWhitespaceAfterNoneWhitespaceIdx < s.size()) { - s = s.substr(0, firstWhitespaceAfterNoneWhitespaceIdx); //Right Trim the string - } - } else { - s = ""; - } - } -} - -} //namespace StringOps - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/string_ops.h b/udf-runner-cpp/v1/base/javacontainer/script_options/string_ops.h deleted file mode 100644 index 7da1e53..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/string_ops.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef SCRIPTOPTIONLINEPARSERSTRINGOPS_H -#define SCRIPTOPTIONLINEPARSERSTRINGOPS_H 1 - -#include -#include - - - -namespace SWIGVMContainers { - -namespace JavaScriptOptions { - -namespace StringOps { - -//Following code is based on https://stackoverflow.com/a/217605 - -inline void ltrim(std::string &s) { - s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) { - return !std::isspace(ch); - })); -} - -inline void rtrim(std::string &s) { - s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { - return !std::isspace(ch); - }).base(), s.end()); -} - -inline void trim(std::string &s) { - ltrim(s); - rtrim(s); -} - -void replaceTrailingEscapeWhitespaces(std::string & s); - -} //namespace StringOps - - -} //namespace JavaScriptOptions - -} //namespace SWIGVMContainers - -#endif //SCRIPTOPTIONLINEPARSERSTRINGOPS_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/test/BUILD b/udf-runner-cpp/v1/base/javacontainer/script_options/test/BUILD deleted file mode 100644 index 76f9bf6..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/test/BUILD +++ /dev/null @@ -1,10 +0,0 @@ - -cc_test( - name = "java-script-options-tests", - srcs = ["ctpg_script_importer_test.cc", "swig_factory_test.cc", - "swig_factory_test.h", "string_ops_tests.cc", "converter_test.cc"], - deps = [ - "//javacontainer/script_options:java_script_option_lines", - "@googletest//:gtest_main", - ], -) diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/test/converter_test.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/test/converter_test.cc deleted file mode 100644 index ba640e7..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/test/converter_test.cc +++ /dev/null @@ -1,92 +0,0 @@ - -#include "include/gtest/gtest.h" -#include "gmock/gmock.h" -#include "javacontainer/script_options/converter_legacy.h" -#include "javacontainer/script_options/converter_v2.h" - -using namespace SWIGVMContainers::JavaScriptOptions; - - -class LegacyConverterJarTest : public ::testing::TestWithParam>> {}; - -TEST_P(LegacyConverterJarTest, jar) { - const std::pair> option_value = GetParam(); - const std::string jar_option_value = option_value.first; - - ConverterLegacy converter; - converter.convertExternalJar(option_value.first); - std::vector result; - converter.iterateJarPaths([&](auto jar) {result.push_back(jar);}); - ASSERT_EQ(result, option_value.second); -} - -const std::vector>> jar_strings = - { - std::make_pair("test.jar:test2.jar", std::vector({"test.jar", "test2.jar"})), //basic splitting - std::make_pair("test.jar:test.jar", std::vector({"test.jar"})), //filter duplicates - std::make_pair("testDEF.jar:testABC.jar", std::vector({"testABC.jar", "testDEF.jar"})), //alphabetical order - }; - -INSTANTIATE_TEST_SUITE_P( - Converter, - LegacyConverterJarTest, - ::testing::ValuesIn(jar_strings) -); - - - -class ConverterV2JarTest : public ::testing::TestWithParam>> {}; - -TEST_P(ConverterV2JarTest, jar) { - const std::pair> option_value = GetParam(); - const std::string jar_option_value = option_value.first; - - ConverterV2 converter; - converter.convertExternalJar(option_value.first); - std::vector result; - converter.iterateJarPaths([&](auto jar) {result.push_back(jar);}); - ASSERT_EQ(result, option_value.second); -} - -const std::vector>> jar_strings_v2 = - { - std::make_pair("test.jar:test2.jar", std::vector({"test.jar", "test2.jar"})), //basic splitting - std::make_pair("test.jar:test.jar", std::vector({"test.jar", "test.jar"})), //keep duplicates - std::make_pair("testDEF.jar:testABC.jar", std::vector({"testDEF.jar", "testABC.jar"})), //maintain order - }; - -INSTANTIATE_TEST_SUITE_P( - Converter, - ConverterV2JarTest, - ::testing::ValuesIn(jar_strings_v2) -); - -class ConverterV2JvmOptionsTest : public ::testing::TestWithParam>> {}; - -TEST_P(ConverterV2JvmOptionsTest, jvm_option) { - const std::pair> option_value = GetParam(); - const std::string jvm_option_value = option_value.first; - - ConverterV2 converter; - converter.convertJvmOption(option_value.first); - std::vector result = std::move(converter.moveJvmOptions()); - ASSERT_EQ(result, option_value.second); -} - -const std::vector>> jvm_options_strings_v2 = - { - std::make_pair("optionA=abc optionB=def", std::vector({"optionA=abc", "optionB=def"})), - std::make_pair("optionA=abc\\ def optionB=ghi", std::vector({"optionA=abc def", "optionB=ghi"})), - std::make_pair("optionA=abc\\tdef optionB=ghi", std::vector({"optionA=abc\tdef", "optionB=ghi"})), - std::make_pair(" optionA=abc\\tdef optionB=ghi", std::vector({"optionA=abc\tdef", "optionB=ghi"})), - std::make_pair(" optionA=abc\\tdef\\\\\t\t optionB=ghi", std::vector({"optionA=abc\tdef\\", "optionB=ghi"})), - std::make_pair(" optionA=abc\\tdef\\\\\\t\\t optionB=ghi", std::vector({"optionA=abc\tdef\\\t\t", "optionB=ghi"})), - std::make_pair(" optionA=abc\\tdef\\\\\\t\\t optionB=ghi ", std::vector({"optionA=abc\tdef\\\t\t", "optionB=ghi"})), - std::make_pair(" optionA=abc\\tdef\\\\\\t\\t optionB=ghi\\ \\t ", std::vector({"optionA=abc\tdef\\\t\t", "optionB=ghi \t"})) - }; - -INSTANTIATE_TEST_SUITE_P( - Converter, - ConverterV2JvmOptionsTest, - ::testing::ValuesIn(jvm_options_strings_v2) -); diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/test/ctpg_script_importer_test.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/test/ctpg_script_importer_test.cc deleted file mode 100644 index 1efbadc..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/test/ctpg_script_importer_test.cc +++ /dev/null @@ -1,66 +0,0 @@ - -#include "include/gtest/gtest.h" -#include "gmock/gmock.h" -#include "javacontainer/script_options/parser_ctpg.h" - -#include "javacontainer/script_options/test/swig_factory_test.h" -#include - -using namespace SWIGVMContainers::JavaScriptOptions; - - -static const char* sc_scriptName = "script"; - - -void checkIndex(size_t currentIdx, const char* scriptKey) { - std::stringstream ss; - ss << sc_scriptName << currentIdx; - if (ss.str() != scriptKey) { - throw std::logic_error(std::string("Script Key does not match: '") + ss.str() + " != '" + scriptKey + "'"); - } -} - -const char* buildNewScriptCode(size_t currentIdx) { - std::stringstream ss; - ss << "%import " << sc_scriptName << currentIdx << ";something"; - static std::string ret; - ret = ss.str(); - return ret.c_str(); -} - -TEST(ScriptImporterTest, max_recursion_depth) { - /** - This test checks that running an infinite recursion of the script import will result in the expected exception. - For that, the test creates new "import scripts" on the fly: - Whenever the parser finds a new 'import script' option, - it calls SWIGVMContainers::SWIGMetadataIf::moduleContent(). - The mocked implementation redirects this request to `buildNewScriptCode()` which creates a - new dummy import script with another '%import ...` option. - */ - - size_t currentIdx = 0; - std::unique_ptr swigFactory = - std::make_unique([&](const char* scriptKey) { - checkIndex(currentIdx, scriptKey); - return buildNewScriptCode(++currentIdx); - }); - ScriptOptionLinesParserCTPG parser(std::move(swigFactory)); - - const std::string code = buildNewScriptCode(currentIdx); - parser.prepareScriptCode(code); - EXPECT_THROW({ - try - { - parser.extractImportScripts(); - } - catch( const std::runtime_error& e ) - { - //We need to deceive "find_duplicate_error_codes.sh" here - const std::string expectedError = - std::string("F-UDF-CL-SL-JAVA-") + "1633: Maximal recursion depth for importing scripts reached."; - EXPECT_STREQ( expectedError.c_str(), e.what()); - throw; - } - }, std::runtime_error ); -} - diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/test/string_ops_tests.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/test/string_ops_tests.cc deleted file mode 100644 index 9615212..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/test/string_ops_tests.cc +++ /dev/null @@ -1,59 +0,0 @@ - -#include "include/gtest/gtest.h" -#include "gmock/gmock.h" -#include "javacontainer/script_options/string_ops.h" - -using namespace SWIGVMContainers::JavaScriptOptions; - - - - -TEST(StringOpsTest, trim) { - std::string sample = " \tHello World \t"; - StringOps::trim(sample); - EXPECT_EQ(sample, "Hello World"); -} - -TEST(StringOpsTest, trimWithNoneASCII) { - /* - Test that trim works correctly with None-ASCII characters - \xa0's bit sequence is '1010 0000', while space bit sequence '0010 0000'. - If StringOps::trim() would not work correctly with characters where MSB is set, it would interpret \xa0 as space. - */ - std::string sample = " \t\xa0Hello World\xa0 \t"; - StringOps::trim(sample); - EXPECT_EQ(sample, "\xa0Hello World\xa0"); -} - -class ReplaceTrailingEscapeWhitespacesTest : public ::testing::TestWithParam> {}; - -TEST_P(ReplaceTrailingEscapeWhitespacesTest, s) { - const std::pair underTest = GetParam(); - - std::string str = underTest.first; - StringOps::replaceTrailingEscapeWhitespaces(str); - ASSERT_EQ(str, underTest.second); -} - -const std::vector> replace_trailing_escape_whitespaces_strings = - { - std::make_pair("hello world", std::string("hello world")), - std::make_pair("hello world ", std::string("hello world")), - std::make_pair("hello world\\t", std::string("hello world\t")), - std::make_pair("hello world\\f", std::string("hello world\f")), - std::make_pair("hello world\\v", std::string("hello world\v")), - std::make_pair("hello world\\\\t", std::string("hello world\\t")), - std::make_pair("hello world\\\\t\t", std::string("hello world\\t")), - std::make_pair("hello world\\\\\\t\t", std::string("hello world\\\t")), - std::make_pair("hello world\\\\\\\\t\t", std::string("hello world\\\\t")), - std::make_pair("hello worl\td\\\\\\\\t\t", std::string("hello worl\td\\\\t")), - std::make_pair("t\t ", std::string("t")), - std::make_pair("hello worl\td\\\\\\\\", std::string("hello worl\td\\\\\\\\")), //If no whitespace escape sequence, backslashes are not interpreted as backslash escape sequence - }; - - -INSTANTIATE_TEST_SUITE_P( - StringOpsTest, - ReplaceTrailingEscapeWhitespacesTest, - ::testing::ValuesIn(replace_trailing_escape_whitespaces_strings) -); \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/test/swig_factory_test.cc b/udf-runner-cpp/v1/base/javacontainer/script_options/test/swig_factory_test.cc deleted file mode 100644 index adca665..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/test/swig_factory_test.cc +++ /dev/null @@ -1,105 +0,0 @@ -#include "javacontainer/script_options/test/swig_factory_test.h" -#include "exaudflib/swig/swig_meta_data.h" - -#include - -class NotImplemented : public std::logic_error -{ -public: - NotImplemented() : std::logic_error("Function not yet implemented") { }; - NotImplemented(const std::string funcName) : std::logic_error("Function " + funcName + " not yet implemented") { }; -}; - -class SWIGMetadataTest : public SWIGVMContainers::SWIGMetadataIf { - -public: - SWIGMetadataTest(std::function callback): m_callback(callback) {} - virtual const char* databaseName() { throw NotImplemented("databaseName"); return nullptr;} - virtual const char* databaseVersion() { throw NotImplemented("databaseVersion"); return nullptr;} - virtual const char* scriptName() { throw NotImplemented("scriptName"); return nullptr;} - virtual const char* scriptSchema() { throw NotImplemented("scriptSchema"); return nullptr;} - virtual const char* currentUser() { throw NotImplemented("currentUser"); return nullptr;} - virtual const char* scopeUser() { throw NotImplemented("scopeUser"); return nullptr;} - virtual const char* currentSchema() { throw NotImplemented("currentSchema"); return nullptr;} - virtual const char* scriptCode() { throw NotImplemented("scriptCode"); return nullptr;} - virtual const unsigned long long sessionID() { throw NotImplemented("sessionID"); return 0;} - virtual const char *sessionID_S() { throw NotImplemented("sessionID_S"); return nullptr;} - virtual const unsigned long statementID() { throw NotImplemented("statementID"); return 0;} - virtual const unsigned int nodeCount() { throw NotImplemented("nodeCount"); return 0;} - virtual const unsigned int nodeID() { throw NotImplemented("nodeID"); return 0;} - virtual const unsigned long long vmID() { throw NotImplemented("vmID"); return 0;} - virtual const unsigned long long memoryLimit() { throw NotImplemented("memoryLimit"); return 0;} - virtual const SWIGVMContainers::VMTYPE vmType() { - throw NotImplemented("vmType"); - return SWIGVMContainers::VM_UNSUPPORTED; - } - virtual const char *vmID_S() { throw NotImplemented("vmID_S"); return nullptr;} - virtual const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name) { - throw NotImplemented("connectionInformation"); return nullptr; - } - virtual const char* moduleContent(const char* name) { - return m_callback(name); - } - virtual const unsigned int inputColumnCount() { throw NotImplemented("inputColumnCount"); return 0;} - virtual const char *inputColumnName(unsigned int col) { - throw NotImplemented("inputColumnName"); - return nullptr; - } - virtual const SWIGVMContainers::SWIGVM_datatype_e inputColumnType(unsigned int col) { - throw NotImplemented("inputColumnType"); - return SWIGVMContainers::UNSUPPORTED; - } - virtual const char *inputColumnTypeName(unsigned int col) { - throw NotImplemented("inputColumnTypeName"); return nullptr; - } - virtual const unsigned int inputColumnSize(unsigned int col) { - throw NotImplemented("inputColumnSize"); return 0; - } - virtual const unsigned int inputColumnPrecision(unsigned int col) { - throw NotImplemented("inputColumnPrecision"); return 0; - } - virtual const unsigned int inputColumnScale(unsigned int col) { - throw NotImplemented("inputColumnScale"); return 0; - } - virtual const SWIGVMContainers::SWIGVM_itertype_e inputType() { - throw NotImplemented("inputType"); - return SWIGVMContainers::EXACTLY_ONCE; - } - virtual const unsigned int outputColumnCount() { throw NotImplemented("outputColumnCount"); return 0;} - virtual const char *outputColumnName(unsigned int col) { throw NotImplemented("outputColumnName"); return nullptr;} - virtual const SWIGVMContainers::SWIGVM_datatype_e outputColumnType(unsigned int col) { - throw NotImplemented("outputColumnType"); - return SWIGVMContainers::UNSUPPORTED; - } - virtual const char *outputColumnTypeName(unsigned int col) { - throw NotImplemented("outputColumnTypeName"); return nullptr; - } - virtual const unsigned int outputColumnSize(unsigned int col) { - throw NotImplemented("outputColumnSize"); return 0; - } - virtual const unsigned int outputColumnPrecision(unsigned int col) { - throw NotImplemented("outputColumnPrecision"); return 0; - } - virtual const unsigned int outputColumnScale(unsigned int col) { - throw NotImplemented("outputColumnScale"); return 0; - } - virtual const SWIGVMContainers::SWIGVM_itertype_e outputType() { - throw NotImplemented("outputType"); - return SWIGVMContainers::EXACTLY_ONCE; - } - virtual const bool isEmittedColumn(unsigned int col) { throw NotImplemented("isEmittedColumn"); return false;} - virtual const char* checkException() { return nullptr;} - virtual const char* pluginLanguageName() { throw NotImplemented("pluginLanguageName"); return nullptr;} - virtual const char* pluginURI() { throw NotImplemented("pluginURI"); return nullptr;} - virtual const char* outputAddress() { throw NotImplemented("outputAddress"); return nullptr;} - -private: - std::function m_callback; -}; - -SwigFactoryTestImpl::SwigFactoryTestImpl(std::function callback) -: m_callback(callback) {} - -SWIGVMContainers::SWIGMetadataIf* SwigFactoryTestImpl::makeSwigMetadata() { - return new SWIGMetadataTest(m_callback); -} diff --git a/udf-runner-cpp/v1/base/javacontainer/script_options/test/swig_factory_test.h b/udf-runner-cpp/v1/base/javacontainer/script_options/test/swig_factory_test.h deleted file mode 100644 index b98f144..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/script_options/test/swig_factory_test.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef SWIG_FACTORY_TEST_H -#define SWIG_FACTORY_TEST_H 1 - -#include -#include -#include "swig_factory/swig_factory.h" -#include - -struct SwigFactoryTestImpl : public SWIGVMContainers::SwigFactory { - - SwigFactoryTestImpl(std::function callback); - - virtual SWIGVMContainers::SWIGMetadataIf* makeSwigMetadata() override; - -private: - std::function m_callback; -}; - - -#endif //namespace SWIG_FACTORY_TEST_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/test/BUILD b/udf-runner-cpp/v1/base/javacontainer/test/BUILD deleted file mode 100644 index 69ff84f..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -java_library( - name = "ExaStackTraceCleaner", - srcs = ["//javacontainer:ExaStackTraceCleaner.java"] -) - -java_test( - name = "ExaStackTraceCleanerTest", - srcs = ["java/com/exasol/ExaStackTraceCleanerTest.java"], - size = "small", - deps = [":ExaStackTraceCleaner"] -) - -JAVACONTAINER_TEST_SRCS = ["cpp/javacontainer_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", - "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"] - -JAVACONTAINER_PERF_TEST_SRCS = ["cpp/javacontainer_perf_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h", - "cpp/swig_factory_test.h", "cpp/swig_factory_test.cc"] - -cc_test( - name = "javacontainer-test-extractor-legacy", - srcs = JAVACONTAINER_TEST_SRCS, - deps = [ - "//javacontainer:javacontainer", - "@googletest//:gtest_main", - ], - data = ["test.jar", "other_test.jar"] -) - -cc_test( - name = "javacontainer-test-extractor-v2", - srcs = JAVACONTAINER_TEST_SRCS + ["cpp/javacontainer_extractor_v2_test.cc"], - deps = [ - "//javacontainer:javacontainer", - "@googletest//:gtest_main", - ], - defines = ["USE_EXTRACTOR_V2"], - data = ["test.jar", "other_test.jar"] -) - - -cc_test( - name = "javacontainer-perf-test-legacy-parser", - srcs = JAVACONTAINER_PERF_TEST_SRCS, - deps = [ - "//javacontainer:javacontainer", - "@googletest//:gtest_main", - ], - data = ["test.jar", "other_test.jar"] -) - -cc_test( - name = "javacontainer-perf-test-ctpg-parser", - srcs = JAVACONTAINER_PERF_TEST_SRCS, - deps = [ - "//javacontainer:javacontainer", - "@googletest//:gtest_main", - ], - defines = ["USE_EXTRACTOR_V2"], - data = ["test.jar", "other_test.jar"] -) diff --git a/udf-runner-cpp/v1/base/javacontainer/test/cpp/exaudf_wrapper.cc b/udf-runner-cpp/v1/base/javacontainer/test/cpp/exaudf_wrapper.cc deleted file mode 100644 index 274c19b..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/cpp/exaudf_wrapper.cc +++ /dev/null @@ -1,12 +0,0 @@ -#include "exaudflib/swig/swig_common.h" - -//Dummy implementation to calm down the linker -void* load_dynamic(const char* name) { - return nullptr; -} - -//Globalinstance of the SWIGVM_params which we used to communicate with JavaVMImpl; -namespace SWIGVMContainers { -SWIGVM_params_t gSWIGVM_params_t; -__thread SWIGVM_params_t * SWIGVM_params = &gSWIGVM_params_t; -} \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_extractor_v2_test.cc b/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_extractor_v2_test.cc deleted file mode 100644 index 3f5308e..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_extractor_v2_test.cc +++ /dev/null @@ -1,133 +0,0 @@ - -#include "include/gtest/gtest.h" -#include "gmock/gmock.h" -#include "javacontainer/test/cpp/javavm_test.h" -#include "javacontainer/test/cpp/swig_factory_test.h" -#include "javacontainer/javacontainer.h" -#include -#include - -using ::testing::MatchesRegex; - -class JavaContainerEscapeSequenceTest : public ::testing::TestWithParam> {}; - -TEST_P(JavaContainerEscapeSequenceTest, quoted_jvm_option) { -const std::pair option_value = GetParam(); - const std::string script_code = - "%jvmoption " + option_value.first + ";\n\n" - "class JVMOPTION_TEST_WITH_SPACE {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n\n\n" - "class JVMOPTION_TEST_WITH_SPACE {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(expected_script_code, vm.getJavaVMInternalStatus().m_scriptCode); - EXPECT_EQ("/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", vm.getJavaVMInternalStatus().m_exaJarPath); - EXPECT_EQ("/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", vm.getJavaVMInternalStatus().m_classpath); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - - const std::vector expectedJVMOptions = { option_value.second, "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(expectedJVMOptions, vm.getJavaVMInternalStatus().m_jvmOptions); -} - -const std::vector> escape_sequences = - { - std::make_pair("-Dhttp.agent=ABC\\nDEF", "-Dhttp.agent=ABC\nDEF"), - std::make_pair("-Dhttp.agent=ABC\\rDEF", "-Dhttp.agent=ABC\rDEF"), - std::make_pair("-Dhttp.agent=ABC\\;DEF", "-Dhttp.agent=ABC;DEF"), - std::make_pair("-Dhttp.agent=ABC\\aDEF", "-Dhttp.agent=ABC\\aDEF"), //any other escape sequence must stay as is - std::make_pair("\\n-Dhttp.agent=ABCDEF", "\n-Dhttp.agent=ABCDEF"), - std::make_pair("\\r-Dhttp.agent=ABCDEF", "\r-Dhttp.agent=ABCDEF"), - std::make_pair("\\;-Dhttp.agent=ABCDEF", ";-Dhttp.agent=ABCDEF"), - std::make_pair("-Dhttp.agent=ABCDEF\\n", "-Dhttp.agent=ABCDEF\n"), - std::make_pair("-Dhttp.agent=ABCDEF\\r", "-Dhttp.agent=ABCDEF\r"), - std::make_pair("-Dhttp.agent=ABCDEF\\;", "-Dhttp.agent=ABCDEF;"), - std::make_pair("\\ -Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), - std::make_pair("\\t-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), - std::make_pair("\\f-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), - std::make_pair("\\v-Dhttp.agent=ABCDEF", "-Dhttp.agent=ABCDEF"), - std::make_pair("\\v-Dhttp.agent=ABC\\tDEF", "-Dhttp.agent=ABC\tDEF"), - std::make_pair("\\v-Dhttp.agent=ABC\\ DEF", "-Dhttp.agent=ABC DEF"), - std::make_pair("\\v-Dhttp.agent=ABC\\fDEF", "-Dhttp.agent=ABC\fDEF"), - std::make_pair("\\v-Dhttp.agent=ABC\\vDEF", "-Dhttp.agent=ABC\vDEF"), - std::make_pair("\\v-Dhttp.agent=\"ABC\\tDEF\"", "-Dhttp.agent=\"ABC\tDEF\"") - }; - -INSTANTIATE_TEST_SUITE_P( - JavaContainer, - JavaContainerEscapeSequenceTest, - ::testing::ValuesIn(escape_sequences) -); - -TEST(JavaContainer, import_script_with_escaped_options) { - const std::string script_code = - "%import other_script;\n\n" - "%jvmoption -Dsomeoption=\"ABC\";\n\n" - "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" - "%jar javacontainer/test/test.jar;" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - std::unique_ptr swigFactory = std::make_unique(); - - const std::string other_script_code = - "%jvmoption -Dsomeotheroption=\"DE\\nF\";\n\n" - "%jar javacontainer/test/other_test.jar;" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script", other_script_code); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n\n\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/other_test.jar:javacontainer/test/test.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dexasol.scriptclass=com.exasol.udf_profiling.UdfProfiler", - "-Dsomeotheroption=\"DE\nF\"", "-Dsomeoption=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/other_test.jar:javacontainer/test/test.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, basic_jar_with_trailing_escape) { - const std::string script_code = "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" - "%jar javacontainer/test/test.jar\\t\t;"; - EXPECT_THROW({ - try - { - JavaVMTest vm(script_code); - } - catch( const SWIGVMContainers::JavaVMach::exception& e ) - { - EXPECT_THAT( e.what(), MatchesRegex("^.*Java VM cannot find 'javacontainer/test/test\\.jar\t': No such file or directory$")); - throw; - } - }, SWIGVMContainers::JavaVMach::exception ); -} diff --git a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_perf_test.cc b/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_perf_test.cc deleted file mode 100644 index b116241..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_perf_test.cc +++ /dev/null @@ -1,58 +0,0 @@ -#include "include/gtest/gtest.h" -#include "gmock/gmock.h" -#include "javacontainer/test/cpp/javavm_test.h" -#include "javacontainer/test/cpp/swig_factory_test.h" -#include - -const uint32_t NumInlineJavaLines = 500000; -const uint32_t NumInlineJavaWordsPerLine = 100; - - -TEST(JavaContainerPerformance, large_inline_java_udf_test) { - std::string script_code = - "%jvmoption option1=abc;\n" - "%jvmoption option2=def;\n" - "class JVMOPTION_TEST_WITH_SPACE {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n"; - - for (uint32_t idxLine(0); idxLine < NumInlineJavaLines; ++idxLine) { - for (uint32_t idxWord(0); idxWord < NumInlineJavaWordsPerLine; ++idxWord) - script_code.append("somecode "); - script_code.append("\n"); - } - script_code.append(" }\n}\n"); - JavaVMTest vm(script_code); -} - -TEST(JavaContainerPerformance, large_inline_single_line_full_java_udf_test) { - std::string script_code = - "%jvmoption option1=abc;" - "%jvmoption option2=def;" - "class JVMOPTION_TEST_WITH_SPACE {" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {"; - - for (uint32_t idxLine(0); idxLine < NumInlineJavaLines; ++idxLine) { - for (uint32_t idxWord(0); idxWord < NumInlineJavaWordsPerLine; ++idxWord) - script_code.append("somecode "); - - } - script_code.append(" }}"); - JavaVMTest vm(script_code); -} - -TEST(JavaContainerPerformance, large_inline_single_line_slim_java_udf_test) { - std::string script_code = - "%jvmoption option1=abc;" - "%jvmoption option2=def;" - "class JVMOPTION_TEST_WITH_SPACE {" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {"; - - for (uint32_t idxLine(0); idxLine < NumInlineJavaLines / 10; ++idxLine) { - for (uint32_t idxWord(0); idxWord < NumInlineJavaWordsPerLine / 10; ++idxWord) - script_code.append("someco%de ; \\t"); - - } - script_code.append(" }}"); - JavaVMTest vm(script_code); -} - diff --git a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_test.cc b/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_test.cc deleted file mode 100644 index aad470e..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javacontainer_test.cc +++ /dev/null @@ -1,609 +0,0 @@ - -#include "include/gtest/gtest.h" -#include "gmock/gmock.h" -#include "javacontainer/test/cpp/javavm_test.h" -#include "javacontainer/javacontainer.h" -#include "javacontainer/test/cpp/swig_factory_test.h" -#include - -using ::testing::MatchesRegex; - - -TEST(JavaContainer, basic_jar) { - const std::string script_code = "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" - "%jar javacontainer/test/test.jar;"; - JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, "\n"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/test.jar"); - EXPECT_FALSE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dexasol.scriptclass=com.exasol.udf_profiling.UdfProfiler", - "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/test.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, basic_jar_script_class_with_white_spaces) { - const std::string script_code = "%scriptclass com.exasol.udf_profiling.UdfProfiler\t ;\n" - "%jar javacontainer/test/test.jar;"; - JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, "\n"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/test.jar"); - EXPECT_FALSE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dexasol.scriptclass=com.exasol.udf_profiling.UdfProfiler", - "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/test.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - - -TEST(JavaContainer, basic_jar_with_white_spaces) { - const std::string script_code = "%jar javacontainer/test/test.jar \t ;"; - - JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/test.jar"); -} - -TEST(JavaContainer, basic_jars_ordering) { - /* - * This test validates correct behavior of collecting the %jar options in Extractor V1/V2. - * Both jar files referenced in `script_code` do not exist. - * The JavaVM will throw an exception with the first JAR file it gets from Extractor. - * For extractor V1: Our assumption is that the exception will be for `base/javacontainer/test/abc.jar` - * as this is alphabetically the first JAR file. (re-ordering) - * For extractor V2: Our assumption is that the exception will be for `base/javacontainer/test/test1.jar` - * as this is the first JAR file. (no re-ordering) - */ - const std::string script_code = "%jar javacontainer/test/test1.jar:javacontainer/test/abc.jar;"; - -#ifndef USE_EXTRACTOR_V2 - const char* regexExpectedException = "^.*Java VM cannot find 'javacontainer/test/abc\\.jar': No such file or directory$"; -#else - const char* regexExpectedException = "^.*Java VM cannot find 'javacontainer/test/test1\\.jar': No such file or directory$"; -#endif - EXPECT_THROW({ - try - { - JavaVMTest vm(script_code); - } - catch( const SWIGVMContainers::JavaVMach::exception& e ) - { - EXPECT_THAT( e.what(), MatchesRegex(regexExpectedException)); - throw; - } - }, SWIGVMContainers::JavaVMach::exception ); -} - -TEST(JavaContainer, basic_inline) { - const std::string script_code = "import java.time.LocalDateTime;" - "import java.time.ZoneOffset;" - "import java.time.format.DateTimeFormatter;" - "class SIMPLE {" - "static void main(String[] args) {" - "int i = 0;" - "}" - "static int run(ExaMetadata exa, ExaIterator ctx) {" - "return 0;" - "}" - "}"; - JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = std::string("package com.exasol;\r\n") + script_code; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - - - -TEST(JavaContainer, combined_inline_jar) { - const std::string script_code = "import java.time.LocalDateTime;" - "import java.time.ZoneOffset;" - "import java.time.format.DateTimeFormatter;" - "class SIMPLE {" - "static void main(String[] args) {" - "int i = 0;" - "}" - "static int run(ExaMetadata exa, ExaIterator ctx) {" - "return 0;" - "}" - "}"; - const std::string script_code_with_jar = std::string("%jar javacontainer/test/test.jar;") + script_code; - JavaVMTest vm(script_code_with_jar); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = std::string("package com.exasol;\r\n") + script_code; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/test.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/test.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - - -TEST(JavaContainer, quoted_jvm_option) { - const std::string script_code = - "%jvmoption -Dhttp.agent=\"ABC DEF\";\n\n" - "class JVMOPTION_TEST_WITH_SPACE {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n\n\n" - "class JVMOPTION_TEST_WITH_SPACE {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - /* - * Note: The option "DEF" is wrong and causes UDF's to crash! - * The correct option would be '-Dhttp.agent=\"ABC DEF\"' - */ - const std::vector expectedJVMOptions = { "-Dhttp.agent=\"ABC", "DEF\"", "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, simple_import_script) { - const std::string script_code = - "%import other_script;\n\n" - "%jvmoption -Dhttp.agent=\"ABC\";\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_script_code = - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script", other_script_code); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dhttp.agent=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, simple_import_script_with_white_space) { - const std::string script_code = - "%import other_script\t ;\n\n" - "%jvmoption -Dhttp.agent=\"ABC\";\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_script_code = - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script", other_script_code); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dhttp.agent=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, simple_import_script_with_quotes) { - const std::string script_code = - "%import \"other_script\t \";\n\n" - "%jvmoption -Dhttp.agent=\"ABC\";\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_script_code = - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("\"other_script\t \"", other_script_code); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dhttp.agent=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, import_script_with_recursion) { - const std::string script_code = - "%import other_script;\n\n" - "%jvmoption -Dhttp.agent=\"ABC\";\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_script_code = - "%import other_script;\n\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script", other_script_code); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n\n\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dhttp.agent=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, import_script_with_jvmoption) { - const std::string script_code = - "%import other_script;\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_script_code = - "%jvmoption -Dhttp.agent=\"ABC\";\n\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script", other_script_code); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n\n\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dhttp.agent=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, multiple_import_scripts) { - const std::string script_code = - "%import other_script_A;\n\n" - "%import other_script_C;\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_scipt_code_A = - "%import other_script_B;\n\n" - "class OtherClassA {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - const std::string other_scipt_code_B = - "class OtherClassB {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - const std::string other_scipt_code_C = - "%import other_script_B;\n\n" - "class OtherClassC {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script_A", other_scipt_code_A); - swigFactory->addModule("other_script_B", other_scipt_code_B); - swigFactory->addModule("other_script_C", other_scipt_code_C); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n" - "class OtherClassB {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n" - "class OtherClassA {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n\n\n" - "class OtherClassC {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, import_script_with_mixed_options) { - const std::string script_code = - "%import other_script;\n\n" - "%jvmoption -Dsomeoption=\"ABC\";\n\n" - "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" - "%jar javacontainer/test/test.jar;" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_script_code = - "%jvmoption -Dsomeotheroption=\"DEF\";\n\n" - "%jar javacontainer/test/other_test.jar;" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script", other_script_code); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n\n\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/other_test.jar:javacontainer/test/test.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Dexasol.scriptclass=com.exasol.udf_profiling.UdfProfiler", - "-Dsomeotheroption=\"DEF\"", "-Dsomeoption=\"ABC\"", "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar:javacontainer/test/other_test.jar:javacontainer/test/test.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - -TEST(JavaContainer, import_script_script_class_option_ignored) { - const std::string script_code = - "%import other_script;\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_script_code = - "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script", other_script_code); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n" -#ifndef USE_EXTRACTOR_V2 //The parsers behave differently: The legacy parser incorrectly keeps imported scriptclass options - "%scriptclass com.exasol.udf_profiling.UdfProfiler;" -#endif - "\n" - "class OtherClass {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} - - -TEST(JavaContainer, import_scripts_deep_recursion) { - const std::string script_code = - "%import other_script_A;\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - auto swigFactory = std::make_unique(); - - const std::string other_scipt_code_A = - "%import other_script_B;\n\n" - "class OtherClassA {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - const std::string other_scipt_code_B = - "%import other_script_C;\n\n" - "class OtherClassB {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - const std::string other_scipt_code_C = - "%import other_script_D;\n\n" - "class OtherClassC {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - const std::string other_scipt_code_D = - "%import other_script_A;\n\n" - "class OtherClassD {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n"; - swigFactory->addModule("other_script_A", other_scipt_code_A); - swigFactory->addModule("other_script_B", other_scipt_code_B); - swigFactory->addModule("other_script_C", other_scipt_code_C); - swigFactory->addModule("other_script_D", other_scipt_code_D); - JavaVMTest vm(script_code, std::move(swigFactory)); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/external/exaudfclient_base+/javacontainer"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); - const std::string expected_script_code = - "package com.exasol;\r\n\n\n" - "class OtherClassD {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n" - "class OtherClassC {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n" - "class OtherClassB {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n" - "class OtherClassA {\n" - "static void doSomething() {\n\n" - " }\n" - "}\n\n\n" - "class JVMOPTION_TEST {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - "\tctx.emit(\"Success!\");\n" - " }\n}\n"; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); - const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/external/exaudfclient_base+/javacontainer/exaudf_deploy.jar", - "-XX:+UseSerialGC" }; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); -} diff --git a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javavm_test.cc b/udf-runner-cpp/v1/base/javacontainer/test/cpp/javavm_test.cc deleted file mode 100644 index fe73a89..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javavm_test.cc +++ /dev/null @@ -1,37 +0,0 @@ -#include "javacontainer/test/cpp/javavm_test.h" -#include "javacontainer/test/cpp/swig_factory_test.h" -#include "javacontainer/javacontainer_impl.h" -#include "javacontainer/script_options/extractor_impl.h" -#include - -std::unique_ptr makeDefaultSwigFactory() { - return std::make_unique(); -} - -JavaVMTest::JavaVMTest(std::string scriptCode) : javaVMInternalStatus() { - run(scriptCode, std::move(makeDefaultSwigFactory())); -} - -JavaVMTest::JavaVMTest(std::string scriptCode, std::unique_ptr swigFactory) : javaVMInternalStatus() { - run(scriptCode, std::move(swigFactory)); -} - -void JavaVMTest::run(std::string scriptCode, std::unique_ptr swigFactory) { - SWIGVMContainers::SWIGVM_params->script_code = scriptCode.data(); -#ifndef USE_EXTRACTOR_V2 - std::unique_ptr extractor = - std::make_unique(std::move(swigFactory)); -#else - std::unique_ptr extractor = - std::make_unique(std::move(swigFactory)); -#endif - SWIGVMContainers::JavaVMImpl javaVMImpl(false, true, std::move(extractor)); - javaVMInternalStatus.m_exaJavaPath = javaVMImpl.m_exaJavaPath; - javaVMInternalStatus.m_localClasspath = javaVMImpl.m_localClasspath; - javaVMInternalStatus.m_scriptCode = javaVMImpl.m_scriptCode; - javaVMInternalStatus.m_exaJarPath = javaVMImpl.m_exaJarPath; - javaVMInternalStatus.m_classpath = javaVMImpl.m_classpath; - javaVMInternalStatus.m_jvmOptions = javaVMImpl.m_jvmOptions; - javaVMInternalStatus.m_needsCompilation = javaVMImpl.m_needsCompilation; -} - diff --git a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javavm_test.h b/udf-runner-cpp/v1/base/javacontainer/test/cpp/javavm_test.h deleted file mode 100644 index a0e8e0b..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/cpp/javavm_test.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef JAVA_VM_TEST -#define JAVA_VM_TEST - -#include -#include -#include -#include - -struct JavaVMInternalStatus { - std::string m_exaJavaPath; - std::string m_localClasspath; - std::string m_scriptCode; - std::string m_exaJarPath; - std::string m_classpath; - bool m_needsCompilation; - std::vector m_jvmOptions; -}; - -class SwigFactoryTestImpl; - -class JavaVMTest { - public: - JavaVMTest(std::string scriptCode); - - JavaVMTest(std::string scriptCode, std::unique_ptr swigFactory); - - const JavaVMInternalStatus& getJavaVMInternalStatus() {return javaVMInternalStatus;} - - private: - void run(std::string scriptCode, std::unique_ptr swigFactory); - private: - JavaVMInternalStatus javaVMInternalStatus; -}; - -#endif \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/test/cpp/swig_factory_test.cc b/udf-runner-cpp/v1/base/javacontainer/test/cpp/swig_factory_test.cc deleted file mode 100644 index 750789f..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/cpp/swig_factory_test.cc +++ /dev/null @@ -1,128 +0,0 @@ -#include "javacontainer/test/cpp/swig_factory_test.h" -#include "exaudflib/swig/swig_meta_data.h" - -#include - -class NotImplemented : public std::logic_error -{ -public: - NotImplemented() : std::logic_error("Function not yet implemented") { }; - NotImplemented(const std::string funcName) : std::logic_error("Function " + funcName + " not yet implemented") { }; -}; - -class SWIGMetadataTest : public SWIGVMContainers::SWIGMetadataIf { - -public: - SWIGMetadataTest(const std::map & moduleContent, const std::string & exceptionMsg) - : m_exceptionMsg(exceptionMsg) - , m_moduleContent(moduleContent) {} - virtual const char* databaseName() { throw NotImplemented("databaseName"); return nullptr;} - virtual const char* databaseVersion() { throw NotImplemented("databaseVersion"); return nullptr;} - virtual const char* scriptName() { throw NotImplemented("scriptName"); return nullptr;} - virtual const char* scriptSchema() { throw NotImplemented("scriptSchema"); return nullptr;} - virtual const char* currentUser() { throw NotImplemented("currentUser"); return nullptr;} - virtual const char* scopeUser() { throw NotImplemented("scopeUser"); return nullptr;} - virtual const char* currentSchema() { throw NotImplemented("currentSchema"); return nullptr;} - virtual const char* scriptCode() { throw NotImplemented("scriptCode"); return nullptr;} - virtual const unsigned long long sessionID() { throw NotImplemented("sessionID"); return 0;} - virtual const char *sessionID_S() { throw NotImplemented("sessionID_S"); return nullptr;} - virtual const unsigned long statementID() { throw NotImplemented("statementID"); return 0;} - virtual const unsigned int nodeCount() { throw NotImplemented("nodeCount"); return 0;} - virtual const unsigned int nodeID() { throw NotImplemented("nodeID"); return 0;} - virtual const unsigned long long vmID() { throw NotImplemented("vmID"); return 0;} - virtual const unsigned long long memoryLimit() { throw NotImplemented("memoryLimit"); return 0;} - virtual const SWIGVMContainers::VMTYPE vmType() { - throw NotImplemented("vmType"); - return SWIGVMContainers::VM_UNSUPPORTED; - } - virtual const char *vmID_S() { throw NotImplemented("vmID_S"); return nullptr;} - virtual const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name) { - throw NotImplemented("connectionInformation"); return nullptr; - } - virtual const char* moduleContent(const char* name) { - auto it = m_moduleContent.find(std::string(name)); - if (m_moduleContent.end() == it) { - throw std::invalid_argument("Script not found."); - } - return it->second.c_str(); - } - virtual const unsigned int inputColumnCount() { throw NotImplemented("inputColumnCount"); return 0;} - virtual const char *inputColumnName(unsigned int col) { - throw NotImplemented("inputColumnName"); - return nullptr; - } - virtual const SWIGVMContainers::SWIGVM_datatype_e inputColumnType(unsigned int col) { - throw NotImplemented("inputColumnType"); - return SWIGVMContainers::UNSUPPORTED; - } - virtual const char *inputColumnTypeName(unsigned int col) { - throw NotImplemented("inputColumnTypeName"); return nullptr; - } - virtual const unsigned int inputColumnSize(unsigned int col) { - throw NotImplemented("inputColumnSize"); return 0; - } - virtual const unsigned int inputColumnPrecision(unsigned int col) { - throw NotImplemented("inputColumnPrecision"); return 0; - } - virtual const unsigned int inputColumnScale(unsigned int col) { - throw NotImplemented("inputColumnScale"); return 0; - } - virtual const SWIGVMContainers::SWIGVM_itertype_e inputType() { - throw NotImplemented("inputType"); - return SWIGVMContainers::EXACTLY_ONCE; - } - virtual const unsigned int outputColumnCount() { throw NotImplemented("outputColumnCount"); return 0;} - virtual const char *outputColumnName(unsigned int col) { throw NotImplemented("outputColumnName"); return nullptr;} - virtual const SWIGVMContainers::SWIGVM_datatype_e outputColumnType(unsigned int col) { - throw NotImplemented("outputColumnType"); - return SWIGVMContainers::UNSUPPORTED; - } - virtual const char *outputColumnTypeName(unsigned int col) { - throw NotImplemented("outputColumnTypeName"); return nullptr; - } - virtual const unsigned int outputColumnSize(unsigned int col) { - throw NotImplemented("outputColumnSize"); return 0; - } - virtual const unsigned int outputColumnPrecision(unsigned int col) { - throw NotImplemented("outputColumnPrecision"); return 0; - } - virtual const unsigned int outputColumnScale(unsigned int col) { - throw NotImplemented("outputColumnScale"); return 0; - } - virtual const SWIGVMContainers::SWIGVM_itertype_e outputType() { - throw NotImplemented("outputType"); - return SWIGVMContainers::EXACTLY_ONCE; - } - virtual const bool isEmittedColumn(unsigned int col) { throw NotImplemented("isEmittedColumn"); return false;} - virtual const char* checkException() { - if (m_exceptionMsg.empty()) { - return nullptr; - } - return m_exceptionMsg.c_str(); - } - virtual const char* pluginLanguageName() { throw NotImplemented("pluginLanguageName"); return nullptr;} - virtual const char* pluginURI() { throw NotImplemented("pluginURI"); return nullptr;} - virtual const char* outputAddress() { throw NotImplemented("outputAddress"); return nullptr;} - -private: - std::string m_exceptionMsg; - - std::map m_moduleContent; -}; - - - -SwigFactoryTestImpl::SwigFactoryTestImpl() {} - - -void SwigFactoryTestImpl::addModule(const std::string key, const std::string script) { - m_moduleContent.insert(std::make_pair(key, script)); -} - -void SwigFactoryTestImpl::setException(const std::string msg) { - m_exceptionMsg = msg; -} - -SWIGVMContainers::SWIGMetadataIf* SwigFactoryTestImpl::makeSwigMetadata() { - return new SWIGMetadataTest(m_moduleContent, m_exceptionMsg); -} diff --git a/udf-runner-cpp/v1/base/javacontainer/test/cpp/swig_factory_test.h b/udf-runner-cpp/v1/base/javacontainer/test/cpp/swig_factory_test.h deleted file mode 100644 index fedd1a4..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/cpp/swig_factory_test.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef SWIG_FACTORY_TEST_H -#define SWIG_FACTORY_TEST_H 1 - -#include -#include -#include "swig_factory/swig_factory.h" - -struct SwigFactoryTestImpl : public SWIGVMContainers::SwigFactory { - - SwigFactoryTestImpl(); - - void addModule(const std::string key, const std::string script); - - void setException(const std::string msg); - - virtual SWIGVMContainers::SWIGMetadataIf* makeSwigMetadata() override; - -private: - std::map m_moduleContent; - std::string m_exceptionMsg; -}; - - -#endif //namespace SWIG_FACTORY_TEST_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/test/java/com/exasol/ExaStackTraceCleanerTest.java b/udf-runner-cpp/v1/base/javacontainer/test/java/com/exasol/ExaStackTraceCleanerTest.java deleted file mode 100644 index d522aca..0000000 --- a/udf-runner-cpp/v1/base/javacontainer/test/java/com/exasol/ExaStackTraceCleanerTest.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.exasol; - -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import com.exasol.ExaStackTraceCleaner; -import java.lang.Exception; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.containsString; - -//First build some artificial exception chain - -final class CustomTestException extends Exception { - public CustomTestException(final String msg, final Throwable th) { - super(msg, th); - } -} - - -final class CustomTestClass { - public CustomTestClass() {} - - public void throwEx() throws CustomTestException { - try { - try { - throw new RuntimeException("ex1"); - } catch (final RuntimeException ex) { - throw new RuntimeException("ex2", ex); - } - } catch(final RuntimeException ex) { - throw new CustomTestException("ex3", ex); - } - } -} - -class ExaWrapper { - public static void wrap_custom_test_class() throws ClassNotFoundException, NoSuchMethodException, - InstantiationException, IllegalAccessException, - InvocationTargetException { - Class a = Class.forName("com.exasol.CustomTestClass"); - Constructor constructor = a.getConstructor(); - Object obj = constructor.newInstance(); - Method m = a.getMethod("throwEx"); - m.invoke(obj); - } -} - -public class ExaStackTraceCleanerTest { - - - @Test - public void runSimpleTest() { - Throwable th = null; - try { - ExaWrapper.wrap_custom_test_class(); - } catch(final Exception ex) { - th = ex; - } - ExaStackTraceCleaner exaStackTraceCleaner = new ExaStackTraceCleaner(ExaWrapper.class.getName()); - final String result = exaStackTraceCleaner.cleanStackTrace(th); - String[] resultLines = result.split("\n"); - //Check that the - assertTrue(resultLines.length > 4); - assertEquals(resultLines[0], "com.exasol.CustomTestException: ex3"); - assertThat(resultLines[1], containsString("com.exasol.CustomTestClass.throwEx(ExaStackTraceCleanerTest.java")); - assertThat(resultLines[2], containsString("com.exasol.ExaWrapper.wrap_custom_test_class(ExaStackTraceCleanerTest.java:47)")); - assertThat(resultLines[3], containsString("com.exasol.ExaStackTraceCleanerTest.runSimpleTest(ExaStackTraceCleanerTest.java")); - } - -} \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/javacontainer/test/other_test.jar b/udf-runner-cpp/v1/base/javacontainer/test/other_test.jar deleted file mode 100644 index e69de29..0000000 diff --git a/udf-runner-cpp/v1/base/javacontainer/test/test.jar b/udf-runner-cpp/v1/base/javacontainer/test/test.jar deleted file mode 100644 index e69de29..0000000 diff --git a/udf-runner-cpp/v1/base/python/BUILD b/udf-runner-cpp/v1/base/python/BUILD deleted file mode 100644 index 4db4d8b..0000000 --- a/udf-runner-cpp/v1/base/python/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -package(default_visibility = ["//visibility:public"]) -exports_files([ - "exascript_python_preset_core.py", - "exascript_python_wrap.py", - "extend_exascript_python_preset_py.sh", - "pythoncontainer.cc" - ]) - -cc_library( - name = "pythoncontainer_header", - hdrs = ["pythoncontainer.h"] -) \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/python/exascript_python_preset_core.py b/udf-runner-cpp/v1/base/python/exascript_python_preset_core.py deleted file mode 100644 index b2dcc77..0000000 --- a/udf-runner-cpp/v1/base/python/exascript_python_preset_core.py +++ /dev/null @@ -1,185 +0,0 @@ -import sys -unicode = str -decodeUTF8 = lambda x: x -encodeUTF8 = lambda x: x - -from exascript_python import * -import decimal -import datetime -import types - -class ExaUDFError(Exception): - pass - -def clean_stacktrace_line(line): - import re - match = re.match(r"""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line) - if match is not None: - filename=match.group(1) - lineno=match.group(2) - text=match.group(3) - if filename!="" and filename!="": - return "{filename}:{lineno} {text}\n".format( - filename=filename, - lineno=lineno, - text=text) - else: - return "" - elif "Traceback (most recent call last):" in line: - return "" - else: - return line - -def create_clean_stacktrace(etype, evalue, etb): - import traceback as tb - st=tb.format_exception(etype, evalue, etb) - new_st=[clean_stacktrace_line(line) for line in st] - return "".join(new_st) - -def create_exception_with_complete_backtrace(error_code, error_message, exc_info): - exception_type = exc_info[0] - exception_message = exc_info[1] - exception_bt = exc_info[2] - backtrace = create_clean_stacktrace(exception_type,exception_message,exception_bt) - new_exception_message = "%s: %s \n%s" % (error_code, error_message, backtrace) - print(new_exception_message) - return ExaUDFError(new_exception_message) - -class exa: - def __init__(self): - self.__modules = {} - self.__meta = meta = Metadata() - class exameta: pass - mo = exameta() - mo.database_name = meta.databaseName() - mo.database_version = meta.databaseVersion() - mo.script_name = decodeUTF8(meta.scriptName()) - mo.script_schema = decodeUTF8(meta.scriptSchema()) - mo.current_user = decodeUTF8(meta.currentUser()) - mo.scope_user = decodeUTF8(meta.scopeUser()) - mo.current_schema = decodeUTF8(meta.currentSchema()) - mo.scope_user = decodeUTF8(meta.scopeUser()) - mo.script_code = decodeUTF8(meta.scriptCode()) - if type(sys.version_info) == tuple: - mo.script_language = "Python %d.%d.%d" % (sys.version_info[0], sys.version_info[1], sys.version_info[2]) - else: mo.script_language = "Python %d.%d.%d" % (sys.version_info.major, sys.version_info.minor, sys.version_info.micro) - mo.session_id = meta.sessionID_S() - mo.statement_id = meta.statementID() - mo.node_count = meta.nodeCount() - mo.node_id = meta.nodeID() - mo.memory_limit = meta.memoryLimit() - mo.vm_id = meta.vmID_S() - mo.input_column_count = meta.inputColumnCount() - mo.input_column_name = [decodeUTF8(meta.inputColumnName(x)) for x in range(mo.input_column_count)] - if meta.inputType() == EXACTLY_ONCE: mo.input_type = "SCALAR" - else: mo.input_type = "SET" - mo.output_column_count = meta.outputColumnCount() - if meta.outputType() == EXACTLY_ONCE: mo.output_type = "RETURN" - else: mo.output_type = "EMIT" - def ci(x, tbl): - if tbl == 'input': - colname = decodeUTF8(self.__meta.inputColumnName(x)) - coltype = self.__meta.inputColumnType(x) - colprec = self.__meta.inputColumnPrecision(x) - colscale = self.__meta.inputColumnScale(x) - colsize = self.__meta.inputColumnSize(x) - coltn = self.__meta.inputColumnTypeName(x) - elif tbl == 'output': - colname = decodeUTF8(self.__meta.outputColumnName(x)) - coltype = self.__meta.outputColumnType(x) - colprec = self.__meta.outputColumnPrecision(x) - colscale = self.__meta.outputColumnScale(x) - colsize = self.__meta.outputColumnSize(x) - coltn = self.__meta.outputColumnTypeName(x) - class exacolumn: - def __init__(self, cn, ct, st, cp, cs, l): - self.name = cn - self.type = ct - self.sql_type = st - self.precision = cp - self.scale = cs - self.length = l - if coltype == INT32: return exacolumn(colname, int, coltn, colprec, 0, None) - elif coltype == INT64: return exacolumn(colname, int, coltn, colprec, 0, None) - elif coltype == DOUBLE: return exacolumn(colname, float, coltn, None, None, None) - elif coltype == STRING: return exacolumn(colname, unicode, coltn, None, None, colsize) - elif coltype == BOOLEAN: return exacolumn(colname, bool, coltn, None, None, None) - elif coltype == NUMERIC and colscale == 0: return exacolumn(colname, int, coltn, colprec, 0, None) - elif coltype == NUMERIC: return exacolumn(colname, decimal.Decimal, coltn, colprec, colscale, None) - elif coltype == DATE: return exacolumn(colname, datetime.date, coltn, None, None, None) - elif coltype == TIMESTAMP: return exacolumn(colname, datetime.datetime, coltn, None, None, None) - return exacolumn(colname, coltype, coltn, colprec, colscale, colsize) - mo.input_columns = [ci(x, 'input') for x in range(mo.input_column_count)] - mo.output_columns = [ci(x, 'output') for x in range(mo.output_column_count)] - self.meta = mo - - def import_script(self, script): - modobj = None - modname = unicode(script) - code = self.__meta.moduleContent(encodeUTF8(modname)) - msg = self.__meta.checkException() - - if msg: raise ImportError(u"F-UDF-CL-SL-PYTHON-1119: Importing module %s failed: %s" % (modname, msg)) - code = decodeUTF8(code) - if str(code) in self.__modules: - print("%%% found code", modname, repr(code)) - modobj = self.__modules[str(code)] - else: - print("%%% new code", modname, repr(code), code in self.__modules) - modobj = types.ModuleType(modname) - modobj.__file__ = "<%s>" % modname - modobj.__dict__['exa'] = self - self.__modules[str(code)] = modobj - try: - if sys.version_info[0] >= 3: - exec(compile(code, script, 'exec'), modobj.__dict__) - else: - exec(compile(code, script, 'exec')) in modobj.__dict__ - except BaseException as err: - raise create_exception_with_complete_backtrace( - "F-UDF-CL-SL-PYTHON-1120", - "Importing module %s failed"%modname, - sys.exc_info()) - return modobj - - - class ConnectionInformation: - def __init__(self,type,address,user,password): - self.type = type - self.address = address - self.user = user - self.password = password - - def __str__(self): - return "{\"type\":\""+self.type+"\",\"address\":\""+self.address+"\",\"user\":\""+self.user+"\",\"password\":}" - - - def get_connection(self, name): - connection_name = unicode(name) - connectionInfo = self.__meta.connectionInformation(encodeUTF8(connection_name)) - msg = self.__meta.checkException() - if msg: raise ImportError(u"F-UDF-CL-SL-PYTHON-1121: get_connection for connection name %s failed: %s" % (name, msg)) - return exa.ConnectionInformation(decodeUTF8(connectionInfo.copyKind()), decodeUTF8(connectionInfo.copyAddress()), decodeUTF8(connectionInfo.copyUser()), decodeUTF8(connectionInfo.copyPassword())) - - - def redirect_output(self, target = ('192.168.1.1', 5000)): - import socket - class activate_remote_output: - def __init__(self): - self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.s.connect(target) - sys.stdout = sys.stderr = self - def write(self, data): return self.s.sendall(data) - def close(self): self.s.close() - activate_remote_output() - -exa = exa() - -def __pythonvm_wrapped_parse(env): - try: - exec(compile(exa.meta.script_code, exa.meta.script_name, 'exec'), env) - except BaseException as err: - raise create_exception_with_complete_backtrace( - "F-UDF-CL-SL-PYTHON-1122", - "Exception while parsing UDF", - sys.exc_info()) diff --git a/udf-runner-cpp/v1/base/python/exascript_python_wrap.py b/udf-runner-cpp/v1/base/python/exascript_python_wrap.py deleted file mode 100644 index d79fd1b..0000000 --- a/udf-runner-cpp/v1/base/python/exascript_python_wrap.py +++ /dev/null @@ -1,360 +0,0 @@ -import sys -import os -pyextdataframe_pkg = None - -unicode = str -decodeUTF8 = lambda x: x -encodeUTF8 = lambda x: x -long = int - -if 'LIBPYEXADATAFRAME_DIR' in os.environ: - path_to_pyexadataframe=os.environ['LIBPYEXADATAFRAME_DIR'] - #print("sys.path append",path_to_pyexadataframe) - sys.path.append(path_to_pyexadataframe) -else: - #Path "external/exaudfclient_base+/" comes from external module "exaudfclient_base" - path_to_pyexadataframe="/exaudf/external/exaudfclient_base+/python/python3" - #print("sys.path append",path_to_pyexadataframe) - sys.path.append(path_to_pyexadataframe) - - - -class exaiter(object): - def __init__(self, meta, inp, out): - self.__meta = meta - self.__inp = inp - self.__out = out - self.__intype = self.__meta.inputType() - incount = self.__meta.inputColumnCount() - data = {} - self.__cache = [None]*incount - self.__finished = False - self.__dataframe_finished = False - def rd(get, null, col, postfun = None): - if postfun == None: - newget = lambda: (get(col), null()) - else: - def newget(): - v = get(col) - n = null() - if n: return (v, True) - return (postfun(v), False) - def resget(): - val = self.__cache[col] - if val == None: - val = self.__cache[col] = newget() - return val - return resget - def convert_date(x): - val = datetime.datetime.strptime(x, "%Y-%m-%d") - return datetime.date(val.year, val.month, val.day) - def convert_timestamp(x): - return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f") - self.__incoltypes = [] - for col in range(self.__meta.inputColumnCount()): - self.__incoltypes.append(self.__meta.inputColumnType(col)) - self.__incolnames = [] - for col in range(self.__meta.inputColumnCount()): - self.__incolnames.append(decodeUTF8(self.__meta.inputColumnName(col))) - for col in range(incount): - colname = self.__incolnames[col] - if self.__incoltypes[col] == DOUBLE: - data[colname] = rd(inp.getDouble, inp.wasNull, col) - elif self.__incoltypes[col] == STRING: - data[colname] = rd(inp.getString, inp.wasNull, col, lambda x: decodeUTF8(x)) - elif self.__incoltypes[col] == INT32: - data[colname] = rd(inp.getInt32, inp.wasNull, col) - elif self.__incoltypes[col] == INT64: - data[colname] = rd(inp.getInt64, inp.wasNull, col) - elif self.__incoltypes[col] == NUMERIC: - if self.__meta.inputColumnScale(col) == 0: - data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: int(str(x))) - else: data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: decimal.Decimal(str(x))) - elif self.__incoltypes[col] == DATE: - data[colname] = rd(inp.getDate, inp.wasNull, col, convert_date) - elif self.__incoltypes[col] == TIMESTAMP: - data[colname] = rd(inp.getTimestamp, inp.wasNull, col, convert_timestamp) - elif self.__incoltypes[col] == BOOLEAN: - data[colname] = rd(inp.getBoolean, inp.wasNull, col) - data[col] = data[colname] - self.__outcoltypes = [] - for col in range(self.__meta.outputColumnCount()): - self.__outcoltypes.append(self.__meta.outputColumnType(col)) - self.__data = data - def __getitem__(self, key): - if self.__finished: - raise RuntimeError("E-UDF-CL-SL-PYTHON-1081: Iteration finished") - if key not in self.__data: - key = unicode(key) - if key not in self.__data: - raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1082: Column with name '%s' does not exist" % key) - ret, null = self.__data[key]() - msg = self.__inp.checkException() - if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1083: "+msg) - if null: return None - return ret - def __getattr__(self, key): - if self.__finished: - raise RuntimeError("E-UDF-CL-SL-PYTHON-1084: Iteration finished") - if key not in self.__data: - key = unicode(key) - if key not in self.__data: - raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1085: Iterator has no object with name '%s'" % key) - ret, null = self.__data[key]() - msg = self.__inp.checkException() - if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1086: "+msg) - if null: return None - return ret - def emit(self, *output): - k = 0 - type_names = { - DOUBLE: "float", - BOOLEAN: "bool", - INT32: "int", - INT64: "long", - STRING: "unicode", - NUMERIC: "decimal.Decimal", - DATE: "datetime.date", - TIMESTAMP: "datetime.datetime" } - if len(output) == 1 and output[0].__class__.__name__ == 'DataFrame': - global pyextdataframe_pkg - if pyextdataframe_pkg is None: - import pyextdataframe - pyextdataframe_pkg = pyextdataframe - - v = output[0] - if v.shape[0] == 0: - raise RuntimeError("E-UDF-CL-SL-PYTHON-1087: emit DataFrame is empty") - if v.shape[1] != len(self.__outcoltypes): - exp_num_out = len(self.__outcoltypes) - raise TypeError("E-UDF-CL-SL-PYTHON-1088: emit() takes exactly %d argument%s (%d given)" % (exp_num_out, 's' if exp_num_out > 1 else '', v.shape[1])) - pyextdataframe_pkg.emit_dataframe(self, v) - return - if len(output) != len(self.__outcoltypes): - if len(self.__outcoltypes) > 1: - raise TypeError("E-UDF-CL-SL-PYTHON-1089: emit() takes exactly %d arguments (%d given)" % (len(self.__outcoltypes), len(output))) - else: raise TypeError("E-UDF-CL-SL-PYTHON-1090: emit() takes exactly %d argument (%d given)" % (len(self.__outcoltypes), len(output))) - for v in output: - if v == None: self.__out.setNull(k) - elif type(v) in (int, long): - if self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) - elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) - elif self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, str(int(v))) - elif self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) - else: - raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1091: emit column '%s' is of type %s but data given have type %s" \ - % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - elif type(v) == float: - if self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) - elif self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) - elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) - elif self.__outcoltypes[k] == NUMERIC: self.__out.setInt64(k, str(v)) - else: - raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1092: emit column '%s' is of type %s but data given have type %s" \ - % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - elif type(v) == bool: - if self.__outcoltypes[k] != BOOLEAN: - raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1093: emit column '%s' is of type %s but data given have type %s" \ - % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - self.__out.setBoolean(k, bool(v)) - elif type(v) in (str, unicode): - v = encodeUTF8(v) - vl = len(v) - if self.__outcoltypes[k] != STRING: - raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1094: emit column '%s' is of type %s but data given have type %s" \ - % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - self.__out.setString(k, v, vl) - elif type(v) == decimal.Decimal: - if self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, str(v)) - elif self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) - elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) - elif self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) - else: - raise RuntimeError("E-UDF-CL-SL-PYTHON-1095: emit column '%s' is of type %s but data given have type %s" \ - % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - elif type(v) == datetime.date: - if self.__outcoltypes[k] != DATE: - raise RuntimeError("E-UDF-CL-SL-PYTHON-1096: emit column '%s' is of type %s but data given have type %s" \ - % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - self.__out.setDate(k, v.isoformat()) - elif type(v) == datetime.datetime: - if self.__outcoltypes[k] != TIMESTAMP: - raise RuntimeError("E-UDF-CL-SL-PYTHON-1097: emit column '%s' is of type %s but data given have type %s" \ - % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - self.__out.setTimestamp(k, v.isoformat(' ')) - else: raise RuntimeError("E-UDF-CL-SL-PYTHON-1098: data type %s is not supported" % str(type(v))) - msg = self.__out.checkException() - if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1099: "+msg) - k += 1 - ret = self.__out.next() - msg = self.__out.checkException() - if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1100: "+msg) - if ret != True: raise RuntimeError("F-UDF-CL-SL-PYTHON-1101: Internal error on emiting row") - def next(self, reset = False): - self.__cache = [None] * len(self.__cache) - if reset: - self.__inp.reset() - self.__finished = False - val = True - elif self.__finished: return False - else: val = self.__inp.next() - msg = self.__inp.checkException() - if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1102: "+msg) - if not val: - self.__finished = True - return val - def get_dataframe(self, num_rows=1, start_col=0): - global pyextdataframe_pkg - if pyextdataframe_pkg is None: - import pyextdataframe - pyextdataframe_pkg = pyextdataframe - - if not (num_rows == "all" or (type(num_rows) in (int, long) and num_rows > 0)): - raise RuntimeError("E-UDF-CL-SL-PYTHON-1103: get_dataframe() parameter 'num_rows' must be 'all' or an integer > 0") - if (type(start_col) not in (int, long) or start_col < 0): - raise RuntimeError("E-UDF-CL-SL-PYTHON-1104: get_dataframe() parameter 'start_col' must be an integer >= 0") - if (start_col > len(self.__incolnames)): - raise RuntimeError("E-UDF-CL-SL-PYTHON-1105: get_dataframe() parameter 'start_col' is %d, but there are only %d input columns" % (start_col, len(self.__incolnames))) - if num_rows == "all": - num_rows = sys.maxsize - if self.__dataframe_finished: - # Exception after None already returned - raise RuntimeError("E-UDF-CL-SL-PYTHON-1106: Iteration finished") - elif self.__finished: - # Return None the first time there is no data - self.__dataframe_finished = True - return None - self.__cache = [None] * len(self.__cache) - df = pyextdataframe_pkg.get_dataframe(self, num_rows, start_col) - return df - def reset(self): - self.__dataframe_finished = False - return self.next(reset = True) - def size(self): - return self.__inp.rowsInGroup() - -def __disallowed_function(*args, **kw): - raise RuntimeError("F-UDF-CL-SL-PYTHON-1107: next(), reset() and emit() functions are not allowed in scalar context") - -def __pythonvm_wrapped_cleanup(): - cleanupfunc = None - try: cleanupfunc = globals()['cleanup'] - except: raise RuntimeError("F-UDF-CL-SL-PYTHON-1108: function 'cleanup' is not defined") - try: - cleanupfunc() - except BaseException as err: - raise create_exception_with_complete_backtrace( - "F-UDF-CL-SL-PYTHON-1109", - "Exception during cleanup", - sys.exc_info()) - -def __pythonvm_wrapped_run(): - runfunc = None - try: runfunc = globals()['run'] - except: raise RuntimeError("F-UDF-CL-SL-PYTHON-1110: function 'run' is not defined") - inp = TableIterator(); msg = inp.checkException(); - if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1111: "+msg) - out = ResultHandler(inp); msg = out.checkException(); - if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1112: "+msg) - meta = Metadata(); msg = meta.checkException(); - if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1113: "+msg) - try: - iter = exaiter(meta, inp, out); iter_next = iter.next; iter_emit = iter.emit - if meta.outputType() == EXACTLY_ONCE: - iter.emit = __disallowed_function - if meta.inputType() == EXACTLY_ONCE: - iter.next = iter.reset = __disallowed_function - if meta.inputType() == MULTIPLE: - if meta.outputType() == EXACTLY_ONCE: iter_emit(runfunc(iter)) - else: runfunc(iter) - else: - if meta.outputType() == EXACTLY_ONCE: - while(True): - iter_emit(runfunc(iter)) - if not iter_next(): break - else: - while(True): - runfunc(iter) - if not iter_next(): break - out.flush() - except BaseException as err: - raise create_exception_with_complete_backtrace( - "F-UDF-CL-SL-PYTHON-1114", - "Exception during run", - sys.exc_info()) - - -class __ImportSpecification: - def __init__(self, d): - self.d = d - - def __getattr__(self, key): - return self.d[key] - - def _setConnectionInformation(self,value): - self.d["connection"] = value - - def __getitem__(self, key): - return self.d[key] - - -class __ExportSpecification: - def __init__(self, d): - self.d = d - - def __getattr__(self, key): - return self.d[key] - - def _setConnectionInformation(self,value): - self.d["connection"] = value - - def __getitem__(self, key): - return self.d[key] - - -class __ConnectionInformation: - def __init__(self, d): - self.d = d - - def __getattr__(self, key): - return self.d[key] - - def __getitem__(self, key): - return self.d[key] - - -def __pythonvm_wrapped_singleCall(fn,arg=None): - if arg: - if "generate_sql_for_import_spec" in globals() and fn == generate_sql_for_import_spec: - imp_spec = __ImportSpecification(arg) - if imp_spec.connection: - imp_spec._setConnectionInformation(__ConnectionInformation(imp_spec.connection)) - try: - return fn(imp_spec) - except BaseException as err: - raise create_exception_with_complete_backtrace( - "F-UDF-CL-SL-PYTHON-1115", - "Exception during singleCall %s"%fn.__name__, - sys.exc_info()) - elif "generate_sql_for_export_spec" in globals() and fn == generate_sql_for_export_spec: - exp_spec = __ExportSpecification(arg) - if exp_spec.connection: - exp_spec._setConnectionInformation(__ConnectionInformation(exp_spec.connection)) - try: - return fn(exp_spec) - except BaseException as err: - raise create_exception_with_complete_backtrace( - "F-UDF-CL-SL-PYTHON-1116", - "Exception during singleCall %s"%fn.__name__, - sys.exc_info()) - else: - raise RuntimeError("F-UDF-CL-SL-PYTHON-1117: Unknown single call function: "+str(fn)) - else: - try: - return fn() - except BaseException as err: - raise create_exception_with_complete_backtrace( - "F-UDF-CL-SL-PYTHON-1118", - "Exception during singleCall %s"%fn.__name__, - sys.exc_info()) - diff --git a/udf-runner-cpp/v1/base/python/extend_exascript_python_preset_py.sh b/udf-runner-cpp/v1/base/python/extend_exascript_python_preset_py.sh deleted file mode 100644 index 0b8e8a8..0000000 --- a/udf-runner-cpp/v1/base/python/extend_exascript_python_preset_py.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e -INPUT=$1 -OUTPUT=$2 -PYTHON_PREFIX=$3 -PYTHON_VERSION=$4 -PYTHON_SYSPATH=$5 -echo "import sys, os" > extension -CURRENT_SYSPATH=$("$PYTHON_PREFIX/bin/$PYTHON_VERSION" -c 'import sys; import site; print(sys.path)') -echo "PYTHON_CURRENT_SYSPATH=$CURRENT_SYSPATH" - -echo "sys.path.extend($CURRENT_SYSPATH)" >> extension -if [ ! "X$PYTHON_SYSPATH" = "X" ]; then - echo "PYTHON_SYSPATH=$PYTHON_SYSPATH" - echo "sys.path.extend($PYTHON_SYSPATH)" >> extension -fi -cat extension "$INPUT" >> "$OUTPUT" diff --git a/udf-runner-cpp/v1/base/python/python3/BUILD b/udf-runner-cpp/v1/base/python/python3/BUILD deleted file mode 100644 index 99b90d6..0000000 --- a/udf-runner-cpp/v1/base/python/python3/BUILD +++ /dev/null @@ -1,118 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -genrule( - name = "exascript_python_tmp_cc", - cmd = """ - INCLUDES=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION-config --includes` - mkdir -p build_exascript_python_tmp_cc/exaudflib - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib - cd build_exascript_python_tmp_cc - swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python_tmp.cc)" exaudflib/exascript.i - #Run replace_swig_import_helper.py because the generated exascript_python.py is not compatible with Python 3.12 - #TODO: Update Swig to a new version which produces code compatible with Python 3.12 - cd .. - python3 "$(location //:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)" - """, - outs = ["exascript_python_tmp.cc", "exascript_python.py"], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h"], - tools = ["//:replace_swig_import_helper.py"], -) - -genrule( - name = "exascript_python_tmp_h", - cmd = """ - INCLUDES=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION-config --includes` - mkdir build_exascript_python_tmp_h - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_python_tmp_h - cp "$(location exascript_python_tmp.cc)" "$(location exascript_python.py)" build_exascript_python_tmp_h - cd build_exascript_python_tmp_h - swig -v $$INCLUDES -DEXTERNAL_PROCESS -c++ -python -py3 -external-runtime "../$(location exascript_python_tmp.h)" - """, - outs = ["exascript_python_tmp.h"], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h", ":exascript_python_tmp.cc", "exascript_python.py"] -) - -genrule( - name = "extend_exascript_python_preset_py", - cmd = 'bash $(location //python:extend_exascript_python_preset_py.sh) "$(location //python:exascript_python_preset_core.py)" "$(location exascript_python_preset.py)" "$$PYTHON3_PREFIX" "$$PYTHON3_VERSION" ""', - outs = ["exascript_python_preset.py"], - srcs = ["//python:exascript_python_preset_core.py"], - tools = ["//python:extend_exascript_python_preset_py.sh"] -) - -genrule( - name = "exascript_python_int", - cmd = """ - cp $(SRCS) . - python3 $(location //:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" - """, - outs = ["exascript_python_int.h"], - srcs = [":exascript_python_tmp_cc", "//python:exascript_python_wrap.py", ":extend_exascript_python_preset_py"], - tools = ["//:build_integrated.py"] -) - -genrule( - name = "filter_swig_code_exascript_python_h", - cmd = """ - ACTUAL_PYTHON_VERSION=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION -c 'import sys; print(".".join(map(str, sys.version_info[:3])))'` - if [[ $$ACTUAL_PYTHON_VERSION == 2* ]] ; then - python3 $(location //:filter_swig_code.py) "$@" "$<" - else - cp "$<" "$@" - fi - """, - outs = ["exascript_python.h"], - srcs = [":exascript_python_tmp_h"], - tools = ["//:filter_swig_code.py"] -) - -genrule( - name = "filter_swig_code_exascript_python_cc", - cmd = """ - ACTUAL_PYTHON_VERSION=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION -c 'import sys; print(".".join(map(str, sys.version_info[:3])))'` - cp $(locations exascript_python_tmp_cc) . - if [[ $$ACTUAL_PYTHON_VERSION == 2* ]] ; then - python $(location //:filter_swig_code.py) "$@" exascript_python_tmp.cc - else - cp exascript_python_tmp.cc "$@" - fi - """, - outs = ["exascript_python.cc"], - srcs = [":exascript_python_tmp_cc"], - tools = ["//:filter_swig_code.py"] -) - -cc_library( - name = "exascript_python", - srcs = [":filter_swig_code_exascript_python_cc",":filter_swig_code_exascript_python_h"], - hdrs = [":filter_swig_code_exascript_python_h"], - deps = ["@python3//:python3","//exaudflib:exaudflib-deps","//exaudflib:header"], - alwayslink=True, -) - -cc_library( - name = "pythoncontainer", - srcs = ["//python:pythoncontainer.cc", ":dummy.h"], - data = [":extend_exascript_python_preset_py"], - hdrs = [":exascript_python_int", ":filter_swig_code_exascript_python_h"], - include_prefix = ".", - deps = ["@python3//:python3",":exascript_python","//exaudflib:header", - "//utils:utils", "//python:pythoncontainer_header"], - alwayslink=True, -) - -#workaround to build pyextdataframe.so together with pythoncontainer c++ library -genrule( - name = "dummy", - cmd = 'touch "$@"', - outs = ["dummy.h"], - srcs = [":pyextdataframe.so"], -) - -cc_binary( - name = "pyextdataframe.so", - linkshared = 1, - srcs = [":python_ext_dataframe.cc"], - deps = ["@python3//:python3","@numpy//:numpy", - "//exaudflib:header", "//utils:utils"], -) diff --git a/udf-runner-cpp/v1/base/python/python3/python_ext_dataframe.cc b/udf-runner-cpp/v1/base/python/python3/python_ext_dataframe.cc deleted file mode 100644 index 106fc0a..0000000 --- a/udf-runner-cpp/v1/base/python/python3/python_ext_dataframe.cc +++ /dev/null @@ -1,1557 +0,0 @@ -#include -#include "exaudflib/swig/swig_common.h" -#include "utils/debug_message.h" - -#include - -#define NPY_NO_DEPRECATED_API NPY_API_VERSION -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" { - -#define PY_INT (NPY_USERDEF+1) -#define PY_DECIMAL (NPY_USERDEF+2) -#define PY_STR (NPY_USERDEF+3) -#define PY_DATE (NPY_USERDEF+4) -#define PY_NONETYPE (NPY_USERDEF+5) -#define PY_BOOL (NPY_USERDEF+6) -#define PY_FLOAT (NPY_USERDEF+7) -#define PY_TIMESTAMP (NPY_USERDEF+8) - -std::map pandasDTypeStrToNumpyCTypeMap { - - {"int", NPY_INT32}, - {"intc", NPY_INT32}, - {"intp", NPY_INT64}, - {"int8", NPY_INT8}, - {"int16", NPY_INT16}, - {"int32", NPY_INT32}, - {"int64", NPY_INT64}, - {"int8[pyarrow]", NPY_OBJECT}, - {"int16[pyarrow]", NPY_OBJECT}, - {"int32[pyarrow]", NPY_OBJECT}, - {"int64[pyarrow]", NPY_OBJECT}, - - {"uint8", NPY_UINT8}, - {"uint16", NPY_UINT16}, - {"uint32", NPY_UINT32}, - {"uint64", NPY_UINT64}, - {"uint8[pyarrow]", NPY_OBJECT}, - {"uint16[pyarrow]", NPY_OBJECT}, - {"uint32[pyarrow]", NPY_OBJECT}, - {"uint64[pyarrow]", NPY_OBJECT}, - - {"float32", NPY_FLOAT32}, - {"float64", NPY_FLOAT64}, - {"float", NPY_FLOAT32}, - {"double", NPY_FLOAT64}, - {"float32[pyarrow]", NPY_OBJECT}, - {"float64[pyarrow]", NPY_OBJECT}, - {"float[pyarrow]", NPY_OBJECT}, - {"double[pyarrow]", NPY_OBJECT}, - // We let numpy convert float16 to float (32 bit) and then use the C conversion from float to double, because a proper conversion from float16 to double in C is very complicated. - {"float16", NPY_FLOAT32}, - {"halffloat", NPY_FLOAT32}, - {"float16[pyarrow]", NPY_OBJECT}, - {"halffloat[pyarrow]", NPY_OBJECT}, - - {"string[pyarrow]", NPY_OBJECT}, - {"string[python]", NPY_OBJECT}, - {"string", NPY_OBJECT}, - - {"bool[pyarrow]", NPY_OBJECT}, - {"boolean", NPY_OBJECT}, - {"bool", NPY_BOOL}, - - {"datetime64[ns]", NPY_DATETIME}, - {"timestamp[ns, tz=UTC][pyarrow]", NPY_OBJECT}, - - {"object", NPY_OBJECT}, - - {"py_NAType", PY_NONETYPE}, - {"py_NoneType", PY_NONETYPE}, - {"py_bool", PY_BOOL}, - {"py_int", PY_INT}, - {"py_float", PY_FLOAT}, - {"py_decimal.Decimal", PY_DECIMAL}, - {"py_str", PY_STR}, - {"py_datetime.date", PY_DATE}, - {"py_Timestamp", PY_TIMESTAMP} -}; - -std::map numpyCTypeToNumpyDTypeStrMap { - {NPY_BOOL, "bool"}, - {NPY_INT8, "int8"}, - {NPY_INT16, "int16"}, - {NPY_INT32, "int32"}, - {NPY_INT64, "int64"}, - {NPY_UINT8, "uint8"}, - {NPY_UINT16, "uint16"}, - {NPY_UINT32, "uint32"}, - {NPY_UINT64, "uint64"}, - // We don't list NPY_FLOAT16 here, because we let numpy convert float16 to float (32 bit) and then use the C conversion from float to double, because a proper conversion from float16 to double in C is very complicated. - {NPY_FLOAT32, "float32"}, - {NPY_FLOAT64, "float64"}, -}; - -std::map emitTypeMap { - {SWIGVMContainers::UNSUPPORTED, "UNSUPPORTED"}, - {SWIGVMContainers::DOUBLE, "DOUBLE"}, - {SWIGVMContainers::INT32, "INT32"}, - {SWIGVMContainers::INT64, "INT64"}, - {SWIGVMContainers::NUMERIC, "NUMERIC"}, - {SWIGVMContainers::TIMESTAMP, "TIMESTAMP"}, - {SWIGVMContainers::DATE, "DATE"}, - {SWIGVMContainers::STRING, "STRING"}, - {SWIGVMContainers::BOOLEAN, "BOOLEAN"}, - {SWIGVMContainers::INTERVALYM, "INTERVALYM"}, - {SWIGVMContainers::INTERVALDS, "INTERVALDS"}, - {SWIGVMContainers::GEOMETRY, "GEOMETRY"} -}; - - - -inline void checkPyObjectIsNull(const PyObject *obj, const std::string& error_code) { - // Error message set by Python - if (!obj) - // formerly F-UDF-CL-SL-PYTHON-1038, now we use different error codes - // for the calling locations to get better clues what is happing - throw std::runtime_error(error_code+": PyObject is unexpectedly a null pointer"); -} - - -struct PyUniquePtrDeleter { - void operator()(PyObject *obj) const noexcept { - Py_XDECREF(obj); - } -}; - -using PyUniquePtr = std::unique_ptr; - -struct PyPtr { - explicit PyPtr() { - } - explicit PyPtr(PyObject *obj) { - checkPyObjectIsNull(obj,"F-UDF-CL-SL-PYTHON-1130"); - ptr.reset(obj); - } - void reset(PyObject *obj) { - ptr.reset(obj); - } - PyObject *get() const { - return ptr.get(); - } - PyObject *release() { - return ptr.release(); - } - explicit operator bool() const { - return (ptr.get() != nullptr); - } - PyUniquePtr ptr; -}; - -inline void checkPyPtrIsNotNull(const PyPtr& obj) { - // Error message set by Python - if (!obj) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1039"); -} - -inline void checkPyObjIsNotNull(const PyObject *obj) { - // Error message set by Python - if (!obj) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1142"); -} - - - -struct ColumnInfo -{ - ColumnInfo(std::string const& name, long type) { - this->name = name; - this->type = static_cast(type); - } - - std::string name; - SWIGVMContainers::SWIGVM_datatype_e type; -}; - - - -// Global modules -PyPtr datetimeModule(PyImport_ImportModule("datetime")); -PyPtr decimalModule(PyImport_ImportModule("decimal")); -PyPtr pandasModule(PyImport_ImportModule("pandas")); - - - -inline void getColumnInfo(PyObject *ctxIter, PyObject *colNames, long startCol, std::vector& colInfo) -{ - const char *ctxColumnTypeList = "_exaiter__incoltypes"; - - PyPtr pyColTypes(PyObject_GetAttrString(ctxIter, ctxColumnTypeList)); - if (!PyList_Check(pyColTypes.get())) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1040: " << "getColumnInfo: " << ctxColumnTypeList << " is not a list"; - throw std::runtime_error(ss.str().c_str()); - } - - if (!PyList_Check(colNames)) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1041: " << "getColumnInfo: colNames is not a list"; - throw std::runtime_error(ss.str().c_str()); - } - - Py_ssize_t pyNumCols = PyList_Size(pyColTypes.get()); - if (pyNumCols != PyList_Size(colNames)) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1042" << "getColumnInfo: "; - ss << ctxColumnTypeList << " has length " << pyNumCols << ", but "; - ss << "colNames has length " << PyList_Size(colNames); - throw std::runtime_error(ss.str().c_str()); - } - - for (Py_ssize_t i = startCol; i < pyNumCols; i++) { - PyObject *pyColType = PyList_GetItem(pyColTypes.get(), i); - checkPyObjectIsNull(pyColType,"F-UDF-CL-SL-PYTHON-1131"); - int colType = PyLong_AsLong(pyColType); - if (colType < 0 && PyErr_Occurred()) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1043 getColumnInfo(): PyLong_AsLong error"); - - PyObject *pyColName = PyList_GetItem(colNames, i); - checkPyObjectIsNull(pyColName,"F-UDF-CL-SL-PYTHON-1132"); - const char *colName = PyUnicode_AsUTF8(pyColName); - if (!colName) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1044"); - - colInfo.push_back(ColumnInfo(std::string(colName), colType)); - } -} - - -inline PyObject *getDateFromString(PyObject *value) -{ - PyPtr datetime(PyObject_GetAttrString(datetimeModule.get(), "datetime")); - PyPtr pyDatetime(PyObject_CallMethod(datetime.get(), "strptime", "(Os)", value, "%Y-%m-%d")); - PyPtr pyYear(PyObject_GetAttrString(pyDatetime.get(), "year")); - PyPtr pyMonth(PyObject_GetAttrString(pyDatetime.get(), "month")); - PyPtr pyDay(PyObject_GetAttrString(pyDatetime.get(), "day")); - PyPtr pyDate(PyObject_CallMethod(datetimeModule.get(), "date", "(OOO)", pyYear.get(), pyMonth.get(), pyDay.get())); - - return pyDate.release(); -} - -inline PyObject *getDatetimeFromString(PyObject *value) -{ - PyPtr datetime(PyObject_GetAttrString(datetimeModule.get(), "datetime")); - PyPtr pyDatetime(PyObject_CallMethod(datetime.get(), "strptime", "(Os)", value, "%Y-%m-%d %H:%M:%S.%f")); - - return pyDatetime.release(); -} - -inline PyObject *getLongFromString(PyObject *value) -{ - PyPtr pyLong(PyLong_FromUnicodeObject(value, 10)); - - return pyLong.release(); -} - -inline PyObject *getDecimalFromString(PyObject *value) -{ - PyPtr pyDecimal(PyObject_CallMethod(decimalModule.get(), "Decimal", "(O)", value)); - - return pyDecimal.release(); -} - - -PyObject *getColumnData(std::vector& colInfo, PyObject *tableIter, long numRows, long startCol, bool isSetInput, bool& isFinished) -{ - const long numCols = colInfo.size(); - std::vector>> pyColGetMethods; - - for (long i = 0; i < numCols; i++) { - PyPtr pyColNum(PyLong_FromLong(i + startCol)); - std::function postFunction; - - std::string methodName; - switch(colInfo[i].type) { - case SWIGVMContainers::INT32: - methodName = "getInt32"; - break; - case SWIGVMContainers::INT64: - methodName = "getInt64"; - break; - case SWIGVMContainers::DOUBLE: - methodName = "getDouble"; - break; - case SWIGVMContainers::NUMERIC: - postFunction = &getDecimalFromString; - methodName = "getNumeric"; - break; - case SWIGVMContainers::STRING: - methodName = "getString"; - break; - case SWIGVMContainers::BOOLEAN: - methodName = "getBoolean"; - break; - case SWIGVMContainers::DATE: - postFunction = &getDateFromString; - methodName = "getDate"; - break; - case SWIGVMContainers::TIMESTAMP: - postFunction = &getDatetimeFromString; - methodName = "getTimestamp"; - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1045" << "getColumnData(): unexpected type " << colInfo[i].type; - throw std::runtime_error(ss.str().c_str()); - } - } - PyPtr pyMethodName(PyUnicode_FromString(methodName.c_str())); - - Py_ssize_t colNum = PyLong_AsSsize_t(pyColNum.get()); - if (colNum < 0 && PyErr_Occurred()) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1046: getColumnData(): PyLong_AsSsize_t error"); - - pyColGetMethods.push_back(std::make_tuple(colNum, std::move(pyColNum), std::move(pyMethodName), postFunction)); - } - - PyPtr pyWasNullMethodName(PyUnicode_FromString("wasNull")); - PyPtr pyNextMethodName(PyUnicode_FromString("next")); - PyPtr pyCheckExceptionMethodName(PyUnicode_FromString("checkException")); - - PyPtr pyData(PyList_New(0)); - for (long r = 0; r < numRows; r++) { - PyPtr pyRow(PyList_New(numCols)); - - for (long c = 0; c < numCols; c++) { - PyPtr pyVal(PyObject_CallMethodObjArgs(tableIter, std::get<2>(pyColGetMethods[c]).get(), std::get<1>(pyColGetMethods[c]).get(), NULL)); - if (!pyVal) { - PyObject *ptype, *pvalue, *ptraceback; - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1047: getColumnData(): Error fetching value for row " << r << ", column " << c << ": "; - ss << PyUnicode_AsUTF8(pvalue); - throw std::runtime_error(ss.str().c_str()); - } - - PyPtr pyCheckException(PyObject_CallMethodObjArgs(tableIter, pyCheckExceptionMethodName.get(), NULL)); - if (pyCheckException.get() != Py_None) { - const char *exMsg = PyUnicode_AsUTF8(pyCheckException.get()); - if (exMsg) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1048: getColumnData(): get row " << r << ", column " << c << ": " << exMsg; - throw std::runtime_error(ss.str().c_str()); - } - } - - PyPtr pyWasNull(PyObject_CallMethodObjArgs(tableIter, pyWasNullMethodName.get(), NULL)); - int wasNull = PyObject_IsTrue(pyWasNull.get()); - if (wasNull < 0) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1049: getColumnData(): wasNull() PyObject_IsTrue() error"); - - if (wasNull) { - Py_INCREF(Py_None); - pyVal.reset(Py_None); - } - else if (std::get<3>(pyColGetMethods[c])) { - // Call post function - pyVal.reset(std::get<3>(pyColGetMethods[c])(pyVal.get())); - } - - PyList_SET_ITEM(pyRow.get(), std::get<0>(pyColGetMethods[c]) - startCol, pyVal.release()); - } - - int ok = PyList_Append(pyData.get(), pyRow.get()); - if (ok < 0) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1050: getColumnData(): PyList_Append error"); - - if (isSetInput) { - PyPtr pyNext(PyObject_CallMethodObjArgs(tableIter, pyNextMethodName.get(), NULL)); - - PyPtr pyCheckException(PyObject_CallMethodObjArgs(tableIter, pyCheckExceptionMethodName.get(), NULL)); - if (pyCheckException.get() != Py_None) { - const char *exMsg = PyUnicode_AsUTF8(pyCheckException.get()); - if (exMsg) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1051: getColumnData(): next(): " << exMsg; - throw std::runtime_error(ss.str().c_str()); - } - } - - int next = PyObject_IsTrue(pyNext.get()); - if (next < 0) { - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1052: getColumnData(): next() PyObject_IsTrue() error"); - } - else if (!next) { - isFinished = true; - break; - } - } - } - - return pyData.release(); -} - -inline void getColumnSetMethods(std::vector& colInfo, std::vector>& pyColSetMethods){ - for (unsigned int i = 0; i < colInfo.size(); i++) { - PyPtr pyColNum(PyLong_FromLong(i)); - - std::string methodName; - switch(colInfo[i].type) { - case SWIGVMContainers::INT32: - methodName = "setInt32"; - break; - case SWIGVMContainers::INT64: - methodName = "setInt64"; - break; - case SWIGVMContainers::DOUBLE: - methodName = "setDouble"; - break; - case SWIGVMContainers::NUMERIC: - methodName = "setNumeric"; - break; - case SWIGVMContainers::STRING: - methodName = "setString"; - break; - case SWIGVMContainers::BOOLEAN: - methodName = "setBoolean"; - break; - case SWIGVMContainers::DATE: - methodName = "setDate"; - break; - case SWIGVMContainers::TIMESTAMP: - methodName = "setTimestamp"; - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1053: emit(): unexpected type " << emitTypeMap.at(colInfo[i].type); - throw std::runtime_error(ss.str().c_str()); - } - } - PyPtr pyMethodName(PyUnicode_FromString(methodName.c_str())); - - pyColSetMethods.push_back(std::make_pair(std::move(pyColNum), std::move(pyMethodName))); - } -} - -inline bool endsWith(std::string const &str, std::string const &suffix) { - if (str.length() < suffix.length()) { - return false; - } - return str.rfind(suffix) == str.size() - suffix.size(); -} - -inline bool startsWith(std::string const &str, std::string const &prefix) { - if (str.length() < prefix.length()) { - return false; - } - return str.rfind(prefix, 0) == 0; -} - -inline bool isNumpyDatetime64(const char* typeName){ - return startsWith(std::string(typeName),"datetime64["); -} - -inline bool isArrowDecimal128(const char* typeName){ - // example decimal128(3, 2)[pyarrow] - const std::string typeNameStr(typeName); - return startsWith(typeNameStr, "decimal128(") && endsWith(typeNameStr, "[pyarrow]"); -} - -inline void getColumnTypeInfo(PyObject *numpyTypes, std::vector>& colTypes){ - PyPtr numpyTypeIter(PyObject_GetIter(numpyTypes)); - for (PyPtr numpyType(PyIter_Next(numpyTypeIter.get())); numpyType.get(); numpyType.reset(PyIter_Next(numpyTypeIter.get()))) { - const char *typeName = PyUnicode_AsUTF8(numpyType.get()); - std::map::iterator it = pandasDTypeStrToNumpyCTypeMap.find(typeName); - if (it != pandasDTypeStrToNumpyCTypeMap.end()) { - colTypes.push_back(*it); - } else if(isArrowDecimal128(typeName)){ - colTypes.push_back({typeName, NPY_OBJECT}); - } else if(isNumpyDatetime64(typeName)){ - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1138: emit: unsupported datetime type: " << typeName << - " emit dataframe only supports datetime64[ns] which represents datetime with nanosecond resolution without time zone. " << - "If you need time zone please use an additional result column. " << - "Note: the nanosecond resolution gets truncated by the database to milliseconds"; - throw std::runtime_error(ss.str().c_str()); - } else { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1054: emit: unexpected type: " << typeName; - throw std::runtime_error(ss.str().c_str()); - } - } - -} - -inline void printPyObject(PyObject* obj, const std::string& error_code){ - PyTypeObject* type = obj->ob_type; - const char* p = type->tp_name; - PyObject* objectsRepresentation = PyObject_Repr(obj); - const char* s = PyUnicode_AsUTF8(objectsRepresentation); - DBG_STREAM_MSG(std::cerr, error_code << ": " << std::string(s) << " " << std::string(p)); -} - -inline const PyPtr& getPandasNA(){ - static const PyPtr pdNA(PyObject_GetAttrString(pandasModule.get(), "NA")); - return pdNA; -} - -inline bool isNoneOrNA(PyObject* pyVal){ - const PyPtr& pdNA = getPandasNA(); - return pyVal == Py_None || pyVal == pdNA.get(); -} - -inline void getColumnArrays(PyObject *colArray, int numCols, int numRows, - std::vector>& colTypes, std::vector& columnArrays){ - for (int c = 0; c < numCols; c++) { - PyPtr pyStart(PyLong_FromLong(c)); - PyPtr pyStop(PyLong_FromLong(c + 1)); - PyPtr slice(PySlice_New(pyStart.get(), pyStop.get(), Py_None)); - PyPtr arraySlice(PyObject_GetItem(colArray, slice.get())); - - PyPtr pyZero(PyLong_FromLong(0L)); - PyPtr array(PyObject_GetItem(arraySlice.get(), pyZero.get())); - - - if (colTypes[c].second == NPY_OBJECT) { - // Convert numpy array to python list - PyPtr pyList(PyObject_CallMethod(array.get(), "tolist", NULL)); - if (!PyList_Check(pyList.get())) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1055: emit(): column array " << c << " is not a list"; - throw std::runtime_error(ss.str().c_str()); - } - - // Get type of first non-None, non-NA item in list - PyObject *pyVal = PyList_GetItem(pyList.get(), 0); - checkPyObjectIsNull(pyVal,"F-UDF-CL-SL-PYTHON-1126"); - std::string pyTypeName(std::string("py_") + Py_TYPE(pyVal)->tp_name); - bool pyValIsNoneOrNA = isNoneOrNA(pyVal); - for (int r = 1; r < numRows && pyValIsNoneOrNA; r++) { - pyVal = PyList_GetItem(pyList.get(), r); - pyValIsNoneOrNA = isNoneOrNA(pyVal); - checkPyObjectIsNull(pyVal,"F-UDF-CL-SL-PYTHON-1127"); - if (!pyValIsNoneOrNA) { - pyTypeName = std::string("py_") + Py_TYPE(pyVal)->tp_name; - break; - } - } - - // Update type in column type info - std::map::iterator userDefIt; - userDefIt = pandasDTypeStrToNumpyCTypeMap.find(pyTypeName); - if (userDefIt != pandasDTypeStrToNumpyCTypeMap.end()) { - colTypes[c] = *userDefIt; - } else { - // TODO accept pandas.Timestamp values - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1056: emit: column " << c << ", unexpected python type: " << pyTypeName; - throw std::runtime_error(ss.str().c_str()); - } - - columnArrays.push_back(std::move(pyList)); - } - else if (colTypes[c].second == NPY_DATETIME) { - - - // Convert numpy array to python list - PyPtr pyList(PyObject_CallMethod(array.get(), "tolist", NULL)); - - if (!PyList_Check(pyList.get())) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1057: emit(): column array " << c << " is not a list"; - throw std::runtime_error(ss.str().c_str()); - } - - columnArrays.push_back(std::move(pyList)); - } - else { - PyPtr asType (PyObject_GetAttrString(array.get(), "astype")); - PyPtr keywordArgs(PyDict_New()); - PyDict_SetItemString(keywordArgs.get(), "copy", Py_False); - const std::string numpyDTypeStr = numpyCTypeToNumpyDTypeStrMap.at(colTypes[c].second); - PyPtr funcArgs(Py_BuildValue("(s)", numpyDTypeStr.c_str())); - PyPtr scalarArr(PyObject_Call(asType.get(), funcArgs.get(), keywordArgs.get())); - columnArrays.push_back(std::move(scalarArr)); - } - } - -} - -inline void handleEmitNpyUint64( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - int64_t value = *((int64_t*)PyArray_GETPTR1((PyArrayObject*)(columnArrays[c].get()), r)); - if (npy_isnan(value)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - pyValue.reset(PyLong_FromLong(value)); - break; - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyUnicode_FromString(std::to_string(value).c_str())); - break; - case SWIGVMContainers::DOUBLE: - pyValue.reset(PyFloat_FromDouble(static_cast(value))); - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1058: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - checkPyPtrIsNotNull(pyValue); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitNpyUint32( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - int32_t value = *((int32_t*)PyArray_GETPTR1((PyArrayObject*)(columnArrays[c].get()), r)); - if (npy_isnan(value)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - pyValue.reset(PyLong_FromLong(value)); - break; - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyUnicode_FromString(std::to_string(value).c_str())); - break; - case SWIGVMContainers::DOUBLE: - pyValue.reset(PyFloat_FromDouble(static_cast(value))); - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1059: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - checkPyPtrIsNotNull(pyValue); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitNpyUint16( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - int16_t value = *((int16_t*)PyArray_GETPTR1((PyArrayObject*)(columnArrays[c].get()), r)); - if (npy_isnan(value)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - pyValue.reset(PyLong_FromLong(value)); - break; - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyUnicode_FromString(std::to_string(value).c_str())); - break; - case SWIGVMContainers::DOUBLE: - pyValue.reset(PyFloat_FromDouble(static_cast(value))); - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1060: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - checkPyPtrIsNotNull(pyValue); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitNpyUint8( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - int8_t value = *((int8_t*)PyArray_GETPTR1((PyArrayObject*)(columnArrays[c].get()), r)); - if (npy_isnan(value)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - pyValue.reset(PyLong_FromLong(value)); - break; - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyUnicode_FromString(std::to_string(value).c_str())); - break; - case SWIGVMContainers::DOUBLE: - pyValue.reset(PyFloat_FromDouble(static_cast(value))); - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1061: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - checkPyPtrIsNotNull(pyValue); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitNpyFloat64( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - double value = *((double*)PyArray_GETPTR1((PyArrayObject*)(columnArrays[c].get()), r)); - if (npy_isnan(value)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - pyValue.reset(PyLong_FromLong(static_cast(value))); - break; - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyUnicode_FromString(std::to_string(value).c_str())); - break; - case SWIGVMContainers::DOUBLE: - pyValue.reset(PyFloat_FromDouble(value)); - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1062: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - checkPyPtrIsNotNull(pyValue); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitNpyFloat32( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - double value = *((float*)PyArray_GETPTR1((PyArrayObject*)(columnArrays[c].get()), r)); - if (npy_isnan(value)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - pyValue.reset(PyLong_FromLong(static_cast(value))); - break; - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyUnicode_FromString(std::to_string(value).c_str())); - break; - case SWIGVMContainers::DOUBLE: - pyValue.reset(PyFloat_FromDouble(value)); - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1063: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - checkPyPtrIsNotNull(pyValue); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitNpyBool( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - bool value = *((bool*)PyArray_GETPTR1((PyArrayObject*)(columnArrays[c].get()), r)); - if (npy_isnan(value)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - switch (colInfo[c].type) { - case SWIGVMContainers::BOOLEAN: - if (value) { - Py_INCREF(Py_True); - pyValue.reset(Py_True); - } - else { - Py_INCREF(Py_False); - pyValue.reset(Py_False); - } - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1065: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - checkPyPtrIsNotNull(pyValue); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitPyBool( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - //PyList_GetItem returns a 'borrowed' reference. Must not call XDECREF() on it. - PyObject *pyBool = PyList_GetItem(columnArrays[c].get(), r); - checkPyObjIsNotNull(pyBool); - if (isNoneOrNA(pyBool)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - switch (colInfo[c].type) { - case SWIGVMContainers::BOOLEAN: - if (pyBool == Py_True) { - Py_INCREF(Py_True); - pyValue.reset(Py_True); - } - else if (pyBool == Py_False) { - Py_INCREF(Py_False); - pyValue.reset(Py_False); - } - else { - pyValue.reset(nullptr); - } - break; - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1066: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - checkPyPtrIsNotNull(pyValue); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitPyInt( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - //PyList_GetItem returns a 'borrowed' reference. Must not call XDECREF() on it. - PyObject *pyInt = PyList_GetItem(columnArrays[c].get(), r); - checkPyObjIsNotNull(pyInt); - if (isNoneOrNA(pyInt)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - { - //pyInt points to a 'borrowed' reference. We need to explicitly increase the ref counter here, as pyValue will decrease it again later. - Py_INCREF(pyInt); - pyValue.reset(pyInt); - break; - } - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyInt)); - break; - case SWIGVMContainers::DOUBLE: - { - double value = PyFloat_AsDouble(pyInt); - if (value < 0 && PyErr_Occurred()) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1067: emit() PY_INT: PyFloat_AsDouble error"); - pyValue.reset(PyFloat_FromDouble(value)); - break; - } - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1068: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitPyFloat( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - //PyList_GetItem returns a 'borrowed' reference. Must not call XDECREF() on it. - PyObject *pyFloat = PyList_GetItem(columnArrays[c].get(), r); - checkPyObjIsNotNull(pyFloat); - if (isNoneOrNA(pyFloat)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - { - double value = PyFloat_AsDouble(pyFloat); - if (value < 0 && PyErr_Occurred()) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1139: emit() PY_FLOAT: PyFloat_AsDouble error"); - if (npy_isnan(value)) { - pyResult.reset( - PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - pyValue.reset(PyLong_FromLong(static_cast(value))); - break; - } - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyFloat)); - break; - case SWIGVMContainers::DOUBLE: - { - //pyFloat points to a 'borrowed' reference. We need to explicitly increase the ref counter here, as pyValue will decrease it again later. - Py_INCREF(pyFloat); - pyValue.reset(pyFloat); - break; - } - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1140: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitPyDecimal( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName, - PyPtr& pyIntMethodName, - PyPtr& pyFloatMethodName - ){ - //PyList_GetItem returns a 'borrowed' reference. Must not call XDECREF() on it. - PyObject *pyDecimal = PyList_GetItem(columnArrays[c].get(), r); - if (isNoneOrNA(pyDecimal)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - - switch (colInfo[c].type) { - case SWIGVMContainers::INT64: - case SWIGVMContainers::INT32: - { - PyPtr pyInt(PyObject_CallMethodObjArgs(pyDecimal, pyIntMethodName.get(), NULL)); - pyValue.reset(pyInt.release()); - break; - } - case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyDecimal)); - break; - case SWIGVMContainers::DOUBLE: - { - PyPtr pyFloat(PyObject_CallMethodObjArgs(pyDecimal, pyFloatMethodName.get(), NULL)); - pyValue.reset(pyFloat.release()); - break; - } - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1069: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyValue.get(), NULL)); -} - -inline void handleEmitPyStr( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - //PyList_GetItem returns a 'borrowed' reference. Must not call XDECREF() on it. - PyObject *pyString = PyList_GetItem(columnArrays[c].get(), r); - if (isNoneOrNA(pyString)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - - switch (colInfo[c].type) { - case SWIGVMContainers::NUMERIC: - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyString, NULL)); - break; - case SWIGVMContainers::STRING: - { - Py_ssize_t size = -1; - const char *str = PyUnicode_AsUTF8AndSize(pyString, &size); - if (!str && size < 0) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1137: invalid size of string"); - PyPtr pySize(PyLong_FromSsize_t(size)); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyString, pySize.get(), NULL)); - break; - } - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1070: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } -} - -inline void handleEmitPyDate( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName, - PyPtr& pyIsoformatMethodName){ - //PyList_GetItem returns a 'borrowed' reference. Must not call XDECREF() on it. - PyObject *pyDate = PyList_GetItem(columnArrays[c].get(), r); - if (isNoneOrNA(pyDate)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - - switch (colInfo[c].type) { - case SWIGVMContainers::DATE: - { - PyPtr pyIsoDate(PyObject_CallMethodObjArgs(pyDate, pyIsoformatMethodName.get(), NULL)); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDate.get(), NULL)); - break; - } - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1071: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } -} -inline void handleEmitPyTimestamp( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName){ - //PyList_GetItem returns a 'borrowed' reference. Must not call XDECREF() on it. - PyObject *pyTimestamp = PyList_GetItem(columnArrays[c].get(), r); - if (isNoneOrNA(pyTimestamp)) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - - switch (colInfo[c].type) { - case SWIGVMContainers::TIMESTAMP: - { - // We call here pandas.Timestamp.tz_localize(None), because we need to remove the timezone from the timestamp. - // Exasol doesn't support timezones, and if we don't remove the timezone, pandas.Timestamp.isoformat will add - // it to the generated string. - PyPtr pyTzLocalize(PyObject_CallMethod(pyTimestamp, "tz_localize", "z", NULL)); - PyPtr pyIsoDatetime(PyObject_CallMethod(pyTzLocalize.get(), "isoformat", "s", " ")); - pyResult.reset(PyObject_CallMethodObjArgs( - resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDatetime.get(), NULL)); - break; - } - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1141: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } -} - - -inline void handleEmitNpyDateTime( - int c, int r, - std::vector& columnArrays, - std::vector>& pyColSetMethods, - std::vector& colInfo, - std::vector>& colTypes, - PyObject *resultHandler, - PyPtr& pyValue, - PyPtr& pyResult, - PyPtr& pySetNullMethodName, - PyPtr& pdNaT){ - //PyList_GetItem returns a 'borrowed' reference. Must not call XDECREF() on it. - PyObject *pyTimestamp = PyList_GetItem(columnArrays[c].get(), r); - if (pyTimestamp == pdNaT.get()) { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - return; - } - - switch (colInfo[c].type) { - case SWIGVMContainers::TIMESTAMP: - { - PyPtr pyIsoDatetime(PyObject_CallMethod(pyTimestamp, "isoformat", "s", " ")); - pyResult.reset(PyObject_CallMethodObjArgs( - resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDatetime.get(), NULL)); - break; - } - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1072: emit column " << c << " of type " << emitTypeMap.at(colInfo[c].type) << " but data given have type " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } -} - -void emit(PyObject *resultHandler, std::vector& colInfo, PyObject *dataframe, PyObject *numpyTypes) -{ - std::vector> pyColSetMethods; - try{ - getColumnSetMethods(colInfo, pyColSetMethods); - } catch (std::exception& err){ - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1133: "+std::string(err.what())); - } - - std::vector> colTypes; - try{ - getColumnTypeInfo(numpyTypes, colTypes); - } catch (std::exception& err){ - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1134: "+std::string(err.what())); - } - - int numRows = -1; - int numCols = -1; - - - PyPtr data; - PyPtr arrayPtr; - PyArrayObject *pyArray; - PyPtr colArray; - bool allColsAreDateTime = - std::all_of(colTypes.begin(), colTypes.end(), - [](std::pair colType) { - return colType.second == NPY_DATETIME; - }); - if(allColsAreDateTime) { - // if we get an dataframe with only datetime columns with type datetime[ns], - // As we call PyArray_FROM_OTF(data.get(), NPY_OBJECT, NPY_ARRAY_IN_ARRAY) with parameter NPY_OBJECT we need - // to explicitly cast the values to type 'object'. Per default the dtypes of values are datetime[ns]. - PyPtr asTypeFunc (PyObject_GetAttrString(dataframe, "astype")); - PyPtr keywordArgs(PyDict_New()); - PyDict_SetItemString(keywordArgs.get(), "copy", Py_False); - PyPtr funcArgs(Py_BuildValue("(s)", "object")); - PyPtr castedValues(PyObject_Call(asTypeFunc.get(), funcArgs.get(), keywordArgs.get())); - data.reset(PyObject_GetAttrString(castedValues.get(), "values")); - arrayPtr = PyPtr(PyArray_FROM_OTF(data.get(), NPY_OBJECT, NPY_ARRAY_IN_ARRAY)); - pyArray = reinterpret_cast(arrayPtr.get()); - numRows = PyArray_DIM(pyArray, 0); - numCols = PyArray_DIM(pyArray, 1); - // Transpose to column-major - colArray = PyPtr(PyArray_Transpose(pyArray, NULL)); - }else{ - data=PyPtr(PyObject_GetAttrString(dataframe, "values")); - arrayPtr = PyPtr(PyArray_FROM_OTF(data.get(), NPY_OBJECT, NPY_ARRAY_IN_ARRAY)); - pyArray = reinterpret_cast(arrayPtr.get()); - numRows = PyArray_DIM(pyArray, 0); - numCols = PyArray_DIM(pyArray, 1); - // Transpose to column-major - colArray = PyPtr(PyArray_Transpose(pyArray, NULL)); - } - - // Get column arrays - std::vector columnArrays; - try{ - getColumnArrays(colArray.get(), numCols, numRows, colTypes, columnArrays); - } catch (std::exception& err){ - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1135: "+std::string(err.what())); - } - - try{ - PyPtr pySetNullMethodName(PyUnicode_FromString("setNull")); - PyPtr pyNextMethodName(PyUnicode_FromString("next")); - PyPtr pyCheckExceptionMethodName(PyUnicode_FromString("checkException")); - PyPtr pyIntMethodName(PyUnicode_FromString("__int__")); - PyPtr pyFloatMethodName(PyUnicode_FromString("__float__")); - PyPtr pyIsoformatMethodName(PyUnicode_FromString("isoformat")); - PyPtr pdNaT(PyObject_GetAttrString(pandasModule.get(), "NaT")); - - // Emit data - PyPtr pyValue; - PyPtr pyResult; - for (int r = 0; r < numRows; r++) { - for (int c = 0; c < numCols; c++) { - switch (colTypes[c].second) { - case NPY_INT64: - case NPY_UINT64: - { - handleEmitNpyUint64(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case NPY_INT32: - case NPY_UINT32: - { - handleEmitNpyUint32(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case NPY_INT16: - case NPY_UINT16: - { - handleEmitNpyUint16(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case NPY_INT8: - case NPY_UINT8: - { - handleEmitNpyUint8(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case NPY_FLOAT64: - { - handleEmitNpyFloat64(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case NPY_FLOAT32: - { - handleEmitNpyFloat32(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case NPY_BOOL: - { - handleEmitNpyBool(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case PY_BOOL: - { - handleEmitPyBool(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case PY_INT: - { - handleEmitPyInt(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case PY_FLOAT: - { - handleEmitPyFloat(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case PY_DECIMAL: - { - handleEmitPyDecimal(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, - pySetNullMethodName, pyIntMethodName, pyFloatMethodName); - break; - } - case PY_STR: - { - handleEmitPyStr(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, pySetNullMethodName); - break; - } - case PY_DATE: - { - handleEmitPyDate(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, - pySetNullMethodName, pyIsoformatMethodName); - break; - } - case PY_TIMESTAMP: - { - handleEmitPyTimestamp(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, - pySetNullMethodName); - break; - } - case NPY_DATETIME: - { - handleEmitNpyDateTime(c, r, columnArrays, pyColSetMethods, colInfo, colTypes, resultHandler, pyValue, pyResult, - pySetNullMethodName, pdNaT); - break; - } - case PY_NONETYPE: - { - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pySetNullMethodName.get(), pyColSetMethods[c].first.get(), NULL)); - break; - } - default: - { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1073: emit: unexpected type: " << colTypes[c].first; - throw std::runtime_error(ss.str().c_str()); - } - } - - if (!pyResult) { - PyObject *ptype, *pvalue, *ptraceback; - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - if (pvalue) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1074: emit(): Error setting value for row " << r << ", column " << c << ": "; - ss << PyUnicode_AsUTF8(pvalue); - throw std::runtime_error(ss.str().c_str()); - } - } - - PyPtr pyCheckException(PyObject_CallMethodObjArgs(resultHandler, pyCheckExceptionMethodName.get(), NULL)); - if (pyCheckException.get() != Py_None) { - const char *exMsg = PyUnicode_AsUTF8(pyCheckException.get()); - if (exMsg) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1075: emit(): " << exMsg; - throw std::runtime_error(ss.str().c_str()); - } - } - } - - PyPtr pyNext(PyObject_CallMethodObjArgs(resultHandler, pyNextMethodName.get(), NULL)); - } - }catch (std::exception& err){ - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1136: "+std::string(err.what())); - } -} - -PyObject *createDataFrame(PyObject *data, std::vector& colInfo) -{ - PyPtr pdDataFrame(PyObject_GetAttrString(pandasModule.get(), "DataFrame")); - - Py_ssize_t numCols = static_cast(colInfo.size()); - PyPtr pyColumnNames(PyList_New(numCols)); - for (Py_ssize_t i = 0; i < numCols; i++) { - PyPtr pyColName(PyUnicode_FromString(colInfo[i].name.c_str())); - PyList_SET_ITEM(pyColumnNames.get(), i, pyColName.release()); - } - - PyPtr funcArgs(Py_BuildValue("(O)", data)); - - PyPtr keywordArgs(PyDict_New()); - PyDict_SetItemString(keywordArgs.get(), "columns", pyColumnNames.get()); - - PyPtr pyDataFrame(PyObject_Call(pdDataFrame.get(), funcArgs.get(), keywordArgs.get())); - - return pyDataFrame.release(); -} - -PyObject *getNumpyTypes(PyObject *dataframe) -{ - PyPtr pyDtypes(PyObject_GetAttrString(dataframe, "dtypes")); - PyPtr pyDtypeValues(PyObject_CallMethod(pyDtypes.get(), "tolist", NULL)); - - if (!PyList_Check(pyDtypeValues.get())) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1076: DataFrame.dtypes.values is not a list"; - throw std::runtime_error(ss.str().c_str()); - } - - Py_ssize_t pyNumCols = PyList_Size(pyDtypeValues.get()); - PyPtr pyColumnDtypes(PyList_New(pyNumCols)); - for (Py_ssize_t i = 0; i < pyNumCols; i++) { - PyObject *pyColDtype = PyList_GetItem(pyDtypeValues.get(), i); - checkPyObjectIsNull(pyColDtype,"F-UDF-CL-SL-PYTHON-1128"); - PyPtr pyColDtypeString(PyObject_Str(pyColDtype)); - PyList_SET_ITEM(pyColumnDtypes.get(), i, pyColDtypeString.release()); - } - - return pyColumnDtypes.release(); -} - -void getOutputColumnTypes(PyObject *colTypes, std::vector& colInfo) -{ - if (!PyList_Check(colTypes)) { - std::stringstream ss; - ss << "F-UDF-CL-SL-PYTHON-1077: getOutputColumnTypes(): colTypes is not a list"; - throw std::runtime_error(ss.str().c_str()); - } - - Py_ssize_t pyNumCols = PyList_Size(colTypes); - for (Py_ssize_t i = 0; i < pyNumCols; i++) { - PyObject *pyColType = PyList_GetItem(colTypes, i); - checkPyObjectIsNull(pyColType,"F-UDF-CL-SL-PYTHON-1129"); - int colType = PyLong_AsLong(pyColType); - if (colType < 0 && PyErr_Occurred()) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1078: getColumnInfo(): PyLong_AsLong error"); - - colInfo.push_back(ColumnInfo(std::to_string(i), colType)); - } -} - - -static PyObject *getDataframe(PyObject *self, PyObject *args) -{ - PyObject *ctxIter = NULL; - long numRows = 0; - long startCol = 0; - - if (!PyArg_ParseTuple(args, "Oll", &ctxIter, &numRows, &startCol)) - return NULL; - - PyPtr pyDataFrame; - try { - PyPtr tableIter(PyObject_GetAttrString(ctxIter, "_exaiter__inp")); - PyPtr colNames(PyObject_GetAttrString(ctxIter, "_exaiter__incolnames")); - PyPtr pyInputType(PyObject_GetAttrString(ctxIter, "_exaiter__intype")); - int inputType = PyLong_AsLong(pyInputType.get()); - if (inputType < 0 && PyErr_Occurred()) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1079: getDataframe(): PyLong_AsLong error"); - - // Get script input type - bool isSetInput = (static_cast(inputType) == SWIGVMContainers::MULTIPLE); - // Get input column info - std::vector colInfo; - getColumnInfo(ctxIter, colNames.get(), startCol, colInfo); - // Get input data - if (!isSetInput && numRows > 1) - numRows = 1; - bool isFinished = false; - PyPtr pyData(getColumnData(colInfo, tableIter.get(), numRows, startCol, isSetInput, isFinished)); - if (isFinished) { - int ok = PyObject_SetAttrString(ctxIter, "_exaiter__finished", Py_True); - if (ok < 0) - throw std::runtime_error("F-UDF-CL-SL-PYTHON-1080: getDataframe(): error setting exaiter.__finished"); - } - // Create DataFrame - pyDataFrame.reset(createDataFrame(pyData.get(), colInfo)); - } - catch (std::exception &ex) { - if (ex.what() && strlen(ex.what())) - PyErr_SetString(PyExc_RuntimeError, ex.what()); - return NULL; - } - - return pyDataFrame.release(); -} - -static PyObject *emitDataframe(PyObject *self, PyObject *args) -{ - PyObject *ctxIter = NULL; - PyObject *dataframe = NULL; - - if (!PyArg_ParseTuple(args, "OO", &ctxIter, &dataframe)) - return NULL; - - try { - PyPtr resultHandler(PyObject_GetAttrString(ctxIter, "_exaiter__out")); - PyPtr colTypes(PyObject_GetAttrString(ctxIter, "_exaiter__outcoltypes")); - // Get output column info - std::vector colInfo; - getOutputColumnTypes(colTypes.get(), colInfo); - // Get NumPy types - PyPtr pyNumpyTypes(getNumpyTypes(dataframe)); - // Emit output data - emit(resultHandler.get(), colInfo, dataframe, pyNumpyTypes.get()); - } - catch (std::exception &ex) { - if (ex.what() && strlen(ex.what())) - PyErr_SetString(PyExc_RuntimeError, ex.what()); - return NULL; - } - - Py_RETURN_NONE; -} - -static PyMethodDef dataframeModuleMethods[] = { - {"get_dataframe", getDataframe, METH_VARARGS}, - {"emit_dataframe", emitDataframe, METH_VARARGS}, - {NULL, NULL} -}; - -static struct PyModuleDef dataframeModule = { - PyModuleDef_HEAD_INIT, - "pyextdataframe", /* name of module */ - NULL, - -1, - dataframeModuleMethods -}; - -PyMODINIT_FUNC -PyInit_pyextdataframe(void) -{ - // Call NumPy import_array() for initialization - import_array(); - - return PyModule_Create(&dataframeModule); -} - -} diff --git a/udf-runner-cpp/v1/base/python/pythoncontainer.cc b/udf-runner-cpp/v1/base/python/pythoncontainer.cc deleted file mode 100644 index ed85ba7..0000000 --- a/udf-runner-cpp/v1/base/python/pythoncontainer.cc +++ /dev/null @@ -1,494 +0,0 @@ -#include "pythoncontainer.h" -#include -#ifdef _POSIX_C_SOURCE -#undef _POSIX_C_SOURCE -#endif -#ifdef _XOPEN_SOURCE -#undef _XOPEN_SOURCE -#endif -#include -#include "exascript_python_int.h" -#include "exascript_python.h" -#include "utils/debug_message.h" - -#include "exaudflib/swig/script_data_transfer_objects.h" - -#include -#include - -#define DISABLE_PYTHON_SUBINTERP - -using namespace SWIGVMContainers; -using namespace std; - -extern "C" PyObject* PyInit__exascript_python(void); - - -static void check(const std::string& error_code) { - PyObject *pt, *pv, *tb, *s = NULL, *pvc, *pvcn; - string pvcns(""); - if (PyErr_Occurred() == NULL) return; - PyErr_Fetch(&pt, &pv, &tb); if (pt == NULL) return; - PyErr_NormalizeException(&pt, &pv, &tb); if (pt == NULL) return; - s = PyObject_Str(pv); - - // Get Exception name - if (NULL != (pvc = PyObject_GetAttrString(pv, "__class__"))) { - if (NULL != (pvcn = PyObject_GetAttrString(pvc, "__name__"))) { - PyObject* repr = PyObject_Str(pvcn); - PyObject* p3str = PyUnicode_AsEncodedString(repr, "utf-8", "ignore"); - const char *bytes = PyBytes_AS_STRING(p3str); - pvcns = string(bytes) + string(": "); - Py_XDECREF(pvcn); - } - Py_XDECREF(pvc); - } - - - string exception_string(""); - PyObject* repr = PyObject_Str(s); - PyObject* p3str = PyUnicode_AsEncodedString(repr, "utf-8", "ignore"); - const char *bytes = PyBytes_AS_STRING(p3str); - exception_string = error_code+": "+pvcns + string(bytes); - PythonVM::exception x(exception_string.c_str()); - Py_XDECREF(s); - PyErr_Clear(); - throw x; -} - -class SWIGVMContainers::PythonVMImpl { - public: - PythonVMImpl(bool checkOnly); - ~PythonVMImpl() {} - bool run(); - const char* singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args, string& calledUndefinedSingleCall); - void shutdown(); - private: - string script_code; - bool m_checkOnly; - PyObject *globals, *code, *script; - PyObject *exatable, *runobj, *cleanobj, *clean_wrap_obj; - PyObject *retvalue; -#ifndef DISABLE_PYTHON_SUBINTERP - PyThreadState *pythread; - static PyThreadState *main_thread; -#endif -}; - -#ifndef DISABLE_PYTHON_SUBINTERP -class PythonThreadBlock { - PyGILState_STATE state; - PyThreadState *save; - public: - PythonThreadBlock(): state(PyGILState_Ensure()), save(PyThreadState_Get()) {} - ~PythonThreadBlock() { PyThreadState_Swap(save); PyGILState_Release(state); } -}; -#endif - -PythonVM::PythonVM(bool checkOnly) { - try { - DBG_FUNC_CALL(cerr, m_impl = new PythonVMImpl(checkOnly)); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - DBG_EXCEPTION(cerr, err); - exception_msg = "F-UDF-CL-SL-PYTHON-1000: " + std::string(err.what()); - } catch (...) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-PYTHON-1125: python crashed for unknown reasons"; - } - -} - -void PythonVM::shutdown() { - try { - DBG_FUNC_CALL(cerr, m_impl->shutdown()); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-PYTHON-1001: " + std::string(err.what()); - } catch (...) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-PYTHON-1124: python crashed for unknown reasons"; - } -} - -bool PythonVM::run() { - try { - DBG_FUNC_CALL(cerr, bool result = m_impl->run()); - return result; - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-PYTHON-1002: "+ std::string(err.what()); - } catch (...) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-PYTHON-1003: python crashed for unknown reasons"; - } - return false; -} - - -const char* PythonVM::singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args) { - try { - return m_impl->singleCall(fn, args,calledUndefinedSingleCall); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-PYTHON-1004: "+std::string(err.what()); - } catch (...) { - lock_guard lock(exception_msg_mtx); - exception_msg = "F-UDF-CL-SL-PYTHON-1123: python crashed for unknown reasons"; - } - return strdup(""); -} - - -#ifndef DISABLE_PYTHON_SUBINTERP -PyThreadState *PythonVMImpl::main_thread = NULL; -#endif - -PythonVMImpl::PythonVMImpl(bool checkOnly): m_checkOnly(checkOnly) -{ - DBG_FUNC_BEGIN( cerr ); - script_code = string("\xEF\xBB\xBF") + string(SWIGVM_params->script_code); // Magic-Number of UTF-8 files - - script = exatable = globals = retvalue = NULL; -#ifndef DISABLE_PYTHON_SUBINTERP - pythread = NULL; -#endif - - if (!Py_IsInitialized()) { - ::setlocale(LC_ALL, "en_US.utf8"); - Py_NoSiteFlag = 1; - PyImport_AppendInittab("_exascript_python",PyInit__exascript_python); - Py_Initialize(); - PyEval_InitThreads(); -#ifndef DISABLE_PYTHON_SUBINTERP - main_thread = PyEval_SaveThread(); -#endif - } - - - { -#ifndef DISABLE_PYTHON_SUBINTERP - PythonThreadBlock block; -#endif - - globals = PyDict_New(); - PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()); - script = Py_CompileString(script_code.c_str(), SWIGVM_params->script_name, Py_file_input); check("F-UDF-CL-SL-PYTHON-1005"); - if (script == NULL) throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1006: Failed to compile script"); - -#ifndef DISABLE_PYTHON_SUBINTERP - pythread = PyThreadState_New(main_thread->interp); - if (pythread == NULL) - throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1007: Failed to create Python interpreter"); -#endif - } - - if (!checkOnly) { -#ifndef DISABLE_PYTHON_SUBINTERP - PythonThreadBlock block; - PyThreadState_Swap(pythread); -#endif - code = Py_CompileString(integrated_exascript_python_py, "exascript_python.py", Py_file_input); check("F-UDF-CL-SL-PYTHON-1008"); - if (code == NULL) throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1009: Failed to compile internal module"); - exatable = PyImport_ExecCodeModule((char*)"exascript_python", code); - check("F-UDF-CL-SL-PYTHON-1010"); - if (exatable == NULL) throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1011: Failed to import code module"); - - code = Py_CompileString(integrated_exascript_python_preset_py, "", Py_file_input); check("F-UDF-CL-SL-PYTHON-1012"); - if (code == NULL) {check("F-UDF-CL-SL-PYTHON-1013");} - - PyEval_EvalCode(code, globals, globals); check("F-UDF-CL-SL-PYTHON-1014"); - Py_DECREF(code); - - PyObject *runobj = PyDict_GetItemString(globals, "__pythonvm_wrapped_parse"); check("F-UDF-CL-SL-PYTHON-1016"); - //PyObject *retvalue = PyObject_CallFunction(runobj, NULL); check(); - PyObject *retvalue = PyObject_CallFunctionObjArgs(runobj, globals, NULL); check("F-UDF-CL-SL-PYTHON-1017"); - Py_XDECREF(retvalue); retvalue = NULL; - - code = Py_CompileString(integrated_exascript_python_wrap_py, "", Py_file_input); check("F-UDF-CL-SL-PYTHON-1018"); - if (code == NULL) throw PythonVM::exception("Failed to compile wrapping script"); - - PyEval_EvalCode(code, globals, globals); check("F-UDF-CL-SL-PYTHON-1019"); - Py_XDECREF(code); - } - DBG_FUNC_END( cerr ); -} - -void PythonVMImpl::shutdown() { - { -#ifndef DISABLE_PYTHON_SUBINTERP - PythonThreadBlock block; - if (pythread != NULL) - PyThreadState_Swap(pythread); -#endif - Py_XDECREF(retvalue); - if (!m_checkOnly) { - cleanobj = PyDict_GetItemString(globals, "cleanup"); check("F-UDF-CL-SL-PYTHON-1021"); - if (cleanobj){ - clean_wrap_obj = PyDict_GetItemString(globals, "__pythonvm_wrapped_cleanup"); check("F-UDF-CL-SL-PYTHON-1022"); - if (clean_wrap_obj) { - retvalue = PyObject_CallObject(clean_wrap_obj, NULL); - check("F-UDF-CL-SL-PYTHON-1023"); - } - } - } - Py_XDECREF(retvalue); retvalue = NULL; - Py_XDECREF(script); - Py_XDECREF(exatable); - Py_XDECREF(globals); - } -} - -bool PythonVMImpl::run() { - DBG_FUNC_BEGIN( cerr ); - - if (m_checkOnly) throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1024: Python VM in check only mode"); - - { -#ifndef DISABLE_PYTHON_SUBINTERP - PythonThreadBlock block; - PyThreadState_Swap(pythread); -#endif - DBG_FUNC_CALL(cerr, runobj = PyDict_GetItemString(globals, "__pythonvm_wrapped_run")); check("F-UDF-CL-SL-PYTHON-1025"); - DBG_FUNC_CALL(cerr, retvalue = PyObject_CallFunction(runobj, NULL)); check("F-UDF-CL-SL-PYTHON-1026"); - if (retvalue == NULL) { - throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1027: Python VM: calling 'run' failed without an exception)"); - } - Py_XDECREF(retvalue); retvalue = NULL; - } - return true; -} - - -static string singleCallResult; - -const char* PythonVMImpl::singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args , string& calledUndefinedSingleCall) { - if (m_checkOnly) throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1028: Python VM in check only mode (singleCall)"); // @@@@ TODO: better exception text - //{ -#ifndef DISABLE_PYTHON_SUBINTERP - PythonThreadBlock block; - PyThreadState_Swap(pythread); -#endif - - const char* func = NULL; - switch (fn) { - case SC_FN_NIL: break; - case SC_FN_DEFAULT_OUTPUT_COLUMNS: func = "default_output_columns"; break; - case SC_FN_VIRTUAL_SCHEMA_ADAPTER_CALL: func = "adapter_call"; break; - case SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC: func = "generate_sql_for_import_spec"; break; - case SC_FN_GENERATE_SQL_FOR_EXPORT_SPEC: func = "generate_sql_for_export_spec"; break; - } - if (func == NULL) - { - throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1029: Unknown single call function "+fn); - } - PyObject* argObject = NULL; - - if (fn==SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC) - { - argObject= PyDict_New(); - const ExecutionGraph::ImportSpecification* imp_spec = dynamic_cast(&args); - if (imp_spec == NULL) - { - throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1030: Internal Python VM error: cannot cast argument DTO to import specification"); - } - // import_spec.is_subselect - PyDict_SetItemString(argObject,"is_subselect", (imp_spec->isSubselect())?Py_True:Py_False); - - if (imp_spec->hasSubselectColumnNames()) { - size_t numSubselectColumnNames = imp_spec->getSubselectColumnNames().size(); - PyObject* names = PyList_New(numSubselectColumnNames); - for (size_t i=0; i< numSubselectColumnNames; i++) - { - PyList_SetItem(names,i,PyString_FromString(imp_spec->getSubselectColumnNames()[i].c_str())); - } - PyDict_SetItemString(argObject,"subselect_column_names",names); - Py_XDECREF(names); - } else { - PyDict_SetItemString(argObject,"subselect_column_names", Py_None); - } - if (imp_spec->hasSubselectColumnTypes()) { - size_t numSubselectColumnTypes = imp_spec->getSubselectColumnTypes().size(); - PyObject* types = PyList_New(numSubselectColumnTypes); - for (size_t i=0; i< numSubselectColumnTypes; i++) - { - PyList_SetItem(types,i,PyString_FromString(imp_spec->getSubselectColumnTypes()[i].c_str())); - } - PyDict_SetItemString(argObject,"subselect_column_types",types); - Py_XDECREF(types); - } else { - PyDict_SetItemString(argObject,"subselect_column_types", Py_None); - } - - if (imp_spec->hasConnectionName()) { - PyObject* connection_name = PyString_FromString(imp_spec->getConnectionName().c_str()); - PyDict_SetItemString(argObject,"connection_name",connection_name); - Py_XDECREF(connection_name); - } else { - PyDict_SetItemString(argObject,"connection_name", Py_None); - } - - if (imp_spec->hasConnectionInformation()) { - PyObject* connectionObject = PyDict_New(); - - PyObject* kind = PyString_FromString(imp_spec->getConnectionInformation().getKind().c_str()); - PyDict_SetItemString(connectionObject,"type",kind); - Py_XDECREF(kind); - - PyObject* address = PyString_FromString(imp_spec->getConnectionInformation().getAddress().c_str()); - PyDict_SetItemString(connectionObject,"address",address); - Py_XDECREF(address); - - PyObject* user = PyString_FromString(imp_spec->getConnectionInformation().getUser().c_str()); - PyDict_SetItemString(connectionObject,"user",user); - Py_XDECREF(user); - - PyObject* password = PyString_FromString(imp_spec->getConnectionInformation().getPassword().c_str()); - PyDict_SetItemString(connectionObject,"password",password); - Py_XDECREF(password); - - PyDict_SetItemString(argObject,"connection",connectionObject); - Py_XDECREF(connectionObject); - } else { - PyDict_SetItemString(argObject,"connection", Py_None); - } - - PyObject* paramObject= PyDict_New(); - for (std::map::const_iterator i = imp_spec->getParameters().begin(); - i != imp_spec->getParameters().end(); - ++i) - { - PyObject* key = PyString_FromString(i->first.c_str()); - PyObject* value = PyString_FromString(i->second.c_str()); - PyDict_SetItem(paramObject,key,value); - Py_XDECREF(key); - Py_XDECREF(value); - } - PyDict_SetItemString(argObject,"parameters",paramObject); - Py_XDECREF(paramObject); - } else if (fn==SC_FN_GENERATE_SQL_FOR_EXPORT_SPEC) { - argObject= PyDict_New(); - const ExecutionGraph::ExportSpecification* exp_spec = dynamic_cast(&args); - if (exp_spec == NULL) - { - throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1031: Internal Python VM error: cannot cast argument DTO to export specification"); - } - - if (exp_spec->hasConnectionName()) { - PyObject* connection_name = PyString_FromString(exp_spec->getConnectionName().c_str()); - PyDict_SetItemString(argObject,"connection_name",connection_name); - Py_XDECREF(connection_name); - } else { - PyDict_SetItemString(argObject,"connection_name", Py_None); - } - - if (exp_spec->hasConnectionInformation()) { - PyObject* connectionObject = PyDict_New(); - - PyObject* kind = PyString_FromString(exp_spec->getConnectionInformation().getKind().c_str()); - PyDict_SetItemString(connectionObject,"type",kind); - Py_XDECREF(kind); - - PyObject* address = PyString_FromString(exp_spec->getConnectionInformation().getAddress().c_str()); - PyDict_SetItemString(connectionObject,"address",address); - Py_XDECREF(address); - - PyObject* user = PyString_FromString(exp_spec->getConnectionInformation().getUser().c_str()); - PyDict_SetItemString(connectionObject,"user",user); - Py_XDECREF(user); - - PyObject* password = PyString_FromString(exp_spec->getConnectionInformation().getPassword().c_str()); - PyDict_SetItemString(connectionObject,"password",password); - Py_XDECREF(password); - - PyDict_SetItemString(argObject,"connection",connectionObject); - Py_XDECREF(connectionObject); - } else { - PyDict_SetItemString(argObject,"connection", Py_None); - } - - PyObject* paramObject= PyDict_New(); - for (std::map::const_iterator i = exp_spec->getParameters().begin(); - i != exp_spec->getParameters().end(); - ++i) - { - PyObject* key = PyString_FromString(i->first.c_str()); - PyObject* value = PyString_FromString(i->second.c_str()); - PyDict_SetItem(paramObject,key,value); - Py_XDECREF(key); - Py_XDECREF(value); - } - PyDict_SetItemString(argObject,"parameters",paramObject); - Py_XDECREF(paramObject); - - PyDict_SetItemString(argObject,"has_truncate", (exp_spec->hasTruncate())?Py_True:Py_False); - PyDict_SetItemString(argObject,"has_replace", (exp_spec->hasReplace())?Py_True:Py_False); - if (exp_spec->hasCreatedBy()) { - PyObject* created_by = PyString_FromString(exp_spec->getCreatedBy().c_str()); - PyDict_SetItemString(argObject,"created_by",created_by); - Py_XDECREF(created_by); - } else { - PyDict_SetItemString(argObject,"created_by", Py_None); - } - - size_t numSourceColumnNames = exp_spec->getSourceColumnNames().size(); - if (numSourceColumnNames > 0) { - PyObject* names = PyList_New(numSourceColumnNames); - for (size_t i=0; i < numSourceColumnNames; i++) - { - PyList_SetItem(names,i,PyString_FromString(exp_spec->getSourceColumnNames()[i].c_str())); - } - PyDict_SetItemString(argObject,"source_column_names",names); - Py_XDECREF(names); - } else { - PyDict_SetItemString(argObject,"source_column_names", Py_None); - } - } - -// Py_XINCREF(argObject); - - PyObject* funcToCall = PyDict_GetItemString(globals, func); check("F-UDF-CL-SL-PYTHON-1032"); - if (funcToCall == NULL) { - calledUndefinedSingleCall = func; - return strdup(""); - //throw swig_undefined_single_call_exception(func); // no such call is defined. - } - if (fn==SC_FN_VIRTUAL_SCHEMA_ADAPTER_CALL) { - // Call directly - // TODO VS This will all be refactored - const ExecutionGraph::StringDTO* argDto = dynamic_cast(&args); - string string_arg = argDto->getArg(); - runobj = PyDict_GetItemString(globals, func); check("F-UDF-CL-SL-PYTHON-1033"); - retvalue = PyObject_CallFunction(runobj, (char *)"s", string_arg.c_str()); - } else { - runobj = PyDict_GetItemString(globals, "__pythonvm_wrapped_singleCall"); check("F-UDF-CL-SL-PYTHON-1034"); - if (runobj == NULL) { - throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1035: Cannot find function __pythonvm_wrapped_singleCall"); - } - // Call indirectly - if (argObject == NULL) { - retvalue = PyObject_CallFunctionObjArgs(runobj, funcToCall, Py_None, NULL); - } else { - retvalue = PyObject_CallFunctionObjArgs(runobj, funcToCall, argObject, NULL); - } - } - check("F-UDF-CL-SL-PYTHON-1036"); - - Py_XDECREF(argObject); - - if (!PyString_Check(retvalue) && !PyUnicode_Check(retvalue)) - { - std::stringstream sb; - sb << "F-UDF-CL-SL-PYTHON-1037: "; - sb << fn; - sb << " did not return string type (singleCall)"; - throw PythonVM::exception(sb.str().c_str()); - } - - PyObject* repr = PyObject_Str(retvalue); - PyObject* p3str = PyUnicode_AsEncodedString(repr, "utf-8", "ignore"); - const char *bytes = PyBytes_AS_STRING(p3str); - singleCallResult = string(bytes); - Py_XDECREF(retvalue); retvalue = NULL; - return singleCallResult.c_str(); -} diff --git a/udf-runner-cpp/v1/base/python/pythoncontainer.h b/udf-runner-cpp/v1/base/python/pythoncontainer.h deleted file mode 100644 index 27092a9..0000000 --- a/udf-runner-cpp/v1/base/python/pythoncontainer.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef PYTHONVM_H -#define PYTHONVM_H - -#include "exaudflib/vm/swig_vm.h" - -#ifdef ENABLE_PYTHON_VM - -namespace SWIGVMContainers { - -class PythonVMImpl; - -class PythonVM: public SWIGVM { - public: - PythonVM(bool checkOnly); - virtual ~PythonVM() {}; - virtual void shutdown(); - virtual bool run(); - virtual const char* singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args); - private: - PythonVMImpl *m_impl; -}; - -} //namespace SWIGVMContainers - -#endif //ENABLE_PYTHON_VM - -#endif //PYTHONVM_H \ No newline at end of file diff --git a/udf-runner-cpp/v1/base/python_repository.bzl b/udf-runner-cpp/v1/base/python_repository.bzl deleted file mode 100644 index 549eabc..0000000 --- a/udf-runner-cpp/v1/base/python_repository.bzl +++ /dev/null @@ -1,138 +0,0 @@ -def _get_actual_python_version(binary,p_repository_ctx): - command_result = p_repository_ctx.execute([binary, "-c", """import sys; print(".".join(map(str, sys.version_info[:3])))"""]) - if command_result.return_code != 0: - fail("Could not acquire actual python version, got return code %s stderr: \n %s" - % (command_result.return_code, command_result.stderr)) - actual_version = command_result.stdout.strip("\n") - print("python actual_version: %s"%actual_version) - return actual_version - -def _get_sysconfig_value(binary,key,p_repository_ctx): - script = "import sysconfig; print(sysconfig.get_config_var('{key}'))".format(key=key) - command_result = p_repository_ctx.execute([binary,"-c", script]) - if command_result.return_code != 0: - fail("Could not acquire {key} for python, got return code {return_code} stderr: \n {stderr}".format( - key=key, return_code=command_result.return_code, stderr=command_result.stderr)) - stripped_command_result = command_result.stdout.strip("\n") - return stripped_command_result - -def _get_include_dir(binary,version,p_repository_ctx): - key = "INCLUDEDIR" - base_include_dir = _get_sysconfig_value(binary,key,p_repository_ctx) #example: /usr/include - include_dir = base_include_dir+"/"+version #example /usr/include/python3.10 - print("python {key}: {include_dir}".format(key=key, include_dir=include_dir)) - return include_dir - -def _get_lib_glob(binary, version, p_repository_ctx): - key = "LIBDIR" - base_lib_dir = _get_sysconfig_value(binary,key,p_repository_ctx) #example: /usr/lib - lib_glob = "%s/**/lib%s*.so" % (base_lib_dir,version) - print("python {key}_glob: {lib_glob}".format(key=key, lib_glob=lib_glob)) - return lib_glob - -def _python_local_repository_impl(repository_ctx): - python_version = repository_ctx.attr.python_version - python_prefix_env_var = python_version.upper() + "_PREFIX" - if python_prefix_env_var in repository_ctx.os.environ: - prefix = repository_ctx.os.environ[python_prefix_env_var] - else: - fail("Environment Variable %s not found"%python_prefix_env_var) - print("python prefix in environment specified; %s"%prefix) - - python_version_env_var = python_version.upper() + "_VERSION" - if python_version_env_var in repository_ctx.os.environ: - version = repository_ctx.os.environ[python_version_env_var] - if python_version == "python3" and version.startswith("2"): - fail("Wrong python version specified in environment variable %s, got binary name '%s', but version number '%s'"%(python_version_env_var,python_version,version)) - if python_version == "python" or version.startswith("2"): - fail("Python 2 is not supported anymore, but specified in environment variable %s, got %s, %s"%(python_version_env_var,python_version,version)) - else: - fail("Environment Variable %s not found"%python_version_env_var) - print("python version in environment specified; %s"%version) - - binary = prefix+"/bin/"+version - actual_version = _get_actual_python_version(binary,repository_ctx) - - include_dir = _get_include_dir(binary, version, repository_ctx) - lib_glob = _get_lib_glob(binary, version, repository_ctx) - defines = ['"ENABLE_PYTHON_VM"'] - if actual_version[0]=="3": - defines.append('"ENABLE_PYTHON3"') - defines_str = ",".join(defines) - build_file_content = """ -cc_library( - name = "{name}", - srcs = glob(["{lib_glob}"]), - hdrs = glob(["{include_dir}/**/*.h"]), - includes = ["{include_dir}"], - defines = [{defines}], - visibility = ["//visibility:public"] -)""".format(lib_glob=lib_glob[1:], include_dir=include_dir[1:], name=python_version, defines=defines_str) - print(build_file_content) - - repository_ctx.symlink(prefix, "."+prefix) - repository_ctx.file("BUILD", build_file_content) - -python_local_repository = repository_rule( - implementation=_python_local_repository_impl, - local = True, - environ = ["PYTHON3_PREFIX", "PYTHON3_VERSION"], - attrs = {"python_version": attr.string()}) - - -def _get_numpy_include_dir(binary,p_repository_ctx): - command_result = p_repository_ctx.execute([binary, "-c", """import numpy as np; print(np.get_include())"""]) - if command_result.return_code != 0: - fail("Could not acquire numpy include dir, got return code %s stderr: \n %s" - % (command_result.return_code, command_result.stderr)) - numpy_include_dir = command_result.stdout.strip("\n")[1:] - print("python numpy_include_dir: %s"%numpy_include_dir) - return numpy_include_dir - - -def _numpy_local_repository_impl(repository_ctx): - python_prefix_env_var = "PYTHON3_PREFIX" - if python_prefix_env_var in repository_ctx.os.environ: - prefix = repository_ctx.os.environ[python_prefix_env_var] - else: - fail("Environment Variable %s not found"%python_prefix_env_var) - print("python prefix in environment specified; %s"%prefix) - - python_version_env_var = "PYTHON3_VERSION" - if python_version_env_var in repository_ctx.os.environ: - version = repository_ctx.os.environ[python_version_env_var] - else: - fail("Environment Variable %s not found"%python_version_env_var) - if version.startswith("2"): - fail("Wrong python version specified in environment variable %s, got binary name '%s', but version number '%s'"%(python_version_env_var,repository_ctx.name,version)) - print("python version in environment specified; %s"%version) - - binary = prefix+"/bin/"+version - actual_version = _get_actual_python_version(binary,repository_ctx) - - defines = ['"ENABLE_PYTHON_VM"'] - if actual_version[0]=="3": - defines.append('"ENABLE_PYTHON3"') - defines_str = ",".join(defines) - - numpy_include_dir =_get_numpy_include_dir(binary,repository_ctx) - hdrs = numpy_include_dir + "/*/*.h" - build_file_content = """ -cc_library( - name = "numpy", - srcs = [], - hdrs = glob(["{hdrs}"]), - includes = ["{includes}"], - defines = [{defines}], - visibility = ["//visibility:public"] -)""".format(hdrs=hdrs, includes=numpy_include_dir, defines=defines_str) - print(build_file_content) - - repository_ctx.symlink(prefix, "."+prefix) - repository_ctx.file("BUILD", build_file_content) - -numpy_local_repository = repository_rule( - implementation=_numpy_local_repository_impl, - local = True, - environ = ["PYTHON3_PREFIX","PYTHON3_VERSION"]) - diff --git a/udf-runner-cpp/v1/base/script_options_parser/ctpg/test/script_option_lines_test.cpp b/udf-runner-cpp/v1/base/script_options_parser/ctpg/test/script_option_lines_test.cpp index bcc4c98..46fc9a9 100644 --- a/udf-runner-cpp/v1/base/script_options_parser/ctpg/test/script_option_lines_test.cpp +++ b/udf-runner-cpp/v1/base/script_options_parser/ctpg/test/script_option_lines_test.cpp @@ -43,9 +43,9 @@ std::vector prefixes = {"", " ", "\t", "\f", "\v", "\n", "\r\n", " std::vector suffixes = {"", " ", "\t", "\f", "\v"}; //"" for case if there is suffix std::vector new_lines = {"", "\n", "\r", "\r\n"}; //"" for case if there is no newline std::vector delimeters = {" ", "\t", "\f", "\v", " \t", "\t ", "\t\f", "\f\t", "\f ", " \f", "\t\v", "\v\t", "\v ", " \v", "\f\v", "\v\f", " \t", " \t "}; -std::vector keywords = {"import", "jvmoption", "scriptclass", "jar", "env"}; -std::vector values = {"something", "com.mycompany.MyScriptClass", "LD_LIBRARY_PATH=/nvdriver", "-Xms128m -Xmx1024m -Xss512k", "/buckets/bfsdefault/default/my_code.jar", "something "}; -std::vector payloads = {"anything", "\n\ndef my_func:\n\tpass", "class MyJava\n public static void Main() {\n};\n"}; +std::vector keywords = {"import", "scriptclass", "env", "some_option"}; +std::vector values = {"something", "com.mycompany.MyScriptClass", "LD_LIBRARY_PATH=/nvdriver", "some-value", "something "}; +std::vector payloads = {"anything", "\n\ndef my_func:\n\tpass", "class AnyClass\n public static void Main() {\n};\n"}; INSTANTIATE_TEST_SUITE_P( ScriptOptionLines, @@ -174,70 +174,27 @@ INSTANTIATE_TEST_SUITE_P( ); -TEST(ScriptOptionLinesTest, test_when_two_options_plus_code_in_same_line_then_options_parsed_successfully) { - /** - Verify the correct behavior of new parser for situation as described in https://github.com/exasol/script-languages-release/issues/652. - */ - const std::string code = "%jar /buckets/bucketfs1/jars/exajdbc.jar; %jvmoption -Xms4m; class JAVA_UDF_3 {static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {String host_name = ctx.getString(\"col1\");}}\n/\n;"; - options_map_t result; - parseOptions(code, result); - ASSERT_EQ(result.size(), 2); - - const auto jar_option_result = result.find("jar"); - ASSERT_NE(jar_option_result, result.end()); - ASSERT_EQ(jar_option_result->second.size(), 1); - ASSERT_EQ(jar_option_result->second[0], buildOption("/buckets/bucketfs1/jars/exajdbc.jar", 0, 41)); - - const auto jvm_option_result = result.find("jvmoption"); - ASSERT_NE(jvm_option_result, result.end()); - ASSERT_EQ(jvm_option_result->second.size(), 1); - ASSERT_EQ(jvm_option_result->second[0], buildOption("-Xms4m", 42, 18)); -} - - -TEST(ScriptOptionLinesTest, test_values_can_contain_spaces) { - /** - Verify assumptions as described in https://github.com/exasol/script-languages-release/issues/878 - The parser is actually correct, but the client code incorrectly parses the result (see javacontainer_test.cc - quoted_jvm_option) - */ - const std::string code = - "%jvmoption -Dhttp.agent=\"ABC DEF\";\n\n" - "class JVMOPTION_TEST_WITH_SPACE {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - options_map_t result; - parseOptions(code, result); - ASSERT_EQ(result.size(), 1); - - const auto jvm_option_result = result.find("jvmoption"); - ASSERT_NE(jvm_option_result, result.end()); - ASSERT_EQ(jvm_option_result->second.size(), 1); - ASSERT_EQ(jvm_option_result->second[0], buildOption("-Dhttp.agent=\"ABC DEF\"", 0, 34)); -} - TEST(ScriptOptionLinesTest, test_multiple_lines_with_code) { /** Verify that the parser can read options coming after some code. */ const std::string code = - "%jvmoption -Dhttp.agent=\"ABC DEF\"; class Abc{};\n\n" - "%jar /buckets/bucketfs1/jars/exajdbc.jar; class DEF{};\n"; + "%some_option alpha beta; class Abc{};\n\n" + "%otheroption gamma; class DEF{};\n"; options_map_t result; parseOptions(code, result); ASSERT_EQ(result.size(), 2); - const auto jvm_option_result = result.find("jvmoption"); - ASSERT_NE(jvm_option_result, result.end()); - ASSERT_EQ(jvm_option_result->second.size(), 1); - ASSERT_EQ(jvm_option_result->second[0], buildOption("-Dhttp.agent=\"ABC DEF\"", 0, 34)); + const auto option_result = result.find("some_option"); + ASSERT_NE(option_result, result.end()); + ASSERT_EQ(option_result->second.size(), 1); + ASSERT_EQ(option_result->second[0], buildOption("alpha beta", 0, 24)); - const auto jar_option_result = result.find("jar"); - ASSERT_NE(jar_option_result, result.end()); - ASSERT_EQ(jar_option_result->second.size(), 1); - ASSERT_EQ(jar_option_result->second[0], buildOption("/buckets/bucketfs1/jars/exajdbc.jar", 49, 41)); + const auto other_option_result = result.find("otheroption"); + ASSERT_NE(other_option_result, result.end()); + ASSERT_EQ(other_option_result->second.size(), 1); + ASSERT_EQ(other_option_result->second[0], buildOption("gamma", 39, 19)); } @@ -249,22 +206,22 @@ TEST_P(ScriptOptionLinesEscapeSequenceTest, test_escape_seq_in_option_value) { Verify that the parser replaces escape sequences correctly. */ const std::string code = - "%jvmoption " + option_value.first + "; class Abc{};\n" - "%jar /buckets/bucketfs1/jars/exajdbc.jar; class DEF{};\n"; + "%some_option " + option_value.first + "; class Abc{};\n" + "%otheroption gamma; class DEF{};\n"; options_map_t result; parseOptions(code, result); ASSERT_EQ(result.size(), 2); - const auto jvm_option_result = result.find("jvmoption"); - ASSERT_NE(jvm_option_result, result.end()); - ASSERT_EQ(jvm_option_result->second.size(), 1); - EXPECT_EQ(jvm_option_result->second[0].value, option_value.second); + const auto option_result = result.find("some_option"); + ASSERT_NE(option_result, result.end()); + ASSERT_EQ(option_result->second.size(), 1); + EXPECT_EQ(option_result->second[0].value, option_value.second); - const auto jar_option_result = result.find("jar"); - ASSERT_NE(jar_option_result, result.end()); - ASSERT_EQ(jar_option_result->second.size(), 1); - ASSERT_EQ(jar_option_result->second[0].value, "/buckets/bucketfs1/jars/exajdbc.jar"); + const auto other_option_result = result.find("otheroption"); + ASSERT_NE(other_option_result, result.end()); + ASSERT_EQ(other_option_result->second.size(), 1); + ASSERT_EQ(other_option_result->second[0].value, "gamma"); } /* @@ -278,26 +235,26 @@ TEST_P(ScriptOptionLinesEscapeSequenceTest, test_escape_seq_in_option_value) { */ const std::vector> escape_sequences = { - std::make_pair("-Dhttp.agent=ABC\\nDEF", "-Dhttp.agent=ABC\nDEF"), - std::make_pair("-Dhttp.agent=ABC\\rDEF", "-Dhttp.agent=ABC\rDEF"), - std::make_pair("-Dhttp.agent=ABC\\;DEF", "-Dhttp.agent=ABC;DEF"), - std::make_pair("-Dhttp.agent=ABC\\\\rDEF", "-Dhttp.agent=ABC\\rDEF"), - std::make_pair("-Dhttp.agent=ABC\\aDEF", "-Dhttp.agent=ABC\\aDEF"), //any other escape sequence must stay as is - std::make_pair("\\n-Dhttp.agent=ABCDEF", "\n-Dhttp.agent=ABCDEF"), - std::make_pair("\\r-Dhttp.agent=ABCDEF", "\r-Dhttp.agent=ABCDEF"), - std::make_pair("\\;-Dhttp.agent=ABCDEF", ";-Dhttp.agent=ABCDEF"), - std::make_pair("\\\\r-Dhttp.agent=ABCDEF", "\\r-Dhttp.agent=ABCDEF"), - std::make_pair("-Dhttp.agent=ABCDEF\\n", "-Dhttp.agent=ABCDEF\n"), - std::make_pair("-Dhttp.agent=ABCDEF\\r", "-Dhttp.agent=ABCDEF\r"), - std::make_pair("-Dhttp.agent=ABCDEF\\;", "-Dhttp.agent=ABCDEF;"), - std::make_pair("-Dhttp.agent=ABCDEF\\\\;", "-Dhttp.agent=ABCDEF\\"), - std::make_pair("-Dhttp.agent=ABCDEF\\\\\\;", "-Dhttp.agent=ABCDEF\\;"), - std::make_pair("-Dhttp.agent=ABC\\ DEF", "-Dhttp.agent=ABC\\ DEF"), //escaped white space in middle of string must stay as is - std::make_pair("\\ -Dhttp.agent=ABCDEF", " -Dhttp.agent=ABCDEF"), - std::make_pair("\\ \t -Dhttp.agent=ABCDEF", " \t -Dhttp.agent=ABCDEF"), - std::make_pair("\\t-Dhttp.agent=ABCDEF", "\t-Dhttp.agent=ABCDEF"), - std::make_pair("\\f-Dhttp.agent=ABCDEF", "\f-Dhttp.agent=ABCDEF"), - std::make_pair("\\v-Dhttp.agent=ABCDEF", "\v-Dhttp.agent=ABCDEF") + std::make_pair("ABC\\nDEF", "ABC\nDEF"), + std::make_pair("ABC\\rDEF", "ABC\rDEF"), + std::make_pair("ABC\\;DEF", "ABC;DEF"), + std::make_pair("ABC\\\\rDEF", "ABC\\rDEF"), + std::make_pair("ABC\\aDEF", "ABC\\aDEF"), //any other escape sequence must stay as is + std::make_pair("\\nABCDEF", "\nABCDEF"), + std::make_pair("\\rABCDEF", "\rABCDEF"), + std::make_pair("\\;ABCDEF", ";ABCDEF"), + std::make_pair("\\\\rABCDEF", "\\rABCDEF"), + std::make_pair("ABCDEF\\n", "ABCDEF\n"), + std::make_pair("ABCDEF\\r", "ABCDEF\r"), + std::make_pair("ABCDEF\\;", "ABCDEF;"), + std::make_pair("ABCDEF\\\\;", "ABCDEF\\"), + std::make_pair("ABCDEF\\\\\\;", "ABCDEF\\;"), + std::make_pair("ABC\\ DEF", "ABC\\ DEF"), //escaped white space in middle of string must stay as is + std::make_pair("\\ ABCDEF", " ABCDEF"), + std::make_pair("\\ \t ABCDEF", " \t ABCDEF"), + std::make_pair("\\tABCDEF", "\tABCDEF"), + std::make_pair("\\fABCDEF", "\fABCDEF"), + std::make_pair("\\vABCDEF", "\vABCDEF") }; INSTANTIATE_TEST_SUITE_P( @@ -315,16 +272,16 @@ TEST_P(ScriptOptionLinesRestTest, test_rest_with_tokens) { after the options in a line. */ const std::string code = - "%jvmoption -Dhttp.agent=abc; class Abc{};" + rest; + "%some_option alpha; class Abc{};" + rest; options_map_t result; parseOptions(code, result); ASSERT_EQ(result.size(), 1); - const auto jvm_option_result = result.find("jvmoption"); - ASSERT_NE(jvm_option_result, result.end()); - ASSERT_EQ(jvm_option_result->second.size(), 1); - ASSERT_EQ(jvm_option_result->second[0], buildOption("-Dhttp.agent=abc", 0, 28)); + const auto option_result = result.find("some_option"); + ASSERT_NE(option_result, result.end()); + ASSERT_EQ(option_result->second.size(), 1); + ASSERT_EQ(option_result->second[0], buildOption("alpha", 0, 19)); } const std::vector rest_strings = @@ -334,4 +291,4 @@ INSTANTIATE_TEST_SUITE_P( ScriptOptionLines, ScriptOptionLinesRestTest, ::testing::ValuesIn(rest_strings) -); \ No newline at end of file +); diff --git a/udf-runner-cpp/v1/base/script_options_parser/legacy/test/script_option_lines_test.cpp b/udf-runner-cpp/v1/base/script_options_parser/legacy/test/script_option_lines_test.cpp index 550f999..af10963 100644 --- a/udf-runner-cpp/v1/base/script_options_parser/legacy/test/script_option_lines_test.cpp +++ b/udf-runner-cpp/v1/base/script_options_parser/legacy/test/script_option_lines_test.cpp @@ -26,9 +26,9 @@ TEST_P(ScriptOptionLinesWhitespaceTest, WhitespaceExtractOptionLineTest) { } std::vector white_space_strings = {"", " ", "\t", "\f", "\v", "\n", " \t", "\t ", "\t\f", "\f\t", "\f ", " \f", "\t\v", "\v\t", "\v ", " \v", "\f\v", "\v\f", " \t", " \t "}; -std::vector keywords = {"%import", "jvmoption", "%scriptclass", "%jar", "%env"}; -std::vector values = {"something", "com.mycompany.MyScriptClass", "LD_LIBRARY_PATH=/nvdriver", "-Xms128m -Xmx1024m -Xss512k", "/buckets/bfsdefault/default/my_code.jar"}; -std::vector payloads = {"anything", "\n\ndef my_func:\n\tpass", "class MyJava\n public static void Main() {\n};\n"}; +std::vector keywords = {"%import", "%scriptclass", "%env", "%some_option"}; +std::vector values = {"something", "com.mycompany.MyScriptClass", "LD_LIBRARY_PATH=/nvdriver", "some-value"}; +std::vector payloads = {"anything", "\n\ndef my_func:\n\tpass", "class AnyClass\n public static void Main() {\n};\n"}; INSTANTIATE_TEST_SUITE_P( ScriptOptionLines, @@ -120,81 +120,25 @@ TEST(ScriptOptionLinesTest, ignores_any_other_option) { } -TEST(ScriptOptionLinesTest, test_all_in_one_line_does_second_option_does_not_work) { - /** - Verify the wrong behavior and assumptions as described in https://github.com/exasol/script-languages-release/issues/652. - Here we call `extractOptionLine()` with the keys in the order of the new implementation (first for key 'jvmoption'). - This is supposed to not work. - */ - size_t pos; - const std::string original_code = "%jar /buckets/bucketfs1/jars/exajdbc.jar; %jvmoption -Xms4m; class JAVA_UDF_3 {static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {String host_name = ctx.getString(\"col1\");}}\n/\n;"; - std::string code = original_code; - const std::string res = extractOptionLine(code, "%jvmoption", whitespace, lineEnd, pos); - EXPECT_TRUE(res.empty()); - EXPECT_EQ(code, original_code); -} - -TEST(ScriptOptionLinesTest, test_all_in_one_line_does_first_option_does_work) { - /** - Verify the wrong behavior and assumptions as described in https://github.com/exasol/script-languages-release/issues/652. - Here we call `extractOptionLine()` with the keys in the order of the old implementation (first for key '%jar', then for key 'jvmoption'). - This is supposed to work. - */ - size_t pos; - const std::string original_code = "%jar /buckets/bucketfs1/jars/exajdbc.jar; %jvmoption -Xms4m; class JAVA_UDF_3 {static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {String host_name = ctx.getString(\"col1\");}}\n/\n;"; - std::string code = original_code; - std::string res = extractOptionLine(code, "%jar", whitespace, lineEnd, pos); - EXPECT_EQ(res, "/buckets/bucketfs1/jars/exajdbc.jar"); - EXPECT_EQ(code, " %jvmoption -Xms4m; class JAVA_UDF_3 {static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {String host_name = ctx.getString(\"col1\");}}\n/\n;"); - res = extractOptionLine(code, "%jvmoption", whitespace, lineEnd, pos); - EXPECT_EQ(code, " class JAVA_UDF_3 {static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {String host_name = ctx.getString(\"col1\");}}\n/\n;"); -} - -TEST(ScriptOptionLinesTest, test_values_must_not_contain_spaces) { - /** - Verify the wrong behavior and assumptions as described in https://github.com/exasol/script-languages-release/issues/878 - The parser is actually correct, but the client code incorrectly parses the result (see javacontainer_test.cc - quoted_jvm_option) - */ - size_t pos; - const std::string original_code = - "%jvmoption -Dhttp.agent=\"ABC DEF\";\n\n" - "class JVMOPTION_TEST_WITH_SPACE {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - std::string code = original_code; - std::string res = extractOptionLine(code, "%jvmoption", whitespace, lineEnd, pos); - const std::string expected_result_code = - "\n\n" - "class JVMOPTION_TEST_WITH_SPACE {\n" - "static void run(ExaMetadata exa, ExaIterator ctx) throws Exception {\n\n" - " ctx.emit(\"Success!\");\n" - " }\n" - "}\n"; - EXPECT_EQ(res, "-Dhttp.agent=\"ABC DEF\""); - EXPECT_EQ(code, expected_result_code); -} - TEST(ScriptOptionLinesTest, test_multiple_lines_with_code) { /** Verify that the parser can read options coming after some code. */ size_t pos; const std::string original_code = - "%jvmoption -Dhttp.agent=\"ABC DEF\"; class Abc{};\n\n" - "%jar /buckets/bucketfs1/jars/exajdbc.jar; class DEF{};\n"; + "%option alpha beta; class Abc{};\n\n" + "%otheroption gamma; class DEF{};\n"; std::string code = original_code; - std::string res = extractOptionLine(code, "%jvmoption", whitespace, lineEnd, pos); - EXPECT_EQ(res, "-Dhttp.agent=\"ABC DEF\""); + std::string res = extractOptionLine(code, "%option", whitespace, lineEnd, pos); + EXPECT_EQ(res, "alpha beta"); std::string expected_result_code = " class Abc{};\n\n" - "%jar /buckets/bucketfs1/jars/exajdbc.jar; class DEF{};\n"; + "%otheroption gamma; class DEF{};\n"; EXPECT_EQ(code, expected_result_code); - res = extractOptionLine(code, "%jar", whitespace, lineEnd, pos); - EXPECT_EQ(res, "/buckets/bucketfs1/jars/exajdbc.jar"); + res = extractOptionLine(code, "%otheroption", whitespace, lineEnd, pos); + EXPECT_EQ(res, "gamma"); expected_result_code = " class Abc{};\n\n" " class DEF{};\n"; diff --git a/udf-runner-cpp/v1/base/variables.bzl b/udf-runner-cpp/v1/base/variables.bzl index ef06b43..64e17fc 100644 --- a/udf-runner-cpp/v1/base/variables.bzl +++ b/udf-runner-cpp/v1/base/variables.bzl @@ -6,16 +6,8 @@ STREAMING_VM_ENABLED_DEFINE=select({ "//:bash": ["ENABLE_STREAMING_VM"], "//conditions:default": [] }) -PYTHON_VM_ENABLED_DEFINE=select({ - "//:python": ["ENABLE_PYTHON_VM"], - "//conditions:default": [] - }) -JAVA_VM_ENABLED_DEFINE=select({ - "//:java": ["ENABLE_JAVA_VM"], - "//conditions:default": [] - }) PLUGIN_CLIENT_ENABLED_DEFINE=select({ "//:plugin_client": ["UDF_PLUGIN_CLIENT"], "//conditions:default": [] }) -VM_ENABLED_DEFINES=BENCHMARK_VM_ENABLED_DEFINE+PYTHON_VM_ENABLED_DEFINE+JAVA_VM_ENABLED_DEFINE+STREAMING_VM_ENABLED_DEFINE+PLUGIN_CLIENT_ENABLED_DEFINE +VM_ENABLED_DEFINES=BENCHMARK_VM_ENABLED_DEFINE+STREAMING_VM_ENABLED_DEFINE+PLUGIN_CLIENT_ENABLED_DEFINE diff --git a/udf-runner-cpp/v1/build_local_all.sh b/udf-runner-cpp/v1/build_local_all.sh index c91abe5..abd8bde 100755 --- a/udf-runner-cpp/v1/build_local_all.sh +++ b/udf-runner-cpp/v1/build_local_all.sh @@ -1,3 +1,3 @@ #!/bin/bash -bash build_local.sh "$@" --config no-tty --config python --config java --config slow-wrapper +bash build_local.sh "$@" --config no-tty --config slow-wrapper diff --git a/udf-runner-cpp/v1/exa_udfclient_binary_common/exa_udf_client.cc b/udf-runner-cpp/v1/exa_udfclient_binary_common/exa_udf_client.cc index 04dc00a..34031a4 100644 --- a/udf-runner-cpp/v1/exa_udfclient_binary_common/exa_udf_client.cc +++ b/udf-runner-cpp/v1/exa_udfclient_binary_common/exa_udf_client.cc @@ -38,9 +38,7 @@ bool ExaUdfClient::validate_arguments(int argc, char** argv) { return false; } - if (!((strcmp(argv[2], "lang=python") == 0) - || (strcmp(argv[2], "lang=java") == 0) - || (strcmp(argv[2], "lang=streaming") == 0) + if (!((strcmp(argv[2], "lang=streaming") == 0) || (strcmp(argv[2], "lang=benchmark") == 0))) { usage(argv[0]); return false; @@ -51,7 +49,7 @@ bool ExaUdfClient::validate_arguments(int argc, char** argv) { void ExaUdfClient::usage(const std::string& programName) { std::cerr << "Usage: " << programName - << " lang=python|lang=java|lang=streaming|lang=benchmark " + << " lang=streaming|lang=benchmark " << std::endl; } diff --git a/udf-runner-cpp/v1/exa_udfclient_binary_common/exa_vm_factory.cc b/udf-runner-cpp/v1/exa_udfclient_binary_common/exa_vm_factory.cc index c288ff3..c510fc0 100644 --- a/udf-runner-cpp/v1/exa_udfclient_binary_common/exa_vm_factory.cc +++ b/udf-runner-cpp/v1/exa_udfclient_binary_common/exa_vm_factory.cc @@ -2,14 +2,6 @@ #include #include "exa_vm_factory.h" -#ifdef ENABLE_JAVA_VM -#include "javacontainer/javacontainer_builder.h" -#endif //ENABLE_JAVA_VM - -#ifdef ENABLE_PYTHON_VM -#include "python/pythoncontainer.h" -#endif //ENABLE_PYTHON_VM - #ifdef ENABLE_STREAMING_VM #include "streaming_container/streamingcontainer.h" #endif @@ -19,33 +11,7 @@ #endif std::function create_vm(const std::string& argv_lang, bool use_ctpg_options_parser) { - if(argv_lang.compare("lang=python") == 0) { - #ifdef ENABLE_PYTHON_VM - char *path_var = getenv("PATH"); - if(path_var != NULL) { - std::string path_var_str = std::string(path_var); - path_var_str.insert(0, "/opt/conda/bin:"); - if (::setenv("PATH", path_var_str.c_str(), 1) == -1) { - std::cerr << "Unable to prefix PATH env variable with /opt/conda/bin"; - } - } - return []() { return new SWIGVMContainers::PythonVM(false); }; - #else - throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Python support"); - #endif - } - else if(argv_lang.compare("lang=java") == 0) { - #ifdef ENABLE_JAVA_VM - if (use_ctpg_options_parser) { - return [&](){return SWIGVMContainers::JavaContainerBuilder().useCtpgParser().build();}; - } else { - return [&](){return SWIGVMContainers::JavaContainerBuilder().build();}; - } - #else - throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Java support"); - #endif - } - else if(argv_lang.compare("lang=streaming") == 0) { + if(argv_lang.compare("lang=streaming") == 0) { #ifdef ENABLE_STREAMING_VM return []() { return new SWIGVMContainers::StreamingVM(false); }; #else @@ -62,4 +28,4 @@ std::function create_vm(const std::string& argv_lan else { throw SWIGVMContainers::SWIGVM::exception("unsupported language specified in argv"); } -} \ No newline at end of file +} diff --git a/udf-runner-cpp/v1/run_local.sh b/udf-runner-cpp/v1/run_local.sh index 64486f1..5b9cd03 100644 --- a/udf-runner-cpp/v1/run_local.sh +++ b/udf-runner-cpp/v1/run_local.sh @@ -4,4 +4,4 @@ #shellcheck disable=SC1091 source .env export VERBOSE_BUILD="--subcommands --verbose_failures" -bash run.sh --define streaming=true --define python=true --define java=true --define benchmark=true --define r=true "$@" \ No newline at end of file +bash run.sh --define bash=true --define benchmark=true "$@" diff --git a/udf-runner-cpp/v1/variables.bzl b/udf-runner-cpp/v1/variables.bzl index 7448649..02a57b8 100644 --- a/udf-runner-cpp/v1/variables.bzl +++ b/udf-runner-cpp/v1/variables.bzl @@ -6,13 +6,4 @@ STREAMING_VM_ENABLED_DEFINE=select({ "@exaudfclient_base//:bash": ["ENABLE_STREAMING_VM"], "//conditions:default": [] }) -PYTHON_VM_ENABLED_DEFINE=select({ - "@exaudfclient_base//:python": ["ENABLE_PYTHON_VM"], - "//conditions:default": [] - }) -JAVA_VM_ENABLED_DEFINE=select({ - "@exaudfclient_base//:java": ["ENABLE_JAVA_VM"], - "//conditions:default": [] - }) - -VM_ENABLED_DEFINES=BENCHMARK_VM_ENABLED_DEFINE+PYTHON_VM_ENABLED_DEFINE+JAVA_VM_ENABLED_DEFINE+STREAMING_VM_ENABLED_DEFINE +VM_ENABLED_DEFINES=BENCHMARK_VM_ENABLED_DEFINE+STREAMING_VM_ENABLED_DEFINE