From 2b66372e30358b44b432037cca1afe187cbd3647 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 14 Aug 2024 06:02:25 -0300 Subject: [PATCH] #685: Update bazel (#439) related to exasol/script-languages-release#685 --- .github/workflows/check_bazel_tests.yml | 12 +- exaudfclient/{base => }/.bazelrc | 4 +- exaudfclient/{base => }/.env.template | 0 exaudfclient/{base => }/.gitignore | 4 +- exaudfclient/BUILD | 156 +++++++++++++++ exaudfclient/MODULE.bazel | 3 + exaudfclient/{base => }/README.md | 2 +- exaudfclient/base/BUILD | 180 ++---------------- exaudfclient/base/WORKSPACE | 81 -------- exaudfclient/base/base.MODULE.bazel | 49 +++++ exaudfclient/base/benchmark_container/BUILD | 2 +- .../benchmark_container.cc | 2 +- .../benchmark_container/benchmark_container.h | 8 +- exaudfclient/base/exaudfclient.template.sh | 2 +- exaudfclient/base/exaudflib/BUILD | 6 +- exaudfclient/base/exaudflib/exascript.i | 6 +- exaudfclient/base/exaudflib/impl/check.cc | 6 +- .../base/exaudflib/impl/exaudflib_main.cc | 16 +- exaudfclient/base/exaudflib/impl/global.cc | 2 +- exaudfclient/base/exaudflib/impl/global.h | 6 +- .../base/exaudflib/impl/msg_conversion.cc | 2 +- .../base/exaudflib/impl/socket_high_level.cc | 16 +- .../base/exaudflib/impl/socket_info.cc | 2 +- .../base/exaudflib/impl/socket_low_level.cc | 6 +- .../base/exaudflib/impl/swig/swig_factory.cc | 6 +- .../impl/swig/swig_general_iterator.cc | 2 +- .../impl/swig/swig_general_iterator.h | 4 +- .../exaudflib/impl/swig/swig_meta_data.cc | 2 +- .../base/exaudflib/impl/swig/swig_meta_data.h | 12 +- .../impl/swig/swig_result_handler.cc | 2 +- .../exaudflib/impl/swig/swig_result_handler.h | 14 +- .../impl/swig/swig_table_iterator.cc | 2 +- .../exaudflib/impl/swig/swig_table_iterator.h | 12 +- .../base/exaudflib/swig/swig_common.cc | 2 +- .../base/exaudflib/swig/swig_meta_data.cc | 2 +- .../base/exaudflib/swig/swig_meta_data.h | 6 +- .../exaudflib/swig/swig_result_handler.cc | 2 +- .../base/exaudflib/swig/swig_result_handler.h | 4 +- .../exaudflib/swig/swig_table_iterator.cc | 2 +- .../base/exaudflib/swig/swig_table_iterator.h | 2 +- exaudfclient/base/exaudflib/test/BUILD | 4 +- .../script_data_transfer_objects_test.cpp | 2 +- .../test/script_option_lines_test.cpp | 2 +- exaudfclient/base/exaudflib/vm/swig_vm.cc | 2 +- exaudfclient/base/exaudflib/vm/swig_vm.h | 4 +- exaudfclient/base/java_repository.bzl | 4 +- exaudfclient/base/javacontainer/BUILD | 140 +++++++------- .../base/javacontainer/javacontainer.cc | 4 +- .../base/javacontainer/javacontainer.h | 2 +- .../base/javacontainer/javacontainer_impl.cc | 14 +- .../base/javacontainer/javacontainer_impl.h | 2 +- exaudfclient/base/javacontainer/test/BUILD | 4 +- .../javacontainer/test/cpp/exaudf_wrapper.cc | 2 +- .../test/cpp/javacontainer_test.cc | 42 ++-- .../javacontainer/test/cpp/javavm_test.cc | 4 +- exaudfclient/base/load_dynamic.cc | 2 +- exaudfclient/base/protobuf_repository.bzl | 4 +- .../base/python/exascript_python_wrap.py | 2 +- exaudfclient/base/python/python3/BUILD | 38 ++-- .../python/python3/python_ext_dataframe.cc | 4 +- exaudfclient/base/python/pythoncontainer.cc | 8 +- exaudfclient/base/python/pythoncontainer.h | 2 +- exaudfclient/base/python_repository.bzl | 22 ++- exaudfclient/base/streaming_container/BUILD | 2 +- exaudfclient/base/zmq_repository.bzl | 4 +- exaudfclient/{base => }/build.sh | 0 exaudfclient/{base => }/build_local.sh | 0 exaudfclient/{base => }/build_local_all.sh | 0 .../{base => }/docs/exaudfclient.drawio | 0 exaudfclient/{base => }/docs/exaudfclient.png | Bin exaudfclient/{base => }/exaudfclient.cc | 10 +- exaudfclient/{base => }/run.sh | 0 exaudfclient/{base => }/run_local.sh | 0 exaudfclient/{base => }/variables.bzl | 0 exaudfclient/{base => }/visualize_all.sh | 0 exaudfclient/{base => }/visualize_deps.sh | 0 .../install_scripts/install_via_conda.pl | 2 +- .../base_test_build_run/Dockerfile | 6 +- .../build_deps/packages/conda_packages | 9 +- .../flavor_base/build_run/Dockerfile | 5 +- .../flavor_base/build_steps.py | 4 +- .../language_deps/packages/conda_packages | 6 - .../udfclient_deps/packages/conda_packages | 3 - .../base_test_build_run/Dockerfile | 6 +- .../build_deps/packages/conda_packages | 9 +- .../flavor_base/build_run/Dockerfile | 5 +- .../flavor_base/build_steps.py | 4 +- .../language_deps/packages/conda_packages | 6 - .../udfclient_deps/packages/conda_packages | 3 - .../base_test_build_run/Dockerfile | 6 +- .../flavor_base/build_deps/Dockerfile | 2 +- .../flavor_base/build_run/Dockerfile | 5 +- .../flavor_base/build_steps.py | 4 +- 93 files changed, 523 insertions(+), 541 deletions(-) rename exaudfclient/{base => }/.bazelrc (81%) rename exaudfclient/{base => }/.env.template (100%) rename exaudfclient/{base => }/.gitignore (59%) create mode 100644 exaudfclient/BUILD create mode 100644 exaudfclient/MODULE.bazel rename exaudfclient/{base => }/README.md (89%) delete mode 100644 exaudfclient/base/WORKSPACE create mode 100644 exaudfclient/base/base.MODULE.bazel rename exaudfclient/{base => }/build.sh (100%) rename exaudfclient/{base => }/build_local.sh (100%) rename exaudfclient/{base => }/build_local_all.sh (100%) rename exaudfclient/{base => }/docs/exaudfclient.drawio (100%) rename exaudfclient/{base => }/docs/exaudfclient.png (100%) rename exaudfclient/{base => }/exaudfclient.cc (96%) rename exaudfclient/{base => }/run.sh (100%) rename exaudfclient/{base => }/run_local.sh (100%) rename exaudfclient/{base => }/variables.bzl (100%) rename exaudfclient/{base => }/visualize_all.sh (100%) rename exaudfclient/{base => }/visualize_deps.sh (100%) diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index fb77880dc..7d6215364 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -27,12 +27,12 @@ jobs: sudo apt-get install -y openjdk-11-jdk libzmq3-dev - name: Java Tests run: | - export USE_BAZEL_VERSION=6.4.0 - bazel test //javacontainer/test/... - working-directory: ./exaudfclient/base + export USE_BAZEL_VERSION=7.2.1 + bazel test //base/javacontainer/test/... + working-directory: ./exaudfclient/ - name: ExaudfLib Tests run: | - export USE_BAZEL_VERSION=6.4.0 - bazel test //exaudflib/test/... - working-directory: ./exaudfclient/base + export USE_BAZEL_VERSION=7.2.1 + bazel test //base/exaudflib/test/... + working-directory: ./exaudfclient/ diff --git a/exaudfclient/base/.bazelrc b/exaudfclient/.bazelrc similarity index 81% rename from exaudfclient/base/.bazelrc rename to exaudfclient/.bazelrc index 1607ca871..032c4ce23 100644 --- a/exaudfclient/base/.bazelrc +++ b/exaudfclient/.bazelrc @@ -1,9 +1,9 @@ -build --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX +build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX # TODO add environment variables for R libraries build:benchmark --define benchmark=true build:java --define java=true --action_env=JAVA_PREFIX build:python --define python=true --action_env=PYTHON2_SYSPATH --action_env=PYTHON2_PREFIX --action_env=PYTHON2_VERSION --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION -build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient_py3 +build:fast-binary-py3 --copt='-DCUSTOM_LIBEXAUDFLIB_PATH="/exaudf/base/libexaudflib_complete.so"' --define binary_type=fast_binary //:exaudfclient_py3 build:slow-wrapper-py3 --define binary_type=slow_wrapper //:exaudfclient_py3 build:static-binary-py3 //:exaudfclient_py3_static build:test-binaries-py3 --config=static-binary-py3 --config=slow-wrapper-py3 diff --git a/exaudfclient/base/.env.template b/exaudfclient/.env.template similarity index 100% rename from exaudfclient/base/.env.template rename to exaudfclient/.env.template diff --git a/exaudfclient/base/.gitignore b/exaudfclient/.gitignore similarity index 59% rename from exaudfclient/base/.gitignore rename to exaudfclient/.gitignore index 3d3f25179..3876f6726 100644 --- a/exaudfclient/base/.gitignore +++ b/exaudfclient/.gitignore @@ -3,5 +3,7 @@ bazel-genfiles bazel-out bazel-src bazel-testlogs +bazel-exaudfclient graph.in -graph.png \ No newline at end of file +graph.png +MODULE.bazel.lock diff --git a/exaudfclient/BUILD b/exaudfclient/BUILD new file mode 100644 index 000000000..bc72b2f91 --- /dev/null +++ b/exaudfclient/BUILD @@ -0,0 +1,156 @@ + +load("//:variables.bzl", "VM_ENABLED_DEFINES") + +config_setting( + name = "benchmark", + define_values = {"benchmark": "true"}, +) + +config_setting( + name = "python", + define_values = {"python": "true"}, +) + +config_setting( + name = "java", + define_values = {"java": "true"}, +) + +config_setting( + name = "bash", + define_values = {"bash": "true"}, +) + +config_setting( + name = "fast_binary", + define_values = { + "binary_type": "fast_binary", + }, +) + +config_setting( + name = "slow_wrapper", + define_values = { + "binary_type": "slow_wrapper", + }, +) + +config_setting( + name = "valgrind_wrapper", + define_values = { + "wrapper_type": "valgrind_wrapper", + }, +) + +config_setting( + name = "valgrind_massif_wrapper", + define_values = { + "wrapper_type": "valgrind_massif_wrapper", + }, +) + +config_setting( + name = "stdout_to_bucketfs", + define_values = { + "wrapper_type": "stdout_to_bucketfs", + }, +) + + +VM_ENABLED_DEPS=select({ + "//:benchmark": ["//base/benchmark_container:benchmark_container"], + "//conditions:default": [] + }) + select({ + "//:java": ["//base/javacontainer:javacontainer"], + "//conditions:default": [] + }) + select({ + "//:bash": ["//base/streaming_container:streamingcontainer"], + "//conditions:default": [] + }) + +VM_PYTHON3_DEPS=select({ + "//:python": ["//base/python/python3:pythoncontainer"], + "//conditions:default": [] + }) + +cc_binary( + name = "exaudfclient_py3_bin", + srcs = ["exaudfclient.cc", "//base:load_dynamic"], + linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace + deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ + ["//base/exaudflib:exaudflib-deps"], + defines = VM_ENABLED_DEFINES, + data = ["//base:libexaudflib_complete.so"] +) + +## The purpose of the static binaries is to verify if the linker namespace test work correctly. +## It is crucial that all linker symbols of exaudflib and all it's dependency are loaded into a new linker namespace, +## so that the user can load it's own versions of those dependencies without any conflict. +## With this binary we simulate an error in our build system, that is a direct depedency to protobuf/zmq, +## which then must be detected with the linker namespace tests: +## test/linker_namespace_sanity/linker_namespace_sanity.py checks the wrong configuration +## Besides this the test under test/python3/all/linker_namespace.py checks the normal build, which expects +## not to find any occurence of the dependencies (protobuf/zmq) in the primary linker namespace. +## +## We need to explicitly declare the dependency of protobuf/zmq here, as the exaudflib is a static lib (//base/exaudflib:exaudflib) +## and hence does not contain dependency information. We cannot declare the shared lib (:exaudflib_complete.so) +## as dependency as it is a binary for bazel. + +cc_binary( + name = "exaudfclient_py3_static_bin", + srcs = ["exaudfclient.cc", "//base:load_dynamic"], + linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace + deps = ["//base/exaudflib:header", "//base:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ + ["//base/exaudflib:exaudflib-deps"] + [ "@zmq//:zmq", "@protobuf//:protobuf"], + defines = VM_ENABLED_DEFINES, + data = ["//base:libexaudflib_complete.so"], +) + +# Workarround for the hardcoded paths in exaudfclient for libexaudflib_complete.so and python_ext_dataframe.cc +# - libexaudflib_complete.so and python_ext_dataframe.cc get dynamically loaded, therefore the exaudfclient needs to know their paths +# - Most flexible way to provides these paths would environment variables +# - The exasol database can't provide these paths, because they depend on the container +# - A workarround to provide these paths would be wrapper bash script which set these environment variables +# - For performance reason, we can not wrap the binary into a shell script, as such this is only for local debugging and testing +# and we hardcode the paths for the production container + +sh_library( + name = "wrapper_generator_bin", + srcs=select({ + "//:valgrind_wrapper": ["//base:create_binary_wrapper_valgrind.sh"], + "//:valgrind_massif_wrapper": ["//base:create_binary_wrapper_valgrind_massif.sh"], + "//:stdout_to_bucketfs": ["//base:create_binary_wrapper_stdout_to_bucketfs.sh"], + "//conditions:default": ["//base:create_binary_wrapper.sh"] + }) +) + +SLOW_WRAPPER_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" "$(location //base:exaudfclient.template.sh)" """ +FAST_BINARY_PY3="""cp "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" """ +CREATE_BINARY_PY3_SCRIPT=select({ + "//:fast_binary": FAST_BINARY_PY3, + "//:slow_wrapper": SLOW_WRAPPER_BINARY_PY3, + "//conditions:default": FAST_BINARY_PY3 + }) +genrule( + name = "exaudfclient_py3", + cmd = CREATE_BINARY_PY3_SCRIPT, + outs = ["exaudfclient_py3"], + srcs = [":exaudfclient_py3_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"], + output_to_bindir = True +) + +SLOW_WRAPPER_STATIC_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" "$(location //base:exaudfclient.template.sh)" """ +FAST_BINARY_STATIC_PY3="""cp "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" """ +CREATE_STATIC_BINARY_PY3_SCRIPT=select({ + "//:fast_binary": FAST_BINARY_STATIC_PY3, + "//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY_PY3, + "//conditions:default": FAST_BINARY_STATIC_PY3 + }) + +genrule( + name = "exaudfclient_py3_static", + cmd = CREATE_STATIC_BINARY_PY3_SCRIPT, + outs = ["exaudfclient_py3_static"], + srcs = [":exaudfclient_py3_static_bin", "//base:libexaudflib_complete.so", "//base:exaudfclient.template.sh", "//:wrapper_generator_bin"], + output_to_bindir = True +) diff --git a/exaudfclient/MODULE.bazel b/exaudfclient/MODULE.bazel new file mode 100644 index 000000000..3c7561918 --- /dev/null +++ b/exaudfclient/MODULE.bazel @@ -0,0 +1,3 @@ +module(name="exaudfclient", version = "1.0") + +include("//base:base.MODULE.bazel") \ No newline at end of file diff --git a/exaudfclient/base/README.md b/exaudfclient/README.md similarity index 89% rename from exaudfclient/base/README.md rename to exaudfclient/README.md index 12538e4ef..848c6212a 100644 --- a/exaudfclient/base/README.md +++ b/exaudfclient/README.md @@ -70,7 +70,7 @@ The usage of multiple linker namespace requires some precautions in the build pr ## Precautions in the build process -In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have to main output targets //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language container live in the same namespace as //:exaudfclient. This namespace must not know anyhing about protobuf and zeromq, because it is possible that a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //exaudflib:exaudflib in //:exaudfclient or the langauge container, because this would leak zeromq and protobuf. If you need to depend on the other dependency of //exaudflib:exaudflib which not depend on protobuf or zeromq them self, such as //exaudflib:script_data_transfer_objects, //exaudflib:script_data_transfer_objects_wrapper, //exaudflib:scriptoptionlines, use either their target as self, the collection of libraries //exaudflib:exaudflib-deps or the collection of headers //exaudflib:header. +In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have to main output targets //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language container live in the same namespace as //:exaudfclient. This namespace must not know anyhing about protobuf and zeromq, because it is possible that a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //base/exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //base/exaudflib:exaudflib in //:exaudfclient or the langauge container, because this would leak zeromq and protobuf. If you need to depend on the other dependency of //base/exaudflib:exaudflib which not depend on protobuf or zeromq them self, such as //base/exaudflib:script_data_transfer_objects, //base/exaudflib:script_data_transfer_objects_wrapper, //base/exaudflib:scriptoptionlines, use either their target as self, the collection of libraries //base/exaudflib:exaudflib-deps or the collection of headers //base/exaudflib:header. ## Precautions in the implementations diff --git a/exaudfclient/base/BUILD b/exaudfclient/base/BUILD index 43de10fe0..687ba0d2a 100644 --- a/exaudfclient/base/BUILD +++ b/exaudfclient/base/BUILD @@ -1,64 +1,11 @@ + package(default_visibility = ["//visibility:public"]) exports_files(["filter_swig_code.py", "build_integrated.py", - "create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh", "create_binary_wrapper_valgrind_massif.sh"]) + "create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh", + "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh"]) load("//:variables.bzl", "VM_ENABLED_DEFINES") -config_setting( - name = "benchmark", - define_values = {"benchmark": "true"}, -) - -config_setting( - name = "python", - define_values = {"python": "true"}, -) - -config_setting( - name = "java", - define_values = {"java": "true"}, -) - -config_setting( - name = "bash", - define_values = {"bash": "true"}, -) - -config_setting( - name = "fast_binary", - define_values = { - "binary_type": "fast_binary", - }, -) - -config_setting( - name = "slow_wrapper", - define_values = { - "binary_type": "slow_wrapper", - }, -) - -config_setting( - name = "valgrind_wrapper", - define_values = { - "wrapper_type": "valgrind_wrapper", - }, -) - -config_setting( - name = "valgrind_massif_wrapper", - define_values = { - "wrapper_type": "valgrind_massif_wrapper", - }, -) - -config_setting( - name = "stdout_to_bucketfs", - define_values = { - "wrapper_type": "stdout_to_bucketfs", - }, -) - cc_library( name = "debug_message_h", hdrs = [ @@ -66,117 +13,28 @@ cc_library( ], ) +cc_library( + name = "load_dynamic", + srcs = [ + "load_dynamic.cc" + ], + deps = ["//base/exaudflib:header", "//base:debug_message_h", "//base/exaudflib:exaudflib-deps"], + defines = VM_ENABLED_DEFINES, +) + + # Using cc_binary to build a shared library as output target is a workaround, because -# Bazel's cc_library are only intermediate stages and can be used as output target. -# It is necessary to include //exaudflib:exaudflib into deps and srcs, because -# otherwise dlmopen won't find the symbols of exaudflib and its dependencies. The target //exaudflib:exaudflib-deps +# Bazel's cc_library are only intermediate stages and can't be used as output target. +# It is necessary to include //base/exaudflib:exaudflib into deps and srcs, because +# otherwise dlmopen won't find the symbols of exaudflib and its dependencies. The target //base/exaudflib:exaudflib-deps # won't work either, because it only contains the libraries as dependencies. # Bazel builts a static library (.a) and a dynamic library (.so) out of exaudflib:exaudflib. # In the libexaudflib_complete.so-2.params, we saw that Bazel uses both libraries to built libexaudflib_complete.so. -# Experiments have shown that we get missing symbols during loading libexaudflib_complete.so, if we only built the static library. +# Experiments have shown that we get missing symbols during loading libexaudflib_complete.so, if we only built the static library. cc_binary( name = "libexaudflib_complete.so", linkshared = 1, - srcs = ["//exaudflib:exaudflib"], - deps = ["//exaudflib:exaudflib"], - defines = VM_ENABLED_DEFINES, -) - - -VM_ENABLED_DEPS=select({ - ":benchmark": ["//benchmark_container:benchmark_container"], - "//conditions:default": [] - }) + select({ - ":java": ["//javacontainer:javacontainer"], - "//conditions:default": [] - }) + select({ - ":bash": ["//streaming_container:streamingcontainer"], - "//conditions:default": [] - }) - -VM_PYTHON3_DEPS=select({ - ":python": ["//python/python3:pythoncontainer"], - "//conditions:default": [] - }) - -cc_binary( - name = "exaudfclient_py3_bin", - srcs = ["exaudfclient.cc", "load_dynamic.cc"], - linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace - deps = ["//exaudflib:header", "//:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ - ["//exaudflib:exaudflib-deps"], - defines = VM_ENABLED_DEFINES, - data = [":libexaudflib_complete.so"] -) - -## The purpose of the static binaries is to verify if the linker namespace test work correctly. -## It is crucial that all linker symbols of exaudflib and all it's dependency are loaded into a new linker namespace, -## so that the user can load it's own versions of those dependencies without any conflict. -## With this binary we simulate an error in our build system, that is a direct depedency to protobuf/zmq, -## which then must be detected with the linker namespace tests: -## test/linker_namespace_sanity/linker_namespace_sanity.py checks the wrong configuration -## Besides this the test under test/python3/all/linker_namespace.py checks the normal build, which expects -## not to find any occurence of the dependencies (protobuf/zmq) in the primary linker namespace. -## -## We need to explicitly declare the dependency of protobuf/zmq here, as the exaudflib is a static lib (//exaudflib:exaudflib) -## and hence does not contain dependency information. We cannot declare the shared lib (:exaudflib_complete.so) -## as dependency as it is a binary for bazel. - -cc_binary( - name = "exaudfclient_py3_static_bin", - srcs = ["exaudfclient.cc", "load_dynamic.cc"], - linkopts = ["-ldl"], # needed for dynamicly loading libexaudflib_complete.so into another linker namespace - deps = ["//exaudflib:header", "//:debug_message_h"]+VM_ENABLED_DEPS+VM_PYTHON3_DEPS+ - ["//exaudflib:exaudflib-deps"] + [ "@zmq//:zmq", "@protobuf//:protobuf"], + srcs = ["//base/exaudflib:exaudflib"], + deps = ["//base/exaudflib:exaudflib"], defines = VM_ENABLED_DEFINES, - data = [":libexaudflib_complete.so"], -) - -# Workarround for the hardcoded paths in exaudfclient for libexaudflib_complete.so and python_ext_dataframe.cc -# - libexaudflib_complete.so and python_ext_dataframe.cc get dynamically loaded, therefore the exaudfclient needs to know their paths -# - Most flexible way to provides these paths would environment variables -# - The exasol database can't provide these paths, because they depend on the container -# - A workarround to provide these paths would be wrapper bash script which set these environment variables -# - For performance reason, we can not wrap the binary into a shell script, as such this is only for local debugging and testing -# and we hardcode the paths for the production container - -sh_library( - name = "wrapper_generator_bin", - srcs=select({ - "//:valgrind_wrapper": ["//:create_binary_wrapper_valgrind.sh"], - "//:valgrind_massif_wrapper": ["//:create_binary_wrapper_valgrind_massif.sh"], - "//:stdout_to_bucketfs": ["//:create_binary_wrapper_stdout_to_bucketfs.sh"], - "//conditions:default": ["//:create_binary_wrapper.sh"] - }) -) - -SLOW_WRAPPER_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" "$(location exaudfclient.template.sh)" """ -FAST_BINARY_PY3="""cp "$(location exaudfclient_py3_bin)" "$(location exaudfclient_py3)" """ -CREATE_BINARY_PY3_SCRIPT=select({ - "//:fast_binary": FAST_BINARY_PY3, - "//:slow_wrapper": SLOW_WRAPPER_BINARY_PY3, - "//conditions:default": FAST_BINARY_PY3 - }) -genrule( - name = "exaudfclient_py3", - cmd = CREATE_BINARY_PY3_SCRIPT, - outs = ["exaudfclient_py3"], - srcs = [":exaudfclient_py3_bin", "//:libexaudflib_complete.so", "exaudfclient.template.sh", "//:wrapper_generator_bin"], - output_to_bindir = True -) - -SLOW_WRAPPER_STATIC_BINARY_PY3="""$(location //:wrapper_generator_bin) "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" "$(location exaudfclient.template.sh)" """ -FAST_BINARY_STATIC_PY3="""cp "$(location exaudfclient_py3_static_bin)" "$(location exaudfclient_py3_static)" """ -CREATE_STATIC_BINARY_PY3_SCRIPT=select({ - "//:fast_binary": FAST_BINARY_STATIC_PY3, - "//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY_PY3, - "//conditions:default": FAST_BINARY_STATIC_PY3 - }) - -genrule( - name = "exaudfclient_py3_static", - cmd = CREATE_STATIC_BINARY_PY3_SCRIPT, - outs = ["exaudfclient_py3_static"], - srcs = [":exaudfclient_py3_static_bin", "//:libexaudflib_complete.so", "exaudfclient.template.sh", "//:wrapper_generator_bin"], - output_to_bindir = True ) diff --git a/exaudfclient/base/WORKSPACE b/exaudfclient/base/WORKSPACE deleted file mode 100644 index a5f73eaf7..000000000 --- a/exaudfclient/base/WORKSPACE +++ /dev/null @@ -1,81 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -skylib_version = "0.8.0" - -http_archive( - name = "bazel_skylib", - sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", - type = "tar.gz", - url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format(skylib_version, skylib_version), -) - -RULES_JVM_EXTERNAL_TAG = "4.2" -RULES_JVM_EXTERNAL_SHA = "cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca" - -http_archive( - name = "rules_jvm_external", - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, - sha256 = RULES_JVM_EXTERNAL_SHA, - url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, -) - - -load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") - -rules_jvm_external_deps() - -load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") - -rules_jvm_external_setup() - -load("@rules_jvm_external//:defs.bzl", "maven_install") - -maven_install( - artifacts = [ - "com.exasol:udf-api-java:1.0.5", - ], - repositories = [ - "https://repo1.maven.org/maven2", - ], -) - -load("//:python_repository.bzl", "python_local_repository") - -python_local_repository(name = "python3") - -load("//:python_repository.bzl", "numpy_local_repository") - -numpy_local_repository(name = "numpy") - -load("//:java_repository.bzl", "java_local_repository") - -java_local_repository(name = "java") - -load("//:protobuf_repository.bzl", "protobuf_local_repository") - -protobuf_local_repository(name = "protobuf") - -load("//:zmq_repository.bzl", "zmq_local_repository") -zmq_local_repository(name = "zmq") - -new_local_repository( - name = "ssl", - build_file_content = """ -cc_library( - name = "ssl", - srcs = ["usr/lib/x86_64-linux-gnu/libssl.so","usr/lib/x86_64-linux-gnu/libcrypto.so"], - hdrs = glob(["usr/include/openssl/*.h"]), - includes = ["usr/include/openssl"], - visibility = ["//visibility:public"] -) - """, - path = "/", -) - -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -git_repository( - name = "googletest", - remote = "https://github.com/google/googletest", - tag = "release-1.11.0", -) diff --git a/exaudfclient/base/base.MODULE.bazel b/exaudfclient/base/base.MODULE.bazel new file mode 100644 index 000000000..0dc34e5df --- /dev/null +++ b/exaudfclient/base/base.MODULE.bazel @@ -0,0 +1,49 @@ +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "googletest", version = "1.15.0") +bazel_dep(name = "rules_java", version = "6.1.1") +bazel_dep(name = "rules_jvm_external", version = "6.2") + +python_local_repository = use_repo_rule("//base:python_repository.bzl", "python_local_repository") +python_local_repository(name = "python3", python_version="python3") + +numpy_local_repository = use_repo_rule("//base:python_repository.bzl", "numpy_local_repository") + +numpy_local_repository(name = "numpy") + +zmq_local_repository = use_repo_rule("//base:zmq_repository.bzl", "zmq_local_repository") +zmq_local_repository(name = "zmq") + +protobuf_local_repository = use_repo_rule("//base:protobuf_repository.bzl", "protobuf_local_repository") +protobuf_local_repository(name = "protobuf") + + +new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +new_local_repository( + name = "ssl", + build_file_content = """ +cc_library( + name = "ssl", + srcs = ["usr/lib/x86_64-linux-gnu/libssl.so","usr/lib/x86_64-linux-gnu/libcrypto.so"], + hdrs = glob(["usr/include/openssl/*.h"]), + includes = ["usr/include/openssl"], + visibility = ["//visibility:public"] +) + """, + path = "/", +) + + +java_local_repository = use_repo_rule("//base:java_repository.bzl", "java_local_repository") +java_local_repository(name = "java") + +maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") +maven.install( + artifacts = [ + "com.exasol:udf-api-java:1.0.5", + ], + repositories = [ + "https://repo1.maven.org/maven2", + ], +) +use_repo(maven, "maven") diff --git a/exaudfclient/base/benchmark_container/BUILD b/exaudfclient/base/benchmark_container/BUILD index 7d544bdb0..dcdffdcdd 100644 --- a/exaudfclient/base/benchmark_container/BUILD +++ b/exaudfclient/base/benchmark_container/BUILD @@ -4,5 +4,5 @@ cc_library( name = "benchmark_container", srcs = ["benchmark_container.cc","benchmark_container.h"], hdrs = ["benchmark_container.h"], - deps = ["//exaudflib:exaudflib-deps","//exaudflib:header", "//:debug_message_h"]+["//exaudflib:scriptoptionlines"] + deps = ["//base/exaudflib:exaudflib-deps","//base/exaudflib:header", "//base:debug_message_h"]+["//base/exaudflib:scriptoptionlines"] ) \ No newline at end of file diff --git a/exaudfclient/base/benchmark_container/benchmark_container.cc b/exaudfclient/base/benchmark_container/benchmark_container.cc index bf4b5f562..3efa97cd0 100644 --- a/exaudfclient/base/benchmark_container/benchmark_container.cc +++ b/exaudfclient/base/benchmark_container/benchmark_container.cc @@ -2,7 +2,7 @@ #define ENABLE_BENCHMARK_VM #endif -#include "debug_message.h" +#include "base/debug_message.h" #include "benchmark_container.h" #include #include diff --git a/exaudfclient/base/benchmark_container/benchmark_container.h b/exaudfclient/base/benchmark_container/benchmark_container.h index e107f4b34..eacf1f153 100644 --- a/exaudfclient/base/benchmark_container/benchmark_container.h +++ b/exaudfclient/base/benchmark_container/benchmark_container.h @@ -1,7 +1,7 @@ -#include "exaudflib/vm/swig_vm.h" -#include "exaudflib/swig/swig_meta_data.h" -#include "exaudflib/swig/swig_result_handler.h" -#include "exaudflib/swig/swig_table_iterator.h" +#include "base/exaudflib/vm/swig_vm.h" +#include "base/exaudflib/swig/swig_meta_data.h" +#include "base/exaudflib/swig/swig_result_handler.h" +#include "base/exaudflib/swig/swig_table_iterator.h" using namespace SWIGVMContainers; using namespace std; diff --git a/exaudfclient/base/exaudfclient.template.sh b/exaudfclient/base/exaudfclient.template.sh index edd33c565..5e93af145 100644 --- a/exaudfclient/base/exaudfclient.template.sh +++ b/exaudfclient/base/exaudfclient.template.sh @@ -3,5 +3,5 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" echo "Changing to script directory $SCRIPT_DIR" cd "$SCRIPT_DIR" || return 1 -export LIBEXAUDFLIB_PATH="$SCRIPT_DIR/libexaudflib_complete.so" +export LIBEXAUDFLIB_PATH="$SCRIPT_DIR/base/libexaudflib_complete.so" export LIBPYEXADATAFRAME_DIR="$SCRIPT_DIR/python/python3" diff --git a/exaudfclient/base/exaudflib/BUILD b/exaudfclient/base/exaudflib/BUILD index 81dfd335e..d82726c02 100644 --- a/exaudfclient/base/exaudflib/BUILD +++ b/exaudfclient/base/exaudflib/BUILD @@ -82,8 +82,8 @@ cc_library( # The only target which must depend on it is //:libexaudflib-complete.so. # If you depend on this target you will load protobuf into your linker namespace # which might cause problems with tensorflow. If you need -# "//exaudflib:scriptoptionlines","//exaudflib:script_data_transfer_objects" or -# "//exaudflib:script_data_transfer_objects_wrapper" in the language container +# "//base/exaudflib:scriptoptionlines","//base/exaudflib:script_data_transfer_objects" or +# "//base/exaudflib:script_data_transfer_objects_wrapper" in the language container # depend directly on them or depend on exaudflib-deps cc_library( name = "exaudflib", @@ -103,5 +103,5 @@ cc_library( "impl/swig/swig_general_iterator.h", "impl/swig/swig_general_iterator.cc"], defines = VM_ENABLED_DEFINES, linkstatic = False, # Needs to be false, because otherwise we might get missing symbols when loading exaudflib_complete.so - deps = [":exaudflib-deps", ":zmqcontainer", "@zmq//:zmq", ":header", "//:debug_message_h"], + deps = [":exaudflib-deps", ":zmqcontainer", "@zmq//:zmq", ":header", "//base:debug_message_h"], ) diff --git a/exaudfclient/base/exaudflib/exascript.i b/exaudfclient/base/exaudflib/exascript.i index bc6abf9bd..9838e88a5 100644 --- a/exaudfclient/base/exaudflib/exascript.i +++ b/exaudfclient/base/exaudflib/exascript.i @@ -3,11 +3,11 @@ %} %{ -#include "exaudflib/swig/swig_result_handler.h" -#include "exaudflib/swig/swig_meta_data.h" +#include "base/exaudflib/swig/swig_result_handler.h" +#include "base/exaudflib/swig/swig_meta_data.h" using namespace SWIGVMContainers; -#include "exaudflib/swig/script_data_transfer_objects_wrapper.h" +#include "base/exaudflib/swig/script_data_transfer_objects_wrapper.h" %} %ignore ExecutionGraph::ConnectionInformationWrapper::ConnectionInformationWrapper; diff --git a/exaudfclient/base/exaudflib/impl/check.cc b/exaudfclient/base/exaudflib/impl/check.cc index 3dc1152f7..44ca9345b 100644 --- a/exaudfclient/base/exaudflib/impl/check.cc +++ b/exaudfclient/base/exaudflib/impl/check.cc @@ -1,12 +1,12 @@ -#include "exaudflib/impl/check.h" -#include "exaudflib/impl/socket_info.h" +#include "base/exaudflib/impl/check.h" +#include "base/exaudflib/impl/socket_info.h" #include #include #include #include -#include "debug_message.h" +#include "base/debug_message.h" namespace exaudflib { namespace check { diff --git a/exaudfclient/base/exaudflib/impl/exaudflib_main.cc b/exaudfclient/base/exaudflib/impl/exaudflib_main.cc index 135e2445a..cd74f2405 100644 --- a/exaudfclient/base/exaudflib/impl/exaudflib_main.cc +++ b/exaudfclient/base/exaudflib/impl/exaudflib_main.cc @@ -10,20 +10,20 @@ #include #include -#include "debug_message.h" +#include "base/debug_message.h" // swig lib #include -#include "exaudflib/impl/check.h" -#include "exaudflib/impl/socket_info.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/impl/msg_conversion.h" -#include "exaudflib/impl/global.h" -#include "exaudflib/impl/socket_high_level.h" +#include "base/exaudflib/impl/check.h" +#include "base/exaudflib/impl/socket_info.h" +#include "base/exaudflib/impl/socket_low_level.h" +#include "base/exaudflib/impl/msg_conversion.h" +#include "base/exaudflib/impl/global.h" +#include "base/exaudflib/impl/socket_high_level.h" -#include "exaudflib/vm/swig_vm.h" +#include "base/exaudflib/vm/swig_vm.h" #ifndef NDEBUG diff --git a/exaudfclient/base/exaudflib/impl/global.cc b/exaudfclient/base/exaudflib/impl/global.cc index d5f570b1f..d29100bef 100644 --- a/exaudfclient/base/exaudflib/impl/global.cc +++ b/exaudfclient/base/exaudflib/impl/global.cc @@ -1,4 +1,4 @@ -#include "exaudflib/impl/global.h" +#include "base/exaudflib/impl/global.h" #include namespace exaudflib { diff --git a/exaudfclient/base/exaudflib/impl/global.h b/exaudfclient/base/exaudflib/impl/global.h index 61e0b8ccf..09402eb1b 100644 --- a/exaudfclient/base/exaudflib/impl/global.h +++ b/exaudfclient/base/exaudflib/impl/global.h @@ -1,9 +1,9 @@ #ifndef EXAUDFCLIENT_EXUDFLIB_GLOBAL_H #define EXAUDFCLIENT_EXUDFLIB_GLOBAL_H -#include "exaudflib/swig/script_data_transfer_objects.h" -#include "exaudflib/zmqcontainer.pb.h" -#include "exaudflib/swig/swig_common.h" +#include "base/exaudflib/swig/script_data_transfer_objects.h" +#include "base/exaudflib/zmqcontainer.pb.h" +#include "base/exaudflib/swig/swig_common.h" #include namespace exaudflib { diff --git a/exaudfclient/base/exaudflib/impl/msg_conversion.cc b/exaudfclient/base/exaudflib/impl/msg_conversion.cc index c09ee024d..654ac0f50 100644 --- a/exaudfclient/base/exaudflib/impl/msg_conversion.cc +++ b/exaudfclient/base/exaudflib/impl/msg_conversion.cc @@ -1,4 +1,4 @@ -#include "exaudflib/impl/msg_conversion.h" +#include "base/exaudflib/impl/msg_conversion.h" std::string exaudflib::msg_conversion::convert_message_type_to_string(int message_type){ switch (message_type) diff --git a/exaudfclient/base/exaudflib/impl/socket_high_level.cc b/exaudfclient/base/exaudflib/impl/socket_high_level.cc index 4f50ecbe8..64721cef2 100644 --- a/exaudfclient/base/exaudflib/impl/socket_high_level.cc +++ b/exaudfclient/base/exaudflib/impl/socket_high_level.cc @@ -1,13 +1,13 @@ -#include "exaudflib/impl/socket_high_level.h" +#include "base/exaudflib/impl/socket_high_level.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/zmqcontainer.pb.h" -#include "exaudflib/impl/msg_conversion.h" -#include "exaudflib/impl/global.h" -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/vm/swig_vm.h" +#include "base/exaudflib/impl/socket_low_level.h" +#include "base/exaudflib/zmqcontainer.pb.h" +#include "base/exaudflib/impl/msg_conversion.h" +#include "base/exaudflib/impl/global.h" +#include "base/exaudflib/swig/swig_common.h" +#include "base/exaudflib/vm/swig_vm.h" #include -#include "debug_message.h" +#include "base/debug_message.h" namespace exaudflib { namespace socket_high_level { diff --git a/exaudfclient/base/exaudflib/impl/socket_info.cc b/exaudfclient/base/exaudflib/impl/socket_info.cc index 9df03c255..1c94ff637 100644 --- a/exaudfclient/base/exaudflib/impl/socket_info.cc +++ b/exaudfclient/base/exaudflib/impl/socket_info.cc @@ -1,4 +1,4 @@ -#include "exaudflib/impl/socket_info.h" +#include "base/exaudflib/impl/socket_info.h" namespace exaudflib { diff --git a/exaudfclient/base/exaudflib/impl/socket_low_level.cc b/exaudfclient/base/exaudflib/impl/socket_low_level.cc index 26aca38ab..5bd14a30a 100644 --- a/exaudfclient/base/exaudflib/impl/socket_low_level.cc +++ b/exaudfclient/base/exaudflib/impl/socket_low_level.cc @@ -1,6 +1,6 @@ -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/impl/check.h" -#include "debug_message.h" +#include "base/exaudflib/impl/socket_low_level.h" +#include "base/exaudflib/impl/check.h" +#include "base/debug_message.h" #include #include #include diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc b/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc index 88af711db..67f6d3744 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc +++ b/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc @@ -1,6 +1,6 @@ -#include "exaudflib/impl/swig/swig_meta_data.h" -#include "exaudflib/impl/swig/swig_table_iterator.h" -#include "exaudflib/impl/swig/swig_result_handler.h" +#include "base/exaudflib/impl/swig/swig_meta_data.h" +#include "base/exaudflib/impl/swig/swig_table_iterator.h" +#include "base/exaudflib/impl/swig/swig_result_handler.h" extern "C" { diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.cc b/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.cc index ec1ffcf3b..d5e51689e 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.cc +++ b/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.cc @@ -1,3 +1,3 @@ //Check that swig_generial_iterator.h is self-contained -#include "exaudflib/impl/swig/swig_general_iterator.h" +#include "base/exaudflib/impl/swig/swig_general_iterator.h" diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.h b/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.h index 781274c6f..7521955f9 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.h +++ b/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.h @@ -1,8 +1,8 @@ #ifndef EXAUDFCLIENT_SWIGGENERALITERATOR_H #define EXAUDFCLIENT_SWIGGENERALITERATOR_H -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/impl/global.h" +#include "base/exaudflib/swig/swig_common.h" +#include "base/exaudflib/impl/global.h" namespace SWIGVMContainers { diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.cc b/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.cc index 0a100b72d..016dbca7e 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.cc +++ b/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.cc @@ -1,3 +1,3 @@ //Check that swig_meta_data.h is self-contained -#include "exaudflib/impl/swig/swig_meta_data.h" +#include "base/exaudflib/impl/swig/swig_meta_data.h" diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h b/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h index 23e9f7cd8..bebb5612f 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h +++ b/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h @@ -1,12 +1,12 @@ #ifndef EXAUDFCLIENT_SWIGMETADATA_H #define EXAUDFCLIENT_SWIGMETADATA_H -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/swig_meta_data.h" -#include "exaudflib/zmqcontainer.pb.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/impl/msg_conversion.h" -#include "exaudflib/impl/global.h" +#include "base/exaudflib/swig/swig_common.h" +#include "base/exaudflib/swig/swig_meta_data.h" +#include "base/exaudflib/zmqcontainer.pb.h" +#include "base/exaudflib/impl/socket_low_level.h" +#include "base/exaudflib/impl/msg_conversion.h" +#include "base/exaudflib/impl/global.h" #include #include #include diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.cc b/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.cc index 987b5e3ec..8a16c0221 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.cc +++ b/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.cc @@ -1,3 +1,3 @@ //Check that swig_result_handler.h is self-contained -#include "exaudflib/impl/swig/swig_result_handler.h" +#include "base/exaudflib/impl/swig/swig_result_handler.h" diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.h b/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.h index cb143db9f..9fd2c2c85 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.h +++ b/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.h @@ -1,13 +1,13 @@ #ifndef EXAUDFCLIENT_SWIGRESULTHANDLER_H #define EXAUDFCLIENT_SWIGRESULTHANDLER_H -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/swig_result_handler.h" -#include "exaudflib/impl/swig/swig_general_iterator.h" -#include "exaudflib/impl/global.h" -#include "exaudflib/impl/msg_conversion.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/zmqcontainer.pb.h" +#include "base/exaudflib/swig/swig_common.h" +#include "base/exaudflib/swig/swig_result_handler.h" +#include "base/exaudflib/impl/swig/swig_general_iterator.h" +#include "base/exaudflib/impl/global.h" +#include "base/exaudflib/impl/msg_conversion.h" +#include "base/exaudflib/impl/socket_low_level.h" +#include "base/exaudflib/zmqcontainer.pb.h" namespace SWIGVMContainers { diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.cc b/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.cc index 853e79e82..d45ee4711 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.cc +++ b/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.cc @@ -1,3 +1,3 @@ //Check that swig_table_iterator.h is self-contained -#include "exaudflib/impl/swig/swig_table_iterator.h" +#include "base/exaudflib/impl/swig/swig_table_iterator.h" diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.h b/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.h index bf5f014ba..3ebddecc9 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.h +++ b/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.h @@ -1,12 +1,12 @@ #ifndef EXAUDFCLIENT_SWIGTABLEITERATOR_H #define EXAUDFCLIENT_SWIGTABLEITERATOR_H -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/swig_table_iterator.h" -#include "exaudflib/impl/swig/swig_general_iterator.h" -#include "exaudflib/impl/global.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/impl/msg_conversion.h" +#include "base/exaudflib/swig/swig_common.h" +#include "base/exaudflib/swig/swig_table_iterator.h" +#include "base/exaudflib/impl/swig/swig_general_iterator.h" +#include "base/exaudflib/impl/global.h" +#include "base/exaudflib/impl/socket_low_level.h" +#include "base/exaudflib/impl/msg_conversion.h" namespace SWIGVMContainers { diff --git a/exaudfclient/base/exaudflib/swig/swig_common.cc b/exaudfclient/base/exaudflib/swig/swig_common.cc index ac2f69f2b..c65009b8f 100644 --- a/exaudfclient/base/exaudflib/swig/swig_common.cc +++ b/exaudfclient/base/exaudflib/swig/swig_common.cc @@ -1,2 +1,2 @@ //Check that swig_common.h is self-contained -#include "exaudflib/swig/swig_common.h" \ No newline at end of file +#include "base/exaudflib/swig/swig_common.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_meta_data.cc b/exaudfclient/base/exaudflib/swig/swig_meta_data.cc index 1b1c6c76f..a85ea9ff9 100644 --- a/exaudfclient/base/exaudflib/swig/swig_meta_data.cc +++ b/exaudfclient/base/exaudflib/swig/swig_meta_data.cc @@ -1,2 +1,2 @@ //Check that swig_meta_data.h is self-contained -#include "exaudflib/swig/swig_meta_data.h" \ No newline at end of file +#include "base/exaudflib/swig/swig_meta_data.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_meta_data.h b/exaudfclient/base/exaudflib/swig/swig_meta_data.h index 9f23f06f4..cad380dc2 100644 --- a/exaudfclient/base/exaudflib/swig/swig_meta_data.h +++ b/exaudfclient/base/exaudflib/swig/swig_meta_data.h @@ -1,9 +1,9 @@ #ifndef SWIG_META_DATA_H #define SWIG_META_DATA_H -#include "exaudflib/load_dynamic.h" -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/script_data_transfer_objects_wrapper.h" +#include "base/exaudflib/load_dynamic.h" +#include "base/exaudflib/swig/swig_common.h" +#include "base/exaudflib/swig/script_data_transfer_objects_wrapper.h" namespace SWIGVMContainers { diff --git a/exaudfclient/base/exaudflib/swig/swig_result_handler.cc b/exaudfclient/base/exaudflib/swig/swig_result_handler.cc index 72365f498..952b32557 100644 --- a/exaudfclient/base/exaudflib/swig/swig_result_handler.cc +++ b/exaudfclient/base/exaudflib/swig/swig_result_handler.cc @@ -1,2 +1,2 @@ //Check that swig_result_handler.h is self-contained -#include "exaudflib/swig/swig_result_handler.h" \ No newline at end of file +#include "base/exaudflib/swig/swig_result_handler.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_result_handler.h b/exaudfclient/base/exaudflib/swig/swig_result_handler.h index 546bfb25f..7140a59a6 100644 --- a/exaudfclient/base/exaudflib/swig/swig_result_handler.h +++ b/exaudfclient/base/exaudflib/swig/swig_result_handler.h @@ -4,8 +4,8 @@ #include #include #include -#include "exaudflib/load_dynamic.h" -#include "exaudflib/swig/swig_table_iterator.h" +#include "base/exaudflib/load_dynamic.h" +#include "base/exaudflib/swig/swig_table_iterator.h" #define SWIG_MAX_VAR_DATASIZE 4000000 diff --git a/exaudfclient/base/exaudflib/swig/swig_table_iterator.cc b/exaudfclient/base/exaudflib/swig/swig_table_iterator.cc index 2e30676c0..9bb159ca7 100644 --- a/exaudfclient/base/exaudflib/swig/swig_table_iterator.cc +++ b/exaudfclient/base/exaudflib/swig/swig_table_iterator.cc @@ -1,2 +1,2 @@ //Check that swig_table_iterator.h is self-contained -#include "exaudflib/swig/swig_table_iterator.h" \ No newline at end of file +#include "base/exaudflib/swig/swig_table_iterator.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_table_iterator.h b/exaudfclient/base/exaudflib/swig/swig_table_iterator.h index 89f854296..95a31cbac 100644 --- a/exaudfclient/base/exaudflib/swig/swig_table_iterator.h +++ b/exaudfclient/base/exaudflib/swig/swig_table_iterator.h @@ -4,7 +4,7 @@ #include #include #include -#include "exaudflib/load_dynamic.h" +#include "base/exaudflib/load_dynamic.h" namespace SWIGVMContainers { diff --git a/exaudfclient/base/exaudflib/test/BUILD b/exaudfclient/base/exaudflib/test/BUILD index f6c58a753..231096441 100644 --- a/exaudfclient/base/exaudflib/test/BUILD +++ b/exaudfclient/base/exaudflib/test/BUILD @@ -2,8 +2,8 @@ cc_test( name = "exaudflib-test", srcs = ["script_data_transfer_objects_test.cpp", "script_option_lines_test.cpp"], deps = [ - "//exaudflib:script_data_transfer_objects", - "//exaudflib:scriptoptionlines", + "//base/exaudflib:script_data_transfer_objects", + "//base/exaudflib:scriptoptionlines", "@googletest//:gtest_main", ], ) \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/test/script_data_transfer_objects_test.cpp b/exaudfclient/base/exaudflib/test/script_data_transfer_objects_test.cpp index 6e146c3eb..8ef9b96a4 100644 --- a/exaudfclient/base/exaudflib/test/script_data_transfer_objects_test.cpp +++ b/exaudfclient/base/exaudflib/test/script_data_transfer_objects_test.cpp @@ -1,4 +1,4 @@ -#include "exaudflib/swig/script_data_transfer_objects.h" +#include "base/exaudflib/swig/script_data_transfer_objects.h" #include using namespace ExecutionGraph; diff --git a/exaudfclient/base/exaudflib/test/script_option_lines_test.cpp b/exaudfclient/base/exaudflib/test/script_option_lines_test.cpp index a83deb18a..5866169a7 100644 --- a/exaudfclient/base/exaudflib/test/script_option_lines_test.cpp +++ b/exaudfclient/base/exaudflib/test/script_option_lines_test.cpp @@ -1,4 +1,4 @@ -#include "exaudflib/vm/scriptoptionlines.h" +#include "base/exaudflib/vm/scriptoptionlines.h" #include #include #include diff --git a/exaudfclient/base/exaudflib/vm/swig_vm.cc b/exaudfclient/base/exaudflib/vm/swig_vm.cc index 3e2d98f5a..c21533ee6 100644 --- a/exaudfclient/base/exaudflib/vm/swig_vm.cc +++ b/exaudfclient/base/exaudflib/vm/swig_vm.cc @@ -1,2 +1,2 @@ //Check that swig_vm.h is self-contained -#include "exaudflib/vm/swig_vm.h" \ No newline at end of file +#include "base/exaudflib/vm/swig_vm.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/vm/swig_vm.h b/exaudfclient/base/exaudflib/vm/swig_vm.h index 3f88c7a77..fcac97312 100644 --- a/exaudfclient/base/exaudflib/vm/swig_vm.h +++ b/exaudfclient/base/exaudflib/vm/swig_vm.h @@ -5,8 +5,8 @@ #include #include #include -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/script_data_transfer_objects_wrapper.h" +#include "base/exaudflib/swig/swig_common.h" +#include "base/exaudflib/swig/script_data_transfer_objects_wrapper.h" namespace SWIGVMContainers { diff --git a/exaudfclient/base/java_repository.bzl b/exaudfclient/base/java_repository.bzl index f4a0d34c5..76cdb7436 100644 --- a/exaudfclient/base/java_repository.bzl +++ b/exaudfclient/base/java_repository.bzl @@ -33,14 +33,14 @@ def _java_local_repository_impl(repository_ctx): defines = '"ENABLE_JAVA_VM"' build_file_content = """ cc_library( - name = "{name}", + name = "java", srcs = glob(["{prefix}/include/*.h"], allow_empty=False), hdrs = glob(["{prefix}/include/*.h","{prefix}/include/linux/*.h"], allow_empty=False), includes = ["{prefix}/include","{prefix}/include/linux"], defines = [{defines}], linkopts = ["-ljvm","-L{rpath_libjvm}",'-Wl,-rpath','{rpath_libjvm}'], visibility = ["//visibility:public"] -)""".format( name=repository_ctx.name, defines=defines, prefix="java", rpath_libjvm=path_to_libjvm) +)""".format( defines=defines, prefix="java", rpath_libjvm=path_to_libjvm) print(build_file_content) repository_ctx.symlink(prefix, "./java") diff --git a/exaudfclient/base/javacontainer/BUILD b/exaudfclient/base/javacontainer/BUILD index 47c15af73..b5224f1ba 100644 --- a/exaudfclient/base/javacontainer/BUILD +++ b/exaudfclient/base/javacontainer/BUILD @@ -3,94 +3,98 @@ package(default_visibility = ["//visibility:public"]) genrule( name = "exascript_java_tmp_cc", cmd = """ - mkdir -p $(location java_src)/com/exasol/swig + mkdir -p java_src/com/exasol/swig mkdir -p build_exascript_java_tmp_cc/exaudflib - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_java_tmp_cc/exaudflib - cd build_exascript_java_tmp_cc - swig -v -O -DEXTERNAL_PROCESS -Wall -c++ -java -addextern -module exascript_java -package com.exasol.swig -outdir "../$(location java_src)/com/exasol/swig" -o "../$(location exascript_java_tmp.cc)" exaudflib/exascript.i - """, - outs = ["exascript_java_tmp.cc","java_src"], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h"] + cp "$(location //base/exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //base/exaudflib:exascript.i)" build_exascript_java_tmp_cc/exaudflib + cd build_exascript_java_tmp_cc + swig -v -O -DEXTERNAL_PROCESS -Wall -c++ -java -addextern -module exascript_java -package com.exasol.swig -outdir "../java_src/com/exasol/swig" -o "../exascript_java_tmp.cc" exaudflib/exascript.i + cd .. + cp java_src/com/exasol/swig/ConnectionInformationWrapper.java $(location ConnectionInformationWrapper.java) + cp java_src/com/exasol/swig/exascript_java.java $(location exascript_java.java) + cp java_src/com/exasol/swig/exascript_javaJNI.java $(location raw/exascript_javaJNI.java) + cp java_src/com/exasol/swig/ExportSpecificationWrapper.java $(location ExportSpecificationWrapper.java) + cp java_src/com/exasol/swig/ImportSpecificationWrapper.java $(location ImportSpecificationWrapper.java) + cp java_src/com/exasol/swig/Metadata.java $(location Metadata.java) + cp java_src/com/exasol/swig/ResultHandler.java $(location raw/ResultHandler.java) + cp java_src/com/exasol/swig/SWIGTYPE_p_ExecutionGraph__ExportSpecification.java $(location SWIGTYPE_p_ExecutionGraph__ExportSpecification.java) + cp java_src/com/exasol/swig/SWIGTYPE_p_ExecutionGraph__ImportSpecification.java $(location SWIGTYPE_p_ExecutionGraph__ImportSpecification.java) + cp java_src/com/exasol/swig/SWIGVM_datatype_e.java $(location SWIGVM_datatype_e.java) + cp java_src/com/exasol/swig/SWIGVM_itertype_e.java $(location SWIGVM_itertype_e.java) + cp java_src/com/exasol/swig/TableIterator.java $(location raw/TableIterator.java) + cp exascript_java_tmp.cc $(location exascript_java_tmp.cc) + """, + outs = ["ConnectionInformationWrapper.java", + "exascript_java.java", + "ExportSpecificationWrapper.java", + "ImportSpecificationWrapper.java", + "Metadata.java", + "SWIGTYPE_p_ExecutionGraph__ExportSpecification.java", + "SWIGTYPE_p_ExecutionGraph__ImportSpecification.java", + "SWIGVM_datatype_e.java", + "SWIGVM_itertype_e.java", + "raw/exascript_javaJNI.java", + "raw/ResultHandler.java", + "raw/TableIterator.java", + "exascript_java_tmp.cc" + ], + srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"] ) genrule( name = "exascript_java_tmp_h", cmd = """ mkdir build_exascript_java_tmp_h - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_java_tmp_h - cp -r "$(location exascript_java_tmp.cc)" "$(location java_src)" build_exascript_java_tmp_h + cp "$(location //base/exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //base/exaudflib:exascript.i)" build_exascript_java_tmp_h + cp -r "$(location exascript_java_tmp.cc)" build_exascript_java_tmp_h cd build_exascript_java_tmp_h swig -v -DEXTERNAL_PROCESS -c++ -java -external-runtime "../$(location exascript_java_tmp.h)" """, outs = ["exascript_java_tmp.h"], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h", ":exascript_java_tmp.cc", ":java_src"] + srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h", ":exascript_java_tmp.cc"] ) -genrule( - name = "exascript_java_int", - cmd = """ - cp -r $(locations :exascript_java_tmp_cc) . - FILES=`ls java_src/com/exasol/swig/*.java` - python3 $(location //:build_integrated.py) "$(location exascript_java_int.h)" $$FILES - """, - outs = ["exascript_java_int.h"], - srcs = [":exascript_java_tmp_cc"], - tools = ["//:build_integrated.py"] -) genrule( name = "filter_swig_code_exascript_java_h", - cmd = 'python3 $(location //:filter_swig_code.py) "$@" "$<"', + cmd = 'python3 $(location //base:filter_swig_code.py) "$@" "$<"', outs = ["exascript_java.h"], srcs = [":exascript_java_tmp_h"], - tools = ["//:filter_swig_code.py"] + tools = ["//base:filter_swig_code.py"] ) genrule( name = "filter_swig_code_exascript_java_cc", cmd = """ TMPDIR=`mktemp -d` - cp -r -L $(location //:filter_swig_code.py) $(locations :exascript_java_tmp_cc) $$TMPDIR - (cd $$TMPDIR && + mkdir -p "$$TMPDIR"/java_src/com/exasol/swig + cp $(location :raw/TableIterator.java) "$$TMPDIR"/java_src/com/exasol/swig/ + cp $(location :raw/ResultHandler.java) "$$TMPDIR"/java_src/com/exasol/swig/ + cp $(location :raw/exascript_javaJNI.java) "$$TMPDIR"/java_src/com/exasol/swig/ + find "$$TMPDIR"/java_src/com/exasol/swig/ -name *.java -type f -exec chmod 644 {} \\; + cp -r -L $(location //base:filter_swig_code.py) $(location :exascript_java_tmp.cc) "$$TMPDIR" + (cd "$$TMPDIR" python3 filter_swig_code.py "exascript_java.cc" "exascript_java_tmp.cc") - cp $$TMPDIR/exascript_java.cc "$(location exascript_java.cc)" - cp -r $$TMPDIR/java_src/com/exasol/swig/ConnectionInformationWrapper.java "$(location ConnectionInformationWrapper.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/exascript_java.java "$(location exascript_java.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/exascript_javaJNI.java "$(location exascript_javaJNI.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/ExportSpecificationWrapper.java "$(location ExportSpecificationWrapper.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/ImportSpecificationWrapper.java "$(location ImportSpecificationWrapper.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/Metadata.java "$(location Metadata.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/ResultHandler.java "$(location ResultHandler.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/SWIGTYPE_p_ExecutionGraph__ExportSpecification.java "$(location SWIGTYPE_p_ExecutionGraph__ExportSpecification.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/SWIGTYPE_p_ExecutionGraph__ImportSpecification.java "$(location SWIGTYPE_p_ExecutionGraph__ImportSpecification.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/SWIGVM_datatype_e.java "$(location SWIGVM_datatype_e.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/SWIGVM_itertype_e.java "$(location SWIGVM_itertype_e.java)" - cp -r $$TMPDIR/java_src/com/exasol/swig/TableIterator.java "$(location TableIterator.java)" - rm -rf $$TMPDIR + cp "$$TMPDIR"/exascript_java.cc $(location :exascript_java.cc) + cp "$$TMPDIR"/java_src/com/exasol/swig/TableIterator.java $(location TableIterator.java) + cp "$$TMPDIR"/java_src/com/exasol/swig/ResultHandler.java $(location ResultHandler.java) + cp "$$TMPDIR"/java_src/com/exasol/swig/exascript_javaJNI.java $(location exascript_javaJNI.java) + rm -rf "$$TMPDIR" """, outs = [ "exascript_java.cc", - "ConnectionInformationWrapper.java", - "exascript_java.java", - "exascript_javaJNI.java", - "ExportSpecificationWrapper.java", - "ImportSpecificationWrapper.java", - "Metadata.java", - "ResultHandler.java", - "SWIGTYPE_p_ExecutionGraph__ExportSpecification.java", - "SWIGTYPE_p_ExecutionGraph__ImportSpecification.java", - "SWIGVM_datatype_e.java", - "SWIGVM_itertype_e.java", "TableIterator.java", + "ResultHandler.java", + "exascript_javaJNI.java" ], - srcs = [":exascript_java_tmp_cc"], - tools = ["//:filter_swig_code.py"] + srcs = [":exascript_java_tmp.cc", ":raw/TableIterator.java", + ":raw/ResultHandler.java", ":raw/exascript_javaJNI.java"], + tools = ["//base:filter_swig_code.py"] ) cc_library( name = "exascript_java", - srcs = [":filter_swig_code_exascript_java_cc",], - deps = ["@java//:java","//exaudflib:exaudflib-deps","//exaudflib:header"], + srcs = [":exascript_java.cc",], + deps = ["@java//:java","//base/exaudflib:exaudflib-deps","//base/exaudflib:header"], copts= ["-O0","-fno-lto"], # We limit this target to -O0 (no optimization) and -fno-lto. because otherwise we get compiler warnings of the sort # note: "code may be misoptimized unless -fno-strict-aliasing is used." @@ -103,8 +107,8 @@ cc_library( cc_library( name = "javacontainer", srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", ":dummy"], - hdrs = [":exascript_java_int", ":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], - deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//exaudflib:header", "//:debug_message_h","//exaudflib:scriptoptionlines"], + hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], + deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//base/exaudflib:header", "//base:debug_message_h","//base/exaudflib:scriptoptionlines"], # copts= ["-O0","-fno-lto"], alwayslink=True, ) @@ -122,18 +126,18 @@ java_binary( srcs = glob(["*.java"])+ #The following are generated classes [ - "ConnectionInformationWrapper.java", - "exascript_java.java", - "exascript_javaJNI.java", - "ExportSpecificationWrapper.java", - "ImportSpecificationWrapper.java", - "Metadata.java", - "ResultHandler.java", - "SWIGTYPE_p_ExecutionGraph__ExportSpecification.java", - "SWIGTYPE_p_ExecutionGraph__ImportSpecification.java", - "SWIGVM_datatype_e.java", - "SWIGVM_itertype_e.java", - "TableIterator.java" + ":ConnectionInformationWrapper.java", + ":exascript_java.java", + ":exascript_javaJNI.java", + ":ExportSpecificationWrapper.java", + ":ImportSpecificationWrapper.java", + ":Metadata.java", + ":ResultHandler.java", + ":SWIGTYPE_p_ExecutionGraph__ExportSpecification.java", + ":SWIGTYPE_p_ExecutionGraph__ImportSpecification.java", + ":SWIGVM_datatype_e.java", + ":SWIGVM_itertype_e.java", + ":TableIterator.java" ], deps = ["@maven//:com_exasol_udf_api_java"], create_executable=False, diff --git a/exaudfclient/base/javacontainer/javacontainer.cc b/exaudfclient/base/javacontainer/javacontainer.cc index 66a126d06..bb40b5722 100644 --- a/exaudfclient/base/javacontainer/javacontainer.cc +++ b/exaudfclient/base/javacontainer/javacontainer.cc @@ -1,5 +1,5 @@ -#include "javacontainer/javacontainer.h" -#include "javacontainer/javacontainer_impl.h" +#include "base/javacontainer/javacontainer.h" +#include "base/javacontainer/javacontainer_impl.h" using namespace SWIGVMContainers; using namespace std; diff --git a/exaudfclient/base/javacontainer/javacontainer.h b/exaudfclient/base/javacontainer/javacontainer.h index 38038e1df..f97c860e5 100644 --- a/exaudfclient/base/javacontainer/javacontainer.h +++ b/exaudfclient/base/javacontainer/javacontainer.h @@ -2,7 +2,7 @@ #define JAVACONTAINER_H -#include "exaudflib/vm/swig_vm.h" +#include "base/exaudflib/vm/swig_vm.h" #include #ifdef ENABLE_JAVA_VM diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc index 27a4e40c9..35786801d 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.cc +++ b/exaudfclient/base/javacontainer/javacontainer_impl.cc @@ -7,13 +7,13 @@ #include #include -#include "exaudflib/swig/swig_meta_data.h" +#include "base/exaudflib/swig/swig_meta_data.h" #include "exascript_java_jni_decl.h" -#include "debug_message.h" -#include "javacontainer/javacontainer.h" -#include "javacontainer/javacontainer_impl.h" -#include "exaudflib/vm/scriptoptionlines.h" +#include "base/debug_message.h" +#include "base/javacontainer/javacontainer.h" +#include "base/javacontainer/javacontainer_impl.h" +#include "base/exaudflib/vm/scriptoptionlines.h" using namespace SWIGVMContainers; using namespace std; @@ -22,7 +22,7 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI): m_checkOnly(checkOnly), m_ex m_scriptCode(SWIGVM_params->script_code), m_exceptionThrown(false), m_jvm(NULL), m_env(NULL), m_needsCompilation(true) { stringstream ss; - m_exaJavaPath = "/exaudf/javacontainer"; // TODO hardcoded path + m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path DBG_FUNC_CALL(cerr,getScriptClassName()); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used DBG_FUNC_CALL(cerr,importScripts()); DBG_FUNC_CALL(cerr,getExternalJvmOptions()); @@ -433,7 +433,7 @@ void JavaVMImpl::registerFunctions() { } void JavaVMImpl::setClasspath() { - m_exaJarPath = m_exaJavaPath + "/libexaudf.jar"; + m_exaJarPath = m_exaJavaPath + "/exaudf_deploy.jar"; m_classpath = m_exaJarPath; } diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.h b/exaudfclient/base/javacontainer/javacontainer_impl.h index af7828336..79f67318f 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.h +++ b/exaudfclient/base/javacontainer/javacontainer_impl.h @@ -5,7 +5,7 @@ #include #include -#include "exaudflib/vm/swig_vm.h" +#include "base/exaudflib/vm/swig_vm.h" #include #ifdef ENABLE_JAVA_VM diff --git a/exaudfclient/base/javacontainer/test/BUILD b/exaudfclient/base/javacontainer/test/BUILD index 8b6c9f53e..c0103e818 100644 --- a/exaudfclient/base/javacontainer/test/BUILD +++ b/exaudfclient/base/javacontainer/test/BUILD @@ -1,6 +1,6 @@ java_library( name = "ExaStackTraceCleaner", - srcs = ["//javacontainer:ExaStackTraceCleaner.java"] + srcs = ["//base/javacontainer:ExaStackTraceCleaner.java"] ) java_test( @@ -15,7 +15,7 @@ cc_test( name = "javacontainer-test", srcs = ["cpp/javacontainer_test.cc", "cpp/exaudf_wrapper.cc", "cpp/javavm_test.cc", "cpp/javavm_test.h"], deps = [ - "//javacontainer:javacontainer", + "//base/javacontainer:javacontainer", "@googletest//:gtest_main", ], data = ["test.jar"] diff --git a/exaudfclient/base/javacontainer/test/cpp/exaudf_wrapper.cc b/exaudfclient/base/javacontainer/test/cpp/exaudf_wrapper.cc index 274c19b7b..7222716bc 100644 --- a/exaudfclient/base/javacontainer/test/cpp/exaudf_wrapper.cc +++ b/exaudfclient/base/javacontainer/test/cpp/exaudf_wrapper.cc @@ -1,4 +1,4 @@ -#include "exaudflib/swig/swig_common.h" +#include "base/exaudflib/swig/swig_common.h" //Dummy implementation to calm down the linker void* load_dynamic(const char* name) { diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc index 5ddfd23d9..387cfda6d 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc @@ -1,25 +1,25 @@ #include "include/gtest/gtest.h" #include "gmock/gmock.h" -#include "javacontainer/test/cpp/javavm_test.h" +#include "base/javacontainer/test/cpp/javavm_test.h" #include TEST(JavaContainer, basic_jar) { const std::string script_code = "%scriptclass com.exasol.udf_profiling.UdfProfiler;\n" - "%jar javacontainer/test/test.jar;"; + "%jar base/javacontainer/test/test.jar;"; JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/javacontainer"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, "\n"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/javacontainer/libexaudf.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/exaudf/javacontainer/libexaudf.jar:javacontainer/test/test.jar"); - const std::vector expectedJarPaths = {"javacontainer/test/test.jar"}; + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/test.jar"); + const std::vector expectedJarPaths = {"base/javacontainer/test/test.jar"}; EXPECT_EQ(vm.getJavaVMInternalStatus().m_jarPaths, expectedJarPaths); EXPECT_FALSE(vm.getJavaVMInternalStatus().m_needsCompilation); const std::vector expectedJVMOptions = { "-Dexasol.scriptclass=com.exasol.udf_profiling.UdfProfiler", "-Xms128m", "-Xmx128m", "-Xss512k", "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/exaudf/javacontainer/libexaudf.jar:javacontainer/test/test.jar", + "-Djava.class.path=/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/test.jar", "-XX:+UseSerialGC" }; EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); } @@ -38,17 +38,17 @@ TEST(JavaContainer, basic_inline) { "}" "}"; JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/javacontainer"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = std::string("package com.exasol;\r\n") + script_code; EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/javacontainer/libexaudf.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/javacontainer/libexaudf.jar"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar"); EXPECT_TRUE(vm.getJavaVMInternalStatus().m_jarPaths.empty()); EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/javacontainer/libexaudf.jar", + "-Djava.class.path=/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar", "-XX:+UseSerialGC" }; EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); } @@ -67,20 +67,20 @@ TEST(JavaContainer, combined_inline_jar) { "return 0;" "}" "}"; - const std::string script_code_with_jar = std::string("%jar javacontainer/test/test.jar;") + script_code; + const std::string script_code_with_jar = std::string("%jar base/javacontainer/test/test.jar;") + script_code; JavaVMTest vm(script_code_with_jar); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/javacontainer"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = std::string("package com.exasol;\r\n") + script_code; EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/javacontainer/libexaudf.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/javacontainer/libexaudf.jar:javacontainer/test/test.jar"); - const std::vector expectedJarPaths = {"javacontainer/test/test.jar"}; + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/test.jar"); + const std::vector expectedJarPaths = {"base/javacontainer/test/test.jar"}; EXPECT_EQ(vm.getJavaVMInternalStatus().m_jarPaths, expectedJarPaths); EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/javacontainer/libexaudf.jar:javacontainer/test/test.jar", + "-Djava.class.path=/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/test.jar", "-XX:+UseSerialGC" }; EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); } @@ -95,7 +95,7 @@ TEST(JavaContainer, quoted_jvm_option) { " }\n" "}\n"; JavaVMTest vm(script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/javacontainer"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJavaPath, "/exaudf/base/javacontainer"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_localClasspath, "/tmp"); const std::string expected_script_code = "package com.exasol;\r\n\n\n" @@ -104,8 +104,8 @@ TEST(JavaContainer, quoted_jvm_option) { "\tctx.emit(\"Success!\");\n" " }\n}\n"; EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/javacontainer/libexaudf.jar"); - EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/javacontainer/libexaudf.jar"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); + EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar"); const std::vector expectedJarPaths = {}; EXPECT_EQ(vm.getJavaVMInternalStatus().m_jarPaths, expectedJarPaths); EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); @@ -115,7 +115,7 @@ TEST(JavaContainer, quoted_jvm_option) { */ const std::vector expectedJVMOptions = { "-Dhttp.agent=\"ABC", "DEF\"", "-Xms128m", "-Xmx128m", "-Xss512k", "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Djava.class.path=/tmp:/exaudf/javacontainer/libexaudf.jar", + "-Djava.class.path=/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar", "-XX:+UseSerialGC" }; EXPECT_EQ(vm.getJavaVMInternalStatus().m_jvmOptions, expectedJVMOptions); } diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc index 59e873f0b..f4c0310c4 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc @@ -1,5 +1,5 @@ -#include "javacontainer/test/cpp/javavm_test.h" -#include "javacontainer/javacontainer_impl.h" +#include "base/javacontainer/test/cpp/javavm_test.h" +#include "base/javacontainer/javacontainer_impl.h" #include diff --git a/exaudfclient/base/load_dynamic.cc b/exaudfclient/base/load_dynamic.cc index 38a742717..4b6949d27 100644 --- a/exaudfclient/base/load_dynamic.cc +++ b/exaudfclient/base/load_dynamic.cc @@ -3,7 +3,7 @@ #endif #include -#include "exaudflib/vm/swig_vm.h" +#include "base/exaudflib/vm/swig_vm.h" void* handle; diff --git a/exaudfclient/base/protobuf_repository.bzl b/exaudfclient/base/protobuf_repository.bzl index 4fe685f2c..c8c8e38d9 100644 --- a/exaudfclient/base/protobuf_repository.bzl +++ b/exaudfclient/base/protobuf_repository.bzl @@ -12,12 +12,12 @@ def _protobuf_local_repository_impl(repository_ctx): print("protobuf include prefix in environment specified; %s"%include_prefix) build_file_content = """ cc_library( - name = "{name}", + name = "protobuf", srcs = glob(["protobuf/lib/**/libprotobuf*.so"]), hdrs = glob(["protobuf/include/**"]), includes = ["protobuf/include/"], visibility = ["//visibility:public"] -)""".format( name=repository_ctx.name) +)""" print(build_file_content) repository_ctx.symlink(library_prefix, "./protobuf/lib") diff --git a/exaudfclient/base/python/exascript_python_wrap.py b/exaudfclient/base/python/exascript_python_wrap.py index f2d6b2da1..00ce02a8d 100644 --- a/exaudfclient/base/python/exascript_python_wrap.py +++ b/exaudfclient/base/python/exascript_python_wrap.py @@ -12,7 +12,7 @@ #print("sys.path append",path_to_pyexadataframe) sys.path.append(path_to_pyexadataframe) else: - path_to_pyexadataframe="/exaudf/python/python3" + path_to_pyexadataframe="/exaudf/base/python/python3" #print("sys.path append",path_to_pyexadataframe) sys.path.append(path_to_pyexadataframe) diff --git a/exaudfclient/base/python/python3/BUILD b/exaudfclient/base/python/python3/BUILD index 29c0defbd..422693eb2 100644 --- a/exaudfclient/base/python/python3/BUILD +++ b/exaudfclient/base/python/python3/BUILD @@ -5,12 +5,12 @@ genrule( cmd = """ INCLUDES=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION-config --includes` mkdir -p build_exascript_python_tmp_cc/exaudflib - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib + cp "$(location //base/exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //base/exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib cd build_exascript_python_tmp_cc swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python_tmp.cc)" exaudflib/exascript.i """, outs = ["exascript_python_tmp.cc", "exascript_python.py"], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h"] + srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"] ) genrule( @@ -18,32 +18,32 @@ genrule( cmd = """ INCLUDES=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION-config --includes` mkdir build_exascript_python_tmp_h - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_python_tmp_h + cp "$(location //base/exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //base/exaudflib:exascript.i)" build_exascript_python_tmp_h cp "$(location exascript_python_tmp.cc)" "$(location exascript_python.py)" build_exascript_python_tmp_h cd build_exascript_python_tmp_h swig -v $$INCLUDES -DEXTERNAL_PROCESS -c++ -python -py3 -external-runtime "../$(location exascript_python_tmp.h)" """, outs = ["exascript_python_tmp.h"], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h", ":exascript_python_tmp.cc", "exascript_python.py"] + srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h", ":exascript_python_tmp.cc", "exascript_python.py"] ) genrule( name = "extend_exascript_python_preset_py", - cmd = 'bash $(location //python:extend_exascript_python_preset_py.sh) "$(location //python:exascript_python_preset_core.py)" "$(location exascript_python_preset.py)" "$$PYTHON3_PREFIX" "$$PYTHON3_VERSION" ""', + cmd = 'bash $(location //base/python:extend_exascript_python_preset_py.sh) "$(location //base/python:exascript_python_preset_core.py)" "$(location exascript_python_preset.py)" "$$PYTHON3_PREFIX" "$$PYTHON3_VERSION" ""', outs = ["exascript_python_preset.py"], - srcs = ["//python:exascript_python_preset_core.py"], - tools = ["//python:extend_exascript_python_preset_py.sh"] + srcs = ["//base/python:exascript_python_preset_core.py"], + tools = ["//base/python:extend_exascript_python_preset_py.sh"] ) genrule( name = "exascript_python_int", cmd = """ cp $(SRCS) . - python3 $(location //:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" + python3 $(location //base:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" """, outs = ["exascript_python_int.h"], - srcs = [":exascript_python_tmp_cc", "//python:exascript_python_wrap.py", ":extend_exascript_python_preset_py"], - tools = ["//:build_integrated.py"] + srcs = [":exascript_python_tmp_cc", "//base/python:exascript_python_wrap.py", ":extend_exascript_python_preset_py"], + tools = ["//base:build_integrated.py"] ) genrule( @@ -51,14 +51,14 @@ genrule( cmd = """ ACTUAL_PYTHON_VERSION=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION -c 'import sys; print(".".join(map(str, sys.version_info[:3])))'` if [[ $$ACTUAL_PYTHON_VERSION == 2* ]] ; then - python3 $(location //:filter_swig_code.py) "$@" "$<" + python3 $(location //base:filter_swig_code.py) "$@" "$<" else cp "$<" "$@" fi """, outs = ["exascript_python.h"], srcs = [":exascript_python_tmp_h"], - tools = ["//:filter_swig_code.py"] + tools = ["//base:filter_swig_code.py"] ) genrule( @@ -67,32 +67,32 @@ genrule( ACTUAL_PYTHON_VERSION=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION -c 'import sys; print(".".join(map(str, sys.version_info[:3])))'` cp $(locations exascript_python_tmp_cc) . if [[ $$ACTUAL_PYTHON_VERSION == 2* ]] ; then - python $(location //:filter_swig_code.py) "$@" exascript_python_tmp.cc + python $(location //base:filter_swig_code.py) "$@" exascript_python_tmp.cc else cp exascript_python_tmp.cc "$@" fi """, outs = ["exascript_python.cc"], srcs = [":exascript_python_tmp_cc"], - tools = ["//:filter_swig_code.py"] + tools = ["//base:filter_swig_code.py"] ) cc_library( name = "exascript_python", srcs = [":filter_swig_code_exascript_python_cc",":filter_swig_code_exascript_python_h"], hdrs = [":filter_swig_code_exascript_python_h"], - deps = ["@python3//:python3","//exaudflib:exaudflib-deps","//exaudflib:header"], + deps = ["@python3//:python3","//base/exaudflib:exaudflib-deps","//base/exaudflib:header"], alwayslink=True, ) cc_library( name = "pythoncontainer", - srcs = ["//python:pythoncontainer.cc", ":dummy.h"], + srcs = ["//base/python:pythoncontainer.cc", ":dummy.h"], data = [":extend_exascript_python_preset_py"], hdrs = [":exascript_python_int", ":filter_swig_code_exascript_python_h"], include_prefix = ".", - deps = ["@python3//:python3",":exascript_python","//exaudflib:header", - "//:debug_message_h","//exaudflib:scriptoptionlines", "//python:pythoncontainer_header"], + deps = ["@python3//:python3",":exascript_python","//base/exaudflib:header", + "//base:debug_message_h","//base/exaudflib:scriptoptionlines", "//base/python:pythoncontainer_header"], alwayslink=True, ) @@ -109,5 +109,5 @@ cc_binary( linkshared = 1, srcs = [":python_ext_dataframe.cc"], deps = ["@python3//:python3","@numpy//:numpy", - "//exaudflib:header", "//:debug_message_h"], + "//base/exaudflib:header", "//base:debug_message_h"], ) diff --git a/exaudfclient/base/python/python3/python_ext_dataframe.cc b/exaudfclient/base/python/python3/python_ext_dataframe.cc index 972560c66..3e3902b8f 100644 --- a/exaudfclient/base/python/python3/python_ext_dataframe.cc +++ b/exaudfclient/base/python/python3/python_ext_dataframe.cc @@ -1,6 +1,6 @@ #include -#include "exaudflib/swig/swig_common.h" -#include "debug_message.h" +#include "base/exaudflib/swig/swig_common.h" +#include "base/debug_message.h" #include diff --git a/exaudfclient/base/python/pythoncontainer.cc b/exaudfclient/base/python/pythoncontainer.cc index 43c6265a6..d87d373e1 100644 --- a/exaudfclient/base/python/pythoncontainer.cc +++ b/exaudfclient/base/python/pythoncontainer.cc @@ -1,5 +1,5 @@ #include "pythoncontainer.h" -#include "exaudflib/swig/swig_meta_data.h" +#include "base/exaudflib/swig/swig_meta_data.h" #include #ifdef _POSIX_C_SOURCE #undef _POSIX_C_SOURCE @@ -10,10 +10,10 @@ #include #include "exascript_python_int.h" #include "exascript_python.h" -#include "debug_message.h" -#include "exaudflib/vm/scriptoptionlines.h" +#include "base/debug_message.h" +#include "base/exaudflib/vm/scriptoptionlines.h" -#include "exaudflib/swig/script_data_transfer_objects.h" +#include "base/exaudflib/swig/script_data_transfer_objects.h" #include #include diff --git a/exaudfclient/base/python/pythoncontainer.h b/exaudfclient/base/python/pythoncontainer.h index 27092a91f..aab6eb7a9 100644 --- a/exaudfclient/base/python/pythoncontainer.h +++ b/exaudfclient/base/python/pythoncontainer.h @@ -1,7 +1,7 @@ #ifndef PYTHONVM_H #define PYTHONVM_H -#include "exaudflib/vm/swig_vm.h" +#include "base/exaudflib/vm/swig_vm.h" #ifdef ENABLE_PYTHON_VM diff --git a/exaudfclient/base/python_repository.bzl b/exaudfclient/base/python_repository.bzl index e342d16ad..549eabc89 100644 --- a/exaudfclient/base/python_repository.bzl +++ b/exaudfclient/base/python_repository.bzl @@ -31,20 +31,21 @@ def _get_lib_glob(binary, version, p_repository_ctx): return lib_glob def _python_local_repository_impl(repository_ctx): - python_prefix_env_var = repository_ctx.name.upper() + "_PREFIX" + python_version = repository_ctx.attr.python_version + python_prefix_env_var = python_version.upper() + "_PREFIX" if python_prefix_env_var in repository_ctx.os.environ: prefix = repository_ctx.os.environ[python_prefix_env_var] else: fail("Environment Variable %s not found"%python_prefix_env_var) print("python prefix in environment specified; %s"%prefix) - python_version_env_var = repository_ctx.name.upper() + "_VERSION" + python_version_env_var = python_version.upper() + "_VERSION" if python_version_env_var in repository_ctx.os.environ: version = repository_ctx.os.environ[python_version_env_var] - if repository_ctx.name == "python3" and version.startswith("2"): - fail("Wrong python version specified in environment variable %s, got binary name '%s', but version number '%s'"%(python_version_env_var,repository_ctx.name,version)) - if repository_ctx.name == "python" or version.startswith("2"): - fail("Python 2 is not supported anymore, but specified in environment variable %s, got %s, %s"%(python_version_env_var,repository_ctx.name,version)) + if python_version == "python3" and version.startswith("2"): + fail("Wrong python version specified in environment variable %s, got binary name '%s', but version number '%s'"%(python_version_env_var,python_version,version)) + if python_version == "python" or version.startswith("2"): + fail("Python 2 is not supported anymore, but specified in environment variable %s, got %s, %s"%(python_version_env_var,python_version,version)) else: fail("Environment Variable %s not found"%python_version_env_var) print("python version in environment specified; %s"%version) @@ -66,7 +67,7 @@ cc_library( includes = ["{include_dir}"], defines = [{defines}], visibility = ["//visibility:public"] -)""".format(lib_glob=lib_glob[1:], include_dir=include_dir[1:], name=repository_ctx.name, defines=defines_str) +)""".format(lib_glob=lib_glob[1:], include_dir=include_dir[1:], name=python_version, defines=defines_str) print(build_file_content) repository_ctx.symlink(prefix, "."+prefix) @@ -75,7 +76,8 @@ cc_library( python_local_repository = repository_rule( implementation=_python_local_repository_impl, local = True, - environ = ["PYTHON3_PREFIX", "PYTHON3_VERSION"]) + environ = ["PYTHON3_PREFIX", "PYTHON3_VERSION"], + attrs = {"python_version": attr.string()}) def _get_numpy_include_dir(binary,p_repository_ctx): @@ -117,13 +119,13 @@ def _numpy_local_repository_impl(repository_ctx): hdrs = numpy_include_dir + "/*/*.h" build_file_content = """ cc_library( - name = "{name}", + name = "numpy", srcs = [], hdrs = glob(["{hdrs}"]), includes = ["{includes}"], defines = [{defines}], visibility = ["//visibility:public"] -)""".format(hdrs=hdrs, includes=numpy_include_dir, name=repository_ctx.name, defines=defines_str) +)""".format(hdrs=hdrs, includes=numpy_include_dir, defines=defines_str) print(build_file_content) repository_ctx.symlink(prefix, "."+prefix) diff --git a/exaudfclient/base/streaming_container/BUILD b/exaudfclient/base/streaming_container/BUILD index b57363916..848d7774b 100644 --- a/exaudfclient/base/streaming_container/BUILD +++ b/exaudfclient/base/streaming_container/BUILD @@ -4,5 +4,5 @@ cc_library( name = "streamingcontainer", srcs = ["streamingcontainer.cc","streamingcontainer.h"], hdrs = ["streamingcontainer.h"], - deps = ["//exaudflib:exaudflib-deps","//exaudflib:header"] + deps = ["//base/exaudflib:exaudflib-deps","//base/exaudflib:header"] ) \ No newline at end of file diff --git a/exaudfclient/base/zmq_repository.bzl b/exaudfclient/base/zmq_repository.bzl index 9cad5095f..64882597b 100644 --- a/exaudfclient/base/zmq_repository.bzl +++ b/exaudfclient/base/zmq_repository.bzl @@ -12,12 +12,12 @@ def _zmq_local_repository_impl(repository_ctx): print("zmq include prefix in environment specified; %s"%include_prefix) build_file_content = """ cc_library( - name = "{name}", + name = "zmq", srcs = glob(["zmq/lib/**/libzmq.so"]), hdrs = glob(["zmq/include/zmq*"]), includes = ["zmq/include/"], visibility = ["//visibility:public"] -)""".format( name=repository_ctx.name) +)""" print(build_file_content) repository_ctx.symlink(library_prefix, "./zmq/lib") diff --git a/exaudfclient/base/build.sh b/exaudfclient/build.sh similarity index 100% rename from exaudfclient/base/build.sh rename to exaudfclient/build.sh diff --git a/exaudfclient/base/build_local.sh b/exaudfclient/build_local.sh similarity index 100% rename from exaudfclient/base/build_local.sh rename to exaudfclient/build_local.sh diff --git a/exaudfclient/base/build_local_all.sh b/exaudfclient/build_local_all.sh similarity index 100% rename from exaudfclient/base/build_local_all.sh rename to exaudfclient/build_local_all.sh diff --git a/exaudfclient/base/docs/exaudfclient.drawio b/exaudfclient/docs/exaudfclient.drawio similarity index 100% rename from exaudfclient/base/docs/exaudfclient.drawio rename to exaudfclient/docs/exaudfclient.drawio diff --git a/exaudfclient/base/docs/exaudfclient.png b/exaudfclient/docs/exaudfclient.png similarity index 100% rename from exaudfclient/base/docs/exaudfclient.png rename to exaudfclient/docs/exaudfclient.png diff --git a/exaudfclient/base/exaudfclient.cc b/exaudfclient/exaudfclient.cc similarity index 96% rename from exaudfclient/base/exaudfclient.cc rename to exaudfclient/exaudfclient.cc index f4269e6e2..158e5d19c 100644 --- a/exaudfclient/base/exaudfclient.cc +++ b/exaudfclient/exaudfclient.cc @@ -15,8 +15,8 @@ #include //This is required for dynamic linking in new linker namespace, not required for plugins #endif #include -#include "exaudflib/vm/swig_vm.h" -#include "exaudflib/load_dynamic.h" +#include "base/exaudflib/vm/swig_vm.h" +#include "base/exaudflib/load_dynamic.h" #ifdef ENABLE_BENCHMARK_VM #include "benchmark_container/benchmark_container.h" #endif @@ -24,7 +24,7 @@ #include "streaming_container/streamingcontainer.h" #endif #include -#include "debug_message.h" +#include "base/debug_message.h" #include #include #include @@ -38,11 +38,11 @@ #ifdef ENABLE_JAVA_VM -#include "javacontainer/javacontainer.h" +#include "base/javacontainer/javacontainer.h" #endif //ENABLE_JAVA_VM #ifdef ENABLE_PYTHON_VM -#include "python/pythoncontainer.h" +#include "base/python/pythoncontainer.h" #endif //ENABLE_PYTHON_VM #ifdef UDF_PLUGIN_CLIENT diff --git a/exaudfclient/base/run.sh b/exaudfclient/run.sh similarity index 100% rename from exaudfclient/base/run.sh rename to exaudfclient/run.sh diff --git a/exaudfclient/base/run_local.sh b/exaudfclient/run_local.sh similarity index 100% rename from exaudfclient/base/run_local.sh rename to exaudfclient/run_local.sh diff --git a/exaudfclient/base/variables.bzl b/exaudfclient/variables.bzl similarity index 100% rename from exaudfclient/base/variables.bzl rename to exaudfclient/variables.bzl diff --git a/exaudfclient/base/visualize_all.sh b/exaudfclient/visualize_all.sh similarity index 100% rename from exaudfclient/base/visualize_all.sh rename to exaudfclient/visualize_all.sh diff --git a/exaudfclient/base/visualize_deps.sh b/exaudfclient/visualize_deps.sh similarity index 100% rename from exaudfclient/base/visualize_deps.sh rename to exaudfclient/visualize_deps.sh diff --git a/ext/scripts/install_scripts/install_via_conda.pl b/ext/scripts/install_scripts/install_via_conda.pl index d4e878a58..c86000d83 100755 --- a/ext/scripts/install_scripts/install_via_conda.pl +++ b/ext/scripts/install_scripts/install_via_conda.pl @@ -122,7 +122,7 @@ sub generate_channel_args{ sub run_install_command{ my ($file, $dry_run, $with_versions, $allow_no_version, $channel_args) = @_; my $element_separator = '\\|'; - my $combining_template = "$conda_binary install --yes --freeze-installed $channel_args <<<<0>>>>"; + my $combining_template = "$conda_binary install --yes $channel_args <<<<0>>>>"; my @templates=("'<<<<0>>>>==<<<<1>>>>'"); my @separators = (" "); diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile index b5779ed30..09812f998 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile @@ -5,15 +5,17 @@ RUN mkdir /conf /buckets RUN ldconfig RUN mkdir /exaudfclient /exaudf -COPY /exaudfclient/base /exaudfclient/base +COPY /exaudfclient/ /exaudfclient/ -WORKDIR /exaudfclient/base +WORKDIR /exaudfclient/ # We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3 RUN cp -r -L bazel-bin/* /exaudf + +WORKDIR /exaudfclient/base RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_deps/packages/conda_packages index 741e947e9..a6aea162c 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_deps/packages/conda_packages @@ -1,8 +1,7 @@ protobuf|4.25.3 pcre|8.45 -chrpath|0.16 -openjdk|11.0.15 -bazel|5.2.0 -gxx_linux-64|11.2.0 -gcc_linux-64|11.2.0 +bazel|7.2.1 +gxx_linux-64|12.4.0 +gcc_linux-64|12.4.0 make|4.3 +chrpath|0.16 diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile index 823e0e37b..434fced7c 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_run/Dockerfile @@ -1,15 +1,16 @@ FROM {{build_deps}} RUN mkdir /exaudfclient /exaudf -COPY exaudfclient/base /exaudfclient/base +COPY exaudfclient/ /exaudfclient/ -WORKDIR /exaudfclient/base +WORKDIR /exaudfclient/ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3 RUN cp -r -L bazel-bin/* /exaudf +WORKDIR /exaudfclient/base RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 WORKDIR /exaudf diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_steps.py b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_steps.py index e6d5ca01e..7e8ec309a 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_steps.py +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/build_steps.py @@ -68,7 +68,7 @@ def requires_tasks(self): return {"build_deps": AnalyzeBuildDeps} def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"exaudfclient/base": "exaudfclient/base"} + return {"exaudfclient": "exaudfclient"} def get_path_in_flavor(self): return "flavor_base" @@ -95,7 +95,7 @@ def requires_tasks(self): return {"base_test_deps": AnalyzeBaseTestDeps} def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"exaudfclient/base": "exaudfclient/base", "emulator": "emulator"} + return {"exaudfclient": "exaudfclient", "emulator": "emulator"} def get_path_in_flavor(self): return "flavor_base" diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_deps/packages/conda_packages index d18b3a297..1f4bade7c 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/language_deps/packages/conda_packages @@ -1,11 +1,5 @@ -libxcrypt|4.4.36 # Needed for building the udfclient and because libcrypt got removed from conda sysroot, see https://github.com/conda-forge/linux-sysroot-feedstock/issues/52 python|3.10.4 numpy|1.26.2 pandas|2.2.2 pyarrow|16.0.0 -libblas|3.9.0=15_linux64_mkl mamba|1.5.1 -ld_impl_linux-64|2.36.1 -cryptography|42.0.7 -libxml2|2.10.3=hca2bb57_4 # Pinned for OpenJDK in build_deps -icu|70.1=h27087fc_0 # Pinned for OpenJDK in build_deps diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/udfclient_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/udfclient_deps/packages/conda_packages index 30a55fd9a..45e2852af 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/udfclient_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/udfclient_deps/packages/conda_packages @@ -1,7 +1,4 @@ libprotobuf|4.25.3 zeromq|4.3.5 -openssl|3.3.0 nss|3.100 cppzmq|4.10.0 -_openmp_mutex|4.5=2_kmp_llvm # We need this, such that we can later install libblas with mkl, which is for example needed by pytorch -libzlib|1.2.13 # We need to pin this, such that we can later install libxml2 and other libraries which are not compatible with libzlib >= 1.3.0 \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile index a9751eaca..fddfb7be3 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile @@ -5,15 +5,17 @@ RUN mkdir /conf /buckets RUN ldconfig RUN mkdir /exaudfclient /exaudf -COPY /exaudfclient/base /exaudfclient/base +COPY /exaudfclient/ /exaudfclient/ -WORKDIR /exaudfclient/base +WORKDIR /exaudfclient/ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ # We need to build with optimize here, because the debug build fails to run with a runtime linking error regarding protobuf RUN ./build.sh --config no-tty -c dbg --config python --config test-binaries-py3 RUN cp -r -L bazel-bin/* /exaudf + +WORKDIR /exaudfclient/base RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_deps/packages/conda_packages index 336600f96..a6aea162c 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_deps/packages/conda_packages @@ -1,8 +1,7 @@ protobuf|4.25.3 pcre|8.45 -chrpath|0.16 -openjdk|11.0.15 -bazel|5.2.0 -gxx_linux-64|12.1.0 -gcc_linux-64|12.1.0 +bazel|7.2.1 +gxx_linux-64|12.4.0 +gcc_linux-64|12.4.0 make|4.3 +chrpath|0.16 diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_run/Dockerfile index 823e0e37b..434fced7c 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_run/Dockerfile @@ -1,15 +1,16 @@ FROM {{build_deps}} RUN mkdir /exaudfclient /exaudf -COPY exaudfclient/base /exaudfclient/base +COPY exaudfclient/ /exaudfclient/ -WORKDIR /exaudfclient/base +WORKDIR /exaudfclient/ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/ ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ ENV C_INCLUDE_PATH=$C_INCLUDE_PATH:$MAMBA_ROOT_PREFIX/include/ RUN ./build.sh --config no-tty --config optimize --config python --config fast-binary-py3 RUN cp -r -L bazel-bin/* /exaudf +WORKDIR /exaudfclient/base RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 WORKDIR /exaudf diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_steps.py b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_steps.py index 8c96e8994..b1110c10b 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_steps.py +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/build_steps.py @@ -82,7 +82,7 @@ def requires_tasks(self): return {"build_deps": AnalyzeBuildDeps} def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"exaudfclient/base": "exaudfclient/base"} + return {"exaudfclient": "exaudfclient"} def get_path_in_flavor(self): return "flavor_base" @@ -109,7 +109,7 @@ def requires_tasks(self): return {"base_test_deps": AnalyzeBaseTestDeps} def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"exaudfclient/base": "exaudfclient/base", "emulator": "emulator"} + return {"exaudfclient": "exaudfclient", "emulator": "emulator"} def get_path_in_flavor(self): return "flavor_base" diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/language_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/language_deps/packages/conda_packages index e12eec1a1..4d6141141 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/language_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/language_deps/packages/conda_packages @@ -1,13 +1,7 @@ -libxcrypt|4.4.36 # Needed for building the udfclient and because libcrypt got removed from conda sysroot, see https://github.com/conda-forge/linux-sysroot-feedstock/issues/52 python|3.10.4 numpy|1.26.2 pandas|2.2.2 pyarrow|16.0.0 libarrow|16.0.0=he216f8a_1_cuda -libblas|3.9.0=15_linux64_mkl mamba|1.5.1 -ld_impl_linux-64|2.36.1 -cryptography|42.0.7 -libxml2|2.10.3=hca2bb57_4 # Pinned for OpenJDK in build_deps -icu|70.1=h27087fc_0 # Pinned for OpenJDK in build_deps cuda-toolkit|11.8.0 diff --git a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/udfclient_deps/packages/conda_packages b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/udfclient_deps/packages/conda_packages index 3d8f59e03..45e2852af 100644 --- a/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/udfclient_deps/packages/conda_packages +++ b/flavors/template-Exasol-all-python-3.10-cuda-conda/flavor_base/udfclient_deps/packages/conda_packages @@ -1,7 +1,4 @@ libprotobuf|4.25.3 zeromq|4.3.5 -openssl|3.3.0 nss|3.100 cppzmq|4.10.0 -_openmp_mutex|4.5=2_kmp_llvm # We need this, such that we can later install libblas with mkl, which is for example needed by pytorch -libzlib|1.2.13 # We need to pin this, such that we can later install libxml2 and other libraries which are not compatible with libzlib >= 1.3.0 diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile index 66ee9bdfd..3038297bb 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile @@ -24,11 +24,13 @@ RUN true # workaround for https://github.com/moby/moby/issues/37965 RUN ldconfig RUN mkdir /exaudfclient /exaudf -COPY /exaudfclient/base /exaudfclient/base +COPY /exaudfclient/ /exaudfclient/ -WORKDIR /exaudfclient/base +WORKDIR /exaudfclient/ RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries-py3"] RUN cp -r -L bazel-bin/* /exaudf + +WORKDIR /exaudfclient/base RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 RUN ./test_udfclient.sh /exaudf/exaudfclient_py3_static diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/build_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/build_deps/Dockerfile index d304fdcf9..a640cae9a 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/build_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/build_deps/Dockerfile @@ -10,7 +10,7 @@ COPY scripts /scripts RUN mkdir -p /build_info/packages COPY build_deps/packages /build_info/packages/build_deps -ENV BAZEL_PACKAGE_VERSION="5.2.0" +ENV BAZEL_PACKAGE_VERSION="7.2.1" ENV BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" ENV BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/build_run/Dockerfile index 92ede8bd1..1ac4548f8 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/build_run/Dockerfile @@ -22,12 +22,13 @@ RUN true # workaround for https://github.com/moby/moby/issues/37965 RUN ldconfig RUN mkdir /exaudfclient /exaudf -COPY exaudfclient/base /exaudfclient/base +COPY exaudfclient/ /exaudfclient/ -WORKDIR /exaudfclient/base +WORKDIR /exaudfclient/ RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary-py3"] RUN cp -r -L bazel-bin/* /exaudf +WORKDIR /exaudfclient/base RUN ./test_udfclient.sh /exaudf/exaudfclient_py3 WORKDIR /exaudf diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/build_steps.py b/flavors/template-Exasol-all-python-3.10/flavor_base/build_steps.py index 191755abe..45ce72f8b 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/build_steps.py +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/build_steps.py @@ -52,7 +52,7 @@ def requires_tasks(self): "language_deps": AnalyzeLanguageDeps} def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"exaudfclient/base": "exaudfclient/base"} + return {"exaudfclient": "exaudfclient"} def get_path_in_flavor(self): return "flavor_base" @@ -80,7 +80,7 @@ def requires_tasks(self): "language_deps": AnalyzeLanguageDeps} def get_additional_build_directories_mapping(self) -> Dict[str, str]: - return {"exaudfclient/base": "exaudfclient/base", "emulator": "emulator"} + return {"exaudfclient": "exaudfclient", "emulator": "emulator"} def get_path_in_flavor(self): return "flavor_base"