From a712232ac16e237fe80e5739b903285b67239cbd Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Mon, 1 Jul 2024 13:15:11 +0100 Subject: [PATCH 01/15] add the prefix path --- .builders/images/linux-aarch64/build_script.sh | 2 +- .builders/images/linux-x86_64/build_script.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index 0675c46311dce..20676de3b5604 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -23,7 +23,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --enable-sasl --enable-curl + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel diff --git a/.builders/images/linux-x86_64/build_script.sh b/.builders/images/linux-x86_64/build_script.sh index 25cbcd668b0ee..dde744276e862 100644 --- a/.builders/images/linux-x86_64/build_script.sh +++ b/.builders/images/linux-x86_64/build_script.sh @@ -20,7 +20,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --enable-sasl --enable-curl + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel From bb9a50c86f97c0e794aa21e20848bedc6a8e6300 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Mon, 1 Jul 2024 15:36:24 +0100 Subject: [PATCH 02/15] commit --- .builders/images/linux-aarch64/build_script.sh | 2 +- .builders/images/linux-x86_64/build_script.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index 20676de3b5604..0675c46311dce 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -23,7 +23,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl + bash install-from-source.sh --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel diff --git a/.builders/images/linux-x86_64/build_script.sh b/.builders/images/linux-x86_64/build_script.sh index dde744276e862..25cbcd668b0ee 100644 --- a/.builders/images/linux-x86_64/build_script.sh +++ b/.builders/images/linux-x86_64/build_script.sh @@ -20,7 +20,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl + bash install-from-source.sh --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel From b96234dde956298107457f8c721fb2e66e305126 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Mon, 1 Jul 2024 15:48:17 +0100 Subject: [PATCH 03/15] commit --- .builders/images/linux-aarch64/Dockerfile | 4 ++-- .builders/images/linux-aarch64/build_script.sh | 2 +- .builders/images/linux-x86_64/Dockerfile | 2 +- .builders/images/linux-x86_64/build_script.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.builders/images/linux-aarch64/Dockerfile b/.builders/images/linux-aarch64/Dockerfile index 23e4cf6464ffe..0be66bd3ba194 100644 --- a/.builders/images/linux-aarch64/Dockerfile +++ b/.builders/images/linux-aarch64/Dockerfile @@ -137,7 +137,7 @@ RUN \ RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \ # No ./configure, use a NOOP CONFIGURE_SCRIPT="true" \ - bash install-from-source.sh + bash install-from-source.sh RUN \ DOWNLOAD_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v{{version}}/e2fsprogs-{{version}}.tar.gz" \ VERSION="1.47.0" \ @@ -148,7 +148,7 @@ RUN \ # Add -fPIC to let librdkafka link against it statically CFLAGS="${CFLAGS} -fPIC" \ # Explicitly ask the linker to use gssapi_krb5, otherwise static compilation fails - LDFLAGS="${LDFLAGS} -L/usr/local/lib -lgssapi_krb5" \ + LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -L/usr/local/lib -lgssapi_krb5" \ DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \ VERSION="2.1.28" \ SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index 0675c46311dce..b9c05c9ac01ce 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -18,7 +18,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then # thus we extract the version from the requirements file. kafka_version=$(grep 'confluent-kafka==' /home/requirements.in | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/') # Libraries need to be explicitly specified for static linking to work properly - LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ + LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \ VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile index 053008c7c495e..dde04267098ec 100644 --- a/.builders/images/linux-x86_64/Dockerfile +++ b/.builders/images/linux-x86_64/Dockerfile @@ -121,7 +121,7 @@ RUN \ # Add -fPIC to let librdkafka link against it statically CFLAGS="${CFLAGS} -fPIC" \ # Explicitly ask the linker to use gssapi_krb5, otherwise static compilation fails - LDFLAGS="${LDFLAGS} -L/usr/local/lib -lgssapi_krb5" \ + LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -L/usr/local/lib -lgssapi_krb5" \ DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \ VERSION="2.1.28" \ SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ diff --git a/.builders/images/linux-x86_64/build_script.sh b/.builders/images/linux-x86_64/build_script.sh index 25cbcd668b0ee..b43c2811e9e93 100644 --- a/.builders/images/linux-x86_64/build_script.sh +++ b/.builders/images/linux-x86_64/build_script.sh @@ -15,7 +15,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then # thus we extract the version from the requirements file. kafka_version=$(grep 'confluent-kafka==' /home/requirements.in | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/') # Libraries need to be explicitly specified for static linking to work properly - LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ + LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \ VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ From 1f7b7a1fd680b5fe11308562bada5962fb6a55a7 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Mon, 1 Jul 2024 16:04:49 +0100 Subject: [PATCH 04/15] commit --- .builders/images/linux-aarch64/Dockerfile | 6 +++--- .builders/images/linux-aarch64/build_script.sh | 4 ++-- .builders/images/linux-x86_64/Dockerfile | 6 +++--- .builders/images/linux-x86_64/build_script.sh | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.builders/images/linux-aarch64/Dockerfile b/.builders/images/linux-aarch64/Dockerfile index 0be66bd3ba194..db01b2e9efb8c 100644 --- a/.builders/images/linux-aarch64/Dockerfile +++ b/.builders/images/linux-aarch64/Dockerfile @@ -137,7 +137,7 @@ RUN \ RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \ # No ./configure, use a NOOP CONFIGURE_SCRIPT="true" \ - bash install-from-source.sh + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" RUN \ DOWNLOAD_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v{{version}}/e2fsprogs-{{version}}.tar.gz" \ VERSION="1.47.0" \ @@ -148,13 +148,13 @@ RUN \ # Add -fPIC to let librdkafka link against it statically CFLAGS="${CFLAGS} -fPIC" \ # Explicitly ask the linker to use gssapi_krb5, otherwise static compilation fails - LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -L/usr/local/lib -lgssapi_krb5" \ + LDFLAGS="${LDFLAGS} -L/usr/local/lib -lgssapi_krb5" \ DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \ VERSION="2.1.28" \ SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ RELATIVE_PATH="cyrus-sasl-{{version}}" \ bash install-from-source.sh --with-dblib=lmdb --enable-gssapi=/usr/local \ - --enable-static --disable-shared + --enable-static --disable-shared --prefix="${DD_PREFIX_PATH}" # curl RUN \ DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \ diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index b9c05c9ac01ce..747aee80c8f05 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -18,12 +18,12 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then # thus we extract the version from the requirements file. kafka_version=$(grep 'confluent-kafka==' /home/requirements.in | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/') # Libraries need to be explicitly specified for static linking to work properly - LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ + LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \ VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --enable-sasl --enable-curl + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile index dde04267098ec..ee5f3cae8caa2 100644 --- a/.builders/images/linux-x86_64/Dockerfile +++ b/.builders/images/linux-x86_64/Dockerfile @@ -110,7 +110,7 @@ RUN \ RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \ # No ./configure, use a NOOP CONFIGURE_SCRIPT="true" \ - bash install-from-source.sh + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" RUN \ DOWNLOAD_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v{{version}}/e2fsprogs-{{version}}.tar.gz" \ VERSION="1.47.0" \ @@ -121,13 +121,13 @@ RUN \ # Add -fPIC to let librdkafka link against it statically CFLAGS="${CFLAGS} -fPIC" \ # Explicitly ask the linker to use gssapi_krb5, otherwise static compilation fails - LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -L/usr/local/lib -lgssapi_krb5" \ + LDFLAGS="${LDFLAGS} -L/usr/local/lib -lgssapi_krb5" \ DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \ VERSION="2.1.28" \ SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ RELATIVE_PATH="cyrus-sasl-{{version}}" \ bash install-from-source.sh --with-dblib=lmdb --enable-gssapi=/usr/local \ - --enable-static --disable-shared + --enable-static --disable-shared --prefix="${DD_PREFIX_PATH}" # curl RUN \ DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \ diff --git a/.builders/images/linux-x86_64/build_script.sh b/.builders/images/linux-x86_64/build_script.sh index b43c2811e9e93..dde744276e862 100644 --- a/.builders/images/linux-x86_64/build_script.sh +++ b/.builders/images/linux-x86_64/build_script.sh @@ -15,12 +15,12 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then # thus we extract the version from the requirements file. kafka_version=$(grep 'confluent-kafka==' /home/requirements.in | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/') # Libraries need to be explicitly specified for static linking to work properly - LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ + LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \ VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --enable-sasl --enable-curl + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel From 4478ac1d019075fb840ac743baf816fda6f855eb Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Tue, 2 Jul 2024 11:28:45 +0100 Subject: [PATCH 05/15] add debug --- .builders/images/linux-aarch64/Dockerfile | 6 +++--- .builders/images/linux-x86_64/Dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.builders/images/linux-aarch64/Dockerfile b/.builders/images/linux-aarch64/Dockerfile index db01b2e9efb8c..f00521cde0cf9 100644 --- a/.builders/images/linux-aarch64/Dockerfile +++ b/.builders/images/linux-aarch64/Dockerfile @@ -27,7 +27,7 @@ RUN yum install -y openssl-devel && \ INSTALL_COMMAND="make altinstall" \ bash install-from-source.sh \ --prefix=/opt/python/${PYTHON2_VERSION} --with-ensurepip=yes --enable-ipv6 --enable-unicode=ucs4 \ - && yum remove -y openssl-devel + && yum remove -y openssl-devel --debug # Set up virtual environment for Python 2 RUN /opt/python/${PYTHON2_VERSION}/bin/python2.7 -m pip install --no-warn-script-location virtualenv \ @@ -137,7 +137,7 @@ RUN \ RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \ # No ./configure, use a NOOP CONFIGURE_SCRIPT="true" \ - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" + bash install-from-source.sh RUN \ DOWNLOAD_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v{{version}}/e2fsprogs-{{version}}.tar.gz" \ VERSION="1.47.0" \ @@ -154,7 +154,7 @@ RUN \ SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ RELATIVE_PATH="cyrus-sasl-{{version}}" \ bash install-from-source.sh --with-dblib=lmdb --enable-gssapi=/usr/local \ - --enable-static --disable-shared --prefix="${DD_PREFIX_PATH}" + --enable-static --disable-shared # curl RUN \ DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \ diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile index ee5f3cae8caa2..f005ea6673931 100644 --- a/.builders/images/linux-x86_64/Dockerfile +++ b/.builders/images/linux-x86_64/Dockerfile @@ -26,7 +26,7 @@ RUN yum install -y openssl-devel && \ RELATIVE_PATH=Python-{{version}} \ bash install-from-source.sh \ --prefix=/opt/python/${PYTHON2_VERSION} --with-ensurepip=yes --enable-ipv6 --enable-unicode=ucs4 \ - && yum remove -y openssl-devel + && yum remove -y openssl-devel --debug # Set up virtual environment for Python 2 RUN /opt/python/${PYTHON2_VERSION}/bin/python -m pip install --no-warn-script-location virtualenv \ @@ -110,7 +110,7 @@ RUN \ RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \ # No ./configure, use a NOOP CONFIGURE_SCRIPT="true" \ - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" + bash install-from-source.sh RUN \ DOWNLOAD_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v{{version}}/e2fsprogs-{{version}}.tar.gz" \ VERSION="1.47.0" \ @@ -127,7 +127,7 @@ RUN \ SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ RELATIVE_PATH="cyrus-sasl-{{version}}" \ bash install-from-source.sh --with-dblib=lmdb --enable-gssapi=/usr/local \ - --enable-static --disable-shared --prefix="${DD_PREFIX_PATH}" + --enable-static --disable-shared # curl RUN \ DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \ From fed2439c362c308d8fca2facafc109813cea57cb Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Tue, 2 Jul 2024 12:50:58 +0100 Subject: [PATCH 06/15] enable ssl --- .builders/images/linux-aarch64/Dockerfile | 2 +- .builders/images/linux-aarch64/build_script.sh | 2 +- .builders/images/linux-x86_64/Dockerfile | 2 +- .builders/images/linux-x86_64/build_script.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.builders/images/linux-aarch64/Dockerfile b/.builders/images/linux-aarch64/Dockerfile index f00521cde0cf9..23e4cf6464ffe 100644 --- a/.builders/images/linux-aarch64/Dockerfile +++ b/.builders/images/linux-aarch64/Dockerfile @@ -27,7 +27,7 @@ RUN yum install -y openssl-devel && \ INSTALL_COMMAND="make altinstall" \ bash install-from-source.sh \ --prefix=/opt/python/${PYTHON2_VERSION} --with-ensurepip=yes --enable-ipv6 --enable-unicode=ucs4 \ - && yum remove -y openssl-devel --debug + && yum remove -y openssl-devel # Set up virtual environment for Python 2 RUN /opt/python/${PYTHON2_VERSION}/bin/python2.7 -m pip install --no-warn-script-location virtualenv \ diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index 747aee80c8f05..a13bfc1531f82 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -23,7 +23,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl + bash install-from-source.sh --with-ssl=/usr/local --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile index f005ea6673931..48c2d9ee6dbc8 100644 --- a/.builders/images/linux-x86_64/Dockerfile +++ b/.builders/images/linux-x86_64/Dockerfile @@ -26,7 +26,7 @@ RUN yum install -y openssl-devel && \ RELATIVE_PATH=Python-{{version}} \ bash install-from-source.sh \ --prefix=/opt/python/${PYTHON2_VERSION} --with-ensurepip=yes --enable-ipv6 --enable-unicode=ucs4 \ - && yum remove -y openssl-devel --debug + && yum remove -y openssl-devel # Set up virtual environment for Python 2 RUN /opt/python/${PYTHON2_VERSION}/bin/python -m pip install --no-warn-script-location virtualenv \ diff --git a/.builders/images/linux-x86_64/build_script.sh b/.builders/images/linux-x86_64/build_script.sh index dde744276e862..14003d3a93930 100644 --- a/.builders/images/linux-x86_64/build_script.sh +++ b/.builders/images/linux-x86_64/build_script.sh @@ -20,7 +20,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl + bash install-from-source.sh --with-ssl=/usr/local --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel From a7edfbebcb0d9b801d06ecec90091b59ceac5ec5 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Tue, 2 Jul 2024 13:44:38 +0100 Subject: [PATCH 07/15] enable ssl in prefix folder --- .builders/images/linux-aarch64/build_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index a13bfc1531f82..747aee80c8f05 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -23,7 +23,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --with-ssl=/usr/local --enable-sasl --enable-curl + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel From 1a59031e3a7b82ab003e4f1519a35ad46837006f Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Tue, 2 Jul 2024 14:34:45 +0100 Subject: [PATCH 08/15] enable ssl in prefix folder --- .builders/images/linux-x86_64/Dockerfile | 2 +- .builders/images/linux-x86_64/build_script.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile index 48c2d9ee6dbc8..053008c7c495e 100644 --- a/.builders/images/linux-x86_64/Dockerfile +++ b/.builders/images/linux-x86_64/Dockerfile @@ -121,7 +121,7 @@ RUN \ # Add -fPIC to let librdkafka link against it statically CFLAGS="${CFLAGS} -fPIC" \ # Explicitly ask the linker to use gssapi_krb5, otherwise static compilation fails - LDFLAGS="${LDFLAGS} -L/usr/local/lib -lgssapi_krb5" \ + LDFLAGS="${LDFLAGS} -L/usr/local/lib -lgssapi_krb5" \ DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \ VERSION="2.1.28" \ SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ diff --git a/.builders/images/linux-x86_64/build_script.sh b/.builders/images/linux-x86_64/build_script.sh index 14003d3a93930..dde744276e862 100644 --- a/.builders/images/linux-x86_64/build_script.sh +++ b/.builders/images/linux-x86_64/build_script.sh @@ -20,7 +20,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --with-ssl=/usr/local --enable-sasl --enable-curl + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel From e18bf1ce6897ac121a53669c89f8430269895043 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Wed, 3 Jul 2024 12:57:43 +0100 Subject: [PATCH 09/15] set default ssl path --- .builders/images/linux-aarch64/build_script.sh | 2 +- .builders/images/linux-x86_64/build_script.sh | 2 +- kafka_consumer/datadog_checks/kafka_consumer/config.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index 747aee80c8f05..ae90ef2b5b816 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -23,7 +23,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl + bash install-from-source.sh --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel diff --git a/.builders/images/linux-x86_64/build_script.sh b/.builders/images/linux-x86_64/build_script.sh index dde744276e862..25cbcd668b0ee 100644 --- a/.builders/images/linux-x86_64/build_script.sh +++ b/.builders/images/linux-x86_64/build_script.sh @@ -20,7 +20,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl + bash install-from-source.sh --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index c9bfe10bfd2dd..d1b29cfef3f08 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -13,6 +13,8 @@ class KafkaConfig: def __init__(self, init_config, instance, log) -> None: + if os.path.exists('/opt/datadog-agent/embedded/ssl/certs/cacert.pem'): + default_ca_cert_path = '/opt/datadog-agent/embedded/ssl/certs/cacert.pem' self.instance = instance self.init_config = init_config self._context_limit = int(init_config.get('max_partition_contexts', CONTEXT_UPPER_BOUND)) @@ -55,7 +57,7 @@ def __init__(self, init_config, instance, log) -> None: self._sasl_kerberos_keytab = instance.get('sasl_kerberos_keytab', os.environ.get("KRB5_CLIENT_KTNAME")) self._sasl_kerberos_principal = instance.get('sasl_kerberos_principal', 'kafkaclient') self._sasl_oauth_token_provider = instance.get('sasl_oauth_token_provider') - self._tls_ca_cert = instance.get("tls_ca_cert") or instance.get("ssl_cafile") + self._tls_ca_cert = instance.get("tls_ca_cert", default_ca_cert_path) or instance.get("ssl_cafile", default_ca_cert_path) self._tls_cert = instance.get("tls_cert") or instance.get("ssl_certfile") self._tls_private_key = instance.get("tls_private_key") or instance.get("ssl_keyfile") self._tls_private_key_password = instance.get("tls_private_key_password") or instance.get("ssl_password") From 61722164c979eb5e593d52313f49fe9e422d0539 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Wed, 3 Jul 2024 13:03:31 +0100 Subject: [PATCH 10/15] set default ssl path --- kafka_consumer/datadog_checks/kafka_consumer/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index d1b29cfef3f08..d7a16b9cad9e8 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -57,7 +57,9 @@ def __init__(self, init_config, instance, log) -> None: self._sasl_kerberos_keytab = instance.get('sasl_kerberos_keytab', os.environ.get("KRB5_CLIENT_KTNAME")) self._sasl_kerberos_principal = instance.get('sasl_kerberos_principal', 'kafkaclient') self._sasl_oauth_token_provider = instance.get('sasl_oauth_token_provider') - self._tls_ca_cert = instance.get("tls_ca_cert", default_ca_cert_path) or instance.get("ssl_cafile", default_ca_cert_path) + self._tls_ca_cert = instance.get("tls_ca_cert", default_ca_cert_path) or instance.get( + "ssl_cafile", default_ca_cert_path + ) self._tls_cert = instance.get("tls_cert") or instance.get("ssl_certfile") self._tls_private_key = instance.get("tls_private_key") or instance.get("ssl_keyfile") self._tls_private_key_password = instance.get("tls_private_key_password") or instance.get("ssl_password") From 755ac1bb80b808dc4792d64a84ad97f3d3ac2556 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Wed, 3 Jul 2024 14:43:04 +0100 Subject: [PATCH 11/15] set default ssl path --- kafka_consumer/datadog_checks/kafka_consumer/config.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index d7a16b9cad9e8..b682416d33abd 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -13,8 +13,6 @@ class KafkaConfig: def __init__(self, init_config, instance, log) -> None: - if os.path.exists('/opt/datadog-agent/embedded/ssl/certs/cacert.pem'): - default_ca_cert_path = '/opt/datadog-agent/embedded/ssl/certs/cacert.pem' self.instance = instance self.init_config = init_config self._context_limit = int(init_config.get('max_partition_contexts', CONTEXT_UPPER_BOUND)) @@ -57,9 +55,7 @@ def __init__(self, init_config, instance, log) -> None: self._sasl_kerberos_keytab = instance.get('sasl_kerberos_keytab', os.environ.get("KRB5_CLIENT_KTNAME")) self._sasl_kerberos_principal = instance.get('sasl_kerberos_principal', 'kafkaclient') self._sasl_oauth_token_provider = instance.get('sasl_oauth_token_provider') - self._tls_ca_cert = instance.get("tls_ca_cert", default_ca_cert_path) or instance.get( - "ssl_cafile", default_ca_cert_path - ) + self._tls_ca_cert = instance.get("tls_ca_cert") or instance.get("ssl_cafile") self._tls_cert = instance.get("tls_cert") or instance.get("ssl_certfile") self._tls_private_key = instance.get("tls_private_key") or instance.get("ssl_keyfile") self._tls_private_key_password = instance.get("tls_private_key_password") or instance.get("ssl_password") @@ -75,6 +71,9 @@ def __init__(self, init_config, instance, log) -> None: self._tls_verify = "true" else: self._tls_verify = "true" if is_affirmative(instance.get("tls_verify", True)) else "false" + + if not self._tls_ca_cert and os.name != 'nt': + self._tls_ca_cert = '/opt/datadog-agent/embedded/ssl/certs/cacert.pem' def validate_config(self): if not self._kafka_connect_str: From 0c9321fbb38a131179ddc70eac4576d2cad8521a Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Wed, 3 Jul 2024 14:48:07 +0100 Subject: [PATCH 12/15] set default ssl path --- kafka_consumer/datadog_checks/kafka_consumer/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index b682416d33abd..4ec3766df1471 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -71,9 +71,9 @@ def __init__(self, init_config, instance, log) -> None: self._tls_verify = "true" else: self._tls_verify = "true" if is_affirmative(instance.get("tls_verify", True)) else "false" - + if not self._tls_ca_cert and os.name != 'nt': - self._tls_ca_cert = '/opt/datadog-agent/embedded/ssl/certs/cacert.pem' + self._tls_ca_cert = '/opt/datadog-agent/embedded/ssl/certs/cacert.pem' def validate_config(self): if not self._kafka_connect_str: From e610b00914dbbc708155ce05cfc0b9ad5b3572ae Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Wed, 3 Jul 2024 15:58:41 +0100 Subject: [PATCH 13/15] set default ssl path --- kafka_consumer/datadog_checks/kafka_consumer/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index 4ec3766df1471..e63781cdb3cf0 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -72,7 +72,7 @@ def __init__(self, init_config, instance, log) -> None: else: self._tls_verify = "true" if is_affirmative(instance.get("tls_verify", True)) else "false" - if not self._tls_ca_cert and os.name != 'nt': + if not self._tls_ca_cert and os.name != 'nt' and os.path.exists('/opt/datadog-agent/embedded/ssl/certs/cacert.pem'): self._tls_ca_cert = '/opt/datadog-agent/embedded/ssl/certs/cacert.pem' def validate_config(self): From 25e22672827805355933a5b212cbc4232528f150 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Wed, 3 Jul 2024 16:08:51 +0100 Subject: [PATCH 14/15] linter --- kafka_consumer/datadog_checks/kafka_consumer/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index e63781cdb3cf0..a82ffce56ffca 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -72,7 +72,11 @@ def __init__(self, init_config, instance, log) -> None: else: self._tls_verify = "true" if is_affirmative(instance.get("tls_verify", True)) else "false" - if not self._tls_ca_cert and os.name != 'nt' and os.path.exists('/opt/datadog-agent/embedded/ssl/certs/cacert.pem'): + if ( + not self._tls_ca_cert + and os.name != 'nt' + and os.path.exists('/opt/datadog-agent/embedded/ssl/certs/cacert.pem') + ): self._tls_ca_cert = '/opt/datadog-agent/embedded/ssl/certs/cacert.pem' def validate_config(self): From 6201a86d10a1b52dc3f4ba1b5159744aca2d6304 Mon Sep 17 00:00:00 2001 From: HadhemiDD Date: Wed, 3 Jul 2024 16:33:32 +0100 Subject: [PATCH 15/15] linter --- .builders/images/linux-aarch64/build_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index ae90ef2b5b816..0675c46311dce 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -23,7 +23,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then VERSION="${kafka_version}" \ SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ - bash install-from-source.sh --enable-sasl --enable-curl + bash install-from-source.sh --enable-sasl --enable-curl always_build+=("confluent-kafka") # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel