From 104dc71082eab32b10ac7e16999d88daa913112d Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Thu, 17 Aug 2023 11:31:18 +0200 Subject: [PATCH 01/12] fix: Hopefully fixed the build of the cpp-example on windows systems --- example/client-cpp-example/src/CMakeLists.txt | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/example/client-cpp-example/src/CMakeLists.txt b/example/client-cpp-example/src/CMakeLists.txt index 6e9e60e089a8..b3b117c5dda7 100644 --- a/example/client-cpp-example/src/CMakeLists.txt +++ b/example/client-cpp-example/src/CMakeLists.txt @@ -33,12 +33,6 @@ IF (DEFINED BOOST_INCLUDEDIR) INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIR}) ENDIF() -# Add the libs for thrift -IF(MSVC) - SET(THRIFT_STATIC_LIB "${CMAKE_SOURCE_DIR}/thrift/lib/Release/thriftmd.lib") -ELSE() - SET(THRIFT_STATIC_LIB "${CMAKE_SOURCE_DIR}/thrift/lib/libthrift.a") -ENDIF() # Add the libs for the cpp-client LINK_DIRECTORIES(${CMAKE_SOURCE_DIR}/client/lib) @@ -46,8 +40,8 @@ ADD_EXECUTABLE(SessionExample SessionExample.cpp) ADD_EXECUTABLE(AlignedTimeseriesSessionExample AlignedTimeseriesSessionExample.cpp) IF(MSVC) - TARGET_LINK_LIBRARIES(SessionExample iotdb_session thriftmd) - TARGET_LINK_LIBRARIES(AlignedTimeseriesSessionExample iotdb_session thriftmd) + TARGET_LINK_LIBRARIES(SessionExample iotdb_session "${CMAKE_SOURCE_DIR}/thrift/lib/Release/thriftmd.lib) + TARGET_LINK_LIBRARIES(AlignedTimeseriesSessionExample iotdb_session "${CMAKE_SOURCE_DIR}/thrift/lib/Release/thriftmd.lib) ELSE() TARGET_LINK_LIBRARIES(SessionExample iotdb_session pthread) TARGET_LINK_LIBRARIES(AlignedTimeseriesSessionExample iotdb_session pthread) From e1c6f722e95736e64527b64758c7659f2eb6c379 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Tue, 7 Nov 2023 22:31:47 +0100 Subject: [PATCH 02/12] fix: Increase the timeout even more to finally make the build succeed. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index cd3ba4c85997..49d95a32a602 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,7 +38,7 @@ pipeline { } options { - timeout(time: 4, unit: 'HOURS') + timeout(time: 8, unit: 'HOURS') // When we have test-fails e.g. we don't need to run the remaining steps skipStagesAfterUnstable() } From 24f78424f5d7c72e1b6757166e95a5ec20c40407 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Thu, 14 Mar 2024 22:02:37 +0100 Subject: [PATCH 03/12] fix: Fix the build. --- .../src/test/java/org/apache/iotdb/tool/WriteCsvFileTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/WriteCsvFileTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/WriteCsvFileTest.java index 4b491f18cc88..c5b271916936 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/WriteCsvFileTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/WriteCsvFileTest.java @@ -38,7 +38,7 @@ public void writeCsvFileTest() { List row3 = new ArrayList<>(Arrays.asList(3, "100", "hello world!!!", false)); ArrayList> records = new ArrayList<>(Arrays.asList(row1, row2, row3)); - assertTrue(AbstractCsvTool.writeCsvFile(headerNames, records, "./test0.csv")); - assertTrue(AbstractCsvTool.writeCsvFile(null, records, "./test1.csv")); + assertTrue(AbstractCsvTool.writeCsvFile(headerNames, records, "./target/test0.csv")); + assertTrue(AbstractCsvTool.writeCsvFile(null, records, "./target/test1.csv")); } } From 4738ade00c9d26256f83648b5555709537c6a8d7 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Mon, 23 Sep 2024 11:46:31 +0200 Subject: [PATCH 04/12] feat: Added a check that fails the build if the projects list of dependencies doesn't match the one on-file in "dependencies.txt" in the root of the project. --- dependencies.txt | 232 +++++++++++++++++++++++ pom.xml | 81 ++++++++ src/main/groovy/checkDependencies.groovy | 36 ++++ src/main/xslt/sbom-filter.xsl | 67 +++++++ 4 files changed, 416 insertions(+) create mode 100644 dependencies.txt create mode 100644 src/main/groovy/checkDependencies.groovy create mode 100644 src/main/xslt/sbom-filter.xsl diff --git a/dependencies.txt b/dependencies.txt new file mode 100644 index 000000000000..645b426f7028 --- /dev/null +++ b/dependencies.txt @@ -0,0 +1,232 @@ + +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + + +Project libraries: +org.apache.iotdb:client-example +org.apache.iotdb:customize-mqtt-example +org.apache.iotdb:external-api +org.apache.iotdb:iotdb-antlr +org.apache.iotdb:iotdb-api +org.apache.iotdb:iotdb-cli +org.apache.iotdb:iotdb-cli +org.apache.iotdb:iotdb-client +org.apache.iotdb:iotdb-confignode +org.apache.iotdb:iotdb-confignode +org.apache.iotdb:iotdb-consensus +org.apache.iotdb:iotdb-core +org.apache.iotdb:iotdb-distribution +org.apache.iotdb:iotdb-examples +org.apache.iotdb:iotdb-jdbc +org.apache.iotdb:iotdb-metrics +org.apache.iotdb:iotdb-protocol +org.apache.iotdb:iotdb-python-api +org.apache.iotdb:iotdb-relational-grammar +org.apache.iotdb:iotdb-server +org.apache.iotdb:iotdb-server +org.apache.iotdb:iotdb-session +org.apache.iotdb:iotdb-thrift +org.apache.iotdb:iotdb-thrift-ainode +org.apache.iotdb:iotdb-thrift-commons +org.apache.iotdb:iotdb-thrift-confignode +org.apache.iotdb:iotdb-thrift-consensus +org.apache.iotdb:isession +org.apache.iotdb:jdbc-example +org.apache.iotdb:library-udf +org.apache.iotdb:metrics-core +org.apache.iotdb:metrics-interface +org.apache.iotdb:mqtt-example +org.apache.iotdb:node-commons +org.apache.iotdb:openapi +org.apache.iotdb:pipe-api +org.apache.iotdb:pipe-count-point-processor-example +org.apache.iotdb:pipe-opc-ua-sink-example +org.apache.iotdb:rest-java-example +org.apache.iotdb:schema-example +org.apache.iotdb:service-rpc +org.apache.iotdb:trigger-api +org.apache.iotdb:trigger-example +org.apache.iotdb:udf-api +org.apache.iotdb:udf-example + +Dependencies: +cglib:cglib +ch.qos.logback:logback-classic +ch.qos.logback:logback-core +ch.qos.reload4j:reload4j +com.bugsnag:bugsnag +com.digitalpetri.fsm:strict-machine +com.digitalpetri.netty:netty-channel-fsm +com.fasterxml.jackson.core:jackson-annotations +com.fasterxml.jackson.core:jackson-core +com.fasterxml.jackson.core:jackson-databind +com.fasterxml.jackson.dataformat:jackson-dataformat-yaml +com.fasterxml.jackson.datatype:jackson-datatype-jsr310 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-base +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider +com.fasterxml.jackson.module:jackson-module-jaxb-annotations +com.github.ben-manes.caffeine:caffeine +com.github.luben:zstd-jni +com.github.stephenc.jcip:jcip-annotations +com.github.wendykierp:JTransforms +com.google.code.findbugs:jsr305 +com.google.code.gson:gson +com.google.errorprone:error_prone_annotations +com.google.guava:failureaccess +com.google.guava:guava +com.google.guava:listenablefuture +com.google.j2objc:j2objc-annotations +com.h2database:h2-mvstore +com.librato.metrics:librato-java +com.librato.metrics:metrics-librato +com.lmax:disruptor +com.nimbusds:content-type +com.nimbusds:lang-tag +com.nimbusds:nimbus-jose-jwt +com.nimbusds:oauth2-oidc-sdk +com.sun.istack:istack-commons-runtime +com.zaxxer:HikariCP +commons-cli:commons-cli +commons-codec:commons-codec +commons-io:commons-io +commons-logging:commons-logging +io.airlift:airline +io.airlift:concurrent +io.airlift:log +io.airlift:units +io.dropwizard.metrics:metrics-core +io.dropwizard.metrics:metrics-jvm +io.jsonwebtoken:jjwt-api +io.micrometer:micrometer-commons +io.micrometer:micrometer-core +io.micrometer:micrometer-observation +io.moquette:moquette-broker +io.netty:netty-buffer +io.netty:netty-codec +io.netty:netty-codec-dns +io.netty:netty-codec-http +io.netty:netty-codec-http2 +io.netty:netty-codec-mqtt +io.netty:netty-codec-socks +io.netty:netty-common +io.netty:netty-handler +io.netty:netty-handler-proxy +io.netty:netty-resolver +io.netty:netty-resolver-dns +io.netty:netty-resolver-dns-classes-macos +io.netty:netty-resolver-dns-native-macos +io.netty:netty-transport +io.netty:netty-transport-classes-epoll +io.netty:netty-transport-native-epoll +io.netty:netty-transport-native-unix-common +io.projectreactor:reactor-core +io.projectreactor.netty:reactor-netty-core +io.projectreactor.netty:reactor-netty-http +io.swagger:swagger-annotations +io.swagger:swagger-core +io.swagger:swagger-jaxrs +io.swagger:swagger-models +jakarta.activation:jakarta.activation-api +jakarta.annotation:jakarta.annotation-api +jakarta.servlet:jakarta.servlet-api +jakarta.validation:jakarta.validation-api +jakarta.ws.rs:jakarta.ws.rs-api +jakarta.xml.bind:jakarta.xml.bind-api +net.java.dev.jna:jna +net.minidev:accessors-smart +net.minidev:json-smart +org.antlr:antlr4-runtime +org.apache.commons:commons-collections4 +org.apache.commons:commons-csv +org.apache.commons:commons-jexl3 +org.apache.commons:commons-lang3 +org.apache.commons:commons-math3 +org.apache.commons:commons-pool2 +org.apache.httpcomponents:httpclient +org.apache.httpcomponents:httpcore +org.apache.ratis:ratis-client +org.apache.ratis:ratis-common +org.apache.ratis:ratis-grpc +org.apache.ratis:ratis-metrics-api +org.apache.ratis:ratis-proto +org.apache.ratis:ratis-server +org.apache.ratis:ratis-server-api +org.apache.ratis:ratis-thirdparty-misc +org.apache.thrift:libthrift +org.apache.tsfile:common +org.apache.tsfile:tsfile +org.bouncycastle:bcpkix-jdk18on +org.bouncycastle:bcprov-jdk18on +org.bouncycastle:bcutil-jdk18on +org.checkerframework:checker-qual +org.eclipse.collections:eclipse-collections +org.eclipse.collections:eclipse-collections-api +org.eclipse.jetty:jetty-http +org.eclipse.jetty:jetty-io +org.eclipse.jetty:jetty-security +org.eclipse.jetty:jetty-server +org.eclipse.jetty:jetty-servlet +org.eclipse.jetty:jetty-util +org.eclipse.jetty:jetty-util-ajax +org.eclipse.milo:bsd-core +org.eclipse.milo:bsd-generator +org.eclipse.milo:sdk-client +org.eclipse.milo:sdk-core +org.eclipse.milo:sdk-server +org.eclipse.milo:stack-client +org.eclipse.milo:stack-core +org.eclipse.milo:stack-server +org.fusesource.hawtbuf:hawtbuf +org.fusesource.hawtdispatch:hawtdispatch +org.fusesource.hawtdispatch:hawtdispatch-transport +org.fusesource.mqtt-client:mqtt-client +org.glassfish.hk2:hk2-api +org.glassfish.hk2:hk2-locator +org.glassfish.hk2:hk2-utils +org.glassfish.hk2:osgi-resource-locator +org.glassfish.hk2.external:aopalliance-repackaged +org.glassfish.hk2.external:jakarta.inject +org.glassfish.jaxb:jaxb-runtime +org.glassfish.jaxb:txw2 +org.glassfish.jersey.containers:jersey-container-servlet-core +org.glassfish.jersey.core:jersey-client +org.glassfish.jersey.core:jersey-common +org.glassfish.jersey.core:jersey-server +org.glassfish.jersey.inject:jersey-hk2 +org.glassfish.jersey.media:jersey-media-multipart +org.hdrhistogram:HdrHistogram +org.java-websocket:Java-WebSocket +org.javassist:javassist +org.jline:jline +org.jvnet.mimepull:mimepull +org.latencyutils:LatencyUtils +org.lz4:lz4-java +org.ops4j.pax.jdbc:pax-jdbc-common +org.osgi:osgi.cmpn +org.osgi:osgi.core +org.ow2.asm:asm +org.reactivestreams:reactive-streams +org.reflections:reflections +org.slf4j:slf4j-api +org.slf4j:slf4j-reload4j +org.tukaani:xz +org.xerial.snappy:snappy-java +org.yaml:snakeyaml +pl.edu.icm:JLargeArrays diff --git a/pom.xml b/pom.xml index cf45ff69e1a1..1f25e342ce02 100644 --- a/pom.xml +++ b/pom.xml @@ -1360,6 +1360,87 @@ + + + org.cyclonedx + cyclonedx-maven-plugin + + false + + apache-${project.artifactId}-${project.version}-sbom + + + + package + + makeAggregateBom + + + + + + org.codehaus.mojo + xml-maven-plugin + 1.1.0 + + false + + + package + + transform + + + + + ${project.basedir}/target/ + apache-${project.artifactId}-${project.version}-sbom.xml + src/main/xslt/sbom-filter.xsl + ${project.basedir}/target/ + + + .txt + + + + + + + + + + net.sf.saxon + Saxon-HE + 12.5 + + + + + org.codehaus.gmaven + groovy-maven-plugin + 2.1.1 + + false + + + compare-with-reference-list + verify + + execute + + + src/main/groovy/checkDependencies.groovy + + + + + + org.apache.groovy + groovy + 4.0.22 + + + diff --git a/src/main/groovy/checkDependencies.groovy b/src/main/groovy/checkDependencies.groovy new file mode 100644 index 000000000000..736445029536 --- /dev/null +++ b/src/main/groovy/checkDependencies.groovy @@ -0,0 +1,36 @@ +package src.main.groovy +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +var referenceFile = new File(basedir, "dependencies.txt") +if(!referenceFile.exists()) { + throw new RuntimeException("Missing Reference: dependencies.txt") +} +var curBuildFile = new File(project.build.directory, "apache-${project.artifactId}-${project.version}-sbom.txt") +if(!curBuildFile.exists()) { + throw new RuntimeException("Missing Build: apache-${project.artifactId}-${project.version}-sbom.txt") +} + +// Simply compare the content of the two files for equality. +// TODO: Possibly a detailed diff output on the screen might be better, but for now this should do. +String referenceText = referenceFile.text +String buildText = curBuildFile.text +if(!referenceText.equals(buildText)) { + throw new RuntimeException("The content of ${referenceFile.getPath()} and ${curBuildFile.getPath()} do not match") +} diff --git a/src/main/xslt/sbom-filter.xsl b/src/main/xslt/sbom-filter.xsl new file mode 100644 index 000000000000..ffd12c570b36 --- /dev/null +++ b/src/main/xslt/sbom-filter.xsl @@ -0,0 +1,67 @@ + + + + + + + +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +Project libraries: + + + + : + +Dependencies: + + + + : + + + + \ No newline at end of file From f5aa633b4bab88ac46cb0f0962a647a86f3cc7ce Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Mon, 23 Sep 2024 11:47:20 +0200 Subject: [PATCH 05/12] chore: Updated the pyproject.toml file to match the version of the main project. --- iotdb-core/ainode/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml index 6211d2301365..c7f773b05908 100644 --- a/iotdb-core/ainode/pyproject.toml +++ b/iotdb-core/ainode/pyproject.toml @@ -21,7 +21,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "apache-iotdb-ainode" -version = "1.4.0.dev" +version = "2.0.0.dev" description = "Apache IoTDB AINode" readme = "README.md" authors = ["Apache Software Foundation "] From c6a302db234cade5f1b27137e29d3a61b51ca40a Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Tue, 24 Sep 2024 14:06:45 +0200 Subject: [PATCH 06/12] fix: changed the format of the dependencies.txt to json to avoid cross-platform differences --- dependencies.json | 167 ++++++++++++++++ dependencies.txt | 232 ----------------------- pom.xml | 2 +- src/main/groovy/checkDependencies.groovy | 39 +++- src/main/xslt/sbom-filter.xsl | 36 +--- 5 files changed, 205 insertions(+), 271 deletions(-) create mode 100644 dependencies.json delete mode 100644 dependencies.txt diff --git a/dependencies.json b/dependencies.json new file mode 100644 index 000000000000..0c8ff00acdd6 --- /dev/null +++ b/dependencies.json @@ -0,0 +1,167 @@ +{ + "dependencies": [ + "cglib:cglib", + "ch.qos.logback:logback-classic", + "ch.qos.logback:logback-core", + "ch.qos.reload4j:reload4j", + "com.bugsnag:bugsnag", + "com.digitalpetri.fsm:strict-machine", + "com.digitalpetri.netty:netty-channel-fsm", + "com.fasterxml.jackson.core:jackson-annotations", + "com.fasterxml.jackson.core:jackson-core", + "com.fasterxml.jackson.core:jackson-databind", + "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", + "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", + "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base", + "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider", + "com.fasterxml.jackson.module:jackson-module-jaxb-annotations", + "com.github.ben-manes.caffeine:caffeine", + "com.github.luben:zstd-jni", + "com.github.stephenc.jcip:jcip-annotations", + "com.github.wendykierp:JTransforms", + "com.google.code.findbugs:jsr305", + "com.google.code.gson:gson", + "com.google.errorprone:error_prone_annotations", + "com.google.guava:failureaccess", + "com.google.guava:guava", + "com.google.guava:listenablefuture", + "com.google.j2objc:j2objc-annotations", + "com.h2database:h2-mvstore", + "com.librato.metrics:librato-java", + "com.librato.metrics:metrics-librato", + "com.lmax:disruptor", + "com.nimbusds:content-type", + "com.nimbusds:lang-tag", + "com.nimbusds:nimbus-jose-jwt", + "com.nimbusds:oauth2-oidc-sdk", + "com.sun.istack:istack-commons-runtime", + "com.zaxxer:HikariCP", + "commons-cli:commons-cli", + "commons-codec:commons-codec", + "commons-io:commons-io", + "commons-logging:commons-logging", + "io.airlift:airline", + "io.airlift:concurrent", + "io.airlift:log", + "io.airlift:units", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-jvm", + "io.jsonwebtoken:jjwt-api", + "io.micrometer:micrometer-commons", + "io.micrometer:micrometer-core", + "io.micrometer:micrometer-observation", + "io.moquette:moquette-broker", + "io.netty:netty-buffer", + "io.netty:netty-codec", + "io.netty:netty-codec-dns", + "io.netty:netty-codec-http", + "io.netty:netty-codec-http2", + "io.netty:netty-codec-mqtt", + "io.netty:netty-codec-socks", + "io.netty:netty-common", + "io.netty:netty-handler", + "io.netty:netty-handler-proxy", + "io.netty:netty-resolver", + "io.netty:netty-resolver-dns", + "io.netty:netty-resolver-dns-classes-macos", + "io.netty:netty-resolver-dns-native-macos", + "io.netty:netty-transport", + "io.netty:netty-transport-classes-epoll", + "io.netty:netty-transport-native-epoll", + "io.netty:netty-transport-native-unix-common", + "io.projectreactor:reactor-core", + "io.projectreactor.netty:reactor-netty-core", + "io.projectreactor.netty:reactor-netty-http", + "io.swagger:swagger-annotations", + "io.swagger:swagger-core", + "io.swagger:swagger-jaxrs", + "io.swagger:swagger-models", + "jakarta.activation:jakarta.activation-api", + "jakarta.annotation:jakarta.annotation-api", + "jakarta.servlet:jakarta.servlet-api", + "jakarta.validation:jakarta.validation-api", + "jakarta.ws.rs:jakarta.ws.rs-api", + "jakarta.xml.bind:jakarta.xml.bind-api", + "net.java.dev.jna:jna", + "net.minidev:accessors-smart", + "net.minidev:json-smart", + "org.antlr:antlr4-runtime", + "org.apache.commons:commons-collections4", + "org.apache.commons:commons-csv", + "org.apache.commons:commons-jexl3", + "org.apache.commons:commons-lang3", + "org.apache.commons:commons-math3", + "org.apache.commons:commons-pool2", + "org.apache.httpcomponents:httpclient", + "org.apache.httpcomponents:httpcore", + "org.apache.ratis:ratis-client", + "org.apache.ratis:ratis-common", + "org.apache.ratis:ratis-grpc", + "org.apache.ratis:ratis-metrics-api", + "org.apache.ratis:ratis-proto", + "org.apache.ratis:ratis-server", + "org.apache.ratis:ratis-server-api", + "org.apache.ratis:ratis-thirdparty-misc", + "org.apache.thrift:libthrift", + "org.apache.tsfile:common", + "org.apache.tsfile:tsfile", + "org.bouncycastle:bcpkix-jdk18on", + "org.bouncycastle:bcprov-jdk18on", + "org.bouncycastle:bcutil-jdk18on", + "org.checkerframework:checker-qual", + "org.eclipse.collections:eclipse-collections", + "org.eclipse.collections:eclipse-collections-api", + "org.eclipse.jetty:jetty-http", + "org.eclipse.jetty:jetty-io", + "org.eclipse.jetty:jetty-security", + "org.eclipse.jetty:jetty-server", + "org.eclipse.jetty:jetty-servlet", + "org.eclipse.jetty:jetty-util", + "org.eclipse.jetty:jetty-util-ajax", + "org.eclipse.milo:bsd-core", + "org.eclipse.milo:bsd-generator", + "org.eclipse.milo:sdk-client", + "org.eclipse.milo:sdk-core", + "org.eclipse.milo:sdk-server", + "org.eclipse.milo:stack-client", + "org.eclipse.milo:stack-core", + "org.eclipse.milo:stack-server", + "org.fusesource.hawtbuf:hawtbuf", + "org.fusesource.hawtdispatch:hawtdispatch", + "org.fusesource.hawtdispatch:hawtdispatch-transport", + "org.fusesource.mqtt-client:mqtt-client", + "org.glassfish.hk2:hk2-api", + "org.glassfish.hk2:hk2-locator", + "org.glassfish.hk2:hk2-utils", + "org.glassfish.hk2:osgi-resource-locator", + "org.glassfish.hk2.external:aopalliance-repackaged", + "org.glassfish.hk2.external:jakarta.inject", + "org.glassfish.jaxb:jaxb-runtime", + "org.glassfish.jaxb:txw2", + "org.glassfish.jersey.containers:jersey-container-servlet-core", + "org.glassfish.jersey.core:jersey-client", + "org.glassfish.jersey.core:jersey-common", + "org.glassfish.jersey.core:jersey-server", + "org.glassfish.jersey.inject:jersey-hk2", + "org.glassfish.jersey.media:jersey-media-multipart", + "org.hdrhistogram:HdrHistogram", + "org.java-websocket:Java-WebSocket", + "org.javassist:javassist", + "org.jline:jline", + "org.jvnet.mimepull:mimepull", + "org.latencyutils:LatencyUtils", + "org.lz4:lz4-java", + "org.ops4j.pax.jdbc:pax-jdbc-common", + "org.osgi:osgi.cmpn", + "org.osgi:osgi.core", + "org.ow2.asm:asm", + "org.reactivestreams:reactive-streams", + "org.reflections:reflections", + "org.slf4j:slf4j-api", + "org.slf4j:slf4j-reload4j", + "org.tukaani:xz", + "org.xerial.snappy:snappy-java", + "org.yaml:snakeyaml", + "pl.edu.icm:JLargeArrays" + ] +} diff --git a/dependencies.txt b/dependencies.txt deleted file mode 100644 index 645b426f7028..000000000000 --- a/dependencies.txt +++ /dev/null @@ -1,232 +0,0 @@ - -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* https://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ - - -Project libraries: -org.apache.iotdb:client-example -org.apache.iotdb:customize-mqtt-example -org.apache.iotdb:external-api -org.apache.iotdb:iotdb-antlr -org.apache.iotdb:iotdb-api -org.apache.iotdb:iotdb-cli -org.apache.iotdb:iotdb-cli -org.apache.iotdb:iotdb-client -org.apache.iotdb:iotdb-confignode -org.apache.iotdb:iotdb-confignode -org.apache.iotdb:iotdb-consensus -org.apache.iotdb:iotdb-core -org.apache.iotdb:iotdb-distribution -org.apache.iotdb:iotdb-examples -org.apache.iotdb:iotdb-jdbc -org.apache.iotdb:iotdb-metrics -org.apache.iotdb:iotdb-protocol -org.apache.iotdb:iotdb-python-api -org.apache.iotdb:iotdb-relational-grammar -org.apache.iotdb:iotdb-server -org.apache.iotdb:iotdb-server -org.apache.iotdb:iotdb-session -org.apache.iotdb:iotdb-thrift -org.apache.iotdb:iotdb-thrift-ainode -org.apache.iotdb:iotdb-thrift-commons -org.apache.iotdb:iotdb-thrift-confignode -org.apache.iotdb:iotdb-thrift-consensus -org.apache.iotdb:isession -org.apache.iotdb:jdbc-example -org.apache.iotdb:library-udf -org.apache.iotdb:metrics-core -org.apache.iotdb:metrics-interface -org.apache.iotdb:mqtt-example -org.apache.iotdb:node-commons -org.apache.iotdb:openapi -org.apache.iotdb:pipe-api -org.apache.iotdb:pipe-count-point-processor-example -org.apache.iotdb:pipe-opc-ua-sink-example -org.apache.iotdb:rest-java-example -org.apache.iotdb:schema-example -org.apache.iotdb:service-rpc -org.apache.iotdb:trigger-api -org.apache.iotdb:trigger-example -org.apache.iotdb:udf-api -org.apache.iotdb:udf-example - -Dependencies: -cglib:cglib -ch.qos.logback:logback-classic -ch.qos.logback:logback-core -ch.qos.reload4j:reload4j -com.bugsnag:bugsnag -com.digitalpetri.fsm:strict-machine -com.digitalpetri.netty:netty-channel-fsm -com.fasterxml.jackson.core:jackson-annotations -com.fasterxml.jackson.core:jackson-core -com.fasterxml.jackson.core:jackson-databind -com.fasterxml.jackson.dataformat:jackson-dataformat-yaml -com.fasterxml.jackson.datatype:jackson-datatype-jsr310 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider -com.fasterxml.jackson.module:jackson-module-jaxb-annotations -com.github.ben-manes.caffeine:caffeine -com.github.luben:zstd-jni -com.github.stephenc.jcip:jcip-annotations -com.github.wendykierp:JTransforms -com.google.code.findbugs:jsr305 -com.google.code.gson:gson -com.google.errorprone:error_prone_annotations -com.google.guava:failureaccess -com.google.guava:guava -com.google.guava:listenablefuture -com.google.j2objc:j2objc-annotations -com.h2database:h2-mvstore -com.librato.metrics:librato-java -com.librato.metrics:metrics-librato -com.lmax:disruptor -com.nimbusds:content-type -com.nimbusds:lang-tag -com.nimbusds:nimbus-jose-jwt -com.nimbusds:oauth2-oidc-sdk -com.sun.istack:istack-commons-runtime -com.zaxxer:HikariCP -commons-cli:commons-cli -commons-codec:commons-codec -commons-io:commons-io -commons-logging:commons-logging -io.airlift:airline -io.airlift:concurrent -io.airlift:log -io.airlift:units -io.dropwizard.metrics:metrics-core -io.dropwizard.metrics:metrics-jvm -io.jsonwebtoken:jjwt-api -io.micrometer:micrometer-commons -io.micrometer:micrometer-core -io.micrometer:micrometer-observation -io.moquette:moquette-broker -io.netty:netty-buffer -io.netty:netty-codec -io.netty:netty-codec-dns -io.netty:netty-codec-http -io.netty:netty-codec-http2 -io.netty:netty-codec-mqtt -io.netty:netty-codec-socks -io.netty:netty-common -io.netty:netty-handler -io.netty:netty-handler-proxy -io.netty:netty-resolver -io.netty:netty-resolver-dns -io.netty:netty-resolver-dns-classes-macos -io.netty:netty-resolver-dns-native-macos -io.netty:netty-transport -io.netty:netty-transport-classes-epoll -io.netty:netty-transport-native-epoll -io.netty:netty-transport-native-unix-common -io.projectreactor:reactor-core -io.projectreactor.netty:reactor-netty-core -io.projectreactor.netty:reactor-netty-http -io.swagger:swagger-annotations -io.swagger:swagger-core -io.swagger:swagger-jaxrs -io.swagger:swagger-models -jakarta.activation:jakarta.activation-api -jakarta.annotation:jakarta.annotation-api -jakarta.servlet:jakarta.servlet-api -jakarta.validation:jakarta.validation-api -jakarta.ws.rs:jakarta.ws.rs-api -jakarta.xml.bind:jakarta.xml.bind-api -net.java.dev.jna:jna -net.minidev:accessors-smart -net.minidev:json-smart -org.antlr:antlr4-runtime -org.apache.commons:commons-collections4 -org.apache.commons:commons-csv -org.apache.commons:commons-jexl3 -org.apache.commons:commons-lang3 -org.apache.commons:commons-math3 -org.apache.commons:commons-pool2 -org.apache.httpcomponents:httpclient -org.apache.httpcomponents:httpcore -org.apache.ratis:ratis-client -org.apache.ratis:ratis-common -org.apache.ratis:ratis-grpc -org.apache.ratis:ratis-metrics-api -org.apache.ratis:ratis-proto -org.apache.ratis:ratis-server -org.apache.ratis:ratis-server-api -org.apache.ratis:ratis-thirdparty-misc -org.apache.thrift:libthrift -org.apache.tsfile:common -org.apache.tsfile:tsfile -org.bouncycastle:bcpkix-jdk18on -org.bouncycastle:bcprov-jdk18on -org.bouncycastle:bcutil-jdk18on -org.checkerframework:checker-qual -org.eclipse.collections:eclipse-collections -org.eclipse.collections:eclipse-collections-api -org.eclipse.jetty:jetty-http -org.eclipse.jetty:jetty-io -org.eclipse.jetty:jetty-security -org.eclipse.jetty:jetty-server -org.eclipse.jetty:jetty-servlet -org.eclipse.jetty:jetty-util -org.eclipse.jetty:jetty-util-ajax -org.eclipse.milo:bsd-core -org.eclipse.milo:bsd-generator -org.eclipse.milo:sdk-client -org.eclipse.milo:sdk-core -org.eclipse.milo:sdk-server -org.eclipse.milo:stack-client -org.eclipse.milo:stack-core -org.eclipse.milo:stack-server -org.fusesource.hawtbuf:hawtbuf -org.fusesource.hawtdispatch:hawtdispatch -org.fusesource.hawtdispatch:hawtdispatch-transport -org.fusesource.mqtt-client:mqtt-client -org.glassfish.hk2:hk2-api -org.glassfish.hk2:hk2-locator -org.glassfish.hk2:hk2-utils -org.glassfish.hk2:osgi-resource-locator -org.glassfish.hk2.external:aopalliance-repackaged -org.glassfish.hk2.external:jakarta.inject -org.glassfish.jaxb:jaxb-runtime -org.glassfish.jaxb:txw2 -org.glassfish.jersey.containers:jersey-container-servlet-core -org.glassfish.jersey.core:jersey-client -org.glassfish.jersey.core:jersey-common -org.glassfish.jersey.core:jersey-server -org.glassfish.jersey.inject:jersey-hk2 -org.glassfish.jersey.media:jersey-media-multipart -org.hdrhistogram:HdrHistogram -org.java-websocket:Java-WebSocket -org.javassist:javassist -org.jline:jline -org.jvnet.mimepull:mimepull -org.latencyutils:LatencyUtils -org.lz4:lz4-java -org.ops4j.pax.jdbc:pax-jdbc-common -org.osgi:osgi.cmpn -org.osgi:osgi.core -org.ow2.asm:asm -org.reactivestreams:reactive-streams -org.reflections:reflections -org.slf4j:slf4j-api -org.slf4j:slf4j-reload4j -org.tukaani:xz -org.xerial.snappy:snappy-java -org.yaml:snakeyaml -pl.edu.icm:JLargeArrays diff --git a/pom.xml b/pom.xml index 1f25e342ce02..1416fa5b831e 100644 --- a/pom.xml +++ b/pom.xml @@ -1399,7 +1399,7 @@ ${project.basedir}/target/ - .txt + transformed.json diff --git a/src/main/groovy/checkDependencies.groovy b/src/main/groovy/checkDependencies.groovy index 736445029536..632d362ef000 100644 --- a/src/main/groovy/checkDependencies.groovy +++ b/src/main/groovy/checkDependencies.groovy @@ -18,19 +18,38 @@ package src.main.groovy * under the License. */ -var referenceFile = new File(basedir, "dependencies.txt") +import groovy.json.JsonSlurper + +def jsonSlurper = new JsonSlurper() + +var referenceFile = new File(basedir, "dependencies.json") if(!referenceFile.exists()) { - throw new RuntimeException("Missing Reference: dependencies.txt") + throw new RuntimeException("Missing Reference: dependencies.json") } -var curBuildFile = new File(project.build.directory, "apache-${project.artifactId}-${project.version}-sbom.txt") +def referenceJson = jsonSlurper.parse(referenceFile) + +var curBuildFile = new File(project.build.directory, "apache-${project.artifactId}-${project.version}-sbom.transformed.json") if(!curBuildFile.exists()) { - throw new RuntimeException("Missing Build: apache-${project.artifactId}-${project.version}-sbom.txt") + throw new RuntimeException("Missing Build: apache-${project.artifactId}-${project.version}-sbom.transformed.json") } +def curBuildJson = jsonSlurper.parse(curBuildFile) -// Simply compare the content of the two files for equality. -// TODO: Possibly a detailed diff output on the screen might be better, but for now this should do. -String referenceText = referenceFile.text -String buildText = curBuildFile.text -if(!referenceText.equals(buildText)) { - throw new RuntimeException("The content of ${referenceFile.getPath()} and ${curBuildFile.getPath()} do not match") +def differencesFound = false +referenceJson.dependencies.each { + if(!curBuildJson.dependencies.contains(it)) { + println "current build has removed a previously existing dependency: " + it + differencesFound = true + } } +curBuildJson.dependencies.each { + if(!referenceJson.dependencies.contains(it)) { + println "current build has added a new dependency: " + it + differencesFound = true + } +} + +if(differencesFound) { + println "Differences were found between the information in ${referenceFile.getPath()} and ${curBuildFile.toPath()}" + println "The simplest fix for this, is to replace the content of ${referenceFile.getPath()} with that of ${curBuildFile.toPath()} and to inspect the diff of the resulting file in your IDE of choice." + throw new RuntimeException("Differences found.") +} \ No newline at end of file diff --git a/src/main/xslt/sbom-filter.xsl b/src/main/xslt/sbom-filter.xsl index ffd12c570b36..0e8f7d57a792 100644 --- a/src/main/xslt/sbom-filter.xsl +++ b/src/main/xslt/sbom-filter.xsl @@ -21,7 +21,6 @@ --> - -/* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* https://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ - -Project libraries: - + { + "dependencies": [ - : + ":", + ] +} - \ No newline at end of file From 7681b71bbafbec849da00f88d42a9586de372eb9 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Tue, 24 Sep 2024 14:26:34 +0200 Subject: [PATCH 07/12] feat: Added a switch to skip the dependencies check. --- pom.xml | 4 ++++ src/main/groovy/checkDependencies.groovy | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1416fa5b831e..852047118172 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ 2.11.1 4.4 false + false 3.4.4 1.21.1 4.2.19 @@ -1429,6 +1430,9 @@ execute + + ${dependencyCheck.skip} + src/main/groovy/checkDependencies.groovy diff --git a/src/main/groovy/checkDependencies.groovy b/src/main/groovy/checkDependencies.groovy index 632d362ef000..8df42473d349 100644 --- a/src/main/groovy/checkDependencies.groovy +++ b/src/main/groovy/checkDependencies.groovy @@ -20,6 +20,11 @@ package src.main.groovy import groovy.json.JsonSlurper +if(Boolean.parseBoolean(properties['skipDependencyCheck']).booleanValue()) { + println "Skipping dependency check" + return +} + def jsonSlurper = new JsonSlurper() var referenceFile = new File(basedir, "dependencies.json") @@ -37,13 +42,13 @@ def curBuildJson = jsonSlurper.parse(curBuildFile) def differencesFound = false referenceJson.dependencies.each { if(!curBuildJson.dependencies.contains(it)) { - println "current build has removed a previously existing dependency: " + it + println "current build has removed a dependency: " + it differencesFound = true } } curBuildJson.dependencies.each { if(!referenceJson.dependencies.contains(it)) { - println "current build has added a new dependency: " + it + println "current build has added a dependency: " + it differencesFound = true } } From 6534bbc0f79ff11d325e0dd396bff4b915c50ee2 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Tue, 24 Sep 2024 15:01:29 +0200 Subject: [PATCH 08/12] chore: Disabled the dependency check per default and added a github workflow for running the test. --- .github/workflows/dependency-check.yml | 59 ++++++++++++++++++++++++++ pom.xml | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/dependency-check.yml diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml new file mode 100644 index 000000000000..c285b6fd73dc --- /dev/null +++ b/.github/workflows/dependency-check.yml @@ -0,0 +1,59 @@ +# This workflow will check if dependencies have changed (adding new dependencies or removing existing ones) + +name: Dependency Check + +on: + push: + branches: + - master + - 'rel/*' + - "rc/*" + paths-ignore: + - 'docs/**' + - 'site/**' + pull_request: + branches: + - master + - 'rel/*' + - "rc/*" + paths-ignore: + - 'docs/**' + - 'site/**' + # allow manually run the action: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + +jobs: + dependency-check: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: corretto + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Test Datanode Module with Maven + shell: bash + run: mvn verify -Dmaven.test.skip=true -DdependencyCheck.skip=false -Dmdep.analyze.skip=true diff --git a/pom.xml b/pom.xml index 852047118172..6ef32d6c1096 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,7 @@ 2.11.1 4.4 false - false + true 3.4.4 1.21.1 4.2.19 From 782919763629052bf1c1819e1b267cd1426bf427 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Tue, 24 Sep 2024 15:01:52 +0200 Subject: [PATCH 09/12] chore: Disabled the dependency check per default and added a github workflow for running the test. --- .github/workflows/dependency-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index c285b6fd73dc..21bce48887da 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -54,6 +54,6 @@ jobs: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2- - - name: Test Datanode Module with Maven + - name: Do the dependency check shell: bash run: mvn verify -Dmaven.test.skip=true -DdependencyCheck.skip=false -Dmdep.analyze.skip=true From 3817d729adbd09ef2fe0b574677acbf5d24768f0 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Tue, 24 Sep 2024 15:21:45 +0200 Subject: [PATCH 10/12] chore: Added a dummy dependency to trigger a dependency check failure --- iotdb-core/datanode/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index b85da1476721..101404b9a8e0 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -378,6 +378,12 @@ awaitility test + + + org.apache.plc4x + plc4j-spi + 0.12.0 + From 17241bc8a3ac051ed1923344204d8b7223772216 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Tue, 24 Sep 2024 15:25:12 +0200 Subject: [PATCH 11/12] chore: Removed the dummy dependency to trigger a dependency check failure after it has proven to work --- iotdb-core/datanode/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index 101404b9a8e0..b85da1476721 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -378,12 +378,6 @@ awaitility test - - - org.apache.plc4x - plc4j-spi - 0.12.0 - From f3d839d651abc928a201390ceab0b0079e5dc074 Mon Sep 17 00:00:00 2001 From: Christofer Dutz Date: Wed, 25 Sep 2024 09:32:00 +0200 Subject: [PATCH 12/12] chore: Removed the commented out parts of the xslt --- src/main/xslt/sbom-filter.xsl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/xslt/sbom-filter.xsl b/src/main/xslt/sbom-filter.xsl index 0e8f7d57a792..3450837c23a9 100644 --- a/src/main/xslt/sbom-filter.xsl +++ b/src/main/xslt/sbom-filter.xsl @@ -29,13 +29,7 @@ encoding="utf-8" /> - { + { "dependencies": [