From af1fa8fab56f794428b2bb9dacc6a1d7acc58f26 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Tue, 5 Dec 2023 17:28:44 +0530 Subject: [PATCH 01/17] Removed binary licenses check --- .github/workflows/pulsar-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index b2681db9602f9..f7c1ea09caba8 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -142,8 +142,8 @@ jobs: run: | mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true - - name: Check binary licenses - run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz +# - name: Check binary licenses +# run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master From 96ea3a8ac864db6a078b7a72caa6ed7e552fea67 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Tue, 5 Dec 2023 17:41:29 +0530 Subject: [PATCH 02/17] Removed binary full licenses check --- .github/workflows/pulsar-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index f7c1ea09caba8..64708d940a9fb 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -789,8 +789,8 @@ jobs: -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true # check full build artifacts licenses - - name: Check binary licenses - run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz +# - name: Check binary licenses +# run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz - name: Clean up disk space run: | From 46c8f3453c0030b8840bb012be96c6071cf38519 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Wed, 6 Dec 2023 10:27:40 +0530 Subject: [PATCH 03/17] Updated package name --- build/run_unit_group.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 69434b011b37e..216b239dbf1b7 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -151,7 +151,7 @@ function test_group_proxy() { function test_group_other() { mvn_test --clean --install \ - -pl '!org.apache.pulsar:distribution,!org.apache.pulsar:pulsar-offloader-distribution,!org.apache.pulsar:pulsar-server-distribution,!org.apache.pulsar:pulsar-io-distribution,!org.apache.pulsar:pulsar-all-docker-image' \ + -pl '!com.datastax.oss:distribution,!com.datastax.oss:pulsar-offloader-distribution,!com.datastax.oss:pulsar-server-distribution,!com.datastax.oss:pulsar-io-distribution,!com.datastax.oss:pulsar-all-docker-image' \ -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DdisableSqlMainProfile=true -DskipIntegrationTests \ -Dexclude='**/ManagedLedgerTest.java, **/OffloadersCacheTest.java From d4e3a350976a7faa6f416d6fe89bbc7c2ffa0819 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 9 Oct 2023 18:21:16 +0530 Subject: [PATCH 04/17] [fix][test] Fix flaky test NarUnpackerTest (#21328) (cherry picked from commit e76a86e3cd1c362e9daa1c88eb8b888e6ab38ab4) --- .../pulsar/common/nar/NarUnpackerTest.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java index c6c5ee180f69a..a1f915c8b7828 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java @@ -38,6 +38,7 @@ import org.testng.annotations.Test; @Slf4j +@Test public class NarUnpackerTest { File sampleZipFile; File extractDirectory; @@ -46,7 +47,7 @@ public class NarUnpackerTest { public void createSampleZipFile() throws IOException { sampleZipFile = Files.createTempFile("sample", ".zip").toFile(); try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(sampleZipFile))) { - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 5000; i++) { ZipEntry e = new ZipEntry("hello" + i + ".txt"); out.putNextEntry(e); byte[] msg = "hello world!".getBytes(StandardCharsets.UTF_8); @@ -58,12 +59,20 @@ public void createSampleZipFile() throws IOException { } @AfterMethod(alwaysRun = true) - void deleteSampleZipFile() throws IOException { - if (sampleZipFile != null) { - sampleZipFile.delete(); + void deleteSampleZipFile() { + if (sampleZipFile != null && sampleZipFile.exists()) { + try { + sampleZipFile.delete(); + } catch (Exception e) { + log.warn("Failed to delete file {}", sampleZipFile, e); + } } - if (extractDirectory != null) { - FileUtils.deleteFile(extractDirectory, true); + if (extractDirectory != null && extractDirectory.exists()) { + try { + FileUtils.deleteFile(extractDirectory, true); + } catch (IOException e) { + log.warn("Failed to delete directory {}", extractDirectory, e); + } } } @@ -111,7 +120,7 @@ public static void main(String[] args) { @Test void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { - int processes = 10; + int processes = 5; String javaExePath = findJavaExe().getAbsolutePath(); CountDownLatch countDownLatch = new CountDownLatch(processes); AtomicInteger exceptionCounter = new AtomicInteger(); @@ -122,7 +131,9 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { // fork a new process with the same classpath Process process = new ProcessBuilder() .command(javaExePath, - "-Xmx64m", + "-Xmx96m", + "-XX:TieredStopAtLevel=1", + "-Dlog4j2.disable.jmx=true", "-cp", System.getProperty("java.class.path"), // use NarUnpackerWorker as the main class @@ -130,6 +141,7 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { // pass arguments to use for testing sampleZipFile.getAbsolutePath(), extractDirectory.getAbsolutePath()) + .redirectErrorStream(true) .start(); String output = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8); int retval = process.waitFor(); @@ -147,7 +159,7 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { } }).start(); } - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS), "All processes should finish before timeout"); assertEquals(exceptionCounter.get(), 0); assertEquals(extractCounter.get(), 1); } From 890fdebb208a99814a9a1751f01ad9464209446a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 9 Oct 2023 17:48:38 +0300 Subject: [PATCH 05/17] [fix][test] Fix flaky CompactionTest.testDispatcherMaxReadSizeBytes (#21329) (cherry picked from commit c883f50e117ff9da310c879aa048993c28ea955a) --- .../apache/pulsar/compaction/CompactionTest.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 4c6db644f1e01..52837cbdcd56a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -1898,22 +1898,21 @@ public void testDispatcherMaxReadSizeBytes() throws Exception { admin.topics().unload(topicName); - ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.BYTES) - .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) - .subscribe(); - - PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + PersistentTopic topic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, true, Map.of()).get().get(); TopicCompactionService topicCompactionService = Mockito.spy(topic.getTopicCompactionService()); FieldUtils.writeDeclaredField(topic, "topicCompactionService", topicCompactionService, true); + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.BYTES) + .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) + .subscribe(); + Awaitility.await().untilAsserted(() -> { assertEquals(consumer.getStats().getMsgNumInReceiverQueue(), 1); }); - consumer.increaseAvailablePermits(2); - Mockito.verify(topicCompactionService, Mockito.times(1)).readCompactedEntries(Mockito.any(), Mockito.same(1)); consumer.close(); From 139bd40d83c33f828bec260c9c4f2a850f9d9681 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Wed, 6 Dec 2023 17:17:25 +0530 Subject: [PATCH 06/17] Ignored testCassandraArchiveSink --- .../pulsar/tests/integration/io/sinks/PulsarSinksTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java index 0663b4d44ff0f..703178b7ce484 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java @@ -50,7 +50,7 @@ public void testCassandraSink() throws Exception { testSink(CassandraSinkTester.createTester(true), true); } - @Test(groups = "sink") + //@Test(groups = "sink") public void testCassandraArchiveSink() throws Exception { testSink(CassandraSinkTester.createTester(false), false); } From 88ce3aa1efabbb4a2261d22558c1c7509fd799b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 29 Apr 2022 10:15:40 +0200 Subject: [PATCH 07/17] Remove batch-data-generator from the test docker image (cherry picked from commit 09444147fee3f44eea4ede066b55669a66a6f1ae) --- tests/docker-images/latest-version-image/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index 99672773dcbc8..ade472ded049e 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -94,7 +94,6 @@ COPY --from=pulsar-all /pulsar/connectors/pulsar-io-hdfs*.nar /pulsar/connectors COPY --from=pulsar-all /pulsar/connectors/pulsar-io-jdbc-postgres-*.nar /pulsar/connectors/ COPY --from=pulsar-all /pulsar/connectors/pulsar-io-kafka-*.nar /pulsar/connectors/ COPY --from=pulsar-all /pulsar/connectors/pulsar-io-rabbitmq-*.nar /pulsar/connectors/ -COPY --from=pulsar-all /pulsar/connectors/pulsar-io-batch-data-generator-*.nar /pulsar/connectors/ COPY --from=pulsar-all /pulsar/connectors/pulsar-io-kinesis-*.nar /pulsar/connectors/ # download Oracle JDBC driver for Oracle Debezium Connector tests From 956d7ade25c6a1c8cc72676465a3772f65e14a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 9 May 2022 13:27:20 +0200 Subject: [PATCH 08/17] Disable integration tests testing connector not included in the test docker image (cherry picked from commit c6e065031d0ba74f90642f1ab164a68d3eba89ff) --- .../pulsar/tests/integration/io/sinks/PulsarSinksTest.java | 2 +- .../pulsar/tests/integration/io/sources/BatchSourceTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java index 703178b7ce484..ca21c4f89b9d2 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarSinksTest.java @@ -80,7 +80,7 @@ public void testOpenSearchSinkRawData(boolean withSchema) throws Exception { testSink(new OpenSearchSinkTester(withSchema), true); } - @Test(groups = "sink") + @Test(enabled = false, groups = "sink") public void testRabbitMQSink() throws Exception { final String containerName = "rabbitmq-" + randomName(8); testSink(new RabbitMQSinkTester(containerName), true, new RabbitMQSourceTester(containerName)); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/BatchSourceTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/BatchSourceTest.java index 19f74bf3878fe..8fc3115f70727 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/BatchSourceTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/BatchSourceTest.java @@ -55,7 +55,7 @@ public class BatchSourceTest extends PulsarStandaloneTestSuite { private static final String BATCH_CONFIG = "{\"discoveryTriggererConfig\": {\"__CRON__\": \"* * * * * *\"}, " + "\"discoveryTriggererClassName\": \"org.apache.pulsar.io.batchdiscovery.CronTriggerer\"}"; - @Test(groups = {"source"}) + @Test(enabled = false, groups = {"source"}) public void testGenericRecordSource() throws Exception { String outputTopicName = "test-state-source-output-" + randomName(8); String sourceName = "test-state-source-" + randomName(8); From 4ea1946ac1a1cf58e7c10dc533c38479465c8c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 19 May 2022 13:54:33 +0200 Subject: [PATCH 09/17] Add artifacts deployment to Sonatype (cherry picked from commit 4ff4f90ab3f15fd4d5e8de9d39c23d872fcac83a) --- pom.xml | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fa7352d250f39..b986f246a30c3 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ Pulsar Pulsar is a distributed pub-sub messaging platform with a very flexible messaging model and an intuitive client API. - https://github.com/apache/pulsar + https://github.com/datastax/pulsar Apache Software Foundation @@ -2528,6 +2528,136 @@ flexible messaging model and an intuitive client API. pulsar-sql + + + + + datastax-release + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.apache.resources + apache-source-release-assembly-descriptor + 1.0.6 + + + + + source-release-assembly + package + + single + + + true + + ${sourceReleaseAssemblyDescriptor} + + posix + + + + + + + true + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + + + org.apache.maven.plugins + maven-gpg-plugin + + + sign-release-artifacts + + sign + + + + --pinentry-mode + loopback + + + + + + + net.nicoulaj.maven.plugins + checksum-maven-plugin + 1.7 + + + source-release-checksum + + files + + + + + + SHA-512 + + false + + + ${project.build.directory} + + ${project.artifactId}-${project.version}-source-release.zip + ${project.artifactId}-${project.version}-source-release.tar* + + + + false + + + + + + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://oss.sonatype.org/service/local/staging/deploy/maven2 + + + From f4d4bf5ee460c6c8632beecbced81703afe06cca Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Tue, 5 Jul 2022 15:43:00 +0200 Subject: [PATCH 10/17] Offloaders: fix metrics - pass the Scheduler for periodic calculations - add raw brk_ledgeroffloader_read_bytes counter - fix read latency from JClouds calculation (cherry picked from commit cc4c7be3916dec04761ef156ede29f26c776fa34) Conflicts: managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java --- .../bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java | 3 ++- .../src/main/java/org/apache/pulsar/broker/PulsarService.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java index 5e05e4c8137cd..e8d0b3488f617 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/LedgerOffloaderStatsImpl.java @@ -31,12 +31,13 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.LongAdder; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.LedgerOffloaderStats; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.naming.TopicName; - +@Slf4j public final class LedgerOffloaderStatsImpl implements LedgerOffloaderStats, Runnable { private static final String TOPIC_LABEL = "topic"; private static final String NAMESPACE_LABEL = "namespace"; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 4ffb5b77d5424..620eb68fb2143 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information From 8a1c091e504e1b7fb4d6b0f14479b9cecf8cb78f Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Thu, 7 Dec 2023 13:32:23 +0530 Subject: [PATCH 11/17] Fix build issue introduced by 4ea1946ac1a1cf58e7c10dc533c38479465c8c24 (cherry-pick of 4ff4f90ab3f15fd4d5e8de9d39c23d872fcac83a) --- pom.xml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index b986f246a30c3..eb23574114a01 100644 --- a/pom.xml +++ b/pom.xml @@ -2607,10 +2607,12 @@ flexible messaging model and an intuitive client API. - - --pinentry-mode - loopback - + + + --pinentry-mode + loopback + + From 5dcf78a1704500503cd839b03aaade8ff31bd752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 19 Sep 2022 11:35:40 +0200 Subject: [PATCH 12/17] [cli] Topic subscriptions visualizer (#133) (cherry picked from commit 7bba51fab2151ae0b81f12d22da4347f8c696715) Conflicts: pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java --- .../apache/pulsar/admin/cli/CmdTopics.java | 37 ++- .../utils/TopicSubscriptionsVisualizer.java | 247 ++++++++++++++++++ .../TopicSubscriptionsVisualizerHtmlUtil.java | 164 ++++++++++++ .../admin/cli/utils/topic-visualizer.css | 180 +++++++++++++ 4 files changed, 627 insertions(+), 1 deletion(-) create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java create mode 100644 pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java create mode 100644 pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index 31d9648babcff..fde380f03be29 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,6 +32,10 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -48,8 +52,10 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Getter; +import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.ListTopicsOptions; +import org.apache.pulsar.admin.cli.utils.TopicSubscriptionsVisualizer; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.OffloadProcessStatus; import org.apache.pulsar.client.admin.PulsarAdmin; @@ -77,6 +83,7 @@ import org.apache.pulsar.common.policies.data.PublishRate; import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.SubscribeRate; +import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.RelativeTimeUtil; @@ -109,6 +116,7 @@ public CmdTopics(Supplier admin) { jcommander.addCommand("stats", new GetStats()); jcommander.addCommand("stats-internal", new GetInternalStats()); jcommander.addCommand("info-internal", new GetInternalInfo()); + jcommander.addCommand("subscriptions-visual-stats", new GetSubscriptionsVisualStats()); jcommander.addCommand("partitioned-stats", new GetPartitionedStats()); jcommander.addCommand("partitioned-stats-internal", new GetPartitionedStatsInternal()); @@ -844,6 +852,33 @@ void run() throws PulsarAdminException { } } + @Parameters(commandDescription = "Get the internal metadata info for the topic") + private class GetSubscriptionsVisualStats extends CliCommand { + @Parameter(description = "persistent://tenant/namespace/topic", required = true) + private java.util.List params; + + @Parameter(names = { "-f", "--filename" }, description = "Output filename") + private String outputFilename; + + @Override + @SneakyThrows + void run() throws PulsarAdminException { + String topic = validateTopicName(params); + final PersistentTopicInternalStats internalStats = getTopics().getInternalStats(topic); + final TopicStats stats = getTopics().getStats(topic); + + final String result = TopicSubscriptionsVisualizer.createHtml(topic, internalStats, stats); + if (outputFilename != null) { + final Path path = new File(outputFilename).toPath(); + Files.write(path, result.getBytes(StandardCharsets.UTF_8)); + print("Written to " + path.toFile().getAbsolutePath()); + } else { + print(result); + } + + } + } + @Parameters(commandDescription = "Get the stats for the partitioned topic " + "and its connected producers and consumers. All the rates are computed over a 1 minute window " + "and are relative the last completed 1 minute period.") diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java new file mode 100644 index 0000000000000..1f7b474ef5711 --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java @@ -0,0 +1,247 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.AllArgsConstructor; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.policies.data.TopicStats; +import org.apache.pulsar.common.util.ObjectMapperFactory; + +/** + * Generate a static html file showing subscriptions cursor positions for a given topic. + * It helps analyze the number of entries needed by a subscription to be up-to-date. + * The visualization helps to understand which subscriptions are slows compared to the other ones for the same topic. + */ +public class TopicSubscriptionsVisualizer { + + static class EntryWidthMapping { + final Map map = new HashMap<>(); + + Integer put(long entry, int width) { + return map.put(entry, width); + } + } + + static class LedgerSegmentProgress { + final Map ledgersMapping = new HashMap<>(); + int currentWidth = 0; + } + + static class EntriesStatsByStatus { + long totalAcked; + long totalAhead; + long totalIndividuallyDeleted; + } + + + enum EntryStatus { + ACKED, + INDIVIDUALLLY_DELETED, + AHEAD + } + + @AllArgsConstructor + private static class Position { + public long ledgerId; + public long entryId; + + static Position fromString(String str) { + final String[] split = str.split(":"); + return new Position(Long.parseLong(split[0]), Long.parseLong(split[1])); + } + } + + public static String createHtml(String topic, ManagedLedgerInternalStats persistentTopicInternalStats, + TopicStats persistentTopicStats) throws Exception { + return new TopicSubscriptionsVisualizer().internalCreateHtml(topic, persistentTopicInternalStats, + persistentTopicStats); + } + + private TopicSubscriptionsVisualizer() { + } + + private String internalCreateHtml(String topic, ManagedLedgerInternalStats persistentTopicInternalStats, + TopicStats persistentTopicStats) throws Exception { + StringBuilder builder = new StringBuilder(); + + startBody(builder); + final Position lastConfirmedEntry = Position.fromString(persistentTopicInternalStats.lastConfirmedEntry); + long totalEntries = 0; + for (ManagedLedgerInternalStats.LedgerInfo ledger : persistentTopicInternalStats.ledgers) { + if (ledger.entries == 0 && ledger.ledgerId == lastConfirmedEntry.ledgerId) { + ledger.entries = lastConfirmedEntry.entryId + 1; + } + totalEntries += ledger.entries; + } + generateTopicLine(topic, persistentTopicInternalStats, builder, totalEntries); + generateSubscriptionLines(persistentTopicInternalStats, persistentTopicStats, builder, totalEntries); + endBody(builder); + return builder.toString(); + } + + private static void endBody(StringBuilder builder) { + builder.append(""); + } + + private static void startBody(StringBuilder builder) throws IOException { + builder.append(""); + } + + private static void generateSubscriptionLines(ManagedLedgerInternalStats persistentTopicInternalStats, + TopicStats persistentTopicStats, + StringBuilder builder, + long totalEntries) { + List> subscriptionsLines = new ArrayList<>(); + persistentTopicInternalStats.cursors.forEach((name, cursor) -> { + StringBuilder subBuilder = new StringBuilder(); + final EntriesStatsByStatus entriesStatsByStatus = + generateSubscriptionLine(subBuilder, persistentTopicInternalStats, + persistentTopicStats, totalEntries, name, cursor); + subscriptionsLines.add(Pair.of(entriesStatsByStatus, subBuilder)); + }); + subscriptionsLines.sort(Comparator + .comparing((Pair pair) -> pair.getLeft().totalAhead) + .reversed()); + + subscriptionsLines.forEach(pair -> builder.append(pair.getRight())); + } + + private static void generateTopicLine(String topic, + ManagedLedgerInternalStats persistentTopicInternalStats, + StringBuilder builder, + long totalEntries) { + builder.append("

"); + builder.append(topic); + builder.append("

"); + final LedgerSegmentProgress ledgerSegmentProgress = new LedgerSegmentProgress(); + boolean first = true; + for (ManagedLedgerInternalStats.LedgerInfo ledger : persistentTopicInternalStats.ledgers) { + builder.append(TopicSubscriptionsVisualizerHtmlUtil.genLedgerSegment(ledger, + totalEntries, ledgerSegmentProgress, first)); + first = false; + } + builder.append("
"); + } + + private static EntriesStatsByStatus generateSubscriptionLine( + StringBuilder builder, + ManagedLedgerInternalStats persistentTopicInternalStats, + TopicStats persistentTopicStats, + long totalEntries, + String name, + ManagedLedgerInternalStats.CursorStats cursor) { + Map> entryStatuses = new HashMap<>(); + final Position markDeletePos = Position.fromString(cursor.markDeletePosition); + final Position readPos = Position.fromString(cursor.readPosition); + + final ObjectMapper mapper = ObjectMapperFactory.getThreadLocal(); + + for (ManagedLedgerInternalStats.LedgerInfo ledger : persistentTopicInternalStats.ledgers) { + Map entryStatusesForLedger = new HashMap<>(); + entryStatuses.put(ledger.ledgerId, entryStatusesForLedger); + + if (markDeletePos.ledgerId > ledger.ledgerId) { + for (long i = 0; i < ledger.entries; i++) { + entryStatusesForLedger.put(i, EntryStatus.ACKED); + } + } else if (markDeletePos.ledgerId == ledger.ledgerId) { + for (long i = 0; i < markDeletePos.entryId + 1; i++) { + entryStatusesForLedger.put(i, EntryStatus.ACKED); + } + } else { + for (long i = 0; i < ledger.entries; i++) { + entryStatusesForLedger.put(i, EntryStatus.AHEAD); + } + } + + if (readPos.ledgerId > ledger.ledgerId) { + for (long i = 0; i < ledger.entries; i++) { + if (!entryStatusesForLedger.containsKey(i)) { + entryStatusesForLedger.put(i, EntryStatus.AHEAD); + } + } + } else if (readPos.ledgerId == ledger.ledgerId) { + for (long i = 0; i < readPos.entryId - 1; i++) { + if (!entryStatusesForLedger.containsKey(i)) { + entryStatusesForLedger.put(i, EntryStatus.AHEAD); + } + } + } + + } + parseRanges(cursor.individuallyDeletedMessages).forEach(range -> { + for (long i = range.getLeft().entryId; i < range.getRight().entryId; i++) { + final long ledgerId = range.getLeft().ledgerId; + final Map entryStatusMap = entryStatuses + .computeIfAbsent(ledgerId, l -> new HashMap<>()); + entryStatusMap.put(i, EntryStatus.INDIVIDUALLLY_DELETED); + } + }); + EntriesStatsByStatus entriesStatsByStatus = new EntriesStatsByStatus(); + + final String line = TopicSubscriptionsVisualizerHtmlUtil.genSubscriptionLine(entryStatuses, + persistentTopicInternalStats.ledgers, totalEntries, entriesStatsByStatus); + + Map> details = new LinkedHashMap<>(); + details.put("Subscription", mapper.convertValue(persistentTopicStats.getSubscriptions().get(name), Map.class)); + details.put("Cursor", mapper.convertValue(cursor, Map.class)); + final String description = TopicSubscriptionsVisualizerHtmlUtil.genSubscriptionDescription(name, + entriesStatsByStatus.totalAhead, details); + builder.append("
"); + builder.append(description); + builder.append(line); + builder.append("
"); + return entriesStatsByStatus; + } + + private static List> parseRanges(String str) { + List> result = new ArrayList<>(); + str = str + .replace("[", "") + .replace("]", "") + .replace("(", "") + .replace(")", ""); + if (str.isEmpty()) { + return result; + } + final String[] splitByComma = str.split(","); + for (String range : splitByComma) { + final String[] splitRange = range.split("\\.\\."); + result.add(Pair.of(Position.fromString(splitRange[0]), + Position.fromString(splitRange[1]))); + } + return result; + } +} diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java new file mode 100644 index 0000000000000..f87c83267f0ed --- /dev/null +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.admin.cli.utils; + +import java.util.List; +import java.util.Map; +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.util.ObjectMapperFactory; + +class TopicSubscriptionsVisualizerHtmlUtil { + + static String genSubscriptionLine(Map> entryStatuses, + List ledgers, + long totalEntries, + TopicSubscriptionsVisualizer.EntriesStatsByStatus entriesStatsByStatus) { + + StringBuilder builder = new StringBuilder("
"); + final float singleEntryPercent = (float) 100 / totalEntries; + + TopicSubscriptionsVisualizer.EntryStatus currentStatus = null; + long currentCount = 0; + for (ManagedLedgerInternalStats.LedgerInfo ledger : ledgers) { + for (long i = 0; i < ledger.entries; i++) { + TopicSubscriptionsVisualizer.EntryStatus entryStatus = entryStatuses.get(ledger.ledgerId).get(i); + if (entryStatus == null) { + entryStatus = TopicSubscriptionsVisualizer.EntryStatus.AHEAD; + } + + if (currentStatus == null) { + currentStatus = entryStatus; + } + + if (entryStatus == currentStatus) { + currentCount++; + continue; + } + if (currentStatus == TopicSubscriptionsVisualizer.EntryStatus.ACKED + || currentStatus == TopicSubscriptionsVisualizer.EntryStatus.INDIVIDUALLLY_DELETED) { + String cssClass; + if (currentStatus == TopicSubscriptionsVisualizer.EntryStatus.ACKED) { + entriesStatsByStatus.totalAcked += currentCount; + cssClass = "acked"; + } else { + entriesStatsByStatus.totalIndividuallyDeleted += currentCount; + cssClass = "idel"; + } + builder.append("
"); + } else { + entriesStatsByStatus.totalAhead += currentCount; + builder.append("
"); + } + currentCount = 1; + currentStatus = entryStatus; + } + } + if (currentCount > 0) { + + if (currentStatus == TopicSubscriptionsVisualizer.EntryStatus.ACKED + || currentStatus == TopicSubscriptionsVisualizer.EntryStatus.INDIVIDUALLLY_DELETED) { + String cssClass; + if (currentStatus == TopicSubscriptionsVisualizer.EntryStatus.ACKED) { + entriesStatsByStatus.totalAcked += currentCount; + cssClass = "acked"; + } else { + entriesStatsByStatus.totalIndividuallyDeleted += currentCount; + cssClass = "idel"; + } + builder.append("
"); + } else { + entriesStatsByStatus.totalAhead += currentCount; + builder.append("
"); + } + + } + builder.append("
"); + return builder.toString(); + } + + static String genLedgerSegment(ManagedLedgerInternalStats.LedgerInfo ledger, + long totalEntries, + TopicSubscriptionsVisualizer.LedgerSegmentProgress ledgerSegmentProgress, + boolean addFirstClass) { + String tooltip = genLedgerInfoDescription(ledger); + double percentWidth = (double) 100 / totalEntries * ledger.entries; + ledgerSegmentProgress.ledgersMapping.put(ledger.ledgerId, new TopicSubscriptionsVisualizer.EntryWidthMapping()); + for (long i = 0; i < ledger.entries; i++) { + ledgerSegmentProgress.ledgersMapping.get(ledger.ledgerId) + .put(i, ledgerSegmentProgress.currentWidth++); + } + + return "
" + + "" + + tooltip + + "
"; + } + + + static String genLedgerInfoDescription(ManagedLedgerInternalStats.LedgerInfo ledger) { + return genDetailsDescription(ObjectMapperFactory.getThreadLocal() + .convertValue(ledger, Map.class)); + + } + + static String genDetailsDescription(Map details) { + StringBuilder builder = new StringBuilder("
"); + details.forEach((k, v) -> { + builder.append("
"); + builder.append(escapeHtml(k)); + builder.append(""); + builder.append(v == null ? "" : escapeHtml(v.toString())); + builder.append("
"); + }); + builder.append("
"); + return builder.toString(); + } + + static String genSubscriptionDescription(String name, long distance, Map> details) { + StringBuilder builder = new StringBuilder("
"); + builder.append(escapeHtml(name)); + builder.append("
"); + details.forEach((k, data) -> { + builder.append("
"); + builder.append(escapeHtml(k)); + builder.append(""); + builder.append(genDetailsDescription(data)); + builder.append("
"); + }); + builder.append("
" + (distance == 0 ? "0" : "-" + distance) + ""); + builder.append("
"); + return builder.toString(); + } + + private static String escapeHtml(String str) { + return StringEscapeUtils.escapeHtml(str); + } +} diff --git a/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css b/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css new file mode 100644 index 0000000000000..8c7520b98522a --- /dev/null +++ b/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +:root { + --c-black: #03071e; + + --c-topic: #277DA1; + --c-topic-hover: #577590; + + --c-ahead: #F3722C; + --c-ahead-hover: #F94144; + + --c-acked: #90BE6D; + --c-acked-hover: #43AA8B; + + --c-idel: #b9a44c; + --c-idel-hover: #566e3d; + + + --c-tooltip: #F8961E; +} +body { + color: var(--c-black); + font-family: verdana; +} +span { + font-size: 14px +} +.line-container { + margin: 6px 0; + box-shadow: 0 2px 4px 0 rgba(0,0,0,0.2); + border-radius: 5px; + padding: 12px 16px; + display: flex; + flex-direction: column; + +} +.line-container:hover { + box-shadow: 0 4px 8px 0 rgba(0,0,0,0.2); +} +.line-container .subscription { + height: 20px; + display: flex; + margin-top: 5px; +} +.line-container .subscription div { + border-top: 1px var(--c-black) solid; + border-bottom: 1px var(--c-black) solid; +} +.line-container .subscription .segment-acked { + background-color: var(--c-acked); +} +.line-container .subscription .segment-acked:hover { + background-color: var(--c-acked-hover); +} +.line-container .subscription .segment-ahead { + background-color: var(--c-ahead); +} +.line-container .subscription .segment-ahead:hover { + background-color: var(--c-ahead-hover); +} +.line-container .subscription .segment-idel { + background-color: var(--c-idel); +} +.line-container .subscription .segment-idel:hover { + background-color: var(--c-idel-hover); +} + +.line-container .topic { + display: flex; + +} +.line-container .topic .ledger { + height: 15px; + background-color: var(--c-topic); + border-top: 1px white solid; + border-bottom: 1px white solid; + border-right: 1px white solid; +} +.line-container .topic .ledger:hover { + background-color: var(--c-topic-hover); +} +.line-container .topic .ledger.first { + border-left: 1px white solid; +} +.line-container .description { + font-size: 18px; + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: flex-end; +} +.line-container .description .title { + cursor: pointer; + font-size: 20px; + font-weight: 800; + padding: 4px 0; +} + +details > summary { + list-style: none; +} +details > summary::-webkit-details-marker { + display: none; +} + +.details-sections-container { + display: flex; +} +.details-sections-container .details-section { + display: flex; + flex-direction: column; +} +.details-sections-container .details-section .title { + font-size: 14px; + font-weight: 800; +} +.details-section .description-details .detail-item span { + font-size: 10px; +} +.description-details { + display: flex; + flex-direction: column; +} +.description-details .detail-item { + display: flex; + +} +.description-details .detail-item span { + font-size: 14px; + padding: 2px; +} +.description-details .detail-item .detail-item-key { + padding-right: 4px; + font-weight: 400; +} +.description-details .detail-item .detail-item-value { + font-weight: 900; +} + +.tooltip { + position: relative; + display: inline-block; +} +.tooltip .tooltiptext { + visibility: hidden; + min-width: 30px; + background-color: var(--c-tooltip); + border-radius: 1px; + border: 2px var(--c-black) solid; + margin: 6px; + position: absolute; + z-index: 1; + + display: flex; + +} +.tooltip .tooltiptext .tooltip-section { + padding: 8px 4px; + display: flex; + flex-direction: column; +} +.tooltip:hover .tooltiptext { + visibility: visible; +} \ No newline at end of file From 044ea3fb00b6d22b4b20ffea50c66303024344cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Tue, 20 Sep 2022 10:09:51 +0200 Subject: [PATCH 13/17] Move to the new datastax internal artifactory (cherry picked from commit 65de5a06edf054d514c8e8d2fb5deded8bb1cb91) Conflicts: buildtools/pom.xml pom.xml pulsar-sql/presto-distribution/pom.xml --- buildtools/pom.xml | 29 +++++++++++++++++ pom.xml | 44 ++++++++++++++++++++++++++ pulsar-sql/presto-distribution/pom.xml | 12 +++++++ 3 files changed, 85 insertions(+) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 5359f274e81b4..c2f993387bd9a 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -265,4 +265,33 @@ + + + jdk11 + + [11,) + + + + + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + + + + + + + + + datastax-releases + DataStax Local Releases + https://repo.aws.dsinternal.org/artifactory/datastax-public-releases-local/ + + + datastax-snapshots-local + DataStax Local Snapshots + https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local/ + + diff --git a/pom.xml b/pom.xml index eb23574114a01..8c8571597b84f 100644 --- a/pom.xml +++ b/pom.xml @@ -2686,5 +2686,49 @@ flexible messaging model and an intuitive client API. false + + datastax-releases + https://repo.aws.dsinternal.org/artifactory/datastax-releases-local + + false + + + true + + + + datastax-snapshots-local + https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local + + true + + + false + + + + public-datastax-releases + https://repo.datastax.com/datastax-public-releases-local + + false + + + true + + + + + + datastax-releases + DataStax Local Releases + https://repo.aws.dsinternal.org/artifactory/datastax-public-releases-local/ + + + datastax-snapshots-local + DataStax Local Snapshots + https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local/ + + + diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 353445fd436ae..694eaf366058c 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -381,4 +381,16 @@ + + + datastax-releases + DataStax Local Releases + https://repo.aws.dsinternal.org/artifactory/datastax-public-releases-local/ + + + datastax-snapshots-local + DataStax Local Snapshots + https://repo.aws.dsinternal.org/artifactory/datastax-snapshots-local/ + + From 5283e10d627a91a9319114b6498099364d0eb274 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 21 Sep 2022 09:00:49 +0200 Subject: [PATCH 14/17] Fix NPE on subscriptions-visual-stats command (cherry picked from commit b835048543255437d0b28c22df8cc638d58d1226) --- .../admin/cli/utils/TopicSubscriptionsVisualizer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java index 1f7b474ef5711..b82918b3399aa 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java @@ -32,6 +32,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.apache.pulsar.common.policies.data.TopicStats; import org.apache.pulsar.common.util.ObjectMapperFactory; @@ -215,7 +216,10 @@ private static EntriesStatsByStatus generateSubscriptionLine( persistentTopicInternalStats.ledgers, totalEntries, entriesStatsByStatus); Map> details = new LinkedHashMap<>(); - details.put("Subscription", mapper.convertValue(persistentTopicStats.getSubscriptions().get(name), Map.class)); + final SubscriptionStats subscriptionStats = persistentTopicStats.getSubscriptions().get(name); + if (subscriptionStats != null) { + details.put("Subscription", mapper.convertValue(subscriptionStats, Map.class)); + } details.put("Cursor", mapper.convertValue(cursor, Map.class)); final String description = TopicSubscriptionsVisualizerHtmlUtil.genSubscriptionDescription(name, entriesStatsByStatus.totalAhead, details); From 3021595254ee3bc153bc70f2b15dce550958bae5 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Thu, 7 Dec 2023 20:48:53 +0530 Subject: [PATCH 15/17] License Header update --- .../src/main/java/org/apache/pulsar/broker/PulsarService.java | 2 +- .../src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java | 2 +- .../pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java | 2 +- .../admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java | 2 +- .../org/apache/pulsar/admin/cli/utils/topic-visualizer.css | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 620eb68fb2143..4ffb5b77d5424 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index fde380f03be29..411e6e6d84257 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java index b82918b3399aa..8aab0418339d9 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java index f87c83267f0ed..d1104e8ff7c92 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/utils/TopicSubscriptionsVisualizerHtmlUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css b/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css index 8c7520b98522a..0822b2a7b0423 100644 --- a/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css +++ b/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information From 86250655ba8b2e70215e01c1bf82785afcc6e691 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Thu, 7 Dec 2023 21:48:26 +0530 Subject: [PATCH 16/17] Fix license header for topic-visualizer css --- .../org/apache/pulsar/admin/cli/utils/topic-visualizer.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css b/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css index 0822b2a7b0423..8c7520b98522a 100644 --- a/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css +++ b/pulsar-client-tools/src/main/resources/org/apache/pulsar/admin/cli/utils/topic-visualizer.css @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information From 644ced2a524b261daf8c75984017a42e508978d9 Mon Sep 17 00:00:00 2001 From: nikhil-ctds Date: Fri, 8 Dec 2023 10:28:28 +0530 Subject: [PATCH 17/17] Fix import order of TopicSubscriptionsVisualizer --- .../src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java index 411e6e6d84257..47d5326b002cf 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdTopics.java @@ -54,8 +54,8 @@ import lombok.Getter; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.admin.ListTopicsOptions; import org.apache.pulsar.admin.cli.utils.TopicSubscriptionsVisualizer; +import org.apache.pulsar.client.admin.ListTopicsOptions; import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.admin.OffloadProcessStatus; import org.apache.pulsar.client.admin.PulsarAdmin;