Hubspot to pubsub batch

dataflow/flex-templates/hubspot-to-pubsub/pom.xml

@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  ~ Copyright 2023 Akvelon Inc.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License"); you may not
+  ~ use this file except in compliance with the License. You may obtain a copy of
+  ~ the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  ~ License for the specific language governing permissions and limitations under
+  ~ the License.
+  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>data-and-analytics</artifactId>
+        <groupId>com.akvelon</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <artifactId>hubspot-to-pubsub</artifactId>
+    <modelVersion>4.0.0</modelVersion>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.cdap.cdap</groupId>
+            <artifactId>cdap-api-common</artifactId>
+            <version>${cdap.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.cdap.plugin</groupId>
+            <artifactId>salesforce-plugins</artifactId>
+            <version>1.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.cdap.plugin</groupId>
+            <artifactId>hydrator-common</artifactId>
+            <version>2.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.akvelon</groupId>
+            <artifactId>cdap-hubspot-plugins</artifactId>
+            <version>1.1.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>io.cdap.plugin</groupId>
+                    <artifactId>hydrator-common</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>io.cdap.cdap</groupId>
+            <artifactId>cdap-api</artifactId>
+            <version>${cdap.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.auto.value</groupId>
+            <artifactId>auto-value</artifactId>
+            <version>1.7.4</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.cdap.cdap</groupId>
+            <artifactId>cdap-etl-api</artifactId>
+            <version>${cdap.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.cdap.cdap</groupId>
+            <artifactId>cdap-common</artifactId>
+            <version>${cdap.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.13.5</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-scala_2.11</artifactId>
+            <version>2.13.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>30.1-jre</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.api-client</groupId>
+            <artifactId>google-api-client</artifactId>
+            <version>1.31.1</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.cloud.bigdataoss</groupId>
+            <artifactId>util</artifactId>
+            <version>2.2.6</version>
+        </dependency>
+    </dependencies>
+</project>

dataflow/flex-templates/hubspot-to-pubsub/src/main/java/com/akvelon/hubspot/options/BaseCdapOptions.java

@@ -0,0 +1,28 @@
+package com.akvelon.hubspot.options;
+
+import io.cdap.plugin.common.Constants;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.Validation;
+
+/**
+ * The {@link BaseCdapOptions} interface provides the custom execution options passed by the
+ * executor at the command-line for examples with Cdap plugins.
+ */
+public interface BaseCdapOptions extends PipelineOptions {
+
+    @Validation.Required
+    @Description(Constants.Reference.REFERENCE_NAME_DESCRIPTION)
+    String getReferenceName();
+
+    void setReferenceName(String referenceName);
+
+    @Description(
+            "The Cloud Pub/Sub topic to publish to. "
+                    + "The name should be in the format of "
+                    + "projects/<project-id>/topics/<topic-name>.")
+    @Validation.Required
+    String getOutputTopic();
+
+    void setOutputTopic(String outputTopic);
+}

dataflow/flex-templates/hubspot-to-pubsub/src/main/java/com/akvelon/hubspot/options/CdapHubspotOptions.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.akvelon.hubspot.options;
+
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Validation;
+
+/**
+ * The {@link CdapHubspotOptions} interface provides the custom execution options passed by the
+ * executor at the command-line for examples with Cdap Hubspot plugins.
+ */
+public interface CdapHubspotOptions extends BaseCdapOptions {
+
+  @Description("Hubspot api server url. If not specified then the default url will be used.")
+  String getApiServerUrl();
+
+  void setApiServerUrl(String apiServerUrl);
+
+  @Validation.Required
+  @Description("Hubspot Private Application Access Token.")
+  String getAuthToken();
+
+  void setAuthToken(String authToken);
+
+  @Validation.Required
+  @Description("Name of object to pull from Hubspot (e.g. Contacts).")
+  String getObjectType();
+
+  void setObjectType(String objectType);
+}

dataflow/flex-templates/hubspot-to-pubsub/src/main/java/com/akvelon/hubspot/options/CdapHubspotStreamingSourceOptions.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.akvelon.hubspot.options;
+
+import org.apache.beam.sdk.options.Description;
+
+/**
+ * The {@link CdapHubspotOptions} interface provides the custom execution options passed by
+ * the executor at the command-line for Cdap Hubspot Source examples.
+ */
+public interface CdapHubspotStreamingSourceOptions extends CdapHubspotOptions {
+
+  @Description("Delay in seconds between polling for new records updates.")
+  Long getPullFrequencySec();
+
+  void setPullFrequencySec(Long pullFrequencySec);
+
+  @Description("Inclusive start offset from which the reading should be started.")
+  Long getStartOffset();
+
+  void setStartOffset(Long startOffset);
+}

dataflow/flex-templates/hubspot-to-pubsub/src/main/java/com/akvelon/hubspot/options/package-info.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2023 Akvelon Inc.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Cdap Hubspot template. */
+package com.akvelon.hubspot.options;