Adding proxy support for control file generation and finding column

headers
socrata · Nov 6, 2014 · c250966 · c250966
1 parent 1337755
commit c250966
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -50,5 +50,5 @@ mvn clean compile -Dmaven.test.skip=true assembly:single
 This puts the JAR file into the "target" directory inside the repo.  So to open DataSync, simply:
 ```
 cd target
-java -jar DataSync-1.5-jar-with-dependencies.jar
+java -jar DataSync-1.5.2-jar-with-dependencies.jar
 ```
diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>DataSync</groupId>
     <artifactId>DataSync</artifactId>
-    <version>1.5</version>
+    <version>1.5.2</version>
     <developers>
         <developer>
             <name>Adrian Laurenzi</name>

diff --git a/src/main/java/com/socrata/datasync/DatasetUtils.java b/src/main/java/com/socrata/datasync/DatasetUtils.java
@@ -1,10 +1,19 @@
 package com.socrata.datasync;
 
-import com.socrata.api.SodaDdl;
-import com.socrata.exceptions.SodaError;
+import com.socrata.datasync.config.userpreferences.UserPreferencesJava;
 import com.socrata.model.importer.Column;
 import com.socrata.model.importer.Dataset;
+import org.apache.http.HttpException;
+import org.apache.http.HttpStatus;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.utils.URIBuilder;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
 
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -14,6 +23,34 @@
 public class DatasetUtils {
 
     private static final String LOCATION_DATATYPE_NAME = "location";
+    private static HttpUtility http = new HttpUtility(new UserPreferencesJava(), true);
+    private static ObjectMapper mapper = new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY);
+
+    public static Dataset getDatasetInfo(String domain, String viewId) throws URISyntaxException, IOException, HttpException {
+        String[] schemaAndDomain = domain.trim().split("//");
+        String justDomain = schemaAndDomain[schemaAndDomain.length -1];
+        URI absolutePath = new URIBuilder()
+                .setScheme("https")
+                .setHost(justDomain)
+                .setPath("/api/views/" + viewId)
+                .build();
+        StatusLine statusLine;
+        int status;
+        int retriesAvailable = 3;
+        int retries = 0;
+        do {
+            try (CloseableHttpResponse resp = http.get(absolutePath, "application/json")) {
+                statusLine = resp.getStatusLine();
+                status = statusLine.getStatusCode();
+                if (status != HttpStatus.SC_OK) {
+                    retries += 1;
+                } else {
+                    return mapper.readValue(resp.getEntity().getContent(), Dataset.class);
+                }
+            }
+        } while (status != HttpStatus.SC_OK && retries < retriesAvailable);
+        throw new HttpException(statusLine.toString());
+    }
 
     /**
      * Retruns the field name of the row identifier, if there is one, else null
@@ -27,15 +64,16 @@ public static String getRowIdentifierName(Dataset schema) {
         }
     }
 
+
     /**
      * Returns list of dataset field names in the form: "col1","col2",...
      *
-     * @param ddl
+     * @param domain
      * @param datasetId
      * @return list of field names or null if there
      */
-    public static String getFieldNamesString(SodaDdl ddl, String datasetId) throws SodaError, InterruptedException {
-        Dataset datasetInfo = (Dataset) ddl.loadDatasetInfo(datasetId);
+    public static String getFieldNamesString(String domain, String datasetId) throws HttpException, IOException, URISyntaxException {
+        Dataset datasetInfo = getDatasetInfo(domain, datasetId);
         return getFieldNamesString(datasetInfo);
     }
 

diff --git a/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java b/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java
@@ -8,8 +8,8 @@
 import com.socrata.datasync.config.userpreferences.UserPreferencesJava;
 import com.socrata.datasync.job.JobStatus;
 import com.socrata.datasync.validation.IntegrationJobValidity;
-import com.socrata.exceptions.SodaError;
 import com.socrata.model.importer.Dataset;
+import org.apache.http.HttpException;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.codehaus.jackson.map.SerializationConfig;
 
@@ -19,6 +19,7 @@
 import java.awt.*;
 import java.io.File;
 import java.io.IOException;
+import java.net.URISyntaxException;
 
 /**
  * Author: Adrian Laurenzi
@@ -113,6 +114,8 @@ public class IntegrationJobTab implements JobTab {
     private boolean usingControlFile;
     private boolean regenerateControlFile;
 
+    private UserPreferences userPrefs = new UserPreferencesJava();
+
     // build Container with all tab components populated with given job data
     public IntegrationJobTab(IntegrationJob job, JFrame containingFrame) {
         mainFrame = containingFrame;
@@ -507,7 +510,7 @@ public void actionPerformed(ActionEvent evnt) {
                 } else {
                     try {
                         controlFileContent = generateControlFileContent(
-                                getSodaDdl(),
+                                userPrefs.getDomain(),
                                 fileToPublishTextField.getText(),
                                 (PublishMethod) publishMethodComboBox.getSelectedItem(),
                                 datasetIDTextField.getText(),
@@ -542,7 +545,7 @@ private boolean fileToPublishIsSelected() {
         /**
          * Generates default content of control.json based on given job parameters
          *
-         * @param ddl Soda 2 ddl object
+         * @param domain The domain this job applies to
          * @param publishMethod to use to publish (upsert, append, replace, or delete)
          *               NOTE: this option will be overriden if userPrefs has pathToFTPControlFile or pathToControlFile set
          * @param datasetId id of the Socrata dataset to publish to
@@ -554,9 +557,9 @@ private boolean fileToPublishIsSelected() {
          * @throws com.socrata.exceptions.SodaError
          * @throws InterruptedException
          */
-        private String generateControlFileContent(SodaDdl ddl, String fileToPublish, PublishMethod publishMethod,
-                                                  String datasetId, boolean containsHeaderRow) throws SodaError, InterruptedException, IOException {
-            Dataset datasetInfo = (Dataset) ddl.loadDatasetInfo(datasetId);
+        private String generateControlFileContent(String domain, String fileToPublish, PublishMethod publishMethod,
+                                                  String datasetId, boolean containsHeaderRow) throws IOException, URISyntaxException, HttpException {
+            Dataset datasetInfo = DatasetUtils.getDatasetInfo(domain, datasetId);
             boolean useGeocoding = DatasetUtils.hasLocationColumn(datasetInfo);
 
             String[] columns = null;
@@ -600,8 +603,7 @@ public void actionPerformed(ActionEvent evnt) {
             String datasetFieldNames = null;
             if(datasetIdValid()) {
                 try {
-                    datasetFieldNames = DatasetUtils.getFieldNamesString(
-                            getSodaDdl(), datasetIDTextField.getText());
+                    datasetFieldNames = DatasetUtils.getFieldNamesString(userPrefs.getDomain(), datasetIDTextField.getText());
                 } catch (Exception e) {
                     e.printStackTrace();
                     errorMessage = "Error getting column IDs for dataset with ID" +
@@ -645,13 +647,4 @@ private boolean datasetIdValid() {
         String datasetId = datasetIDTextField.getText();
         return Utils.uidIsValid(datasetId);
     }
-
-    private SodaDdl getSodaDdl() {
-        UserPreferences userPrefs = new UserPreferencesJava();
-        return SodaDdl.newDdl(
-                userPrefs.getDomain(),
-                userPrefs.getUsername(),
-                userPrefs.getPassword(),
-                userPrefs.getAPIKey());
-    }
 }
diff --git a/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java b/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java
@@ -1,6 +1,5 @@
 package com.socrata.datasync.validation;
 
-import com.socrata.api.SodaImporter;
 import com.socrata.datasync.DatasetUtils;
 import com.socrata.datasync.HttpUtility;
 import com.socrata.datasync.PublishMethod;
@@ -14,7 +13,6 @@
 import com.socrata.datasync.job.JobStatus;
 import com.socrata.model.importer.Dataset;
 import org.apache.commons.cli.CommandLine;
-import org.apache.commons.lang3.ArrayUtils;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.entity.ContentType;
 import org.codehaus.jackson.map.ObjectMapper;
@@ -81,10 +79,9 @@ public static JobStatus validateJobParams(SocrataConnectionInfo connectionInfo,
         if(!allowedFileToPublishExtensions.contains(fileExtension))
             return JobStatus.FILE_TO_PUBLISH_INVALID_FORMAT;
 
-        final SodaImporter importer = SodaImporter.newImporter(connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken());
         Dataset schema;
         try {
-            schema = (Dataset) importer.loadDatasetInfo(job.getDatasetID());
+            schema = DatasetUtils.getDatasetInfo(connectionInfo.getUrl(), job.getDatasetID());
 
             if(job.getPublishViaDi2Http() || job.getPublishViaFTP()) {
 

diff --git a/src/test/java/com/socrata/datasync/VersionProviderTest.java b/src/test/java/com/socrata/datasync/VersionProviderTest.java
@@ -35,7 +35,7 @@ public void testVersionMajorNumbers() {
 
     @Test
     public void testGetLatestVersion() throws URISyntaxException {
-        TestCase.assertEquals("1.0", VersionProvider.getLatestVersion());
+        TestCase.assertEquals("1.5.2", VersionProvider.getLatestVersion());
     }
 
     @Test
@@ -48,6 +48,6 @@ public void testIsLatestVersion() throws URISyntaxException {
 
     @Test
     public void testGetThisVersion() {
-        TestCase.assertEquals("1.5", VersionProvider.getThisVersion());
+        TestCase.assertEquals("1.5.2", VersionProvider.getThisVersion());
     }
 }
diff --git a/src/test/java/com/socrata/datasync/utilities/IntegrationUtilityTest.java b/src/test/java/com/socrata/datasync/utilities/IntegrationUtilityTest.java
@@ -15,10 +15,12 @@
 import com.socrata.model.UpsertResult;
 import com.socrata.model.importer.Dataset;
 import junit.framework.TestCase;
+import org.apache.http.HttpException;
 import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 
 /**
@@ -306,11 +308,11 @@ public void testAddLogEntryInvalidLogDatasetId() {
     }
 
     @Test
-    public void testGetDatasetFieldNames() throws IOException, SodaError, InterruptedException {
+    public void testGetDatasetFieldNames() throws InterruptedException, HttpException, URISyntaxException, IOException, SodaError {
         //UserPreferences userPrefs = new UserPreferencesJava();
         //System.out.println(IntegrationUtility.getFieldNamesString(ddl, "6qkn-8xvw"));
         final SodaDdl ddl = createSodaDdl();
-        String datasetFieldNamesString = DatasetUtils.getFieldNamesString(ddl, UNITTEST_DATASET_ID);
+        String datasetFieldNamesString = DatasetUtils.getFieldNamesString(DOMAIN, UNITTEST_DATASET_ID);
         TestCase.assertEquals("\"id\",\"name\",\"another_name\",\"date\"", datasetFieldNamesString);
 
         Dataset datasetInfo = (Dataset) ddl.loadDatasetInfo(UNITTEST_DATASET_ID);