From c250966b3f3ac93fdc7806c94cc2599439adce7a Mon Sep 17 00:00:00 2001 From: ayn leslie Date: Thu, 6 Nov 2014 09:19:23 -0800 Subject: [PATCH] Adding proxy support for control file generation and finding column headers --- README.md | 2 +- pom.xml | 2 +- .../com/socrata/datasync/DatasetUtils.java | 48 +++++++++++++++++-- .../datasync/ui/IntegrationJobTab.java | 27 ++++------- .../validation/IntegrationJobValidity.java | 5 +- .../socrata/datasync/VersionProviderTest.java | 4 +- .../utilities/IntegrationUtilityTest.java | 6 ++- 7 files changed, 62 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 83e46e1d..d5d23560 100644 --- a/README.md +++ b/README.md @@ -50,5 +50,5 @@ mvn clean compile -Dmaven.test.skip=true assembly:single This puts the JAR file into the "target" directory inside the repo. So to open DataSync, simply: ``` cd target -java -jar DataSync-1.5-jar-with-dependencies.jar +java -jar DataSync-1.5.2-jar-with-dependencies.jar ``` diff --git a/pom.xml b/pom.xml index 2d95d6de..4798299d 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 DataSync DataSync - 1.5 + 1.5.2 Adrian Laurenzi diff --git a/src/main/java/com/socrata/datasync/DatasetUtils.java b/src/main/java/com/socrata/datasync/DatasetUtils.java index 892176b7..b5dd53f1 100644 --- a/src/main/java/com/socrata/datasync/DatasetUtils.java +++ b/src/main/java/com/socrata/datasync/DatasetUtils.java @@ -1,10 +1,19 @@ package com.socrata.datasync; -import com.socrata.api.SodaDdl; -import com.socrata.exceptions.SodaError; +import com.socrata.datasync.config.userpreferences.UserPreferencesJava; import com.socrata.model.importer.Column; import com.socrata.model.importer.Dataset; +import org.apache.http.HttpException; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.utils.URIBuilder; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -14,6 +23,34 @@ public class DatasetUtils { private static final String LOCATION_DATATYPE_NAME = "location"; + private static HttpUtility http = new HttpUtility(new UserPreferencesJava(), true); + private static ObjectMapper mapper = new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY); + + public static Dataset getDatasetInfo(String domain, String viewId) throws URISyntaxException, IOException, HttpException { + String[] schemaAndDomain = domain.trim().split("//"); + String justDomain = schemaAndDomain[schemaAndDomain.length -1]; + URI absolutePath = new URIBuilder() + .setScheme("https") + .setHost(justDomain) + .setPath("/api/views/" + viewId) + .build(); + StatusLine statusLine; + int status; + int retriesAvailable = 3; + int retries = 0; + do { + try (CloseableHttpResponse resp = http.get(absolutePath, "application/json")) { + statusLine = resp.getStatusLine(); + status = statusLine.getStatusCode(); + if (status != HttpStatus.SC_OK) { + retries += 1; + } else { + return mapper.readValue(resp.getEntity().getContent(), Dataset.class); + } + } + } while (status != HttpStatus.SC_OK && retries < retriesAvailable); + throw new HttpException(statusLine.toString()); + } /** * Retruns the field name of the row identifier, if there is one, else null @@ -27,15 +64,16 @@ public static String getRowIdentifierName(Dataset schema) { } } + /** * Returns list of dataset field names in the form: "col1","col2",... * - * @param ddl + * @param domain * @param datasetId * @return list of field names or null if there */ - public static String getFieldNamesString(SodaDdl ddl, String datasetId) throws SodaError, InterruptedException { - Dataset datasetInfo = (Dataset) ddl.loadDatasetInfo(datasetId); + public static String getFieldNamesString(String domain, String datasetId) throws HttpException, IOException, URISyntaxException { + Dataset datasetInfo = getDatasetInfo(domain, datasetId); return getFieldNamesString(datasetInfo); } diff --git a/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java b/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java index 80950a33..17a3adac 100644 --- a/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java +++ b/src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java @@ -8,8 +8,8 @@ import com.socrata.datasync.config.userpreferences.UserPreferencesJava; import com.socrata.datasync.job.JobStatus; import com.socrata.datasync.validation.IntegrationJobValidity; -import com.socrata.exceptions.SodaError; import com.socrata.model.importer.Dataset; +import org.apache.http.HttpException; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; @@ -19,6 +19,7 @@ import java.awt.*; import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; /** * Author: Adrian Laurenzi @@ -113,6 +114,8 @@ public class IntegrationJobTab implements JobTab { private boolean usingControlFile; private boolean regenerateControlFile; + private UserPreferences userPrefs = new UserPreferencesJava(); + // build Container with all tab components populated with given job data public IntegrationJobTab(IntegrationJob job, JFrame containingFrame) { mainFrame = containingFrame; @@ -507,7 +510,7 @@ public void actionPerformed(ActionEvent evnt) { } else { try { controlFileContent = generateControlFileContent( - getSodaDdl(), + userPrefs.getDomain(), fileToPublishTextField.getText(), (PublishMethod) publishMethodComboBox.getSelectedItem(), datasetIDTextField.getText(), @@ -542,7 +545,7 @@ private boolean fileToPublishIsSelected() { /** * Generates default content of control.json based on given job parameters * - * @param ddl Soda 2 ddl object + * @param domain The domain this job applies to * @param publishMethod to use to publish (upsert, append, replace, or delete) * NOTE: this option will be overriden if userPrefs has pathToFTPControlFile or pathToControlFile set * @param datasetId id of the Socrata dataset to publish to @@ -554,9 +557,9 @@ private boolean fileToPublishIsSelected() { * @throws com.socrata.exceptions.SodaError * @throws InterruptedException */ - private String generateControlFileContent(SodaDdl ddl, String fileToPublish, PublishMethod publishMethod, - String datasetId, boolean containsHeaderRow) throws SodaError, InterruptedException, IOException { - Dataset datasetInfo = (Dataset) ddl.loadDatasetInfo(datasetId); + private String generateControlFileContent(String domain, String fileToPublish, PublishMethod publishMethod, + String datasetId, boolean containsHeaderRow) throws IOException, URISyntaxException, HttpException { + Dataset datasetInfo = DatasetUtils.getDatasetInfo(domain, datasetId); boolean useGeocoding = DatasetUtils.hasLocationColumn(datasetInfo); String[] columns = null; @@ -600,8 +603,7 @@ public void actionPerformed(ActionEvent evnt) { String datasetFieldNames = null; if(datasetIdValid()) { try { - datasetFieldNames = DatasetUtils.getFieldNamesString( - getSodaDdl(), datasetIDTextField.getText()); + datasetFieldNames = DatasetUtils.getFieldNamesString(userPrefs.getDomain(), datasetIDTextField.getText()); } catch (Exception e) { e.printStackTrace(); errorMessage = "Error getting column IDs for dataset with ID" + @@ -645,13 +647,4 @@ private boolean datasetIdValid() { String datasetId = datasetIDTextField.getText(); return Utils.uidIsValid(datasetId); } - - private SodaDdl getSodaDdl() { - UserPreferences userPrefs = new UserPreferencesJava(); - return SodaDdl.newDdl( - userPrefs.getDomain(), - userPrefs.getUsername(), - userPrefs.getPassword(), - userPrefs.getAPIKey()); - } } diff --git a/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java b/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java index f2ccb5a9..1ffcaa78 100644 --- a/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java +++ b/src/main/java/com/socrata/datasync/validation/IntegrationJobValidity.java @@ -1,6 +1,5 @@ package com.socrata.datasync.validation; -import com.socrata.api.SodaImporter; import com.socrata.datasync.DatasetUtils; import com.socrata.datasync.HttpUtility; import com.socrata.datasync.PublishMethod; @@ -14,7 +13,6 @@ import com.socrata.datasync.job.JobStatus; import com.socrata.model.importer.Dataset; import org.apache.commons.cli.CommandLine; -import org.apache.commons.lang3.ArrayUtils; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.entity.ContentType; import org.codehaus.jackson.map.ObjectMapper; @@ -81,10 +79,9 @@ public static JobStatus validateJobParams(SocrataConnectionInfo connectionInfo, if(!allowedFileToPublishExtensions.contains(fileExtension)) return JobStatus.FILE_TO_PUBLISH_INVALID_FORMAT; - final SodaImporter importer = SodaImporter.newImporter(connectionInfo.getUrl(), connectionInfo.getUser(), connectionInfo.getPassword(), connectionInfo.getToken()); Dataset schema; try { - schema = (Dataset) importer.loadDatasetInfo(job.getDatasetID()); + schema = DatasetUtils.getDatasetInfo(connectionInfo.getUrl(), job.getDatasetID()); if(job.getPublishViaDi2Http() || job.getPublishViaFTP()) { diff --git a/src/test/java/com/socrata/datasync/VersionProviderTest.java b/src/test/java/com/socrata/datasync/VersionProviderTest.java index a84dcdf7..2a0e68bd 100644 --- a/src/test/java/com/socrata/datasync/VersionProviderTest.java +++ b/src/test/java/com/socrata/datasync/VersionProviderTest.java @@ -35,7 +35,7 @@ public void testVersionMajorNumbers() { @Test public void testGetLatestVersion() throws URISyntaxException { - TestCase.assertEquals("1.0", VersionProvider.getLatestVersion()); + TestCase.assertEquals("1.5.2", VersionProvider.getLatestVersion()); } @Test @@ -48,6 +48,6 @@ public void testIsLatestVersion() throws URISyntaxException { @Test public void testGetThisVersion() { - TestCase.assertEquals("1.5", VersionProvider.getThisVersion()); + TestCase.assertEquals("1.5.2", VersionProvider.getThisVersion()); } } diff --git a/src/test/java/com/socrata/datasync/utilities/IntegrationUtilityTest.java b/src/test/java/com/socrata/datasync/utilities/IntegrationUtilityTest.java index 8a4dc324..9886446e 100644 --- a/src/test/java/com/socrata/datasync/utilities/IntegrationUtilityTest.java +++ b/src/test/java/com/socrata/datasync/utilities/IntegrationUtilityTest.java @@ -15,10 +15,12 @@ import com.socrata.model.UpsertResult; import com.socrata.model.importer.Dataset; import junit.framework.TestCase; +import org.apache.http.HttpException; import org.junit.Test; import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; import java.util.ArrayList; /** @@ -306,11 +308,11 @@ public void testAddLogEntryInvalidLogDatasetId() { } @Test - public void testGetDatasetFieldNames() throws IOException, SodaError, InterruptedException { + public void testGetDatasetFieldNames() throws InterruptedException, HttpException, URISyntaxException, IOException, SodaError { //UserPreferences userPrefs = new UserPreferencesJava(); //System.out.println(IntegrationUtility.getFieldNamesString(ddl, "6qkn-8xvw")); final SodaDdl ddl = createSodaDdl(); - String datasetFieldNamesString = DatasetUtils.getFieldNamesString(ddl, UNITTEST_DATASET_ID); + String datasetFieldNamesString = DatasetUtils.getFieldNamesString(DOMAIN, UNITTEST_DATASET_ID); TestCase.assertEquals("\"id\",\"name\",\"another_name\",\"date\"", datasetFieldNamesString); Dataset datasetInfo = (Dataset) ddl.loadDatasetInfo(UNITTEST_DATASET_ID);