Merge branch 'master' of https://github.com/Esri/geoportal-server-har…

…vester.git
Esri · Sep 30, 2019 · 7eb1091 · 7eb1091
2 parents 9fb6cf6 + 254bbad
commit 7eb1091
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 11 deletions.
diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/tasks/Tasks.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/tasks/Tasks.js
@@ -193,7 +193,7 @@ define(["dojo/_base/declare",
           }),
           lang.hitch(this,function(error){
             console.error(error);
-            topic.publish("msg", new Error(this.i18n.tasks.errors.execute));
+            topic.publish("msg", new Error(this.i18n.tasks.errors.execute + ": " + data.taskDefinition.name));
           })
         );
       },

diff --git a/...ommons-utils/src/main/java/com/esri/geoportal/commons/utils/HttpClientContextBuilder.java b/...ommons-utils/src/main/java/com/esri/geoportal/commons/utils/HttpClientContextBuilder.java
@@ -39,9 +39,12 @@ public class HttpClientContextBuilder {
    */
   public static HttpClientContext createHttpClientContext(URL url, SimpleCredentials cred) {
     HttpHost targetHost = new HttpHost(url.getHost(), url.getPort(), url.getProtocol());
-    CredentialsProvider credsProvider = new BasicCredentialsProvider();
-    credsProvider.setCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()),
-            new UsernamePasswordCredentials(cred.getUserName(),cred.getPassword()));    
+    CredentialsProvider credsProvider = null;
+    if (cred!=null && !cred.isEmpty()) {
+      credsProvider = new BasicCredentialsProvider();
+      credsProvider.setCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()),
+              new UsernamePasswordCredentials(cred.getUserName(),cred.getPassword()));    
+    }
 
     // Create AuthCache instance
     AuthCache authCache = new BasicAuthCache();
@@ -51,7 +54,9 @@ public static HttpClientContext createHttpClientContext(URL url, SimpleCredentia
 
     // Add AuthCache to the execution context
     HttpClientContext context = HttpClientContext.create();
-    context.setCredentialsProvider(credsProvider);
+    if (credsProvider!=null) {
+      context.setCredentialsProvider(credsProvider);
+    }
     context.setAuthCache(authCache);
 
     return context;

diff --git a/...oportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/HtmlUrlScrapper.java b/...oportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/HtmlUrlScrapper.java
@@ -56,17 +56,20 @@ public HtmlUrlScrapper(CloseableHttpClient httpClient, SimpleCredentials creds)
    * @throws URISyntaxException if invalid URL
    */
   public List<URL> scrap(URL root) throws IOException, URISyntaxException {
-    ContentAnalyzer analyzer = new ContentAnalyzer(root);
     HttpGet method = new HttpGet(root.toExternalForm());
     method.setConfig(DEFAULT_REQUEST_CONFIG);
     method.setHeader("User-Agent", HttpConstants.getUserAgent());
-    HttpClientContext context = creds!=null && !creds.isEmpty()? createHttpClientContext(root, creds): null;
+    HttpClientContext context = createHttpClientContext(root, creds);
 
     try (CloseableHttpResponse httpResponse = httpClient.execute(method, context); InputStream input = httpResponse.getEntity().getContent();) {
       if (httpResponse.getStatusLine().getStatusCode()>=400) {
         throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase());
       }
       String content = IOUtils.toString(input, "UTF-8");
+      if (context.getRedirectLocations()!=null && !context.getRedirectLocations().isEmpty()) {
+        root = context.getRedirectLocations().get(context.getRedirectLocations().size() - 1).toURL();
+      }
+      ContentAnalyzer analyzer = new ContentAnalyzer(root);
       return analyzer.analyze(content);
     }
   }

diff --git a/...ors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafBroker.java b/...ors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafBroker.java
@@ -32,6 +32,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Set;
@@ -152,6 +153,9 @@ public WafIterator(IteratorContext iteratorContext) {
 
     @Override
     public boolean hasNext() throws DataInputException {
+      if (Thread.currentThread().isInterrupted()) {
+        return false;
+      }
 
       try {
         if (files!=null && !files.isEmpty()) {

diff --git a/...ctors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFile.java b/...ctors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFile.java
@@ -26,7 +26,6 @@
 import com.esri.geoportal.harvester.api.base.SimpleDataReference;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.time.ZonedDateTime;
@@ -81,6 +80,9 @@ public SimpleDataReference readContent(CloseableHttpClient httpClient, Date sinc
       if (httpResponse.getStatusLine().getStatusCode()>=400) {
         throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase());
       }
+      if (Thread.currentThread().isInterrupted()) {
+        return new SimpleDataReference(broker.getBrokerUri(), broker.getEntityDefinition().getLabel(), fileUrl.toExternalForm(), null, fileUrl.toURI(), broker.td.getSource().getRef(), broker.td.getRef());
+      }
       Date lastModifiedDate = readLastModifiedDate(httpResponse);
       MimeType contentType = readContentType(httpResponse);
       boolean readBody = since==null || lastModifiedDate==null || lastModifiedDate.getTime()>=since.getTime();

diff --git a/...ors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFolder.java b/...ors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFolder.java
@@ -18,6 +18,8 @@
 import static com.esri.geoportal.commons.utils.CrlfUtils.formatForLog;
 import com.esri.geoportal.commons.utils.SimpleCredentials;
 import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.FileSystem;
@@ -26,6 +28,7 @@
 import java.nio.file.PathMatcher;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.apache.commons.lang3.StringUtils;
@@ -76,14 +79,16 @@ public WafFolderContent readContent(CloseableHttpClient httpClient) throws IOExc
 
     try {
       List<URL> urls = scrapper.scrap(folderUrl);
-
-      urls.forEach(u -> {
+      for (URL u: urls) {
+        if (Thread.currentThread().isInterrupted()) {
+          return new WafFolderContent(this, Collections.emptyList(), Collections.emptyList());
+        }
         if (u.toExternalForm().endsWith("/") || !cutOff(u.toExternalForm(),"/").contains(".")) {
           subFolders.add(new WafFolder(broker, u, matchPattern, creds));
         } else if (StringUtils.isBlank(matchPattern) || multiMatchUrl(u,matchPattern)) {
           files.add(new WafFile(broker, u, creds));
         }
-      });
+      }
     } catch (HttpResponseException ex) {
       if (ex.getStatusCode()!=403) {
         throw ex;