diff --git a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/tasks/Tasks.js b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/tasks/Tasks.js index d4747601f..bc409d1d2 100644 --- a/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/tasks/Tasks.js +++ b/geoportal-application/geoportal-harvester-war/src/main/webapp/hrv/ui/tasks/Tasks.js @@ -193,7 +193,7 @@ define(["dojo/_base/declare", }), lang.hitch(this,function(error){ console.error(error); - topic.publish("msg", new Error(this.i18n.tasks.errors.execute)); + topic.publish("msg", new Error(this.i18n.tasks.errors.execute + ": " + data.taskDefinition.name)); }) ); }, diff --git a/geoportal-commons/geoportal-commons-utils/src/main/java/com/esri/geoportal/commons/utils/HttpClientContextBuilder.java b/geoportal-commons/geoportal-commons-utils/src/main/java/com/esri/geoportal/commons/utils/HttpClientContextBuilder.java index 4c9d9d34e..ab62ba172 100644 --- a/geoportal-commons/geoportal-commons-utils/src/main/java/com/esri/geoportal/commons/utils/HttpClientContextBuilder.java +++ b/geoportal-commons/geoportal-commons-utils/src/main/java/com/esri/geoportal/commons/utils/HttpClientContextBuilder.java @@ -39,9 +39,12 @@ public class HttpClientContextBuilder { */ public static HttpClientContext createHttpClientContext(URL url, SimpleCredentials cred) { HttpHost targetHost = new HttpHost(url.getHost(), url.getPort(), url.getProtocol()); - CredentialsProvider credsProvider = new BasicCredentialsProvider(); - credsProvider.setCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()), - new UsernamePasswordCredentials(cred.getUserName(),cred.getPassword())); + CredentialsProvider credsProvider = null; + if (cred!=null && !cred.isEmpty()) { + credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort()), + new UsernamePasswordCredentials(cred.getUserName(),cred.getPassword())); + } // Create AuthCache instance AuthCache authCache = new BasicAuthCache(); @@ -51,7 +54,9 @@ public static HttpClientContext createHttpClientContext(URL url, SimpleCredentia // Add AuthCache to the execution context HttpClientContext context = HttpClientContext.create(); - context.setCredentialsProvider(credsProvider); + if (credsProvider!=null) { + context.setCredentialsProvider(credsProvider); + } context.setAuthCache(authCache); return context; diff --git a/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/HtmlUrlScrapper.java b/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/HtmlUrlScrapper.java index 8270b8e14..2d1ab5c32 100644 --- a/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/HtmlUrlScrapper.java +++ b/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/HtmlUrlScrapper.java @@ -56,17 +56,20 @@ public HtmlUrlScrapper(CloseableHttpClient httpClient, SimpleCredentials creds) * @throws URISyntaxException if invalid URL */ public List scrap(URL root) throws IOException, URISyntaxException { - ContentAnalyzer analyzer = new ContentAnalyzer(root); HttpGet method = new HttpGet(root.toExternalForm()); method.setConfig(DEFAULT_REQUEST_CONFIG); method.setHeader("User-Agent", HttpConstants.getUserAgent()); - HttpClientContext context = creds!=null && !creds.isEmpty()? createHttpClientContext(root, creds): null; + HttpClientContext context = createHttpClientContext(root, creds); try (CloseableHttpResponse httpResponse = httpClient.execute(method, context); InputStream input = httpResponse.getEntity().getContent();) { if (httpResponse.getStatusLine().getStatusCode()>=400) { throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); } String content = IOUtils.toString(input, "UTF-8"); + if (context.getRedirectLocations()!=null && !context.getRedirectLocations().isEmpty()) { + root = context.getRedirectLocations().get(context.getRedirectLocations().size() - 1).toURL(); + } + ContentAnalyzer analyzer = new ContentAnalyzer(root); return analyzer.analyze(content); } } diff --git a/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafBroker.java b/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafBroker.java index 8579cb44a..eb0745dc0 100644 --- a/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafBroker.java +++ b/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafBroker.java @@ -32,6 +32,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.Set; @@ -152,6 +153,9 @@ public WafIterator(IteratorContext iteratorContext) { @Override public boolean hasNext() throws DataInputException { + if (Thread.currentThread().isInterrupted()) { + return false; + } try { if (files!=null && !files.isEmpty()) { diff --git a/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFile.java b/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFile.java index c12e9e96a..37fa74da8 100644 --- a/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFile.java +++ b/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFile.java @@ -26,7 +26,6 @@ import com.esri.geoportal.harvester.api.base.SimpleDataReference; import java.io.IOException; import java.io.InputStream; -import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.time.ZonedDateTime; @@ -81,6 +80,9 @@ public SimpleDataReference readContent(CloseableHttpClient httpClient, Date sinc if (httpResponse.getStatusLine().getStatusCode()>=400) { throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); } + if (Thread.currentThread().isInterrupted()) { + return new SimpleDataReference(broker.getBrokerUri(), broker.getEntityDefinition().getLabel(), fileUrl.toExternalForm(), null, fileUrl.toURI(), broker.td.getSource().getRef(), broker.td.getRef()); + } Date lastModifiedDate = readLastModifiedDate(httpResponse); MimeType contentType = readContentType(httpResponse); boolean readBody = since==null || lastModifiedDate==null || lastModifiedDate.getTime()>=since.getTime(); diff --git a/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFolder.java b/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFolder.java index ee729ef84..3b52fc25c 100644 --- a/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFolder.java +++ b/geoportal-connectors/geoportal-harvester-waf/src/main/java/com/esri/geoportal/harvester/waf/WafFolder.java @@ -18,6 +18,8 @@ import static com.esri.geoportal.commons.utils.CrlfUtils.formatForLog; import com.esri.geoportal.commons.utils.SimpleCredentials; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.FileSystem; @@ -26,6 +28,7 @@ import java.nio.file.PathMatcher; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -76,14 +79,16 @@ public WafFolderContent readContent(CloseableHttpClient httpClient) throws IOExc try { List urls = scrapper.scrap(folderUrl); - - urls.forEach(u -> { + for (URL u: urls) { + if (Thread.currentThread().isInterrupted()) { + return new WafFolderContent(this, Collections.emptyList(), Collections.emptyList()); + } if (u.toExternalForm().endsWith("/") || !cutOff(u.toExternalForm(),"/").contains(".")) { subFolders.add(new WafFolder(broker, u, matchPattern, creds)); } else if (StringUtils.isBlank(matchPattern) || multiMatchUrl(u,matchPattern)) { files.add(new WafFile(broker, u, creds)); } - }); + } } catch (HttpResponseException ex) { if (ex.getStatusCode()!=403) { throw ex;