diff --git a/build.gradle b/build.gradle index d0c09a7ab..0c9dc72d8 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ apply from: "${project.rootDir}/gradle/eclipse.gradle" sourceCompatibility = 1.8 targetCompatibility = 1.8 group = 'de.mediathekview' -version = '3.1.15' +version = '3.1.16' def jarName = 'MServer.jar' def mainClass = 'mServer.Main' diff --git a/src/main/java/mServer/crawler/sender/newsearch/ZDFSearchPageTask.java b/src/main/java/mServer/crawler/sender/newsearch/ZDFSearchPageTask.java index e98fa8bea..77e43df6e 100644 --- a/src/main/java/mServer/crawler/sender/newsearch/ZDFSearchPageTask.java +++ b/src/main/java/mServer/crawler/sender/newsearch/ZDFSearchPageTask.java @@ -51,16 +51,16 @@ protected Collection compute() { zdfEntryDTOList.forEach(zdfEntryDTO -> { if(zdfEntryDTO != null) { final ZDFEntryTask entryTask = new ZDFEntryTask(zdfEntryDTO); - + entryTask.fork(); subTasks.add(entryTask); if (MserverDaten.debug) Log.sysLog("EntryTask " + entryTask.hashCode() + " added."); } }); + // wait till entry tasks are finished - filmList.addAll(invokeAll(subTasks).parallelStream().map(ForkJoinTask::join). - collect(Collectors.toList())); + subTasks.forEach(t -> filmList.add(t.join())); if (MserverDaten.debug) Log.sysLog("All EntryTasks finished."); } diff --git a/src/main/java/mServer/crawler/sender/newsearch/ZDFSearchTask.java b/src/main/java/mServer/crawler/sender/newsearch/ZDFSearchTask.java index 286b85643..c0cffe411 100644 --- a/src/main/java/mServer/crawler/sender/newsearch/ZDFSearchTask.java +++ b/src/main/java/mServer/crawler/sender/newsearch/ZDFSearchTask.java @@ -16,15 +16,15 @@ public class ZDFSearchTask extends RecursiveTask> { private static final String JSON_ELEMENT_NEXT = "next"; - + private static final long serialVersionUID = 1L; private final Collection filmList; private final ZDFClient client; - + private int page; private final int days; - + public ZDFSearchTask(int aDays) { super(); @@ -38,35 +38,34 @@ public ZDFSearchTask(int aDays) @Override protected Collection compute() { - if(!Config.getStop()) { + if (!Config.getStop()) + { try { Collection subTasks = ConcurrentHashMap.newKeySet(); JsonObject baseObject; - do { + do + { baseObject = client.executeSearch(page, days, 1); - if(baseObject != null) { + if (baseObject != null) + { ZDFSearchPageTask task = new ZDFSearchPageTask(baseObject); - subTasks.add(task); task.fork(); + subTasks.add(task); if (MserverDaten.debug) Log.sysLog("SearchTask " + task.hashCode() + " added."); } page++; - } while(!Config.getStop() && baseObject != null && baseObject.has(JSON_ELEMENT_NEXT)); - filmList.addAll(subTasks.parallelStream() - .map(ForkJoinTask::join) - .flatMap(Collection::stream) - .collect(Collectors.toList()) - ); - + } while (!Config.getStop() && baseObject != null && baseObject.has(JSON_ELEMENT_NEXT)); + subTasks.forEach(t -> filmList.addAll(t.join())); if (MserverDaten.debug) Log.sysLog("All SearchTasks finished."); - } catch (Exception ex) { + } catch (Exception ex) + { Log.errorLog(496583201, ex); } } diff --git a/src/main/java/mServer/crawler/sender/newsearch/ZDFVideoDTODeserializer.java b/src/main/java/mServer/crawler/sender/newsearch/ZDFVideoDTODeserializer.java index 8dc9d6586..3896f68f1 100644 --- a/src/main/java/mServer/crawler/sender/newsearch/ZDFVideoDTODeserializer.java +++ b/src/main/java/mServer/crawler/sender/newsearch/ZDFVideoDTODeserializer.java @@ -18,7 +18,8 @@ /** * A JSON deserializer to gather the needed information for a {@link VideoDTO}. */ -public class ZDFVideoDTODeserializer implements JsonDeserializer { +public class ZDFVideoDTODeserializer implements JsonDeserializer +{ private static final String JSON_ELEMENT_BEGIN = "airtimeBegin"; private static final String JSON_ELEMENT_BRAND = "http://zdf.de/rels/brand"; @@ -27,7 +28,7 @@ public class ZDFVideoDTODeserializer implements JsonDeserializer { private static final String JSON_ELEMENT_DURATION = "duration"; private static final String JSON_ELEMENT_EDITORIALDATE = "editorialDate"; private static final String JSON_ELEMENT_LEADPARAGRAPH = "leadParagraph"; - private static final String JSON_ELEMENT_MAINVIDEO = "mainVideoContent"; + private static final String JSON_ELEMENT_MAINVIDEO = "mainVideoContent"; private static final String JSON_ELEMENT_PROGRAMMITEM = "programmeItem"; private static final String JSON_ELEMENT_SHARING_URL = "http://zdf.de/rels/sharing-url"; private static final String JSON_ELEMENT_SUBTITLE = "subtitle"; @@ -41,26 +42,23 @@ public class ZDFVideoDTODeserializer implements JsonDeserializer { private final FastDateFormat sdfOutDay = FastDateFormat.getInstance("dd.MM.yyyy"); @Override - public VideoDTO deserialize(final JsonElement aJsonElement, final Type aTypeOfT, final JsonDeserializationContext aJsonDeserializationContext) throws JsonParseException { + public VideoDTO deserialize(final JsonElement aJsonElement, final Type aTypeOfT, final JsonDeserializationContext aJsonDeserializationContext) throws JsonParseException + { VideoDTO dto = null; - try { + try + { dto = new VideoDTO(); JsonObject rootNode = aJsonElement.getAsJsonObject(); - JsonArray programmItem = rootNode.getAsJsonArray(JSON_ELEMENT_PROGRAMMITEM); JsonObject programmItemTarget = null; - if(programmItem != null) { - if(programmItem.size() > 1) { - throw new RuntimeException("Element does not contain programmitem" + rootNode.getAsString()); - } - - if(programmItem.size() == 1) { - programmItemTarget = programmItem.get(0).getAsJsonObject().get(JSON_ELEMENT_TARGET).getAsJsonObject(); - } + if (rootNode.has(JSON_ELEMENT_PROGRAMMITEM) && !rootNode.get(JSON_ELEMENT_PROGRAMMITEM).isJsonNull()) + { + JsonArray programmItem = rootNode.getAsJsonArray(JSON_ELEMENT_PROGRAMMITEM); + programmItemTarget = programmItem.get(0).getAsJsonObject().get(JSON_ELEMENT_TARGET).getAsJsonObject(); } - parseTitle(dto, rootNode, programmItemTarget); + parseTitle(dto, rootNode, programmItemTarget); parseTopic(dto, rootNode); parseDescription(dto, rootNode); @@ -68,140 +66,175 @@ public VideoDTO deserialize(final JsonElement aJsonElement, final Type aTypeOfT, parseAirtime(dto, rootNode, programmItemTarget); parseDuration(dto, rootNode); - } catch (UnsupportedOperationException ex) { + } catch (UnsupportedOperationException ex) + { if (MserverDaten.debug) Log.errorLog(496583256, ex); - } catch (Exception ex) { + } catch (Exception ex) + { dto = null; Log.errorLog(496583256, ex); - } - + } + return dto; } - - private void parseAirtime(VideoDTO dto, JsonObject rootNode, JsonObject programmItemTarget) { + + private void parseAirtime(VideoDTO dto, JsonObject rootNode, JsonObject programmItemTarget) + { String date; FastDateFormat sdf; - + // use broadcast airtime if found - if(programmItemTarget != null) { + if (programmItemTarget != null) + { JsonArray broadcastArray = programmItemTarget.getAsJsonArray(JSON_ELEMENT_BROADCAST); - if(broadcastArray == null || broadcastArray.size() < 1) { + if (broadcastArray == null || broadcastArray.size() < 1) + { date = getEditorialDate(rootNode); sdf = sdfEditorialDate; - } else { + } else + { // array is ordered ascending though the oldest broadcast is the first entry date = broadcastArray.get(0).getAsJsonObject().get(JSON_ELEMENT_BEGIN).getAsString(); sdf = sdfAirtimeBegin; } - } else { + } else + { // use editorialdate date = getEditorialDate(rootNode); sdf = sdfEditorialDate; } - if(!date.isEmpty()) { + if (!date.isEmpty()) + { dto.setDate(convertDate(date, sdf)); dto.setTime(convertTime(date, sdf)); } } - - private String getEditorialDate(JsonObject rootNode) { + + private String getEditorialDate(JsonObject rootNode) + { return rootNode.get(JSON_ELEMENT_EDITORIALDATE).getAsString(); } - private void parseWebsiteUrl(VideoDTO dto, JsonObject rootNode) { + private void parseWebsiteUrl(VideoDTO dto, JsonObject rootNode) + { String websiteUrl = rootNode.get(JSON_ELEMENT_SHARING_URL).getAsString(); dto.setWebsiteUrl(websiteUrl); } - - private void parseDuration(VideoDTO dto, JsonObject rootNode) { + + private void parseDuration(VideoDTO dto, JsonObject rootNode) + { JsonElement mainVideoElement = rootNode.get(JSON_ELEMENT_MAINVIDEO); - if(mainVideoElement != null) { + if (mainVideoElement != null) + { JsonObject mainVideo = mainVideoElement.getAsJsonObject(); JsonObject targetMainVideo = mainVideo.get(JSON_ELEMENT_TARGET).getAsJsonObject(); JsonElement duration = targetMainVideo.get(JSON_ELEMENT_DURATION); - if(duration != null) { - dto.setDuration(duration.getAsInt()); + if (duration != null) + { + dto.setDuration(duration.getAsInt()); } } } - - private void parseDescription(VideoDTO dto, JsonObject rootNode) { + + private void parseDescription(VideoDTO dto, JsonObject rootNode) + { JsonElement leadParagraph = rootNode.get(JSON_ELEMENT_LEADPARAGRAPH); - if(leadParagraph != null) { + if (leadParagraph != null) + { dto.setDescription(leadParagraph.getAsString()); - } else { + } else + { JsonElement teaserText = rootNode.get(JSON_ELEMENT_TEASERTEXT); - if(teaserText != null) { + if (teaserText != null) + { dto.setDescription(teaserText.getAsString()); - } + } } } - - private void parseTitle(VideoDTO dto, JsonObject rootNode, JsonObject target) { + + private void parseTitle(VideoDTO dto, JsonObject rootNode, JsonObject target) + { // use property "title" if found JsonElement titleElement = rootNode.get(JSON_ELEMENT_TITLE); - if(titleElement != null) { + if (titleElement != null) + { JsonElement subTitleElement = rootNode.get(JSON_ELEMENT_SUBTITLE); - if(subTitleElement != null) { + if (subTitleElement != null) + { dto.setTitle(titleElement.getAsString() + " - " + subTitleElement.getAsString()); - } else { + } else + { dto.setTitle(titleElement.getAsString()); } - } else { + } else + { // programmItem target required to determine title String title = target.get(JSON_ELEMENT_TITLE).getAsString(); String subTitle = target.get(JSON_ELEMENT_SUBTITLE).getAsString(); - if(subTitle.isEmpty()) { + if (subTitle.isEmpty()) + { dto.setTitle(title); - } else { + } else + { dto.setTitle(title + " - " + subTitle); - } + } } } - - private void parseTopic(VideoDTO dto, JsonObject rootNode) { + + private void parseTopic(VideoDTO dto, JsonObject rootNode) + { JsonObject brand = rootNode.getAsJsonObject(JSON_ELEMENT_BRAND); JsonObject category = rootNode.getAsJsonObject(JSON_ELEMENT_CATEGORY); - if(brand != null) { + if (brand != null) + { // first use brand JsonElement topic = brand.get(JSON_ELEMENT_TITLE); - if(topic != null) { + if (topic != null) + { dto.setTopic(topic.getAsString()); return; } } - - if(category != null) { + + if (category != null) + { // second use category JsonElement topic = category.get(JSON_ELEMENT_TITLE); - if(topic != null) { + if (topic != null) + { dto.setTopic(topic.getAsString()); return; } } - + // if no topic found, set topic to title dto.setTopic(dto.getTitle()); } - private String convertDate(String dateValue, FastDateFormat sdf) { - try { + private String convertDate(String dateValue, FastDateFormat sdf) + { + try + { return sdfOutDay.format(sdf.parse(dateValue)); - } catch (ParseException ex) { + } catch (ParseException ex) + { throw new RuntimeException("Date parse exception: " + dateValue); } } - private String convertTime(String dateValue, FastDateFormat sdf) { - try { + private String convertTime(String dateValue, FastDateFormat sdf) + { + try + { return sdfOutTime.format(sdf.parse(dateValue)); - } catch (ParseException ex) { + } catch (ParseException ex) + { throw new RuntimeException("Date parse exception: " + dateValue); } - } + } }