From 2d1755d4e79136733d58bae271d6d7ebdc5e1ee4 Mon Sep 17 00:00:00 2001 From: "W.Xaver" Date: Tue, 6 Dec 2016 19:54:06 +0100 Subject: [PATCH 1/8] build.gradle: start server --- build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle b/build.gradle index bfec51a6a..8bb729741 100644 --- a/build.gradle +++ b/build.gradle @@ -113,6 +113,11 @@ distributions { } } +task run(type: JavaExec, dependsOn: classes) { + main = 'mServer.Main' + classpath = sourceSets.main.runtimeClasspath +} + /** * HOWTO debug: * 1. run "gradle debug" From 9f475cb94aeca3c29af86c0b05ae0ac74f511a3e Mon Sep 17 00:00:00 2001 From: Nicklas Wiegandt Date: Fri, 9 Dec 2016 18:41:13 +0100 Subject: [PATCH 2/8] Updated and cleaned up the gradle build file. --- build.gradle | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index 8bb729741..392dcf689 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ task copyRuntimeLibs(type: Copy) { repositories { maven { - url "https://repo.mediathekview.de/repository/maven-public/" + url "https://repo.mediathekview.de/repository/maven-public/" } } @@ -43,7 +43,7 @@ dependencies { ext { propsFile = file('src/main/resources/version.properties').absoluteFile - if(!propsFile.exists()) { + if (!propsFile.exists()) { Files.createFile(propsFile.toPath()) } } @@ -57,11 +57,11 @@ def loadVersionProperties() { task updateVersion << { Properties props = loadVersionProperties() def oldVersion = props.getProperty('VERSION') - if(!oldVersion.equals(project.version)) { + if (!oldVersion.equals(project.version)) { logger.lifecycle "==mserver======================" logger.lifecycle "Version: $project.version" logger.lifecycle "==mserver======================" - props.setProperty('VERSION', project.version) + props.setProperty('VERSION', project.version) props.store(propsFile.newWriter(), null) } } @@ -113,20 +113,33 @@ distributions { } } +/** + * This is how you pass arguments: "./gradlew run -Pargs=arg1,arg2,arg3 + */ task run(type: JavaExec, dependsOn: classes) { - main = 'mServer.Main' + main = mainClass classpath = sourceSets.main.runtimeClasspath + + if (project.hasProperty('args')) { + args(project.args.split(',')) + } + } /** * HOWTO debug: * 1. run "gradle debug" * 2. Call your IDE to connect to a remote java application on port 5005. + * + * This is how you pass arguments: "./gradlew debug -Pargs=arg1,arg2,arg3 */ -task debug (type: JavaExec, dependsOn: classes){ - main = 'mServer.Main' +task debug(type: JavaExec, dependsOn: classes) { + main = mainClass classpath = sourceSets.main.runtimeClasspath - args '-gui' debug true + + if (project.hasProperty('args')) { + args(project.args.split(',')) + } } From f03a5a4e236a498015a87ecc920108461cd5ac00 Mon Sep 17 00:00:00 2001 From: "W.Xaver" Date: Wed, 14 Dec 2016 18:14:40 +0100 Subject: [PATCH 3/8] upgrade url --- .../crawler/sender/MediathekReader.java | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/src/main/java/mServer/crawler/sender/MediathekReader.java b/src/main/java/mServer/crawler/sender/MediathekReader.java index 84d2ecf12..dc6ec8a21 100644 --- a/src/main/java/mServer/crawler/sender/MediathekReader.java +++ b/src/main/java/mServer/crawler/sender/MediathekReader.java @@ -29,10 +29,10 @@ import mSearch.daten.DatenFilm; import mSearch.tool.GermanStringSorter; import mSearch.tool.Log; +import mServer.crawler.CrawlerTool; import mServer.crawler.FilmeSuchen; import mServer.crawler.GetUrl; import mServer.crawler.RunSender; -import mServer.crawler.CrawlerTool; public class MediathekReader implements Runnable { @@ -139,6 +139,18 @@ void addFilm(DatenFilm film) { if (film.arr[DatenFilm.FILM_GROESSE].isEmpty()) { film.arr[DatenFilm.FILM_GROESSE] = mSearchFilmeSuchen.listeFilmeAlt.getFileSizeUrl(film.arr[DatenFilm.FILM_URL], film.arr[DatenFilm.FILM_SENDER]); } + +// // zum Testen +// if (film.isHD()) { +// return; +// } + + upgradeUrl(film); + +// if (!film.isHD()) { +// return; +// } + film.setUrlHistory(); CrawlerTool.setGeo(film); if (mSearchFilmeSuchen.listeFilmeNeu.addFilmVomSender(film)) { @@ -223,6 +235,50 @@ synchronized void meldungThreadUndFertig() { final static int TIMEOUT = 3000; // ms //ToDo evtl. wieder kürzen!! + private void upgradeUrl(DatenFilm film) { + // versuchen HD anhand der URL zu suchen, wo noch nicht vorhanden + if (film.isHD()) { + return; + } + + // http://media.ndr.de/progressive/2016/0817/TV-20160817-1113-2300.hq.mp4 + // http://media.ndr.de/progressive/2016/0817/TV-20160817-1113-2300.hd.mp4 -> HD + if (film.arr[DatenFilm.FILM_URL].startsWith("http://media.ndr.de") && film.arr[DatenFilm.FILM_URL].endsWith(".hq.mp4")) { + String from = film.arr[DatenFilm.FILM_URL]; + String to = film.arr[DatenFilm.FILM_URL].replace(".hq.mp4", ".hd.mp4"); + updateHd(from, to, film); + } + + // http://cdn-storage.br.de/iLCpbHJGNLT6NK9HsLo6s61luK4C_2rc571S/_AJS/_ArG_2bP_71S/583da0ef-3e92-4648-bb22-1b14d739aa91_C.mp4 + // http://cdn-storage.br.de/iLCpbHJGNLT6NK9HsLo6s61luK4C_2rc571S/_AJS/_ArG_2bP_71S/583da0ef-3e92-4648-bb22-1b14d739aa91_X.mp4 -> HD + if (film.arr[DatenFilm.FILM_URL].startsWith("http://cdn-storage.br.de") && film.arr[DatenFilm.FILM_URL].endsWith("_C.mp4")) { + String from = film.arr[DatenFilm.FILM_URL]; + String to = film.arr[DatenFilm.FILM_URL].replace("_C.mp4", "_X.mp4"); + updateHd(from, to, film); + } + + // http://pd-ondemand.swr.de/das-erste/buffet/904278.l.mp4 + // http://pd-ondemand.swr.de/das-erste/buffet/904278.xl.mp4 -> HD + if (film.arr[DatenFilm.FILM_URL].startsWith("http://pd-ondemand.swr.de") && film.arr[DatenFilm.FILM_URL].endsWith(".l.mp4")) { + String from = film.arr[DatenFilm.FILM_URL]; + String to = film.arr[DatenFilm.FILM_URL].replace(".l.mp4", ".xl.mp4"); + updateHd(from, to, film); + } + } + + private void updateHd(String from, String to, DatenFilm film) { + if (film.arr[DatenFilm.FILM_URL_HD].isEmpty() && film.arr[DatenFilm.FILM_URL].endsWith(from)) { + String url_ = film.arr[DatenFilm.FILM_URL].substring(0, film.arr[DatenFilm.FILM_URL].lastIndexOf(from)) + to; + // zum Testen immer machen!! + if (urlExists(url_)) { + CrawlerTool.addUrlHd(film, url_, ""); + //Log.sysLog("upgradeUrl: " + film.arr[DatenFilm.FILM_SENDER]); + } else { + Log.errorLog(945120347, "upgradeUrl: " + from); + } + } + } + public static boolean urlExists(String url) { // liefert liefert true, wenn es die URL gibt // brauchts, um Filmurls zu prüfen From 1cbcd936150499225bfa87fc1558bde1ac84779b Mon Sep 17 00:00:00 2001 From: Nicklas Wiegandt Date: Wed, 14 Dec 2016 09:23:07 +0100 Subject: [PATCH 4/8] Fixed travis deploy Added missing subfolder --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b4e0b3304..60d4f51fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,8 +27,8 @@ deploy: file_glob: true file: - "build/libs/MServer-.*.jar" - - "build/MServer.*.zip" - - "build/MServer.*.tar" + - "build/distributions/MServer.*.zip" + - "build/distributions/MServer.*.tar" skip_cleanup: true on: tags: true From 6a2916df37b0a4ae6115a7cb4b276dd2f3a76432 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 16 Dec 2016 06:12:50 +0100 Subject: [PATCH 5/8] Version auf 2.2.0 angehoben. --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 392dcf689..c94ad17ed 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ import org.apache.tools.ant.filters.ReplaceTokens sourceCompatibility = 1.8 targetCompatibility = 1.8 -version = '2.1.0' +version = '2.2.0' def jarName = 'MServer.jar' def mainClass = 'mServer.Main' From 60212aa0305e5c680028f5e1166ef80136368905 Mon Sep 17 00:00:00 2001 From: "W.Xaver" Date: Fri, 16 Dec 2016 15:31:12 +0100 Subject: [PATCH 6/8] ARTE.de, ARTE.FR --- .../crawler/sender/MediathekArte_de.java | 447 +++++++++++------- .../crawler/sender/MediathekArte_fr.java | 4 + 2 files changed, 272 insertions(+), 179 deletions(-) diff --git a/src/main/java/mServer/crawler/sender/MediathekArte_de.java b/src/main/java/mServer/crawler/sender/MediathekArte_de.java index a4e4c5aeb..b848f5d09 100644 --- a/src/main/java/mServer/crawler/sender/MediathekArte_de.java +++ b/src/main/java/mServer/crawler/sender/MediathekArte_de.java @@ -20,15 +20,16 @@ package mServer.crawler.sender; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import mSearch.Config; import mSearch.Const; import mSearch.daten.DatenFilm; import mSearch.tool.Log; import mSearch.tool.MSStringBuilder; +import mServer.crawler.CrawlerTool; import mServer.crawler.FilmeSuchen; import mServer.crawler.GetUrl; -import mServer.crawler.CrawlerTool; public class MediathekArte_de extends MediathekReader implements Runnable { @@ -40,6 +41,9 @@ public class MediathekArte_de extends MediathekReader implements Runnable { String URL_ARTE = "http://www.arte.tv/papi/tvguide/epg/schedule/D/L3/"; String URL_CONCERT = "http://concert.arte.tv/de/videos/all"; String URL_CONCERT_NOT_CONTAIN = "-STF"; + String URL_ARTE_MEDIATHEK = "http://www.arte.tv/guide/de/plus7/alle-videos?date=j-"; + String TIME_1 = "
  • Sendetermine:
    • "; + String TIME_2 = "um"; public MediathekArte_de(FilmeSuchen ssearch, int startPrio) { super(ssearch, SENDERNAME,/* threads */ 2, /* urlWarten */ 500, startPrio); @@ -60,7 +64,7 @@ public void addToList() { addTage(); if (Config.getStop()) { meldungThreadUndFertig(); - } else if (listeThemen.size() == 0) { + } else if (listeThemen.isEmpty()) { if (CrawlerTool.loadLongMax()) { addConcert(); } else { @@ -90,15 +94,9 @@ private void addConcert() { } private void addTage() { - // http://www.arte.tv/papi/tvguide/epg/schedule/D/L3/2013-08-04/2013-8-04.json - Date d = new Date(); - String out1, out2, u; - SimpleDateFormat formatter1 = new SimpleDateFormat("yyyy-MM-dd"); - SimpleDateFormat formatter2 = new SimpleDateFormat("yyyy-M-dd"); + // http://www.arte.tv/guide/de/plus7/alle-videos?date=j-0 for (int i = 0; i <= 14; ++i) { - out1 = formatter1.format(new Date(d.getTime() - i * (1000 * 60 * 60 * 24))); - out2 = formatter2.format(new Date(d.getTime() - i * (1000 * 60 * 60 * 24))); - u = URL_ARTE + out1 + "/" + out2 + ".json"; + String u = URL_ARTE_MEDIATHEK + i; listeThemen.add(new String[]{u}); } } @@ -231,10 +229,10 @@ private void addConcert(int start, int anz) { DatenFilm film = new DatenFilm(sendername, THEMA, urlWeb, titel, urlNormal, "" /*urlRtmp*/, datum, "" /*zeit*/, duration, beschreibung); if (!urlHd.isEmpty()) { - CrawlerTool.addUrlHd(film,urlHd, ""); + CrawlerTool.addUrlHd(film, urlHd, ""); } if (!urlLow.isEmpty()) { - CrawlerTool.addUrlKlein(film,urlLow, ""); + CrawlerTool.addUrlKlein(film, urlLow, ""); } addFilm(film); } @@ -253,6 +251,8 @@ class ThemaLaden implements Runnable { GetUrl getUrl = new GetUrl(wartenSeiteLaden); private final MSStringBuilder seite1 = new MSStringBuilder(Const.STRING_BUFFER_START_BUFFER); private final MSStringBuilder seite2 = new MSStringBuilder(Const.STRING_BUFFER_START_BUFFER); + private final MSStringBuilder seite3 = new MSStringBuilder(Const.STRING_BUFFER_START_BUFFER); + private final ArrayList liste = new ArrayList<>(); @Override public void run() { @@ -261,211 +261,300 @@ public void run() { String link[]; while (!Config.getStop() && (link = listeThemen.getListeThemen()) != null) { meldungProgress(link[0] /* url */); - addTheman(seite1, seite2, link[0]); + addThemen(link[0]); } } catch (Exception ex) { Log.errorLog(894330854, ex, ""); } meldungThreadUndFertig(); } - } - private void addTheman(MSStringBuilder seite1, MSStringBuilder seite2, String startUrl) { - // Datum, Zeit: "BAD":"04/08/2013","BAT":"13:20" - final String MUSTER_START = "{\"programId\":"; - final String MUSTER_URL_JSON = "\"videoStreamUrl\":\""; - final String MUSTER_DATUM = "\"BAD\":\""; - final String MUSTER_ZEIT = "\"BAT\":\""; - final String MUSTER_TITEL = "\"TIT\":\""; - final String MUSTER_THEMA = "\"GEN\":\""; - String[] arr; - seite1 = getUrlIo.getUri_Utf(sendername, startUrl, seite1, ""); - int posStart = 0, posStop; - int pos1; - int pos2; - int pos; - String urlJson; - String datum; - String zeit; - String titel, thema; - while ((posStart = seite1.indexOf(MUSTER_START, posStart)) != -1) { - posStart += MUSTER_START.length(); - posStop = seite1.indexOf(MUSTER_START, posStart); - urlJson = ""; - datum = ""; - zeit = ""; - titel = ""; - thema = ""; - if ((pos1 = seite1.indexOf(MUSTER_URL_JSON, posStart)) != -1) { - pos1 += MUSTER_URL_JSON.length(); - if (posStop == -1 || pos1 < posStop) { - if ((pos2 = seite1.indexOf("\"", pos1)) != -1) { - urlJson = seite1.substring(pos1, pos2); - } + private void addThemen(String startUrl) { + getUrl.getUri_Utf(sendername, startUrl, seite1, ""); + seite1.extractList("url":"http:\\/\\/www.arte.tv", """, liste); + for (String s : liste) { + if (Config.getStop()) { + break; + } + s = "http://www.arte.tv" + s.replace("\\", ""); + //Datum: url: xx-0 => heute, xx-1 => gestern, ... + String date = ""; + try { + String d = startUrl.substring(startUrl.lastIndexOf("-") + 1); + int iD = Integer.parseInt(d); + SimpleDateFormat form = new SimpleDateFormat("dd.MM.yyyy"); + date = form.format(new Date(new Date().getTime() - iD * (1000 * 60 * 60 * 24))); + } catch (Exception ignore) { } + getFilm1(s, date); } - pos = posStart; - while ((pos = seite1.indexOf(MUSTER_DATUM, pos)) != -1) { - pos += MUSTER_DATUM.length(); - if (posStop != -1 && pos > posStop) { - break; + } + + private void getFilm1(String filmWebsite, String date) { + getUrl.getUri_Utf(sendername, filmWebsite, seite1, ""); + String title = seite1.extract("

    ", "<"); + String subtitle = seite1.extract("

    ", "<"); + if (!subtitle.isEmpty() && !title.equals(subtitle)) { + title = title + " - " + subtitle; + } + String thema = seite1.extract("", "<"); + String description = seite1.extract("

    ", "<"); + String duration = seite1.extract("", "<"); + long dauer = 0; + try { + duration = duration.replace("Min.", ""); + duration = duration.replace("\n", "").trim(); + dauer = Integer.parseInt(duration) * 60; + } catch (Exception ignore) { + } + String time = seite1.extract(TIME_1, TIME_2, "<"); //Donnerstag, 15. Dezember um 23.30 Uhr + time = time.replace("\n", ""); + time = time.replace("Uhr", "").trim(); + time = time.replace(".", ":"); + time = time.replace("h", ":"); + time = time + ":00"; + if (time.length() < 8) { + time = "0" + time; + } + + String fUrl = seite1.extract("src=\"http://www.arte.tv/player", "\""); + + if (!fUrl.isEmpty()) { + fUrl = "http://www.arte.tv/player" + fUrl; + meldung(fUrl); + getFilm2(fUrl, filmWebsite, thema, title, description, dauer, date, time); + } + } + + private void getFilm2(String urlWeb, String filmWebsite, String thema, String title, String description, long dauer, String date, String time) { + getUrl.getUri_Utf(sendername, urlWeb, seite1, ""); + String urlHd = seite1.extract("\"id\":\"HTTP_MP4_SQ_1\"", "\"url\":\"", "\"").replace("\\", ""); + String urlNorm = seite1.extract("\"id\":\"HTTP_MP4_EQ_1\"", "\"url\":\"", "\"").replace("\\", ""); + String urlKlein = seite1.extract("\"id\":\"HTTP_MP4_HQ_1\"", "\"url\":\"", "\"").replace("\\", ""); + + if (urlNorm.isEmpty() && !urlKlein.isEmpty()) { + urlNorm = urlKlein; + urlKlein = ""; + } + if (urlNorm.isEmpty() && !urlHd.isEmpty()) { + urlNorm = urlHd; + urlHd = ""; + } + + if (!urlNorm.isEmpty() && !urlNorm.endsWith("EXTRAIT.mp4")) { + // http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_SQ_1_VA_00604871_MP4-2200_AMM-HBBTV_EXTRAIT.mp4 + // sind nur Trailer + DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, title, urlNorm, "" /*urlRtmp*/, + date, time, dauer, description); + if (!urlKlein.isEmpty()) { + CrawlerTool.addUrlKlein(film, urlKlein, ""); } - if (posStop == -1 || pos < posStop) { - if ((pos2 = seite1.indexOf("\"", pos)) != -1) { - datum = seite1.substring(pos, pos2); - } + if (!urlHd.isEmpty()) { + CrawlerTool.addUrlHd(film, urlHd, ""); } + addFilm(film); + } else { + Log.errorLog(915263647, "Keine URL: " + filmWebsite); } - pos = posStart; - while ((pos = seite1.indexOf(MUSTER_ZEIT, pos)) != -1) { - pos += MUSTER_ZEIT.length(); - if (posStop != -1 && pos > posStop) { - break; + } + + private void addTheman(String startUrl) { + // Datum, Zeit: "BAD":"04/08/2013","BAT":"13:20" + final String MUSTER_START = "{\"programId\":"; + final String MUSTER_URL_JSON = "\"videoStreamUrl\":\""; + final String MUSTER_DATUM = "\"BAD\":\""; + final String MUSTER_ZEIT = "\"BAT\":\""; + final String MUSTER_TITEL = "\"TIT\":\""; + final String MUSTER_THEMA = "\"GEN\":\""; + String[] arr; + getUrlIo.getUri_Utf(sendername, startUrl, seite1, ""); + int posStart = 0, posStop; + int pos1; + int pos2; + int pos; + String urlJson; + String datum; + String zeit; + String titel, thema; + while ((posStart = seite1.indexOf(MUSTER_START, posStart)) != -1) { + posStart += MUSTER_START.length(); + posStop = seite1.indexOf(MUSTER_START, posStart); + urlJson = ""; + datum = ""; + zeit = ""; + titel = ""; + thema = ""; + if ((pos1 = seite1.indexOf(MUSTER_URL_JSON, posStart)) != -1) { + pos1 += MUSTER_URL_JSON.length(); + if (posStop == -1 || pos1 < posStop) { + if ((pos2 = seite1.indexOf("\"", pos1)) != -1) { + urlJson = seite1.substring(pos1, pos2); + } + } } - if (posStop == -1 || pos < posStop) { - if ((pos2 = seite1.indexOf("\"", pos)) != -1) { - zeit = seite1.substring(pos, pos2); + pos = posStart; + while ((pos = seite1.indexOf(MUSTER_DATUM, pos)) != -1) { + pos += MUSTER_DATUM.length(); + if (posStop != -1 && pos > posStop) { + break; + } + if (posStop == -1 || pos < posStop) { + if ((pos2 = seite1.indexOf("\"", pos)) != -1) { + datum = seite1.substring(pos, pos2); + } } } - } - if ((pos1 = seite1.indexOf(MUSTER_TITEL, posStart)) != -1) { - pos1 += MUSTER_TITEL.length(); - if (posStop == -1 || pos1 < posStop) { - if ((pos2 = seite1.indexOf("\",", pos1)) != -1) { - titel = seite1.substring(pos1, pos2); - titel = titel.replace("\\", ""); + pos = posStart; + while ((pos = seite1.indexOf(MUSTER_ZEIT, pos)) != -1) { + pos += MUSTER_ZEIT.length(); + if (posStop != -1 && pos > posStop) { + break; + } + if (posStop == -1 || pos < posStop) { + if ((pos2 = seite1.indexOf("\"", pos)) != -1) { + zeit = seite1.substring(pos, pos2); + } } } - } - if ((pos1 = seite1.indexOf(MUSTER_THEMA, posStart)) != -1) { - pos1 += MUSTER_THEMA.length(); - if (posStop == -1 || pos1 < posStop) { - if ((pos2 = seite1.indexOf("\"", pos1)) != -1) { - thema = seite1.substring(pos1, pos2); + if ((pos1 = seite1.indexOf(MUSTER_TITEL, posStart)) != -1) { + pos1 += MUSTER_TITEL.length(); + if (posStop == -1 || pos1 < posStop) { + if ((pos2 = seite1.indexOf("\",", pos1)) != -1) { + titel = seite1.substring(pos1, pos2); + titel = titel.replace("\\", ""); + } } } - } - if (!urlJson.isEmpty()) { - arr = new String[]{urlJson, datum, zeit, titel, thema}; - filmeLaden(seite2, arr); - } else { + if ((pos1 = seite1.indexOf(MUSTER_THEMA, posStart)) != -1) { + pos1 += MUSTER_THEMA.length(); + if (posStop == -1 || pos1 < posStop) { + if ((pos2 = seite1.indexOf("\"", pos1)) != -1) { + thema = seite1.substring(pos1, pos2); + } + } + } + if (!urlJson.isEmpty()) { + arr = new String[]{urlJson, datum, zeit, titel, thema}; + filmeLaden(seite2, arr); + } else { // Log.fehlerMeldung(-956230147, Log.FEHLER_ART_MREADER, "MediathekArte_de.addThemen", "Keine URL: " + startUrl + "**" + count); + } } } - } - void filmeLaden(MSStringBuilder seite, String[] arr) { - // url_hd url, url_klein - //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"SQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_SQ_2_VOF_00604879_MP4-2200_AMM-HBBTV_EXTRAIT.mp4"}, - //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"EQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_EQ_2_VOF_00604878_MP4-1500_AMM-HBBTV_EXTRAIT.mp4"}, - //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"HQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_HQ_2_VOF_00604876_MP4-800_AMM-HBBTV_EXTRAIT.mp4"}, + void filmeLaden(MSStringBuilder seite, String[] arr) { + // url_hd url, url_klein + //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"SQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_SQ_2_VOF_00604879_MP4-2200_AMM-HBBTV_EXTRAIT.mp4"}, + //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"EQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_EQ_2_VOF_00604878_MP4-1500_AMM-HBBTV_EXTRAIT.mp4"}, + //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"HQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_HQ_2_VOF_00604876_MP4-800_AMM-HBBTV_EXTRAIT.mp4"}, - String datum = "", zeit = ""; - String urlHd = "", urlKlein = "", url = ""; - String beschreibung = ""; - String filmWebsite = ""; - String dauerStr = ""; - String titel = "", thema = "", subTitle = ""; - long dauer = 0; - final String MUSTER_BESCHREIBUNG = "\"VDE\":\""; - final String MUSTER_FILM_WEBSITE = "\"VUP\":\""; - final String MUSTER_URL_HD = "\"HBBTV\",\"VQU\":\"SQ\",\"VMT\":\"mp4\",\"VUR\":\""; - final String MUSTER_URL = "HBBTV\",\"VQU\":\"EQ\",\"VMT\":\"mp4\",\"VUR\":\""; - final String MUSTER_URL_KLEIN = "HBBTV\",\"VQU\":\"HQ\",\"VMT\":\"mp4\",\"VUR\":\""; - final String MUSTER_DAUER = "\"videoDurationSeconds\":"; - int pos1, pos2; - if (Config.getStop()) { - return; - } - meldung(arr[0]); - seite = getUrlIo.getUri_Utf(sendername, arr[0], seite, ""); - if ((pos1 = seite.indexOf(MUSTER_BESCHREIBUNG)) != -1) { - pos1 += MUSTER_BESCHREIBUNG.length(); - if ((pos2 = seite.indexOf("\",", pos1)) != -1) { - beschreibung = seite.substring(pos1, pos2); - if (!beschreibung.isEmpty() && beschreibung.endsWith("\"")) { - beschreibung = beschreibung.substring(0, beschreibung.length() - 2); + String datum = "", zeit = ""; + String urlHd = "", urlKlein = "", url = ""; + String beschreibung = ""; + String filmWebsite = ""; + String dauerStr = ""; + String titel = "", thema = "", subTitle = ""; + long dauer = 0; + final String MUSTER_BESCHREIBUNG = "\"VDE\":\""; + final String MUSTER_FILM_WEBSITE = "\"VUP\":\""; + final String MUSTER_URL_HD = "\"HBBTV\",\"VQU\":\"SQ\",\"VMT\":\"mp4\",\"VUR\":\""; + final String MUSTER_URL = "HBBTV\",\"VQU\":\"EQ\",\"VMT\":\"mp4\",\"VUR\":\""; + final String MUSTER_URL_KLEIN = "HBBTV\",\"VQU\":\"HQ\",\"VMT\":\"mp4\",\"VUR\":\""; + final String MUSTER_DAUER = "\"videoDurationSeconds\":"; + int pos1, pos2; + if (Config.getStop()) { + return; + } + meldung(arr[0]); + seite = getUrlIo.getUri_Utf(sendername, arr[0], seite, ""); + if ((pos1 = seite.indexOf(MUSTER_BESCHREIBUNG)) != -1) { + pos1 += MUSTER_BESCHREIBUNG.length(); + if ((pos2 = seite.indexOf("\",", pos1)) != -1) { + beschreibung = seite.substring(pos1, pos2); + if (!beschreibung.isEmpty() && beschreibung.endsWith("\"")) { + beschreibung = beschreibung.substring(0, beschreibung.length() - 2); + } } } - } - if ((pos1 = seite.indexOf(MUSTER_FILM_WEBSITE)) != -1) { - pos1 += MUSTER_FILM_WEBSITE.length(); - if ((pos2 = seite.indexOf("\"", pos1)) != -1) { - filmWebsite = seite.substring(pos1, pos2); + if ((pos1 = seite.indexOf(MUSTER_FILM_WEBSITE)) != -1) { + pos1 += MUSTER_FILM_WEBSITE.length(); + if ((pos2 = seite.indexOf("\"", pos1)) != -1) { + filmWebsite = seite.substring(pos1, pos2); + } } - } - if ((pos1 = seite.indexOf(MUSTER_URL_HD)) != -1) { - pos1 += MUSTER_URL_HD.length(); - if ((pos2 = seite.indexOf("\"", pos1)) != -1) { - urlHd = seite.substring(pos1, pos2); + if ((pos1 = seite.indexOf(MUSTER_URL_HD)) != -1) { + pos1 += MUSTER_URL_HD.length(); + if ((pos2 = seite.indexOf("\"", pos1)) != -1) { + urlHd = seite.substring(pos1, pos2); + } } - } - if ((pos1 = seite.indexOf(MUSTER_URL_KLEIN)) != -1) { - pos1 += MUSTER_URL_KLEIN.length(); - if ((pos2 = seite.indexOf("\"", pos1)) != -1) { - urlKlein = seite.substring(pos1, pos2); + if ((pos1 = seite.indexOf(MUSTER_URL_KLEIN)) != -1) { + pos1 += MUSTER_URL_KLEIN.length(); + if ((pos2 = seite.indexOf("\"", pos1)) != -1) { + urlKlein = seite.substring(pos1, pos2); + } } - } - if ((pos1 = seite.indexOf(MUSTER_URL)) != -1) { - pos1 += MUSTER_URL.length(); - if ((pos2 = seite.indexOf("\"", pos1)) != -1) { - url = seite.substring(pos1, pos2); + if ((pos1 = seite.indexOf(MUSTER_URL)) != -1) { + pos1 += MUSTER_URL.length(); + if ((pos2 = seite.indexOf("\"", pos1)) != -1) { + url = seite.substring(pos1, pos2); + } } - } - if ((pos1 = seite.indexOf(MUSTER_DAUER)) != -1) { - pos1 += MUSTER_DAUER.length(); - if ((pos2 = seite.indexOf(",", pos1)) != -1) { - dauerStr = seite.substring(pos1, pos2); - if (!dauerStr.isEmpty()) { - try { - dauer = Long.parseLong(dauerStr); - } catch (Exception ex) { - dauer = 0; - } + if ((pos1 = seite.indexOf(MUSTER_DAUER)) != -1) { + pos1 += MUSTER_DAUER.length(); + if ((pos2 = seite.indexOf(",", pos1)) != -1) { + dauerStr = seite.substring(pos1, pos2); + if (!dauerStr.isEmpty()) { + try { + dauer = Long.parseLong(dauerStr); + } catch (Exception ex) { + dauer = 0; + } + } } } - } - // Datum ändern - // arr = new String[]{urlJson, datum, zeit, titel, thema}; - datum = convertDatum(arr[1]); - zeit = convertZeit(arr[2]); - titel = arr[3]; - subTitle = seite.extract("\"VSU\":\"", "\","); - subTitle = subTitle.replace("\\\"", "\""); - if (!subTitle.isEmpty() && !titel.equals(subTitle)) { - titel = titel + " - " + subTitle; - } - thema = arr[4]; - if (!url.isEmpty()) { - // public DatenFilm(String ssender, String tthema, String filmWebsite, String ttitel, String uurl, String uurlRtmp, - // String datum, String zeit, long dauerSekunden, String description, String thumbnailUrl, String imageUrl, String[] keywords) { - if (!url.endsWith("EXTRAIT.mp4")) { - // http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_SQ_1_VA_00604871_MP4-2200_AMM-HBBTV_EXTRAIT.mp4 - // sind nur Trailer - DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, url, "" /*urlRtmp*/, - datum, zeit, dauer, beschreibung); - if (!urlKlein.isEmpty()) { - CrawlerTool.addUrlKlein(film,urlKlein, ""); + // Datum ändern + // arr = new String[]{urlJson, datum, zeit, titel, thema}; + datum = convertDatum(arr[1]); + zeit = convertZeit(arr[2]); + titel = arr[3]; + subTitle = seite.extract("\"VSU\":\"", "\","); + subTitle = subTitle.replace("\\\"", "\""); + if (!subTitle.isEmpty() && !titel.equals(subTitle)) { + titel = titel + " - " + subTitle; + } + thema = arr[4]; + if (!url.isEmpty()) { + // public DatenFilm(String ssender, String tthema, String filmWebsite, String ttitel, String uurl, String uurlRtmp, + // String datum, String zeit, long dauerSekunden, String description, String thumbnailUrl, String imageUrl, String[] keywords) { + if (!url.endsWith("EXTRAIT.mp4")) { + // http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_SQ_1_VA_00604871_MP4-2200_AMM-HBBTV_EXTRAIT.mp4 + // sind nur Trailer + DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, url, "" /*urlRtmp*/, + datum, zeit, dauer, beschreibung); + if (!urlKlein.isEmpty()) { + CrawlerTool.addUrlKlein(film, urlKlein, ""); + } + if (!urlHd.isEmpty()) { + CrawlerTool.addUrlHd(film, urlHd, ""); + } + addFilm(film); } + } else if (!urlKlein.isEmpty()) { + DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, urlKlein, "" /*urlRtmp*/, + datum, zeit, dauer, beschreibung); if (!urlHd.isEmpty()) { - CrawlerTool.addUrlHd(film,urlHd, ""); + CrawlerTool.addUrlHd(film, urlHd, ""); } addFilm(film); + } else if (!urlHd.isEmpty()) { + DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, urlHd, "" /*urlRtmp*/, + datum, zeit, dauer, beschreibung); + addFilm(film); + } else { + Log.errorLog(963025874, "Keine URL: " + arr[0]); } - } else if (!urlKlein.isEmpty()) { - DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, urlKlein, "" /*urlRtmp*/, - datum, zeit, dauer, beschreibung); - if (!urlHd.isEmpty()) { - CrawlerTool.addUrlHd(film,urlHd, ""); - } - addFilm(film); - } else if (!urlHd.isEmpty()) { - DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, urlHd, "" /*urlRtmp*/, - datum, zeit, dauer, beschreibung); - addFilm(film); - } else { - Log.errorLog(963025874, "Keine URL: " + arr[0]); } } diff --git a/src/main/java/mServer/crawler/sender/MediathekArte_fr.java b/src/main/java/mServer/crawler/sender/MediathekArte_fr.java index 4b5454d88..2af195375 100644 --- a/src/main/java/mServer/crawler/sender/MediathekArte_fr.java +++ b/src/main/java/mServer/crawler/sender/MediathekArte_fr.java @@ -36,5 +36,9 @@ public MediathekArte_fr(FilmeSuchen ssearch, int startPrio) { URL_ARTE = "http://www.arte.tv/papi/tvguide/epg/schedule/F/L3/"; URL_CONCERT = "http://concert.arte.tv/fr/videos/all"; URL_CONCERT_NOT_CONTAIN = "-STA"; + URL_ARTE_MEDIATHEK = "http://www.arte.tv/guide/fr/plus7/toutes-les-videos?date=j-"; + TIME_1 = "

  • Diffusion :
    • "; + TIME_2 = "à"; + } } From 859739faf80972bd16294023880b8dd3e69a2a88 Mon Sep 17 00:00:00 2001 From: "W.Xaver" Date: Sun, 18 Dec 2016 13:28:21 +0100 Subject: [PATCH 7/8] ARTE.de, ARTE.FR --- .../crawler/sender/MediathekArte_de.java | 207 ------------------ 1 file changed, 207 deletions(-) diff --git a/src/main/java/mServer/crawler/sender/MediathekArte_de.java b/src/main/java/mServer/crawler/sender/MediathekArte_de.java index b848f5d09..fc4bf93dd 100644 --- a/src/main/java/mServer/crawler/sender/MediathekArte_de.java +++ b/src/main/java/mServer/crawler/sender/MediathekArte_de.java @@ -358,213 +358,6 @@ private void getFilm2(String urlWeb, String filmWebsite, String thema, String ti } } - private void addTheman(String startUrl) { - // Datum, Zeit: "BAD":"04/08/2013","BAT":"13:20" - final String MUSTER_START = "{\"programId\":"; - final String MUSTER_URL_JSON = "\"videoStreamUrl\":\""; - final String MUSTER_DATUM = "\"BAD\":\""; - final String MUSTER_ZEIT = "\"BAT\":\""; - final String MUSTER_TITEL = "\"TIT\":\""; - final String MUSTER_THEMA = "\"GEN\":\""; - String[] arr; - getUrlIo.getUri_Utf(sendername, startUrl, seite1, ""); - int posStart = 0, posStop; - int pos1; - int pos2; - int pos; - String urlJson; - String datum; - String zeit; - String titel, thema; - while ((posStart = seite1.indexOf(MUSTER_START, posStart)) != -1) { - posStart += MUSTER_START.length(); - posStop = seite1.indexOf(MUSTER_START, posStart); - urlJson = ""; - datum = ""; - zeit = ""; - titel = ""; - thema = ""; - if ((pos1 = seite1.indexOf(MUSTER_URL_JSON, posStart)) != -1) { - pos1 += MUSTER_URL_JSON.length(); - if (posStop == -1 || pos1 < posStop) { - if ((pos2 = seite1.indexOf("\"", pos1)) != -1) { - urlJson = seite1.substring(pos1, pos2); - } - } - } - pos = posStart; - while ((pos = seite1.indexOf(MUSTER_DATUM, pos)) != -1) { - pos += MUSTER_DATUM.length(); - if (posStop != -1 && pos > posStop) { - break; - } - if (posStop == -1 || pos < posStop) { - if ((pos2 = seite1.indexOf("\"", pos)) != -1) { - datum = seite1.substring(pos, pos2); - } - } - } - pos = posStart; - while ((pos = seite1.indexOf(MUSTER_ZEIT, pos)) != -1) { - pos += MUSTER_ZEIT.length(); - if (posStop != -1 && pos > posStop) { - break; - } - if (posStop == -1 || pos < posStop) { - if ((pos2 = seite1.indexOf("\"", pos)) != -1) { - zeit = seite1.substring(pos, pos2); - } - } - } - if ((pos1 = seite1.indexOf(MUSTER_TITEL, posStart)) != -1) { - pos1 += MUSTER_TITEL.length(); - if (posStop == -1 || pos1 < posStop) { - if ((pos2 = seite1.indexOf("\",", pos1)) != -1) { - titel = seite1.substring(pos1, pos2); - titel = titel.replace("\\", ""); - } - } - } - if ((pos1 = seite1.indexOf(MUSTER_THEMA, posStart)) != -1) { - pos1 += MUSTER_THEMA.length(); - if (posStop == -1 || pos1 < posStop) { - if ((pos2 = seite1.indexOf("\"", pos1)) != -1) { - thema = seite1.substring(pos1, pos2); - } - } - } - if (!urlJson.isEmpty()) { - arr = new String[]{urlJson, datum, zeit, titel, thema}; - filmeLaden(seite2, arr); - } else { -// Log.fehlerMeldung(-956230147, Log.FEHLER_ART_MREADER, "MediathekArte_de.addThemen", "Keine URL: " + startUrl + "**" + count); - } - } - } - - void filmeLaden(MSStringBuilder seite, String[] arr) { - // url_hd url, url_klein - //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"SQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_SQ_2_VOF_00604879_MP4-2200_AMM-HBBTV_EXTRAIT.mp4"}, - //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"EQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_EQ_2_VOF_00604878_MP4-1500_AMM-HBBTV_EXTRAIT.mp4"}, - //{"version":"VOF","versionProg":"1","VFO":"HBBTV","VQU":"HQ","VMT":"mp4","VUR":"http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_HQ_2_VOF_00604876_MP4-800_AMM-HBBTV_EXTRAIT.mp4"}, - - String datum = "", zeit = ""; - String urlHd = "", urlKlein = "", url = ""; - String beschreibung = ""; - String filmWebsite = ""; - String dauerStr = ""; - String titel = "", thema = "", subTitle = ""; - long dauer = 0; - final String MUSTER_BESCHREIBUNG = "\"VDE\":\""; - final String MUSTER_FILM_WEBSITE = "\"VUP\":\""; - final String MUSTER_URL_HD = "\"HBBTV\",\"VQU\":\"SQ\",\"VMT\":\"mp4\",\"VUR\":\""; - final String MUSTER_URL = "HBBTV\",\"VQU\":\"EQ\",\"VMT\":\"mp4\",\"VUR\":\""; - final String MUSTER_URL_KLEIN = "HBBTV\",\"VQU\":\"HQ\",\"VMT\":\"mp4\",\"VUR\":\""; - final String MUSTER_DAUER = "\"videoDurationSeconds\":"; - int pos1, pos2; - if (Config.getStop()) { - return; - } - meldung(arr[0]); - seite = getUrlIo.getUri_Utf(sendername, arr[0], seite, ""); - if ((pos1 = seite.indexOf(MUSTER_BESCHREIBUNG)) != -1) { - pos1 += MUSTER_BESCHREIBUNG.length(); - if ((pos2 = seite.indexOf("\",", pos1)) != -1) { - beschreibung = seite.substring(pos1, pos2); - if (!beschreibung.isEmpty() && beschreibung.endsWith("\"")) { - beschreibung = beschreibung.substring(0, beschreibung.length() - 2); - } - } - } - if ((pos1 = seite.indexOf(MUSTER_FILM_WEBSITE)) != -1) { - pos1 += MUSTER_FILM_WEBSITE.length(); - if ((pos2 = seite.indexOf("\"", pos1)) != -1) { - filmWebsite = seite.substring(pos1, pos2); - } - } - if ((pos1 = seite.indexOf(MUSTER_URL_HD)) != -1) { - pos1 += MUSTER_URL_HD.length(); - if ((pos2 = seite.indexOf("\"", pos1)) != -1) { - urlHd = seite.substring(pos1, pos2); - } - } - if ((pos1 = seite.indexOf(MUSTER_URL_KLEIN)) != -1) { - pos1 += MUSTER_URL_KLEIN.length(); - if ((pos2 = seite.indexOf("\"", pos1)) != -1) { - urlKlein = seite.substring(pos1, pos2); - } - } - if ((pos1 = seite.indexOf(MUSTER_URL)) != -1) { - pos1 += MUSTER_URL.length(); - if ((pos2 = seite.indexOf("\"", pos1)) != -1) { - url = seite.substring(pos1, pos2); - } - } - if ((pos1 = seite.indexOf(MUSTER_DAUER)) != -1) { - pos1 += MUSTER_DAUER.length(); - if ((pos2 = seite.indexOf(",", pos1)) != -1) { - dauerStr = seite.substring(pos1, pos2); - if (!dauerStr.isEmpty()) { - try { - dauer = Long.parseLong(dauerStr); - } catch (Exception ex) { - dauer = 0; - } - - } - } - } - // Datum ändern - // arr = new String[]{urlJson, datum, zeit, titel, thema}; - datum = convertDatum(arr[1]); - zeit = convertZeit(arr[2]); - titel = arr[3]; - subTitle = seite.extract("\"VSU\":\"", "\","); - subTitle = subTitle.replace("\\\"", "\""); - if (!subTitle.isEmpty() && !titel.equals(subTitle)) { - titel = titel + " - " + subTitle; - } - thema = arr[4]; - if (!url.isEmpty()) { - // public DatenFilm(String ssender, String tthema, String filmWebsite, String ttitel, String uurl, String uurlRtmp, - // String datum, String zeit, long dauerSekunden, String description, String thumbnailUrl, String imageUrl, String[] keywords) { - if (!url.endsWith("EXTRAIT.mp4")) { - // http://artestras.vo.llnwxd.net/o35/nogeo/HBBTV/042975-013-B_EXT_SQ_1_VA_00604871_MP4-2200_AMM-HBBTV_EXTRAIT.mp4 - // sind nur Trailer - DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, url, "" /*urlRtmp*/, - datum, zeit, dauer, beschreibung); - if (!urlKlein.isEmpty()) { - CrawlerTool.addUrlKlein(film, urlKlein, ""); - } - if (!urlHd.isEmpty()) { - CrawlerTool.addUrlHd(film, urlHd, ""); - } - addFilm(film); - } - } else if (!urlKlein.isEmpty()) { - DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, urlKlein, "" /*urlRtmp*/, - datum, zeit, dauer, beschreibung); - if (!urlHd.isEmpty()) { - CrawlerTool.addUrlHd(film, urlHd, ""); - } - addFilm(film); - } else if (!urlHd.isEmpty()) { - DatenFilm film = new DatenFilm(sendername, thema, filmWebsite, titel, urlHd, "" /*urlRtmp*/, - datum, zeit, dauer, beschreibung); - addFilm(film); - } else { - Log.errorLog(963025874, "Keine URL: " + arr[0]); - } - } } - String convertDatum(String datum) { - // "BAD":"04/08/2013","BAT":"13:20" - return datum.replace("/", "."); - } - - String convertZeit(String zeit) { - // "BAD":"04/08/2013","BAT":"13:20" - return zeit + ":00"; - } } From 10eb5278741e393af9b333d04825922e463d3c04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Finkh=C3=A4user?= Date: Sun, 18 Dec 2016 23:49:34 +0100 Subject: [PATCH 8/8] Version angehoben auf 2.3.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c94ad17ed..c86634653 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ import org.apache.tools.ant.filters.ReplaceTokens sourceCompatibility = 1.8 targetCompatibility = 1.8 -version = '2.2.0' +version = '2.3.0' def jarName = 'MServer.jar' def mainClass = 'mServer.Main'