Merge branch 'release/2.4.0'

ARTE: Filme werden nicht immer zeitgleich mit Mediathek gefunden Einfügen von Repo Deployment für MServer closed #41 closed #43
mediathekview · Dec 23, 2016 · c24e672 · c24e672
2 parents 388e8bf + f16fa0c
commit c24e672
Show file tree

Hide file tree

Showing 7 changed files with 86 additions and 22 deletions.
diff --git a/.gitignore b/.gitignore
@@ -21,4 +21,5 @@ start*.sh
 build/
 
 version.properties
+RepoZugang.properties
 
diff --git a/.travis.yml b/.travis.yml
@@ -1,33 +1,40 @@
 language: java
 jdk:
 - oraclejdk8
-install:
-- cd ..
-- git clone https://github.com/mediathekview/MSearch.git MSearch
-- cd -
 addons:
   sonarqube: true
 env:
   global:
-  - secure: pHmSb+3bIvSj1iA5j4MIgRv1ogRH1w7NKlfPxAPDXYq+AQZOyyAhWm/nTMq0mtX9x5RsrM8c7YUN7ds+pTpE5JpcA+rlOSibj+ziDeLhpmDzcBt+YUltTOxP7EVrAAmHBTu7DGjPmTLvfzLfpyGNkv6yuIXNo/oIs6BRMwQ2xHa1QdkQ/gkS6nxgh7UhqtDmNix/idHjX/a5PEX8Qse3ZvKBvt1kgrKcVN1ysOtcHKvhQ5ddMAj+7PM+aLuAPTlysfqMYN1gW+cpWOJrHriV7vGCU6lz34s2yM19wFmRFOo7yht3FhAwH8Rt6aTtwpC3HN5DzwqhcicHQkoyv59KMiBbUBN9w5SGqit+/HtFOQ9vgyV5qo5Il/UkbVGUXXl9WcY+zPV2XTlO3LH5o36Bn+da/HRdmlvveMZT8tJ5137YSQhrQRZdOVEDBWuAZmUiTh94c9u3owU85TFzS97fqWi5bvpbyIU749oGhWGv/zxmOEJmtinnQpUAJqS9fFrXrFFtjdLF6vNYWc1f1y7Xg57pLZYK0NcClWc77QwCX5WHy6VPdj3elIJHyv+3DKQw9VBrfV+bRE4b6uvLuDkllBztYECstYR1uJTOumVnhufc12GrD51m+IRqtzEp4+OlNunKsoInNGsJYyaz3t8VcdAwEN4lTE7Se7ijehIpHRA=
+    - secure: pHmSb+3bIvSj1iA5j4MIgRv1ogRH1w7NKlfPxAPDXYq+AQZOyyAhWm/nTMq0mtX9x5RsrM8c7YUN7ds+pTpE5JpcA+rlOSibj+ziDeLhpmDzcBt+YUltTOxP7EVrAAmHBTu7DGjPmTLvfzLfpyGNkv6yuIXNo/oIs6BRMwQ2xHa1QdkQ/gkS6nxgh7UhqtDmNix/idHjX/a5PEX8Qse3ZvKBvt1kgrKcVN1ysOtcHKvhQ5ddMAj+7PM+aLuAPTlysfqMYN1gW+cpWOJrHriV7vGCU6lz34s2yM19wFmRFOo7yht3FhAwH8Rt6aTtwpC3HN5DzwqhcicHQkoyv59KMiBbUBN9w5SGqit+/HtFOQ9vgyV5qo5Il/UkbVGUXXl9WcY+zPV2XTlO3LH5o36Bn+da/HRdmlvveMZT8tJ5137YSQhrQRZdOVEDBWuAZmUiTh94c9u3owU85TFzS97fqWi5bvpbyIU749oGhWGv/zxmOEJmtinnQpUAJqS9fFrXrFFtjdLF6vNYWc1f1y7Xg57pLZYK0NcClWc77QwCX5WHy6VPdj3elIJHyv+3DKQw9VBrfV+bRE4b6uvLuDkllBztYECstYR1uJTOumVnhufc12GrD51m+IRqtzEp4+OlNunKsoInNGsJYyaz3t8VcdAwEN4lTE7Se7ijehIpHRA=
 script:
-- "./gradlew check build"
-- sonar-scanner -Dsonar.login=$SONAR_TOKEN
+  - "./gradlew check build"
+  - if [ "$TRAVIS_BRANCH" == "develop" ]; then
+      openssl aes-256-cbc -K $encrypted_3a9121b37456_key -iv $encrypted_3a9121b37456_iv 
+        -in scripte/deploy/RepoZugang.properties.enc -out scripte/deploy/RepoZugang.properties -d;
+      ./gradlew releaseSnapshot;
+    fi
+after_success:
+  - sonar-scanner -Dsonar.login=$SONAR_TOKEN
 before_cache:
-- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
+  - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
 cache:
   directories:
   - "$HOME/.gradle/caches/"
   - "$HOME/.gradle/wrapper/"
   - "$HOME/.sonar/cache/"
+
+before_deploy:
+  - openssl aes-256-cbc -K $encrypted_3a9121b37456_key -iv $encrypted_3a9121b37456_iv 
+    -in scripte/deploy/RepoZugang.properties.enc -out scripte/deploy/RepoZugang.properties -d
+  - ./gradlew release
 deploy:
   provider: releases
   api_key:
     secure: yZv3QsIr8M1rFEuRpgyYZwrAgeHtd0LVkpG9DIsLLNhboApz9LCMcU0dbYxRYi8A5V3MCD+X/rtWsWHfapIjRcoVAyi8f7YWWvz+dnjwOxKi+gHabonnUvIyFAEr8WE+Br5MYOr0GTYiEhiUMrSLg20wBWlqH5LD1Mp/dYd2e80EWN8jcZJRK/rO1CDa78esgaaBMp1Pb/6+E9jS+fuEpLuKocqacA0eHtRrPtPFbTJdVH9amQxfcF3UfEV7/0D8tGlL2cH3J45zYQ0sg/qnWqv6Vtt6W0x6H3pFrEfrkAww8jvV5sKGxyz1/bqLOQJkF8mCotjGox9pkL97/cyf+s+L3Mcux98fK9u7NOzizfFMYz5GgY1K/q0l7hzfzh6JSVPcZ/hHdH0VRqcUVgqXB//wPgs2JNGLj/nGLyxY7T53U5hJOhsqFyGGcbnpZ+thEWlB5wrbPi5PwKCtWvf+Y6H/AePxCRQEie25lwi+M2MCIPjSUB65mXoQnXy3oQahiYWPU0HIpD0Om9fp6UK5x8LXMAx3DeH6wbI7qB2gylygOxuUm0AmojieOeLaQFVK7cvcy9c2W16xyfDUDczkEnceosCe8b2nK/yifKeOfFU/eUiEdzl9ZZvOMf9y4YImQl0ZlQlvYknPGLK/myGLPlJrJhP92XtSiYESsrWOEFw=
   file_glob: true
   file:
-      - build/libs/*.jar
-      - build/distributions/*
+    - build/libs/*.jar
+    - build/distributions/*
   skip_cleanup: true
   on:
     tags: true
diff --git a/build.gradle b/build.gradle
@@ -1,13 +1,15 @@
 apply plugin: 'java'
 apply plugin: 'distribution'
+apply plugin: 'maven'
 apply from: "${project.rootDir}/gradle/eclipse.gradle"
 import java.nio.file.Files
 import java.nio.file.Paths
 import org.apache.tools.ant.filters.ReplaceTokens
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
-version = '2.3.0'
+group = 'de.mediathekview'
+version = '2.4.0'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'
@@ -33,7 +35,7 @@ repositories {
 }
 
 dependencies {
-    compile project(':MSearch')
+    compile 'de.mediathekview:MSearch:2.1.1'
     compile 'commons-net:commons-net:3.5'
     compile 'org.apache.commons:commons-compress:1.12'
     compile 'org.apache.commons:commons-lang3:3.5'
@@ -46,6 +48,10 @@ ext {
     if (!propsFile.exists()) {
         Files.createFile(propsFile.toPath())
     }
+    repoZugangFile = file('scripte/deploy/RepoZugang.properties').absoluteFile
+    if (!repoZugangFile.exists()) {
+        Files.createFile(repoZugangFile.toPath())
+    }
 }
 
 def loadVersionProperties() {
@@ -54,6 +60,27 @@ def loadVersionProperties() {
     return props
 }
 
+def loadRepoZugangProperties() {
+    Properties props = new Properties()
+    props.load(repoZugangFile.newDataInputStream())
+    return props
+}
+
+uploadArchives {
+    repositories.mavenDeployer {
+        Properties props = loadRepoZugangProperties()
+        def nexusUser = props.getProperty('repoUser')
+        def nexusPw = props.getProperty('repoPw')
+        repository(url: "https://repo.mediathekview.de/repository/maven-releases/") {
+            authentication(userName:  nexusUser, password:  nexusPw)
+        }
+        snapshotRepository(url: "https://repo.mediathekview.de/repository/maven-snapshots/") {
+            authentication(userName:  nexusUser, password:  nexusPw)
+        }
+    }
+}
+build.dependsOn(install)
+
 task updateVersion << {
     Properties props = loadVersionProperties()
     def oldVersion = props.getProperty('VERSION')
@@ -143,3 +170,29 @@ task debug(type: JavaExec, dependsOn: classes) {
     }
 }
 
+/**
+ * <b>You don't have to call this. Travis will do it for you if a new releass (tag) will be build!<b/>
+ * Call this so: './gradlew build release -PnexusUser=[NEXUS_USER] -PnexusPw=[NEXUS_PASSWORD]'
+ */
+task release(dependsOn: 'uploadArchives') {
+    doLast {
+        println 'Released Version '+version
+    }
+}
+
+/**
+ * <b>You don't have to call this. Travis will do it for you if you push to develop!<b/>
+ * Call this so: './gradlew build releaseSnapshot -PnexusUser=[NEXUS_USER] -PnexusPw=[NEXUS_PASSWORD]'
+ */
+task releaseSnapshot(dependsOn: 'uploadArchives') {
+    doLast {
+        println 'Released Snapshot Version '+version
+    }
+}
+
+gradle.taskGraph.whenReady {taskGraph ->
+    if (!taskGraph.hasTask(release)) {
+        version = version+'-SNAPSHOT'
+    }
+}
+
diff --git a/scripte/deploy/RepoZugang.properties.enc b/scripte/deploy/RepoZugang.properties.enc
@@ -0,0 +1,2 @@
+��R��n^��)[`
+~E^Ȥt,x��8����������ܽ�QXJ�:

diff --git a/settings.gradle b/settings.gradle
diff --git a/src/main/java/mServer/crawler/sender/MediathekArte_de.java b/src/main/java/mServer/crawler/sender/MediathekArte_de.java
@@ -41,7 +41,8 @@ public class MediathekArte_de extends MediathekReader implements Runnable {
     String URL_ARTE = "http://www.arte.tv/papi/tvguide/epg/schedule/D/L3/";
     String URL_CONCERT = "http://concert.arte.tv/de/videos/all";
     String URL_CONCERT_NOT_CONTAIN = "-STF";
-    String URL_ARTE_MEDIATHEK = "http://www.arte.tv/guide/de/plus7/alle-videos?date=j-";
+    String URL_ARTE_MEDIATHEK_1 = "http://www.arte.tv/guide/de/plus7/videos?day=-";
+    String URL_ARTE_MEDIATHEK_2 = "&page=1&isLoading=true&sort=newest&country=DE";
     String TIME_1 = "<li>Sendetermine:</li>";
     String TIME_2 = "um";
 
@@ -94,9 +95,9 @@ private void addConcert() {
     }
 
     private void addTage() {
-        // http://www.arte.tv/guide/de/plus7/alle-videos?date=j-0
+        // http://www.arte.tv/guide/de/plus7/videos?day=-2&page=1&isLoading=true&sort=newest&country=DE
         for (int i = 0; i <= 14; ++i) {
-            String u = URL_ARTE_MEDIATHEK + i;
+            String u = URL_ARTE_MEDIATHEK_1 + i + URL_ARTE_MEDIATHEK_2;
             listeThemen.add(new String[]{u});
         }
     }
@@ -270,8 +271,9 @@ public void run() {
         }
 
         private void addThemen(String startUrl) {
+            // http://www.arte.tv/guide/de/plus7/videos?day=-2&page=1&isLoading=true&sort=newest&country=DE
             getUrl.getUri_Utf(sendername, startUrl, seite1, "");
-            seite1.extractList("url&quot;:&quot;http:\\/\\/www.arte.tv", "&quot", liste);
+            seite1.extractList("\"url\":\"http:\\/\\/www.arte.tv", "\"", liste);
             for (String s : liste) {
                 if (Config.getStop()) {
                     break;
@@ -280,7 +282,7 @@ private void addThemen(String startUrl) {
                 //Datum: url: xx-0 => heute, xx-1 => gestern, ...
                 String date = "";
                 try {
-                    String d = startUrl.substring(startUrl.lastIndexOf("-") + 1);
+                    String d = startUrl.substring(startUrl.lastIndexOf("day=-") + "day=-".length(), startUrl.indexOf("&page="));
                     int iD = Integer.parseInt(d);
                     SimpleDateFormat form = new SimpleDateFormat("dd.MM.yyyy");
                     date = form.format(new Date(new Date().getTime() - iD * (1000 * 60 * 60 * 24)));
@@ -302,6 +304,7 @@ private void getFilm1(String filmWebsite, String date) {
             String duration = seite1.extract("<span class=\"duration\">", "<");
             long dauer = 0;
             try {
+                duration = duration.replace("min", "");
                 duration = duration.replace("Min.", "");
                 duration = duration.replace("\n", "").trim();
                 dauer = Integer.parseInt(duration) * 60;

diff --git a/src/main/java/mServer/crawler/sender/MediathekArte_fr.java b/src/main/java/mServer/crawler/sender/MediathekArte_fr.java
@@ -36,7 +36,9 @@ public MediathekArte_fr(FilmeSuchen ssearch, int startPrio) {
         URL_ARTE = "http://www.arte.tv/papi/tvguide/epg/schedule/F/L3/";
         URL_CONCERT = "http://concert.arte.tv/fr/videos/all";
         URL_CONCERT_NOT_CONTAIN = "-STA";
-        URL_ARTE_MEDIATHEK = "http://www.arte.tv/guide/fr/plus7/toutes-les-videos?date=j-";
+        //http://www.arte.tv/guide/fr/plus7/videos?day=-1&page=2&isLoading=true&limit=24&sort=newest&country=DE
+        URL_ARTE_MEDIATHEK_1 = "http://www.arte.tv/guide/fr/plus7/videos?day=-";
+        URL_ARTE_MEDIATHEK_2 = "&page=1&isLoading=true&sort=newest&country=DE";
         TIME_1 = "<li>Diffusion :</li>";
         TIME_2 = "à";