Deleted install script and optimized Dockerfile

Z-Jais · Sep 22, 2023 · e07f064 · e07f064
1 parent e8822a8
commit e07f064
Show file tree

Hide file tree

Showing 13 changed files with 164 additions and 114 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -10,11 +10,59 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: 'adopt'
 
       - name: Tests
-        run: mvn test
+        run: mvn test
+
+  docker-amd64:
+    runs-on: ubuntu-latest
+
+    needs:
+      - tests
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build
+        id: docker
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/amd64
+          push: false
+          context: .
+          tags: scraper:amd64
+
+  docker-arm64:
+    runs-on: ubuntu-latest
+
+    needs:
+      - tests
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build
+        id: docker
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/arm64
+          push: false
+          context: .
+          tags: scraper:arm64
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -12,10 +12,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: 'adopt'
 
       - name: Tests
@@ -30,10 +30,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: 'adopt'
 
       - name: Build

diff --git a/Dockerfile b/Dockerfile
@@ -1,10 +1,32 @@
-FROM maven:3.8.6-jdk-11-slim
+FROM maven:3.9.4-amazoncorretto-17 AS build
 WORKDIR /app
 COPY . /app
-RUN apt-get update && apt-get install -y libgtk-3-0 libasound2 libx11-6 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libxrender1 libxtst6 libfreetype6 libfontconfig1 libpangocairo-1.0-0 libpangocairo-1.0-0 libpango-1.0-0 libatk1.0-0 libcairo-gobject2 libcairo2 libgdk-pixbuf-2.0-0 libglib2.0-0 libdbus-glib-1-2 libdbus-1-3 libxcb-shm0 libx11-xcb1 libxcb1 libxcursor1 libxi6 libnss3 libnspr4 libdrm2 libgbm1
 RUN mvn clean package -DskipTests
-RUN ls -lash
-RUN mv target/scraper-1.0-SNAPSHOT-jar-with-dependencies.jar /scraper.jar
-RUN rm -rf /app/**
-RUN mv /scraper.jar /app/scraper.jar
-CMD ["java", "-jar", "scraper.jar"]
+
+FROM mcr.microsoft.com/playwright:v1.38.0-jammy
+
+ARG version=17.0.8.8-1
+RUN set -eux \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl ca-certificates gnupg software-properties-common fontconfig java-common \
+    && curl -fL https://apt.corretto.aws/corretto.key | apt-key add - \
+    && add-apt-repository 'deb https://apt.corretto.aws stable main' \
+    && mkdir -p /usr/share/man/man1 || true \
+    && apt-get update \
+    && apt-get install -y java-17-amazon-corretto-jdk=1:$version \
+    && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+        curl gnupg software-properties-common
+
+ENV LANG C.UTF-8
+ENV JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto
+
+COPY --from=build /app/target/scraper-1.0-SNAPSHOT-jar-with-dependencies.jar /app/app.jar
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install tzdata
+ENV TZ=Europe/Paris
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+RUN dpkg-reconfigure --frontend noninteractive tzdata
+
+WORKDIR /app
+CMD ["java", "-jar", "app.jar"]
diff --git a/create_images.sh b/create_images.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+docker buildx create --use
+docker buildx build --load --platform linux/amd64 -t jais-scraper:amd64 .
+docker buildx build --load --platform linux/arm64 -t jais-scraper:arm64 .
diff --git a/install.sh b/install.sh
diff --git a/pom.xml b/pom.xml
@@ -14,7 +14,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <kotlin.code.style>official</kotlin.code.style>
-        <kotlin.compiler.jvmTarget>11</kotlin.compiler.jvmTarget>
+        <kotlin.compiler.jvmTarget>17</kotlin.compiler.jvmTarget>
         <kotlin.version>1.9.10</kotlin.version>
         <mainClass>fr.jais.scraper.ScraperKt</mainClass>
         <junit-jupiter.version>5.10.0</junit-jupiter.version>
@@ -157,7 +157,7 @@
         <dependency>
             <groupId>org.quartz-scheduler</groupId>
             <artifactId>quartz</artifactId>
-            <version>2.3.2</version>
+            <version>2.5.0-rc1</version>
         </dependency>
     </dependencies>
 </project>
diff --git a/src/main/kotlin/fr/jais/scraper/Scraper.kt b/src/main/kotlin/fr/jais/scraper/Scraper.kt
@@ -28,17 +28,22 @@ class Scraper {
     fun startThreadCommand() {
         ThreadManager.start("Command") {
             while (true) {
-                val command = readlnOrNull() ?: continue
-                val args = command.split(" ")
+                try {
+                    val command = readlnOrNull() ?: continue
+                    val args = command.split(" ")
 
-                when (args[0]) {
-                    "ayane" -> {
-                        AyaneJob().execute(null)
-                    }
 
-                    else -> {
-                        Logger.info("Unknown command")
+                    when (args[0]) {
+                        "ayane" -> {
+                            AyaneJob().execute(null)
+                        }
+
+                        else -> {
+                            Logger.info("Unknown command")
+                        }
                     }
+                } catch (_: Exception) {
+                    Thread.sleep(1000)
                 }
             }
         }

diff --git a/src/main/kotlin/fr/jais/scraper/converters/AnimationDigitalNetworkConverter.kt b/src/main/kotlin/fr/jais/scraper/converters/AnimationDigitalNetworkConverter.kt
@@ -105,6 +105,11 @@ class AnimationDigitalNetworkConverter(private val platform: AnimationDigitalNet
 
         // ----- NUMBER -----
         Logger.info("Get number...")
+
+        if ("Bande-annonce" == jsonObject["shortNumber"]?.asString) {
+            throw EpisodeNotAvailableException("Trailer detected")
+        }
+
         val number = jsonObject["shortNumber"]?.asString()?.toIntOrNull() ?: run {
             Logger.warning("No number found, using -1...")
             -1

diff --git a/src/main/kotlin/fr/jais/scraper/platforms/AnimationDigitalNetworkPlatform.kt b/src/main/kotlin/fr/jais/scraper/platforms/AnimationDigitalNetworkPlatform.kt
@@ -47,10 +47,7 @@ class AnimationDigitalNetworkPlatform(scraper: Scraper) :
                 try {
                     converter.convertEpisode(country, calendar, it, cachedEpisodes)
                 } catch (e: Exception) {
-                    if (e !is EpisodeException) {
-                        Logger.log(Level.SEVERE, "Error while converting episode", e)
-                    }
-
+                    Logger.log(Level.SEVERE, "Error while converting episode", e)
                     null
                 }
             } ?: emptyList()

diff --git a/src/main/kotlin/fr/jais/scraper/platforms/CrunchyrollPlatform.kt b/src/main/kotlin/fr/jais/scraper/platforms/CrunchyrollPlatform.kt
@@ -58,48 +58,42 @@ class CrunchyrollPlatform(scraper: Scraper) : IPlatform(
     }
 
     private fun checkSimulcasts(iCountry: ICountry) {
-        Logger.info("Checking simulcasts for ${iCountry.name}...")
-        // Clear simulcast for this country if exists
-        simulcasts.remove(iCountry)
         val countryTag = converter.getCountryTag(iCountry)
         Logger.info("Loading simulcasts for ${iCountry.name}...")
 
-        val content = try {
-            Browser(
-                Browser.BrowserType.FIREFOX,
-                "https://www.crunchyroll.com/$countryTag/simulcasts"
-            ).launchAndWaitForSelector("#content > div > div.app-body-wrapper > div > div > div.erc-browse-collection > div > div:nth-child(1) > div > div > h4 > a")
-        } catch (_: Exception) {
-            Logger.warning("No simulcasts found for ${iCountry.name}! (Empty page or error)")
-
-            Browser(
-                Browser.BrowserType.FIREFOX,
-                "https://www.crunchyroll.com/$countryTag/simulcasts"
-            ).launchAndWaitForSelector("#content > div > div.app-body-wrapper > div > div > div.info-box--wxFEW.erc-simulcast-no-results")
-        }
+        val selector = "#content > div > div.app-body-wrapper > div > div > div.erc-browse-collection > div > div:nth-child(1) > div > div > h4 > a"
+        val simulcastSelector = ".erc-browse-cards-collection > .browse-card > div > div > h4 > a"
 
-        val currentSimulcastAnimes = content.select(".erc-browse-cards-collection > .browse-card > div > div > h4 > a")
-            .map { it.text().lowercase() }.toSet()
+        val contentCurrentSimulcast = Browser(
+            Browser.BrowserType.FIREFOX,
+            "https://www.crunchyroll.com/$countryTag/simulcasts"
+        ).launchAndWaitForSelector(selector)
 
         val simulcastName =
-            content.select("#content > div > div.app-body-wrapper > div > div > div.header > div > div > span.call-to-action--PEidl.call-to-action--is-m--RVdkI.select-trigger__title-cta--C5-uH.select-trigger__title-cta--is-displayed-on-mobile--6oNk1")
-                .text()
+            contentCurrentSimulcast.select("#content > div > div.app-body-wrapper > div > div > div.header > div > div > span.call-to-action--PEidl.call-to-action--is-m--RVdkI.select-trigger__title-cta--C5-uH.select-trigger__title-cta--is-displayed-on-mobile--6oNk1").text()
         val simulcastCode = getSimulcastCode(simulcastName)
-        Logger.info("Simulcast code for ${iCountry.name}: $simulcastCode")
+        Logger.info("Current simulcast code for ${iCountry.name}: $simulcastCode")
+
+        val currentSimulcastAnimes = contentCurrentSimulcast.select(simulcastSelector)
+            .map { it.text().lowercase() }.toSet()
+        Logger.config("Found ${currentSimulcastAnimes.size} animes for the current simulcast")
+
         val previousSimulcastCode = getPreviousSimulcastCode(simulcastCode)
         Logger.info("Previous simulcast code for ${iCountry.name}: $previousSimulcastCode")
 
-        val content2 = Browser(
+        val contentPreviousSimulcast = Browser(
             Browser.BrowserType.FIREFOX,
             "https://www.crunchyroll.com/$countryTag/simulcasts/seasons/$previousSimulcastCode"
-        ).launchAndWaitForSelector("#content > div > div.app-body-wrapper > div > div > div.erc-browse-collection > div > div:nth-child(1) > div > div > h4 > a")
+        ).launchAndWaitForSelector(selector)
+
         val previousSimulcastAnimes =
-            content2.select(".erc-browse-cards-collection > .browse-card > div > div > h4 > a")
-                .map { it.text().lowercase() }.toSet()
+            contentPreviousSimulcast.select(simulcastSelector).map { it.text().lowercase() }.toSet()
+        Logger.config("Found ${previousSimulcastAnimes.size} animes for the previous simulcast")
 
-        simulcasts[iCountry] = (currentSimulcastAnimes + previousSimulcastAnimes).toSet()
-        Logger.info("Found ${simulcasts[iCountry]?.size} simulcasts for ${iCountry.name}!")
-        Logger.config("Simulcasts: ${simulcasts[iCountry]?.joinToString(", ")}")
+        val combinedSimulcastAnimes = (currentSimulcastAnimes + previousSimulcastAnimes).toSet()
+        simulcasts[iCountry] = combinedSimulcastAnimes
+        Logger.info("Found ${combinedSimulcastAnimes.size} simulcasts for ${iCountry.name}!")
+        Logger.config("Simulcasts: ${combinedSimulcastAnimes.joinToString(", ")}")
     }
 
     fun xmlToJson(content: String) =
@@ -137,12 +131,11 @@ class CrunchyrollPlatform(scraper: Scraper) : IPlatform(
             countries.forEach {
                 try {
                     checkSimulcasts(it)
+                    lastSimulcastCheck = System.currentTimeMillis()
                 } catch (e: Exception) {
                     Logger.log(Level.SEVERE, "Error while checking simulcasts", e)
                 }
             }
-
-            lastSimulcastCheck = System.currentTimeMillis()
         }
 
         return countries.flatMap { country ->