Deleted install script and optimized Dockerfile

.github/workflows/pull_request.yml

@@ -4,17 +4,35 @@ on:
   pull_request:
 
 jobs:
-  tests:
+  test:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: 'adopt'
 
       - name: Tests
-        run: mvn test
+        run: mvn test
+
+  build:
+    runs-on: ubuntu-latest
+
+    needs:
+      - test
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: 'adopt'
+
+      - name: Build
+        run: mvn -B package -DskipTests

.github/workflows/push.yml

@@ -6,16 +6,16 @@ on:
       - master
 
 jobs:
-  tests:
+  test:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: 'adopt'
 
       - name: Tests
@@ -25,16 +25,76 @@ jobs:
     runs-on: ubuntu-latest
 
     needs:
-      - tests
+      - test
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: 'adopt'
 
       - name: Build
-        run: mvn -B package -DskipTests
+        run: mvn -B package -DskipTests
+
+  docker:
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: registry.ziedelth.fr:5000
+          username: ziedelth
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Build and push snapshot
+        uses: docker/build-push-action@v5
+        if: github.ref == 'refs/heads/master'
+        with:
+          context: .
+          push: true
+          tags: registry.ziedelth.fr:5000/jais-scraper:snapshot
+
+      - name: Build and push production
+        uses: docker/build-push-action@v5
+        if: github.ref == 'refs/heads/stable'
+        with:
+          context: .
+          push: true
+          tags: registry.ziedelth.fr:5000/jais-scraper:latest
+
+  release:
+    if: github.ref == 'refs/heads/stable'
+    name: Release
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    steps:
+      - name: Install SSH KEY
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          known_hosts: 'just-a-placeholder-so-we-dont-get-errors'
+
+      - name: Adding Known Hosts
+        run: ssh-keyscan -p 2222 -H ${{ secrets.SSH_HOST_IP }} >> ~/.ssh/known_hosts
+
+      - name: SSH Commands
+        run: |
+          ssh -C -p 2222 ${{ secrets.SSH_DESTINATION }} "docker pull localhost:5000/jais-scraper:latest && cd ${{ secrets.SSH_FOLDER }} && docker compose down jais-scraper && docker compose up jais-scraper -d"

Dockerfile

@@ -1,10 +1,32 @@
-FROM maven:3.8.6-jdk-11-slim
+FROM maven:3.9.4-amazoncorretto-17 AS build
 WORKDIR /app
 COPY . /app
-RUN apt-get update && apt-get install -y libgtk-3-0 libasound2 libx11-6 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libxrender1 libxtst6 libfreetype6 libfontconfig1 libpangocairo-1.0-0 libpangocairo-1.0-0 libpango-1.0-0 libatk1.0-0 libcairo-gobject2 libcairo2 libgdk-pixbuf-2.0-0 libglib2.0-0 libdbus-glib-1-2 libdbus-1-3 libxcb-shm0 libx11-xcb1 libxcb1 libxcursor1 libxi6 libnss3 libnspr4 libdrm2 libgbm1
 RUN mvn clean package -DskipTests
-RUN ls -lash
-RUN mv target/scraper-1.0-SNAPSHOT-jar-with-dependencies.jar /scraper.jar
-RUN rm -rf /app/**
-RUN mv /scraper.jar /app/scraper.jar
+
+FROM mcr.microsoft.com/playwright:v1.38.0-jammy
+
+ARG version=17.0.8.8-1
+RUN set -eux \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl ca-certificates gnupg software-properties-common fontconfig java-common \
+    && curl -fL https://apt.corretto.aws/corretto.key | apt-key add - \
+    && add-apt-repository 'deb https://apt.corretto.aws stable main' \
+    && mkdir -p /usr/share/man/man1 || true \
+    && apt-get update \
+    && apt-get install -y java-17-amazon-corretto-jdk=1:$version \
+    && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+        curl gnupg software-properties-common
+
+ENV LANG C.UTF-8
+ENV JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto
+
+COPY --from=build /app/target/scraper-1.0-SNAPSHOT-jar-with-dependencies.jar /app/scraper.jar
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install tzdata
+ENV TZ=Europe/Paris
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+RUN dpkg-reconfigure --frontend noninteractive tzdata
+
+WORKDIR /app
 CMD ["java", "-jar", "scraper.jar"]

create_images.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+docker buildx create --use
+docker buildx build --load --platform linux/amd64 -t jais-scraper:amd64 .
+docker buildx build --load --platform linux/arm64 -t jais-scraper:arm64 .

pom.xml

@@ -14,7 +14,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <kotlin.code.style>official</kotlin.code.style>
-        <kotlin.compiler.jvmTarget>11</kotlin.compiler.jvmTarget>
+        <kotlin.compiler.jvmTarget>17</kotlin.compiler.jvmTarget>
         <kotlin.version>1.9.10</kotlin.version>
         <mainClass>fr.jais.scraper.ScraperKt</mainClass>
         <junit-jupiter.version>5.10.0</junit-jupiter.version>
@@ -157,7 +157,7 @@
         <dependency>
             <groupId>org.quartz-scheduler</groupId>
             <artifactId>quartz</artifactId>
-            <version>2.3.2</version>
+            <version>2.5.0-rc1</version>
         </dependency>
     </dependencies>
 </project>

src/main/kotlin/fr/jais/scraper/Scraper.kt

@@ -8,16 +8,17 @@ import fr.jais.scraper.jobs.JobManager
 import fr.jais.scraper.platforms.AnimationDigitalNetworkPlatform
 import fr.jais.scraper.platforms.CrunchyrollPlatform
 import fr.jais.scraper.platforms.IPlatform
-import fr.jais.scraper.platforms.NetflixPlatform
 import fr.jais.scraper.utils.Const
 import fr.jais.scraper.utils.Logger
 import fr.jais.scraper.utils.ThreadManager
+import java.io.File
+import java.util.*
 
 class Scraper {
     val platforms = listOf(
         AnimationDigitalNetworkPlatform(this),
         CrunchyrollPlatform(this),
-        NetflixPlatform(this)
+        // NetflixPlatform(this)
     )
     val countries = platforms.flatMap { it.countries }.distinct().mapNotNull { it.getConstructor().newInstance() }
     private val jobManager = JobManager()
@@ -27,18 +28,17 @@ class Scraper {
 
     fun startThreadCommand() {
         ThreadManager.start("Command") {
+            val scanner = Scanner(System.`in`)
+
             while (true) {
-                val command = readlnOrNull() ?: continue
-                val args = command.split(" ")
+                try {
+                    val line = scanner.nextLine()
 
-                when (args[0]) {
-                    "ayane" -> {
+                    if (line == "ayane") {
                         AyaneJob().execute(null)
                     }
-
-                    else -> {
-                        Logger.info("Unknown command")
-                    }
+                } catch (_: Exception) {
+                    Thread.sleep(1000)
                 }
             }
         }
@@ -58,6 +58,7 @@ class Scraper {
 
 fun main() {
     Logger.info("Initializing...")
+    File("data").mkdirs()
     Const.gson
     Logger.info("Initialization done!")
 

src/main/kotlin/fr/jais/scraper/converters/AnimationDigitalNetworkConverter.kt

@@ -13,7 +13,7 @@ import java.io.File
 import java.util.*
 
 class AnimationDigitalNetworkConverter(private val platform: AnimationDigitalNetworkPlatform) {
-    private val file = File("animation_digital_network.json")
+    private val file = File("data/animation_digital_network.json")
 
     /// Convert anime from AnimationDigitalNetworkPlatform jsonObject to entity Anime
     private fun convertAnime(checkedCountry: ICountry, calendar: Calendar, jsonObject: JsonObject): Anime {
@@ -105,6 +105,11 @@ class AnimationDigitalNetworkConverter(private val platform: AnimationDigitalNet
 
         // ----- NUMBER -----
         Logger.info("Get number...")
+
+        if ("Bande-annonce" == jsonObject["shortNumber"]?.asString) {
+            throw EpisodeNotAvailableException("Trailer detected")
+        }
+
         val number = jsonObject["shortNumber"]?.asString()?.toIntOrNull() ?: run {
             Logger.warning("No number found, using -1...")
             -1

src/main/kotlin/fr/jais/scraper/converters/CrunchyrollConverter.kt

@@ -32,7 +32,7 @@ class CrunchyrollConverter(private val platform: CrunchyrollPlatform) {
     private val useApi = false
     val cache = mutableListOf<CrunchyrollAnime>()
     private val sessionId: String = crunchyrollSession()
-    private val file = File("crunchyroll.json")
+    private val file = File("data/crunchyroll.json")
 
     fun getCountryTag(checkedCountry: ICountry): String {
         val country = when (checkedCountry) {

src/main/kotlin/fr/jais/scraper/jobs/AyaneJob.kt

@@ -23,7 +23,7 @@ class AyaneJob : Job {
 
     override fun execute(p0: JobExecutionContext?) {
         Logger.info("Starting AyaneJob...")
-        val folder = File("ayane")
+        val folder = File("data/ayane")
         if (!folder.exists()) folder.mkdirs()
 
         val font = File(folder, "Rubik.ttf")

src/main/kotlin/fr/jais/scraper/platforms/AnimationDigitalNetworkPlatform.kt

@@ -47,10 +47,7 @@ class AnimationDigitalNetworkPlatform(scraper: Scraper) :
                 try {
                     converter.convertEpisode(country, calendar, it, cachedEpisodes)
                 } catch (e: Exception) {
-                    if (e !is EpisodeException) {
-                        Logger.log(Level.SEVERE, "Error while converting episode", e)
-                    }
-
+                    Logger.log(Level.SEVERE, "Error while converting episode", e)
                     null
                 }
             } ?: emptyList()