diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ce3de73..bd3ef3c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4,17 +4,35 @@ on: pull_request: jobs: - tests: + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: - java-version: 11 + java-version: 17 distribution: 'adopt' - name: Tests - run: mvn test \ No newline at end of file + run: mvn test + + build: + runs-on: ubuntu-latest + + needs: + - test + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'adopt' + + - name: Build + run: mvn -B package -DskipTests \ No newline at end of file diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index fe18f12..d44c882 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -6,16 +6,16 @@ on: - master jobs: - tests: + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: - java-version: 11 + java-version: 17 distribution: 'adopt' - name: Tests @@ -25,16 +25,76 @@ jobs: runs-on: ubuntu-latest needs: - - tests + - test steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: - java-version: 11 + java-version: 17 distribution: 'adopt' - name: Build - run: mvn -B package -DskipTests \ No newline at end of file + run: mvn -B package -DskipTests + + docker: + runs-on: ubuntu-latest + + needs: + - build + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: registry.ziedelth.fr:5000 + username: ziedelth + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Build and push snapshot + uses: docker/build-push-action@v5 + if: github.ref == 'refs/heads/master' + with: + context: . + push: true + tags: registry.ziedelth.fr:5000/jais-scraper:snapshot + + - name: Build and push production + uses: docker/build-push-action@v5 + if: github.ref == 'refs/heads/stable' + with: + context: . + push: true + tags: registry.ziedelth.fr:5000/jais-scraper:latest + + release: + if: github.ref == 'refs/heads/stable' + name: Release + runs-on: ubuntu-latest + + needs: + - docker + + steps: + - name: Install SSH KEY + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.SSH_PRIVATE_KEY }} + known_hosts: 'just-a-placeholder-so-we-dont-get-errors' + + - name: Adding Known Hosts + run: ssh-keyscan -p 2222 -H ${{ secrets.SSH_HOST_IP }} >> ~/.ssh/known_hosts + + - name: SSH Commands + run: | + ssh -C -p 2222 ${{ secrets.SSH_DESTINATION }} "docker pull localhost:5000/jais-scraper:latest && cd ${{ secrets.SSH_FOLDER }} && docker compose down jais-scraper && docker compose up jais-scraper -d" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index ba697e2..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Release - -on: - push: - branches: - - stable - -env: - FOLDER: /home/pi/scraper/ - -jobs: - build: - name: Build and release - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: 'adopt' - - - name: Build - run: mvn clean package - - - name: Install SSH KEY - uses: shimataro/ssh-key-action@v2 - with: - key: ${{ secrets.SSH_PRIVATE_KEY }} - known_hosts: 'just-a-placeholder-so-we-dont-get-errors' - - - name: Adding Known Hosts - run: ssh-keyscan -H ${{ secrets.SSH_HOST_IP }} >> ~/.ssh/known_hosts - - - name: SSH Commands - run: | - scp target/scraper-1.0-SNAPSHOT-jar-with-dependencies.jar ${{ secrets.SSH_DESTINATION }}:"$FOLDER" - ssh -C ${{ secrets.SSH_DESTINATION }} "screen -XS Scraper quit && cd $FOLDER && ./start.sh" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f6bc2cb..28d02e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,32 @@ -FROM maven:3.8.6-jdk-11-slim +FROM maven:3.9.4-amazoncorretto-17 AS build WORKDIR /app COPY . /app -RUN apt-get update && apt-get install -y libgtk-3-0 libasound2 libx11-6 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libxrender1 libxtst6 libfreetype6 libfontconfig1 libpangocairo-1.0-0 libpangocairo-1.0-0 libpango-1.0-0 libatk1.0-0 libcairo-gobject2 libcairo2 libgdk-pixbuf-2.0-0 libglib2.0-0 libdbus-glib-1-2 libdbus-1-3 libxcb-shm0 libx11-xcb1 libxcb1 libxcursor1 libxi6 libnss3 libnspr4 libdrm2 libgbm1 RUN mvn clean package -DskipTests -RUN ls -lash -RUN mv target/scraper-1.0-SNAPSHOT-jar-with-dependencies.jar /scraper.jar -RUN rm -rf /app/** -RUN mv /scraper.jar /app/scraper.jar + +FROM mcr.microsoft.com/playwright:v1.38.0-jammy + +ARG version=17.0.8.8-1 +RUN set -eux \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates gnupg software-properties-common fontconfig java-common \ + && curl -fL https://apt.corretto.aws/corretto.key | apt-key add - \ + && add-apt-repository 'deb https://apt.corretto.aws stable main' \ + && mkdir -p /usr/share/man/man1 || true \ + && apt-get update \ + && apt-get install -y java-17-amazon-corretto-jdk=1:$version \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + curl gnupg software-properties-common + +ENV LANG C.UTF-8 +ENV JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto + +COPY --from=build /app/target/scraper-1.0-SNAPSHOT-jar-with-dependencies.jar /app/scraper.jar + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install tzdata +ENV TZ=Europe/Paris +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN dpkg-reconfigure --frontend noninteractive tzdata + +WORKDIR /app CMD ["java", "-jar", "scraper.jar"] \ No newline at end of file diff --git a/create_images.sh b/create_images.sh new file mode 100644 index 0000000..672fecc --- /dev/null +++ b/create_images.sh @@ -0,0 +1,4 @@ +#!/bin/sh +docker buildx create --use +docker buildx build --load --platform linux/amd64 -t jais-scraper:amd64 . +docker buildx build --load --platform linux/arm64 -t jais-scraper:arm64 . \ No newline at end of file diff --git a/install.sh b/install.sh deleted file mode 100644 index 88eb4ec..0000000 --- a/install.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -BRANCH=master - -# Check args passed to script -if [ $# -eq 1 ]; then - BRANCH=$1 -fi - -# Check if git is installed -if ! [ -x "$(command -v git)" ]; then - echo 'Error: git is not installed.' >&2 - exit 1 -fi - -# Check if docker is installed -if ! [ -x "$(command -v docker)" ]; then - echo 'Error: docker is not installed.' >&2 - exit 1 -fi - -# Clone repo from github "https://github.com/Z-Jais/Scraper.git" on branch $BRANCH -git clone --single-branch --branch "$BRANCH" https://github.com/Z-Jais/Scraper.git ./tmp - -cd ./tmp || exit - -# Build docker image -docker build -t scraper . -# Run docker container -docker run -d --name scraper scraper - -cd .. || exit -# Remove tmp folder -rm -rf ./tmp \ No newline at end of file diff --git a/pom.xml b/pom.xml index 64f8efb..78b4071 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ UTF-8 official - 11 + 17 1.9.10 fr.jais.scraper.ScraperKt 5.10.0 @@ -157,7 +157,7 @@ org.quartz-scheduler quartz - 2.3.2 + 2.5.0-rc1 \ No newline at end of file diff --git a/src/main/kotlin/fr/jais/scraper/Scraper.kt b/src/main/kotlin/fr/jais/scraper/Scraper.kt index 59bae6a..d3b3932 100644 --- a/src/main/kotlin/fr/jais/scraper/Scraper.kt +++ b/src/main/kotlin/fr/jais/scraper/Scraper.kt @@ -8,16 +8,17 @@ import fr.jais.scraper.jobs.JobManager import fr.jais.scraper.platforms.AnimationDigitalNetworkPlatform import fr.jais.scraper.platforms.CrunchyrollPlatform import fr.jais.scraper.platforms.IPlatform -import fr.jais.scraper.platforms.NetflixPlatform import fr.jais.scraper.utils.Const import fr.jais.scraper.utils.Logger import fr.jais.scraper.utils.ThreadManager +import java.io.File +import java.util.* class Scraper { val platforms = listOf( AnimationDigitalNetworkPlatform(this), CrunchyrollPlatform(this), - NetflixPlatform(this) + // NetflixPlatform(this) ) val countries = platforms.flatMap { it.countries }.distinct().mapNotNull { it.getConstructor().newInstance() } private val jobManager = JobManager() @@ -27,18 +28,17 @@ class Scraper { fun startThreadCommand() { ThreadManager.start("Command") { + val scanner = Scanner(System.`in`) + while (true) { - val command = readlnOrNull() ?: continue - val args = command.split(" ") + try { + val line = scanner.nextLine() - when (args[0]) { - "ayane" -> { + if (line == "ayane") { AyaneJob().execute(null) } - - else -> { - Logger.info("Unknown command") - } + } catch (_: Exception) { + Thread.sleep(1000) } } } @@ -58,6 +58,7 @@ class Scraper { fun main() { Logger.info("Initializing...") + File("data").mkdirs() Const.gson Logger.info("Initialization done!") diff --git a/src/main/kotlin/fr/jais/scraper/converters/AnimationDigitalNetworkConverter.kt b/src/main/kotlin/fr/jais/scraper/converters/AnimationDigitalNetworkConverter.kt index 7b85046..5ba2b1e 100644 --- a/src/main/kotlin/fr/jais/scraper/converters/AnimationDigitalNetworkConverter.kt +++ b/src/main/kotlin/fr/jais/scraper/converters/AnimationDigitalNetworkConverter.kt @@ -13,7 +13,7 @@ import java.io.File import java.util.* class AnimationDigitalNetworkConverter(private val platform: AnimationDigitalNetworkPlatform) { - private val file = File("animation_digital_network.json") + private val file = File("data/animation_digital_network.json") /// Convert anime from AnimationDigitalNetworkPlatform jsonObject to entity Anime private fun convertAnime(checkedCountry: ICountry, calendar: Calendar, jsonObject: JsonObject): Anime { @@ -105,6 +105,11 @@ class AnimationDigitalNetworkConverter(private val platform: AnimationDigitalNet // ----- NUMBER ----- Logger.info("Get number...") + + if ("Bande-annonce" == jsonObject["shortNumber"]?.asString) { + throw EpisodeNotAvailableException("Trailer detected") + } + val number = jsonObject["shortNumber"]?.asString()?.toIntOrNull() ?: run { Logger.warning("No number found, using -1...") -1 diff --git a/src/main/kotlin/fr/jais/scraper/converters/CrunchyrollConverter.kt b/src/main/kotlin/fr/jais/scraper/converters/CrunchyrollConverter.kt index 1594a10..645edca 100644 --- a/src/main/kotlin/fr/jais/scraper/converters/CrunchyrollConverter.kt +++ b/src/main/kotlin/fr/jais/scraper/converters/CrunchyrollConverter.kt @@ -32,7 +32,7 @@ class CrunchyrollConverter(private val platform: CrunchyrollPlatform) { private val useApi = false val cache = mutableListOf() private val sessionId: String = crunchyrollSession() - private val file = File("crunchyroll.json") + private val file = File("data/crunchyroll.json") fun getCountryTag(checkedCountry: ICountry): String { val country = when (checkedCountry) { diff --git a/src/main/kotlin/fr/jais/scraper/jobs/AyaneJob.kt b/src/main/kotlin/fr/jais/scraper/jobs/AyaneJob.kt index 093da0a..2bae0d2 100644 --- a/src/main/kotlin/fr/jais/scraper/jobs/AyaneJob.kt +++ b/src/main/kotlin/fr/jais/scraper/jobs/AyaneJob.kt @@ -23,7 +23,7 @@ class AyaneJob : Job { override fun execute(p0: JobExecutionContext?) { Logger.info("Starting AyaneJob...") - val folder = File("ayane") + val folder = File("data/ayane") if (!folder.exists()) folder.mkdirs() val font = File(folder, "Rubik.ttf") diff --git a/src/main/kotlin/fr/jais/scraper/platforms/AnimationDigitalNetworkPlatform.kt b/src/main/kotlin/fr/jais/scraper/platforms/AnimationDigitalNetworkPlatform.kt index 7a33dfb..50c0037 100644 --- a/src/main/kotlin/fr/jais/scraper/platforms/AnimationDigitalNetworkPlatform.kt +++ b/src/main/kotlin/fr/jais/scraper/platforms/AnimationDigitalNetworkPlatform.kt @@ -47,10 +47,7 @@ class AnimationDigitalNetworkPlatform(scraper: Scraper) : try { converter.convertEpisode(country, calendar, it, cachedEpisodes) } catch (e: Exception) { - if (e !is EpisodeException) { - Logger.log(Level.SEVERE, "Error while converting episode", e) - } - + Logger.log(Level.SEVERE, "Error while converting episode", e) null } } ?: emptyList() diff --git a/src/main/kotlin/fr/jais/scraper/platforms/CrunchyrollPlatform.kt b/src/main/kotlin/fr/jais/scraper/platforms/CrunchyrollPlatform.kt index 0b8c9f8..461385c 100644 --- a/src/main/kotlin/fr/jais/scraper/platforms/CrunchyrollPlatform.kt +++ b/src/main/kotlin/fr/jais/scraper/platforms/CrunchyrollPlatform.kt @@ -58,48 +58,42 @@ class CrunchyrollPlatform(scraper: Scraper) : IPlatform( } private fun checkSimulcasts(iCountry: ICountry) { - Logger.info("Checking simulcasts for ${iCountry.name}...") - // Clear simulcast for this country if exists - simulcasts.remove(iCountry) val countryTag = converter.getCountryTag(iCountry) Logger.info("Loading simulcasts for ${iCountry.name}...") - val content = try { - Browser( - Browser.BrowserType.FIREFOX, - "https://www.crunchyroll.com/$countryTag/simulcasts" - ).launchAndWaitForSelector("#content > div > div.app-body-wrapper > div > div > div.erc-browse-collection > div > div:nth-child(1) > div > div > h4 > a") - } catch (_: Exception) { - Logger.warning("No simulcasts found for ${iCountry.name}! (Empty page or error)") - - Browser( - Browser.BrowserType.FIREFOX, - "https://www.crunchyroll.com/$countryTag/simulcasts" - ).launchAndWaitForSelector("#content > div > div.app-body-wrapper > div > div > div.info-box--wxFEW.erc-simulcast-no-results") - } + val selector = "#content > div > div.app-body-wrapper > div > div > div.erc-browse-collection > div > div:nth-child(1) > div > div > h4 > a" + val simulcastSelector = ".erc-browse-cards-collection > .browse-card > div > div > h4 > a" - val currentSimulcastAnimes = content.select(".erc-browse-cards-collection > .browse-card > div > div > h4 > a") - .map { it.text().lowercase() }.toSet() + val contentCurrentSimulcast = Browser( + Browser.BrowserType.FIREFOX, + "https://www.crunchyroll.com/$countryTag/simulcasts" + ).launchAndWaitForSelector(selector) val simulcastName = - content.select("#content > div > div.app-body-wrapper > div > div > div.header > div > div > span.call-to-action--PEidl.call-to-action--is-m--RVdkI.select-trigger__title-cta--C5-uH.select-trigger__title-cta--is-displayed-on-mobile--6oNk1") - .text() + contentCurrentSimulcast.select("#content > div > div.app-body-wrapper > div > div > div.header > div > div > span.call-to-action--PEidl.call-to-action--is-m--RVdkI.select-trigger__title-cta--C5-uH.select-trigger__title-cta--is-displayed-on-mobile--6oNk1").text() val simulcastCode = getSimulcastCode(simulcastName) - Logger.info("Simulcast code for ${iCountry.name}: $simulcastCode") + Logger.info("Current simulcast code for ${iCountry.name}: $simulcastCode") + + val currentSimulcastAnimes = contentCurrentSimulcast.select(simulcastSelector) + .map { it.text().lowercase() }.toSet() + Logger.config("Found ${currentSimulcastAnimes.size} animes for the current simulcast") + val previousSimulcastCode = getPreviousSimulcastCode(simulcastCode) Logger.info("Previous simulcast code for ${iCountry.name}: $previousSimulcastCode") - val content2 = Browser( + val contentPreviousSimulcast = Browser( Browser.BrowserType.FIREFOX, "https://www.crunchyroll.com/$countryTag/simulcasts/seasons/$previousSimulcastCode" - ).launchAndWaitForSelector("#content > div > div.app-body-wrapper > div > div > div.erc-browse-collection > div > div:nth-child(1) > div > div > h4 > a") + ).launchAndWaitForSelector(selector) + val previousSimulcastAnimes = - content2.select(".erc-browse-cards-collection > .browse-card > div > div > h4 > a") - .map { it.text().lowercase() }.toSet() + contentPreviousSimulcast.select(simulcastSelector).map { it.text().lowercase() }.toSet() + Logger.config("Found ${previousSimulcastAnimes.size} animes for the previous simulcast") - simulcasts[iCountry] = (currentSimulcastAnimes + previousSimulcastAnimes).toSet() - Logger.info("Found ${simulcasts[iCountry]?.size} simulcasts for ${iCountry.name}!") - Logger.config("Simulcasts: ${simulcasts[iCountry]?.joinToString(", ")}") + val combinedSimulcastAnimes = (currentSimulcastAnimes + previousSimulcastAnimes).toSet() + simulcasts[iCountry] = combinedSimulcastAnimes + Logger.info("Found ${combinedSimulcastAnimes.size} simulcasts for ${iCountry.name}!") + Logger.config("Simulcasts: ${combinedSimulcastAnimes.joinToString(", ")}") } fun xmlToJson(content: String) = @@ -137,12 +131,11 @@ class CrunchyrollPlatform(scraper: Scraper) : IPlatform( countries.forEach { try { checkSimulcasts(it) + lastSimulcastCheck = System.currentTimeMillis() } catch (e: Exception) { Logger.log(Level.SEVERE, "Error while checking simulcasts", e) } } - - lastSimulcastCheck = System.currentTimeMillis() } return countries.flatMap { country -> diff --git a/src/main/kotlin/fr/jais/scraper/utils/API.kt b/src/main/kotlin/fr/jais/scraper/utils/API.kt index a9792ee..bcb2ed8 100644 --- a/src/main/kotlin/fr/jais/scraper/utils/API.kt +++ b/src/main/kotlin/fr/jais/scraper/utils/API.kt @@ -11,9 +11,6 @@ import java.net.http.HttpRequest import java.net.http.HttpResponse import java.util.logging.Level -private const val URL = "https://beta-api.ziedelth.fr/" -//private const val URL = "http://localhost:8080/" - object API { private fun get(url: String): HttpResponse { val request = HttpRequest.newBuilder() @@ -32,20 +29,20 @@ object API { } private fun getCountry(country: Country): JsonObject? { - val response = get("${URL}countries") + val response = get("${Const.apiUrl}countries") val json = Const.gson.fromJson(response.body(), JsonArray::class.java) ?: return null return json.firstOrNull { it.asJsonObject["tag"].asString == country.tag }?.asJsonObject } private fun createCountry(country: Country): JsonObject? { - val response = post("${URL}countries", Const.gson.toJson(country)) + val response = post("${Const.apiUrl}countries", Const.gson.toJson(country)) return if (response.statusCode() == 201) Const.gson.fromJson(response.body(), JsonObject::class.java) else null } private fun getAnimeByHash(country: Country, anime: Anime): JsonObject? { val hash = anime.name.lowercase().filter { it.isLetterOrDigit() || it.isWhitespace() || it == '-' }.trim() .replace("\\s+".toRegex(), "-").replace("--", "-") - val response = get("${URL}animes/country/${country.tag}/search/hash/$hash") + val response = get("${Const.apiUrl}animes/country/${country.tag}/search/hash/$hash") return if (response.statusCode() == 200) Const.gson.fromJson(response.body(), JsonObject::class.java) else null } @@ -59,40 +56,40 @@ object API { } private fun createAnime(country: JsonObject, releaseDate: String, anime: Anime): JsonObject? { - val response = post("${URL}animes", toAnime(country, anime, releaseDate).toString()) + val response = post("${Const.apiUrl}animes", toAnime(country, anime, releaseDate).toString()) return if (response.statusCode() == 201) Const.gson.fromJson(response.body(), JsonObject::class.java) else null } private fun getPlatform(platform: Platform): JsonObject? { - val response = get("${URL}platforms") + val response = get("${Const.apiUrl}platforms") val json = Const.gson.fromJson(response.body(), JsonArray::class.java) ?: return null return json.firstOrNull { it.asJsonObject["name"].asString == platform.name }?.asJsonObject } private fun createPlatform(platform: Platform): JsonObject? { - val response = post("${URL}platforms", Const.gson.toJson(platform)) + val response = post("${Const.apiUrl}platforms", Const.gson.toJson(platform)) return if (response.statusCode() == 201) Const.gson.fromJson(response.body(), JsonObject::class.java) else null } private fun getEpisodeType(type: EpisodeType): JsonObject? { - val response = get("${URL}episodetypes") + val response = get("${Const.apiUrl}episodetypes") val json = Const.gson.fromJson(response.body(), JsonArray::class.java) ?: return null return json.firstOrNull { it.asJsonObject["name"].asString == type.name }?.asJsonObject } private fun createEpisodeType(type: EpisodeType): JsonObject? { - val response = post("${URL}episodetypes", JsonObject().apply { addProperty("name", type.name) }.toString()) + val response = post("${Const.apiUrl}episodetypes", JsonObject().apply { addProperty("name", type.name) }.toString()) return if (response.statusCode() == 201) Const.gson.fromJson(response.body(), JsonObject::class.java) else null } private fun getLangType(type: LangType): JsonObject? { - val response = get("${URL}langtypes") + val response = get("${Const.apiUrl}langtypes") val json = Const.gson.fromJson(response.body(), JsonArray::class.java) ?: return null return json.firstOrNull { it.asJsonObject["name"].asString == type.name }?.asJsonObject } private fun createLangType(type: LangType): JsonObject? { - val response = post("${URL}langtypes", JsonObject().apply { addProperty("name", type.name) }.toString()) + val response = post("${Const.apiUrl}langtypes", JsonObject().apply { addProperty("name", type.name) }.toString()) return if (response.statusCode() == 201) Const.gson.fromJson(response.body(), JsonObject::class.java) else null } @@ -154,7 +151,7 @@ object API { return } - post("${URL}episodes/multiple", Const.gson.toJson(episodesApi)) + post("${Const.apiUrl}episodes/multiple", Const.gson.toJson(episodesApi)) } catch (e: Exception) { Logger.log(Level.SEVERE, "Error saving episodes", e) } @@ -162,7 +159,7 @@ object API { fun saveAyane(message: String, images: List) { try { - post("${URL}ayane", Const.gson.toJson(mapOf("message" to message, "images" to images))) + post("${Const.apiUrl}ayane", Const.gson.toJson(mapOf("message" to message, "images" to images))) } catch (e: Exception) { Logger.log(Level.SEVERE, "Error saving episodes", e) } diff --git a/src/main/kotlin/fr/jais/scraper/utils/Browser.kt b/src/main/kotlin/fr/jais/scraper/utils/Browser.kt index 437199d..b40f2bb 100644 --- a/src/main/kotlin/fr/jais/scraper/utils/Browser.kt +++ b/src/main/kotlin/fr/jais/scraper/utils/Browser.kt @@ -28,13 +28,19 @@ class Browser(type: BrowserType = BrowserType.CHROME, val url: String) { context = browser?.newContext() Logger.info("Creating page...") page = context?.newPage() - page?.setDefaultTimeout(30_000.0) - page?.setDefaultNavigationTimeout(30_000.0) + page?.setDefaultTimeout(60_000.0) + page?.setDefaultNavigationTimeout(60_000.0) Logger.config("URL: $url") - Logger.info("Navigating...") - page?.navigate(url) - Logger.info("Waiting for load...") - page?.waitForLoadState() + + try { + Logger.info("Navigating...") + page?.navigate(url) + Logger.info("Waiting for load...") + page?.waitForLoadState() + } catch (e: Exception) { + close() + throw e + } } fun launch(): Document { @@ -46,7 +52,13 @@ class Browser(type: BrowserType = BrowserType.CHROME, val url: String) { } fun launchAndWaitForSelector(selector: String): Document { - page?.waitForSelector(selector) + try { + page?.waitForSelector(selector) + } catch (e: Exception) { + close() + throw e + } + val content = page?.content() close() diff --git a/src/main/kotlin/fr/jais/scraper/utils/Const.kt b/src/main/kotlin/fr/jais/scraper/utils/Const.kt index 7c49db1..3f62e62 100644 --- a/src/main/kotlin/fr/jais/scraper/utils/Const.kt +++ b/src/main/kotlin/fr/jais/scraper/utils/Const.kt @@ -17,4 +17,5 @@ object Const { val firefox: BrowserType = playwright.firefox() val multipleSpaceRegex = "\\s+".toRegex() const val calendarBaseUrl = "https://anime.icotaku.com" + val apiUrl = System.getenv("API_URL") ?: "http://localhost:8080/" } diff --git a/src/main/kotlin/fr/jais/scraper/utils/Database.kt b/src/main/kotlin/fr/jais/scraper/utils/Database.kt index 7483320..bf198f8 100644 --- a/src/main/kotlin/fr/jais/scraper/utils/Database.kt +++ b/src/main/kotlin/fr/jais/scraper/utils/Database.kt @@ -10,7 +10,7 @@ class Database { val episodes: MutableList = mutableListOf() ) - private val file = File("database.txt") + private val file = File("data/database.txt") fun load(): DB = Const.gson.fromJson( diff --git a/src/main/kotlin/fr/jais/scraper/utils/Logger.kt b/src/main/kotlin/fr/jais/scraper/utils/Logger.kt index 79f1b26..0b60ce1 100644 --- a/src/main/kotlin/fr/jais/scraper/utils/Logger.kt +++ b/src/main/kotlin/fr/jais/scraper/utils/Logger.kt @@ -51,7 +51,7 @@ object Logger : Logger("Scraper", null) { consoleHandler.formatter = formatter consoleHandler.level = Level.ALL this.addHandler(consoleHandler) - val logsFolder = File("logs") + val logsFolder = File("data/logs") if (!logsFolder.exists()) logsFolder.mkdirs() val fileHandler = FileHandler("${logsFolder.absolutePath}${File.separator}log.log", 5 * 1024 * 1024, 1, true) fileHandler.formatter = formatter