From 1ba1f0f5159e8ccdb510c9eb421880d2e915eaaa Mon Sep 17 00:00:00 2001 From: MrTyton Date: Mon, 27 May 2024 17:45:06 -0400 Subject: [PATCH] Major Refactor --- .VSCodeCounter/2024-05-20_23-45-31/details.md | 28 + .../2024-05-20_23-45-31/diff-details.md | 15 + .VSCodeCounter/2024-05-20_23-45-31/diff.csv | 2 + .VSCodeCounter/2024-05-20_23-45-31/diff.md | 19 + .VSCodeCounter/2024-05-20_23-45-31/diff.txt | 22 + .../2024-05-20_23-45-31/results.csv | 15 + .../2024-05-20_23-45-31/results.json | 1 + .VSCodeCounter/2024-05-20_23-45-31/results.md | 24 + .../2024-05-20_23-45-31/results.txt | 40 + .github/workflows/docker-image.yml | 5 +- Dockerfile | 22 +- README.md | 93 +- release-versions/fff.txt | 2 +- release-versions/latest.txt | 2 +- release-versions/s6.txt | 1 + requirements.txt | 26 + root/app/calibre_info.py | 89 + root/app/calibre_info_test.py | 196 + root/app/fanfic_info.py | 65 + root/app/fanfic_info_test.py | 49 + root/app/fanficdownload.py | 514 +-- root/app/ff_logging.py | 36 + root/app/ff_logging_test.py | 80 + root/app/ff_waiter.py | 60 + root/app/ff_waiter_test.py | 46 + root/app/notifications.py | 8 - root/app/pushbullet_notification.py | 47 + root/app/pushbullet_notification_test.py | 194 + root/app/regex_parsing.py | 95 + root/app/regex_parsing_test.py | 169 + root/app/run.sh | 7 +- root/app/runner_notify.py | 171 - root/app/url_ingester.py | 89 + root/app/url_ingester_test.py | 120 + root/app/url_worker.py | 269 ++ root/config.default/config.ini | 18 - root/config.default/config.toml | 18 + root/config.default/defaults.ini | 3613 +++++++++++++---- root/config.default/fanfiction_file | 0 root/config.default/personal.ini | 129 +- root/etc/cont-init.d/05-default-confs | 19 + root/etc/cont-init.d/90-user-permissions | 10 + root/etc/cont-init.d/99-init.d-finish | 11 + root/etc/services.d/run | 4 + 44 files changed, 4871 insertions(+), 1572 deletions(-) create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/details.md create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/diff-details.md create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/diff.csv create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/diff.md create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/diff.txt create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/results.csv create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/results.json create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/results.md create mode 100644 .VSCodeCounter/2024-05-20_23-45-31/results.txt create mode 100644 release-versions/s6.txt create mode 100644 requirements.txt create mode 100644 root/app/calibre_info.py create mode 100644 root/app/calibre_info_test.py create mode 100644 root/app/fanfic_info.py create mode 100644 root/app/fanfic_info_test.py create mode 100644 root/app/ff_logging.py create mode 100644 root/app/ff_logging_test.py create mode 100644 root/app/ff_waiter.py create mode 100644 root/app/ff_waiter_test.py delete mode 100644 root/app/notifications.py create mode 100644 root/app/pushbullet_notification.py create mode 100644 root/app/pushbullet_notification_test.py create mode 100644 root/app/regex_parsing.py create mode 100644 root/app/regex_parsing_test.py delete mode 100644 root/app/runner_notify.py create mode 100644 root/app/url_ingester.py create mode 100644 root/app/url_ingester_test.py create mode 100644 root/app/url_worker.py delete mode 100644 root/config.default/config.ini create mode 100644 root/config.default/config.toml delete mode 100644 root/config.default/fanfiction_file create mode 100644 root/etc/cont-init.d/05-default-confs create mode 100644 root/etc/cont-init.d/90-user-permissions create mode 100644 root/etc/cont-init.d/99-init.d-finish create mode 100644 root/etc/services.d/run diff --git a/.VSCodeCounter/2024-05-20_23-45-31/details.md b/.VSCodeCounter/2024-05-20_23-45-31/details.md new file mode 100644 index 0000000..b608c58 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/details.md @@ -0,0 +1,28 @@ +# Details + +Date : 2024-05-20 23:45:31 + +Directory c:\\Users\\Joshua\\Documents\\GitHub\\Hub\\Automated-FFDL + +Total : 13 files, 3793 codes, 119 comments, 1213 blanks, all 5125 lines + +[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) + +## Files +| filename | language | code | comment | blank | total | +| :--- | :--- | ---: | ---: | ---: | ---: | +| [calibre_info.py](/calibre_info.py) | Python | 47 | 13 | 13 | 73 | +| [config.toml](/config.toml) | TOML | 16 | 0 | 2 | 18 | +| [defaults.ini](/defaults.ini) | Ini | 3,177 | 38 | 1,080 | 4,295 | +| [fanfic_info.py](/fanfic_info.py) | Python | 46 | 7 | 5 | 58 | +| [fanficdownload.py](/fanficdownload.py) | Python | 18 | 6 | 13 | 37 | +| [ff_logging.py](/ff_logging.py) | Python | 25 | 1 | 4 | 30 | +| [ff_logging_test.py](/ff_logging_test.py) | Python | 27 | 0 | 5 | 32 | +| [personal.ini](/personal.ini) | Ini | 81 | 0 | 19 | 100 | +| [pushbullet_notification.py](/pushbullet_notification.py) | Python | 31 | 8 | 6 | 45 | +| [regex_parsing.py](/regex_parsing.py) | Python | 71 | 2 | 11 | 84 | +| [requirements.txt](/requirements.txt) | pip requirements | 26 | 0 | 1 | 27 | +| [url_ingester.py](/url_ingester.py) | Python | 55 | 6 | 13 | 74 | +| [url_worker.py](/url_worker.py) | Python | 173 | 38 | 41 | 252 | + +[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2024-05-20_23-45-31/diff-details.md b/.VSCodeCounter/2024-05-20_23-45-31/diff-details.md new file mode 100644 index 0000000..d671925 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/diff-details.md @@ -0,0 +1,15 @@ +# Diff Details + +Date : 2024-05-20 23:45:31 + +Directory c:\\Users\\Joshua\\Documents\\GitHub\\Hub\\Automated-FFDL + +Total : 0 files, 0 codes, 0 comments, 0 blanks, all 0 lines + +[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details + +## Files +| filename | language | code | comment | blank | total | +| :--- | :--- | ---: | ---: | ---: | ---: | + +[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details \ No newline at end of file diff --git a/.VSCodeCounter/2024-05-20_23-45-31/diff.csv b/.VSCodeCounter/2024-05-20_23-45-31/diff.csv new file mode 100644 index 0000000..b7d8d75 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/diff.csv @@ -0,0 +1,2 @@ +"filename", "language", "", "comment", "blank", "total" +"Total", "-", , 0, 0, 0 \ No newline at end of file diff --git a/.VSCodeCounter/2024-05-20_23-45-31/diff.md b/.VSCodeCounter/2024-05-20_23-45-31/diff.md new file mode 100644 index 0000000..6e47be3 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/diff.md @@ -0,0 +1,19 @@ +# Diff Summary + +Date : 2024-05-20 23:45:31 + +Directory c:\\Users\\Joshua\\Documents\\GitHub\\Hub\\Automated-FFDL + +Total : 0 files, 0 codes, 0 comments, 0 blanks, all 0 lines + +[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) + +## Languages +| language | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | + +## Directories +| path | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | + +[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2024-05-20_23-45-31/diff.txt b/.VSCodeCounter/2024-05-20_23-45-31/diff.txt new file mode 100644 index 0000000..d3889f6 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/diff.txt @@ -0,0 +1,22 @@ +Date : 2024-05-20 23:45:31 +Directory : c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL +Total : 0 files, 0 codes, 0 comments, 0 blanks, all 0 lines + +Languages ++----------+------------+------------+------------+------------+------------+ +| language | files | code | comment | blank | total | ++----------+------------+------------+------------+------------+------------+ ++----------+------------+------------+------------+------------+------------+ + +Directories ++------+------------+------------+------------+------------+------------+ +| path | files | code | comment | blank | total | ++------+------------+------------+------------+------------+------------+ ++------+------------+------------+------------+------------+------------+ + +Files ++----------+----------+------------+------------+------------+------------+ +| filename | language | code | comment | blank | total | ++----------+----------+------------+------------+------------+------------+ +| Total | | 0 | 0 | 0 | 0 | ++----------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/.VSCodeCounter/2024-05-20_23-45-31/results.csv b/.VSCodeCounter/2024-05-20_23-45-31/results.csv new file mode 100644 index 0000000..d23fc49 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/results.csv @@ -0,0 +1,15 @@ +"filename", "language", "Ini", "Python", "TOML", "pip requirements", "comment", "blank", "total" +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\calibre_info.py", "Python", 0, 47, 0, 0, 13, 13, 73 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\config.toml", "TOML", 0, 0, 16, 0, 0, 2, 18 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\defaults.ini", "Ini", 3177, 0, 0, 0, 38, 1080, 4295 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\fanfic_info.py", "Python", 0, 46, 0, 0, 7, 5, 58 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\fanficdownload.py", "Python", 0, 18, 0, 0, 6, 13, 37 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\ff_logging.py", "Python", 0, 25, 0, 0, 1, 4, 30 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\ff_logging_test.py", "Python", 0, 27, 0, 0, 0, 5, 32 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\personal.ini", "Ini", 81, 0, 0, 0, 0, 19, 100 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\pushbullet_notification.py", "Python", 0, 31, 0, 0, 8, 6, 45 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\regex_parsing.py", "Python", 0, 71, 0, 0, 2, 11, 84 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\requirements.txt", "pip requirements", 0, 0, 0, 26, 0, 1, 27 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\url_ingester.py", "Python", 0, 55, 0, 0, 6, 13, 74 +"c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\url_worker.py", "Python", 0, 173, 0, 0, 38, 41, 252 +"Total", "-", 3258, 493, 16, 26, 119, 1213, 5125 \ No newline at end of file diff --git a/.VSCodeCounter/2024-05-20_23-45-31/results.json b/.VSCodeCounter/2024-05-20_23-45-31/results.json new file mode 100644 index 0000000..354f2a7 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/results.json @@ -0,0 +1 @@ +{"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/personal.ini":{"language":"Ini","code":81,"comment":0,"blank":19},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/ff_logging.py":{"language":"Python","code":25,"comment":1,"blank":4},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/ff_logging_test.py":{"language":"Python","code":27,"comment":0,"blank":5},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/fanfic_info.py":{"language":"Python","code":46,"comment":7,"blank":5},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/pushbullet_notification.py":{"language":"Python","code":31,"comment":8,"blank":6},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/fanficdownload.py":{"language":"Python","code":18,"comment":6,"blank":13},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/config.toml":{"language":"TOML","code":16,"comment":0,"blank":2},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/requirements.txt":{"language":"pip requirements","code":26,"comment":0,"blank":1},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/url_ingester.py":{"language":"Python","code":55,"comment":6,"blank":13},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/defaults.ini":{"language":"Ini","code":3177,"comment":38,"blank":1080},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/url_worker.py":{"language":"Python","code":173,"comment":38,"blank":41},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/calibre_info.py":{"language":"Python","code":47,"comment":13,"blank":13},"file:///c%3A/Users/Joshua/Documents/GitHub/Hub/Automated-FFDL/regex_parsing.py":{"language":"Python","code":71,"comment":2,"blank":11}} \ No newline at end of file diff --git a/.VSCodeCounter/2024-05-20_23-45-31/results.md b/.VSCodeCounter/2024-05-20_23-45-31/results.md new file mode 100644 index 0000000..96aa407 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/results.md @@ -0,0 +1,24 @@ +# Summary + +Date : 2024-05-20 23:45:31 + +Directory c:\\Users\\Joshua\\Documents\\GitHub\\Hub\\Automated-FFDL + +Total : 13 files, 3793 codes, 119 comments, 1213 blanks, all 5125 lines + +Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) + +## Languages +| language | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| Ini | 2 | 3,258 | 38 | 1,099 | 4,395 | +| Python | 9 | 493 | 81 | 111 | 685 | +| pip requirements | 1 | 26 | 0 | 1 | 27 | +| TOML | 1 | 16 | 0 | 2 | 18 | + +## Directories +| path | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| . | 13 | 3,793 | 119 | 1,213 | 5,125 | + +Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2024-05-20_23-45-31/results.txt b/.VSCodeCounter/2024-05-20_23-45-31/results.txt new file mode 100644 index 0000000..eeb4005 --- /dev/null +++ b/.VSCodeCounter/2024-05-20_23-45-31/results.txt @@ -0,0 +1,40 @@ +Date : 2024-05-20 23:45:31 +Directory : c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL +Total : 13 files, 3793 codes, 119 comments, 1213 blanks, all 5125 lines + +Languages ++------------------+------------+------------+------------+------------+------------+ +| language | files | code | comment | blank | total | ++------------------+------------+------------+------------+------------+------------+ +| Ini | 2 | 3,258 | 38 | 1,099 | 4,395 | +| Python | 9 | 493 | 81 | 111 | 685 | +| pip requirements | 1 | 26 | 0 | 1 | 27 | +| TOML | 1 | 16 | 0 | 2 | 18 | ++------------------+------------+------------+------------+------------+------------+ + +Directories ++--------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| path | files | code | comment | blank | total | ++--------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| . | 13 | 3,793 | 119 | 1,213 | 5,125 | ++--------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ + +Files ++--------------------------------------------------------------------------------+------------------+------------+------------+------------+------------+ +| filename | language | code | comment | blank | total | ++--------------------------------------------------------------------------------+------------------+------------+------------+------------+------------+ +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\calibre_info.py | Python | 47 | 13 | 13 | 73 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\config.toml | TOML | 16 | 0 | 2 | 18 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\defaults.ini | Ini | 3,177 | 38 | 1,080 | 4,295 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\fanfic_info.py | Python | 46 | 7 | 5 | 58 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\fanficdownload.py | Python | 18 | 6 | 13 | 37 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\ff_logging.py | Python | 25 | 1 | 4 | 30 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\ff_logging_test.py | Python | 27 | 0 | 5 | 32 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\personal.ini | Ini | 81 | 0 | 19 | 100 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\pushbullet_notification.py | Python | 31 | 8 | 6 | 45 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\regex_parsing.py | Python | 71 | 2 | 11 | 84 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\requirements.txt | pip requirements | 26 | 0 | 1 | 27 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\url_ingester.py | Python | 55 | 6 | 13 | 74 | +| c:\Users\Joshua\Documents\GitHub\Hub\Automated-FFDL\url_worker.py | Python | 173 | 38 | 41 | 252 | +| Total | | 3,793 | 119 | 1,213 | 5,125 | ++--------------------------------------------------------------------------------+------------------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 5e15111..8e55f9d 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -35,13 +35,15 @@ jobs: echo "CALIBRE_VERSION=$CALIBRE_VERSION" >> $GITHUB_ENV FFF_VERSION=$(cat release-versions/fff.txt) echo "FFF_VERSION=$FFF_VERSION" >> $GITHUB_ENV + S6_OVERLAY_VERSION=$(cat release-versions/s6.txt) + echo "S6_OVERLAY_VERSION=$S6_OVERLAY_VERSION" >> $GITHUB_ENV - name: Print image tag run: | echo "Branch: $CI_ACTION_REF_NAME" echo "Release Version: ${{ env.RELEASE_VERSION }}" echo "Calibre Version: ${{ env.CALIBRE_VERSION }}" echo "FFF Version: ${{ env.FFF_VERSION }}" - + echo "S6 Overlay Version: ${{ env.S6_OVERLAY_VERSION }}" - name: Login to DockerHub uses: docker/login-action@v3.0.0 with: @@ -60,3 +62,4 @@ jobs: VERSION=${{ env.RELEASE_VERSION }} CALIBRE_RELEASE=${{ env.CALIBRE_VERSION }} FFF_RELEASE=${{ env.FFF_VERSION }} + S6_OVERLAY_VERSION=${{ env.S6_OVERLAY_VERSION }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b319ba6..5138579 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ FROM python:3-slim ARG VERSION ARG CALIBRE_RELEASE ARG FFF_RELEASE -LABEL build_version="FFDL-Auto version:- ${VERSION} Calibre: ${CALIBRE_RELEASE} FFF: ${FFF_RELEASE}" +ARG S6_OVERLAY_VERSION +LABEL build_version="FFDL-Auto version:- ${VERSION} Calibre: ${CALIBRE_RELEASE} FFF: ${FFF_RELEASE} S6_OVERLAY_VERSION: ${S6_OVERLAY_VERSION}" ENV PUID="911" \ PGID="911" @@ -36,6 +37,15 @@ RUN echo "**** install calibre ****" && \ apt-get install -y calibre && \ dbus-uuidgen > /etc/machine-id +ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz /tmp +RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz +ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-x86_64.tar.xz /tmp +RUN tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz + +RUN echo "*** Install Other Python Packages ***" && \ +COPY requirements.txt /tmp/ +RUN python3 -m pip install --no-cache-dir -r /tmp/requirements.txt + RUN echo "*** Install FFF ***" && \ if [ -z ${FFF_RELEASE} ]; then \ echo "FFF Using Default Release"; \ @@ -46,9 +56,6 @@ RUN echo "*** Install FFF ***" && \ fi -RUN echo "*** Install Other Python Packages ***" && \ - python3 -m pip --no-cache-dir install pushbullet.py pillow - RUN echo "**** cleanup ****" && \ rm -rf \ /tmp/* \ @@ -57,10 +64,11 @@ RUN echo "**** cleanup ****" && \ COPY root/ / +RUN chmod -R +777 /etc/cont-init.d/ +RUN chmod -R +777 /etc/services.d/ + VOLUME /config WORKDIR /config -RUN groupmod -o -g "$PGID" abc && usermod -o -u "$PUID" abc && chown -R abc:abc /app && chown -R abc:abc /config && chown -R abc:abc /root && chmod +x /app/run.sh - -CMD ["sh", "/app/run.sh"] +ENTRYPOINT ["/init"] diff --git a/README.md b/README.md index 21a1f9c..99b7434 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,101 @@ Automated Fanfiction Download using FanficFare CLI This is a docker image to run the Automated FFF CLI, with pushbullet integration. +[FanFicFare](https://github.com/JimmXinu/FanFicFare) + [Dockerhub Link](https://hub.docker.com/r/mrtyton/automated-ffdl) -## How to Install +- [AutomatedFanfic](#automatedfanfic) + - [Calibre Setup](#calibre-setup) + - [Execution](#execution) + - [How to Install - Docker](#how-to-install---docker) + - [How to Run - Non-Docker](#how-to-run---non-docker) + - [Configuration](#configuration) + - [Email](#email) + - [Calibre](#calibre) + - [Pushbullet](#pushbullet) + + +## Calibre Setup + +1. Setup the Calibre Content Server. Instructions can be found on the [calibre website](https://manual.calibre-ebook.com/server.html) +2. Make note of the IP address, Port and Library for the server. If needed, also make note of the Username and Password. + +## Execution + +### How to Install - Docker 1. Install the docker image with `docker pull mrtyton/automated-ffdl` 2. Map the `/config` volume to someplace on your drive. 3. After running the image once, it will have copied over default configs. Fill them out and everything should start working. + +### How to Run - Non-Docker + +1. Make sure that you have calibre, and more importantly [calibredb](https://manual.calibre-ebook.com/generated/en/calibredb.html) installed on the system that you're running the script on. `calibredb` should be installed standard when you install calibre. +2. Install [Python3](https://www.python.org/downloads/) +3. Clone the Repo +4. Run `python -m pip install -r requirements.txt` +5. Install [FanficFare](https://github.com/JimmXinu/FanFicFare/wiki#command-line-interface-cli-version) +6. Fill out the config.toml file +7. Navigate to `root/app` and run `python fanficdownload.py` + +## Configuration + +The config file is a [TOML](https://toml.io/en/) file that contains the script's specific options. Changes to this file will only take effect upon script startup. + + +### Email + +In order for the script to work, you have to fill out the email login information. + +```toml +[email] +email = "" +password = "" +server = "" +mailbox = "" +sleep_time = 60 +``` + + +- `email`: The email address, username only. +- `password`: The password to the email address. It is recommened that you use an app password (Google's page on [App Password](https://support.google.com/accounts/answer/185833?hl=en)), rather than your email's actual password. +- `server`: Address for the email server. For Gmail, this is going to be `imap.gmail.com`. For other web services, you'll have to search for them. +- `mailbox`: Which mailbox to check, such as `INBOX`, for the unread update emails. +- `sleep_time`: How often to check the email account for new updates, in seconds. Default is 60 seconds, but you can make this as often as you want. Recommended that you don't go too fast though, since some email providers will not be happy. + +### Calibre + +The Calibre information for access and updating. + +```toml +[calibre] +path="" +username="" +password="" +default_ini="" +personal_ini="" +``` + +- `path`: This is the path to your Calibre database. It's the location where your Calibre library is stored on your system. This can be either a directory that contains the `calibre.db` file, or the URL/Port/Library marked down above, such as `https://192.168.1.1:9001/#Fanfiction` This is the only argument that is **required** in this section. +- `username`: If your Calibre database is password protected, this is the username you use to access it. +- `password`: If your Calibre database is password protected, this is the password you use to access it. +- `default_ini`: This is the path to the [default INI configuration file](https://github.com/JimmXinu/FanFicFare/blob/main/fanficfare/defaults.ini) for FanFicFare. +- `personal_ini`: This is the path to your [personal INI configuration file](https://github.com/JimmXinu/FanFicFare/wiki/INI-File) for FanFicFare. + +For both the default and personal INI, any changes made to them will take effect during the next update check, it does not require a restart of the script. + +### Pushbullet + +This script has an _optional_ [Pushbullet](https://pushbullet.com) integration, in case you want to get phone notifications when an update has occurred. The system will also send a notification if it fails to update a story, for whatever reason. + +```toml +[pushbullet] +enabled = false +api_key = "" +device = "" +``` + +- `enabled`: Whether or not to enable the pushbullet notifications +- `api_key`: Your [[Pushbullet API Key](https://docs.pushbullet.com/#authentication)] +- `device`: If you want to send the notification to a specific device rather than the entirety of the pushbullet subscriptions, you can specify which device here with the device name. \ No newline at end of file diff --git a/release-versions/fff.txt b/release-versions/fff.txt index c1ee3a3..1fcaeef 100644 --- a/release-versions/fff.txt +++ b/release-versions/fff.txt @@ -1 +1 @@ -4.34.5 \ No newline at end of file +4.34.6 \ No newline at end of file diff --git a/release-versions/latest.txt b/release-versions/latest.txt index cf1487e..b3a5839 100644 --- a/release-versions/latest.txt +++ b/release-versions/latest.txt @@ -1 +1 @@ -2024.05.20-1 \ No newline at end of file +2024.05.27 \ No newline at end of file diff --git a/release-versions/s6.txt b/release-versions/s6.txt new file mode 100644 index 0000000..daa0ff2 --- /dev/null +++ b/release-versions/s6.txt @@ -0,0 +1 @@ +3.1.6.2z \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3456aea --- /dev/null +++ b/requirements.txt @@ -0,0 +1,26 @@ +beautifulsoup4==4.12.3 +Brotli==1.1.0 +certifi==2024.2.2 +chardet==5.2.0 +charset-normalizer==3.3.2 +cloudscraper==1.2.71 +freezegun==1.5.1 +html2text==2024.2.26 +html5lib==1.1 +idna==3.7 +parameterized==0.9.0 +pillow==10.3.0 +pushbullet.py==0.12.0 +pyparsing==3.1.2 +python-dateutil==2.9.0.post0 +python-magic==0.4.27 +pywin32==306 +requests==2.31.0 +requests-file==2.0.0 +requests-toolbelt==1.0.0 +setuptools==58.1.0 +six==1.16.0 +soupsieve==2.5 +urllib3==2.2.1 +webencodings==0.5.1 +websocket-client==1.8.0 diff --git a/root/app/calibre_info.py b/root/app/calibre_info.py new file mode 100644 index 0000000..ec4b3c2 --- /dev/null +++ b/root/app/calibre_info.py @@ -0,0 +1,89 @@ +import multiprocessing as mp +import os +from subprocess import call + +import ff_logging +import tomllib + + + +class CalibreInfo: + """ + This class represents the Calibre library information. + It reads the configuration from a TOML file and provides access to the Calibre library details. + """ + + def __init__(self, toml_path: str, manager: mp.Manager): + """ + Initialize the CalibreInfo object. + + Args: + toml_path (str): The path to the TOML configuration file. + manager (mp.Manager): A multiprocessing Manager instance. + """ + # Open and load the TOML configuration file + with open(toml_path, "rb") as file: + config = tomllib.load(file) + + # Get the 'calibre' section from the configuration + calibre_config = config.get("calibre", {}) + + # If the 'path' key is not present in the 'calibre' section, log a failure and raise an exception + if not calibre_config.get("path"): + message = "Calibre library location not set in the config file. Cannot search the calibre library or update it." + ff_logging.log_failure(message) + raise ValueError(message) + + # Set the Calibre library details + self.location = calibre_config.get("path") + self.username = calibre_config.get("username") + self.password = calibre_config.get("password") + self.default_ini = self._append_filename(calibre_config.get("default_ini"), "defaults.ini") + self.personal_ini = self._append_filename(calibre_config.get("personal_ini"), "personal.ini") + + # Create a lock for thread-safe operations + self.lock = manager.Lock() + + @staticmethod + def _append_filename(path: str, filename: str) -> str: + """ + Append the filename to the path if it's not already there. + + Args: + path (str): The original path. + filename (str): The filename to append. + + Returns: + str: The path with the filename appended. + """ + # If the path is not None and does not already end with the filename, append the filename + if path and not path.endswith(filename): + return os.path.join(path, filename) + return path + + # Check if Calibre is installed + def check_installed(self) -> bool: + try: + # Try to call calibredb + with open(os.devnull, "w") as nullout: + call(["calibredb"], stdout=nullout, stderr=nullout) + return True + except OSError: + # If calibredb is not found, log a failure and return False + ff_logging.log_failure( + "Calibredb is not installed on this system. Cannot search the calibre library or update it." + ) + return False + except Exception as e: + # If any other error occurs, log a failure + ff_logging.log_failure(f"Some other issue happened. {e}") + return False + + # String representation of the object + def __str__(self): + repr = f' --with-library "{self.location}"' + if self.username: + repr += f' --username "{self.username}"' + if self.password: + repr += f' --password "{self.password}"' + return repr \ No newline at end of file diff --git a/root/app/calibre_info_test.py b/root/app/calibre_info_test.py new file mode 100644 index 0000000..98ccbc8 --- /dev/null +++ b/root/app/calibre_info_test.py @@ -0,0 +1,196 @@ +from typing import NamedTuple, Union +from unittest.mock import MagicMock, mock_open, patch +from parameterized import parameterized +import unittest + +from calibre_info import CalibreInfo + + +class TestCalibreInfo(unittest.TestCase): + class ConfigCase(NamedTuple): + toml_path: str + config: str + expected_config: dict + + @parameterized.expand( + [ + # Test case: Valid configuration + ConfigCase( + toml_path="path/to/config.toml", + config=""" + [calibre] + path = "test_path" + username = "test_username" + password = "test_password" + default_ini = "test_default_ini" + personal_ini = "test_personal_ini" + """, + expected_config={ + "calibre": { + "path": "test_path", + "username": "test_username", + "password": "test_password", + "default_ini": "test_default_ini\\defaults.ini", + "personal_ini": "test_personal_ini\\personal.ini", + } + }, + ), + # Test case: default_ini and personal_ini already end with "/defaults.ini" and "/personal.ini" + ConfigCase( + toml_path="path/to/yet_another_config.toml", + config=""" + [calibre] + path = "yet_another_test_path" + username = "yet_another_test_username" + password = "yet_another_test_password" + default_ini = "yet_another_test_default_ini/defaults.ini" + personal_ini = "yet_another_test_personal_ini/personal.ini" + """, + expected_config={ + "calibre": { + "path": "yet_another_test_path", + "username": "yet_another_test_username", + "password": "yet_another_test_password", + "default_ini": "yet_another_test_default_ini/defaults.ini", + "personal_ini": "yet_another_test_personal_ini/personal.ini", + } + }, + ), + # Test case: Missing path in configuration + ConfigCase( + toml_path="path/to/another_config.toml", + config=""" + [calibre] + username = "another_test_username" + password = "another_test_password" + default_ini = "another_test_default_ini" + personal_ini = "another_test_personal_ini" + """, + expected_config=ValueError, + ), + ] + ) + @patch("builtins.open", new_callable=mock_open) + @patch("multiprocessing.Manager") + @patch("calibre_info.ff_logging.log_failure") + def test_calibre_info_init( + self, toml_path, config, expected_config, mock_log, mock_manager, mock_file + ): + mock_file.return_value.read.return_value = str(config).encode() + mock_manager.return_value = MagicMock() + if isinstance(expected_config, dict): + calibre_info = CalibreInfo(toml_path, mock_manager()) + self.assertEqual(calibre_info.location, expected_config["calibre"]["path"]) + self.assertEqual( + calibre_info.username, expected_config["calibre"]["username"] + ) + self.assertEqual( + calibre_info.password, expected_config["calibre"]["password"] + ) + self.assertEqual( + calibre_info.default_ini, expected_config["calibre"]["default_ini"] + ) + self.assertEqual( + calibre_info.personal_ini, + expected_config["calibre"]["personal_ini"], + ) + mock_log.assert_not_called() # Ensure that log_failure was not called + + else: + with self.assertRaises(expected_config): + CalibreInfo(toml_path, mock_manager) + mock_log.assert_called_once() # Ensure that log_failure was called once + + class CheckInstalledCase(NamedTuple): + call_return: Union[int, Exception] + expected_result: bool + + @parameterized.expand( + [ + CheckInstalledCase(call_return=0, expected_result=True), + CheckInstalledCase(call_return=OSError(), expected_result=False), + ] + ) + @patch("multiprocessing.Manager") + @patch("calibre_info.call") + @patch("builtins.open", new_callable=mock_open) + @patch("calibre_info.ff_logging.log_failure") + def test_check_installed( + self, + call_return: int, + expected_result: bool, + mock_log, + mock_file, + mock_call, + mock_manager, + ): + if isinstance(call_return, Exception): + mock_call.side_effect = call_return + else: + mock_call.return_value = call_return + mock_manager.return_value = MagicMock() + mock_file.return_value.read.return_value = str(""" + [calibre] + path = "test_path" + """).encode() + + calibre_info = CalibreInfo("path/to/config.toml", mock_manager()) + result = calibre_info.check_installed() + + self.assertEqual(result, expected_result) + if expected_result: + mock_log.assert_not_called() + else: + mock_log.assert_called_once() + + class StrRepresentationCase(NamedTuple): + location: str + username: str + password: str + expected_result: str + + @parameterized.expand( + [ + StrRepresentationCase( + location="test_path", + username=None, + password=None, + expected_result=' --with-library "test_path"', + ), + StrRepresentationCase( + location="test_path", + username="test_user", + password=None, + expected_result=' --with-library "test_path" --username "test_user"', + ), + StrRepresentationCase( + location="test_path", + username="test_user", + password="test_pass", + expected_result=' --with-library "test_path" --username "test_user" --password "test_pass"', + ), + ] + ) + @patch("multiprocessing.Manager") + @patch("builtins.open", new_callable=mock_open) + def test_str_representation( + self, location, username, password, expected_result, mock_file, mock_manager + ): + mock_manager.return_value = MagicMock() + + mock_file.return_value.read.return_value = str(""" + [calibre] + path = "test_path" + """).encode() + calibre_info = CalibreInfo("path/to/config.toml", mock_manager()) + calibre_info.location = location + calibre_info.username = username + calibre_info.password = password + + result = str(calibre_info) + + self.assertEqual(result, expected_result) + + +if __name__ == "__main__": + unittest.main() diff --git a/root/app/fanfic_info.py b/root/app/fanfic_info.py new file mode 100644 index 0000000..82ed78b --- /dev/null +++ b/root/app/fanfic_info.py @@ -0,0 +1,65 @@ +from subprocess import CalledProcessError, check_output, PIPE, STDOUT + +from typing import Optional + +import calibre_info +import ff_logging + +class FanficInfo: + def __init__( + self, + url: str, + site: str, + calibre_id: Optional[str] = None, + repeats: Optional[int] = 0, + max_repeats: Optional[int] = 10, + behavior: Optional[str] = None, + title: Optional[str] = None, + ): + self.url: str = url + self.calibre_id: Optional[str] = calibre_id + self.site: str = site + self.repeats: Optional[int] = repeats + self.max_repeats: Optional[int] = max_repeats + self.behavior: Optional[str] = behavior + self.title: Optional[str] = title + + # Increment the repeat counter + def increment_repeat(self) -> None: + if self.repeats is not None: + self.repeats += 1 + + # Check if the URL has been repeated too many times + def reached_maximum_repeats(self) -> bool: + if self.repeats is not None and self.max_repeats is not None: + return self.repeats >= self.max_repeats + return False + + # Check if the story is in the Calibre database + def get_id_from_calibredb( + self, calibre_information: calibre_info.CalibreInfo + ) -> bool: + try: + # Lock the Calibre database to prevent concurrent modifications + with calibre_information.lock: + # Search for the story in the Calibre database + story_id = check_output( + f'calibredb search "Identifiers:{self.url}" {calibre_information}', + shell=True, + stderr=STDOUT, + stdin=PIPE, + ).decode("utf-8") + + # If the story is found, update the id and log a message + self.calibre_id = story_id + ff_logging.log(f"\tStory is in Calibre with Story ID: {self.calibre_id}", "OKBLUE") + return True + except CalledProcessError: + # If the story is not found, log a warning + ff_logging.log("\tStory not in Calibre", "WARNING") + return False + + def __eq__(self, other: object) -> bool: + if not isinstance(other, FanficInfo): + return False + return self.url == other.url and self.site == other.site and self.calibre_id == other.calibre_id \ No newline at end of file diff --git a/root/app/fanfic_info_test.py b/root/app/fanfic_info_test.py new file mode 100644 index 0000000..ebd900f --- /dev/null +++ b/root/app/fanfic_info_test.py @@ -0,0 +1,49 @@ +import unittest +from unittest.mock import Mock, patch, mock_open, MagicMock +from fanfic_info import FanficInfo + +class TestFanficInfo(unittest.TestCase): + def setUp(self): + self.fanfic_info = FanficInfo( + url="https://www.fanfiction.net/s/1234", + site="ffnet", + calibre_id="1234", + repeats=0, + max_repeats=10, + behavior="update", + title="Test Story", + ) + + def test_increment_repeat(self): + self.fanfic_info.increment_repeat() + self.assertEqual(self.fanfic_info.repeats, 1) + + def test_reached_maximum_repeats(self): + self.fanfic_info.repeats = 10 + self.assertTrue(self.fanfic_info.reached_maximum_repeats()) + + @patch("fanfic_info.check_output") + @patch("builtins.open", new_callable=mock_open) + @patch("fanfic_info.ff_logging.log") + def test_get_id_from_calibredb(self, mock_ff_logger, mock_open, mock_check_output): + mock_check_output.return_value = b"1234" + calibre_information = Mock() + calibre_information.lock = MagicMock() + self.assertTrue(self.fanfic_info.get_id_from_calibredb(calibre_information)) + self.assertEqual(self.fanfic_info.calibre_id, "1234") + mock_ff_logger.assert_called_once_with("\tStory is in Calibre with Story ID: 1234", "OKBLUE") + + def test_eq(self): + other_fanfic_info = FanficInfo( + url="https://www.fanfiction.net/s/1234", + site="ffnet", + calibre_id="1234", + repeats=0, + max_repeats=10, + behavior="update", + title="Test Story", + ) + self.assertTrue(self.fanfic_info == other_fanfic_info) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/root/app/fanficdownload.py b/root/app/fanficdownload.py index abe4efe..0bc2118 100644 --- a/root/app/fanficdownload.py +++ b/root/app/fanficdownload.py @@ -1,449 +1,69 @@ -from fanficfare import geturls -from os import listdir, remove, rename, utime, devnull -from os.path import isfile, join -from subprocess import check_output, STDOUT, call, PIPE -import logging -from optparse import OptionParser -import re -from configparser import ConfigParser -from tempfile import mkdtemp -from shutil import rmtree, copyfile -import socket -from time import strftime, localtime -import os -import errno - -from multiprocessing import Pool - -logging.getLogger("fanficfare").setLevel(logging.ERROR) - - -class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - - -def log(msg, color=None, output=True): - if color: - col = bcolors.HEADER - if color == 'BLUE': - col = bcolors.OKBLUE - elif color == 'GREEN': - col = bcolors.OKGREEN - elif color == 'WARNING': - col = bcolors.WARNING - elif color == 'FAIL': - col = bcolors.FAIL - elif color == 'BOLD': - col = bcolors.BOLD - elif color == 'UNDERLINE': - col = bcolors.UNDERLINE - line = '{}{}{}: \t {}{}{}'.format( - bcolors.BOLD, - strftime( - '%m/%d/%Y %H:%M:%S', - localtime()), - bcolors.ENDC, - col, - msg, - bcolors.ENDC) - else: - line = '{}{}{}: \t {}'.format( - bcolors.BOLD, - strftime( - '%m/%d/%Y %H:%M:%S', - localtime()), - bcolors.ENDC, - msg) - if output: - print(line) - return "" - else: - return line + "\n" - - -def touch(fname, times=None): - with open(fname, 'a'): - utime(fname, times) - - -url_parsers = [(re.compile(r'(fanfiction.net/s/\d*/?).*'), "www."), #ffnet - (re.compile(r'(archiveofourown.org/works/\d*)/?.*'), ""), #ao3 - (re.compile(r'(fictionpress.com/s/\d*)/?.*'), ""), #fictionpress - (re.compile(r'(royalroad.com/fiction/\d*)/?.*'), ""), #royalroad - (re.compile(r'https?://(.*)'), "")] #other sites -story_name = re.compile(r'(.*)-.*') - -equal_chapters = re.compile(r'.* already contains \d* chapters.') -chapter_difference = re.compile( - r'.* contains \d* chapters, more than source: \d*.') -bad_chapters = re.compile( - r".* doesn't contain any recognizable chapters, probably from a different source. Not updating.") -no_url = re.compile(r'No story URL found in epub to update.') -more_chapters = re.compile( - r".*File\(.*\.epub\) Updated\(.*\) more recently than Story\(.*\) - Skipping") - - -def parse_url(url): - for cur_parser, cur_prefix in url_parsers: - if cur_parser.search(url): - url = cur_prefix + cur_parser.search(url).group(1) - return url - return url - - -def get_files(mypath, filetype=None, fullpath=False): - ans = [] - if filetype: - ans = [f for f in listdir(mypath) if isfile( - join(mypath, f)) and f.endswith(filetype)] - else: - ans = [f for f in listdir(mypath) if isfile(join(mypath, f))] - if fullpath: - return [join(mypath, f) for f in ans] - else: - return ans - - -def check_regexes(output): - if equal_chapters.search(output): - raise ValueError( - "Issue with story, site is broken. Story likely hasn't updated on site yet.") - if bad_chapters.search(output): - raise ValueError( - "Something is messed up with the site or the epub. No chapters found.") - if no_url.search(output): - raise ValueError("No URL in epub to update from. Fix the metadata.") - - -def downloader(args): - url, inout_file, path, live = args - loc = mkdtemp() - output = "" - output += log("Working with url {}".format(url), 'HEADER', live) - if 'fanfiction.net' in url: - output += log("Skipping FFNET for now due to flaresolverr bug.", 'WARNING', live) - storyId = None - try: - if path: - try: - storyId = check_output( - 'calibredb search "Identifiers:{}" {}'.format( - url, path), shell=True, stderr=STDOUT, stdin=PIPE, ).decode('utf-8') - output += log("\tStory is in calibre with id {}".format(storyId), 'BLUE', live) - output += log("\tExporting file", 'BLUE', live) - res = check_output( - 'calibredb export {} --dont-save-cover --dont-write-opf --single-dir --to-dir "{}" {}'.format( - storyId, loc, path), shell=True, stdin=PIPE, stderr=STDOUT).decode('utf-8') - cur = get_files(loc, ".epub", True)[0] - output += log( - '\tDownloading with fanficfare, updating file "{}"'.format(cur), - 'GREEN', - live) - moving = "" - except BaseException: - # story is not in calibre - output += log("\tStory is not in Calibre", 'WARNING', live) - cur = url - moving = 'cd "{}" && '.format(loc) - copyfile("/config/personal.ini", "{}/personal.ini".format(loc)) - copyfile("/config/defaults.ini", "{}/defaults.ini".format(loc)) - output += log('\tRunning: {}python3 -m fanficfare.cli -u "{}" --update-cover --non-interactive'.format( - moving, cur), 'BLUE', live) - res = check_output('{}python3 -m fanficfare.cli -u "{}" --update-cover --non-interactive --config={}/personal.ini'.format( - moving, cur, loc), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8') - check_regexes(res) - if chapter_difference.search(res) or more_chapters.search(res): - output += log("\tForcing download update due to:", - 'WARNING', live) - for line in res.split("\n"): - if line: - output += log("\t\t{}".format(line), 'WARNING', live) - res = check_output( - '{}python3 -m fanficfare.cli -u "{}" --force --update-cover --non-interactive --config={}/personal.ini'.format( - moving, cur, loc), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8') - check_regexes(res) - cur = get_files(loc, '.epub', True)[0] - - if storyId: - output += log("\tRemoving {} from library".format(storyId), - 'BLUE', live) - try: - res = check_output( - 'calibredb remove {} {}'.format( - path, - storyId), - shell=True, - stderr=STDOUT, - stdin=PIPE, - ).decode('utf-8') - except BaseException: - if not live: - print(output.strip()) - raise - - output += log("\tAdding {} to library".format(cur), 'BLUE', live) - try: - res = check_output( - 'calibredb add -d {} "{}"'.format(path, cur), shell=True, stderr=STDOUT, stdin=PIPE, ).decode('utf-8') - except Exception as e: - output += log(e) - if not live: - print(output.strip()) - raise - try: - res = check_output( - 'calibredb search "Identifiers:{}" {}'.format( - url, path), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8') - output += log("\tAdded {} to library with id {}".format(cur, - res), 'GREEN', live) - except BaseException: - output += log( - "It's been added to library, but not sure what the ID is.", - 'WARNING', - live) - output += log("Added /Story-file to library with id 0", 'GREEN', live) - remove(cur) - else: - res = check_output( - 'cd "{}" && fanficfare -u "{}" --update-cover'.format( - loc, url), shell=True, stderr=STDOUT, stdin=PIPE).decode('utf-8') - check_regexes(res) - cur = get_files(loc, '.epub', True)[0] - name = get_files(loc, '.epub', False)[0] - rename(cur, name) - output += log( - "Downloaded story {} to {}".format( - story_name.search(name).group(1), - name), - 'GREEN', - live) - if not live: - print(output.strip()) - rmtree(loc) - except Exception as e: - output += log("Exception: {}".format(e), 'FAIL', live) - if not live: - print(output.strip()) - try: - rmtree(loc) - except BaseException: - pass - with open(inout_file, "a") as fp: - fp.write("{}\n".format(url)) - - -def main(user, password, server, label, inout_file, path, lib_user, lib_password, live): - - if path: - path = '--with-library "{}" --username {} --password {}'.format( - path, lib_user, lib_password) - try: - with open(devnull, 'w') as nullout: - call(['calibredb'], stdout=nullout, stderr=nullout) - except OSError as e: - if e.errno == errno.ENOENT: - log("Calibredb is not installed on this system. Cannot search the calibre library or update it.", 'FAIL') - return - - touch(inout_file) - - with open(inout_file, "r") as fp: - urls = set([x.replace("\n", "") for x in fp.readlines()]) - - with open(inout_file, "w") as fp: - fp.write("") - - try: - socket.setdefaulttimeout(55) - urls |= geturls.get_urls_from_imap(server, user, password, label) - socket.setdefaulttimeout(None) - except BaseException: - with open(inout_file, "w") as fp: - for cur in urls: - fp.write("{}\n".format(cur)) - return - - if not urls: - return - urls = set(parse_url(x) for x in urls) - log("URLs to parse ({}):".format(len(urls)), 'HEADER') - for url in urls: - log("\t{}".format(url), 'BLUE') - if len(urls) == 1: - downloader([list(urls)[0], inout_file, path, True]) - else: - for url in urls: - downloader([url, inout_file, path, True]) - with open(inout_file, "r") as fp: - urls = set([x.replace("\n", "") for x in fp.readlines()]) - with open(inout_file, "w") as fp: - fp.writelines(["{}\n".format(x) for x in urls]) - return - +import argparse +import multiprocessing as mp +import signal +import sys + +import calibre_info +import ff_waiter +import pushbullet_notification +import regex_parsing +import url_ingester +import url_worker if __name__ == "__main__": - option_parser = OptionParser(usage="usage: %prog [flags]") - - option_parser.add_option( - '-u', - '--user', - action='store', - dest='user', - help='Email Account Username. Required.') - - option_parser.add_option( - '-p', - '--password', - action='store', - dest='password', - help='Email Account Password. Required.') - - option_parser.add_option( - '-s', - '--server', - action='store', - dest='server', - default="imap.gmail.com", - help='Email IMAP Server. Default is "imap.gmail.com".') - - option_parser.add_option( - '-m', - '--mailbox', - action='store', - dest='mailbox', - default='INBOX', - help='Email Label. Default is "INBOX".') - - option_parser.add_option( - '-l', - '--library', - action='store', - dest='library', - help="calibre library db location. If none is passed, then this merely scrapes the email and error file for new stories and downloads them into the current directory.") - - option_parser.add_option( - '-i', - '--input', - action='store', - dest='input', - default="./fanfiction.txt", - help="Error file. Any urls that fail will be output here, and file will be read to find any urls that failed previously. If file does not exist will create. File is overwitten every time the program is run.") - - option_parser.add_option( - '-c', - '--config', - action='store', - dest='config', - help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.') - - option_parser.add_option( - '-o', - '--output', - action='store_true', - dest='live', - help='Include this if you want all the output to be saved and posted live. Useful when multithreading.') - - option_parser.add_option( - '-q', - '--libuser', - action='store', - dest='libuser', - help='Calibre User. Required.') - - option_parser.add_option( - '-w', - '--libpassword', - action='store', - dest='libpassword', - help='Calibre Password. Required.') - - (options, args) = option_parser.parse_args() - - if options.config: - touch(options.config) - config = ConfigParser(allow_no_value=True) - config.read(options.config) - - def updater(option, newval): return newval if newval != "" else option - try: - options.user = updater( - options.user, config.get( - 'login', 'user').strip()) - except BaseException: - pass - - try: - options.password = updater( - options.password, config.get( - 'login', 'password').strip()) - except BaseException: - pass - - try: - options.libuser = updater( - options.libuser, config.get( - 'login', 'libuser').strip()) - except BaseException: - pass - - try: - options.libpassword = updater( - options.libpassword, config.get( - 'login', 'libpassword').strip()) - except BaseException: - pass - - try: - options.server = updater( - options.server, config.get( - 'login', 'server').strip()) - except BaseException: - pass - - try: - options.mailbox = updater( - options.mailbox, config.get( - 'login', 'mailbox').strip()) - except BaseException: - pass - - try: - options.library = updater( - options.library, config.get( - 'locations', 'library').strip()) - except BaseException: - pass - - try: - options.input = updater( - options.input, config.get( - 'locations', 'input').strip()) - except BaseException: - pass - - try: - options.live = updater( - options.live, config.getboolean( - 'output', 'live').strip()) - except BaseException: - pass - - if not (options.user or options.password): - raise ValueError("User or Password not given") - main( - options.user, - options.password, - options.server, - options.mailbox, - options.input, - options.library, - options.libuser, - options.libpassword, - options.live) + # Create an argument parser + parser = argparse.ArgumentParser(description="Process input arguments.") + parser.add_argument( + "--config", default="../config.default/config.toml", help="The location of the config.toml file" + ) + + # Parse the command line arguments + args = parser.parse_args() + + # Initialize CalibreInfo, EmailInfo, and PushbulletNotification objects + email_info = url_ingester.EmailInfo(args.config) + pushbullet_info = pushbullet_notification.PushbulletNotification(args.config) + + with mp.Manager() as manager: + # Create a dictionary of multiprocessing queues for each URL parser + queues = {site: manager.Queue() for site in regex_parsing.url_parsers.keys()} + waiting_queue = manager.Queue() + + cdb_info = calibre_info.CalibreInfo(args.config, manager) + + # Check if Calibre is installed + cdb_info.check_installed() + + def signal_handler(sig, frame): + """Handle received signals and terminate the program.""" + email_watcher.terminate() # Terminate the email watcher process + waiting_watcher.terminate() # Terminate the waiting watcher process + pool.terminate() # Terminate all worker processes in the pool + sys.exit(0) # Exit the program + + # Set the signal handler for SIGTERM + signal.signal(signal.SIGTERM, signal_handler) + + # Start a new process to watch the email account for new URLs + email_watcher = mp.Process( + target=url_ingester.email_watcher, args=(email_info, queues) + ) + email_watcher.start() + + waiting_watcher = mp.Process( + target=ff_waiter.wait_processor, args=(queues, waiting_queue) + ) + waiting_watcher.start() + + # Create a pool of worker processes to process URLs from the queues + workers = [ + (queues[site], cdb_info, pushbullet_info, waiting_queue) + for site in queues.keys() + ] + with mp.Pool(len(queues)) as pool: + pool.starmap(url_worker.url_worker, workers) + + # Wait for the email watcher process to finish + email_watcher.join() + # Wait for the waiting watcher process to finish + waiting_watcher.join() diff --git a/root/app/ff_logging.py b/root/app/ff_logging.py new file mode 100644 index 0000000..268f9fc --- /dev/null +++ b/root/app/ff_logging.py @@ -0,0 +1,36 @@ +from time import localtime, strftime + + +class bcolors: + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +color_map = { + "HEADER": bcolors.HEADER, + "OKBLUE": bcolors.OKBLUE, + "OKGREEN": bcolors.OKGREEN, + "WARNING": bcolors.WARNING, + "FAIL": bcolors.FAIL, + "ENDC": bcolors.ENDC, + "BOLD": bcolors.BOLD, + "UNDERLINE": bcolors.UNDERLINE, +} + + +# Logging Function +def log(msg, color=None) -> None: + using_col = color_map.get(color, bcolors.BOLD) + print( + f'{bcolors.BOLD}{strftime("%Y-%m-%d %I:%M:%S %p", localtime())}{bcolors.ENDC} - {using_col}{msg}{bcolors.ENDC}' + ) + + +def log_failure(msg): + log(msg, "FAIL") diff --git a/root/app/ff_logging_test.py b/root/app/ff_logging_test.py new file mode 100644 index 0000000..6edeab6 --- /dev/null +++ b/root/app/ff_logging_test.py @@ -0,0 +1,80 @@ +from typing import NamedTuple +import unittest +from unittest.mock import patch + +from freezegun import freeze_time +from parameterized import parameterized + +import ff_logging + + +class TestLogFunction(unittest.TestCase): + class CheckLogHeaderTestCase(NamedTuple): + log_type: str + message: str + expected_header: str + expected_color_code: str + + @parameterized.expand( + [ + CheckLogHeaderTestCase( + log_type="header", + message="testing header", + expected_header="HEADER", + expected_color_code="95", + ), + CheckLogHeaderTestCase( + log_type="okblue", + message="testing okblue", + expected_header="OKBLUE", + expected_color_code="94", + ), + CheckLogHeaderTestCase( + log_type="okgreen", + message="testing okgreen", + expected_header="OKGREEN", + expected_color_code="92", + ), + CheckLogHeaderTestCase( + log_type="warning", + message="testing warning", + expected_header="WARNING", + expected_color_code="93", + ), + CheckLogHeaderTestCase( + log_type="fail", + message="testing fail", + expected_header="FAIL", + expected_color_code="91", + ), + CheckLogHeaderTestCase( + log_type="endc", + message="testing endc", + expected_header="ENDC", + expected_color_code="0", + ), + CheckLogHeaderTestCase( + log_type="bold", + message="testing bold", + expected_header="BOLD", + expected_color_code="1", + ), + CheckLogHeaderTestCase( + log_type="underline", + message="testing underline", + expected_header="UNDERLINE", + expected_color_code="4", + ), + ] + ) + @freeze_time("2021-01-01 12:00:00") + @patch("builtins.print") + def test_log_header(self, name, message, color, code, mock_print): + ff_logging.log(message, color) + mock_print.assert_called_once_with( + f"\x1b[1m2021-01-01 07:00:00\x1b[0m - \x1b[{code}m{message}\x1b[0m" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/root/app/ff_waiter.py b/root/app/ff_waiter.py new file mode 100644 index 0000000..3b92416 --- /dev/null +++ b/root/app/ff_waiter.py @@ -0,0 +1,60 @@ +import multiprocessing as mp +import threading +from time import sleep + +import fanfic_info +import ff_logging + +def insert_after_time(queue: mp.Queue, fanfic: fanfic_info.FanficInfo) -> None: + """ + Inserts a fanfic into the queue after a delay. + + Args: + queue (mp.Queue): The queue to insert the fanfic into. + fanfic (fanfic_info.FanficInfo): The fanfic to insert. + """ + # Insert the fanfic into the queue + queue.put(fanfic) + +def process_fanfic(fanfic: fanfic_info.FanficInfo, processor_queues: dict[str, mp.Queue]) -> threading.Timer: + """ + Processes a single fanfic. It calculates a delay based on the number of repeats for the fanfic, + logs a warning message, and starts a timer to insert the fanfic into the appropriate processor queue after the delay. + + Args: + fanfic (fanfic_info.FanficInfo): The fanfic to process. + processor_queues (dict[str, mp.Queue]): A dictionary of processor queues. + + Returns: + threading.Timer: The timer that was started. + """ + # Calculate the delay based on the number of repeats for the fanfic + delay = 60 * fanfic.repeats + # Log a warning message indicating that we're waiting for a certain delay + ff_logging.log(f"Waiting {fanfic.repeats} minutes for {fanfic.url} in queue {fanfic.site}", "WARNING") + # Start a timer to insert the fanfic into the appropriate processor queue after the delay + timer = threading.Timer(delay, insert_after_time, args=(processor_queues[fanfic.site], fanfic)) + timer.start() + + return timer + +def wait_processor(processor_queues: dict[str, mp.Queue], waiting_queue: mp.Queue): + """ + Processes the waiting queue. + + Args: + processor_queues (dict[str, mp.Queue]): A dictionary of processor queues. + waiting_queue (mp.Queue): The waiting queue. + """ + while True: + # Get a fanfic from the waiting queue + fanfic: fanfic_info.FanficInfo = waiting_queue.get() + + # If the fanfic is None, this signals that we should stop processing the waiting queue + if fanfic is None: + break + + # Process the fanfic + process_fanfic(fanfic, processor_queues) + + sleep(5) # Sleep for 5 seconds to avoid busy-waiting diff --git a/root/app/ff_waiter_test.py b/root/app/ff_waiter_test.py new file mode 100644 index 0000000..5cb787e --- /dev/null +++ b/root/app/ff_waiter_test.py @@ -0,0 +1,46 @@ +import multiprocessing as mp +from typing import NamedTuple +import unittest +from unittest.mock import patch, Mock + +from freezegun import freeze_time +from parameterized import parameterized + +import fanfic_info +import ff_waiter + + +class TestWaitFunction(unittest.TestCase): + class CheckTimerProcessingTestCase(NamedTuple): + repeats: int + expected_time: int + + @parameterized.expand( + [ + CheckTimerProcessingTestCase(repeats=0, expected_time=0), + CheckTimerProcessingTestCase(repeats=1, expected_time=60), + CheckTimerProcessingTestCase(repeats=2, expected_time=120), + CheckTimerProcessingTestCase(repeats=3, expected_time=180), + CheckTimerProcessingTestCase(repeats=4, expected_time=240), + CheckTimerProcessingTestCase(repeats=5, expected_time=300), + ] + ) + @freeze_time("2021-01-01 12:00:00") + @patch("builtins.print") + @patch("threading.Timer") + def test_wait(self, repeats, expected_time, mock_timer, mock_print): + fanfic = fanfic_info.FanficInfo(site="site", url="url", repeats=repeats) + queue = mp.Queue() + processor_queues = {"site": queue} + ff_waiter.process_fanfic(fanfic, processor_queues) + mock_print.assert_called_once_with( + f"\x1b[1m2021-01-01 07:00:00\x1b[0m - \x1b[93mWaiting {repeats} minutes for url in queue site\x1b[0m" + ) + mock_timer.assert_called_once_with( + expected_time, ff_waiter.insert_after_time, args=(queue, fanfic) + ) + mock_timer.return_value.start.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/root/app/notifications.py b/root/app/notifications.py deleted file mode 100644 index 340f772..0000000 --- a/root/app/notifications.py +++ /dev/null @@ -1,8 +0,0 @@ -class Notification(): - def __init__(self): - - self.name="Fanfiction" - - def send_notification(self, title, text): - return - diff --git a/root/app/pushbullet_notification.py b/root/app/pushbullet_notification.py new file mode 100644 index 0000000..1eaf8bd --- /dev/null +++ b/root/app/pushbullet_notification.py @@ -0,0 +1,47 @@ +from pushbullet import InvalidKeyError, Pushbullet, PushbulletError + +import ff_logging +import tomllib + +class PushbulletNotification: + # Initialization Function + def __init__(self, toml_path: str): + # Load the configuration from the TOML file + with open(toml_path, "rb") as file: + config = tomllib.load(file) + + # Extract the Pushbullet configuration + pushbullet_config = config["pushbullet"] + + # Check if Pushbullet is enabled + self.enabled = pushbullet_config["enabled"] + if not self.enabled: + return + + try: + # Initialize the Pushbullet client + self.pb = Pushbullet(pushbullet_config["api_key"]) + + # If a device is specified, get the device + device = pushbullet_config["device"] + if device: + self.pb = self.pb.get_device(device) + except InvalidKeyError: + message = "Invalid Pushbullet API key in the config file. Cannot send notifications." + ff_logging.log_failure(message) + self.enabled = False + except PushbulletError as e: + message = f"Pushbullet error: {e}. Cannot send notifications." + ff_logging.log_failure(message) + self.enabled = False + + # Function to send a notification + def send_notification(self, title: str, body: str) -> None: + # If Pushbullet is enabled, send the notification + if self.enabled: + try: + ff_logging.log(f"\tSending Pushbullet notification: {title} - {body}", "OKBLUE") + self.pb.push_note(title, body) + except PushbulletError as e: + message = f"\tFailed to send Pushbullet notification: {e}" + ff_logging.log_failure(message) \ No newline at end of file diff --git a/root/app/pushbullet_notification_test.py b/root/app/pushbullet_notification_test.py new file mode 100644 index 0000000..0a41727 --- /dev/null +++ b/root/app/pushbullet_notification_test.py @@ -0,0 +1,194 @@ +from typing import NamedTuple, Optional, Type +import unittest +from unittest.mock import MagicMock, patch + +from parameterized import parameterized +from pushbullet import InvalidKeyError, PushbulletError + +from pushbullet_notification import PushbulletNotification + + +class TestPushbulletNotification(unittest.TestCase): + class InitTestCase(NamedTuple): + enabled: bool + api_key: str + device: str + side_effect: Optional[Type[BaseException]] + expected: bool + + @parameterized.expand( + [ + # Test case: Pushbullet is enabled and API key is valid, device is specified + InitTestCase( + enabled=True, + api_key="valid_api_key", + device="device1", + side_effect=None, + expected=True, + ), + # Test case: Pushbullet is enabled and API key is valid, device is not specified + InitTestCase( + enabled=True, + api_key="valid_api_key", + device="", + side_effect=None, + expected=True, + ), + # Test case: Pushbullet is enabled but API key is invalid, device is specified + InitTestCase( + enabled=True, + api_key="invalid_api_key", + device="device1", + side_effect=InvalidKeyError, + expected=False, + ), + # Test case: Pushbullet is enabled but there is a Pushbullet error, device is not specified + InitTestCase( + enabled=True, + api_key="valid_api_key", + device="", + side_effect=PushbulletError, + expected=False, + ), + # Test case: Pushbullet is disabled, device is specified + InitTestCase( + enabled=False, + api_key="valid_api_key", + device="device1", + side_effect=None, + expected=False, + ), + ] + ) + @patch("pushbullet_notification.Pushbullet") + @patch("pushbullet_notification.ff_logging.log_failure") + @patch("pushbullet_notification.ff_logging.log") + @patch("pushbullet_notification.tomllib.load") + @patch("builtins.open", new_callable=MagicMock) + def test_init( + self, + enabled, + api_key, + device, + side_effect, + expected_enabled, + mock_open, + mock_load, + mock_log, + mock_log_failure, + mock_pushbullet, + ): + # Setup: Mock the configuration and the Pushbullet client + mock_load.return_value = { + "pushbullet": {"enabled": enabled, "api_key": api_key, "device": device} + } + mock_pushbullet.side_effect = side_effect if side_effect else MagicMock() + + # Execution: Create a PushbulletNotification instance + pb_notification = PushbulletNotification("path/to/config.toml") + + # Assertion: Check that the 'enabled' attribute is as expected + self.assertEqual(pb_notification.enabled, expected_enabled) + + # Assertion: Check that the logging functions were called as expected + if side_effect is InvalidKeyError: + mock_log_failure.assert_called_once_with( + "Invalid Pushbullet API key in the config file. Cannot send notifications." + ) + elif side_effect is PushbulletError: + mock_log_failure.assert_called_once() + self.assertTrue("Pushbullet error:" in mock_log_failure.call_args[0][0]) + else: + mock_log.assert_not_called() + mock_log_failure.assert_not_called() + + class SendingNotificationTestCase(NamedTuple): + title: str + body: str + side_effect: Optional[Type[BaseException]] + enabled: bool + log_called: bool + log_failure_called: bool + + @parameterized.expand( + [ + # Test case: Pushbullet is enabled and the notification is sent successfully + SendingNotificationTestCase( + title="title1", + body="body1", + side_effect=None, + enabled=True, + log_called=True, + log_failure_called=False, + ), + # Test case: Pushbullet is enabled but there is a Pushbullet error when sending the notification + SendingNotificationTestCase( + title="title2", + body="body2", + side_effect=PushbulletError, + enabled=True, + log_called=True, + log_failure_called=True, + ), + # Test case: Pushbullet is disabled + SendingNotificationTestCase( + title="title3", + body="body3", + side_effect=None, + enabled=False, + log_called=False, + log_failure_called=False, + ), + ] + ) + @patch("pushbullet_notification.Pushbullet") + @patch("pushbullet_notification.ff_logging.log_failure") + @patch("pushbullet_notification.ff_logging.log") + @patch("pushbullet_notification.tomllib.load") + @patch("builtins.open", new_callable=MagicMock) + def test_send_notification( + self, + title, + body, + side_effect, + enabled, + log_called, + log_failure_called, + mock_open, + mock_load, + mock_log, + mock_log_failure, + mock_pushbullet, + ): + # Setup: Create a PushbulletNotification instance with a mock Pushbullet client + mock_load.return_value = { + "pushbullet": {"enabled": enabled, "api_key": "valid", "device": "device"} + } + pb_notification = PushbulletNotification("path/to/config.toml") + pb_notification.pb = mock_pushbullet + pb_notification.enabled = enabled + mock_pushbullet.push_note.return_value = None + mock_pushbullet.push_note.side_effect = side_effect + + # Execution: Call send_notification + pb_notification.send_notification(title, body) + # Assertion: Check that the logging functions were called as expected + if log_called: + mock_log.assert_called_once_with( + f"\tSending Pushbullet notification: {title} - {body}", "OKBLUE" + ) + else: + mock_log.assert_not_called() + + if log_failure_called: + mock_log_failure.assert_called_once() + self.assertTrue( + "\tFailed to send Pushbullet notification:" + in mock_log_failure.call_args[0][0] + ) + else: + mock_log_failure.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/root/app/regex_parsing.py b/root/app/regex_parsing.py new file mode 100644 index 0000000..f3b3467 --- /dev/null +++ b/root/app/regex_parsing.py @@ -0,0 +1,95 @@ +import re + +import fanfic_info +import ff_logging + +# Define regular expressions for different URL formats +url_parsers = { + "ffnet": (re.compile(r"(fanfiction.net/s/\d*/?).*"), "www."), + "ao3": (re.compile(r"(archiveofourown.org/works/\d*)/?.*"), ""), + "fictionpress": (re.compile(r"(fictionpress.com/s/\d*)/?.*"), ""), + "royalroad": (re.compile(r"(royalroad.com/fiction/\d*)/?.*"), ""), + "sv": (re.compile(r"(forums.sufficientvelocity.com/threads/.*\.\d*)/?.*"), ""), + "sb": (re.compile(r"(forums.spacebattles.com/threads/.*\.\d*)/?.*"), ""), + "qq": (re.compile(r"(forum.questionablequesting.com/threads/.*\.\d*)/?.*"), ""), + "other": (re.compile(r"https?://(.*)"), ""), +} + +# Define regular expressions for different story formats +story_name = re.compile(r"(.*?)-.*") +equal_chapters = re.compile(r".* already contains \d* chapters.") +chapter_difference = re.compile(r".* contains \d* chapters, more than source: \d*.") +bad_chapters = re.compile( + r".* doesn't contain any recognizable chapters, probably from a different source. Not updating." +) +no_url = re.compile(r"No story URL found in epub to update.") +more_chapters = re.compile( + r".*File\(.*\.epub\) Updated\(.*\) more recently than Story\(.*\) - Skipping" +) +failed_login = re.compile(r".*Login Failed on non-interactive process. Set username and password in personal.ini.") +bad_request = re.compile(r".*400 Client Error: Bad Request for url:.*") + +def extract_filename(filename: str) -> str: + """Extract the title from the filename.""" + match = story_name.search(filename) + if match: + return match.group(1).strip() + return filename + + +def check_regexes(output: str, regex: re.Pattern, message: str) -> bool: + """Check if the output matches the given regular expression.""" + match = regex.search(output) + if match: + ff_logging.log_failure(message) + return True + return False + + +def check_failure_regexes(output: str) -> bool: + """Check if the output matches any of the failure regular expressions.""" + return not any( + check_regexes(output, regex, message) + for regex, message in [ + ( + equal_chapters, + "Issue with story, site is broken. Story likely hasn't updated on site yet.", + ), + ( + bad_chapters, + "Something is messed up with the site or the epub. No chapters found.", + ), + (no_url, "No URL in epub to update from. Fix the metadata."), + (failed_login, "Login failed. Check your username and password."), + (bad_request, "Bad request. Check the URL.") + ] + ) + + +def check_forceable_regexes(output: str) -> bool: + """Check if the output matches any of the forceable regular expressions.""" + return any( + check_regexes(output, regex, message) + for regex, message in [ + ( + chapter_difference, + "Chapter difference between source and destination. Forcing update.", + ), + ( + more_chapters, + "File has been updated more recently than the story, this is likely a metadata bug. Forcing update.", + ), + ] + ) + + +def generate_FanficInfo_from_url(url: str) -> fanfic_info.FanficInfo: + """Generate a FanficInfo object from a URL.""" + site = "other" + for current_site, (current_parser, current_prefix) in url_parsers.items(): + match = current_parser.search(url) + if match: + url = current_prefix + match.group(1) + site = current_site + break + return fanfic_info.FanficInfo(url, site) diff --git a/root/app/regex_parsing_test.py b/root/app/regex_parsing_test.py new file mode 100644 index 0000000..c05f46e --- /dev/null +++ b/root/app/regex_parsing_test.py @@ -0,0 +1,169 @@ +import re +import unittest +from typing import NamedTuple, Optional +from unittest.mock import patch + +from parameterized import parameterized + +import fanfic_info +import regex_parsing + + +class TestRegexParsing(unittest.TestCase): + class CheckFilenameExtractionTestCase(NamedTuple): + input: str + expected: str + + @parameterized.expand( + [ + # Test case: Extract 'story' from 'story-name-1234' + CheckFilenameExtractionTestCase(input="story-name-1234", expected="story"), + # Test case: Extract 'author' from 'author-name' + CheckFilenameExtractionTestCase(input="author-name", expected="author"), + ] + ) + def test_extract_filename(self, input, expected): + self.assertEqual(regex_parsing.extract_filename(input), expected) + + class CheckRegexesTestCase(NamedTuple): + output: str + match: str + message: str + expected: bool + + @parameterized.expand( + [ + # Test case: 'test' is in 'test output' + CheckRegexesTestCase( + output="test output", + match="test", + message="test message", + expected=True, + ), + # Test case: 'not match' is not in 'test output' + CheckRegexesTestCase( + output="test output", + match="not match", + message="test message", + expected=False, + ), + ] + ) + @patch("regex_parsing.ff_logging.log_failure") + def test_check_regexes( + self, test_output, test_pattern, test_message, expected_result, mock_log_failure + ): + self.assertEqual( + regex_parsing.check_regexes( + test_output, re.compile(test_pattern), test_message + ), + expected_result, + ) + if expected_result: + mock_log_failure.assert_called_once_with(test_message) + else: + mock_log_failure.assert_not_called() + + class CheckRegexFailuresTestCase(NamedTuple): + output: str + expected: bool + message: Optional[str] + + @parameterized.expand([ + # Test case: Output contains 5 chapters, expected failure, with a specific message + CheckRegexFailuresTestCase( + output="test output already contains 5 chapters.", + expected=False, + message="Issue with story, site is broken. Story likely hasn't updated on site yet." + ), + + # Test case: Output doesn't contain any recognizable chapters, expected failure, with a specific message + CheckRegexFailuresTestCase( + output="test output doesn't contain any recognizable chapters, probably from a different source. Not updating.", + expected=False, + message="Something is messed up with the site or the epub. No chapters found." + ), + + # Test case: Generic output, expected success, no specific message + CheckRegexFailuresTestCase( + output="test output", + expected=True, + message=None + ), + ]) + @patch("regex_parsing.ff_logging.log_failure") + def test_check_failure_regexes( + self, input, expected, log_message, mock_log_failure + ): + self.assertEqual(regex_parsing.check_failure_regexes(input), expected) + if log_message: + mock_log_failure.assert_called_once_with(log_message) + else: + mock_log_failure.assert_not_called() + + class CheckForceableRegexTestCase(NamedTuple): + output: str + expected: bool + message: Optional[str] + + @parameterized.expand([ + # Test case: Output contains 5 chapters, more than source: 3, expected True, with a specific message + CheckForceableRegexTestCase( + output="test output contains 5 chapters, more than source: 3.", + expected=True, + message="Chapter difference between source and destination. Forcing update." + ), + + # Test case: File has been updated more recently than the story, expected True, with a specific message + CheckForceableRegexTestCase( + output="File(test.epub) Updated(2022-01-01) more recently than Story(2021-12-31) - Skipping", + expected=True, + message="File has been updated more recently than the story, this is likely a metadata bug. Forcing update." + ), + + # Test case: Generic output, expected False, no specific message + CheckForceableRegexTestCase( + output="test output", + expected=False, + message=None + ), + ]) + @patch("regex_parsing.ff_logging.log_failure") + def test_check_forceable_regexes( + self, input, expected, log_message, mock_log_failure + ): + self.assertEqual(regex_parsing.check_forceable_regexes(input), expected) + if log_message: + mock_log_failure.assert_called_once_with(log_message) + else: + mock_log_failure.assert_not_called() + + class CheckGenerateFanficInfoTestCase(NamedTuple): + url: str + expected_url: str + expected_site: str + + @parameterized.expand([ + # Test case: Fanfiction.net URL + CheckGenerateFanficInfoTestCase( + url="https://www.fanfiction.net/s/1234", + expected_url="www.fanfiction.net/s/1234", + expected_site="ffnet" + ), + + # Test case: Archive of Our Own URL + CheckGenerateFanficInfoTestCase( + url="https://archiveofourown.org/works/5678", + expected_url="archiveofourown.org/works/5678", + expected_site="ao3" + ), + ]) + def test_generate_FanficInfo_from_url(self, input_url, expected_url, expected_site): + fanfic = regex_parsing.generate_FanficInfo_from_url(input_url) + self.assertIsInstance(fanfic, fanfic_info.FanficInfo) + self.assertEqual(fanfic.url, expected_url) + self.assertEqual(fanfic.site, expected_site) + + +if __name__ == "__main__": + unittest.main() diff --git a/root/app/run.sh b/root/app/run.sh index f6f017e..b24f279 100644 --- a/root/app/run.sh +++ b/root/app/run.sh @@ -1,8 +1,3 @@ #!/bin/bash -while : -do - python3 /app/runner_notify.py -c /config/config.ini - sleep 60 -done - +python /app/fanficdownload.py --config="/config/config.toml" \ No newline at end of file diff --git a/root/app/runner_notify.py b/root/app/runner_notify.py deleted file mode 100644 index 61e53d0..0000000 --- a/root/app/runner_notify.py +++ /dev/null @@ -1,171 +0,0 @@ -from io import StringIO -import re -from subprocess import check_output, STDOUT -from time import sleep -import ntpath - -from os import utime -from os.path import join - -from notifications import Notification -from pushbullet import Pushbullet - -from optparse import OptionParser -from configparser import ConfigParser - -def enable_notifications(options): - if options.pushbullet: - fail = False - try: - pb = Pushbullet(options.pushbullet) - except BaseException: - print("Problem wtih connecting to pushbullet. API Key likely invalid") - fail = True - if options.pbdevice and not fail: - try: - pb = pb.get_device(options.pbdevice) - except BaseException: - print("Cannot get this device.") - fail = True - pass - if not fail: - temp_note = Notification() - temp_note.send_notification = pb.push_note - yield temp_note - - -def touch(fname, times=None): - with open(fname, 'a'): - utime(fname, times) - - -def main(options): - try: - res = check_output( - f"python3 /app/fanficdownload.py -c {options.config}", - shell=True, - stderr=STDOUT) - except Exception as e: - print(e) - res = None - if not res: - return - else: - res = res.decode('utf-8') - print(res) - buf = StringIO(res) - regex = re.compile(r"Added (?:.*/)?(.*)-.* to library with id \d*") - searcher = regex.search - stripper = False - for line in buf.readlines(): - r = searcher(line) - if r: - story = ntpath.basename(r.group(1).strip()) - stripper = True - for notify in enable_notifications(options): - notify.send_notification("New Fanfiction Download", story) - if stripper and options.tag: - import sqlite3 - with sqlite3.connect(join(options.library_path, "metadata.db")) as conn: - c = conn.cursor() - c.execute("delete from books_tags_link where id in (select id from books_tags_link where tag in (select id from tags where name like '%Last Update%'));") - return - - -if __name__ == "__main__": - option_parser = OptionParser(usage="usage: %prog [flags]") - option_parser.add_option( - '-p', - '--pushbullet', - action='store', - dest='pushbullet', - help='If you want to use pushbullet, pass in your key here.') - option_parser.add_option( - '-d', - '--device', - action='store', - dest='pbdevice', - help='If you wish to only send to a certian pushbullet device, put the device name here. If the device name is invalid, will just send to all pushbullets associated with the acc') - option_parser.add_option( - '-n', - '--notify', - action='store_true', - dest='notify', - help='Enable if you want to use system notifications. Only for Win/Linux.') - option_parser.add_option( - '-c', - '--config', - action='store', - dest='config', - help='Config file for inputs. Blank config file is provided. No default. If an option is present in whatever config file is passed it, the option will overwrite whatever is passed in through command line arguments unless the option is blank. Do not put any quotation marks in the options.') - option_parser.add_option( - '-t', - '--tag', - action='store_true', - dest='tag', - help='Strip Last Updated tags from calibredb. Requires library to be passed in.') - option_parser.add_option( - '-l', - '--library', - action='store', - dest='library', - help='Path to calibre library. If you are connecting to a calibre webserver then this should be the url.') - option_parser.add_option( - '-a', - '--library-path', - action='store', - dest='library_path', - help='Path location of library. Will be equal to library if nothing is passed in.') - - (options, args) = option_parser.parse_args() - - if options.library and not options.library_path: - options.library_path = options.library - - if options.config: - config = ConfigParser(allow_no_value=True) - config.read(options.config) - - def updater(option, newval): return newval if newval != "" else option - - try: - options.pushbullet = updater( - options.pushbullet, config.get( - 'runner', 'pushbullet')) - except BaseException: - pass - - try: - options.pbdevice = updater( - options.pbdevice, config.get( - 'runner', 'pbdevice')) - except BaseException: - pass - - try: - options.tag = updater( - options.tag, config.getboolean( - 'runner', 'tag')) - except BaseException: - pass - - try: - options.library = updater( - options.library, config.get( - 'locations', 'library')) - except BaseException: - pass - - try: - options.library_path = updater( - options.library, config.get( - 'locations', 'library_path')) - except BaseException: - pass - - if options.pbdevice and not options.pushbullet: - raise ValueError("Can't use a pushbullet device without key") - if options.tag and not options.library: - raise ValueError( - "Can't strip tags from calibre library without a library location.") - main(options) diff --git a/root/app/url_ingester.py b/root/app/url_ingester.py new file mode 100644 index 0000000..2105b8d --- /dev/null +++ b/root/app/url_ingester.py @@ -0,0 +1,89 @@ +import multiprocessing as mp +import socket +from contextlib import contextmanager +import time + +from fanficfare import geturls +import ff_logging +import regex_parsing +import tomllib + +@contextmanager +def set_timeout(time): + """Set a timeout for socket operations.""" + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(time) + try: + yield + finally: + socket.setdefaulttimeout(old_timeout) + +class EmailInfo: + """ + A class used to ingest URLs from an email account. + + Attributes: + email (str): The email address. + password (str): The password for the email account. + smtp_server (str): The SMTP server for the email account. + mailbox (str): The mailbox from which to ingest URLs. + + Methods: + get_urls(): Get URLs from the email account. + """ + + def __init__(self, toml_path: str): + """ + Initialize the UrlIngester with the email account information from a TOML file. + + Parameters: + toml_path (str): The path of the TOML file. + """ + with open(toml_path, "rb") as file: + config = tomllib.load(file) + email_config = config.get("email", {}) + self.email = email_config.get("email") + self.password = email_config.get("password") + self.server = email_config.get("server") + self.mailbox = email_config.get("mailbox") + self.sleep_time = email_config.get("sleep_time") + + def get_urls(self) -> set[str]: + """ + Get URLs from the email account. + + Returns: + set[str]: A set of URLs. + """ + urls = set() + with set_timeout(55): + try: + # Get URLs from the email account + urls = geturls.get_urls_from_imap(self.server, self.email, self.password, self.mailbox) + except Exception as e: + ff_logging.log_failure(f"Failed to get URLs: {e}") + return urls + + + +def email_watcher(email_info: EmailInfo, processor_queues: dict[str, mp.Queue]): + """ + Continuously watch an email account for new URLs and add them to the appropriate processor queues. + + Parameters: + email_info (EmailInfo): The email information object. + processor_queues (dict[str, mp.Queue]): A dictionary mapping site names to processor queues. + + Returns: + None + """ + while True: + # Get URLs from the email account + urls = email_info.get_urls() + # For each URL, generate a FanficInfo object and add it to the appropriate processor queue + for url in urls: + fanfic = regex_parsing.generate_FanficInfo_from_url(url) + ff_logging.log(f"Adding {fanfic.url} to the {fanfic.site} processor queue", "OKBLUE") + processor_queues[fanfic.site].put(fanfic) + # Sleep for the specified amount of time before checking the email account again + time.sleep(email_info.sleep_time) diff --git a/root/app/url_ingester_test.py b/root/app/url_ingester_test.py new file mode 100644 index 0000000..958ad77 --- /dev/null +++ b/root/app/url_ingester_test.py @@ -0,0 +1,120 @@ +from parameterized import parameterized +import unittest +from unittest.mock import mock_open, patch + +from url_ingester import EmailInfo + + +class TestUrlIngester(unittest.TestCase): + @parameterized.expand( + [ + ( + "path/to/config.toml", + """ + [email] + email = "test_email" + password = "test_password" + server = "test_server" + mailbox = "test_mailbox" + sleep_time = 10 + """, + { + "email": { + "email": "test_email", + "password": "test_password", + "server": "test_server", + "mailbox": "test_mailbox", + "sleep_time": 10, + } + }, + ), + ( + "path/to/another_config.toml", + """ + [email] + email = "another_test_email" + password = "another_test_password" + server = "another_test_server" + mailbox = "another_test_mailbox" + sleep_time = 20 + """, + { + "email": { + "email": "another_test_email", + "password": "another_test_password", + "server": "another_test_server", + "mailbox": "another_test_mailbox", + "sleep_time": 20, + } + }, + ), + ] + ) + @patch("builtins.open", new_callable=mock_open) + def test_email_info_init(self, toml_path, config, expected_config, mock_file): + mock_file.return_value.read.return_value = str(config).encode() + email_info = EmailInfo(toml_path) + self.assertEqual(email_info.email, expected_config["email"]["email"]) + self.assertEqual(email_info.password, expected_config["email"]["password"]) + self.assertEqual(email_info.server, expected_config["email"]["server"]) + self.assertEqual(email_info.mailbox, expected_config["email"]["mailbox"]) + self.assertEqual(email_info.sleep_time, expected_config["email"]["sleep_time"]) + + @parameterized.expand( + [ + ( + """ + [email] + email = "test_email" + password = "test_password" + server = "test_server" + mailbox = "test_mailbox" + sleep_time = 10 + """, + { + "email": "test_email", + "password": "test_password", + "server": "test_server", + "mailbox": "test_mailbox", + }, + ["url1", "url2"], + ), + ( + """ + [email] + email = "another_test_email" + password = "another_test_password" + server = "another_test_server" + mailbox = "another_test_mailbox" + sleep_time = 20 + """, + { + "email": "another_test_email", + "password": "another_test_password", + "server": "another_test_server", + "mailbox": "another_test_mailbox", + }, + ["url3", "url4"], + ), + ] + ) + @patch("url_ingester.geturls.get_urls_from_imap") + @patch("builtins.open", new_callable=mock_open) + def test_email_info_get_urls( + self, config, expected_config, urls, mock_file, mock_get_urls_from_imap + ): + mock_get_urls_from_imap.return_value = urls + mock_file.return_value.read.return_value = str(config).encode() + email_info = EmailInfo("path/to/config.toml") + result = email_info.get_urls() + mock_get_urls_from_imap.assert_called_once_with( + expected_config["server"], + expected_config["email"], + expected_config["password"], + expected_config["mailbox"], + ) + self.assertEqual(result, urls) + + +if __name__ == "__main__": + unittest.main() diff --git a/root/app/url_worker.py b/root/app/url_worker.py new file mode 100644 index 0000000..ef795a1 --- /dev/null +++ b/root/app/url_worker.py @@ -0,0 +1,269 @@ +from contextlib import contextmanager +import multiprocessing as mp +import os +from os.path import isfile, join +from shutil import rmtree, copyfile +from subprocess import call, check_output, PIPE, STDOUT, DEVNULL +from tempfile import mkdtemp +from time import sleep + +import calibre_info +import fanfic_info +import ff_logging +import pushbullet_notification +import regex_parsing + +@contextmanager +def temporary_directory(): + """Create and clean up a temporary directory.""" + temp_dir = mkdtemp() + try: + yield temp_dir + finally: + rmtree(temp_dir) + +def call_calibre_db(command: str, fanfic_info: fanfic_info.FanficInfo, calibre_info: calibre_info.CalibreInfo): + """ + Call the calibre database with a specific command. + + Parameters: + command (str): The command to be executed on the calibre database. + fanfic_info (fanfic_info.FanficInfo): The fanfic information object. + calibre_info (calibre_info.CalibreInfo): The calibre information object. + + Returns: + None + """ + try: + # Lock the calibre database to prevent concurrent modifications + with calibre_info.lock: + # Call the calibre command line tool with the specified command\ + ff_logging.log(f"\tCommand: calibredb {command} {fanfic_info.calibre_id} {calibre_info}", "OKBLUE") + call( + f"calibredb {command} {fanfic_info.calibre_id} {calibre_info}", + shell=True, + stdin=PIPE, + stdout=DEVNULL, + stderr=DEVNULL, + ) + except Exception as e: + # Log any failures + ff_logging.log_failure(f"\tFailed to {command} {fanfic_info.calibre_id} from Calibre: {e}") + +def export_story(*, fanfic_info: fanfic_info.FanficInfo, location: str, calibre_info: calibre_info.CalibreInfo) -> None: + """ + Export a story from the Calibre library to a specified location. + + Parameters: + fanfic_info (fanfic_info.FanficInfo): The fanfic information object. + location (str): The directory to which the story should be exported. + calibre_info (calibre_info.CalibreInfo): The calibre information object. + + Returns: + None + """ + # Define the command to be executed + command = f'export --dont-save-cover --dont-write-opf --single-dir --to-dir "{location}"' + # Call the calibre database with the specified command + call_calibre_db(command, fanfic_info, calibre_info) + +def remove_story(*, fanfic_info: fanfic_info.FanficInfo, calibre_info: calibre_info.CalibreInfo) -> None: + """ + Remove a story from the Calibre library. + + Parameters: + fanfic_info (fanfic_info.FanficInfo): The fanfic information object. + calibre_info (calibre_info.CalibreInfo): The calibre information object. + + Returns: + None + """ + # Call the calibre database with the "remove" command + call_calibre_db("remove", fanfic_info, calibre_info) + +def add_story(*, location: str, fanfic_info: fanfic_info.FanficInfo, calibre_info: calibre_info.CalibreInfo,) -> None: + """ + Add a story to the Calibre library. + + Parameters: + location (str): The directory where the story file is located. + fanfic_info (fanfic_info.FanficInfo): The fanfic information object. + calibre_info (calibre_info.CalibreInfo): The calibre information object. + + Returns: + None + """ + # Get the first epub file in the location + file_to_add = get_files(location, file_extension=".epub", return_full_path=True)[0] + + # Log the file being added + ff_logging.log(f"\tAdding {file_to_add} to Calibre", "OKGREEN") + + try: + # Lock the calibre database to prevent concurrent modifications + with calibre_info.lock: + # Call the calibre command line tool to add the story + fanfic_info.title = regex_parsing.extract_filename(get_files(location, file_extension=".epub")[0]) + call( + f'calibredb add -d {calibre_info} "{file_to_add}"', + shell=True, + stdin=PIPE, + stderr=STDOUT, + ) + # Update the title of the fanfic_info object + except Exception as e: + # Log any failures + ff_logging.log_failure(f"\tFailed to add {file_to_add} to Calibre: {e}") + + +def continue_failure(fanfic: fanfic_info.FanficInfo, pushbullet: pushbullet_notification.PushbulletNotification, queue: mp.Queue) -> None: + """ + Handle a failure by either logging a failure and returning, or incrementing the repeat count and putting the fanfic back in the queue. + + Parameters: + fanfic (fanfic_info.FanficInfo): The fanfic information object. + pushbullet (pushbullet_notification.PushbulletNotification): The pushbullet notification object. + queue (mp.Queue): The multiprocessing queue object. + + Returns: + None + """ + # If the fanfic has reached the maximum number of repeats, log a failure and return + if fanfic.reached_maximum_repeats(): + ff_logging.log_failure(f"Reached maximum number of repeats for {fanfic.url}. Skipping.") + pushbullet.send_notification("Fanfiction Download Failed", fanfic.url) + else: + # Increment the repeat count and put the fanfic back in the queue + fanfic.increment_repeat() + queue.put(fanfic) + +def get_files(directory_path, file_extension=None, return_full_path=False): + """ + Get files from a directory. If a file extension is specified, filter files by extension. + + Parameters: + directory_path (str): The path of the directory from which to get files. + file_extension (str, optional): The extension of the files to get. Defaults to None. + return_full_path (bool, optional): Whether to return the full path of the files. Defaults to False. + + Returns: + list: A list of file names or file paths, depending on the value of return_full_path. + """ + # Get a list of files in the directory that have the specified extension (or all files if no extension is specified) + files = [ + file + for file in os.listdir(directory_path) + if isfile(join(directory_path, file)) and (not file_extension or file.endswith(file_extension)) + ] + + # If return_full_path is True, replace the list of file names with a list of file paths + if return_full_path: + files = [join(directory_path, file) for file in files] + + return files + +def get_path_or_url(ff_info: fanfic_info.FanficInfo, cdb_info: calibre_info.CalibreInfo, location: str = "") -> str: + """ + Get the path of the exported story if it exists in the Calibre library, otherwise return the URL of the story. + + Parameters: + ff_info (fanfic_info.FanficInfo): The fanfic information object. + cdb_info (calibre_info.CalibreInfo): The calibre information object. + location (str, optional): The directory to which the story should be exported. Defaults to "". + + Returns: + str: The path of the exported story or the URL of the story. + """ + # If the story exists in the Calibre library + if ff_info.get_id_from_calibredb(cdb_info): + # Export the story to the specified location + export_story(fanfic_info=ff_info, location=location, calibre_info=cdb_info) + # Return the path of the exported story + return get_files(location, file_extension=".epub", return_full_path=True)[0] + # If the story does not exist in the Calibre library, return the URL of the story + return ff_info.url + +def url_worker(queue: mp.Queue, cdb: calibre_info.CalibreInfo, pushbullet_info: pushbullet_notification.PushbulletNotification, waiting_queue: mp.Queue) -> None: + """ + Worker function that updates fanfics from a queue. + + Parameters: + queue (mp.Queue): The multiprocessing queue object. + cdb (calibre_info.CalibreInfo): The calibre information object. + pushbullet_info (pushbullet_notification.PushbulletNotification): The pushbullet notification object. + + Returns: + None + """ + # Continuously process fanfics from the queue + while True: + # If the queue is empty, sleep for 5 seconds and then continue to the next iteration + if queue.empty(): + sleep(5) + continue + + # Get a fanfic from the queue + fanfic: fanfic_info.FanficInfo = queue.get() + # If the fanfic is None, continue to the next iteration + if fanfic is None: + continue + + # Create a temporary directory + with temporary_directory() as temp_dir: + + site = fanfic.site + + ff_logging.log(f"\t({site}) Processing {fanfic.url}", "OKGREEN") + + # Get the path of the fanfic if it exists in the Calibre library, otherwise get the URL of the fanfic + path_or_url = get_path_or_url(fanfic, cdb, temp_dir) + + # Log the update + ff_logging.log(f"\t({site}) Updating {path_or_url}", "OKGREEN") + + # Define the command to update the fanfic + command = f'cd {temp_dir} && python -m fanficfare.cli -u "{path_or_url}" --update-cover --non-interactive"' + # If the behavior of the fanfic is "force", add the "--force" option to the command + if fanfic.behavior == "force": + command += " --force" + + ff_logging.log(f"\t({site}) Running Command: {command}", "OKBLUE") + try: + #copy the configs to the temp directory + if cdb.default_ini: + copyfile(cdb.default_ini, join(temp_dir, "defaults.ini")) + if cdb.personal_ini: + copyfile(cdb.personal_ini, join(temp_dir, "personal.ini")) + # Execute the command and get the output + output = check_output(command, shell=True, stderr=STDOUT, stdin=PIPE,).decode("utf-8") + ff_logging.log(f"\t({site}) Output: {output}", "OKBLUE") + except Exception as e: + # If the command fails, log the failure and continue to the next iteration + ff_logging.log_failure(f"\t({site}) Failed to update {path_or_url}: {e}, {output}") + continue_failure(fanfic, pushbullet_info, waiting_queue) + continue + + # If the output indicates a failure, continue to the next iteration + if not regex_parsing.check_failure_regexes(output): + continue_failure(fanfic, pushbullet_info, waiting_queue) + continue + # If the output indicates a forceable error, set the behavior of the fanfic to "force" and put it back in the queue + if regex_parsing.check_forceable_regexes(output): + fanfic.behavior = "force" + queue.put(fanfic) + continue + + # If the fanfic exists in the Calibre library, remove it + if fanfic.calibre_id: + ff_logging.log(f"\t({site}) Going to remove story.", "OKGREEN") + remove_story(fanfic_info=fanfic, calibre_info=cdb) + # Add the fanfic to the Calibre library + add_story(location=temp_dir, fanfic_info=fanfic, calibre_info=cdb) + + # If the fanfic was not added to the Calibre library, log a failure and continue to the next iteration + if not fanfic.get_id_from_calibredb(cdb): + ff_logging.log_failure(f"\t({site}) Failed to add {path_or_url} to Calibre") + continue_failure(fanfic, pushbullet_info, waiting_queue) + else: + # If the fanfic was added to the Calibre library, send a notification + pushbullet_info.send_notification("New Fanfiction Download", fanfic.title) diff --git a/root/config.default/config.ini b/root/config.default/config.ini deleted file mode 100644 index 85aefd7..0000000 --- a/root/config.default/config.ini +++ /dev/null @@ -1,18 +0,0 @@ -[login] -user= -password= -server= -mailbox= -libuser= -libpassword= - -[locations] -library= -input=/config/fanfiction_file - -[runner] -pushbullet=False -pbdevice= - -[output] -live=False diff --git a/root/config.default/config.toml b/root/config.default/config.toml new file mode 100644 index 0000000..39e9a40 --- /dev/null +++ b/root/config.default/config.toml @@ -0,0 +1,18 @@ +[email] +email = "" +password = "" +server = "" +mailbox = "" +sleep_time = 60 + +[calibre] +path="" +username="" +password="" +default_ini="" +personal_ini="" + +[pushbullet] +enabled = false +api_key = "" +device = "" \ No newline at end of file diff --git a/root/config.default/defaults.ini b/root/config.default/defaults.ini index b45ac5e..9808c25 100644 --- a/root/config.default/defaults.ini +++ b/root/config.default/defaults.ini @@ -1,4 +1,4 @@ -# Copyright 2015 Fanficdownloader team, 2016 FanFicFare team +# Copyright 2015 Fanficdownloader team, 2021 FanFicFare team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ ## titlepage_entries: category,genre, status,dateUpdated,rating ## [epub] ## # overrides defaults & site section -## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated +## titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated ## [www.whofic.com:epub] ## # overrides defaults, site section & format section ## titlepage_entries: category,genre, status,datePublished @@ -34,8 +34,8 @@ ## titlepage_entries: category ## Some sites also require the user to confirm they are adult for -## adult content. Uncomment by removing '#' in front of is_adult. -is_adult:true +## adult content. Defaults to false. +is_adult:false ## All available titlepage_entries and the label used for them: ## _label: