diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9294287..473a3fc 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -18,6 +18,8 @@ jobs: dockerfile: ./Dockerfile.ocr - image: c0deplayer/dc-processor dockerfile: ./Dockerfile.processor + - image: c0deplayer/dc-predictor + dockerfile: ./Dockerfile.predictor steps: - name: Checkout diff --git a/.gitignore b/.gitignore index f05cc35..bd413e3 100644 --- a/.gitignore +++ b/.gitignore @@ -162,5 +162,7 @@ cython_debug/ #.idea/ # Other +.DS_Store logs/ -__*.py \ No newline at end of file +__*.py +models/ \ No newline at end of file diff --git a/Dockerfile.ocr b/Dockerfile.ocr index 0ed504f..770aa48 100644 --- a/Dockerfile.ocr +++ b/Dockerfile.ocr @@ -1,20 +1,36 @@ -FROM python:3.12-slim +FROM ghcr.io/astral-sh/uv:latest AS uv +FROM python:3.12-slim AS python + LABEL authors="codeplayer" -WORKDIR /code +ENV VIRTUAL_ENV=/opt/venv + +WORKDIR /app/data/ocr # Update and upgrade the system RUN apt update -y && \ apt upgrade -y \ + # Install required packages + && apt install poppler-utils -y \ # cleanup && apt autoremove -y \ && apt clean -y \ && rm -rf /var/lib/apt/lists -COPY ./requirements.txt /code/requirements.txt +RUN \ + # we use a cache --mount to reuse the uv cache across builds + --mount=type=cache,target=/root/.cache/uv \ + # we use a bind --mount to use the uv binary from the uv stage + --mount=type=bind,from=uv,source=/uv,target=/uv \ + # we use a bind --mount to use the requirements.txt from the host instead of adding a COPY layer + --mount=type=bind,source=requirements.txt,target=requirements.txt \ + /uv venv /opt/venv && \ + /uv pip install -r requirements.txt -RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt +WORKDIR /app/code/ocr -COPY ./src/documentclassification/ocr/ /code/documentclassification/ocr/ +COPY src/ocr/ . +COPY src/configs/ocr_config.py configs/ocr_config.py +COPY src/payload/ocr_models.py payload/ocr_models.py -CMD ["uvicorn", "documentclassification.ocr.ocr:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file +CMD ["/opt/venv/bin/python", "-m", "uvicorn", "ocr:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/Dockerfile.predictor b/Dockerfile.predictor new file mode 100644 index 0000000..388ac7e --- /dev/null +++ b/Dockerfile.predictor @@ -0,0 +1,32 @@ +FROM ghcr.io/astral-sh/uv:latest AS uv +FROM python:3.12-slim AS python + +ENV VIRTUAL_ENV=/opt/venv + +WORKDIR /app/data/predictor + +# Update and upgrade the system +RUN apt update -y && \ + apt upgrade -y \ + # cleanup + && apt autoremove -y \ + && apt clean -y \ + && rm -rf /var/lib/apt/lists + +RUN \ + # we use a cache --mount to reuse the uv cache across builds + --mount=type=cache,target=/root/.cache/uv \ + # we use a bind --mount to use the uv binary from the uv stage + --mount=type=bind,from=uv,source=/uv,target=/uv \ + # we use a bind --mount to use the requirements.txt from the host instead of adding a COPY layer + --mount=type=bind,source=requirements.txt,target=requirements.txt \ + /uv venv /opt/venv && \ + /uv pip install -r requirements.txt + +WORKDIR /app/code/predictor + +COPY src/predictor/ . +COPY src/configs/model_config.py configs/model_config.py +COPY src/payload/model_models.py payload/model_models.py + +CMD ["/opt/venv/bin/python","-m", "uvicorn", "model:app", "--host", "0.0.0.0", "--port", "7070"] \ No newline at end of file diff --git a/Dockerfile.processor b/Dockerfile.processor index 9681e43..b305488 100644 --- a/Dockerfile.processor +++ b/Dockerfile.processor @@ -1,7 +1,11 @@ -FROM python:3.12-slim +FROM ghcr.io/astral-sh/uv:latest AS uv +FROM python:3.12-slim AS python + LABEL authors="codeplayer" -WORKDIR /code +ENV VIRTUAL_ENV=/opt/venv + +# RUN mkdir -p /app/data/logs/processor # Update and upgrade the system RUN apt update -y && \ @@ -11,10 +15,20 @@ RUN apt update -y && \ && apt clean -y \ && rm -rf /var/lib/apt/lists -COPY ./requirements.txt /code/requirements.txt +RUN \ + # we use a cache --mount to reuse the uv cache across builds + --mount=type=cache,target=/root/.cache/uv \ + # we use a bind --mount to use the uv binary from the uv stage + --mount=type=bind,from=uv,source=/uv,target=/uv \ + # we use a bind --mount to use the requirements.txt from the host instead of adding a COPY layer + --mount=type=bind,source=requirements.txt,target=requirements.txt \ + /uv venv /opt/venv && \ + /uv pip install -r requirements.txt -RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt +WORKDIR /app/code/processor -COPY ./src/documentclassification/process/ /code/documentclassification/process/ +COPY src/processor/ . +COPY src/configs/processor_config.py configs/processor_config.py +COPY src/payload/processor_models.py payload/processor_models.py -CMD ["uvicorn", "documentclassification.process.process:app", "--host", "0.0.0.0", "--port", "9090"] \ No newline at end of file +CMD ["/opt/venv/bin/python", "-m", "uvicorn", "processor:app", "--host", "0.0.0.0", "--port", "9090"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 707672e..d5defd4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,13 +2,61 @@ services: ocr: container_name: ocr_service image: c0deplayer/dc-ocr:main + # build: + # dockerfile: Dockerfile.ocr ports: - "8080:8080" + networks: + - document-classification + volumes: + - type: bind + source: logs + target: /app/data/logs + - type: bind + source: models + target: /app/data/models processor: container_name: processor_service image: c0deplayer/dc-processor:main + # build: + # dockerfile: Dockerfile.processor ports: - "9090:9090" + networks: + - document-classification depends_on: - - ocr \ No newline at end of file + - ocr + volumes: + - type: bind + source: logs + target: /app/data/logs + - type: bind + source: models + target: /app/data/models + + + predictor: + container_name: predictor_service + image: c0deplayer/dc-predictor:main + # build: + # dockerfile: Dockerfile.predictor + ports: + - "7070:7070" + networks: + - document-classification + depends_on: + - processor + volumes: + - type: bind + source: logs + target: /app/data/logs + - type: bind + source: models + target: /app/data/models + + + +networks: + document-classification: + driver: bridge \ No newline at end of file diff --git a/pdm.lock b/pdm.lock index d6a7fec..acc4642 100644 --- a/pdm.lock +++ b/pdm.lock @@ -3,22 +3,70 @@ [metadata] groups = ["default"] -strategy = ["inherit_metadata"] +strategy = [] lock_version = "4.5.0" -content_hash = "sha256:8ea02f2da903bab69546a0554af09c5bd9d18e4519204ac5b036209a964de4d9" +content_hash = "sha256:add19ef0fffec4392ca8b724a77712d8e909712598b4be0d062ec33e46dac925" [[metadata.targets]] requires_python = ">=3.12,<3.13" [[package]] -name = "annotated-types" -version = "0.7.0" -requires_python = ">=3.8" -summary = "Reusable constraint types to use with typing.Annotated" -groups = ["default"] +name = "aiohappyeyeballs" +version = "2.4.3" +summary = "" +files = [ + {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, + {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, +] + +[[package]] +name = "aiohttp" +version = "3.11.2" +summary = "" +dependencies = [ + "aiohappyeyeballs", + "aiosignal", + "attrs", + "frozenlist", + "multidict", + "propcache", + "yarl", +] +files = [ + {file = "aiohttp-3.11.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f833a80d9de9307d736b6af58c235b17ef7f90ebea7b9c49cd274dec7a66a2f1"}, + {file = "aiohttp-3.11.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:382f853516664d2ebfc75dc01da4a10fdef5edcb335fe7b45cf471ce758ecb18"}, + {file = "aiohttp-3.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3a2bcf6c81639a165da93469e1e0aff67c956721f3fa9c0560f07dd1e505116"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de3b4d5fb5d69749104b880a157f38baeea7765c93d9cd3837cedd5b84729e10"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0a90a0dc4b054b5af299a900bf950fe8f9e3e54322bc405005f30aa5cacc5c98"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32334f35824811dd20a12cc90825d000e6b50faaeaa71408d42269151a66140d"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cba0b8d25aa2d450762f3dd6df85498f5e7c3ad0ddeb516ef2b03510f0eea32"}, + {file = "aiohttp-3.11.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bbb2dbc2701ab7e9307ca3a8fa4999c5b28246968e0a0202a5afabf48a42e22"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97fba98fc5d9ccd3d33909e898d00f2494d6a9eec7cbda3d030632e2c8bb4d00"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0ebdf5087e2ce903d8220cc45dcece90c2199ae4395fd83ca616fcc81010db2c"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:122768e3ae9ce74f981b46edefea9c6e5a40aea38aba3ac50168e6370459bf20"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5587da333b7d280a312715b843d43e734652aa382cba824a84a67c81f75b338b"}, + {file = "aiohttp-3.11.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85de9904bc360fd29a98885d2bfcbd4e02ab33c53353cb70607f2bea2cb92468"}, + {file = "aiohttp-3.11.2-cp312-cp312-win32.whl", hash = "sha256:b470de64d17156c37e91effc109d3b032b39867000e2c126732fe01d034441f9"}, + {file = "aiohttp-3.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:3f617a48b70f4843d54f52440ea1e58da6bdab07b391a3a6aed8d3b311a4cc04"}, + {file = "aiohttp-3.11.2.tar.gz", hash = "sha256:68d1f46f9387db3785508f5225d3acbc5825ca13d9c29f2b5cce203d5863eb79"}, +] + +[[package]] +name = "aiosignal" +version = "1.3.1" +summary = "" dependencies = [ - "typing-extensions>=4.0.0; python_version < \"3.9\"", + "frozenlist", ] +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +summary = "" files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -27,26 +75,29 @@ files = [ [[package]] name = "anyio" version = "4.6.2.post1" -requires_python = ">=3.9" -summary = "High level compatibility layer for multiple asynchronous event loop implementations" -groups = ["default"] +summary = "" dependencies = [ - "exceptiongroup>=1.0.2; python_version < \"3.11\"", - "idna>=2.8", - "sniffio>=1.1", - "typing-extensions>=4.1; python_version < \"3.11\"", + "idna", + "sniffio", ] files = [ {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, ] +[[package]] +name = "attrs" +version = "24.2.0" +summary = "" +files = [ + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, +] + [[package]] name = "certifi" version = "2024.8.30" -requires_python = ">=3.6" -summary = "Python package for providing Mozilla's CA Bundle." -groups = ["default"] +summary = "" files = [ {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, @@ -55,9 +106,7 @@ files = [ [[package]] name = "charset-normalizer" version = "3.4.0" -requires_python = ">=3.7.0" -summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -groups = ["default"] +summary = "" files = [ {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, @@ -81,12 +130,9 @@ files = [ [[package]] name = "click" version = "8.1.7" -requires_python = ">=3.7" -summary = "Composable command line interface toolkit" -groups = ["default"] +summary = "" dependencies = [ "colorama; platform_system == \"Windows\"", - "importlib-metadata; python_version < \"3.8\"", ] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, @@ -96,21 +142,50 @@ files = [ [[package]] name = "colorama" version = "0.4.6" -requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -summary = "Cross-platform colored terminal text." -groups = ["default"] -marker = "platform_system == \"Windows\" or sys_platform == \"win32\"" +summary = "" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "datasets" +version = "3.1.0" +summary = "" +dependencies = [ + "aiohttp", + "dill", + "filelock", + "fsspec", + "huggingface-hub", + "multiprocess", + "numpy", + "packaging", + "pandas", + "pyarrow", + "pyyaml", + "requests", + "tqdm", + "xxhash", +] +files = [ + {file = "datasets-3.1.0-py3-none-any.whl", hash = "sha256:dc8808a6d17838fe05e13b39aa7ac3ea0fd0806ed7004eaf4d4eb2c2a356bc61"}, + {file = "datasets-3.1.0.tar.gz", hash = "sha256:c92cac049e0f9f85b0dd63739c68e564c657b1624bc2b66b1e13489062832e27"}, +] + +[[package]] +name = "dill" +version = "0.3.8" +summary = "" +files = [ + {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, + {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, +] + [[package]] name = "dnspython" version = "2.7.0" -requires_python = ">=3.9" -summary = "DNS toolkit" -groups = ["default"] +summary = "" files = [ {file = "dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86"}, {file = "dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1"}, @@ -119,21 +194,20 @@ files = [ [[package]] name = "easyocr" version = "1.7.2" -summary = "End-to-End Multi-Lingual Optical Character Recognition (OCR) Solution" -groups = ["default"] +summary = "" dependencies = [ - "Pillow", - "PyYAML", - "Shapely", "ninja", "numpy", "opencv-python-headless", + "pillow", "pyclipper", "python-bidi", + "pyyaml", "scikit-image", "scipy", + "shapely", "torch", - "torchvision>=0.5", + "torchvision", ] files = [ {file = "easyocr-1.7.2-py3-none-any.whl", hash = "sha256:5be12f9b0e595d443c9c3d10b0542074b50f0ec2d98b141a109cd961fd1c177c"}, @@ -142,12 +216,10 @@ files = [ [[package]] name = "email-validator" version = "2.2.0" -requires_python = ">=3.8" -summary = "A robust email address syntax and deliverability validation library." -groups = ["default"] +summary = "" dependencies = [ - "dnspython>=2.0.0", - "idna>=2.0.0", + "dnspython", + "idna", ] files = [ {file = "email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631"}, @@ -156,29 +228,25 @@ files = [ [[package]] name = "fastapi" -version = "0.115.3" -requires_python = ">=3.8" -summary = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" -groups = ["default"] +version = "0.115.5" +summary = "" dependencies = [ - "pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4", - "starlette<0.42.0,>=0.40.0", - "typing-extensions>=4.8.0", + "pydantic", + "starlette", + "typing-extensions", ] files = [ - {file = "fastapi-0.115.3-py3-none-any.whl", hash = "sha256:8035e8f9a2b0aa89cea03b6c77721178ed5358e1aea4cd8570d9466895c0638c"}, - {file = "fastapi-0.115.3.tar.gz", hash = "sha256:c091c6a35599c036d676fa24bd4a6e19fa30058d93d950216cdc672881f6f7db"}, + {file = "fastapi-0.115.5-py3-none-any.whl", hash = "sha256:596b95adbe1474da47049e802f9a65ab2ffa9c2b07e7efee70eb8a66c9f2f796"}, + {file = "fastapi-0.115.5.tar.gz", hash = "sha256:0e7a4d0dc0d01c68df21887cce0945e72d3c48b9f4f79dfe7a7d53aa08fbb289"}, ] [[package]] name = "fastapi-cli" version = "0.0.5" -requires_python = ">=3.8" -summary = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀" -groups = ["default"] +summary = "" dependencies = [ - "typer>=0.12.3", - "uvicorn[standard]>=0.15.0", + "typer", + "uvicorn", ] files = [ {file = "fastapi_cli-0.0.5-py3-none-any.whl", hash = "sha256:e94d847524648c748a5350673546bbf9bcaeb086b33c24f2e82e021436866a46"}, @@ -189,12 +257,10 @@ files = [ name = "fastapi-cli" version = "0.0.5" extras = ["standard"] -requires_python = ">=3.8" -summary = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀" -groups = ["default"] +summary = "" dependencies = [ "fastapi-cli==0.0.5", - "uvicorn[standard]>=0.15.0", + "uvicorn", ] files = [ {file = "fastapi_cli-0.0.5-py3-none-any.whl", hash = "sha256:e94d847524648c748a5350673546bbf9bcaeb086b33c24f2e82e021436866a46"}, @@ -203,56 +269,83 @@ files = [ [[package]] name = "fastapi" -version = "0.115.3" +version = "0.115.5" extras = ["standard"] -requires_python = ">=3.8" -summary = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" -groups = ["default"] +summary = "" dependencies = [ - "email-validator>=2.0.0", - "fastapi-cli[standard]>=0.0.5", - "fastapi==0.115.3", - "httpx>=0.23.0", - "jinja2>=2.11.2", - "python-multipart>=0.0.7", - "uvicorn[standard]>=0.12.0", + "email-validator", + "fastapi-cli", + "fastapi==0.115.5", + "httpx", + "jinja2", + "python-multipart", + "uvicorn", ] files = [ - {file = "fastapi-0.115.3-py3-none-any.whl", hash = "sha256:8035e8f9a2b0aa89cea03b6c77721178ed5358e1aea4cd8570d9466895c0638c"}, - {file = "fastapi-0.115.3.tar.gz", hash = "sha256:c091c6a35599c036d676fa24bd4a6e19fa30058d93d950216cdc672881f6f7db"}, + {file = "fastapi-0.115.5-py3-none-any.whl", hash = "sha256:596b95adbe1474da47049e802f9a65ab2ffa9c2b07e7efee70eb8a66c9f2f796"}, + {file = "fastapi-0.115.5.tar.gz", hash = "sha256:0e7a4d0dc0d01c68df21887cce0945e72d3c48b9f4f79dfe7a7d53aa08fbb289"}, ] [[package]] name = "filelock" version = "3.16.1" -requires_python = ">=3.8" -summary = "A platform independent file lock." -groups = ["default"] +summary = "" files = [ {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] +[[package]] +name = "frozenlist" +version = "1.5.0" +summary = "" +files = [ + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f"}, + {file = "frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8"}, + {file = "frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f"}, + {file = "frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3"}, + {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, +] + [[package]] name = "fsspec" -version = "2024.10.0" -requires_python = ">=3.8" -summary = "File-system specification" -groups = ["default"] +version = "2024.9.0" +summary = "" files = [ - {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, - {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, + {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, + {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, ] [[package]] -name = "h11" -version = "0.14.0" -requires_python = ">=3.7" -summary = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -groups = ["default"] +name = "fsspec" +version = "2024.9.0" +extras = ["http"] +summary = "" dependencies = [ - "typing-extensions; python_version < \"3.8\"", + "aiohttp", + "fsspec==2024.9.0", ] +files = [ + {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"}, + {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"}, +] + +[[package]] +name = "h11" +version = "0.14.0" +summary = "" files = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, @@ -261,12 +354,10 @@ files = [ [[package]] name = "httpcore" version = "1.0.6" -requires_python = ">=3.8" -summary = "A minimal low-level HTTP client." -groups = ["default"] +summary = "" dependencies = [ "certifi", - "h11<0.15,>=0.13", + "h11", ] files = [ {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"}, @@ -276,9 +367,7 @@ files = [ [[package]] name = "httptools" version = "0.6.4" -requires_python = ">=3.8.0" -summary = "A collection of framework independent HTTP protocol utils." -groups = ["default"] +summary = "" files = [ {file = "httptools-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2"}, {file = "httptools-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44"}, @@ -293,13 +382,11 @@ files = [ [[package]] name = "httpx" version = "0.27.2" -requires_python = ">=3.8" -summary = "The next generation HTTP client." -groups = ["default"] +summary = "" dependencies = [ "anyio", "certifi", - "httpcore==1.*", + "httpcore", "idna", "sniffio", ] @@ -311,17 +398,15 @@ files = [ [[package]] name = "huggingface-hub" version = "0.26.1" -requires_python = ">=3.8.0" -summary = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -groups = ["default"] +summary = "" dependencies = [ "filelock", - "fsspec>=2023.5.0", - "packaging>=20.9", - "pyyaml>=5.1", + "fsspec", + "packaging", + "pyyaml", "requests", - "tqdm>=4.42.1", - "typing-extensions>=3.7.4.3", + "tqdm", + "typing-extensions", ] files = [ {file = "huggingface_hub-0.26.1-py3-none-any.whl", hash = "sha256:5927a8fc64ae68859cd954b7cc29d1c8390a5e15caba6d3d349c973be8fdacf3"}, @@ -331,9 +416,7 @@ files = [ [[package]] name = "idna" version = "3.10" -requires_python = ">=3.6" -summary = "Internationalized Domain Names in Applications (IDNA)" -groups = ["default"] +summary = "" files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -342,12 +425,10 @@ files = [ [[package]] name = "imageio" version = "2.36.0" -requires_python = ">=3.9" -summary = "Library for reading and writing a wide range of image, video, scientific, and volumetric data formats." -groups = ["default"] +summary = "" dependencies = [ "numpy", - "pillow>=8.3.2", + "pillow", ] files = [ {file = "imageio-2.36.0-py3-none-any.whl", hash = "sha256:471f1eda55618ee44a3c9960911c35e647d9284c68f077e868df633398f137f0"}, @@ -357,11 +438,9 @@ files = [ [[package]] name = "jinja2" version = "3.1.4" -requires_python = ">=3.7" -summary = "A very fast and expressive template engine." -groups = ["default"] +summary = "" dependencies = [ - "MarkupSafe>=2.0", + "markupsafe", ] files = [ {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, @@ -371,11 +450,8 @@ files = [ [[package]] name = "lazy-loader" version = "0.4" -requires_python = ">=3.7" -summary = "Makes it easy to load subpackages and functions on demand." -groups = ["default"] +summary = "" dependencies = [ - "importlib-metadata; python_version < \"3.8\"", "packaging", ] files = [ @@ -383,14 +459,46 @@ files = [ {file = "lazy_loader-0.4.tar.gz", hash = "sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1"}, ] +[[package]] +name = "lightning" +version = "2.4.0" +summary = "" +dependencies = [ + "fsspec", + "lightning-utilities", + "packaging", + "pytorch-lightning", + "pyyaml", + "torch", + "torchmetrics", + "tqdm", + "typing-extensions", +] +files = [ + {file = "lightning-2.4.0-py3-none-any.whl", hash = "sha256:560163af9711cf59055c448232c473150a299089efce0d2be3cc3288082d8768"}, + {file = "lightning-2.4.0.tar.gz", hash = "sha256:9156604cc56e4b2b603f34fa7f0fe5107375c8e6d85e74544b319a15faa9ed0e"}, +] + +[[package]] +name = "lightning-utilities" +version = "0.11.8" +summary = "" +dependencies = [ + "packaging", + "setuptools", + "typing-extensions", +] +files = [ + {file = "lightning_utilities-0.11.8-py3-none-any.whl", hash = "sha256:a57edb34a44258f0c61eed8b8b88926766e9052f5e60bbe69e4871a2b2bfd970"}, + {file = "lightning_utilities-0.11.8.tar.gz", hash = "sha256:8dfbdc6c52f9847efc948dc462ab8bebb4f4e9a43bd69c82c1b1da484dac20e6"}, +] + [[package]] name = "markdown-it-py" version = "3.0.0" -requires_python = ">=3.8" -summary = "Python port of markdown-it. Markdown parsing, done right!" -groups = ["default"] +summary = "" dependencies = [ - "mdurl~=0.1", + "mdurl", ] files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, @@ -400,9 +508,7 @@ files = [ [[package]] name = "markupsafe" version = "3.0.2" -requires_python = ">=3.9" -summary = "Safely add untrusted strings to HTML/XML markup." -groups = ["default"] +summary = "" files = [ {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, @@ -420,9 +526,7 @@ files = [ [[package]] name = "mdurl" version = "0.1.2" -requires_python = ">=3.7" -summary = "Markdown URL utilities" -groups = ["default"] +summary = "" files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -431,20 +535,56 @@ files = [ [[package]] name = "mpmath" version = "1.3.0" -summary = "Python library for arbitrary-precision floating-point arithmetic" -groups = ["default"] -marker = "python_version >= \"3.9\"" +summary = "" files = [ {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, ] +[[package]] +name = "multidict" +version = "6.1.0" +summary = "" +files = [ + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3"}, + {file = "multidict-6.1.0-cp312-cp312-win32.whl", hash = "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133"}, + {file = "multidict-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1"}, + {file = "multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506"}, + {file = "multidict-6.1.0.tar.gz", hash = "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a"}, +] + +[[package]] +name = "multiprocess" +version = "0.70.16" +summary = "" +dependencies = [ + "dill", +] +files = [ + {file = "multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02"}, + {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, + {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, + {file = "multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435"}, + {file = "multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3"}, + {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, +] + [[package]] name = "networkx" version = "3.4.2" -requires_python = ">=3.10" -summary = "Python package for creating and manipulating graphs and networks" -groups = ["default"] +summary = "" files = [ {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, @@ -453,8 +593,7 @@ files = [ [[package]] name = "ninja" version = "1.11.1.1" -summary = "Ninja is a small build system with a focus on speed" -groups = ["default"] +summary = "" files = [ {file = "ninja-1.11.1.1-py2.py3-none-macosx_10_9_universal2.macosx_10_9_x86_64.macosx_11_0_arm64.macosx_11_0_universal2.whl", hash = "sha256:376889c76d87b95b5719fdd61dd7db193aa7fd4432e5d52d2e44e4c497bdbbee"}, {file = "ninja-1.11.1.1-py2.py3-none-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:ecf80cf5afd09f14dcceff28cb3f11dc90fb97c999c89307aea435889cb66877"}, @@ -475,127 +614,94 @@ files = [ [[package]] name = "numpy" -version = "2.1.2" -requires_python = ">=3.10" -summary = "Fundamental package for array computing in Python" -groups = ["default"] +version = "2.1.3" +summary = "" files = [ - {file = "numpy-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1"}, - {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426"}, - {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0"}, - {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df"}, - {file = "numpy-2.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366"}, - {file = "numpy-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142"}, - {file = "numpy-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550"}, - {file = "numpy-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e"}, - {file = "numpy-2.1.2.tar.gz", hash = "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564"}, + {file = "numpy-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512"}, + {file = "numpy-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b"}, + {file = "numpy-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc"}, + {file = "numpy-2.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0"}, + {file = "numpy-2.1.3-cp312-cp312-win32.whl", hash = "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9"}, + {file = "numpy-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a"}, + {file = "numpy-2.1.3.tar.gz", hash = "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761"}, ] [[package]] name = "nvidia-cublas-cu12" version = "12.4.5.8" -requires_python = ">=3" -summary = "CUBLAS native runtime libraries" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" files = [ {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0f8aa1706812e00b9f19dfe0cdb3999b092ccb8ca168c0db5b8ea712456fd9b3"}, {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b"}, - {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-win_amd64.whl", hash = "sha256:5a796786da89203a0657eda402bcdcec6180254a8ac22d72213abc42069522dc"}, ] [[package]] name = "nvidia-cuda-cupti-cu12" version = "12.4.127" -requires_python = ">=3" -summary = "CUDA profiling tools runtime libs." -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" files = [ {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:79279b35cf6f91da114182a5ce1864997fd52294a87a16179ce275773799458a"}, {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb"}, - {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:5688d203301ab051449a2b1cb6690fbe90d2b372f411521c86018b950f3d7922"}, ] [[package]] name = "nvidia-cuda-nvrtc-cu12" version = "12.4.127" -requires_python = ">=3" -summary = "NVRTC native runtime libraries" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" files = [ {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0eedf14185e04b76aa05b1fea04133e59f465b6f960c0cbf4e37c3cb6b0ea198"}, {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338"}, - {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:a961b2f1d5f17b14867c619ceb99ef6fcec12e46612711bcec78eb05068a60ec"}, ] [[package]] name = "nvidia-cuda-runtime-cu12" version = "12.4.127" -requires_python = ">=3" -summary = "CUDA Runtime native Libraries" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" files = [ {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:961fe0e2e716a2a1d967aab7caee97512f71767f852f67432d572e36cb3a11f3"}, {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5"}, - {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:09c2e35f48359752dfa822c09918211844a3d93c100a715d79b59591130c5e1e"}, ] [[package]] name = "nvidia-cudnn-cu12" version = "9.1.0.70" -requires_python = ">=3" -summary = "cuDNN runtime libraries" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" dependencies = [ "nvidia-cublas-cu12", ] files = [ {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f"}, - {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-win_amd64.whl", hash = "sha256:6278562929433d68365a07a4a1546c237ba2849852c0d4b2262a486e805b977a"}, ] [[package]] name = "nvidia-cufft-cu12" version = "11.2.1.3" -requires_python = ">=3" -summary = "CUFFT native runtime libraries" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" dependencies = [ "nvidia-nvjitlink-cu12", ] files = [ {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5dad8008fc7f92f5ddfa2101430917ce2ffacd86824914c82e28990ad7f00399"}, {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9"}, - {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-win_amd64.whl", hash = "sha256:d802f4954291101186078ccbe22fc285a902136f974d369540fd4a5333d1440b"}, ] [[package]] name = "nvidia-curand-cu12" version = "10.3.5.147" -requires_python = ">=3" -summary = "CURAND native runtime libraries" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" files = [ {file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1f173f09e3e3c76ab084aba0de819c49e56614feae5c12f69883f4ae9bb5fad9"}, {file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b"}, - {file = "nvidia_curand_cu12-10.3.5.147-py3-none-win_amd64.whl", hash = "sha256:f307cc191f96efe9e8f05a87096abc20d08845a841889ef78cb06924437f6771"}, ] [[package]] name = "nvidia-cusolver-cu12" version = "11.6.1.9" -requires_python = ">=3" -summary = "CUDA solver native runtime libraries" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" dependencies = [ "nvidia-cublas-cu12", "nvidia-cusparse-cu12", @@ -604,32 +710,24 @@ dependencies = [ files = [ {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d338f155f174f90724bbde3758b7ac375a70ce8e706d70b018dd3375545fc84e"}, {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260"}, - {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-win_amd64.whl", hash = "sha256:e77314c9d7b694fcebc84f58989f3aa4fb4cb442f12ca1a9bde50f5e8f6d1b9c"}, ] [[package]] name = "nvidia-cusparse-cu12" version = "12.3.1.170" -requires_python = ">=3" -summary = "CUSPARSE native runtime libraries" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" dependencies = [ "nvidia-nvjitlink-cu12", ] files = [ {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9d32f62896231ebe0480efd8a7f702e143c98cfaa0e8a76df3386c1ba2b54df3"}, {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1"}, - {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-win_amd64.whl", hash = "sha256:9bc90fb087bc7b4c15641521f31c0371e9a612fc2ba12c338d3ae032e6b6797f"}, ] [[package]] name = "nvidia-nccl-cu12" version = "2.21.5" -requires_python = ">=3" -summary = "NVIDIA Collective Communication Library (NCCL) Runtime" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" files = [ {file = "nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:8579076d30a8c24988834445f8d633c697d42397e92ffc3f63fa26766d25e0a0"}, ] @@ -637,46 +735,27 @@ files = [ [[package]] name = "nvidia-nvjitlink-cu12" version = "12.4.127" -requires_python = ">=3" -summary = "Nvidia JIT LTO Library" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" files = [ {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83"}, {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57"}, - {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:fd9020c501d27d135f983c6d3e244b197a7ccad769e34df53a42e276b0e25fa1"}, ] [[package]] name = "nvidia-nvtx-cu12" version = "12.4.127" -requires_python = ">=3" -summary = "NVIDIA Tools Extension" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\"" +summary = "" files = [ {file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7959ad635db13edf4fc65c06a6e9f9e55fc2f92596db928d169c0bb031e88ef3"}, {file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a"}, - {file = "nvidia_nvtx_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:641dccaaa1139f3ffb0d3164b4b84f9d253397e38246a4f2f36728b48566d485"}, ] [[package]] name = "opencv-python-headless" version = "4.10.0.84" -requires_python = ">=3.6" -summary = "Wrapper package for OpenCV python bindings." -groups = ["default"] +summary = "" dependencies = [ - "numpy>=1.13.3; python_version < \"3.7\"", - "numpy>=1.17.0; python_version >= \"3.7\"", - "numpy>=1.17.3; python_version >= \"3.8\"", - "numpy>=1.19.3; python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\"", - "numpy>=1.19.3; python_version >= \"3.9\"", - "numpy>=1.21.0; python_version <= \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\"", - "numpy>=1.21.2; python_version >= \"3.10\"", - "numpy>=1.21.4; python_version >= \"3.10\" and platform_system == \"Darwin\"", - "numpy>=1.23.5; python_version >= \"3.11\"", - "numpy>=1.26.0; python_version >= \"3.12\"", + "numpy", ] files = [ {file = "opencv-python-headless-4.10.0.84.tar.gz", hash = "sha256:f2017c6101d7c2ef8d7bc3b414c37ff7f54d64413a1847d89970b6b7069b4e1a"}, @@ -691,19 +770,37 @@ files = [ [[package]] name = "packaging" version = "24.1" -requires_python = ">=3.8" -summary = "Core utilities for Python packages" -groups = ["default"] +summary = "" files = [ {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] +[[package]] +name = "pandas" +version = "2.2.3" +summary = "" +dependencies = [ + "numpy", + "python-dateutil", + "pytz", + "tzdata", +] +files = [ + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, +] + [[package]] name = "pdf2image" version = "1.17.0" -summary = "A wrapper around the pdftoppm and pdftocairo command line tools to convert PDF to a PIL Image list." -groups = ["default"] +summary = "" dependencies = [ "pillow", ] @@ -715,9 +812,7 @@ files = [ [[package]] name = "pillow" version = "11.0.0" -requires_python = ">=3.9" -summary = "Python Imaging Library (Fork)" -groups = ["default"] +summary = "" files = [ {file = "pillow-11.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d2c0a187a92a1cb5ef2c8ed5412dd8d4334272617f532d4ad4de31e0495bd923"}, {file = "pillow-11.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:084a07ef0821cfe4858fe86652fffac8e187b6ae677e9906e192aafcc1b69903"}, @@ -733,11 +828,50 @@ files = [ {file = "pillow-11.0.0.tar.gz", hash = "sha256:72bacbaf24ac003fea9bff9837d1eedb6088758d41e100c1552930151f677739"}, ] +[[package]] +name = "propcache" +version = "0.2.0" +summary = "" +files = [ + {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2"}, + {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7"}, + {file = "propcache-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2e900bad2a8456d00a113cad8c13343f3b1f327534e3589acc2219729237a2e8"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f52a68c21363c45297aca15561812d542f8fc683c85201df0bebe209e349f793"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e41d67757ff4fbc8ef2af99b338bfb955010444b92929e9e55a6d4dcc3c4f09"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a64e32f8bd94c105cc27f42d3b658902b5bcc947ece3c8fe7bc1b05982f60e89"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55346705687dbd7ef0d77883ab4f6fabc48232f587925bdaf95219bae072491e"}, + {file = "propcache-0.2.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6994984550eaf25dd7fc7bd1b700ff45c894149341725bb4edc67f0ffa94efa4"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:56295eb1e5f3aecd516d91b00cfd8bf3a13991de5a479df9e27dd569ea23959c"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:439e76255daa0f8151d3cb325f6dd4a3e93043e6403e6491813bcaaaa8733887"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f6475a1b2ecb310c98c28d271a30df74f9dd436ee46d09236a6b750a7599ce57"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3444cdba6628accf384e349014084b1cacd866fbb88433cd9d279d90a54e0b23"}, + {file = "propcache-0.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a9d9b4d0a9b38d1c391bb4ad24aa65f306c6f01b512e10a8a34a2dc5675d348"}, + {file = "propcache-0.2.0-cp312-cp312-win32.whl", hash = "sha256:69d3a98eebae99a420d4b28756c8ce6ea5a29291baf2dc9ff9414b42676f61d5"}, + {file = "propcache-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ad9c9b99b05f163109466638bd30ada1722abb01bbb85c739c50b6dc11f92dc3"}, + {file = "propcache-0.2.0-py3-none-any.whl", hash = "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036"}, + {file = "propcache-0.2.0.tar.gz", hash = "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70"}, +] + +[[package]] +name = "pyarrow" +version = "18.0.0" +summary = "" +files = [ + {file = "pyarrow-18.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:e7ab04f272f98ebffd2a0661e4e126036f6936391ba2889ed2d44c5006237802"}, + {file = "pyarrow-18.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:03f40b65a43be159d2f97fd64dc998f769d0995a50c00f07aab58b0b3da87e1f"}, + {file = "pyarrow-18.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be08af84808dff63a76860847c48ec0416928a7b3a17c2f49a072cac7c45efbd"}, + {file = "pyarrow-18.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c70c1965cde991b711a98448ccda3486f2a336457cf4ec4dca257a926e149c9"}, + {file = "pyarrow-18.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:00178509f379415a3fcf855af020e3340254f990a8534294ec3cf674d6e255fd"}, + {file = "pyarrow-18.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a71ab0589a63a3e987beb2bc172e05f000a5c5be2636b4b263c44034e215b5d7"}, + {file = "pyarrow-18.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe92efcdbfa0bcf2fa602e466d7f2905500f33f09eb90bf0bcf2e6ca41b574c8"}, + {file = "pyarrow-18.0.0.tar.gz", hash = "sha256:a6aa027b1a9d2970cf328ccd6dbe4a996bc13c39fd427f502782f5bdb9ca20f5"}, +] + [[package]] name = "pyclipper" version = "1.3.0.post6" -summary = "Cython wrapper for the C++ translation of the Angus Johnson's Clipper library (ver. 6.4.2)" -groups = ["default"] +summary = "" files = [ {file = "pyclipper-1.3.0.post6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:6363b9d79ba1b5d8f32d1623e797c1e9f994600943402e68d5266067bdde173e"}, {file = "pyclipper-1.3.0.post6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32cd7fb9c1c893eb87f82a072dbb5e26224ea7cebbad9dc306d67e1ac62dd229"}, @@ -751,14 +885,11 @@ files = [ [[package]] name = "pydantic" version = "2.9.2" -requires_python = ">=3.8" -summary = "Data validation using Python type hints" -groups = ["default"] +summary = "" dependencies = [ - "annotated-types>=0.6.0", - "pydantic-core==2.23.4", - "typing-extensions>=4.12.2; python_version >= \"3.13\"", - "typing-extensions>=4.6.1; python_version < \"3.13\"", + "annotated-types", + "pydantic-core", + "typing-extensions", ] files = [ {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, @@ -768,11 +899,9 @@ files = [ [[package]] name = "pydantic-core" version = "2.23.4" -requires_python = ">=3.8" -summary = "Core functionality for Pydantic validation and serialization" -groups = ["default"] +summary = "" dependencies = [ - "typing-extensions!=4.7.0,>=4.6.0", + "typing-extensions", ] files = [ {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, @@ -793,9 +922,7 @@ files = [ [[package]] name = "pygments" version = "2.18.0" -requires_python = ">=3.8" -summary = "Pygments is a syntax highlighting package written in Python." -groups = ["default"] +summary = "" files = [ {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, @@ -804,8 +931,7 @@ files = [ [[package]] name = "python-bidi" version = "0.6.3" -summary = "Python Bidi layout wrapping the Rust crate unicode-bidi" -groups = ["default"] +summary = "" files = [ {file = "python_bidi-0.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:4bdc9dc1143c558ca6931d6712339a30470959f2b7eecb3d0687db7075c20a87"}, {file = "python_bidi-0.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0775499b8037103278f05b2bf92d25bf04f40a9f77884ec3d42b01a1e52a40fe"}, @@ -824,12 +950,22 @@ files = [ {file = "python_bidi-0.6.3.tar.gz", hash = "sha256:e12114969001a328aea859f79efc30ab9c15241befb86e07029d8961d97fae36"}, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +summary = "" +dependencies = [ + "six", +] +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + [[package]] name = "python-dotenv" version = "1.0.1" -requires_python = ">=3.8" -summary = "Read key-value pairs from a .env file and set them as environment variables" -groups = ["default"] +summary = "" files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, @@ -837,21 +973,45 @@ files = [ [[package]] name = "python-multipart" -version = "0.0.16" -requires_python = ">=3.8" -summary = "A streaming multipart parser for Python" -groups = ["default"] +version = "0.0.17" +summary = "" +files = [ + {file = "python_multipart-0.0.17-py3-none-any.whl", hash = "sha256:15dc4f487e0a9476cc1201261188ee0940165cffc94429b6fc565c4d3045cb5d"}, + {file = "python_multipart-0.0.17.tar.gz", hash = "sha256:41330d831cae6e2f22902704ead2826ea038d0419530eadff3ea80175aec5538"}, +] + +[[package]] +name = "pytorch-lightning" +version = "2.4.0" +summary = "" +dependencies = [ + "fsspec", + "lightning-utilities", + "packaging", + "pyyaml", + "torch", + "torchmetrics", + "tqdm", + "typing-extensions", +] +files = [ + {file = "pytorch-lightning-2.4.0.tar.gz", hash = "sha256:6aa897fd9d6dfa7b7b49f37c2f04e13592861831d08deae584dfda423fdb71c8"}, + {file = "pytorch_lightning-2.4.0-py3-none-any.whl", hash = "sha256:9ac7935229ac022ef06994c928217ed37f525ac6700f7d4fc57009624570e655"}, +] + +[[package]] +name = "pytz" +version = "2024.2" +summary = "" files = [ - {file = "python_multipart-0.0.16-py3-none-any.whl", hash = "sha256:c2759b7b976ef3937214dfb592446b59dfaa5f04682a076f78b117c94776d87a"}, - {file = "python_multipart-0.0.16.tar.gz", hash = "sha256:8dee37b88dab9b59922ca173c35acb627cc12ec74019f5cd4578369c6df36554"}, + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, ] [[package]] name = "pyyaml" version = "6.0.2" -requires_python = ">=3.8" -summary = "YAML parser and emitter for Python" -groups = ["default"] +summary = "" files = [ {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, @@ -868,9 +1028,7 @@ files = [ [[package]] name = "regex" version = "2024.9.11" -requires_python = ">=3.8" -summary = "Alternative regular expression module, to replace re." -groups = ["default"] +summary = "" files = [ {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b0d0a6c64fcc4ef9c69bd5b3b3626cc3776520a1637d8abaa62b9edc147a58f7"}, {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:49b0e06786ea663f933f3710a51e9385ce0cba0ea56b67107fd841a55d56a231"}, @@ -893,14 +1051,12 @@ files = [ [[package]] name = "requests" version = "2.32.3" -requires_python = ">=3.8" -summary = "Python HTTP for Humans." -groups = ["default"] +summary = "" dependencies = [ - "certifi>=2017.4.17", - "charset-normalizer<4,>=2", - "idna<4,>=2.5", - "urllib3<3,>=1.21.1", + "certifi", + "charset-normalizer", + "idna", + "urllib3", ] files = [ {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, @@ -909,26 +1065,21 @@ files = [ [[package]] name = "rich" -version = "13.9.3" -requires_python = ">=3.8.0" -summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -groups = ["default"] +version = "13.9.4" +summary = "" dependencies = [ - "markdown-it-py>=2.2.0", - "pygments<3.0.0,>=2.13.0", - "typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"", + "markdown-it-py", + "pygments", ] files = [ - {file = "rich-13.9.3-py3-none-any.whl", hash = "sha256:9836f5096eb2172c9e77df411c1b009bace4193d6a481d534fea75ebba758283"}, - {file = "rich-13.9.3.tar.gz", hash = "sha256:bc1e01b899537598cf02579d2b9f4a415104d3fc439313a7a2c165d76557a08e"}, + {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, + {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, ] [[package]] name = "safetensors" version = "0.4.5" -requires_python = ">=3.7" summary = "" -groups = ["default"] files = [ {file = "safetensors-0.4.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:473300314e026bd1043cef391bb16a8689453363381561b8a3e443870937cc1e"}, {file = "safetensors-0.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:801183a0f76dc647f51a2d9141ad341f9665602a7899a693207a82fb102cc53e"}, @@ -948,18 +1099,16 @@ files = [ [[package]] name = "scikit-image" version = "0.24.0" -requires_python = ">=3.9" -summary = "Image processing in Python" -groups = ["default"] +summary = "" dependencies = [ - "imageio>=2.33", - "lazy-loader>=0.4", - "networkx>=2.8", - "numpy>=1.23", - "packaging>=21", - "pillow>=9.1", - "scipy>=1.9", - "tifffile>=2022.8.12", + "imageio", + "lazy-loader", + "networkx", + "numpy", + "packaging", + "pillow", + "scipy", + "tifffile", ] files = [ {file = "scikit_image-0.24.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6fccceb54c9574590abcddc8caf6cefa57c13b5b8b4260ab3ff88ad8f3c252b3"}, @@ -973,11 +1122,9 @@ files = [ [[package]] name = "scipy" version = "1.14.1" -requires_python = ">=3.10" -summary = "Fundamental algorithms for scientific computing in Python" -groups = ["default"] +summary = "" dependencies = [ - "numpy<2.3,>=1.23.5", + "numpy", ] files = [ {file = "scipy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d"}, @@ -994,10 +1141,7 @@ files = [ [[package]] name = "setuptools" version = "75.2.0" -requires_python = ">=3.8" -summary = "Easily download, build, install, upgrade, and uninstall Python packages" -groups = ["default"] -marker = "python_version >= \"3.12\"" +summary = "" files = [ {file = "setuptools-75.2.0-py3-none-any.whl", hash = "sha256:a7fcb66f68b4d9e8e66b42f9876150a3371558f98fa32222ffaa5bced76406f8"}, {file = "setuptools-75.2.0.tar.gz", hash = "sha256:753bb6ebf1f465a1912e19ed1d41f403a79173a9acf66a42e7e6aec45c3c16ec"}, @@ -1006,11 +1150,9 @@ files = [ [[package]] name = "shapely" version = "2.0.6" -requires_python = ">=3.7" -summary = "Manipulation and analysis of geometric objects" -groups = ["default"] +summary = "" dependencies = [ - "numpy<3,>=1.14", + "numpy", ] files = [ {file = "shapely-2.0.6-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cec9193519940e9d1b86a3b4f5af9eb6910197d24af02f247afbfb47bcb3fab0"}, @@ -1025,20 +1167,25 @@ files = [ [[package]] name = "shellingham" version = "1.5.4" -requires_python = ">=3.7" -summary = "Tool to Detect Surrounding Shell" -groups = ["default"] +summary = "" files = [ {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] +[[package]] +name = "six" +version = "1.16.0" +summary = "" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "sniffio" version = "1.3.1" -requires_python = ">=3.7" -summary = "Sniff out which async library your code is running under" -groups = ["default"] +summary = "" files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, @@ -1047,12 +1194,9 @@ files = [ [[package]] name = "starlette" version = "0.41.2" -requires_python = ">=3.8" -summary = "The little ASGI library that shines." -groups = ["default"] +summary = "" dependencies = [ - "anyio<5,>=3.4.0", - "typing-extensions>=3.10.0; python_version < \"3.10\"", + "anyio", ] files = [ {file = "starlette-0.41.2-py3-none-any.whl", hash = "sha256:fbc189474b4731cf30fcef52f18a8d070e3f3b46c6a04c97579e85e6ffca942d"}, @@ -1062,12 +1206,9 @@ files = [ [[package]] name = "sympy" version = "1.13.1" -requires_python = ">=3.8" -summary = "Computer algebra system (CAS) in Python" -groups = ["default"] -marker = "python_version >= \"3.9\"" +summary = "" dependencies = [ - "mpmath<1.4,>=1.1.0", + "mpmath", ] files = [ {file = "sympy-1.13.1-py3-none-any.whl", hash = "sha256:db36cdc64bf61b9b24578b6f7bab1ecdd2452cf008f34faa33776680c26d66f8"}, @@ -1077,9 +1218,7 @@ files = [ [[package]] name = "tifffile" version = "2024.9.20" -requires_python = ">=3.10" -summary = "Read and write TIFF files" -groups = ["default"] +summary = "" dependencies = [ "numpy", ] @@ -1091,11 +1230,9 @@ files = [ [[package]] name = "tokenizers" version = "0.20.1" -requires_python = ">=3.7" summary = "" -groups = ["default"] dependencies = [ - "huggingface-hub<1.0,>=0.16.4", + "huggingface-hub", ] files = [ {file = "tokenizers-0.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:407ab666b38e02228fa785e81f7cf79ef929f104bcccf68a64525a54a93ceac9"}, @@ -1115,81 +1252,86 @@ files = [ [[package]] name = "torch" -version = "2.5.0" -requires_python = ">=3.8.0" -summary = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -groups = ["default"] +version = "2.5.1" +summary = "" dependencies = [ "filelock", "fsspec", "jinja2", "networkx", - "nvidia-cublas-cu12==12.4.5.8; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-cuda-cupti-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-cuda-nvrtc-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-cuda-runtime-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-cudnn-cu12==9.1.0.70; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-cufft-cu12==11.2.1.3; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-curand-cu12==10.3.5.147; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-cusolver-cu12==11.6.1.9; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-cusparse-cu12==12.3.1.170; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-nccl-cu12==2.21.5; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-nvjitlink-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "nvidia-nvtx-cu12==12.4.127; platform_system == \"Linux\" and platform_machine == \"x86_64\"", - "setuptools; python_version >= \"3.12\"", - "sympy==1.12.1; python_version == \"3.8\"", - "sympy==1.13.1; python_version >= \"3.9\"", - "triton==3.1.0; platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version < \"3.13\"", - "typing-extensions>=4.8.0", -] -files = [ - {file = "torch-2.5.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:2dd40c885a05ef7fe29356cca81be1435a893096ceb984441d6e2c27aff8c6f4"}, - {file = "torch-2.5.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:bc52d603d87fe1da24439c0d5fdbbb14e0ae4874451d53f0120ffb1f6c192727"}, - {file = "torch-2.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea718746469246cc63b3353afd75698a288344adb55e29b7f814a5d3c0a7c78d"}, - {file = "torch-2.5.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:6de1fd253e27e7f01f05cd7c37929ae521ca23ca4620cfc7c485299941679112"}, + "nvidia-cublas-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-cuda-cupti-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-cuda-nvrtc-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-cuda-runtime-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-cudnn-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-cufft-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-curand-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-cusolver-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-cusparse-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-nccl-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-nvjitlink-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "nvidia-nvtx-cu12; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "setuptools", + "sympy", + "triton; platform_machine == \"x86_64\" and platform_system == \"Linux\"", + "typing-extensions", +] +files = [ + {file = "torch-2.5.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:ed231a4b3a5952177fafb661213d690a72caaad97d5824dd4fc17ab9e15cec03"}, + {file = "torch-2.5.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:3f4b7f10a247e0dcd7ea97dc2d3bfbfc90302ed36d7f3952b0008d0df264e697"}, + {file = "torch-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:73e58e78f7d220917c5dbfad1a40e09df9929d3b95d25e57d9f8558f84c9a11c"}, + {file = "torch-2.5.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:8c712df61101964eb11910a846514011f0b6f5920c55dbf567bff8a34163d5b1"}, ] [[package]] name = "torchaudio" -version = "2.5.0" -summary = "An audio package for PyTorch" -groups = ["default"] +version = "2.5.1" +summary = "" dependencies = [ - "torch==2.5.0", + "torch", ] files = [ - {file = "torchaudio-2.5.0-1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:593258f33de1fa16ebe5718c9717daf3695460d48a0188194a5c574a710838cb"}, - {file = "torchaudio-2.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c835da771701b06fbe8e19ce643d5e587fd385e5f4d8f140551ce04900b1b96b"}, - {file = "torchaudio-2.5.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:3009ca3e095825911917ab865b2ea48abbf3e66f948c4ffd64916fe6e476bfec"}, - {file = "torchaudio-2.5.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:1971302bfc321be5ea18ba60dbc3ffdc6ae53cb47bb6427db25d5b69e4e932ec"}, - {file = "torchaudio-2.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e98ad102e0e574a397759078bc9809afc05de6e6f8ac0cb26d2245e0d3817adf"}, + {file = "torchaudio-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1cbfdfd1bbdfbe7289d47a74f36ff6c5d87c3205606202fef5a7fb693f61cf0"}, + {file = "torchaudio-2.5.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:9c8fb06fbd8d2016e7b7caf15a3231867c792a2e3b0f2f8f9013633e9c2ce412"}, + {file = "torchaudio-2.5.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:6bb65416405f40e00b20701257c16e7493bfdd7188e02e87cc5b389c31c10c2c"}, + {file = "torchaudio-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:ec8f12d6be12aed248a0d65a76c7bb341ee5eef969fe2e9dc3154c7cfba1bdf4"}, +] + +[[package]] +name = "torchmetrics" +version = "1.6.0" +summary = "" +dependencies = [ + "lightning-utilities", + "numpy", + "packaging", + "torch", +] +files = [ + {file = "torchmetrics-1.6.0-py3-none-any.whl", hash = "sha256:a508cdd87766cedaaf55a419812bf9f493aff8fffc02cc19df5a8e2e7ccb942a"}, + {file = "torchmetrics-1.6.0.tar.gz", hash = "sha256:aebba248708fb90def20cccba6f55bddd134a58de43fb22b0c5ca0f3a89fa984"}, ] [[package]] name = "torchvision" -version = "0.20.0" -requires_python = ">=3.8" -summary = "image and video datasets and models for torch deep learning" -groups = ["default"] +version = "0.20.1" +summary = "" dependencies = [ "numpy", - "pillow!=8.3.*,>=5.3.0", - "torch==2.5.0", + "pillow", + "torch", ] files = [ - {file = "torchvision-0.20.0-1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:f8d0213489acfb138369f2455a6893880c194a8195e381c19f872b277f2654c3"}, - {file = "torchvision-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ac0edba534fb071b2b03a2fd5cbbf9b7c259896d17a1d0d830b3c5b7dfae0782"}, - {file = "torchvision-0.20.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:c8f3bc399d9c3e4ba05d74ca6dd5e63fed08ad5c5b302a946c8fcaa56216220f"}, - {file = "torchvision-0.20.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:a78c99ebe1a62857b68e97ff9417b92f299f2ee61f009491a114ddad050c493d"}, - {file = "torchvision-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:bb0da0950d2034a0412c251a3a9117ff9612157f45177d37ba1b20b472c0864b"}, + {file = "torchvision-0.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1a31256ff945d64f006bb306813a7c95a531fe16bfb2535c837dd4c104533d7a"}, + {file = "torchvision-0.20.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:17cd78adddf81dac57d7dccc9277a4d686425b1c55715f308769770cb26cad5c"}, + {file = "torchvision-0.20.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:9f853ba4497ac4691815ad41b523ee23cf5ba4f87b1ce869d704052e233ca8b7"}, + {file = "torchvision-0.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:4a330422c36dbfc946d3a6c1caec3489db07ecdf3675d83369adb2e5a0ca17c4"}, ] [[package]] name = "tqdm" version = "4.66.5" -requires_python = ">=3.7" -summary = "Fast, Extensible Progress Meter" -groups = ["default"] +summary = "" dependencies = [ "colorama; platform_system == \"Windows\"", ] @@ -1200,33 +1342,29 @@ files = [ [[package]] name = "transformers" -version = "4.46.0" -requires_python = ">=3.8.0" -summary = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" -groups = ["default"] +version = "4.46.2" +summary = "" dependencies = [ "filelock", - "huggingface-hub<1.0,>=0.23.2", - "numpy>=1.17", - "packaging>=20.0", - "pyyaml>=5.1", - "regex!=2019.12.17", + "huggingface-hub", + "numpy", + "packaging", + "pyyaml", + "regex", "requests", - "safetensors>=0.4.1", - "tokenizers<0.21,>=0.20", - "tqdm>=4.27", + "safetensors", + "tokenizers", + "tqdm", ] files = [ - {file = "transformers-4.46.0-py3-none-any.whl", hash = "sha256:e161268ae8bee315eb9e9b4c0b27f1bd6980f91e0fc292d75249193d339704c0"}, - {file = "transformers-4.46.0.tar.gz", hash = "sha256:3a9e2eb537094db11c3652334d281afa4766c0e5091c4dcdb454e9921bb0d2b7"}, + {file = "transformers-4.46.2-py3-none-any.whl", hash = "sha256:c921f4406b78e6518c97b618c5acd1cf8a4f2315b6b727f4bf9e01496eef849c"}, + {file = "transformers-4.46.2.tar.gz", hash = "sha256:3d85410881e1c074be767877bf33c83231ec11529f274a6044ecb20c157ba14e"}, ] [[package]] name = "triton" version = "3.1.0" -summary = "A language and compiler for custom Deep Learning operations" -groups = ["default"] -marker = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version < \"3.13\"" +summary = "" dependencies = [ "filelock", ] @@ -1237,14 +1375,12 @@ files = [ [[package]] name = "typer" version = "0.12.5" -requires_python = ">=3.7" -summary = "Typer, build great CLIs. Easy to code. Based on Python type hints." -groups = ["default"] +summary = "" dependencies = [ - "click>=8.0.0", - "rich>=10.11.0", - "shellingham>=1.3.0", - "typing-extensions>=3.7.4.3", + "click", + "rich", + "shellingham", + "typing-extensions", ] files = [ {file = "typer-0.12.5-py3-none-any.whl", hash = "sha256:62fe4e471711b147e3365034133904df3e235698399bc4de2b36c8579298d52b"}, @@ -1254,20 +1390,25 @@ files = [ [[package]] name = "typing-extensions" version = "4.12.2" -requires_python = ">=3.8" -summary = "Backported and Experimental Type Hints for Python 3.8+" -groups = ["default"] +summary = "" files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "tzdata" +version = "2024.2" +summary = "" +files = [ + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, +] + [[package]] name = "urllib3" version = "2.2.3" -requires_python = ">=3.8" -summary = "HTTP library with thread-safe connection pooling, file post, and more." -groups = ["default"] +summary = "" files = [ {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, @@ -1276,13 +1417,10 @@ files = [ [[package]] name = "uvicorn" version = "0.32.0" -requires_python = ">=3.8" -summary = "The lightning-fast ASGI server." -groups = ["default"] +summary = "" dependencies = [ - "click>=7.0", - "h11>=0.8", - "typing-extensions>=4.0; python_version < \"3.11\"", + "click", + "h11", ] files = [ {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, @@ -1293,18 +1431,16 @@ files = [ name = "uvicorn" version = "0.32.0" extras = ["standard"] -requires_python = ">=3.8" -summary = "The lightning-fast ASGI server." -groups = ["default"] +summary = "" dependencies = [ - "colorama>=0.4; sys_platform == \"win32\"", - "httptools>=0.5.0", - "python-dotenv>=0.13", - "pyyaml>=5.1", + "colorama; sys_platform == \"win32\"", + "httptools", + "python-dotenv", + "pyyaml", "uvicorn==0.32.0", - "uvloop!=0.15.0,!=0.15.1,>=0.14.0; (sys_platform != \"cygwin\" and sys_platform != \"win32\") and platform_python_implementation != \"PyPy\"", - "watchfiles>=0.13", - "websockets>=10.4", + "uvloop; platform_python_implementation != \"PyPy\" and (sys_platform != \"cygwin\" and sys_platform != \"win32\")", + "watchfiles", + "websockets", ] files = [ {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, @@ -1314,10 +1450,7 @@ files = [ [[package]] name = "uvloop" version = "0.21.0" -requires_python = ">=3.8.0" -summary = "Fast implementation of asyncio event loop on top of libuv" -groups = ["default"] -marker = "(sys_platform != \"cygwin\" and sys_platform != \"win32\") and platform_python_implementation != \"PyPy\"" +summary = "" files = [ {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c"}, {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2"}, @@ -1331,11 +1464,9 @@ files = [ [[package]] name = "watchfiles" version = "0.24.0" -requires_python = ">=3.8" -summary = "Simple, modern and high performance file watching and code reload in python." -groups = ["default"] +summary = "" dependencies = [ - "anyio>=3.0.0", + "anyio", ] files = [ {file = "watchfiles-0.24.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7211b463695d1e995ca3feb38b69227e46dbd03947172585ecb0588f19b0d87a"}, @@ -1357,9 +1488,7 @@ files = [ [[package]] name = "websockets" version = "13.1" -requires_python = ">=3.8" -summary = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -groups = ["default"] +summary = "" files = [ {file = "websockets-13.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d75baf00138f80b48f1eac72ad1535aac0b6461265a0bcad391fc5aba875cfc"}, {file = "websockets-13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9b6f347deb3dcfbfde1c20baa21c2ac0751afaa73e64e5b693bb2b848efeaa49"}, @@ -1375,3 +1504,56 @@ files = [ {file = "websockets-13.1-py3-none-any.whl", hash = "sha256:a9a396a6ad26130cdae92ae10c36af09d9bfe6cafe69670fd3b6da9b07b4044f"}, {file = "websockets-13.1.tar.gz", hash = "sha256:a3b3366087c1bc0a2795111edcadddb8b3b59509d5db5d7ea3fdd69f954a8878"}, ] + +[[package]] +name = "xxhash" +version = "3.5.0" +summary = "" +files = [ + {file = "xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00"}, + {file = "xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e"}, + {file = "xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8"}, + {file = "xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e"}, + {file = "xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2"}, + {file = "xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f"}, +] + +[[package]] +name = "yarl" +version = "1.17.1" +summary = "" +dependencies = [ + "idna", + "multidict", + "propcache", +] +files = [ + {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:327828786da2006085a4d1feb2594de6f6d26f8af48b81eb1ae950c788d97f61"}, + {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc353841428d56b683a123a813e6a686e07026d6b1c5757970a877195f880c2d"}, + {file = "yarl-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c73df5b6e8fabe2ddb74876fb82d9dd44cbace0ca12e8861ce9155ad3c886139"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdff5e0995522706c53078f531fb586f56de9c4c81c243865dd5c66c132c3b5"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06157fb3c58f2736a5e47c8fcbe1afc8b5de6fb28b14d25574af9e62150fcaac"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1654ec814b18be1af2c857aa9000de7a601400bd4c9ca24629b18486c2e35463"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6595c852ca544aaeeb32d357e62c9c780eac69dcd34e40cae7b55bc4fb1147"}, + {file = "yarl-1.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459e81c2fb920b5f5df744262d1498ec2c8081acdcfe18181da44c50f51312f7"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e48cdb8226644e2fbd0bdb0a0f87906a3db07087f4de77a1b1b1ccfd9e93685"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d9b6b28a57feb51605d6ae5e61a9044a31742db557a3b851a74c13bc61de5172"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e594b22688d5747b06e957f1ef822060cb5cb35b493066e33ceac0cf882188b7"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5f236cb5999ccd23a0ab1bd219cfe0ee3e1c1b65aaf6dd3320e972f7ec3a39da"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a2a64e62c7a0edd07c1c917b0586655f3362d2c2d37d474db1a509efb96fea1c"}, + {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d0eea830b591dbc68e030c86a9569826145df485b2b4554874b07fea1275a199"}, + {file = "yarl-1.17.1-cp312-cp312-win32.whl", hash = "sha256:46ddf6e0b975cd680eb83318aa1d321cb2bf8d288d50f1754526230fcf59ba96"}, + {file = "yarl-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:117ed8b3732528a1e41af3aa6d4e08483c2f0f2e3d3d7dca7cf538b3516d93df"}, + {file = "yarl-1.17.1-py3-none-any.whl", hash = "sha256:f1790a4b1e8e8e028c391175433b9c8122c39b46e1663228158e61e6f915bf06"}, + {file = "yarl-1.17.1.tar.gz", hash = "sha256:067a63fcfda82da6b198fa73079b1ca40b7c9b7994995b6ee38acda728b64d47"}, +] diff --git a/pyproject.toml b/pyproject.toml index 130781c..bc8cb87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,8 @@ dependencies = [ "numpy>=2.1.2", "rich>=13.9.3", "transformers>=4.46.0", + "lightning>=2.4.0", + "datasets>=3.1.0", ] requires-python = ">=3.12,<3.13" readme = "README.md" diff --git a/requirements.txt b/requirements.txt index dacc99b..81d70c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ # This file is @generated by PDM. # Please do not edit it manually. +datasets>=3.1.0 easyocr>=1.7.2 fastapi[standard]>=0.115.3 +lightning>=2.4.0 numpy>=2.1.2 pdf2image>=1.17.0 pillow>=11.0.0 diff --git a/setup-services.sh b/setup-services.sh new file mode 100755 index 0000000..b59fd8c --- /dev/null +++ b/setup-services.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Usage: ./setup.sh [--force|-f] +# Options: +# --force, -f Remove existing models and clone again + +# Initialize FORCE flag +FORCE=false + +export DOCKER_DEFAULT_PLATFORM=linux/arm64 + +# Parse command line arguments +while [[ "$#" -gt 0 ]]; do + case $1 in + -f|--force) + FORCE=true + ;; + *) + echo "Unknown parameter: $1" + echo "Usage: ./setup.sh [--force|-f]" + exit 1 + ;; + esac + shift +done + +# Create models and logs directories if they don't exist +mkdir -p models +mkdir -p logs + +# Check if git-lfs is installed +if ! command -v git-lfs &> /dev/null +then + echo "git-lfs could not be found. Please install it before running this script." + exit 1 +fi + +# Navigate to models directory +cd models + +# Remove existing cloned repositories if FORCE is true +if [ "$FORCE" = true ]; then + echo "Removing existing models..." + rm -rf layoutlmv3-base layoutlmv3-base-finetuned-rvlcdip + + git clone https://huggingface.co/microsoft/layoutlmv3-base + git clone https://huggingface.co/gordonlim/layoutlmv3-base-finetuned-rvlcdip +fi + +# Navigate back to the project root +cd .. + +# Start Docker services +docker-compose up --build --abort-on-container-failure \ No newline at end of file diff --git a/src/configs/model_config.py b/src/configs/model_config.py new file mode 100644 index 0000000..1799810 --- /dev/null +++ b/src/configs/model_config.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass(frozen=True) +class ModelConfig: + """Configuration for document processor.""" + + LOG_DIR: Path = Path("/app/data/logs/predictor") + LOG_LEVEL: str = "DEBUG" + DOCUMENT_CLASSES: list[str] = field( + default_factory=lambda: [ + "letter", + "form", + "email", + "handwritten", + "advertisement", + "scientific report", + "scientific publication", + "specification", + "file folder", + "news article", + "budget", + "invoice", + "presentation", + "questionnaire", + "resume", + "memo", + ] + ) diff --git a/src/configs/ocr_config.py b/src/configs/ocr_config.py new file mode 100644 index 0000000..e136a58 --- /dev/null +++ b/src/configs/ocr_config.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Set + +BYTE_MEGABYTE: Final[int] = 1024 * 1024 + + +@dataclass(frozen=True) +class OCRConfig: + """Configuration settings for OCR processing.""" + + FILE_SIZE_LIMIT: int = 20 * BYTE_MEGABYTE + MAX_PDF_PAGES: int = 15 + TARGET_SIZE: int = 1028 + MAX_WORKERS: int = 4 + LOG_DIR: Path = Path("/app/data/logs/ocr") + ACCEPTED_FILE_TYPES: Set[str] = frozenset( + { + "image/jpeg", + "image/png", + "image/jpg", + "image/webp", + "application/pdf", + } + ) + PROCESSOR_URL: str = "http://processor:9090/text-preprocess" + LOG_LEVEL: str = "DEBUG" diff --git a/src/configs/processor_config.py b/src/configs/processor_config.py new file mode 100644 index 0000000..c8929fc --- /dev/null +++ b/src/configs/processor_config.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class ProcessorConfig: + """Configuration for document processor.""" + + LOG_DIR: Path = Path("/app/data/logs/processor") + IMAGE_TARGET_SIZE: int = 1000 + MAX_SEQUENCE_LENGTH: int = 512 + BATCH_SIZE: int = 1 + MODEL_NAME: str = "microsoft/layoutlmv3-base" + FILE_SIZE_LIMIT_MB: int = 20 + MAX_PDF_PAGES: int = 5 + LOG_LEVEL: str = "DEBUG" + + PREDICT_URL: str = "http://predictor:7070/predict-class" diff --git a/src/documentclassification/ocr/ocr.py b/src/documentclassification/ocr/ocr.py deleted file mode 100644 index 256164b..0000000 --- a/src/documentclassification/ocr/ocr.py +++ /dev/null @@ -1,132 +0,0 @@ -import base64 -import io -import logging -from datetime import datetime -from pathlib import Path - -import requests -from fastapi import FastAPI, HTTPException, UploadFile, status -from pdf2image import convert_from_bytes -from pdf2image.exceptions import PDFPageCountError -from PIL import Image - -from .optimized_ocr import OptimizedOCR - -LOG_DIR = Path("logs/ocr") -LOG_DIR.mkdir(parents=True, exist_ok=True) - -CURRENT_DATE = datetime.now().strftime("%Y-%m-%d") - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[ - logging.FileHandler(LOG_DIR / f"{CURRENT_DATE}.log", mode="a"), - logging.StreamHandler(), - ], -) - -logger = logging.getLogger(__name__) -app = FastAPI() -optimizer = OptimizedOCR(target_size=1024) -FILE_SIZE_LIMIT = 20 * 1024 * 1024 # 20MB -ACCEPTED_FILE_TYPES = { - "image/jpeg", - "image/png", - "image/jpg", - "image/webp", - "application/pdf", -} - - -def validate_file(file: UploadFile) -> None: - logger.info(f"Validating file: {file.filename}") - - if file.content_type not in ACCEPTED_FILE_TYPES: - logger.error(f"Unsupported file type: {file.content_type}") - raise HTTPException( - status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, - detail="Unsupported file type", - ) - - if file.size > FILE_SIZE_LIMIT: - logger.error(f"File size too large: {file.size} bytes") - raise HTTPException( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - detail="File size too large", - ) - - if file.content_type == "application/pdf": - try: - page_count = len(convert_from_bytes(file.file.read())) - if page_count > 15: - logger.error(f"PDF has more than 15 pages: {page_count} pages") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="PDF has more than 15 pages", - ) - except PDFPageCountError: - logger.error( - "Unable to get page count. The document stream might be empty or corrupted." - ) - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Unable to get page count. The document stream might be empty or corrupted.", - ) - finally: - file.file.seek(0) - - -def convert_file_to_image(file: UploadFile) -> list[Image.Image]: - logger.info(f"Converting file to image: {file.filename}") - - if file.content_type == "application/pdf": - return convert_from_bytes(file.file.read(), fmt="jpeg") - - return [Image.open(file.file)] - - -def convert_images_to_base64(images: list[Image.Image]) -> list[str]: - """ - Convert PIL Images to base64 strings properly. - - Args: - images: List of PIL Image objects - - Returns: - List of base64 encoded image strings - """ - encoded_images = [] - - for img in images: - img_byte_arr = io.BytesIO() - img.save(img_byte_arr, format="JPEG") - img_byte_arr = img_byte_arr.getvalue() - - encoded = base64.b64encode(img_byte_arr).decode("utf-8") - encoded_images.append(encoded) - - return encoded_images - - -@app.post("/ocr") -async def read_text_from_file(file: UploadFile) -> None: - logger.info(f"Received file for OCR: {file.filename}") - - validate_file(file) - images = convert_file_to_image(file) - logger.info(f"Processing OCR for file: {file.filename}") - ocr_result = optimizer.process_batch(images) - - logger.info(f"OCR processing completed for file: {file.filename}") - - encoded_images = convert_images_to_base64(images) - - requests.post( - "http://processor:9090/processor", - json={ - "ocr_result": ocr_result, - "images": encoded_images, - }, - ) diff --git a/src/documentclassification/ocr/optimized_ocr.py b/src/documentclassification/ocr/optimized_ocr.py deleted file mode 100644 index dd18fc2..0000000 --- a/src/documentclassification/ocr/optimized_ocr.py +++ /dev/null @@ -1,119 +0,0 @@ -import logging - -import cv2 -import easyocr -import numpy as np -import rich -from PIL import Image - - -class OptimizedOCR: - def __init__(self, max_workers: int = 4, target_size: int = 1280): - """ - Initialize OptimizedOCR with performance optimizations. - - Args: - max_workers: Number of worker threads for parallel processing - target_size: Target image size for resizing - """ - self.max_workers = max_workers - self.target_size = target_size - self.reader = self._initialize_reader() - - def _initialize_reader(self) -> easyocr.Reader: - """Initialize EasyOCR with optimized settings.""" - reader = easyocr.Reader( - ["en", "pl"], - gpu=False, - model_storage_directory="./models", - download_enabled=True, - quantize=True, - ) - return reader - - def preprocess_image(self, image: Image.Image) -> np.ndarray: - """ - Optimize image for OCR processing. - - Args: - image: Input image as numpy array - - Returns: - Preprocessed image - """ - # Convert to grayscale if needed - if image.mode != "L": - image = image.convert("L") - - # Resize image while maintaining aspect ratio - # h, w = image.size - # logging.info(f"Original image size: {w}x{h}") - # scale = min(self.target_size / w, self.target_size / h) - # scale = 0.75 - # new_w, new_h = int(w * scale), int(h * scale) - # image = image.resize((new_w, new_h), resample=Image.Resampling.LANCZOS) - # logging.info(f"Resized image size: {new_w}x{new_h}") - # image.show() - - image = np.array(image) - - # Enhance contrast using CLAHE - clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) - image = clahe.apply(image) - - # Denoise - image = cv2.fastNlMeansDenoising(image) - - return image - - def process_batch(self, images: list[Image.Image]) -> list[list[dict]]: - """ - Process images in batches for better CPU utilization. - - Args: - images: list of images to process - - Returns: - list of OCR results for each image - """ - results = [] - preprocessed = [self.preprocess_image(img) for img in images] - for img in rich.progress.track( - preprocessed, description="Processing images..." - ): - image_results = self._process_single(img) - results.extend(image_results) - - return results - - def _process_single(self, image: np.ndarray) -> list[dict]: - """Process a single image with optimized settings.""" - try: - # Use lower confidence threshold for faster processing - results = self.reader.readtext( - image, - # paragraph=True, # Group text into paragraphs - # batch_size=1, - # workers=8, - # beamWidth=2, # Reduce beam width for faster inference - # contrast_ths=0.1, # Lower contrast threshold - # adjust_contrast=0.5, # Reduce contrast adjustment - canvas_size=self.target_size, # Resize image for better OCR - # low_text=0.3, # Lower text confidence threshold - ) - - return [ - { - "bounding_box": self.create_bounding_box(bbox), - "word": word, - } - for bbox, word, _ in results - ] - except Exception as e: - logging.error(f"Error processing image: {str(e)}") - return [] - - @staticmethod - def create_bounding_box(bbox_data: list[tuple[float, float]]) -> list[int]: - xs, ys = zip(*bbox_data) - return [int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys))] diff --git a/src/documentclassification/process/process.py b/src/documentclassification/process/process.py deleted file mode 100644 index 378b08a..0000000 --- a/src/documentclassification/process/process.py +++ /dev/null @@ -1,113 +0,0 @@ -import base64 -import io -import logging -from datetime import datetime -from pathlib import Path - -import torchvision.transforms.v2 as v2 -from fastapi import FastAPI -from PIL import Image -from transformers import ( - LayoutLMv3ImageProcessor, - LayoutLMv3Processor, - LayoutLMv3TokenizerFast, -) - -# Constants -LOG_DIR = Path("logs/process") -LOG_DIR.mkdir(parents=True, exist_ok=True) - -CURRENT_DATE = datetime.now().strftime("%Y-%m-%d") -LOG_FILE = LOG_DIR / f"{CURRENT_DATE}.log" - -FILE_SIZE_LIMIT_MB = 20 -MAX_PDF_PAGES = 5 -IMAGE_TARGET_SIZE = 1000 - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - handlers=[ - logging.FileHandler(LOG_FILE, mode="a"), - logging.StreamHandler(), - ], -) -logger = logging.getLogger(__name__) - -app = FastAPI() -processor = LayoutLMv3Processor( - LayoutLMv3ImageProcessor(apply_ocr=False), - LayoutLMv3TokenizerFast.from_pretrained("microsoft/layoutlmv3-base"), -) -transform = v2.ToPILImage() - - -def scale_bounding_box( - box: list[int], width_scale: float = 1.0, height_scale: float = 1.0 -) -> list[int]: - """Scales bounding box coordinates based on provided scales.""" - return [ - int(coord * scale) for coord, scale in zip(box, [width_scale, height_scale] * 2) - ] - - -@app.post("/process") -def process_text(body: dict[str, list[dict[str, list[int] | str] | bytes]]) -> None: - """Processes OCR result and images for further analysis.""" - logger.info("Starting text processing.") - - try: - images = [ - Image.open(io.BytesIO(base64.b64decode(img))).convert("RGB") - for img in body.get("images", []) - ] - if not images: - logger.error("No images found in the request body.") - return - - width, height = images[0].size - width_scale = IMAGE_TARGET_SIZE / width - height_scale = IMAGE_TARGET_SIZE / height - - ocr_results = body.get("ocr_result", []) - words, boxes = ( - zip( - *( - ( - item["word"], - scale_bounding_box( - item["bounding_box"], width_scale, height_scale - ), - ) - for item in ocr_results - ) - ) - if ocr_results - else ([], []) - ) - - encoding = processor( - images, - words, - boxes=list(boxes), - max_length=512, - padding="max_length", - truncation=True, - return_tensors="pt", - ) - - logger.info( - f"Encoding shapes - input_ids: {list(encoding['input_ids'].shape)}, " - f"bbox: {list(encoding['bbox'].shape)}, " - f"pixel_values: {list(encoding['pixel_values'].shape)}, " - f"image size: {images[0].size}" - ) - - image_data = transform(encoding["pixel_values"][0]) - # Further processing can be done with image_data - - except Exception as e: - logger.exception("An error occurred during text processing.") - raise e diff --git a/src/ocr/ocr.py b/src/ocr/ocr.py new file mode 100644 index 0000000..1fb8b29 --- /dev/null +++ b/src/ocr/ocr.py @@ -0,0 +1,139 @@ +import base64 +import io +import logging +from datetime import datetime + +import requests +from fastapi import FastAPI, HTTPException, UploadFile, status +from optimized_ocr import OptimizedOCR +from pdf2image import convert_from_bytes +from pdf2image.exceptions import PDFPageCountError +from PIL import Image + +from configs.ocr_config import OCRConfig + +config = OCRConfig() +config.LOG_DIR.mkdir(parents=True, exist_ok=True) + + +logging.basicConfig( + level=logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.FileHandler( + config.LOG_DIR / f"{datetime.now():%Y-%m-%d}.log", mode="a" + ), + logging.StreamHandler(), + ], +) + +logger = logging.getLogger(__name__) + +app = FastAPI() +optimizer = OptimizedOCR(config=config) + + +class OCRProcessor: + """Handle OCR processing operations.""" + + @staticmethod + def validate_file(file: UploadFile) -> None: + """Validate uploaded file against constraints.""" + logger.info("Validating file: %s", file.filename) + + if file.content_type not in config.ACCEPTED_FILE_TYPES: + logger.error("Unsupported file type: %s", file.content_type) + raise HTTPException( + status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, + detail="Unsupported file type", + ) + + if file.size > config.FILE_SIZE_LIMIT: + logger.error("File size too large: %s bytes", file.size) + raise HTTPException( + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + detail="File size too large", + ) + + if file.content_type == "application/pdf": + try: + page_count = len(convert_from_bytes(file.file.read())) + if page_count > config.MAX_PDF_PAGES: + logger.error("PDF has too many pages: %d", page_count) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"PDF exceeds {config.MAX_PDF_PAGES} pages", + ) + except PDFPageCountError as e: + logger.error("PDF processing error: %s", str(e)) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid or corrupted PDF file", + ) from e + finally: + file.file.seek(0) + + @staticmethod + def convert_file_to_images(file: UploadFile) -> list[Image.Image]: + """Convert uploaded file to list of PIL Images.""" + logger.info("Converting file to images: %s", file.filename) + + if file.content_type == "application/pdf": + return convert_from_bytes(file.file.read(), fmt="jpeg") + + return [Image.open(file.file)] + + @staticmethod + def convert_to_base64(images: list[Image.Image]) -> list[str]: + """Convert PIL Images to base64 strings.""" + encoded_images = [] + + for img in images: + with io.BytesIO() as buffer: + img.save(buffer, format="JPEG") + img_byte_arr = buffer.getvalue() + + encoded = base64.b64encode(img_byte_arr).decode("utf-8") + encoded_images.append(encoded) + + return encoded_images + + +@app.post("/ocr") +async def process_document(file: UploadFile) -> dict[str, str]: + """Process document for OCR and forward results.""" + logger.info("Processing document: %s", file.filename) + + processor = OCRProcessor() + + try: + processor.validate_file(file) + images = processor.convert_file_to_images(file) + + ocr_response, preprocessed_images = optimizer.process_batch(images) + encoded_images = processor.convert_to_base64(preprocessed_images) + + logger.debug("Processed images: %d", len(preprocessed_images)) + + response = requests.post( + config.PROCESSOR_URL, + json={ + "ocr_result": ocr_response.model_dump(exclude={"page_count"})[ + "results" + ], + "images": encoded_images, + }, + timeout=30, + ) + + logger.info("Document processing completed: %s", file.filename) + + return response.json() + + except Exception as e: + logger.error("Processing error: %s", str(e)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Document processing failed", + ) from e diff --git a/src/ocr/optimized_ocr.py b/src/ocr/optimized_ocr.py new file mode 100644 index 0000000..c7c1ef8 --- /dev/null +++ b/src/ocr/optimized_ocr.py @@ -0,0 +1,99 @@ +import logging + +import cv2 +import easyocr +import numpy as np +from numpy.typing import NDArray +from PIL import Image + +from configs.ocr_config import OCRConfig +from payload.ocr_models import OCRResponse, OCRResult + + +class ImagePreprocessor: + """Handle image preprocessing operations.""" + + def __init__(self, target_size: int) -> None: + self.target_size = target_size + self._clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) + + def resize_image(self, image: Image.Image) -> Image.Image: + """Resize image while maintaining aspect ratio.""" + if self.target_size == 0 or max(image.size) <= self.target_size: + return image + + width, height = image.size + scale = min(self.target_size / width, self.target_size / height) + new_size = tuple(int(dim * scale) for dim in (width, height)) + + return image.resize(new_size, resample=Image.Resampling.LANCZOS) + + def enhance_image(self, image: NDArray[np.uint8]) -> NDArray[np.uint8]: + """Apply image enhancement techniques.""" + enhanced = self._clahe.apply(image) + + return cv2.fastNlMeansDenoising(enhanced) + + def preprocess(self, image: Image.Image) -> NDArray[np.uint8]: + """Complete image preprocessing pipeline.""" + if image.mode != "L": + image = image.convert("L") + + image = self.resize_image(image) + image_array = np.array(image) + + return self.enhance_image(image_array) + + +class OptimizedOCR: + """Optimized OCR processing with performance enhancements.""" + + def __init__(self, config: OCRConfig) -> None: + """Initialize OCR processor with configuration.""" + self.config = config + self.preprocessor = ImagePreprocessor(config.TARGET_SIZE) + self.reader = self._initialize_reader() + + def _initialize_reader(self) -> easyocr.Reader: + """Initialize EasyOCR with optimized settings.""" + return easyocr.Reader( + ["en", "pl"], + gpu=False, + model_storage_directory="/app/data/models", + download_enabled=True, + quantize=True, + ) + + @staticmethod + def create_bounding_box(bbox_data: list[tuple[float, float]]) -> list[int]: + """Create standardized bounding box from coordinates.""" + xs, ys = zip(*bbox_data) + + return [int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys))] + + def _process_single(self, image: NDArray[np.uint8]) -> list[OCRResult]: + """Process a single image with optimized settings.""" + try: + results = self.reader.readtext(image) + return [ + OCRResult( + bounding_box=self.create_bounding_box(bbox), + word=word, + ) + for bbox, word, _ in results + ] + except Exception as e: + logging.error("Error processing image: %s", str(e)) + return [] + + def process_batch( + self, images: list[Image.Image] + ) -> tuple[OCRResponse, list[Image.Image]]: + """Process multiple images in optimized batch.""" + processed = [self.preprocessor.preprocess(image) for image in images] + resized = [self.preprocessor.resize_image(image) for image in images] + results = [self._process_single(image) for image in processed] + + response = OCRResponse(results=results, page_count=len(results)) + + return response, resized diff --git a/src/payload/model_models.py b/src/payload/model_models.py new file mode 100644 index 0000000..834d0a0 --- /dev/null +++ b/src/payload/model_models.py @@ -0,0 +1,36 @@ +from dataclasses import dataclass + +import torch +from pydantic import BaseModel, ConfigDict + + +@dataclass +class PagePrediction: + """Stores prediction results for a single page.""" + + page_number: int + logits: torch.Tensor + confidence: float + predicted_class: str + text_length: int + + +class ProcessorResult(BaseModel): + """Result data for document processing.""" + + input_ids: list + attention_mask: list + bbox: list + pixel_values: list + + model_config = ConfigDict( + arbitrary_types_allowed=True, + json_schema_extra={ + "example": { + "input_ids": [[1, 2, 3, 4, 5]], + "attention_mask": [[1, 1, 1, 1, 1]], + "bbox": [[0, 0, 100, 50]], + "pixel_values": [[0, 0, 0, 255, 255, 255]], + } + }, + ) diff --git a/src/payload/ocr_models.py b/src/payload/ocr_models.py new file mode 100644 index 0000000..a127e8d --- /dev/null +++ b/src/payload/ocr_models.py @@ -0,0 +1,35 @@ +from pydantic import BaseModel, ConfigDict, Field + + +class OCRResult(BaseModel): + """Structured OCR result using Pydantic BaseModel.""" + + bounding_box: list[int] = Field( + ..., + description="Coordinates of bounding box [x_min, y_min, x_max, y_max]", + min_items=4, + max_items=4, + ) + word: str = Field(..., description="Recognized text") + + model_config = ConfigDict( + frozen=True, + json_schema_extra={ + "example": { + "bounding_box": [100, 100, 200, 150], + "word": "Example", + } + }, + ) + + +class OCRResponse(BaseModel): + """Response model for OCR processing.""" + + results: list[list[OCRResult]] = Field( + ..., description="list of OCR results from the document" + ) + + page_count: int = Field(1, description="Number of pages processed", ge=1) + + model_config = ConfigDict(frozen=True) diff --git a/src/payload/processor_models.py b/src/payload/processor_models.py new file mode 100644 index 0000000..de8e7d7 --- /dev/null +++ b/src/payload/processor_models.py @@ -0,0 +1,72 @@ +import torch +from pydantic import BaseModel, ConfigDict, Field + + +class BoundingBox(BaseModel): + """Represents a bounding box for OCR results.""" + + coordinates: list[int] = Field( + ..., + min_items=4, + max_items=4, + description="Coordinates in format [x_min, y_min, x_max, y_max]", + ) + + +class OCRResult(BaseModel): + """Structured OCR result using Pydantic BaseModel.""" + + bounding_box: list[int] = Field( + ..., + description="Coordinates of bounding box [x_min, y_min, x_max, y_max]", + min_items=4, + max_items=4, + ) + word: str = Field(..., description="Recognized text") + + model_config = ConfigDict( + frozen=True, + json_schema_extra={ + "example": { + "bounding_box": [100, 100, 200, 150], + "word": "Example", + } + }, + ) + + +class ProcessorInput(BaseModel): + """Input data for document processing.""" + + ocr_result: list[list[OCRResult]] + images: list[str] = Field(..., description="Base64 encoded images") + + model_config = ConfigDict( + json_schema_extra={ + "example": { + "ocr_result": [{"word": "Example", "bounding_box": [0, 0, 100, 50]}], + "images": ["base64_encoded_image_string"], + } + } + ) + + +class ProcessorResult(BaseModel): + """Result data for document processing.""" + + input_ids: torch.Tensor + attention_mask: torch.Tensor + bbox: torch.Tensor + pixel_values: torch.Tensor + + model_config = ConfigDict( + arbitrary_types_allowed=True, + json_schema_extra={ + "example": { + "input_ids": [[1, 2, 3, 4, 5]], + "attention_mask": [[1, 1, 1, 1, 1]], + "bbox": [[0, 0, 100, 50]], + "pixel_values": [[0, 0, 0, 255, 255, 255]], + } + }, + ) diff --git a/src/predictor/model.py b/src/predictor/model.py new file mode 100644 index 0000000..8a877f2 --- /dev/null +++ b/src/predictor/model.py @@ -0,0 +1,232 @@ +import logging +from datetime import datetime +from typing import Any + +import lightning.pytorch as pl +import torch +import torch.nn.functional as F +from fastapi import FastAPI, HTTPException +from torchmetrics import Accuracy +from transformers import LayoutLMv3ForSequenceClassification, PreTrainedModel + +from configs.model_config import ModelConfig +from payload.model_models import PagePrediction, ProcessorResult + +config = ModelConfig() +config.LOG_DIR.mkdir(parents=True, exist_ok=True) + +logging.basicConfig( + level=logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.FileHandler( + config.LOG_DIR / f"{datetime.now():%Y-%m-%d}.log", mode="a" + ), + logging.StreamHandler(), + ], +) + +logger = logging.getLogger(__name__) + +app = FastAPI() + + +class ModelModule(pl.LightningModule): + def __init__(self, n_classes: int = 15): + super().__init__() + self.model = LayoutLMv3ForSequenceClassification.from_pretrained( + "/app/data/models/layoutlmv3-base-finetuned-rvlcdip", + num_labels=n_classes, + use_safetensors=True, + ) + self.model.config.id2label = { + k: v for k, v in enumerate(config.DOCUMENT_CLASSES) + } + self.model.config.label2id = { + v: k for k, v in enumerate(config.DOCUMENT_CLASSES) + } + self.train_accuracy = Accuracy(task="multiclass", num_classes=n_classes) + self.val_accuracy = Accuracy(task="multiclass", num_classes=n_classes) + + def forward(self, input_ids, attention_mask, bbox, pixel_values): + return self.model( + input_ids, + attention_mask=attention_mask, + bbox=bbox, + pixel_values=pixel_values, + ) + + def training_step(self, batch, batch_idx): + input_ids = batch["input_ids"] + attention_mask = batch["attention_mask"] + bbox = batch["bbox"] + pixel_values = batch["pixel_values"] + labels = batch["labels"] + + output = self(input_ids, attention_mask, bbox, pixel_values, labels) + + self.log("train_loss", output.loss) + self.log( + "train_acc", + self.train_accuracy(output.logits, labels), + on_step=True, + on_epoch=True, + ) + + return output.loss + + def validation_step(self, batch, batch_idx): + input_ids = batch["input_ids"] + attention_mask = batch["attention_mask"] + bbox = batch["bbox"] + pixel_values = batch["pixel_values"] + labels = batch["labels"] + + output = self(input_ids, attention_mask, bbox, pixel_values, labels) + + self.log("val_loss", output.loss) + self.log( + "val_acc", + self.val_accuracy(output.logits, labels), + on_step=False, + on_epoch=True, + ) + + return output.loss + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.model.parameters(), lr=1e-5) + return optimizer + + +model = ModelModule(n_classes=len(config.DOCUMENT_CLASSES)) +model.eval() + + +def _calculate_page_weights(page_predictions: list[PagePrediction]) -> torch.Tensor: + """Calculate weights for each page based on multiple factors. + + Args: + page_predictions: List of predictions for each page + + Returns: + Tensor of normalized weights for each page + """ + text_lengths = torch.tensor([pred.text_length for pred in page_predictions]) + confidences = torch.tensor([pred.confidence for pred in page_predictions]) + + if text_lengths.max() > 0: + normalized_lengths = text_lengths / text_lengths.max() + else: + normalized_lengths = torch.ones_like(text_lengths) + + weights = (normalized_lengths * confidences) + 1e-6 + + for i, (raw_weight, length, conf) in enumerate( + zip(weights, text_lengths, confidences) + ): + logger.debug( + f"Page {i+1} pre-normalization: " + f"weight={raw_weight:.4f}, " + f"text_length={length}, " + f"confidence={conf:.4f}" + ) + + temperature = 1.0 + normalized_weights = F.softmax(weights / temperature, dim=0) + + for i, weight in enumerate(normalized_weights): + logger.debug(f"Page {i+1} final weight: {weight:.4f}") + + return normalized_weights + + +def aggregate_predictions( + model: PreTrainedModel, page_predictions: list[PagePrediction] +) -> dict[str, Any]: + """Aggregate predictions from multiple pages using weighted voting. + + Args: + page_predictions: List of predictions for each page + + Returns: + Dictionary containing final classification results + """ + if not page_predictions: + raise ValueError("No valid predictions to aggregate") + + weights = _calculate_page_weights(page_predictions) + + stacked_logits = torch.stack([pred.logits for pred in page_predictions]) + weighted_logits = (stacked_logits * weights.unsqueeze(-1)).sum(dim=0) + + probabilities = F.softmax(weighted_logits, dim=-1) + confidence, predicted_class = torch.max(probabilities, dim=-1) + + return { + "predicted_class": model.config.id2label[predicted_class.item()], + "confidence": confidence.item(), + "page_predictions": [ + { + "page": pred.page_number, + "class": pred.predicted_class, + "confidence": pred.confidence, + "weight": weight.item(), + } + for pred, weight in zip(page_predictions, weights) + ], + "probability_distribution": { + model.config.id2label[i]: prob.item() + for i, prob in enumerate(probabilities) + }, + } + + +@app.post("/predict-class") +async def predict(data: ProcessorResult) -> dict[str, str]: + """ + Predict document class from processed data. + + Args: + data: Processed data from document processor + """ + try: + logger.info("Received data for prediction") + input_ids = torch.tensor(data.input_ids) + attention_mask = torch.tensor(data.attention_mask) + bbox = torch.tensor(data.bbox) + pixel_values = torch.tensor(data.pixel_values) + + with torch.inference_mode(): + output = model(input_ids, attention_mask, bbox, pixel_values) + + probabilities = F.softmax(output.logits, dim=-1) + confidence, predicted_class = torch.max(probabilities, dim=-1) + + predictions = [] + for i in range(output.logits.size(0)): + pred_class = predicted_class[i].item() + conf = confidence[i].item() + text_length = torch.sum(attention_mask[i]).item() + + predictions.append( + PagePrediction( + page_number=i, + logits=output.logits[i], + confidence=conf, + predicted_class=model.model.config.id2label[pred_class], + text_length=text_length, + ) + ) + + detailed_results = aggregate_predictions(model.model, predictions) + + logger.debug("Prediction results: %s", detailed_results) + logger.info("Predicted class: %s", detailed_results["predicted_class"]) + + return {"predicted_classes": detailed_results["predicted_class"]} + + except Exception as e: + logger.exception("Prediction failed") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/src/processor/dataset.py b/src/processor/dataset.py new file mode 100644 index 0000000..f2f1f82 --- /dev/null +++ b/src/processor/dataset.py @@ -0,0 +1,205 @@ +import base64 +import io +import logging +from datetime import datetime +from typing import Optional + +import torch +from PIL import Image +from torch.utils.data import Dataset +from transformers import ProcessorMixin + +from configs.processor_config import ProcessorConfig +from payload.processor_models import OCRResult, ProcessorInput + +config = ProcessorConfig() +config.LOG_DIR.mkdir(parents=True, exist_ok=True) + +logging.basicConfig( + level=logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.FileHandler( + config.LOG_DIR / f"{datetime.now():%Y-%m-%d}.log", mode="a" + ), + logging.StreamHandler(), + ], +) + +logger = logging.getLogger(__name__) + + +class DocumentClassificationDataset(Dataset): + """Dataset for document classification using LayoutLMv3.""" + + def __init__( + self, + processor: ProcessorMixin, + data: ProcessorInput, + config: ProcessorConfig, + labels: Optional[list[int]] = None, + *, + encodings: Optional[dict[str, torch.Tensor]] = None, + ) -> None: + """ + Initialize document classification dataset. + + Args: + processor: LayoutLMv3 processor instance + data: Validated input data + config: Processing configuration + labels: Optional classification labels + """ + self.processor = processor + self.config = config + self.labels = labels + self.encodings = ( + encodings if encodings is not None else self._prepare_encodings(data) + ) + + @classmethod + def get_encodings( + cls, + processor: ProcessorMixin, + data: ProcessorInput, + config: ProcessorConfig, + ) -> dict[str, torch.Tensor]: + """ + Get document encodings without instantiating the full dataset. + + This method processes the input data and returns only the encodings, + which is useful when you don't need the full dataset functionality. + + Args: + processor: LayoutLMv3 processor instance + data: Validated input data containing OCR results and images + config: Processing configuration settings + + Returns: + dict[str, torch.Tensor]: Encoded document features + + Raises: + ValueError: If encoding preparation fails + """ + temp_instance = cls( + processor=processor, + data=data, + config=config, + encodings={}, + ) + + return temp_instance._prepare_encodings(data) + + def _prepare_encodings(self, data: ProcessorInput) -> dict[str, torch.Tensor]: + """Prepare encodings from input data for multiple pages.""" + try: + images = self._decode_images(data.images) + if not images: + raise ValueError("No valid images found in input data") + + scales = self._calculate_scales(images[0].size) + words_per_page, boxes_per_page = self._prepare_ocr_data( + data.ocr_result, scales + ) + + # Process each page separately + page_encodings = [] + for img, words, boxes in zip(images, words_per_page, boxes_per_page): + encoding = self.processor( + [img], # Processor expects a list of images + words, + boxes=boxes, + max_length=self.config.MAX_SEQUENCE_LENGTH, + padding="max_length", + truncation=True, + return_tensors="pt", + ) + page_encodings.append(encoding) + + # Combine encodings from all pages + combined_encodings = { + "input_ids": torch.cat( + [enc["input_ids"] for enc in page_encodings], dim=0 + ), + "attention_mask": torch.cat( + [enc["attention_mask"] for enc in page_encodings], dim=0 + ), + "bbox": torch.cat([enc["bbox"] for enc in page_encodings], dim=0), + "pixel_values": torch.cat( + [enc["pixel_values"] for enc in page_encodings], dim=0 + ), + } + + logger.debug( + "Combined encoding shapes: %s", + {k: v.shape for k, v in combined_encodings.items()}, + ) + + return combined_encodings + + except Exception as e: + logger.exception("Error preparing encodings") + raise ValueError(f"Encoding preparation failed: {str(e)}") from e + + @staticmethod + def _decode_images(encoded_images: list[str]) -> list[Image.Image]: + """Decode base64 images to PIL Images.""" + return [ + Image.open(io.BytesIO(base64.b64decode(img))).convert("RGB") + for img in encoded_images + ] + + def _calculate_scales(self, image_size: tuple[int, int]) -> tuple[float, float]: + """Calculate scaling factors for image resizing.""" + width, height = image_size + width_scale = self.config.IMAGE_TARGET_SIZE / width + height_scale = self.config.IMAGE_TARGET_SIZE / height + return width_scale, height_scale + + @staticmethod + def _prepare_ocr_data( + ocr_results: list[list[OCRResult]], scales: tuple[float, float] + ) -> tuple[list[str], list[list[int]]]: + """Prepare OCR data with scaling.""" + if not ocr_results: + return [], [] + + words = [] + boxes = [] + width_scale, height_scale = scales + + for page_results in ocr_results: + tmp_word, tmp_box = [], [] + for result in page_results: + tmp_word.append(result.word) + scaled_box = [ + int(coord * scale) + for coord, scale in zip( + result.bounding_box, [width_scale, height_scale] * 2 + ) + ] + tmp_box.append(scaled_box) + + words.append(tmp_word) + boxes.append(tmp_box) + + return words, boxes + + def __len__(self) -> int: + """Get dataset length.""" + return len(self.encodings["input_ids"]) + + def __getitem__(self, idx: int) -> dict[str, torch.Tensor]: + """Get a single item from the dataset.""" + item = { + key: tensor[idx].flatten() + if key not in ("pixel_values", "bbox") + else tensor[idx].flatten(end_dim=1) + for key, tensor in self.encodings.items() + } + + if self.labels is not None: + item["labels"] = torch.tensor(self.labels[idx]) + + return item diff --git a/src/processor/processor.py b/src/processor/processor.py new file mode 100644 index 0000000..0484f4c --- /dev/null +++ b/src/processor/processor.py @@ -0,0 +1,104 @@ +import logging +from datetime import datetime +from pathlib import Path + +import requests +from dataset import DocumentClassificationDataset +from fastapi import FastAPI, HTTPException +from transformers import ( + LayoutLMv3ImageProcessor, + LayoutLMv3Processor, + LayoutLMv3TokenizerFast, +) + +from configs.processor_config import ProcessorConfig +from payload.processor_models import ProcessorInput, ProcessorResult + +config = ProcessorConfig() +# TMP solution to avoid error during testing +try: + config.LOG_DIR.mkdir(parents=True, exist_ok=True) +except OSError: + config = ProcessorConfig(LOG_DIR=Path("logs/processor")) + config.LOG_DIR.mkdir(parents=True, exist_ok=True) + +logging.basicConfig( + level=logging.DEBUG if config.LOG_LEVEL == "DEBUG" else logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.FileHandler( + config.LOG_DIR / f"{datetime.now():%Y-%m-%d}.log", mode="a" + ), + logging.StreamHandler(), + ], +) + +logger = logging.getLogger(__name__) + +app = FastAPI() + + +class DocumentProcessor: + """Document processing service using LayoutLMv3.""" + + def __init__(self) -> None: + """Initialize document processor.""" + self.processor = LayoutLMv3Processor( + LayoutLMv3ImageProcessor(apply_ocr=False), + LayoutLMv3TokenizerFast.from_pretrained("/app/data/models/layoutlmv3-base"), + ) + + def process_document(self, input_data: ProcessorInput) -> ProcessorResult: + """ + Process document with LayoutLMv3. + + Args: + input_data: Validated input data containing OCR results and images + """ + try: + encodings: list[dict] = DocumentClassificationDataset.get_encodings( + self.processor, input_data, config + ) + + result = ProcessorResult( + input_ids=encodings["input_ids"], + attention_mask=encodings["attention_mask"], + bbox=encodings["bbox"], + pixel_values=encodings["pixel_values"], + ) + + return result + + except Exception as e: + logger.exception("Document processing failed") + raise HTTPException(status_code=500, detail=f"Processing failed: {str(e)}") + + +# Create processor instance +document_processor = DocumentProcessor() + + +@app.post("/text-preprocess") +async def process_text(data: ProcessorInput) -> dict[str, str]: + """ + Process document text and layout. + + Args: + data: Validated input data containing OCR results and images + """ + try: + encodings = document_processor.process_document(data) + + encodings = {k: v.tolist() for k, v in encodings.model_dump().items()} + + response = requests.post( + config.PREDICT_URL, + json=encodings, + timeout=30, + ) + return response.json() + + except Exception as e: + logger.exception("Endpoint processing failed") + raise HTTPException(status_code=500, detail=str(e))