From e4bdd9552e8988278c55f957a664e5b97c0d1578 Mon Sep 17 00:00:00 2001 From: Konstantin Alekseev Date: Sun, 7 Jan 2024 16:05:20 +0200 Subject: [PATCH 1/2] Weasyprint layer for python3.12 lambda --- .github/workflows/ci.yml | 72 +++---------------------------- Makefile | 79 +++++++++++------------------------ fonts/layer_builder.sh | 10 ++--- weasyprint/lambda_function.py | 17 ++------ weasyprint/layer_builder.sh | 41 +++++++++++------- 5 files changed, 64 insertions(+), 155 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb44f85..4447e28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,12 +10,13 @@ jobs: - uses: actions/checkout@v2 - name: Build weasyprint run: | - make build/weasyprint-layer-python3.8.zip + make build/weasyprint-layer-python3.12.zip - name: Test weasyprint run: | mkdir output - TEST_FILENAME=output/report.pdf make test.weasyprint - TEST_FILENAME=output/report.png make test.weasyprint + make test.start.container & + sleep 1 + TEST_FILENAME=output/report.pdf make test.print.report - name: Upload pdf uses: actions/upload-artifact@v1 with: @@ -37,67 +38,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_weasyprint_release.outputs.upload_url }} - asset_path: ./build/weasyprint-layer-python3.8.zip - asset_name: weasyprint-layer-python3.8.zip - asset_content_type: application/zip - - wkhtmltox-build: - name: Build wkhtmltox - runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[ci skip]') " - steps: - - uses: actions/checkout@v2 - - name: Build wkhtmltox - run: | - make build/wkhtmltox-layer.zip - make build/wkhtmltopdf-layer.zip - make build/wkhtmltoimage-layer.zip - - name: Test wkhtmltox - run: | - mkdir output - TEST_FILENAME=output/google.pdf make test.wkhtmltox - TEST_FILENAME=output/google.png make test.wkhtmltox - - name: Upload pdf - uses: actions/upload-artifact@v1 - with: - name: wkhtmltox test results - path: output - - name: Create wkhtmltox Release - id: create_wkhtmltox_release - if: startsWith(github.ref, 'refs/tags/wkhtmltox-') - uses: actions/create-release@latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release wkhtmltox layer - - name: Upload wkhtmltox Layer - if: startsWith(github.ref, 'refs/tags/wkhtmltox-') - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_wkhtmltox_release.outputs.upload_url }} - asset_path: ./build/wkhtmltox-layer.zip - asset_name: wkhtmltox-layer.zip - asset_content_type: application/zip - - name: Upload wkhtmltopdf Layer - if: startsWith(github.ref, 'refs/tags/wkhtmltox-') - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_wkhtmltox_release.outputs.upload_url }} - asset_path: ./build/wkhtmltopdf-layer.zip - asset_name: wkhtmltopdf-layer.zip - asset_content_type: application/zip - - name: Upload wkhtmltoimage Layer - if: startsWith(github.ref, 'refs/tags/wkhtmltox-') - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_wkhtmltox_release.outputs.upload_url }} - asset_path: ./build/wkhtmltoimage-layer.zip - asset_name: wkhtmltoimage-layer.zip + asset_path: ./build/weasyprint-layer-python3.12.zip + asset_name: weasyprint-layer-python3.12.zip asset_content_type: application/zip diff --git a/Makefile b/Makefile index 8565da9..be9d15c 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,30 @@ -RUNTIME ?= python3.8 +PLATFORM ?= linux/amd64 +RUNTIME ?= 3.12 TEST_FILENAME ?= report.pdf +DOCKER_RUN=docker run --rm --platform=${PLATFORM} -.PHONY: stack.deploy.weasyprint clean test.weasyprint +.PHONY: stack.deploy.weasyprint clean test.start.container test.print.report -all: build/weasyprint-layer-$(RUNTIME).zip build/wkhtmltopdf-layer.zip +all: build/weasyprint-layer-python$(RUNTIME).zip -build/weasyprint-layer-$(RUNTIME).zip: weasyprint/layer_builder.sh \ +build/weasyprint-layer-python$(RUNTIME).zip: weasyprint/layer_builder.sh \ build/fonts-layer.zip \ | _build - docker run --rm \ + ${DOCKER_RUN} \ -v `pwd`/weasyprint:/out \ - -t lambci/lambda:build-${RUNTIME} \ - bash /out/layer_builder.sh + --entrypoint "/out/layer_builder.sh" \ + -t public.ecr.aws/lambda/python:${RUNTIME} mv -f ./weasyprint/layer.zip ./build/weasyprint-no-fonts-layer.zip cd build && rm -rf ./opt && mkdir opt \ && unzip fonts-layer.zip -d opt \ && unzip weasyprint-no-fonts-layer.zip -d opt \ - && cd opt && zip -r9 ../weasyprint-layer-${RUNTIME}.zip . + && cd opt && zip -r9 ../weasyprint-layer-python${RUNTIME}.zip . build/fonts-layer.zip: fonts/layer_builder.sh | _build - docker run --rm \ - -e INSTALL_MS_FONTS="${INSTALL_MS_FONTS}" \ + ${DOCKER_RUN} \ -v `pwd`/fonts:/out \ - -t lambci/lambda:build-${RUNTIME} \ - bash /out/layer_builder.sh + --entrypoint "/out/layer_builder.sh" \ + -t public.ecr.aws/lambda/python:${RUNTIME} mv -f ./fonts/layer.zip $@ stack.diff: @@ -34,50 +35,22 @@ stack.deploy: cd cdk-stacks && npm install && npm run build cdk deploy --app ./cdk-stacks/bin/app.js --stack PrintStack --parameters uploadBucketName=${BUCKET} -test.weasyprint: - docker run --rm \ +test.start.container: + ${DOCKER_RUN} \ -e GDK_PIXBUF_MODULE_FILE="/opt/lib/loaders.cache" \ -e FONTCONFIG_PATH="/opt/fonts" \ -e XDG_DATA_DIRS="/opt/lib" \ -v `pwd`/weasyprint:/var/task \ -v `pwd`/build/opt:/opt \ - lambci/lambda:${RUNTIME} \ - lambda_function.lambda_handler \ - '{"url": "https://kotify.github.io/cloud-print-utils/samples/report/", "filename": "${TEST_FILENAME}", "return": "base64"}' \ - | tail -1 | jq .body | tr -d '"' | base64 -d > ${TEST_FILENAME} - @echo "Check ./${TEST_FILENAME}, eg.: xdg-open ${TEST_FILENAME}" - - -build/wkhtmltox-layer.zip: wkhtmltox/layer_builder.sh \ - build/fonts-layer.zip \ - | _build - docker run --rm \ - -v `pwd`/wkhtmltox:/out \ - -t lambci/lambda:build-${RUNTIME} \ - bash /out/layer_builder.sh - mv -f ./wkhtmltox/layer.zip ./build/wkhtmltox-no-fonts-layer.zip - cd build && rm -rf ./opt && mkdir opt \ - && unzip fonts-layer.zip -d opt \ - && unzip wkhtmltox-no-fonts-layer.zip -d opt \ - && cd opt && zip -r9 ../wkhtmltox-layer.zip . - -build/wkhtmltopdf-layer.zip: build/wkhtmltox-layer.zip - cp build/wkhtmltox-layer.zip $@ - zip -d $@ "bin/wkhtmltoimage" - -build/wkhtmltoimage-layer.zip: build/wkhtmltox-layer.zip - cp build/wkhtmltox-layer.zip $@ - zip -d $@ "bin/wkhtmltopdf" - -test.wkhtmltox: - docker run --rm \ - -e FONTCONFIG_PATH="/opt/fonts" \ - -v `pwd`/wkhtmltox:/var/task \ - -v `pwd`/build/opt:/opt \ - lambci/lambda:${RUNTIME} \ - lambda_function.lambda_handler \ - '{"args": "https://google.com", "filename": "${TEST_FILENAME}", "return": "base64"}' \ - | tail -1 | jq .body | tr -d '"' | base64 -d > ${TEST_FILENAME} + -p 9000:8080 \ + public.ecr.aws/lambda/python:${RUNTIME} \ + lambda_function.lambda_handler + +test.print.report: + which jq + curl --fail -s -S -XPOST "http://localhost:9000/2015-03-31/functions/function/invocations" \ + -d '{"return": "base64", "filename": "${TEST_FILENAME}", "url": "https://kotify.github.io/cloud-print-utils/samples/report/"}' \ + | tail -1 | jq .body | tr -d '"' | base64 -d > ${TEST_FILENAME} @echo "Check ./${TEST_FILENAME}, eg.: xdg-open ${TEST_FILENAME}" @@ -86,7 +59,3 @@ _build: clean: rm -rf ./build - -fonts.list: - docker run --rm lambci/lambda:build-${RUNTIME} \ - bash -c "yum search font | grep noarch | grep -v texlive" diff --git a/fonts/layer_builder.sh b/fonts/layer_builder.sh index e8bb413..b4779e9 100755 --- a/fonts/layer_builder.sh +++ b/fonts/layer_builder.sh @@ -2,13 +2,13 @@ # don't fortget to set FONTCONFIG_PATH="/opt/fonts" in your lambda set -e -yum upgrade -y ca-certificates -yum install -y yum-utils rpmdevtools +dnf install -y rpmdevtools + cd /tmp # download fonts -yumdownloader \ - dejavu-sans-fonts \ - dejavu-fonts-common +dnf download dejavu-sans-fonts +dnf download dejavu-serif-fonts +dnf download dejavu-sans-mono-fonts rpmdev-extract -- *rpm diff --git a/weasyprint/lambda_function.py b/weasyprint/lambda_function.py index f851020..f9cb9ac 100644 --- a/weasyprint/lambda_function.py +++ b/weasyprint/lambda_function.py @@ -9,19 +9,10 @@ def lambda_handler(event, context): filename = event["filename"] basename = os.path.basename(filename) tmpfile = f"/tmp/{basename}" - ext = os.path.splitext(basename)[1] - if ext == ".pdf": - content_type = "application/pdf" - method = "write_pdf" - elif ext == ".png": - content_type = "image/png" - method = "write_png" - else: - raise ValueError("File extension {ext} is not supported.") if "url" in event: - getattr(HTML(url=event["url"]), method)(target=tmpfile) + HTML(url=event["url"]).write_pdf(target=tmpfile) else: - getattr(HTML(string=event["html"]), method)( + HTML(string=event["html"]).write_pdf( target=tmpfile, stylesheets=[CSS(string=event["css"])] if "css" in event else None, ) @@ -31,7 +22,7 @@ def lambda_handler(event, context): return { "statusCode": 200, "headers": { - "Content-type": content_type, + "Content-type": "application/pdf", "Content-Disposition": f"attachment;filename={basename}", }, "isBase64Encoded": True, @@ -47,7 +38,7 @@ def lambda_handler(event, context): open(tmpfile, "rb"), bucket, filename, - ExtraArgs={"ContentType": content_type}, + ExtraArgs={"ContentType": "application/pdf"}, ) url = s3.generate_presigned_url( ClientMethod="get_object", diff --git a/weasyprint/layer_builder.sh b/weasyprint/layer_builder.sh index 73cd6da..8d088a9 100755 --- a/weasyprint/layer_builder.sh +++ b/weasyprint/layer_builder.sh @@ -3,18 +3,30 @@ # GDK_PIXBUF_MODULE_FILE="/opt/lib/loaders.cache" # XDG_DATA_DIRS="/opt/lib" set -e -yum install -y yum-utils rpmdevtools +dnf install -y rpmdevtools cd /tmp -yumdownloader --resolve \ - cairo.x86_64 \ - gdk-pixbuf2.x86_64 \ - libffi.x86_64 \ - pango.x86_64 \ - expat.x86_64 \ - libmount.x86_64 \ - libuuid.x86_64 \ - libblkid.x86_64 \ - glib2.x86_64 \ +dnf download cairo +dnf download gdk-pixbuf2 +dnf download libffi +dnf download pango +dnf download expat +dnf download libmount +dnf download libuuid +dnf download libblkid +dnf download glib2 +dnf download libthai +dnf download fribidi +dnf download harfbuzz +dnf download libdatrie +dnf download freetype +dnf download graphite2 +dnf download libbrotli +dnf download libpng +dnf download fontconfig + +# pixbuf need mime database +# https://www.linuxtopia.org/online_books/linux_desktop_guides/gnome_2.14_admin_guide/mimetypes-database.html +dnf download shared-mime-info rpmdev-extract -- *rpm @@ -26,14 +38,11 @@ PIXBUF_BIN=$(find /tmp -name gdk-pixbuf-query-loaders-64) GDK_PIXBUF_MODULEDIR=$(find /opt/lib/gdk-pixbuf-2.0/ -name loaders) export GDK_PIXBUF_MODULEDIR $PIXBUF_BIN > /opt/lib/loaders.cache -# pixbuf need mime database -# https://www.linuxtopia.org/online_books/linux_desktop_guides/gnome_2.14_admin_guide/mimetypes-database.html -cp -r /usr/share/mime /opt/lib/mime -RUNTIME=$(echo "$AWS_EXECUTION_ENV" | cut -d _ -f 3) +RUNTIME=$(grep AWS_EXECUTION_ENV "$LAMBDA_RUNTIME_DIR/bootstrap" | cut -d _ -f 5) export RUNTIME mkdir -p "/opt/python/lib/$RUNTIME/site-packages" -python -m pip install "weasyprint<53.0" -t "/opt/python/lib/$RUNTIME/site-packages" +python -m pip install "weasyprint" -t "/opt/python/lib/$RUNTIME/site-packages" cd /opt zip -r9 /out/layer.zip lib/* python/* From 09a8107972e61406b82e7e0de932c01bc53060ab Mon Sep 17 00:00:00 2001 From: Konstantin Alekseev Date: Sat, 17 Feb 2024 16:39:47 +0200 Subject: [PATCH 2/2] Add ghostscript --- .github/workflows/ci.yml | 58 ++++++++++++++++++++++-------------- Makefile | 11 +++++-- fonts/layer_builder.sh | 10 ++----- ghostscript/layer_builder.sh | 12 ++++++++ 4 files changed, 58 insertions(+), 33 deletions(-) create mode 100755 ghostscript/layer_builder.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4447e28..f1679c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,39 +5,51 @@ jobs: weasyprint-build: name: Build WeasyPrint runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[ci skip]') " steps: - - uses: actions/checkout@v2 - - name: Build weasyprint - run: | - make build/weasyprint-layer-python3.12.zip + - uses: actions/checkout@v4 + - name: Build Layer + run: make build/weasyprint-layer-python3.12.zip - name: Test weasyprint run: | mkdir output make test.start.container & sleep 1 TEST_FILENAME=output/report.pdf make test.print.report - - name: Upload pdf - uses: actions/upload-artifact@v1 + rm -rf build/opt + - name: Upload Build + uses: actions/upload-artifact@v4 + with: + name: WeasyPrint Layer Build + path: build + - name: Upload Test PDF + uses: actions/upload-artifact@v4 with: - name: WeasyPrint test results + name: WeasyPrint Test Results path: output - name: Create WeasyPrint Release - id: create_weasyprint_release if: startsWith(github.ref, 'refs/tags/weasyprint-') - uses: actions/create-release@latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: softprops/action-gh-release@4634c16e79c963813287e889244c50009e7f0981 with: - tag_name: ${{ github.ref }} - release_name: Release WeasyPrint layer - - name: Upload WeasyPrint Layer - if: startsWith(github.ref, 'refs/tags/weasyprint-') - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + files: | + ./build/weasyprint-layer-python3.12.zip + ./build/weasyprint-layer-python3.12-no-fonts.zip + + ghostscript-build: + name: Build GhostScript + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build Layer + run: | + make build/ghostscript-layer.zip + rm -rf build/opt + - name: Upload Build + uses: actions/upload-artifact@v4 + with: + name: GhostScript Layer Build + path: build + - name: Create GhostScript Release + if: startsWith(github.ref, 'refs/tags/ghostscript-') + uses: softprops/action-gh-release@4634c16e79c963813287e889244c50009e7f0981 with: - upload_url: ${{ steps.create_weasyprint_release.outputs.upload_url }} - asset_path: ./build/weasyprint-layer-python3.12.zip - asset_name: weasyprint-layer-python3.12.zip - asset_content_type: application/zip + files: ./build/build/ghostscript-layer.zip diff --git a/Makefile b/Makefile index be9d15c..ad001b6 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ build/weasyprint-layer-python$(RUNTIME).zip: weasyprint/layer_builder.sh \ -v `pwd`/weasyprint:/out \ --entrypoint "/out/layer_builder.sh" \ -t public.ecr.aws/lambda/python:${RUNTIME} - mv -f ./weasyprint/layer.zip ./build/weasyprint-no-fonts-layer.zip + mv -f ./weasyprint/layer.zip ./build/weasyprint-layer-python${RUNTIME}-no-fonts.zip cd build && rm -rf ./opt && mkdir opt \ && unzip fonts-layer.zip -d opt \ - && unzip weasyprint-no-fonts-layer.zip -d opt \ + && unzip weasyprint-layer-python${RUNTIME}-no-fonts.zip -d opt \ && cd opt && zip -r9 ../weasyprint-layer-python${RUNTIME}.zip . build/fonts-layer.zip: fonts/layer_builder.sh | _build @@ -27,6 +27,13 @@ build/fonts-layer.zip: fonts/layer_builder.sh | _build -t public.ecr.aws/lambda/python:${RUNTIME} mv -f ./fonts/layer.zip $@ +build/ghostscript-layer.zip: ghostscript/layer_builder.sh | _build + ${DOCKER_RUN} \ + -v `pwd`/ghostscript:/out \ + --entrypoint "/out/layer_builder.sh" \ + -t public.ecr.aws/lambda/python:${RUNTIME} + mv -f ./ghostscript/layer.zip $@ + stack.diff: cd cdk-stacks && npm install && npm run build cdk diff --app ./cdk-stacks/bin/app.js --stack PrintStack --parameters uploadBucketName=${BUCKET} diff --git a/fonts/layer_builder.sh b/fonts/layer_builder.sh index b4779e9..5a2f1d8 100755 --- a/fonts/layer_builder.sh +++ b/fonts/layer_builder.sh @@ -13,16 +13,10 @@ dnf download dejavu-sans-mono-fonts rpmdev-extract -- *rpm mkdir /opt/fonts +# dnf download urw-base35-nimbus-roman-fonts +# find /tmp/*/usr/share/fonts -name '*.afm' -delete -o -name '*.t1' -delete cp -P -r /tmp/*/usr/share/fonts/* /opt/fonts -if [ "$INSTALL_MS_FONTS" = "yes" ]; then - PYTHON=python2 amazon-linux-extras install epel -y - yum install -y fontconfig xorg-x11-font-utils cabextract - curl -L -O https://downloads.sourceforge.net/project/mscorefonts2/rpms/msttcore-fonts-installer-2.6-1.noarch.rpm - rpm -i msttcore-fonts-installer-2.6-1.noarch.rpm - cp -P -r /usr/share/fonts/msttcore /opt/fonts/ -fi - cat > /opt/fonts/fonts.conf < diff --git a/ghostscript/layer_builder.sh b/ghostscript/layer_builder.sh new file mode 100755 index 0000000..2790fda --- /dev/null +++ b/ghostscript/layer_builder.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +export VERSION="10.02.1" + +dnf install -y gcc tar +cd /tmp/ +curl -L -s https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs${VERSION//./}/ghostscript-${VERSION}.tar.gz | tar zxf - +cd ghostscript-* +./configure --prefix=/opt +make && make install +cd /opt +zip -r9 /out/layer.zip bin/gs