From 7927017e96531b042b70f0de24be40fd88b44b63 Mon Sep 17 00:00:00 2001 From: Aaron Sturm Date: Fri, 6 Sep 2019 16:24:49 -0700 Subject: [PATCH 1/2] Convert compile.sh to Docker - Copy layers.zip from the container to the current folder - Removing compile.sh as it's not needed with Docker --- .gitignore | 1 + compile.sh => Dockerfile | 97 +++++++++++++++++++++++----------------- config/centos.repo | 6 +++ docker-compose.yml | 8 ++++ 4 files changed, 71 insertions(+), 41 deletions(-) rename compile.sh => Dockerfile (59%) create mode 100644 config/centos.repo create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index 7053cda..53cea8e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.tfstate.backup *.backup lo.tar.gz +layers.zip diff --git a/compile.sh b/Dockerfile similarity index 59% rename from compile.sh rename to Dockerfile index 368d3a8..ad4267b 100644 --- a/compile.sh +++ b/Dockerfile @@ -1,12 +1,16 @@ -#!/usr/bin/env bash +FROM amazonlinux:2.0.20190508 as lobuild # see https://stackoverflow.com/questions/2499794/how-to-fix-a-locale-setting-warning-from-perl -export LC_CTYPE=en_US.UTF-8 -export LC_ALL=en_US.UTF-8 +ENV LC_CTYPE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 + +ENV LIBREOFFICE_VERSION=6.2.1.2 # install basic stuff required for compilation -sudo yum-config-manager --enable epel -sudo yum install -y \ +RUN yum install -y yum-utils \ + && yum-config-manager --enable epel \ + && yum install -y \ + google-crosextra-caladea-fonts \ autoconf \ ccache \ expat-devel \ @@ -39,34 +43,27 @@ sudo yum install -y \ nss-devel \ openssl-devel \ perl-Digest-MD5 \ - python34-devel + python34-devel \ + which # Used by autogen.sh -sudo yum groupinstall -y "Development Tools" - -# install liblangtag (not available in Amazon Linux or EPEL repos) -sudo nano /etc/yum.repos.d/centos.repo -# paste repo info from https://unix.stackexchange.com/questions/433046/how-do-i-enable-centos-repositories-on-rhel-red-hat -yum repolist -sudo yum install -y liblangtag -sudo cp -r /usr/share/liblangtag /usr/local/share/liblangtag/ +RUN yum groupinstall -y "Development Tools" +# fetch the LibreOffice source +RUN cd /tmp \ + && curl -L https://github.com/LibreOffice/core/archive/libreoffice-${LIBREOFFICE_VERSION}.tar.gz | tar -xz \ + && mv core-libreoffice-${LIBREOFFICE_VERSION} libreoffice -# clone libreoffice sources -curl -L https://github.com/LibreOffice/core/archive/libreoffice-6.2.1.2.tar.gz | tar -xz -mv core-libreoffice-6.2.1.2 libreoffice -cd libreoffice +WORKDIR /tmp/libreoffice # see https://ask.libreoffice.org/en/question/72766/sourcesver-missing-while-compiling-from-source/ -echo "lo_sources_ver=6.2.1.2" >> sources.ver - -# set this cache if you are going to compile several times -ccache --max-size 32 G && ccache -s +RUN echo "lo_sources_ver=${LIBREOFFICE_VERSION}" >> sources.ver -# See https://git.io/fhAJ0 -sudo yum remove -y gcc48-c++ && sudo yum install -y gcc72-c++ +# install liblangtag (not available in Amazon Linux or EPEL repos) +# paste repo info from https://unix.stackexchange.com/questions/433046/how-do-i-enable-centos-repositories-on-rhel-red-hat +COPY config/centos.repo /etc/yum.repos.d/ +RUN yum repolist && yum install -y liblangtag && cp -r /usr/share/liblangtag /usr/local/share/liblangtag/ -# the most important part. Run ./autogen.sh --help to see wha each option means -./autogen.sh \ +RUN ./autogen.sh \ --disable-avahi \ --disable-cairo-canvas \ --disable-coinmp \ @@ -119,34 +116,52 @@ sudo yum remove -y gcc48-c++ && sudo yum install -y gcc72-c++ --without-system-dicts # Disable flaky unit test failing on macos (and for some reason on Amazon Linux as well) -nano ./vcl/qa/cppunit/pdfexport/pdfexport.cxx # find the line "void PdfExportTest::testSofthyphenPos()" (around 600) # and replace "#if !defined MACOSX && !defined _WIN32" with "#if defined MACOSX && !defined _WIN32" +RUN sed -i '609s/#if !defined MACOSX && !defined _WIN32/#if defined MACOSX \&\& !defined _WIN32/' vcl/qa/cppunit/pdfexport/pdfexport.cxx -# this will take 0-2 hours to compile, depends on your machine -make +# this will take 30 minutes to 2 hours to compile, depends on your machine +RUN make # this will remove ~100 MB of symbols from shared objects -strip ./instdir/**/* +# strip will always return exit code 1 as it generates file warnings when hitting directories +RUN strip ./instdir/**/* || true # remove unneeded stuff for headless mode -rm -rf ./instdir/share/gallery \ +RUN rm -rf ./instdir/share/gallery \ ./instdir/share/config/images_*.zip \ ./instdir/readmes \ ./instdir/CREDITS.fodt \ ./instdir/LICENSE* \ ./instdir/NOTICE -# archive -tar -cvf lo.tar instdir +# test if compilation was successful +RUN echo "hello world" > a.txt \ + && ./instdir/program/soffice --headless --invisible --nodefault --nofirststartwizard \ + --nolockcheck --nologo --norestore --convert-to pdf --outdir $(pwd) a.txt -# install brotli first https://www.howtoforge.com/how-to-compile-brotli-from-source-on-centos-7/ -brotli --best --force ./lo.tar +RUN tar -cvf /tmp/lo.tar instdir/ -# test if compilation was successful -echo "hello world" > a.txt -./instdir/program/soffice --headless --invisible --nodefault --nofirststartwizard \ - --nolockcheck --nologo --norestore --convert-to pdf --outdir $(pwd) a.txt +FROM amazonlinux:2.0.20190508 as brotli + +ENV BROTLI_VERSION=1.0.7 + +WORKDIR /tmp + +# Compile Brotli +RUN yum install -y make zip unzip bc autoconf automake libtool \ + && curl -LO https://github.com/google/brotli/archive/v${BROTLI_VERSION}.zip \ + && unzip v${BROTLI_VERSION}.zip \ + && cd brotli-${BROTLI_VERSION} \ + && ./bootstrap \ + && ./configure \ + && make \ + && make install + +COPY --from=lobuild /tmp/lo.tar . + +RUN brotli --best /tmp/lo.tar && zip -r layers.zip lo.tar.br + +FROM amazonlinux:2.0.20190508 -# download from EC2 to local machine -scp ec2-user@ec2-54-227-212-139.compute-1.amazonaws.com:/home/ec2-user/libreoffice/lo.tar.br $(pwd) +COPY --from=brotli /tmp/layers.zip /tmp diff --git a/config/centos.repo b/config/centos.repo new file mode 100644 index 0000000..9ab902f --- /dev/null +++ b/config/centos.repo @@ -0,0 +1,6 @@ +[centos] +name=CentOS-7 +baseurl=http://ftp.heanet.ie/pub/centos/7/os/x86_64/ +enabled=1 +gpgcheck=1 +gpgkey=http://ftp.heanet.ie/pub/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4484948 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3.7' +services: + libreoffice: + image: vladgolubev/serverless-libreoffice + build: . + volumes: + - .:/host + command: 'cp /tmp/layers.zip /host' From a52ff63ae75b9a146d2d21789331e0035f46f622 Mon Sep 17 00:00:00 2001 From: Aaron Sturm Date: Wed, 11 Sep 2019 07:45:54 -0700 Subject: [PATCH 2/2] Update README to document new steps --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4e9b26f..7f1fe42 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,10 @@ Compiled and ready to use archive can be downloaded under [Releases section](htt > Check out a comprehensive [step-by-step tutorial](STEP_BY_STEP.md) from 0 to deployed function. -1. Go to [Lambda Execution Environment and Available Libraries](https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html) page to get the latest AMI id -2. Click on [this link](https://console.aws.amazon.com/ec2/v2/home#Images:visibility=public-images;search=amzn-ami-hvm-2017.03.1.20170812-x86_64-gp2) to get AMI id for your region -3. Spin up a `c5.2xlarge` spot instance with ~ 100 GB of storage attached -4. Follow the steps in `compile.sh` file in the repo +To run this, you will need to [Docker](https://docs.docker.com/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed. + +1. Install and configure Docker and docker-compose locally or on a `c5.2xlarge` spot instance with ~ 8 GB (the default) of storage attached. +1. In a terminal, run `docker-compose run --rm libreoffice`. It will compile LibreOffice and then copy layers.zip to your local drive. # Help