From b2f8a0d13b2e5c1f368dde11d28ba5bdce2f01f1 Mon Sep 17 00:00:00 2001 From: Sylvain Bellemare Date: Tue, 26 Nov 2019 19:26:13 -0500 Subject: [PATCH] Remove the need of docker credentials on Travis CI * Move base dependencies into separate docker image * Resolves #387 --- .travis.yml | 38 ++---- Dockerfile | 185 ++++------------------------- docs/development/contributing.rst | 2 + docs/development/docker-images.rst | 173 +++++++++++++++++++++++++++ docs/index.rst | 1 + 5 files changed, 205 insertions(+), 194 deletions(-) create mode 100644 docs/development/docker-images.rst diff --git a/.travis.yml b/.travis.yml index fb2dfe72..121de8a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,15 +5,7 @@ cache: pip python: 3.7 sudo: true services: -- docker -env: - global: - # Generated by running: - # $ gem install travis - # $ travis encrypt DOCKER_USER= --add - # $ travis encrypt DOCKER_PASS= --add - # These are needed for the `docker login` call to succeed. - - secure: "J4SqqWBvK+n6GHHACzOyJfZWyoObHRp26QleFyywSt+l6r40QIIyBGBVxUzO2E5EHZCRlE640pbk1wYMFg6GIqHbaWOI7AjbLAmzMLOiIbRLd/y0BNntC6Gl8N/SWTA0JsiDd5/xOXTK2XPk/O0NfHZmHX9LajW9blXOyFlSKCy6sO093sRZJIH9bt6TkyQlpfDhp4bw63WRbekXvxXYVg0fK47n9h1/t5OqY/ppKECpN7K7/qRPtlwdfi0rB4Ud38PolW/qppBKPD19JdIIJFseDNA2k6HFDaw4WA1o5Sj0g1uY2aI6ST+6U7sC0hkf5x2EDei+WP5ktJvRi0Fk1baegUk/mel9+ZbBMIb/moYequBgZuee9xBPzVtiA2zzTIz2nd4PPC5TKtO7ZfLQI3cm9W59+lUMcZm2ZBE9HuWjt7KJvre3CwnO461g1P5KD+DOtn1Vpxk3QCUjxaUypXbOE4LsuZuqS+CkPIQ5fCqlLiH4zZEYBgU6KkupDfr3a27CbtuNfnsp/6/V2GHDEIlM2oMl1gez5EkdifRRMaDZYL7VyAX5e/Es7BkkCo+lF5ihKMVgUu1p71ZFO3VQ8uEhcCUnfXVSLJHiiCy20eYsPWP5rgESnL3qRF6+W0Q+9ak80ojFZdSe1ZHfJlG7cQQs9q8KTh4Y558YbkciXtw=" + - docker jobs: include: - stage: Linting @@ -26,32 +18,16 @@ jobs: - name: Doc8 install: pip install doc8 pygments script: doc8 docs/ - - stage: Build - name: Build Docker Image for Current Commit - install: pip install pyyaml - before_script: bash scripts/docker_login.sh - script: scripts/stager.py -vv build -t prod -c remote -b $TRAVIS_COMMIT --cache-tags $TRAVIS_COMMIT latest - after_success: scripts/stager.py -vv push -t prod -b $TRAVIS_COMMIT - stage: Test name: Sphinx Tests - install: pip install pyyaml - # Not using `scripts/stager.py` as we only need the one image - before_script: bash scripts/docker_image_pull.sh - script: bash scripts/docker_doc_run.sh + install: docker build -t hbmpc-tests --build-arg SETUP_EXTRAS="docs" --target tests . + script: docker run -it hbmpc-tests sphinx-build -M html docs docs/_build -c docs -W after_success: skip - name: Unit Tests - install: pip install --upgrade codecov - # Not using `scripts/stager.py` as we only need the one image - before_script: bash scripts/docker_image_pull.sh - # TODO: currently spits out errors on travis runs - # See: https://travis-ci.org/initc3/HoneyBadgerMPC/jobs/560315742 - script: bash scripts/docker_test_run.sh + install: + - pip install --upgrade codecov + - docker build -t hbmpc-tests --build-arg SETUP_EXTRAS="tests" --target tests . + script: docker run -it hbmpc-tests pytest -v --cov --cov-report=term-missing --cov-report=xml -Wignore::DeprecationWarning -nauto after_success: - docker cp $(docker ps -alq --format "{{.Names}}"):/usr/src/HoneyBadgerMPC/coverage.xml . - codecov -v - - stage: Upload - if: type = push - name: Tag and upload images to Dockerhub - install: pip install pyyaml - script: .ci/tag_and_upload.sh - after_success: skip diff --git a/Dockerfile b/Dockerfile index eba486d9..9e8691d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,157 +1,17 @@ -# Dockerfile is used to create a development environment for running our code -# The dockerfile is composed of several distinct phases: -# - Base -# - Install baseline dependencies used to build our main dependencies (e.g. cffi, -# make, etc.) -# - Commands in this section should be changed as little as possible to improve -# cache performance -# - Dependencies -# - This is composed of a bunch of different targets, each of which inherit from -# the base target, and create a single dependency. -# - It's essential to only copy what's necessary from the build context in this -# stage to improve caching -# - Final target -# - Pull in all created dependencies from the other targets into one streamlined -# target -# - We should create a dev and prod target. +ARG HBMPC_DEPS_DIGEST="46902d869ea881d7b00b72ff6accf2558a5e15849da5fa5cc722b4ff82a507f8" -## -# Base target: -# All used targets should be based off of this target, and as such, changes to this -# should be kept to an absolute minimum, as it causes the longest builds. -# This should contain all setup required by all other targets, such as environment -# variables, and essential apt dependencies. -## -FROM python:3.7.3-slim AS base +FROM initc3/honeybadgermpc-deps@sha256:$HBMPC_DEPS_DIGEST AS build-compute-power-sums +COPY apps/asynchromix/cpp/ /usr/src/apps/asynchromix/cpp/ +RUN make -C /usr/src/apps/asynchromix/cpp -# Allows for log messages to be immediately dumped to the -# stream instead of being buffered. -ENV PYTHONUNBUFFERED 1 +FROM initc3/honeybadgermpc-deps@sha256:$HBMPC_DEPS_DIGEST AS pre-tests -# Path variables needed for Charm -ENV LIBRARY_PATH /usr/local/lib -ENV LD_LIBRARY_PATH /usr/local/lib -ENV LIBRARY_INCLUDE_PATH /usr/local/include +COPY pairing /usr/src/pairing +RUN pip install -v /usr/src/pairing/ -ENV PYTHON_LIBRARY_PATH /opt/venv -ENV PATH ${PYTHON_LIBRARY_PATH}/bin:${PATH} - -# Make sh point to bash -# This is being changed since it will avoid any errors in the `launch_mpc.sh` script -# which relies on certain code that doesn't work in container's default shell. -RUN ln -sf bash /bin/sh - -# Install apt dependencies -# Put apt dependencies here that are needed by all build paths -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - git \ - iproute2 \ - libflint-dev \ - libgmp-dev \ - libffi-dev \ - libmpc-dev \ - libmpfr-dev \ - libssl-dev \ - openssl \ - sudo - -# Setup virtualenv -RUN pip install --upgrade pip virtualenv -RUN python -m virtualenv ${PYTHON_LIBRARY_PATH} - -# Install pip dependencies here that are absolutely required by setup.py for -# better cache performance. These should be changed rarely, as they cause -# long rebuild times. -RUN pip install \ - cffi \ - Cython \ - gmpy2 \ - psutil \ - pycrypto \ - pyzmq \ - zfec - -# This is needed otherwise the build for the power sum solver will fail. -# This is a known issue in the version of libflint-dev in apt. -# https://github.com/wbhart/flint2/issues/217 -# This has been fixed if we pull the latest code from the repo. However, we want -# to avoid compiling the lib from the source since it adds 20 minutes to the build. -RUN sed -i '30c #include "flint/flint.h"' /usr/include/flint/flintxx/flint_classes.h - -## -# Build Target: -# Isolated target to build heavy dependencies -# Built dependencies must be manually copied over in later stages. -## -FROM base AS build -WORKDIR / - -# Install apt dependencies. These dependencies should only be those which are -# needed for building dependencies. Any other dependencies should be installed -# in later targets -RUN apt-get install -y --no-install-recommends \ - bison \ - cmake \ - flex \ - wget - -# Downloads rust and sets it up -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly -ENV PATH "/root/.cargo/bin:${PATH}" - - -# Install NTL -WORKDIR / -RUN curl -so - https://www.shoup.net/ntl/ntl-11.3.2.tar.gz | tar xzvf - -WORKDIR /ntl-11.3.2/src -RUN ./configure CXXFLAGS="-g -O2 -fPIC -march=native -pthread -std=c++11" -RUN make -RUN make install - -# Install better pairing -# Creates dependencies in /usr/local/include/pbc and /usr/local/lib -WORKDIR / -RUN curl -so - https://crypto.stanford.edu/pbc/files/pbc-0.5.14.tar.gz | tar xzvf - -WORKDIR /pbc-0.5.14/ -RUN ./configure -RUN make -RUN make install - -# Install charm -# Creates /charm/dist/Charm_Crypto...x86_64.egg, which gets copied into the venv -# /opt/venv/lib/python3.7/site-packages/Charm_crypto...x86_64.egg -WORKDIR / -RUN git clone https://github.com/JHUISI/charm.git -WORKDIR /charm -RUN git reset --hard be9587ccdd4d61c591fb50728ebf2a4690a2064f -RUN ./configure.sh -RUN make install - -# Copy pairing from build context and install it -COPY pairing/ pairing/ -RUN pip install pairing/ - -# Install final dependencies needed in prod, as well as -# pull in dependencies from the build target. -# The reason there is this pre-prod target is so that we can have a final target, -# prod, which pulls in all remaining files from the build context into the -# docker image. This will give us the best possible caching performance given -# routine file changes. -# -# Furthermore, by splitting these targets, we can continue building off of this -# target for dev targets later with good cache performance by delaying copying -# changed files until the end of the dev targets. -FROM base AS pre-prod -WORKDIR /usr/src/HoneyBadgerMPC/ - -COPY --from=build ${PYTHON_LIBRARY_PATH} ${PYTHON_LIBRARY_PATH} -COPY --from=build /usr/local/include/ /usr/local/include/ -COPY --from=build ${LIBRARY_PATH} ${LIBRARY_PATH} - -COPY apps/asynchromix/cpp/ apps/asynchromix/cpp/ -RUN make -C apps/asynchromix/cpp +ENV HBMPC_HOME /usr/src/HoneyBadgerMPC +WORKDIR $HBMPC_HOME +COPY --from=build-compute-power-sums /usr/local/bin/compute-power-sums /usr/local/bin/ COPY setup.py . COPY README.md . @@ -159,22 +19,17 @@ COPY honeybadgermpc/__version__.py honeybadgermpc/ COPY honeybadgermpc/__init__.py honeybadgermpc/ COPY honeybadgermpc/ntl/ honeybadgermpc/ntl/ COPY apps/asynchromix/solver/ apps/asynchromix/solver/ -RUN pip install -e .['tests,docs'] +ARG SETUP_EXTRAS="tests,docs" +RUN pip install -e .[$SETUP_EXTRAS] -# This is the target that can minimally run the unit tests. -FROM pre-prod AS prod +FROM pre-tests AS tests COPY . . -# This is the target that installs the remaining dependencies we -# want to have on the dev machines. This is the best place to install -# dependencies from pip, npm, apt, etc. for rapid iteration, as -# it will not affect the build times or image sizes of the production image. -# Once a dependency is deemed necessary enough, it can be later moved into -# the production image. -FROM pre-prod AS pre-dev +FROM tests as pre-dev WORKDIR / # Install solidity +# TODO look at https://github.com/ethereum/solidity/blob/e383b2bbbaeae33bd44d1e3b641ca922cf030c8d/.circleci/docker/Dockerfile.ubuntu1904 RUN git clone --recursive https://github.com/ethereum/solidity.git WORKDIR /solidity/ RUN git checkout v0.4.24 # Old version necessary to work??? @@ -189,6 +44,11 @@ WORKDIR / # Bash commands RUN echo "alias cls=\"clear && printf '\e[3J'\"" >> ~/.bashrc +# Make sh point to bash +# This is being changed since it will avoid any errors in the `launch_mpc.sh` script +# which relies on certain code that doesn't work in container's default shell. +RUN ln -sf bash /bin/sh + # Install Nodejs RUN curl -sL https://deb.nodesource.com/setup_8.x | bash @@ -202,9 +62,8 @@ RUN apt-get install -y --no-install-recommends \ RUN npm install -g ganache-cli # Install remaining pip dependencies here -WORKDIR /usr/src/HoneyBadgerMPC/ +WORKDIR $HBMPC_HOME RUN pip install -e .['dev'] -FROM pre-dev AS dev +FROM pre-dev as dev COPY . . - diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst index 6c573e60..ad2022c0 100644 --- a/docs/development/contributing.rst +++ b/docs/development/contributing.rst @@ -1,3 +1,5 @@ +.. _contributing-new-code: + Contributing new code ===================== Since `git`_ and `github`_ are used to version and host the code, one needs diff --git a/docs/development/docker-images.rst b/docs/development/docker-images.rst new file mode 100644 index 00000000..4e6e133b --- /dev/null +++ b/docs/development/docker-images.rst @@ -0,0 +1,173 @@ +HoneyBadgerMPC Docker Image +=========================== +The Docker image for HoneyBadgerMPC is roughly made up of two main +parts: + +1. HoneyBadgerMPC code-independent dependencies + (`honeybadgermpc-deps`_). +2. HoneyBadgerMPC code-dependent and experimental dependencies + (`honeybadgermpc`__). + +In addition to distinguishing between dependencies that depend on +the HoneyBadgerMPC source code from those that do not, it is also +useful to distinguish between dependencies that can be installed via +a package management tool like ``apt`` or ``pip`` from dependencies +that must be built and installed from their source code. + +1. Dependencies obtained from a package repository. (Examples: + `debian packages`_ or `PyPI`_.) +2. Dependencies which must be installed from their source code. + +Some dependencies that must be installed from their source code can +take a lot of time to build and in these cases it may be convenient +to provide standalone Docker images that contain the pre-built binaries +and headers. The parent image of HoneyBadgerMPC makes use of such +images: + +* `FLINT`_ +* `NTL`_ +* `PBC`_ +* `Charm-crypto`_ +* `Rust nightly frozen`_ + + +Code-independent dependencies +----------------------------- +The Docker image for HoneyBadgerMPC is built from the parent image +`honeybadgermpc-deps`_. This parent image contains core +dependencies that do not need any of the HoneyBadgerMPC code or context +from the HoneyBadgerMPC source repository. NTL or Rust are examples of +such dependencies. The ``Dockerfile`` for the parent image is +maintained in `initc3/docker-honeybadgermpc-deps`_. + +Code-dependent and experimental dependencies +-------------------------------------------- +The HoneyBadgerMPC main image contains dependencies that rely on the +source code. Examples of such dependencies are the Rust/Python based +`pairing`_ library and the C++ based asynchromix `compute-power-sums`_ +program. Dependencies required for integration or experimentation +purposes can be added to the main image as well. For example, +``solidity`` is part of the main image as it is necessary for an +integration with the Ethereum testnet (see `asynchromix.sol`_ and +`asynchromix.py`_). The ``Dockerfile`` for the main image is maintained +in `initc3/HoneyBadgerMPC`__. + + +Guidelines for modifying the images +----------------------------------- +Changes to the parent or main Docker image may be needed for various +reasons. This section presents some suggestions on how to go about to +perform the necessary changes. + +Modifications to the main HoneyBadgerMPC image +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The main image can be modified by simply modifying the Dockerfile under +`initc3/HoneyBadgerMPC `_ and +making a pull request. See :ref:`contributing-new-code` for help on +making new contributions to HoneyBadgerMPC. + +Build time +"""""""""" +Changes to the main image should ideally preserve or reduce the build +time since this image is being built on Travis CI for pull requests +and needs to be built or re-built by those who are developing the +HoneyBadgerMPC code base. + +Parent image digest +""""""""""""""""""" +The parent image digest specified in the ``FROM`` statements controls +which version of the parent image will be used. This digest should only +be changed when that is indeed the intention! See +:ref:`use-new-parent-image` to modify the parent image version in the +main image, `honeybadgermpc +`_. + + +Adding, or modifying a core dependency +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +For dependencies that can be added via a package manager like ``apt`` +or ``pip`` you can simply add, or update the version of the dependency +using the package manager. + +For dependencies that must be built from source, adding or modifying +the dependency can be done in three or four phases: + +1. Modify the main image, by modifying the `Dockerfile + `_ + under ``initc3/HoneyBadgerMPC``. +2. Create a standalone image such as `NTL`_. +3. Using the standalone image as a stage add the dependency to the main + or parent image. See `Use an external image as a "stage"`_. +4. If adding the dependency to the parent image, then change the + sha256 digest of the parent image in the ``FROM`` statements in the + main image. + +A core dependency (that does not depend on the source code of +HoneyBadgerMPC) can always be added to the main image first, and moved +to the parent image at a later time. This may be convenient for +testing, experimentation or troubleshooting purposes. + +Modifying an existing core dependency can also be done in the main +image. For instance, if the goal was to upgrade an existing dependency, +like the Rust version, this could be done first in the main image by +installing the new version and making it the default one. + +Modifications of honeybadgermpc-deps (parent image) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Modifications to the parent image must be done via a pull request on +the repository `initc3/docker-honeybadgermpc-deps`_. + +Once a pull request is merged, a new image will be built and pushed to +Docker Hub. When an image is pushed a sha256 digest of the image is +generated. The sha256 digest is used as an identifier in Dockerfiles +in ``FROM`` and ``--from=`` statements. + + +.. _use-new-parent-image: + +Using a new parent image in honeybadgermpc (main image) +""""""""""""""""""""""""""""""""""""""""""""""""""""""" +In order to use a new parent image in the `honeybadgermpc +`_ +image, the sha256 digest in the ``FROM`` statements: + +.. code-block:: dockerfile + + # Dockerfile for honeybadgermpc image + ARG HBMPC_DEPS_DIGEST="46902d869ea881d7b00b72ff6accf2558a5e15849da5fa5cc722b4ff82a507f8" + + FROM initc3/honeybadgermpc-deps@sha256:$HBMPC_DEPS_DIGEST + + +of the `HoneyBadgerMPC Dockerfile`_ must point to the targeted parent +image. Changing the parent image can be done in two ways: + +1. Using ``docker build --build-arg HBMPC_DEPS_DIGEST= ...`` +2. Changing the default value of ``HBMPC_DEPS_DIGEST`` in the + ``Dockerfile``. + +**Image digest generation.** The image digest is generated when the +image is pushed to Docker Hub. For `honeybadgermpc-deps`_ this +happens when a pull request is merged in the ``master`` branch of the +`initc3/docker-honeybadgermpc-deps`_ repository. + + + + +.. __: https://hub.docker.com/repository/docker/initc3/honeybadgermpc +.. __: https://github.com/initc3/HoneyBadgerMPC +.. _honeybadgermpc-deps: https://hub.docker.com/repository/docker/initc3/honeybadgermpc-deps +.. _initc3/docker-honeybadgermpc-deps: https://github.com/initc3/docker-honeybadgermpc-deps +.. _HoneyBadgerMPC Dockerfile: https://github.com/initc3/HoneyBadgerMPC/blob/dev/Dockerfile +.. _pairing: https://github.com/initc3/HoneyBadgerMPC/tree/dev/pairing +.. _compute-power-sums: https://github.com/initc3/HoneyBadgerMPC/blob/dev/apps/asynchromix/cpp/compute-power-sums.cpp +.. _asynchromix.sol: https://github.com/initc3/HoneyBadgerMPC/blob/dev/apps/asynchromix/asynchromix.sol +.. _asynchromix.py: https://github.com/initc3/HoneyBadgerMPC/blob/dev/apps/asynchromix/asynchromix.py +.. _debian packages: https://www.debian.org/distrib/packages +.. _pypi: https://pypi.org/ +.. _FLINT: https://hub.docker.com/repository/docker/initc3/flint2 +.. _NTL: https://hub.docker.com/repository/docker/initc3/ntl +.. _PBC: https://hub.docker.com/repository/docker/initc3/pbc +.. _Charm-crypto: https://hub.docker.com/repository/docker/initc3/charm-crypto +.. _Rust nightly frozen: https://hub.docker.com/repository/docker/initc3/rust-frozen +.. _Use an external image as a "stage": https://docs.docker.com/develop/develop-images/multistage-build/#use-an-external-image-as-a-stage diff --git a/docs/index.rst b/docs/index.rst index bedda67d..3e194283 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,6 +58,7 @@ development/getting-started development/contributing + development/docker-images development/reviewing-contributions ci packaging