Skip to content

Commit

Permalink
Merge pull request #401 from sbellem/issue-387
Browse files Browse the repository at this point in the history 
Remove the need of docker credentials on Travis CI
  • Loading branch information
@sanket1729
sanket1729 authored Dec 4, 2019
2 parents 350bfb1 + 498a686 commit d436e32
Show file tree
Hide file tree
Showing 5 changed files with 206 additions and 205 deletions.
38 changes: 7 additions & 31 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,7 @@ cache: pip
python: 3.7
sudo: true
services:
- docker
env:
global:
# Generated by running:
# $ gem install travis
# $ travis encrypt DOCKER_USER=<username> --add
# $ travis encrypt DOCKER_PASS=<password> --add
# These are needed for the `docker login` call to succeed.
- secure: "J4SqqWBvK+n6GHHACzOyJfZWyoObHRp26QleFyywSt+l6r40QIIyBGBVxUzO2E5EHZCRlE640pbk1wYMFg6GIqHbaWOI7AjbLAmzMLOiIbRLd/y0BNntC6Gl8N/SWTA0JsiDd5/xOXTK2XPk/O0NfHZmHX9LajW9blXOyFlSKCy6sO093sRZJIH9bt6TkyQlpfDhp4bw63WRbekXvxXYVg0fK47n9h1/t5OqY/ppKECpN7K7/qRPtlwdfi0rB4Ud38PolW/qppBKPD19JdIIJFseDNA2k6HFDaw4WA1o5Sj0g1uY2aI6ST+6U7sC0hkf5x2EDei+WP5ktJvRi0Fk1baegUk/mel9+ZbBMIb/moYequBgZuee9xBPzVtiA2zzTIz2nd4PPC5TKtO7ZfLQI3cm9W59+lUMcZm2ZBE9HuWjt7KJvre3CwnO461g1P5KD+DOtn1Vpxk3QCUjxaUypXbOE4LsuZuqS+CkPIQ5fCqlLiH4zZEYBgU6KkupDfr3a27CbtuNfnsp/6/V2GHDEIlM2oMl1gez5EkdifRRMaDZYL7VyAX5e/Es7BkkCo+lF5ihKMVgUu1p71ZFO3VQ8uEhcCUnfXVSLJHiiCy20eYsPWP5rgESnL3qRF6+W0Q+9ak80ojFZdSe1ZHfJlG7cQQs9q8KTh4Y558YbkciXtw="
- docker
jobs:
include:
- stage: Linting
Expand All @@ -26,32 +18,16 @@ jobs:
- name: Doc8
install: pip install doc8 pygments
script: doc8 docs/
- stage: Build
name: Build Docker Image for Current Commit
install: pip install pyyaml
before_script: bash scripts/docker_login.sh
script: scripts/stager.py -vv build -t prod -c remote -b $TRAVIS_COMMIT --cache-tags $TRAVIS_COMMIT latest
after_success: scripts/stager.py -vv push -t prod -b $TRAVIS_COMMIT
- stage: Test
name: Sphinx Tests
install: pip install pyyaml
# Not using `scripts/stager.py` as we only need the one image
before_script: bash scripts/docker_image_pull.sh
script: bash scripts/docker_doc_run.sh
install: docker build -t hbmpc-tests --build-arg SETUP_EXTRAS="docs" --target tests .
script: docker run -it hbmpc-tests sphinx-build -M html docs docs/_build -c docs -W
after_success: skip
- name: Unit Tests
install: pip install --upgrade codecov
# Not using `scripts/stager.py` as we only need the one image
before_script: bash scripts/docker_image_pull.sh
# TODO: currently spits out errors on travis runs
# See: https://travis-ci.org/initc3/HoneyBadgerMPC/jobs/560315742
script: bash scripts/docker_test_run.sh
install:
- pip install --upgrade codecov
- docker build -t hbmpc-tests --build-arg SETUP_EXTRAS="tests" --target tests .
script: docker run -it hbmpc-tests pytest -v --cov --cov-report=term-missing --cov-report=xml -Wignore::DeprecationWarning -nauto
after_success:
- docker cp $(docker ps -alq --format "{{.Names}}"):/usr/src/HoneyBadgerMPC/coverage.xml .
- codecov -v
- stage: Upload
if: type = push
name: Tag and upload images to Dockerhub
install: pip install pyyaml
script: .ci/tag_and_upload.sh
after_success: skip
197 changes: 23 additions & 174 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,194 +1,44 @@
# Dockerfile is used to create a development environment for running our code
# The dockerfile is composed of several distinct phases:
# - Base
# - Install baseline dependencies used to build our main dependencies (e.g. cffi,
# make, etc.)
# - Commands in this section should be changed as little as possible to improve
# cache performance
# - Dependencies
# - This is composed of a bunch of different targets, each of which inherit from
# the base target, and create a single dependency.
# - It's essential to only copy what's necessary from the build context in this
# stage to improve caching
# - Final target
# - Pull in all created dependencies from the other targets into one streamlined
# target
# - We should create a dev and prod target.
ARG HBMPC_DEPS_DIGEST="46902d869ea881d7b00b72ff6accf2558a5e15849da5fa5cc722b4ff82a507f8"

##
# Base target:
# All used targets should be based off of this target, and as such, changes to this
# should be kept to an absolute minimum, as it causes the longest builds.
# This should contain all setup required by all other targets, such as environment
# variables, and essential apt dependencies.
##
FROM python:3.7.3-slim AS base
FROM initc3/honeybadgermpc-deps@sha256:$HBMPC_DEPS_DIGEST AS build-compute-power-sums
COPY apps/asynchromix/cpp/ /usr/src/apps/asynchromix/cpp/
RUN make -C /usr/src/apps/asynchromix/cpp

# Allows for log messages to be immediately dumped to the
# stream instead of being buffered.
ENV PYTHONUNBUFFERED 1
FROM initc3/honeybadgermpc-deps@sha256:$HBMPC_DEPS_DIGEST AS pre-tests

# Path variables needed for Charm
ENV LIBRARY_PATH /usr/local/lib
ENV LD_LIBRARY_PATH /usr/local/lib
ENV LIBRARY_INCLUDE_PATH /usr/local/include
COPY pairing /usr/src/pairing
RUN pip install -v /usr/src/pairing/

ENV PYTHON_LIBRARY_PATH /opt/venv
ENV PATH ${PYTHON_LIBRARY_PATH}/bin:${PATH}

# Make sh point to bash
# This is being changed since it will avoid any errors in the `launch_mpc.sh` script
# which relies on certain code that doesn't work in container's default shell.
RUN ln -sf bash /bin/sh

# Install apt dependencies
# Put apt dependencies here that are needed by all build paths
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
curl \
git \
iproute2 \
libflint-dev \
libgmp-dev \
libffi-dev \
libmpc-dev \
libmpfr-dev \
libssl-dev \
openssl \
sudo

# Setup virtualenv
RUN pip install --upgrade pip virtualenv
RUN python -m virtualenv ${PYTHON_LIBRARY_PATH}

# Install pip dependencies here that are absolutely required by setup.py for
# better cache performance. These should be changed rarely, as they cause
# long rebuild times.
RUN pip install \
cffi \
Cython \
gmpy2 \
psutil \
pycrypto \
pyzmq \
zfec

# This is needed otherwise the build for the power sum solver will fail.
# This is a known issue in the version of libflint-dev in apt.
# https://github.com/wbhart/flint2/issues/217
# This has been fixed if we pull the latest code from the repo. However, we want
# to avoid compiling the lib from the source since it adds 20 minutes to the build.
RUN sed -i '30c #include "flint/flint.h"' /usr/include/flint/flintxx/flint_classes.h

##
# Build Target:
# Isolated target to build heavy dependencies
# Built dependencies must be manually copied over in later stages.
##
FROM base AS build
WORKDIR /

# Install apt dependencies. These dependencies should only be those which are
# needed for building dependencies. Any other dependencies should be installed
# in later targets
RUN apt-get install -y --no-install-recommends \
bison \
cmake \
flex \
wget

# Downloads rust and sets it up
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly
ENV PATH "/root/.cargo/bin:${PATH}"


# Install NTL
WORKDIR /
RUN curl -so - https://www.shoup.net/ntl/ntl-11.3.2.tar.gz | tar xzvf -
WORKDIR /ntl-11.3.2/src
RUN ./configure CXXFLAGS="-g -O2 -fPIC -march=native -pthread -std=c++11"
RUN make
RUN make install

# Install better pairing
# Creates dependencies in /usr/local/include/pbc and /usr/local/lib
WORKDIR /
RUN curl -so - https://crypto.stanford.edu/pbc/files/pbc-0.5.14.tar.gz | tar xzvf -
WORKDIR /pbc-0.5.14/
RUN ./configure
RUN make
RUN make install

# Install charm
# Creates /charm/dist/Charm_Crypto...x86_64.egg, which gets copied into the venv
# /opt/venv/lib/python3.7/site-packages/Charm_crypto...x86_64.egg
WORKDIR /
RUN git clone https://github.com/JHUISI/charm.git
WORKDIR /charm
RUN git reset --hard be9587ccdd4d61c591fb50728ebf2a4690a2064f
RUN ./configure.sh
RUN make install

# Copy pairing from build context and install it
COPY pairing/ pairing/
RUN pip install pairing/

# Install final dependencies needed in prod, as well as
# pull in dependencies from the build target.
# The reason there is this pre-prod target is so that we can have a final target,
# prod, which pulls in all remaining files from the build context into the
# docker image. This will give us the best possible caching performance given
# routine file changes.
#
# Furthermore, by splitting these targets, we can continue building off of this
# target for dev targets later with good cache performance by delaying copying
# changed files until the end of the dev targets.
FROM base AS pre-prod
WORKDIR /usr/src/HoneyBadgerMPC/

COPY --from=build ${PYTHON_LIBRARY_PATH} ${PYTHON_LIBRARY_PATH}
COPY --from=build /usr/local/include/ /usr/local/include/
COPY --from=build ${LIBRARY_PATH} ${LIBRARY_PATH}

COPY apps/asynchromix/cpp/ apps/asynchromix/cpp/
RUN make -C apps/asynchromix/cpp
ENV HBMPC_HOME /usr/src/HoneyBadgerMPC
WORKDIR $HBMPC_HOME
COPY --from=build-compute-power-sums /usr/local/bin/compute-power-sums /usr/local/bin/

COPY setup.py .
COPY README.md .
COPY honeybadgermpc/__version__.py honeybadgermpc/
COPY honeybadgermpc/__init__.py honeybadgermpc/
COPY honeybadgermpc/ntl/ honeybadgermpc/ntl/
COPY apps/asynchromix/solver/ apps/asynchromix/solver/
RUN pip install -e .['tests,docs']
ARG SETUP_EXTRAS="tests,docs"
RUN pip install -e .[$SETUP_EXTRAS]

# This is the target that can minimally run the unit tests.
FROM pre-prod AS prod
FROM pre-tests AS tests
COPY . .

# This is the target that installs the remaining dependencies we
# want to have on the dev machines. This is the best place to install
# dependencies from pip, npm, apt, etc. for rapid iteration, as
# it will not affect the build times or image sizes of the production image.
# Once a dependency is deemed necessary enough, it can be later moved into
# the production image.
FROM pre-prod AS pre-dev
FROM tests as pre-dev
WORKDIR /

# Install solidity
RUN git clone --recursive https://github.com/ethereum/solidity.git
WORKDIR /solidity/
RUN git checkout v0.4.24 # Old version necessary to work???
RUN git submodule update --init --recursive
RUN ./scripts/install_deps.sh
RUN mkdir build/
WORKDIR /solidity/build/
RUN cmake ..
RUN make install
WORKDIR /
# solidity
COPY --from=ethereum/solc:0.4.24 /usr/bin/solc /usr/bin/solc

# Bash commands
RUN echo "alias cls=\"clear && printf '\e[3J'\"" >> ~/.bashrc

# Make sh point to bash
# This is being changed since it will avoid any errors in the `launch_mpc.sh` script
# which relies on certain code that doesn't work in container's default shell.
RUN ln -sf bash /bin/sh

# Install Nodejs
RUN curl -sL https://deb.nodesource.com/setup_8.x | bash

Expand All @@ -202,9 +52,8 @@ RUN apt-get install -y --no-install-recommends \
RUN npm install -g ganache-cli

# Install remaining pip dependencies here
WORKDIR /usr/src/HoneyBadgerMPC/
WORKDIR $HBMPC_HOME
RUN pip install -e .['dev']

FROM pre-dev AS dev
FROM pre-dev as dev
COPY . .

2 changes: 2 additions & 0 deletions docs/development/contributing.rst
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
.. _contributing-new-code:

Contributing new code
=====================
Since `git`_ and `github`_ are used to version and host the code, one needs
Expand Down
Loading

0 comments on commit d436e32

Please sign in to comment.