Skip to content

Commit

Permalink
Merge pull request #1970 from alfred-bratterud/0.13-release
Browse files Browse the repository at this point in the history
0.13 release
  • Loading branch information
alfreb authored Nov 2, 2018
2 parents 367a1dc + e34facc commit 1850d74
Show file tree
Hide file tree
Showing 801 changed files with 24,179 additions and 14,329 deletions.
3 changes: 3 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
build_*
IncludeOS_install/
**/build
16 changes: 16 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,13 @@
*.img
*.img.*
*.gz
*.tar
*nacl.txt
.DS_Store
nbproject
dummy.disk
callgraph.svg
massif.out.*

!Dockerfile

Expand All @@ -35,3 +39,15 @@ test/**/*.log
CMakeFiles*
CMakeCache*
cmake_install.cmake

# Name of installation folder
IncludeOS_install

# Vim
*.swp

#CLion
.idea/

# Starbase disk file
lib/uplink/starbase/disk
64 changes: 33 additions & 31 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ set(SCRIPTS ${CMAKE_INSTALL_PREFIX}/includeos/scripts)
set(CPP_VERSION c++17)

# create OS version string from git describe (used in CXX flags)
execute_process(COMMAND git describe --dirty
execute_process(COMMAND git describe --tags --dirty
WORKING_DIRECTORY ${INCLUDEOS_ROOT}
OUTPUT_VARIABLE OS_VERSION)
string(STRIP ${OS_VERSION} OS_VERSION)
Expand All @@ -48,28 +48,27 @@ else()
set(DEFAULT_VM "vm.cpu_feat.json") # vmrunner
endif(cpu_feat_vanilla)

option(threading "Compile with SMP/threading support" OFF)
option(smp "Compile with SMP (multiprocessing)" OFF)

option(silent "Disable most output during OS boot" OFF)

option (undefined_san "Enable undefined-behavior sanitizer" OFF)
option (thin_lto "Enable the Thin LTO plugin" OFF)
option (full_lto "Enable full LTO (compatibility)" OFF)

# create random hex string as stack protector canary
string(RANDOM LENGTH 16 ALPHABET 0123456789ABCDEF STACK_PROTECTOR_VALUE)

set(CAPABS "${CAPABS} -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE}")
set(CAPABS "${CAPABS} -g -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE}")

# Various global defines
# * NO_DEBUG disables output from the debug macro
# * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures
set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION ")

set(WARNS "-Wall -Wextra")
set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE -D__includeos__")
set(WARNS "-Wall -Wextra") # -Werror

# configure options
option(debug "Build with debugging symbols (OBS: Dramatically increases binary size)" OFF)
option(debug-info "Build like \"all\" but with debugging output (i.e. the 'debug'-macro) enabled" OFF)
option(debug-all "Build with debugging symbols + debugging output, i.e. \"debug\" + \"debug-info\"" OFF)
option(debug "Build with no optimizations" OFF)
option(minimal "Build for minimal size" OFF)
option(stripped "reduce size" OFF)

Expand All @@ -96,15 +95,6 @@ if(minimal)
set(OPTIMIZE "-Os")
endif(minimal)

# Set debug options
if(debug OR debug-all)
set(CAPABS "${CAPABS} -ggdb3 -DGSL_THROW_ON_CONTRACT_VIOLATION")
endif(debug OR debug-all)

if(debug-info OR debug-all)
set(CAPABS "${CAPABS} -UNO_DEBUG")
endif(debug-info OR debug-all)

if(silent)
set(CAPABS "${CAPABS} -DNO-INFO=1")
endif(silent)
Expand All @@ -116,6 +106,11 @@ set(CAPABS "${CAPABS} ${OPTIMIZE}")
if (undefined_san)
set(CAPABS "${CAPABS} -fsanitize=undefined -fno-sanitize=vptr")
endif()
if (thin_lto)
set(CAPABS "${CAPABS} -flto=thin")
elseif(full_lto)
set(CAPABS "${CAPABS} -flto=full")
endif()

# object format needs to be set BEFORE enabling ASM
# see: https://cmake.org/Bug/bug_relationship_graph.php?bug_id=13166
Expand All @@ -131,21 +126,15 @@ endif()

enable_language(ASM_NASM)


if (NOT threading)
add_definitions(-D_LIBCPP_HAS_NO_THREADS)
endif()


# initialize C and C++ compiler flags
if (CMAKE_COMPILER_IS_GNUCC)
# gcc/g++ settings
set(CMAKE_CXX_FLAGS "-std=${CPP_VERSION} -MMD ${CAPABS} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"")
set(CMAKE_C_FLAGS " -MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"")
set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c")
set(CMAKE_C_FLAGS " ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c")
else()
# these kinda work with llvm
set(CMAKE_CXX_FLAGS "-std=${CPP_VERSION} -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"")
set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"")
set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c")
set(CMAKE_C_FLAGS "${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c")
endif()

# either download or cross-compile needed libraries
Expand Down Expand Up @@ -179,7 +168,9 @@ if(vmbuild)
SOURCE_DIR ${INCLUDEOS_ROOT}/vmbuild # Where is project located
BINARY_DIR ${INCLUDEOS_ROOT}/vmbuild/build
INSTALL_DIR ${BIN} # Where to install
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> # Pass installation folder
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
-DINCLUDE_PATH=${CMAKE_INSTALL_PREFIX} # Pass installation folder
DEPENDS PrecompiledLibraries
)
endif(vmbuild)

Expand All @@ -201,8 +192,14 @@ if(examples)
endif(examples)

option(tests "Build unit tests in /test and install lest test framework" OFF)
if(tests)
option(lest "Install lest unittest headers" OFF)

if (lest OR tests)
init_submodule(test/lest)
install(DIRECTORY test/lest/include/lest DESTINATION ${CMAKE_INSTALL_PREFIX}/includeos/include)
endif()

if(tests)
enable_testing()
ExternalProject_Add(unittests
PREFIX unittests
Expand Down Expand Up @@ -272,8 +269,13 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/elf-toolchain.cmake DESTINATION
# Install seed
install(DIRECTORY seed/ DESTINATION includeos/seed)

# Install boot util
# Install executable scripts
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/boot DESTINATION bin)
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-run DESTINATION bin)
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-callgraph DESTINATION bin)
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-debug DESTINATION bin)
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-gprof DESTINATION bin)
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-pgo DESTINATION bin)

# Install scripts
install(PROGRAMS
Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTE.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@

You're very welcome to [clone, edit and send pull-request](https://help.github.com/articles/using-pull-requests).

**Please read the contribution gudelines on the wiki: [Conributing to IncludeOS](https://github.com/hioa-cs/IncludeOS/wiki/Contributing-to-IncludeOS)**
**Please read the contribution guidelines: [Contributing to IncludeOS](http://includeos.readthedocs.io/en/latest/Contributing-to-IncludeOS.html)**

## Thank you!
111 changes: 93 additions & 18 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,31 +1,106 @@
FROM ubuntu:xenial
FROM ubuntu:xenial as base

RUN apt-get update && apt-get -y install \
sudo \
curl \
locales \
&& rm -rf /var/lib/apt/lists/*
RUN locale-gen en_US.UTF-8
ENV LANG=en_US.UTF-8 \
LANGUAGE=en_US:en \
LC_ALL=en_US.UTF-8

# Add fixuid to change permissions for bind-mounts. Set uid to same as host with -u <uid>:<guid>
RUN addgroup --gid 1000 docker && \
adduser --uid 1000 --ingroup docker --home /home/docker --shell /bin/sh --disabled-password --gecos "" docker && \
usermod -aG sudo docker && \
sed -i.bkp -e \
's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
/etc/sudoers
RUN USER=docker && \
GROUP=docker && \
curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.3/fixuid-0.3-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \
chown root:root /usr/local/bin/fixuid && \
chmod 4755 /usr/local/bin/fixuid && \
mkdir -p /etc/fixuid && \
printf "user: $USER\ngroup: $GROUP\npaths:\n - /service\n" > /etc/fixuid/config.yml

ARG CUSTOM_TAG
LABEL org.label-schema.schema-version="1.0" \
org.label-schema.name="IncludeOS builder" \
org.label-schema.vendor="IncludeOS" \
org.label-schema.version=$CUSTOM_TAG \
org.label-schema.vcs-url="https://github.com/hioa-cs/includeos"

RUN echo "LANG=C.UTF-8" > /etc/default/locale

#########################
FROM base as source-build

RUN apt-get update && apt-get -y install \
git \
lsb-release \
net-tools \
sudo \
wget \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/*

RUN useradd --create-home -s /bin/bash ubuntu
RUN adduser ubuntu sudo
RUN echo -n 'ubuntu:ubuntu' | chpasswd
RUN mkdir -p /root/IncludeOS
WORKDIR /root/IncludeOS
COPY . .

# Enable passwordless sudo for users under the "sudo" group
RUN sed -i.bkp -e \
's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
/etc/sudoers
# Ability to specify custom tag that overwrites any existing tag. This will then match what IncludeOS reports itself.
ARG CUSTOM_TAG
RUN git describe --tags --dirty > /ios_version.txt
RUN : ${CUSTOM_TAG:=$(git describe --tags)} && git tag -d $(git describe --tags); git tag $CUSTOM_TAG && git describe --tags --dirty > /custom_tag.txt

# Installation
RUN ./install.sh -n

#############################
FROM base as grubify

RUN apt-get update && apt-get -y install \
dosfstools \
grub-pc

USER ubuntu
COPY --from=source-build /usr/local/includeos/scripts/grubify.sh /home/ubuntu/IncludeOS_install/includeos/scripts/grubify.sh

ADD . /home/ubuntu/IncludeOS
WORKDIR /home/ubuntu/IncludeOS
ENTRYPOINT ["fixuid", "/home/ubuntu/IncludeOS_install/includeos/scripts/grubify.sh"]

RUN sudo apt-get update && \
sudo do_bridge="" ./etc/install_all_source.sh \
&& sudo rm -rf /var/lib/apt/lists/*
###########################
FROM base as build

RUN apt-get update && apt-get -y install \
git \
clang-5.0 \
cmake \
nasm \
python-pip \
&& rm -rf /var/lib/apt/lists/* \
&& pip install pystache antlr4-python2-runtime && \
apt-get remove -y python-pip && \
apt autoremove -y

ARG VCS_REF="Check file /ios_version.txt inside container"
LABEL org.label-schema.description="Build a service using IncludeOS" \
org.label-schema.vcs-ref=$VCS_REF \
org.label-schema.docker.cmd="docker run -v $PWD:/service <container>"

VOLUME /service
WORKDIR /service

CMD ./run.sh
COPY --from=source-build /usr/local/includeos /usr/local/includeos/
COPY --from=source-build /usr/local/bin/boot /usr/local/bin/boot
COPY --from=source-build /root/IncludeOS/etc/install_dependencies_linux.sh /
COPY --from=source-build /root/IncludeOS/etc/use_clang_version.sh /
COPY --from=source-build /root/IncludeOS/lib/uplink/starbase /root/IncludeOS/lib/uplink/starbase/
COPY --from=source-build /ios_version.txt /
COPY --from=source-build /custom_tag.txt /
COPY --from=source-build /root/IncludeOS/etc/docker_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

CMD mkdir -p build && \
cd build && \
cp $(find /usr/local/includeos -name chainloader) /service/build/chainloader && \
echo "IncludeOS version:" $(cat /ios_version.txt) "tag:" $(cat /custom_tag.txt) && \
cmake .. && \
make
2 changes: 1 addition & 1 deletion NaCl
Submodule NaCl updated 50 files
+363 −1,329 NaCl.py
+0 −354 cpp_resolve_values.py
+194 −135 cpp_template.mustache
+0 −508 cpp_transpile_function.py
+3 −4 examples/assignments.nacl
+13 −6 examples/iface.nacl
+39 −0 examples/iface_with_limits.nacl
+19 −0 examples/iface_without_network_configuration.nacl
+31 −0 examples/timers.nacl
+63 −56 examples/vlan.nacl
+44 −0 examples/vlan_routing.nacl
+44 −0 examples/vlan_with_mac.nacl
+16 −17 goldenfiles/assignments.cpp
+5 −5 goldenfiles/cidr.cpp
+9 −9 goldenfiles/config_options.cpp
+6 −6 goldenfiles/conntrack.cpp
+5 −5 goldenfiles/conntrack_with_timeout.cpp
+5 −5 goldenfiles/conntrack_with_timeout_assignments.cpp
+5 −5 goldenfiles/functions.cpp
+11 −12 goldenfiles/gateway_with_forward_chain.cpp
+9 −9 goldenfiles/gateway_with_send_time_exceeded.cpp
+22 −22 goldenfiles/iface.cpp
+62 −0 goldenfiles/iface_with_limits.cpp
+52 −0 goldenfiles/iface_without_network_configuration.cpp
+6 −109 goldenfiles/lb.cpp
+6 −109 goldenfiles/lb_assignment_functionality.cpp
+6 −109 goldenfiles/lb_assignment_functionality_2.cpp
+7 −110 goldenfiles/lb_with_uplink.cpp
+5 −5 goldenfiles/log.cpp
+18 −18 goldenfiles/nacl.cpp
+18 −18 goldenfiles/nacl_one_liner.cpp
+9 −9 goldenfiles/nat_and_gateway.cpp
+7 −7 goldenfiles/syslog.cpp
+149 −0 goldenfiles/timers.cpp
+51 −45 goldenfiles/vlan.cpp
+79 −0 goldenfiles/vlan_routing.cpp
+81 −0 goldenfiles/vlan_with_mac.cpp
+33 −277 shared.py
+42 −0 subtranspilers/__init__.py
+576 −0 subtranspilers/function_transpiler.py
+388 −0 subtranspilers/value_transpiler.py
+5 −0 test/cpp_diff.sh
+39 −0 type_processors/__init__.py
+153 −0 type_processors/conntrack.py
+176 −0 type_processors/function.py
+383 −0 type_processors/gateway.py
+432 −0 type_processors/iface.py
+336 −0 type_processors/load_balancer.py
+125 −0 type_processors/syslog.py
+195 −0 type_processors/timer.py
12 changes: 6 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ IncludeOS is free software, with "no warranties or restrictions of any kind".
* **Extreme memory footprint**: A minimal bootable 64-bit web server, including operating system components and a anything needed from the C/C++ standard libraries is currently 2.5 MB.
* **KVM, VirtualBox and VMWare support** with full virtualization, using [x86 hardware virtualization](https://en.wikipedia.org/wiki/X86_virtualization), available on most modern x86 CPUs. IncludeOS will run on any x86 hardware platform, even on a physical x86 computer, given appropriate drivers. Officially, we develop for- and test on [Linux KVM](http://www.linux-kvm.org/page/Main_Page), and VMWare [ESXi](https://www.vmware.com/products/esxi-and-esx.html)/[Fusion](https://www.vmware.com/products/fusion.html) which means that you can run your IncludeOS service on Linux, Microsoft Windows and macOS, as well as on cloud providers such as [Google Compute Engine](http://www.includeos.org/blog/2017/includeos-on-google-compute-engine.html), [OpenStack](https://www.openstack.org/) and VMWare [vcloud](https://www.vmware.com/products/vcloud-suite.html).
* **Instant boot:** IncludeOS on Qemu/kvm boots in about 300ms but IBM Research has also integrated IncludeOS with [Solo5/uKVM](https://github.com/Solo5/solo5), providing boot times as low as 10 milliseconds.
* **C++11/14 support**
* Full C++11/14 language support with [clang](http://clang.llvm.org) v3.8 and later.
* **Modern C++ support**
* Full C++11/14/17 language support with [clang](http://clang.llvm.org) 5 and later.
* Standard C++ library (STL) [libc++](http://libcxx.llvm.org) from [LLVM](http://llvm.org/).
* Exceptions and stack unwinding (currently using [libgcc](https://gcc.gnu.org/onlinedocs/gccint/Libgcc.html)).
* *Note:* Certain language features, such as threads and filestreams are currently missing backend support.
* **Standard C library** using [newlib](https://sourceware.org/newlib/) from [Red Hat](http://www.redhat.com/).
* **Standard C library** using [musl libc](http://www.musl-libc.org/).
* **Virtio and vmxnet3 Network drivers** with DMA. [Virtio](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=virtio) provides a highly efficient and widely supported I/O virtualization. vmxnet3 is the VMWare equivalent.
* **A highly modular TCP/IP-stack**.

Expand All @@ -42,11 +42,11 @@ A longer list of features and limitations can be found on our [documentation sit

### Set custom location and compiler

By default the project is installed to /usr/local/includeos.
By default the project is installed to `/usr/local/includeos`.

However, it is recommended to choose a custom location as well as select the compiler we want clang to find. In this document we assume you install IncludeOS in your home directory, in the folder ~/includeos.
However, it is recommended to choose a custom location as well as select the compiler we want clang to find. In this document we assume you install IncludeOS in your home directory, in the folder `~/includeos`.

To do this we can edit ~/.bash_profile (mac os) or ~/.bashrc (linux), adding these lines at the end of the file:
To do this we can edit `~/.bash_profile` (mac os) or `~/.bashrc` (linux), adding these lines at the end of the file:

```
export INCLUDEOS_PREFIX=~/includeos/
Expand Down
9 changes: 7 additions & 2 deletions api/arch.hpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// -*-C++-*-
// -*-C++-*-
// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2017 Oslo and Akershus University College of Applied Sciences
Expand Down Expand Up @@ -31,19 +31,24 @@ extern void __arch_poweroff();
extern void __arch_reboot();
extern void __arch_enable_legacy_irq(uint8_t);
extern void __arch_disable_legacy_irq(uint8_t);
extern void __arch_system_deactivate();

extern void __arch_install_irq(uint8_t, void(*)());
extern void __arch_subscribe_irq(uint8_t);
extern void __arch_unsubscribe_irq(uint8_t);
extern void __arch_preempt_forever(void(*)());

extern inline void __arch_read_memory_barrier() noexcept;
extern inline void __arch_write_memory_barrier() noexcept;
inline void __arch_hw_barrier() noexcept;
inline void __sw_barrier() noexcept;

extern uint64_t __arch_system_time() noexcept;
extern timespec __arch_wall_clock() noexcept;
inline uint64_t __arch_cpu_cycles() noexcept;

inline void __arch_hw_barrier() noexcept {
__sync_synchronize();
}

inline void __sw_barrier() noexcept
{
Expand Down
4 changes: 3 additions & 1 deletion api/arch/i686.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
#ifndef i686_ARCH_HPP
#define i686_ARCH_HPP

#include <arch/x86.hpp>
#define ARCH_x86

inline uint64_t __arch_cpu_cycles() noexcept {
uint64_t ret;
Expand All @@ -28,4 +28,6 @@ inline uint64_t __arch_cpu_cycles() noexcept {
}


constexpr uintptr_t __arch_max_canonical_addr = 0xffffffff;

#endif
Loading

0 comments on commit 1850d74

Please sign in to comment.