diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bcf5513 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +# Do not look for these files when docker builds +# https://www.geeksforgeeks.org/how-to-use-a-dockerignore-file/ +# hadolint ignore +*.md +Dockerfile.dockerignore +.git +.vscode +.gitignore +Makefile diff --git a/.github/workflows/workflow.docker.buildx.gha.yaml b/.github/workflows/workflow.docker.buildx.gha.yaml new file mode 100644 index 0000000..1e7f36d --- /dev/null +++ b/.github/workflows/workflow.docker.buildx.gha.yaml @@ -0,0 +1,114 @@ +# vim: se ts=2 sw=2 et: +# https://dev.to/cloudx/multi-arch-docker-images-the-easy-way-with-github-actions-4k54 +# https://docs.docker.com/ci-cd/github-actions/ +# assumes the image to build is the owner/current repo + +name: Build docker buildx image + +# yamllint disable-line rule:truthy +on: + push: + branches: + - main + - 'rich-*' + tags: + - 'v*' + + pull_request: + branches: + - main + +env: + # normal location + # DOCKERFILE: Dockerfile + DOCKERFILE: ./docker/Dockerfile + TEST_TAG: ${{github.repository}}:test + # https://github.com/docker/build-push-action/issues/557 + # only 7GB of memory and two cores so only one job at a time + BUILD_ARGS: NR_JOBS=1 + +# need permissions to write to github repo artifacts +permissions: + packages: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + - name: Setup buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + # https://docs.github.com/en/actions/security-guides/encrypted-secrets + - name: Login to GitHub registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + # https://docs.docker.com/docker-hub/access-tokens/ + # you need to add the docker Personal Access Token to the secrets + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PERSONAL_ACCESS_TOKEN }} + + # type=semvar,pattern={{major}} + # type=semvar,pattern={{major}.{{minor}} + # type=semvar,pattern={{major}}.{{minor}}.{{patch}} + # type=semvar,pattern={{version}} + # https://github.com/docker/build-push-action/blob/master/TROUBLESHOOTING.md + - name: Docker meta tags + id: meta + uses: docker/metadata-action@v4 + with: + # use different registry names here to push to different registries + images: | + ${{github.repository}} + ghcr.io/${{github.repository}} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=sha + latest + + # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md + - name: Build image for test + uses: docker/build-push-action@v3 + with: + context: . + file: ${{ env.DOCKERFILE }} + build-args: ${{ env.BUILD_ARGS }} + # cannot do multi-platform because runs out of memory + # platforms: linux/amd64,linux/arm64 + load: true + tags: ${{ env.TEST_TAG }} + + - name: Test docker image runs + run: | + docker run --rm ${{ env.TEST_TAG }} + + - name: Push image to registries (using cached build) + uses: docker/build-push-action@v3 + with: + context: . + # for covins how parallel is the build too much and it fails + build-args: ${{ env.BUILD_ARGS }} + file: ${{ env.DOCKERFILE }} + # this currently fails probably too much memory used as of Aug 2022 + # see https://github.com/docker/build-push-action/issues/621 + # platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + # this is the step that creates the correct image names from above + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # instead of manual tgs use the metadata-action to generate them + #tags: ${{GITHUB_REPOSITORY}}:${{GITHUB_REF_NAME}}, ${{GITHUB_REPOSITORY}}:${{GITHUB_SHA}} diff --git a/docker/Dockerfile b/docker/Dockerfile index b0610af..88b8204 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,52 +1,61 @@ # This is an auto generated Dockerfile for ros:desktop-full # generated from docker_images/create_ros_image.Dockerfile.em FROM osrf/ros:melodic-desktop-bionic - ENV CATKIN_WS=/root/covins_ws # fix keys see (https://github.com/googlecartographer/cartographer_ros/issues/1323) -RUN apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 -RUN apt-key del 421C365BD9FF1F717815A3895523BAEEB01FA116 +RUN apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' \ + --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 && \ + apt-key del 421C365BD9FF1F717815A3895523BAEEB01FA116 -# install ros packages -RUN apt-get update && apt-get install -y \ - wget \ - ros-melodic-desktop-full \ - ros-melodic-serial \ - ros-melodic-pcl-ros\ - software-properties-common\ - python3-catkin-tools \ - python3-wstool \ - vim \ - sudo \ - net-tools\ - autoconf \ - mesa-utils\ - gedit \ - gdb \ - cgdb \ +# install ros packages determine version pinning with +# apt-cache show _packages_ | grep Version +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget="1.19.*" \ + ros-melodic-desktop-full="1.4.*" \ + ros-melodic-serial="1.2.*" \ + ros-melodic-pcl-ros="1.7.*" \ + software-properties-common="0.96.*" \ + python3-catkin-tools="0.9.*" \ + python3-wstool="0.1.*" \ + vim="2:8.0.*" \ + sudo="1.8.*" \ + net-tools="1.60*" \ + autoconf="2.69*" \ + mesa-utils="8.4.*" \ + gedit="3.28.*" \ + gdb="8.1.*" \ + cgdb="0.6.*" \ && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get upgrade -y +# not sure why this is here +RUN apt-get update && apt-get upgrade -y && apt-get clean && rm -rf /var/lib/apt/lists/* # ------------------------------------------- # Libraries for building COVINS # ------------------------------------------- -RUN apt-get update && apt-get install -y \ - cmake \ - doxygen \ - libgoogle-glog-dev \ - libatlas-base-dev \ - libsuitesparse-dev \ - libyaml-cpp-dev \ - libvtk6-dev \ - libv4l-dev \ - libomp-dev \ - libglew-dev + +RUN apt-get update && apt-get install -y --no-install-recommends \ + cmake="3.10.*" \ + doxygen="1.8.*" \ + libgoogle-glog-dev="0.3.*" \ + libatlas-base-dev="3.10.*" \ + libsuitesparse-dev="1:5.1.*" \ + libyaml-cpp-dev="0.5.*" \ + libvtk6-dev="6.3.*" \ + libv4l-dev="1.14.*" \ + libomp-dev="5.0.*" \ + unzip="6.0*" \ + gcc="4:7.4.*" \ + libglew-dev="2.0.*" && \ + rm -rf /var/lib/apt/lists/* + #-------------------------- # BUILD COVINS #-------------------------- +# note this cache busts because of the copy if any files have changed at all +# including READMEs etc COPY ./ $CATKIN_WS/src/covins ARG NR_JOBS= WORKDIR $CATKIN_WS @@ -57,12 +66,13 @@ RUN catkin init && \ catkin config \ --extend /opt/ros/melodic \ --merge-devel \ - --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo && \ - chmod +x src/covins/install_file.sh && \ + --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo + +# shellcheck disable=SC2016 +RUN chmod +x src/covins/install_file.sh && \ ./src/covins/install_file.sh $NR_JOBS && \ sed -i '/exec "$@"/i \ source "${CATKIN_WS}/devel/setup.bash"' /ros_entrypoint.sh - #-------------------------- # BUILD COVINS ROS NODE #-------------------------- diff --git a/docker/Dockerfile.dockerignore b/docker/Dockerfile.dockerignore new file mode 100644 index 0000000..bcf5513 --- /dev/null +++ b/docker/Dockerfile.dockerignore @@ -0,0 +1,9 @@ +# Do not look for these files when docker builds +# https://www.geeksforgeeks.org/how-to-use-a-dockerignore-file/ +# hadolint ignore +*.md +Dockerfile.dockerignore +.git +.vscode +.gitignore +Makefile diff --git a/docker/Makefile b/docker/Makefile index 32705ae..5d4fb65 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,16 +1,122 @@ -all: help +## Dockerized Make for Covins +SHELL := /usr/bin/env bash +NR_JOBS ?= $$(($$(nproc)/2)) +# NR_JOBS ?= $(shell nproc) -help: - @echo "" - @echo "-- Help Menu" - @echo "" - @echo " 1. make build - build docker image" - @echo " 1. make clean - remove all images" - @echo "" +# registry can be docker.io, ghcr.io, gcr.io for exmaple +REGISTRY ?= docker.io +ORG ?= vis4rob +IMAGE ?= covins +DOCKERFILE ?= Dockerfile +# we are building the parent +BUILD_PATH ?= .. +# use Dockerfile.dockerignore for files like README that should not break the +# cache +# https://github.com/docker/buildx/issues/166 +# the --load not compatible with multiarch images so do not +# use with --platform=linux/arm64,linux/amd64 for example +OUTPUT ?= $(if $(PUSH),--push,--load) -build: - @echo "Use -j$(NR_JOBS) for build commands" - @docker build --tag covins -f ./Dockerfile --build-arg NR_JOBS=$(NR_JOBS) .. +# default tag is just the sha of the current git commit +# overwrite if you want with the specific tag in +# for instance make build TAG=v1.0.1 +TAG ?= $(shell git rev-parse HEAD) +# Cannot run multiple arch if memory <= 7GB +PLATFORM ?= linux/amd64,linux/arm64 +#PLATFORM ?= linux/amd64 +#PLATFORM ?= linux/arm64 +MEMORY != docker info | grep "Total Memory" | sed 's/[^0-9\.]*//g' + +.DEFAULT_GOAL := help +.PHONY: help +# https://swcarpentry.github.io/make-novice/08-self-doc/ is simpler just need +help: $(MAKEFILE_LIST) + @sed -n 's/^##//p' $(MAKEFILE_LIST) + + +## buildx: Use BuildKit if you want multiarch images +# create a builder with large log file +# latest is automatically applied so do not need to tag +.PHONY: buildx +buildx: warning + @time ( \ + docker buildx build -t $(REGISTRY)/$(ORG)/$(IMAGE):$(TAG) \ + --platform $(PLATFORM) \ + --build-arg NR_JOBS=$(NR_JOBS) \ + -f $(DOCKERFILE) \ + $(OUTPUT) \ + $(BUILD_PATH) && \ + docker tag $(REGISTRY)/$(ORG)/$(IMAGE):$(TAG) \ + $(REGISTRY)/$(ORG)/$(IMAGE):latest \ + ) + +## push: push local images +.PHONY: push +push: + docker push --all-tags $(REGISTRY)/$(ORG)/$(IMAGE) + +## build: build one architecture at time to lower memory used +.PHONY: build +build: warning + @time ( \ + docker build -t $(REGISTRY)/$(ORG)/$(IMAGE):$(TAG) \ + --build-arg NR_JOBS=$(NR_JOBS) \ + -f $(DOCKERFILE) \ + $(BUILD_PATH) && \ + docker tag $(REGISTRY)/$(ORG)/$(IMAGE):$(TAG) \ + $(REGISTRY)/$(ORG)/$(IMAGE):latest \ + ) + +## debug: dump all docker buildx messages out +# create a temporary docker context with larger log size +# https://stackoverflow.com/questions/65819424/is-there-a-way-to-increase-the-log-size-in-docker-when-building-a-container +.PHONY: debug +debug: + @time ( \ + if ! docker buildx ls | grep -q build-$(IMAGE); then \ + docker buildx create --name build-$(IMAGE) --use \ + --driver-opt env.BUILDKIT_STEP_LOG_MAX_SIZE=100000000 \ + ; fi && \ + docker buildx use big && \ + docker buildx build -t $(REGISTRY)/$(ORG)/$(IMAGE):$(TAG) \ + $(OUTPUT) \ + --progress plain \ + --platform $(PLATFORM) \ + --build-arg NR_JOBS=$(NR_JOBS) \ + -f $(DOCKERFILE) \ + $(BUILD_PATH) && \ + docker tag $(REGISTRY)/$(ORG)/$(IMAGE):$(TAG) \ + $(REGISTRY)/$(ORG)/$(IMAGE):latest && \ + docker buildx rm build-$(IMAGE) \ + ) + +## tag: Add a github tag you can use with make build TAG=_your tag_ +.PHONY: tag +tag: + @git tag -a $(TAG) -m "tagged with $(TAG)" && \ + git push --tags + +## warning: RAM and processor requirements +# note need to use bc because bash does not support floating comparison +# https://stackoverflow.com/questions/33111191/bash-bc-curl-comparing-variables +# Note that the values in a GiB so slightly more than a GB +.PHONY: warning +warning: + @echo "$(MEMORY) RAM available need at least 7GB and 32GB for two architectures" && \ + if [[ $(PLATFORM) =~ , ]] && (( $$(bc <<<"$(MEMORY) <= 31") )); then \ + echo "WARNING: Multiple build require at least 32GB of RAM"; \ + fi && \ + if (( $$(bc <<<"$(MEMORY) <= 6.8") )); then \ + echo "WARNING: build requires more than 7GB of RAM"; \ + fi + +## stats: memory and other stats of containers +.PHONY: stats +stats: + @docker stats + +## clean: remove docker images +.PHONY: clean clean: - @docker rmi -f covins + @docker rmi -f $(REGIStRY)/$(ORG)/$(IMAGE) diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..c8aa47f --- /dev/null +++ b/docker/README.md @@ -0,0 +1,97 @@ +# Dockerized COVINS build + +In the original documentation, the advice is to set `make build NR_JOBS=14` and +when that fails keep decreasing it. + +## Building Quickstart + +You can now add a make parameter and the image registry, registry organization +and image name by setting. By default you build a single image to the local +docker cache: + +```sh +# builds a single image +make build +``` + +## Pushing + +If you want to push it you can either specify this at build time or you can do +it explicitlyi + + +```sh +# push the local image that was previously +make push +# this pushs by default to docker.io/vis4rbo/covins:v1.0.1 +make build PUSH=1 +# pushes to ghcr.io/netdrones/covins-build:v1.0.1 +make build PUSH=1 REGISTRY=ghcr.io ORG=netdrones IMAGE=covins-build TAG=v1.0.1-netdrones +``` + +## Multiple architecture in a docker image + +In the original system, only Intel images are created, but by changing the +PLATFORM environment variable, you can build for instance for both Intel and +Apple Silicon: + +```sh +# build just you local machine architecture as noted in uname -m +make build +# build for intel and Apple silicon not you must have a push location +make buildx PLATFORM=linux/amd64,linux/arm64 +``` + +### Number of jobs per processor cores and jobs per make + +When you start docker, you can assign processor cores that it can use. The +build benefits from the concurrency of jobs, but at the base and run time +change significantly based on how much concurrency there. And you can also set +how many jobs per make by setting NR_JOBS. + +Note that if you are building multiple images, these builds actually run +concurrently, so if you set NR_JOBS=5 with 5 cores then 5 jobs run for each +image. So if you are building two images, then you are actually scheduling +2*5 or 10 jobs in docker. + +The default assumes you devote half the number of physical cores on your +machine with the utility `nproc` and so you are assigning one job per core. + +```sh +# set to run with 10 jobs in the build +make build NR_JOBS=10 +# this also sets 10 jobs because platform builds run concurrently +make build PLATFORM=linux/amd64,linux/arm64 NR_JOBS=5 +``` + +## How much memory to allocate to docker: 7GB for single, 24GB for dual + +The error that occurs is not easy to understand. It is a C++ compiler error +failing on number of variables being optimized, but it appears to be an out of +memory problem during the catkins process. Some experimentation is needed to +understand what the right build parameters on your configuration. + +The original system only created the container for local use. You can still use +it this way, but given the long build times for the system on the order of +hours for typical laptops, here are some parameters for how much space to +allocate to the docker machine running. On a Docker Desktop for the Mac, this +is done with the graphical interface. Other systems such as colima or multipass +handle this with command line parameters. But here are some rough guidelines +with all tests done with the default GitHub Actions runner and an Apple MacBook +Pro (2021) with an M1 Max 10-core and 64GB memory: + +- Build of single image succeeds using 7GB and 1 job for 2 cores after 2 hours. + This is what can be used with GitHub Actions as that is the +- Build of dual Intel and Apple Silicon image fails with a gcc error using 8GB + for NR_JOBS set to 5 jobs using 5 cores after 3 hours +- Build of dual fails with gcc error using 16GB and 10 jobs for 5 cores after 3 hours +- Build of dual gcc error with 24GB and 5 jobs for 5 cores after 2 hours +- Build of dual images succeeds with 32GB of RAM and 5 jobs/image for 5 cores + after 2 hours of building +- Build of dual succeeds with 32GB and 10 jobs for 5 cores after 2 hours + +## Debugging memory issues with docker stats + +If you run in a separate window `make stats` then docker stats will run and you +can see the dynamic memory requirements, these tend to peak in the later phases +of the catkins build of covins. diff --git a/readme.md b/readme.md index 584c4a4..a254c3e 100644 --- a/readme.md +++ b/readme.md @@ -7,6 +7,7 @@ COVINS is an accurate, scalable, and versatile visual-inertial collaborative SLA COVINS provides a server back-end for collaborative SLAM, running on a local machine or a remote cloud instance, generating collaborative estimates from map data contributed by different agents running Visual-Inertial Odomety (VIO) and sharing their map with the back-end. COVINS also provides a generic communication module to interface the keyframe-based VIO of your choice. Here, we provide an example of COVINS interfaced with the VIO front-end of [ORB-SLAM3](https://github.com/UZ-SLAMLab/ORB_SLAM3). We provide guidance and examples how to run COVINS on the [EuRoC dataset](https://projects.asl.ethz.ch/datasets/doku.php?id=kmavvisualinertialdatasets), as well as information beyond basic deployment, for example how the COVINS back-end can be deployed on a remote cloud computing instance. ## Index + 1. [Related Publications](#related_publications) 2. [License](#license) 3. [Basic Setup](#setup) @@ -29,6 +30,7 @@ COVINS provides a server back-end for collaborative SLAM, running on a local mac 7. [Limitations and Known Issues](#issues) + ## 1 Related Publications [***COVINS***] Patrik Schmuck, Thomas Ziegler, Marco Karrer, Jonathan Perraudin and Margarita Chli. **COVINS: Visual-Inertial SLAM for Centralized Collaboration**. *IEEE International Symposium on Mixed and Augmented Reality (ISMAR)*, 2021. **[PDF](https://www.research-collection.ethz.ch/handle/20.500.11850/507909)** @@ -39,79 +41,89 @@ COVINS provides a server back-end for collaborative SLAM, running on a local mac [*Collaborative VI-SLAM*] Patrik Schmuck, Marco Karrer and Margarita Chli. **CVI-SLAM - Collaborative Visual-Inertial SLAM**. *IEEE Robotics and Automation Letters (RA-L)*, 2018. **[PDF](https://www.research-collection.ethz.ch/bitstream/handle/20.500.11850/294281/2018_IROS_Karrer.pdf?sequence=7&isAllowed=y)** +### Video -#### Video: Mesh + ## 2 License COVINS is released under a [GPLv3 license](https://github.com/VIS4ROB-lab/covins/blob/master/.aux/licencse_gpl.txt). For a list of code/library dependencies (and associated licenses), please see [thirdparty_code.md](https://github.com/VIS4ROB-lab/covins/blob/master/.aux/thirdparty_code.md). - For license-related questions, please contact the authors: collaborative (dot) slam (at) gmail (dot) com. If you use COVINS in an academic work, please cite: +```cite @article{schmuck2021covins, title={COVINS: Visual-Inertial SLAM for Centralized Collaboration}, author={Schmuck, Patrik and Ziegler, Thomas and Karrer, Marco and Perraudin, Jonathan and Chli, Margarita}, journal={arXiv preprint arXiv:2108.05756}, year={2021} } +```` ## 3 Basic Setup + This section explains how you can build the COVINS server back-end, as well as the provided version of the ORB-SLAM3 front-end able to communicate with the back-end. COVINS was developed under Ubuntu *18.04*, and we provide installation instructions for *18.04* as well as *20.04*. Note that we also provide a [Docker implementation](#docker) for simplified deployment of COVINS. -**Note**: Please pay attention to the ```CMAKE_BUILD_TYPE```. Particularly, building parts of the code with ```march=native``` can cause problems on some machines. +**Note**: Please pay attention to the `CMAKE_BUILD_TYPE`. Particularly, +building parts of the code with `march=native` can cause problems on some +machines. + ### Environment Setup #### Dependencies -* ```sudo apt-get update``` -* Install dependencies: ```sudo apt-get install libpthread-stubs0-dev build-essential cmake git doxygen libsuitesparse-dev libyaml-cpp-dev libvtk6-dev python3-wstool libomp-dev libglew-dev``` +* `sudo apt-get update` +* Install dependencies: `sudo apt-get install libpthread-stubs0-dev build-essential cmake git doxygen libsuitesparse-dev libyaml-cpp-dev libvtk6-dev python3-wstool libomp-dev libglew-dev` * _catkin_tools_ (from the [catkin_tools manual](https://catkin-tools.readthedocs.io/en/latest/installing.html)) - * ```sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list'``` - * ```wget http://packages.ros.org/ros.key -O - | sudo apt-key add -``` - * ```sudo apt-get update``` - * ```sudo apt-get install python3-catkin-tools``` + * `sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list'` + * `wget http://packages.ros.org/ros.key -O - | sudo apt-key add -` + * `sudo apt-get update` + * `sudo apt-get install python3-catkin-tools` * _ROS_ * [Melodic](https://wiki.ros.org/melodic/Installation/Ubuntu) (Ubuntu 18) * [Noetic](https://wiki.ros.org/noetic/Installation/Ubuntu) (Ubuntu 20) #### Set up your workspace -This will create a workspace for COVINS as ```~/ws/covins_ws```. All further commands will use this path structure - if you decide to change the workspace path, you will need to adjust the commands accordingly. - -* ```cd ~``` -* ```mkdir -p ws/covins_ws/src``` -* ```cd ~/ws/covins_ws``` -* ```catkin init``` -* ROS Setup - * **U18/Melodic**: ```catkin config --extend /opt/ros/melodic/``` - * **U20/Noetic**: ```catkin config --extend /opt/ros/noetic/``` -* ```catkin config --merge-devel``` -* ```catkin config --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo``` +This will create a workspace for COVINS as `~/ws/covins_ws`. All further commands will use this path structure - if you decide to change the workspace path, you will need to adjust the commands accordingly. + +```sh +cd ~ +mkdir -p ws/covins_ws/src +cd ~/ws/covins_ws +catkin init +# ROS Setup *U18/Melodic**: +catkin config --extend /opt/ros/melodic/ +# **U20/Noetic**: +catkin config --extend /opt/ros/noetic/ +catkin config --merge-devel +catkin config --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo +``` + ### COVINS Installation -We provide a script (```covins/install_file.sh```) that will perform a full installation of COVINS, including back-end, front-end, and third-party packages, if the environment is set up correctly. **If the installation fails, we strongly recommend executing the steps in the build script manually one by one**. The script might not perform a correct installation under certain circumstances if executed multiple times. +We provide a script (`covins/install_file.sh`) that will perform a full installation of COVINS, including back-end, front-end, and third-party packages, if the environment is set up correctly. **If the installation fails, we strongly recommend executing the steps in the build script manually one by one**. The script might not perform a correct installation under certain circumstances if executed multiple times. -* ```cd ~/ws/covins_ws/src``` -* ```git clone https://github.com/VIS4ROB-lab/covins.git``` -* ```cd ~/ws/covins_ws``` -* ```chmod +x src/covins/install_file.sh``` -* ```./src/covins/install_file.sh 8``` - * The argument ```8``` is optional, and specifies the number of jobs the build process should use. +* `cd ~/ws/covins_ws/src` +* `git clone https://github.com/VIS4ROB-lab/covins.git` +* `cd ~/ws/covins_ws` +* `chmod +x src/covins/install_file.sh` +* `./src/covins/install_file.sh 8` + * The argument `8` is optional, and specifies the number of jobs the build process should use. -Generally, when the build process of COVINS or ORB-SLAM3 fails, make sure you have correctly sourced the workspace, and that the libraries in the third-party folders, such as ```DBoW2``` and ```g2o``` are built correctly. +Generally, when the build process of COVINS or ORB-SLAM3 fails, make sure you have correctly sourced the workspace, and that the libraries in the third-party folders, such as `DBoW2` and `g2o` are built correctly. -A remark on ```fix_eigen_deps.sh```: compiling code with dependencies against multiple ```Eigen``` versions is usually fatal and must be avoided. Therefore, we specify and download the ```Eigen``` version explicitly through the ```eigen_catkin``` package, and make sure all ```Eigen``` dependencies point to this package. +A remark on `fix_eigen_deps.sh`: compiling code with dependencies against multiple `Eigen` versions is usually fatal and must be avoided. Therefore, we specify and download the `Eigen` version explicitly through the `eigen_catkin` package, and make sure all `Eigen` dependencies point to this package. ### Installing ROS Support for the ORB-SLAM3 Front-End @@ -119,80 +131,81 @@ A remark on ```fix_eigen_deps.sh```: compiling code with dependencies against mu If you want to use `rosbag` files to pass sensor data to COVINS, you need to explicitly build the ORB-SLAM3 front-end with ROS support. * Install _vision_opencv_: - * ```cd ~/ws/covins_ws/src``` - * Clone: ```git clone https://github.com/ros-perception/vision_opencv.git``` + * `cd ~/ws/covins_ws/src` + * Clone: `git clone https://github.com/ros-perception/vision_opencv.git` * **Check out the correct branch** - * ```cd vision_opencv/``` - * *U18/Melodic*: ```git checkout melodic``` - * *U20/Noetic*: ```git checkout noetic``` - * Open ```~/ws/covins_ws/src/vision_opencv/cv_bridge/CMakeLists.txt``` - * Add the ```opencv3_catkin``` dependency: change the line ```find_package(catkin REQUIRED COMPONENTS rosconsole sensor_msgs)``` to ```find_package(catkin REQUIRED COMPONENTS rosconsole sensor_msgs opencv3_catkin)``` - * If you are running **Ubuntu 20** (or generally have OpenCV 4 installed): remove the lines that search for an OpenCV 4 version in the ```CMakeLists.txt```. It should look like this: + * `cd vision_opencv/` + * *U18/Melodic*: `git checkout melodic` + * *U20/Noetic*: `git checkout noetic` + * Open `~/ws/covins_ws/src/vision_opencv/cv_bridge/CMakeLists.txt` + * Add the `opencv3_catkin` dependency: change the line `find_package(catkin REQUIRED COMPONENTS rosconsole sensor_msgs)` to `find_package(catkin REQUIRED COMPONENTS rosconsole sensor_msgs opencv3_catkin)` + * If you are running **Ubuntu 20** (or generally have OpenCV 4 installed): remove the lines that search for an OpenCV 4 version in the `CMakeLists.txt`. It should look like this: ![cv_bridge OpenCV dependencies](.aux/cv_bridge_opencv_dependency.png) - * ```source ~/ws/covins_ws/devel/setup.bash``` - * ```catkin build cv_bridge``` + * `source ~/ws/covins_ws/devel/setup.bash` + * `catkin build cv_bridge` * [Optional] Check correct linkage: - * ```cd ~/ws/covins_ws/devel/lib``` - * ```ldd libcv_bridge.so | grep opencv_core``` - * This should only list ```libopencv_core.so.3.4``` as a dependency -* ```catkin build ORB_SLAM3``` + * `cd ~/ws/covins_ws/devel/lib` + * `ldd libcv_bridge.so | grep opencv_core` + * This should only list `libopencv_core.so.3.4` as a dependency +* `catkin build ORB_SLAM3` * [Optional] Check correct linkage: - * ```cd ~/ws/covins_ws/src/covins/orb_slam3/Examples/ROS/ORB_SLAM3``` - * ```ldd Mono_Inertial | grep opencv_core``` - * This should mention ```libopencv_core.so.3.4``` as the only ```libopencv_core``` dependency + * `cd ~/ws/covins_ws/src/covins/orb_slam3/Examples/ROS/ORB_SLAM3` + * `ldd Mono_Inertial | grep opencv_core` + * This should mention `libopencv_core.so.3.4` as the only `libopencv_core` dependency ## 4 Running COVINS -This section explains how to run COVINS on the [EuRoC dataset](https://projects.asl.ethz.ch/datasets/doku.php?id=kmavvisualinertialdatasets). If you want to use a different dataset, please do not forget to use a correct parameter file instead of ```covins/orb_slam3/Examples/Monocular-Inertial/EuRoC.yaml```. +This section explains how to run COVINS on the [EuRoC dataset](https://projects.asl.ethz.ch/datasets/doku.php?id=kmavvisualinertialdatasets). If you want to use a different dataset, please do not forget to use a correct parameter file instead of `covins/orb_slam3/Examples/Monocular-Inertial/EuRoC.yaml`. **Note**: We strongly recommend running every agent and the server back-end in a separate terminal each. #### Setting up the environment -* In ```~/ws/covins_ws/src/covins/covins_comm/config/config_comm.yaml```: adjust the value of ```sys.server_ip``` to the IP of the machine where the COVINS back-end is running. -* In *every* of the provided scripts to run the ORB-SLAM3 front-end (e.g., ```euroc_examples_mh1.sh```, in ```orb_slam3/covins_examples/```), adjust ```pathDatasetEuroc``` to the path where the dataset has been uncompressed. The default expected path is ```/MH_01_easy/mav0/...``` (for ```euroc_examples_mh1.sh```, in this case). -* In ```~/ws/covins_ws/src/covins/covins_backend/config/config_backend.yaml```: adjust the path of ```sys.map_path0``` to the directory where you would like to load maps from. +* In `~/ws/covins_ws/src/covins/covins_comm/config/config_comm.yaml`: adjust the value of `sys.server_ip` to the IP of the machine where the COVINS back-end is running. +* In *every* of the provided scripts to run the ORB-SLAM3 front-end (e.g., `euroc_examples_mh1.sh`, in `orb_slam3/covins_examples/`), adjust `pathDatasetEuroc` to the path where the dataset has been uncompressed. The default expected path is `/MH_01_easy/mav0/...` (for `euroc_examples_mh1.sh`, in this case). +* In `~/ws/covins_ws/src/covins/covins_backend/config/config_backend.yaml`: adjust the path of `sys.map_path0` to the directory where you would like to load maps from. + ### Running the COVINS Server Back-End -* Source your workspace: ```source ~/ws/covins_ws/devel/setup.bash``` -* In a terminal, start a roscore: ```roscore``` -* In a new terminal, start the COVINS backend by executing ```rosrun covins_backend covins_backend_node``` +* Source your workspace: `source ~/ws/covins_ws/devel/setup.bash` +* In a terminal, start a roscore: `roscore` +* In a new terminal, start the COVINS backend by executing `rosrun covins_backend covins_backend_node` + ### Running the ORB-SLAM3 Front-End -Example scripts are provided in ```orb_slam3/covins_examples/```. Don't forget to correctly set the dataset path in every script you want to use (see above: *Setting up the environment*). You can also check the original [ORB-SLAM3 Repo](https://github.com/UZ-SLAMLab/ORB_SLAM3) for help on how to use the ORB-SLAM3 front-end. +Example scripts are provided in `orb_slam3/covins_examples/`. Don't forget to correctly set the dataset path in every script you want to use (see above: *Setting up the environment*). You can also check the original [ORB-SLAM3 Repo](https://github.com/UZ-SLAMLab/ORB_SLAM3) for help on how to use the ORB-SLAM3 front-end. * Download the [EuRoC dataset](https://projects.asl.ethz.ch/datasets/doku.php?id=kmavvisualinertialdatasets) (ASL dataset format) * Open a a new terminal. -* Source your workspace: ```source ~/ws/covins_ws/devel/setup.bash``` -* ```cd``` to the folder with the example scripts:```cd ~/ws/covins_ws/src/covins/orb_slam3/covins_examples/``` -* Execute one of the example scripts provided in the ```orb_slam3/covins_examples/``` folder, such as ```euroc_examples_mh123_vigba``` - * **Note**: The example scripts **must be executed from inside the ```covins_examples``` folder** - * ```euroc_examples_mhX.sh``` runs the front-end with a single sequence from EuRoC MH1-5. - * ```euroc_examples_mh123_vigba.sh``` runs a 3-agent collaborative SLAM session (sequential) followed by Bundle Adjustment. - * ```euroc_examples_mh12345_vigba.sh``` runs a 5-agent collaborative SLAM session (sequential) followed by Bundle Adjustment. +* Source your workspace: `source ~/ws/covins_ws/devel/setup.bash` +* `cd` to the folder with the example scripts:`cd ~/ws/covins_ws/src/covins/orb_slam3/covins_examples/` +* Execute one of the example scripts provided in the `orb_slam3/covins_examples/` folder, such as `euroc_examples_mh123_vigba` + * **Note**: The example scripts **must be executed from inside the `covins_examples` folder** + * `euroc_examples_mhX.sh` runs the front-end with a single sequence from EuRoC MH1-5. + * `euroc_examples_mh123_vigba.sh` runs a 3-agent collaborative SLAM session (sequential) followed by Bundle Adjustment. + * `euroc_examples_mh12345_vigba.sh` runs a 5-agent collaborative SLAM session (sequential) followed by Bundle Adjustment. * Multiple front-ends can run in parallel. The front-ends can run on the same machine, or on different machines connected through a wireless network. However, when running multiple front-ends on the same machine, note that the performance of COVINS might degrade if the computational resources are overloaded by running too many agents simultaneously. We recommend running every front-end instance in a separate terminal. * Common error sources: - * If the front-end is stuck after showing ```Loading images for sequence 0...LOADED!```, most likely your dataset path is wrong. - * If the front-end is stuck after showing ```--> Connect to server -``` or shows an error message ```Could no establish connection - exit```, the server is not reachable - the IP might be incorrect, you might have forgotten to start the server, or there is a problem with your network (try pinging the server IP) + * If the front-end is stuck after showing `Loading images for sequence 0...LOADED!`, most likely your dataset path is wrong. + * If the front-end is stuck after showing `--> Connect to server` or shows an error message `Could no establish connection - exit`, the server is not reachable - the IP might be incorrect, you might have forgotten to start the server, or there is a problem with your network (try pinging the server IP) -COVINS does not support resetting the map onboard the agent. Since map resets are more frequent at the beginning of a session or dataset, for example due to faulty initialization, in the current implementation, the COVINS communication module is set up such that it only starts sending data if a pre-specified number of keyframes was already created by the front-end. This number is specified by ```comm.start_sending_after_kf``` in ```covins/covins_comm/config/config_comm.yaml```, and is currently set to 50. Also check [Limitations](#issues) for more details. +COVINS does not support resetting the map onboard the agent. Since map resets are more frequent at the beginning of a session or dataset, for example due to faulty initialization, in the current implementation, the COVINS communication module is set up such that it only starts sending data if a pre-specified number of keyframes was already created by the front-end. This number is specified by `comm.start_sending_after_kf` in `covins/covins_comm/config/config_comm.yaml`, and is currently set to 50. Also check [Limitations](#issues) for more details. ### Visualization -COVINS provides a config file for visualization with RVIZ (```covins.rviz``` in ```covins_backend/config/```) +COVINS provides a config file for visualization with RVIZ (`covins.rviz` in `covins_backend/config/`) -* In a new terminal: run ```tf.launch``` in ```covins_backend/launch/``` to set up the coordinate frames for visualization: ```roslaunch ~/ws/covins_ws/src/covins/covins_backend/launch/tf.launch``` -* In a new terminal: launch RVIZ: ```rviz -d ~/ws/covins_ws/src/covins/covins_backend/config/covins.rviz``` - * Covisibility edges between keyframes from different agents are shown in red, while edges between keyframes from the same agent are colored gray (those are not shown by default, but can be activated by setting ```vis.covgraph_shared_edges_only``` to ```0``` in ```config_backend.yaml```). +* In a new terminal: run `tf.launch` in `covins_backend/launch/` to set up the coordinate frames for visualization: `roslaunch ~/ws/covins_ws/src/covins/covins_backend/launch/tf.launch` +* In a new terminal: launch RVIZ: `rviz -d ~/ws/covins_ws/src/covins/covins_backend/config/covins.rviz` + * Covisibility edges between keyframes from different agents are shown in red, while edges between keyframes from the same agent are colored gray (those are not shown by default, but can be activated by setting `vis.covgraph_shared_edges_only` to `0` in `config_backend.yaml`). * In case keyframes are visualized, removed keyframes are displayed in red (keyframes are not shown by default, but can be activated in RVIZ). - * The section _VISUALIZATION_ in ```config_backend.yaml``` provides several options to modify the visualization. + * The section _VISUALIZATION_ in `config_backend.yaml` provides several options to modify the visualization. ![Covisibilty graphs](.aux/cov_graph_examples.png) @@ -203,51 +216,51 @@ COVINS provides a config file for visualization with RVIZ (```covins.rviz``` in COVINS provides several options to interact with the map held by the back-end. This is implemented through ROS services. -* Make sure your workspace is sourced: ```source ~/ws/covins_ws/devel/setup.bash``` -* **Map save:** ```rosservice call /covins_savemap ``` - this saves the map associated to the agent specified by ```AGENT_ID```. - * The map will be saved to the folder ```..../covins_backend/output/map_data```. Make sure the folder is empty, before you save a map (COVINS performs a brief check - if a folder named ```keyframes/``` or ```mappoints/``` exists in the target directory, it will show an error and abort the map save process. Any other files or folders will not result in an error though). -* **Map load:** ```rosservice call /covins_loadmap 0``` - loads a map stored on disk, from the folder specified by ```sys.map_path0``` in ```config_backend.yaml```. +* Make sure your workspace is sourced: `source ~/ws/covins_ws/devel/setup.bash` +* **Map save:** `rosservice call /covins_savemap ` - this saves the map associated to the agent specified by `AGENT_ID`. + * The map will be saved to the folder `..../covins_backend/output/map_data`. Make sure the folder is empty, before you save a map (COVINS performs a brief check - if a folder named `keyframes/` or `mappoints/` exists in the target directory, it will show an error and abort the map save process. Any other files or folders will not result in an error though). +* **Map load:** `rosservice call /covins_loadmap 0` - loads a map stored on disk, from the folder specified by `sys.map_path0` in `config_backend.yaml`. * Note: map load needs to be performed **before** registering any agent. - * ```0``` specifies the operation mode of the load functionality. ```0``` means "standard" map loading, while ```1``` and ```2``` will perform place recognition (```1```) and place recognition and PGO (```2```). Note that both modes with place recognition are experimental, only "standard" map load is tested and supported for the open-source version of COVINS. -* **Bundle Adjustment:** ```rosservice call /covins_gba -``` - Performs visual-inertial bundle adjustemt on the map associated to the agent specified by ```AGENT_ID```. Modes: 0: BA *without* outlier rejection, 1: BA *with* outlier rejection. -* **Map Compression / Redundancy Removal:** ```rosservice call /covins_prunemap ``` - performs redundancy detection and removal on the map associated to the agent specified by ```AGENT_ID```. - * ```MAX_KFs``` specifies the target number of keyframes held by the compressed map. If ```MAX_KFs=0```, the threshold value for measuring redundancy specified by the parameter ```kf_culling_th_red``` in ```config_backend.yaml``` will be used. + * `0` specifies the operation mode of the load functionality. `0` means "standard" map loading, while `1` and `2` will perform place recognition (`1`) and place recognition and PGO (`2`). Note that both modes with place recognition are experimental, only "standard" map load is tested and supported for the open-source version of COVINS. +* **Bundle Adjustment:** `rosservice call /covins_gba ` +- Performs visual-inertial bundle adjustment on the map associated to the agent specified by `AGENT_ID`. Modes: 0: BA *without* outlier rejection, 1: BA *with* outlier rejection. +* **Map Compression / Redundancy Removal:** `rosservice call /covins_prunemap ` - performs redundancy detection and removal on the map associated to the agent specified by `AGENT_ID`. + * `MAX_KFs` specifies the target number of keyframes held by the compressed map. If `MAX_KFs=0`, the threshold value for measuring redundancy specified by the parameter `kf_culling_th_red` in `config_backend.yaml` will be used. * All experiments with COVINS were performed specifying the target keyframe count. Therefore, we recommend resorting to this functionality. - * The parameter ```kf_culling_max_time_dist``` in ```config_backend.yaml``` specifies a maximum time delta permitted between two consecutive keyframes, in order to ensure limit the error of IMU integration. If no keyframe can removed without violating this constraint, map compression will stop, even if the target number of keyframes is not reached. -* Note: After a map merge of the maps associated to Agent 0 and Agent 1, the merged map is associated to both agents, i.e. ```rosservice call /covins_savemap 0``` and ```rosservice call /covins_savemap 1``` will save the same (shared) map. + * The parameter `kf_culling_max_time_dist` in `config_backend.yaml` specifies a maximum time delta permitted between two consecutive keyframes, in order to ensure limit the error of IMU integration. If no keyframe can removed without violating this constraint, map compression will stop, even if the target number of keyframes is not reached. +* Note: After a map merge of the maps associated to Agent 0 and Agent 1, the merged map is associated to both agents, i.e. `rosservice call /covins_savemap 0` and `rosservice call /covins_savemap 1` will save the same (shared) map. ### Parameters COVINS provides two parameter files to adjust the behavior of the system and algorithms. -* ```../covins_comm/config/config_comm.yaml``` contains all parameters related to communication and the agent front-end. -* ```../covins_backend/config/config_backend.yaml``` contains all parameters related to the server back-end. +* `../covins_comm/config/config_comm.yaml` contains all parameters related to communication and the agent front-end. +* `../covins_backend/config/config_backend.yaml` contains all parameters related to the server back-end. The user should not be required to change any parameters to run COVINS, except paths and the server IP, as explained in this manual. ### Output Files -* COVINS automatically saves the trajectory estimate of each agent to a file in ```covins_backend/output```. The file ```KF_.csv``` stores the poses associated to the agent specified by ```AGENT_ID```. Each row represents a single pose. -* COVINS can save the trajectory in 2 formats: *EuRoC format* and *TUM format*. Which one is used can be controlled via the parameter ```trajectory_format``` in ```config_backend.yaml```. - * **TUM format** (default): ```timestamp[s] t_x t_y t_z q_x q_y q_z q_w``` - * **EuRoC format**: ```timestamp[ns], t_x, t_y, t_z, q_w, q_x, q_y, q_z, vel_x, vel_y, vel_z, bias_gyro_x, bias_gyro_y, bias_gyro_z, bias_acc_x, bias_acc_y, bias_acc_z``` - * Each output file contains a suffix indicating the format: ```_ftum``` or ```_feuroc``` +* COVINS automatically saves the trajectory estimate of each agent to a file in `covins_backend/output`. The file `KF_.csv` stores the poses associated to the agent specified by `AGENT_ID`. Each row represents a single pose. +* COVINS can save the trajectory in 2 formats: *EuRoC format* and *TUM format*. Which one is used can be controlled via the parameter `trajectory_format` in `config_backend.yaml`. + * **TUM format** (default): `timestamp[s] t_x t_y t_z q_x q_y q_z q_w` + * **EuRoC format**: `timestamp[ns], t_x, t_y, t_z, q_w, q_x, q_y, q_z, vel_x, vel_y, vel_z, bias_gyro_x, bias_gyro_y, bias_gyro_z, bias_acc_x, bias_acc_y, bias_acc_z` + * Each output file contains a suffix indicating the format: `_ftum` or `_feuroc` * Trajectories in *TUM format* can be directly evaluated using the [evo evaluation tool](https://github.com/MichaelGrupp/evo). - * Run the evaluation e.g. as ```evo_ape euroc KF_0_ftum.csv gt_data.csv -vas``` to perform a Sim(3) alignment reporting trajectory RMSE and scale error. - * The ground truth data for the individual EuRoC sequences can be found in ```/mav0/state_groundtruth_estimate0/data.csv``` - * To evaluate a multi-agents estimate, the individual trajectory files must be combined, e.g. with ```cat KF_0_ftum.csv KF_1_ftum.csv KF_2_ftum.csv > mh123_est.csv```. Also, the individual ground truth information from the EuRoC sequences used to generate the estimate must be combined into a single file. We recommend doing this manually, since every file contains a header describing the data, which should not be copied multiple times. + * Run the evaluation e.g. as `evo_ape euroc KF_0_ftum.csv gt_data.csv -vas` to perform a Sim(3) alignment reporting trajectory RMSE and scale error. + * The ground truth data for the individual EuRoC sequences can be found in `/mav0/state_groundtruth_estimate0/data.csv` + * To evaluate a multi-agents estimate, the individual trajectory files must be combined, e.g. with `cat KF_0_ftum.csv KF_1_ftum.csv KF_2_ftum.csv > mh123_est.csv`. Also, the individual ground truth information from the EuRoC sequences used to generate the estimate must be combined into a single file. We recommend doing this manually, since every file contains a header describing the data, which should not be copied multiple times. ### Running COVINS with ROS -* Make sure your workspace is sourced: ```source ~/ws/covins_ws/devel/setup.bash``` -* In ```~/ws/covins_ws/src/covins/orb_slam3/Examples/ROS/ORB_SLAM3/launch/launch_ros_euroc.launch```: adjust the paths for ```voc``` and ```cam``` -* ```cd``` to ```orb_slam3/``` and run ```roslaunch ORB_SLAM3 launch_ros_euroc.launch``` -* In a new terminal: run the rosbag file, e.g. ```rosbag play MH_01_easy.bag``` - * When using COVINS with ROS, we recommend skipping the initialization sequence performed at the beginning of each EuRoC MH trajectory. ORB-SLAM3 often performs a map reset after this sequence, which is not supported by COVINS and will therefore cause an error. For example, for MH1, this can be easily done by running ```rosbag play MH_01_easy.bag --start 45```. (Start at: MH01: 45s; MH02: 35s; MH03-05: 15s) +* Make sure your workspace is sourced: `source ~/ws/covins_ws/devel/setup.bash` +* In `~/ws/covins_ws/src/covins/orb_slam3/Examples/ROS/ORB_SLAM3/launch/launch_ros_euroc.launch`: adjust the paths for `voc` and `cam` +* `cd` to `orb_slam3/` and run `roslaunch ORB_SLAM3 launch_ros_euroc.launch` +* In a new terminal: run the rosbag file, e.g. `rosbag play MH_01_easy.bag` + * When using COVINS with ROS, we recommend skipping the initialization sequence performed at the beginning of each EuRoC MH trajectory. ORB-SLAM3 often performs a map reset after this sequence, which is not supported by COVINS and will therefore cause an error. For example, for MH1, this can be easily done by running `rosbag play MH_01_easy.bag --start 45`. (Start at: MH01: 45s; MH02: 35s; MH03-05: 15s) #### Running a second agent in parallel with ROS @@ -256,51 +269,84 @@ When you want to run 2 (or more) agents in parallel, you need to adjust the laun * You need to **remap the input topics**, so that ROS nodes for different agents listen to different camera and IMU topics * You need to **remap the topics of the bagfiles** for the individual agents, so that they 1) match the topics specified in the launch files and 2) the rosbag files played in parallel do not publish IMU and camera data under the same topic. -We provide an example for two agents. For the first agent, you can execute the launch file and play the bag file as described above. For the second agent, we provide ```launch_ros_euroc_second_agent.launch```. -* ```name``` is changed to ```ORB_SLAM3_monoi1``` -* Expected input topics are changed to ```/cam0/image_raw1``` and ```/imu1``` -* Run with ```roslaunch ORB_SLAM3 launch_ros_euroc_second_agent.launch``` -* Play the bag file of your choice with remapped topics by appending ```/cam0/image_raw:=/cam0/image_raw1 /cam1/image_raw:=/cam1/image_raw1 /imu0:=/imu1```. For example, for ```MH_02_easy.bag```, you would run it with ```rosbag play MH_02_easy.bag --start 35 /cam0/image_raw:=/cam0/image_raw1 /cam1/image_raw:=/cam1/image_raw1 /imu0:=/imu1``` +We provide an example for two agents. For the first agent, you can execute the launch file and play the bag file as described above. For the second agent, we provide `launch_ros_euroc_second_agent.launch`. +* `name` is changed to `ORB_SLAM3_monoi1` +* Expected input topics are changed to `/cam0/image_raw1` and `/imu1` +* Run with `roslaunch ORB_SLAM3 launch_ros_euroc_second_agent.launch` +* Play the bag file of your choice with remapped topics by appending `/cam0/image_raw:=/cam0/image_raw1 /cam1/image_raw:=/cam1/image_raw1 /imu0:=/imu1`. For example, for `MH_02_easy.bag`, you would run it with `rosbag play MH_02_easy.bag --start 35 /cam0/image_raw:=/cam0/image_raw1 /cam1/image_raw:=/cam1/image_raw1 /imu0:=/imu1` -Note: if you **run multiple agents sequentially**, you only need to use ```launch_ros_euroc.launch```. After one agent has finished, just start it again using ```roslaunch ORB_SLAM3 launch_ros_euroc.launch``` and run your bagfile. +Note: if you **run multiple agents sequentially**, you only need to use `launch_ros_euroc.launch`. After one agent has finished, just start it again using `roslaunch ORB_SLAM3 launch_ros_euroc.launch` and run your bagfile. + ## 5 Docker Implementation -We provide COVINS also as a Docker implementation. A guide how to install docker can be found [here](https://docs.docker.com/engine/install/). To avoid the need of `sudo` when running the commands below you can add your user to the `docker` group. +We provide COVINS also as a Docker implementation. A guide how to install +docker can be found [here](https://docs.docker.com/engine/install/). To avoid +the need of `sudo` when running the commands below you can add your user to the +`docker` group. (see https://docs.docker.com/engine/install/linux-postinstall/) -```sudo usermod -aG docker $USER``` (see https://docs.docker.com/engine/install/linux-postinstall/) +```sh +sudo usermod -aG docker $USER +``` ### Building the Docker Image -**NOTE:** The provided docker implementation will **not build** if you have previously built your cloned version of COVINS in a catkin workspace using the instructions provided above. If you want to use the docker implementation, please **perform the docker build with a freshly cloned version of COVINS** - -Build the docker file using the Make file provided in the `docker` folder. Provide the number of jobs `make` and `catkin build` should use. This can take a while. If the build fails try again with a reduced number of jobs value. +**NOTE:** The provided docker implementation will **not build** if you have +previously built your cloned version of COVINS in a catkin workspace using the +instructions provided above. If you want to use the docker implementation, +please **perform the docker build with a freshly cloned version of COVINS** + +Build the docker file using the Make file provided in the `docker` folder. +Provide the number of jobs `make` and `catkin build` should use. This can take +a while. If the build fails try again with a reduced number of jobs value. This +appears to be related to the memory you have allocated for docker. With a 7GB +memory allocation for GitHub Action runners as an example, you can build a +single image with NR_JOBS=1 and this will take about two hours. + +The failure is typically something deep in catkins involving a mysterious error +in opengv complication where the gcc compiler gives up. The make file by +default creates both an Intel and an Apple Silicon M1 image. This requires at +least 12GB of RAM to run with parallel builds using Buildkit. + +The makefile will by default push these images to Docker Hub at docker.io (and +optionall the GitHub container registry at ghcr.io) by changing environment +variables before running make build + +```sh +# start at 14 jobs and see if it fails usually with a c++ error +make build NR_JOBS=14 +# decreases until you get a clean build +# for example, build with half the processors on the machine +make build NR_JOBS=$(($(nproc/2))) +# to build for a different registry and a different org +# with a different semvar tag +make build REGISTRY=ghcr.io ORG=netdrones TAG=v1.0.1-net +``` -* ```make build NR_JOBS=14``` - ### Running the Docker Image + The docker image can be used to run different parts of COVINS (e.g. server, ORB-SLAM3 front-end, ...). #### ROSCORE To start the roscore one can either use the host system ROS implementation (if ROS is installed). Otherwise, it can be started using the docker image. -* ```./run.sh -c``` +* `./run.sh -c` #### COVINS Server Back-End The convins server back-end needs a running roscore, how to start one see above. Furthermore, the server needs two configuration files, one for the communication server on one for the back-end. These two files need to be linked when running the docker image. -* ```./run.sh -s ../covins_comm/config/config_comm.yaml ../covins_backend/config/config_backend.yaml``` +* `./run.sh -s ../covins_comm/config/config_comm.yaml ../covins_backend/config/config_backend.yaml` #### ORB-SLAM3 Front-End The ORB-SLAM3 front-end client needs the communication server config file, the file which should be executed, and the path to the dataset. The dataset has to be given seperately since the file system of the docker container differs from the host system. Hence, the `pathDatasetEuroc` variable in the run script gets adapted automatically inside the docker container. -* ```./run.sh -o ../covins_comm/config/config_comm.yaml ../orb_slam3/covins_examples/euroc_examples_mh1 ``` +* `./run.sh -o ../covins_comm/config/config_comm.yaml ../orb_slam3/covins_examples/euroc_examples_mh1 ` #### ORB-SLAM3 ROS Front-End The ROS wrapper of the ORB-SLAM3 front-end can also be started in the docker container. It requires the server config file and the ROS launch file. A bag file can then for example be played on the host system. -* ```./run.sh -r ../covins_comm/config/config_comm.yaml ../orb_slam3/Examples/ROS/ORB_SLAM3/launch/launch_docker_ros_euroc.launch``` +* `./run.sh -r ../covins_comm/config/config_comm.yaml ../orb_slam3/Examples/ROS/ORB_SLAM3/launch/launch_docker_ros_euroc.launch` #### Terminal A terminal within the docker image can also be opened. This can for example be used to send `rosservice` commands. -* ```./run.sh -t``` +* `./run.sh -t` ## 6 Extended Functionalities @@ -313,7 +359,7 @@ We have tested COVINS with remote cloud servers provides by [Amazons AWS](https: #### Cloud Setup -* Choose an AWS instance to launch (EC2 instance, Ubuntu 18 or 20 virtual machine, for example a ```c4.4xlarge``` with 30 GB storage is a good starting point) +* Choose an AWS instance to launch (EC2 instance, Ubuntu 18 or 20 virtual machine, for example a `c4.4xlarge` with 30 GB storage is a good starting point) * AWS virtual machines by default block most incoming and outgoing traffic, so you need to adjust the security groups. Don't forget to modify both inbound and outbound rules. * Some information on setting security groups can be found [here](https://aws.amazon.com/premiumsupport/knowledge-center/connect-http-https-ec2/). * As a straightforward solution, you can just allow **any** traffic: In *Step 6: Configure Security Group* of the AWS instance launch process, add a rule as follows (note that this leaves your remote server also unprotected against unauthorized access): @@ -323,9 +369,9 @@ We have tested COVINS with remote cloud servers provides by [Amazons AWS](https: ![AWS security group setup](.aux/aws_securiy_rules.png) --> * Launch your instance and install COVINS -* On your local machine (or any other machine where you would like to run the front-end), set ```sys.server_ip``` in ```config_comm.yaml``` to the **public** IPv4 address of your instance +* On your local machine (or any other machine where you would like to run the front-end), set `sys.server_ip` in `config_comm.yaml` to the **public** IPv4 address of your instance * Try to ping the IP from your local machine. If this does not work, the security groups are definitely not set up correctly and the back-end on the remote instance will not be reachable. -* Start the covins back-end on the remote instance as usual using ```rosrun covins_backend covins_backend_node``` +* Start the covins back-end on the remote instance as usual using `rosrun covins_backend covins_backend_node` * Start the front-end on your local machine as usual using one of the provided example files. * The front-end should be able to connect to the cloud server, and you should see the server displaying messages that data is arriving there. @@ -334,42 +380,42 @@ We have tested COVINS with remote cloud servers provides by [Amazons AWS](https: We will use [screen](https://linuxize.com/post/how-to-use-linux-screen/) to run COVINS remotely on a cloud server * On the **remote server** (connect via ssh - your AWS instance will give instructions how to do this) - * ```screen -S covins``` - * ```roscore``` - * New window: ```Ctrl+a```, then ```c``` - * ```export ROS_IP=public_IP_of_!SERVER!``` - * ```cd ~/ws/covins_ws/``` - * ```source devel/setup.bash``` - * ```roslaunch src/covins/covins_backend/launch/tf.launch``` - * New window: ```Ctrl+a```, then ```c``` - * ```export ROS_IP=public_IP_of_!SERVER!``` - * ```cd ~/ws/covins_ws/``` - * ```source devel/setup.bash``` - * Run the back-end using ```rosrun covins_backend covins_backend_node``` + * `screen -S covins` + * `roscore` + * New window: `Ctrl+a`, then `c` + * `export ROS_IP=public_IP_of_!SERVER!` + * `cd ~/ws/covins_ws/` + * `source devel/setup.bash` + * `roslaunch src/covins/covins_backend/launch/tf.launch` + * New window: `Ctrl+a`, then `c` + * `export ROS_IP=public_IP_of_!SERVER!` + * `cd ~/ws/covins_ws/` + * `source devel/setup.bash` + * Run the back-end using `rosrun covins_backend covins_backend_node` * In a shell on your **local machine** - * ```export ROS_IP=IP_of_your_!LOCAL!_machine``` - * ```export ROS_MASTER_URI=http://IP_of_the_!SERVER!:11311``` - * Launch RVIZ: ```rviz -d ~/ws/covins_ws/src/covins/covins_backend/config/covins.rviz``` (you need to have ```covins.rviz``` locally. + * `export ROS_IP=IP_of_your_!LOCAL!_machine` + * `export ROS_MASTER_URI=http://IP_of_the_!SERVER!:11311` + * Launch RVIZ: `rviz -d ~/ws/covins_ws/src/covins/covins_backend/config/covins.rviz` (you need to have `covins.rviz` locally. * Now you should be able to remotely visualize the RVIZ topics published from the COVINS backend running on the cloud server. ### Interfacing a Custom VIO System with COVINS -COVINS exports a generic communication interface, that can be integrated into custom keyframe-based VIO systems in order to share map data with the server back-end and generate a collaborative estimate. The code for the communication interface is located in the ```covins_comm``` folder, which builds a library with the same name that facilitates communication between the VIO system onboard the agent and the COVINS server back-end. +COVINS exports a generic communication interface, that can be integrated into custom keyframe-based VIO systems in order to share map data with the server back-end and generate a collaborative estimate. The code for the communication interface is located in the `covins_comm` folder, which builds a library with the same name that facilitates communication between the VIO system onboard the agent and the COVINS server back-end. -For straightforward understanding which steps need to be taken to interface a VIO front-end with COVINS, we have defined the preprocessor macro ```COVINS_MOD``` in ```covins/covins_comm/include/covins/covins_base/typedefs_base.hpp```. **This macro indicates all modifications made to the original ORB-SLAM3 code in order to set up the communication with the server back-end.** +For straightforward understanding which steps need to be taken to interface a VIO front-end with COVINS, we have defined the preprocessor macro `COVINS_MOD` in `covins/covins_comm/include/covins/covins_base/typedefs_base.hpp`. **This macro indicates all modifications made to the original ORB-SLAM3 code in order to set up the communication with the server back-end.** -In a nutshell, the communication interface provides a base communicator class, which is intended to be used to create a derived communicator class tailored to the VIO system. The communicator module runs in a separate thread, taking care of setting up a connection to the server, and exchanging map data. For the derived class, the user only needs to define a function that can be used to pass data to the communicator module and fill the provided data containers, and the ```Run()``` function that is continuously executed by the thread allocated to the communicator module. Furthermore, the communicator module uses the predefined message types ```MsgKeyframe``` and ```MsgLandmark``` for transmitting data to the server, therefore, the user needs to define functions that fill those messages from the custom data structures of the VIO system. +In a nutshell, the communication interface provides a base communicator class, which is intended to be used to create a derived communicator class tailored to the VIO system. The communicator module runs in a separate thread, taking care of setting up a connection to the server, and exchanging map data. For the derived class, the user only needs to define a function that can be used to pass data to the communicator module and fill the provided data containers, and the `Run()` function that is continuously executed by the thread allocated to the communicator module. Furthermore, the communicator module uses the predefined message types `MsgKeyframe` and `MsgLandmark` for transmitting data to the server, therefore, the user needs to define functions that fill those messages from the custom data structures of the VIO system. ### Map Re-Use Onboard the Agent COVINS also provides the functionality to share data from the collaborative estimate on the server-side with the agents participating in the estimate. COVINS provides only the infrastructure to share this data, the method for map re-use needs to be implement by the user. -By default, COVINS is configured to not send any data back to the agent. By setting ```comm.data_to_client``` to ```1``` in ```config_comm.yaml```, this functionality can be activated. By default, the server then regularly sends information about one keyframe back to the agent. The agent will display a message that requests the user to define what to do with the received information. +By default, COVINS is configured to not send any data back to the agent. By setting `comm.data_to_client` to `1` in `config_comm.yaml`, this functionality can be activated. By default, the server then regularly sends information about one keyframe back to the agent. The agent will display a message that requests the user to define what to do with the received information. -* In the function ```CollectDataForAgent()``` in ```covins_backend/src/covins_backend/```, the data to send to the agent can be specified. -* In the function ```ProcessKeyframeMessages()``` in ```orb_slam3/src/```, the processing of the received keyframe can be specified. +* In the function `CollectDataForAgent()` in `covins_backend/src/covins_backend/`, the data to send to the agent can be specified. +* In the function `ProcessKeyframeMessages()` in `orb_slam3/src/`, the processing of the received keyframe can be specified. ## 7 Limitations and Known Issues @@ -377,7 +423,7 @@ By default, COVINS is configured to not send any data back to the agent. By sett * [**TOGGLING VISUALIZATION**] When running multiple agents in parallel, and the maps are not merged yet, the visualization in RVIZ might toggle between the visualization of both trajectories. * [**MAP RESET**] ORB-SLAM3 has the functionality to **start a new map** when tracking is lost, in order to improve robustness. This functionality is **not supported** by COVINS. The COVINS server back-end assumes that keyframes arriving from a specific agent are shared in a continuous fashion and belong to the same map, and if the agent map is reset and a second keyframe with a previously used ID arrives again at the server side, the back-end will detect this inconsistency and throw an error. We have almost never experienced this behavior on the EuRoC sequences when using the ASL dataset format, and rarely when using rosbag files. * Too little computational resources available to the front-end can be a reason for more frequent map resets. - * Map resets are more frequent at the beginning of a dataset, and occur less when the VIO front-end is well initialized and already tracking the agent's pose over some time. Therefore, the communication module will only start sending data to the server once a pre-specified number of keyframes was created by the VIO front-end. This number is specified by ```comm.start_sending_after_kf``` in ```covins/covins_comm/config/config_comm.yaml```, and is currently set to 50. - * Particularly when running with rosbag files, setting the parameter ```orb.imu_stamp_max_diff: 2.0``` in ```covins/covins_comm/config/config_comm.yaml```, instead of the default (```1.0```), helped to significantly reduce map resets. We did not see any negative impact on the accuracy of the COVINS collaborative estimate from this change. + * Map resets are more frequent at the beginning of a dataset, and occur less when the VIO front-end is well initialized and already tracking the agent's pose over some time. Therefore, the communication module will only start sending data to the server once a pre-specified number of keyframes was created by the VIO front-end. This number is specified by `comm.start_sending_after_kf` in `covins/covins_comm/config/config_comm.yaml`, and is currently set to 50. + * Particularly when running with rosbag files, setting the parameter `orb.imu_stamp_max_diff: 2.0` in `covins/covins_comm/config/config_comm.yaml`, instead of the default (`1.0`), helped to significantly reduce map resets. We did not see any negative impact on the accuracy of the COVINS collaborative estimate from this change. * [**Duplicate Files**] The repository contains 2 copies of the ORB vocabularies, as well as 2 versions of the DBoW library. We decided to use this structure in order to keep the code of COVINS and ORB-SLAM3 as much separated as possible. * [**Mixed Notation**] COVINS mainly utilizes the *Google C++ Style Guide*. However, some modules re-use code of other open-source software using *Hungarian notation*, such as CCM-SLAM and ORB-SLAM2, and this code was not ported to the new notation convention yet (particularly, this applies to code parts related to the FeatureMatcher, KFDatabase, PlaceRecognition, Se3Solver).