diff --git a/calico-secure/v05/.DS_Store b/calico-secure/v05/.DS_Store new file mode 100644 index 0000000..69a4e4b Binary files /dev/null and b/calico-secure/v05/.DS_Store differ diff --git a/calico-secure/v05/1a-env.export.sh b/calico-secure/v05/1a-env.export.sh new file mode 100644 index 0000000..6cb53c3 --- /dev/null +++ b/calico-secure/v05/1a-env.export.sh @@ -0,0 +1,37 @@ +tee env.export <<-'EOF' +# This will be customer-specific, so it's at the top +# This export file is used to generate env files for all systemd units +export CALICO_CIDR=172.16.0.0/16 + +# 2379 and 2380 are within the DC/OS service port range, and are used by the etcd included with Calico. +export ETCD_LISTEN_PORT=62379 +export ETCD_TRANSPORT_PORT=62380 + +## Env variables +export MASTER_LIST_NOPORT=$(curl -sS master.mesos:8181/exhibitor/v1/cluster/status | python -c 'import sys,json;j=json.loads(sys.stdin.read());print(",".join([y["hostname"]+"=https://"+y["hostname"]+":ETCD_TRANSPORT_PORT" for y in j]))') +export MASTER_LIST=$(echo $MASTER_LIST_NOPORT | sed "s|ETCD_TRANSPORT_PORT|${ETCD_TRANSPORT_PORT}|g") +export MASTER_LIST_OPEN=$(echo $MASTER_LIST | sed "s|https|http|g") + +export ETCD_ROOT_DIR=/opt/etcd +export ETCD_DATA_DIR=/var/etcd/data +export ETCD_TLS_CERT=etcd.crt +export ETCD_TLS_KEY=etcd.key +export ETCD_CA_CERT=dcos-ca.crt +export LOCAL_HOSTNAME=$(/opt/mesosphere/bin/detect_ip) +export INITIAL_CLUSTER=${MASTER_LIST} +export INITIAL_CLUSTER_OPEN=${MASTER_LIST_OPEN} + +export CALICO_CNI_PLUGIN_DIR=/opt/calico/bin +export CALICO_CNI_CONF_DIR=/etc/calico/cni +export CALICO_CNI_CONF_FILE=dcos.calico.conf +export KUBERNETES_CNI_CONF_FILE=cni.conflist + +export CALICO_NODE_IMAGE=quay.io/calico/node:v2.6.9 + +export ETCD_CERTS_DIR=/etc/etcd/certs +export DOCKER_CLUSTER_CERTS_DIR=/etc/docker/cluster/certs +export CALICO_NODE_CERTS_DIR=/etc/calico/certs/node +export CALICO_CALICOCTL_CERTS_DIR=/etc/calico/certs/calicoctl +export CALICO_CNI_CERTS_DIR=/etc/calico/certs/cni + +EOF diff --git a/calico-secure/v05/1b-prereqs-certs.sh b/calico-secure/v05/1b-prereqs-certs.sh new file mode 100644 index 0000000..d599dd9 --- /dev/null +++ b/calico-secure/v05/1b-prereqs-certs.sh @@ -0,0 +1,61 @@ +source env.export + + +## Scriptlet used to generate certs using DC/OS CA +tee bootstrap-certs.py <<-'EOF' +#!/opt/mesosphere/bin/python + +import sys +sys.path.append('/opt/mesosphere/lib/python3.6/site-packages') + +from dcos_internal_utils import bootstrap + +if len(sys.argv) == 1: + print("Usage: ./bootstrap-certs.py | ./bootstrap-certs.py etcd /var/lib/dcos/etcd/certs") + sys.exit(1) + +b = bootstrap.Bootstrapper(bootstrap.parse_args()) +b.read_agent_secrets() + +cn = sys.argv[1] +location = sys.argv[2] + +keyfile = location + '/' + cn + '.key' +crtfile = location + '/' + cn + '.crt' + +b.ensure_key_certificate(cn, keyfile, crtfile, service_account='dcos_bootstrap_agent') +EOF +chmod +x bootstrap-certs.py + +## Etcd certs +sudo mkdir -p ${ETCD_CERTS_DIR} + +sudo ./bootstrap-certs.py etcd ${ETCD_CERTS_DIR} +sudo curl -kL https://master.mesos/ca/dcos-ca.crt -o ${ETCD_CERTS_DIR}/dcos-ca.crt + +## Docker certs +sudo mkdir -p ${DOCKER_CLUSTER_CERTS_DIR} + +sudo ./bootstrap-certs.py docker-etcd ${DOCKER_CLUSTER_CERTS_DIR} +sudo curl -kL http://master.mesos/ca/dcos-ca.crt -o ${DOCKER_CLUSTER_CERTS_DIR}/dcos-ca.crt + +## Calico Node certs +sudo mkdir -p ${CALICO_NODE_CERTS_DIR} + +sudo ./bootstrap-certs.py calico ${CALICO_NODE_CERTS_DIR} +sudo curl -kL http://master.mesos/ca/dcos-ca.crt -o ${CALICO_NODE_CERTS_DIR}/dcos-ca.crt + +## Calicoctl certs +sudo mkdir -p ${CALICO_CALICOCTL_CERTS_DIR} + +sudo ./bootstrap-certs.py calico ${CALICO_CALICOCTL_CERTS_DIR} +sudo curl -kL http://master.mesos/ca/dcos-ca.crt -o ${CALICO_CALICOCTL_CERTS_DIR}/dcos-ca.crt + +## CNI Certs +sudo mkdir -p ${CALICO_CNI_CERTS_DIR} + +sudo ./bootstrap-certs.py calico ${CALICO_CNI_CERTS_DIR} +sudo curl -kL https://master.mesos/ca/dcos-ca.crt -o ${CALICO_CNI_CERTS_DIR}/dcos-ca.crt + +## Other misc. directories +sudo mkdir -p ${ETCD_DATA_DIR} diff --git a/calico-secure/v05/1c-prereqs-systemd.sh b/calico-secure/v05/1c-prereqs-systemd.sh new file mode 100644 index 0000000..87d2bea --- /dev/null +++ b/calico-secure/v05/1c-prereqs-systemd.sh @@ -0,0 +1,293 @@ +source env.export + + +sudo mkdir -p /etc/etcd +sudo mkdir -p /etc/calico + + +#### etcd systemd environment file +sudo rm -f /etc/etcd/etcd.env +echo "ETCD_ROOT_DIR=${ETCD_ROOT_DIR}" | sudo tee -a /etc/etcd/etcd.env +echo "ETCD_DATA_DIR=${ETCD_DATA_DIR}" | sudo tee -a /etc/etcd/etcd.env +echo "ETCD_CERTS_DIR=${ETCD_CERTS_DIR}" | sudo tee -a /etc/etcd/etcd.env +echo "ETCD_TLS_CERT=${ETCD_TLS_CERT}" | sudo tee -a /etc/etcd/etcd.env +echo "ETCD_TLS_KEY=${ETCD_TLS_KEY}" | sudo tee -a /etc/etcd/etcd.env +echo "ETCD_CA_CERT=${ETCD_CA_CERT}" | sudo tee -a /etc/etcd/etcd.env +echo "LOCAL_HOSTNAME=${LOCAL_HOSTNAME}" | sudo tee -a /etc/etcd/etcd.env +echo "INITIAL_CLUSTER=${INITIAL_CLUSTER}" | sudo tee -a /etc/etcd/etcd.env +echo "INITIAL_CLUSTER_OPEN=${INITIAL_CLUSTER_OPEN}" | sudo tee -a /etc/etcd/etcd.env +echo "ETCD_LISTEN_PORT=${ETCD_LISTEN_PORT}" | sudo tee -a /etc/etcd/etcd.env +echo "ETCD_TRANSPORT_PORT=${ETCD_TRANSPORT_PORT}" | sudo tee -a /etc/etcd/etcd.env + +sed "s/^/export /g" /etc/etcd/etcd.env | sudo tee /etc/etcd/etcd.env.export + + +#### calico node systemd environment file +sudo tee /etc/calico/calico.env <<-'EOF' +ETCD_ENDPOINTS="https://localhost:ETCD_LISTEN_PORT" +ETCD_ENDPOINTS_OPEN="http://localhost:ETCD_LISTEN_PORT" +ETCD_CERT_DIR="ETCD_CERT_DIR_ENV" +ETCD_CONTAINER_CERT_DIR="/etc/certs" +ETCD_CA_CERT_FILE="dcos-ca.crt" +ETCD_CERT_FILE="calico.crt" +ETCD_KEY_FILE="calico.key" +CALICO_NODENAME="" +CALICO_NO_DEFAULT_POOLS="" +CALICO_IP="DETECT_IP_OUTPUT" +CALICO_IP6="" +CALICO_AS="" +CALICO_LIBNETWORK_ENABLED=true +CALICO_NETWORKING_BACKEND=bird +CALICO_DOCKER_IMAGE=CALICO_NODE_IMAGE +EOF + +sudo sed -i "s|ETCD_CERT_DIR_ENV|${CALICO_NODE_CERTS_DIR}|g" /etc/calico/calico.env +sudo sed -i "s|CALICO_NODE_IMAGE|${CALICO_NODE_IMAGE}|g" /etc/calico/calico.env +sudo sed -i "s|ETCD_LISTEN_PORT|${ETCD_LISTEN_PORT}|g" /etc/calico/calico.env +sudo sed -i "s/DETECT_IP_OUTPUT/$(/opt/mesosphere/bin/detect_ip)/g" /etc/calico/calico.env + +sed "s/^/export /g" /etc/calico/calico.env | sudo tee /etc/calico/calico.env.export + + +#### etcd systemd unit file (for masters) +sudo tee /etc/etcd/dcos-etcd.service <<-'EOF' +[Unit] +Description=etcd +Documentation=https://github.com/coreos/etcd +Conflicts=etcd.service +Conflicts=etcd2.service + +[Service] +Type=notify +Restart=always +RestartSec=5s +LimitNOFILE=40000 +TimeoutStartSec=0 + +EnvironmentFile=/etc/etcd/etcd.env + +# Listen on 0.0.0.0, advertise on IP address +ExecStart=/opt/etcd/etcd --name ${LOCAL_HOSTNAME} \ + --data-dir ${ETCD_DATA_DIR} \ + --listen-client-urls https://0.0.0.0:${ETCD_LISTEN_PORT} \ + --advertise-client-urls https://${LOCAL_HOSTNAME}:${ETCD_LISTEN_PORT} \ + --listen-peer-urls https://0.0.0.0:${ETCD_TRANSPORT_PORT} \ + --initial-advertise-peer-urls https://${LOCAL_HOSTNAME}:${ETCD_TRANSPORT_PORT} \ + --initial-cluster ${INITIAL_CLUSTER} \ + --initial-cluster-token tkn \ + --initial-cluster-state new \ + --client-cert-auth \ + --trusted-ca-file ${ETCD_CERTS_DIR}/${ETCD_CA_CERT} \ + --cert-file ${ETCD_CERTS_DIR}/${ETCD_TLS_CERT} \ + --key-file ${ETCD_CERTS_DIR}/${ETCD_TLS_KEY} \ + --peer-client-cert-auth \ + --peer-trusted-ca-file ${ETCD_CERTS_DIR}/${ETCD_CA_CERT} \ + --peer-cert-file ${ETCD_CERTS_DIR}/${ETCD_TLS_CERT} \ + --peer-key-file ${ETCD_CERTS_DIR}/${ETCD_TLS_KEY} + +[Install] +WantedBy=multi-user.target +EOF + +#### etcd systemd unit file (for masters) +sudo tee /etc/etcd/dcos-etcd-open.service <<-'EOF' +[Unit] +Description=etcd +Documentation=https://github.com/coreos/etcd +Conflicts=etcd.service +Conflicts=etcd2.service + +[Service] +Type=notify +Restart=always +RestartSec=5s +LimitNOFILE=40000 +TimeoutStartSec=0 + +EnvironmentFile=/etc/etcd/etcd.env + +# Listen on 0.0.0.0, advertise on IP address +ExecStart=/opt/etcd/etcd --name ${LOCAL_HOSTNAME} \ + --data-dir ${ETCD_DATA_DIR} \ + --listen-client-urls http://0.0.0.0:${ETCD_LISTEN_PORT} \ + --advertise-client-urls http://${LOCAL_HOSTNAME}:${ETCD_LISTEN_PORT} \ + --listen-peer-urls http://0.0.0.0:${ETCD_TRANSPORT_PORT} \ + --initial-advertise-peer-urls http://${LOCAL_HOSTNAME}:${ETCD_TRANSPORT_PORT} \ + --initial-cluster ${INITIAL_CLUSTER_OPEN} \ + --initial-cluster-token tkn \ + --initial-cluster-state new + +[Install] +WantedBy=multi-user.target +EOF + +#### etcd-proxy systemd unit file (for slaves) +sudo tee /etc/etcd/dcos-etcd-proxy.service <<-'EOF' +[Unit] +Description=etcd-proxy +Documentation=https://github.com/coreos/etcd +Conflicts=etcd.service +Conflicts=etcd2.service + +[Service] +Type=notify +Restart=always +RestartSec=5s +LimitNOFILE=40000 +TimeoutStartSec=0 + +EnvironmentFile=/etc/etcd/etcd.env + +# Listen on 0.0.0.0, advertise on IP address +ExecStart=/opt/etcd/etcd --proxy on \ + --data-dir ${ETCD_DATA_DIR} \ + --listen-client-urls https://0.0.0.0:${ETCD_LISTEN_PORT} \ + --key-file ${ETCD_CERTS_DIR}/${ETCD_TLS_KEY} \ + --cert-file ${ETCD_CERTS_DIR}/${ETCD_TLS_CERT} \ + --peer-key-file ${ETCD_CERTS_DIR}/${ETCD_TLS_KEY} \ + --peer-cert-file ${ETCD_CERTS_DIR}/${ETCD_TLS_CERT} \ + --trusted-ca-file ${ETCD_CERTS_DIR}/${ETCD_CA_CERT} \ + --peer-trusted-ca-file ${ETCD_CERTS_DIR}/${ETCD_CA_CERT} \ + --client-cert-auth \ + --peer-client-cert-auth \ + --initial-cluster ${INITIAL_CLUSTER} + +[Install] +WantedBy=multi-user.target +EOF + +#### etcd-proxy systemd unit file (for slaves) +sudo tee /etc/etcd/dcos-etcd-proxy-open.service <<-'EOF' +[Unit] +Description=etcd-proxy +Documentation=https://github.com/coreos/etcd +Conflicts=etcd.service +Conflicts=etcd2.service + +[Service] +Type=notify +Restart=always +RestartSec=5s +LimitNOFILE=40000 +TimeoutStartSec=0 + +EnvironmentFile=/etc/etcd/etcd.env + +# Listen on 0.0.0.0, advertise on IP address +ExecStart=/opt/etcd/etcd --proxy on \ + --data-dir ${ETCD_DATA_DIR} \ + --listen-client-urls http://0.0.0.0:${ETCD_LISTEN_PORT} \ + --initial-cluster ${INITIAL_CLUSTER_OPEN} + +[Install] +WantedBy=multi-user.target +EOF + + +#### calico node (Docker container) systemd unit file +sudo tee /etc/calico/dcos-calico-node.service <<-'EOF' +[Unit] +Description=calico-node +After=docker.service +Requires=docker.service + +[Service] +EnvironmentFile=/etc/calico/calico.env +ExecStartPre=-/usr/bin/docker rm -f calico-node +ExecStart=/usr/bin/docker run --net=host --privileged \ + --name=calico-node \ + -e NODENAME=${CALICO_NODENAME} \ + -e IP=${CALICO_IP} \ + -e IP6=${CALICO_IP6} \ + -e CALICO_NETWORKING_BACKEND=${CALICO_NETWORKING_BACKEND} \ + -e AS=${CALICO_AS} \ + -e NO_DEFAULT_POOLS=${CALICO_NO_DEFAULT_POOLS} \ + -e CALICO_LIBNETWORK_ENABLED=${CALICO_LIBNETWORK_ENABLED} \ + -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \ + -e ETCD_CA_CERT_FILE=${ETCD_CONTAINER_CERT_DIR}/${ETCD_CA_CERT_FILE} \ + -e ETCD_CERT_FILE=${ETCD_CONTAINER_CERT_DIR}/${ETCD_CERT_FILE} \ + -e ETCD_KEY_FILE=${ETCD_CONTAINER_CERT_DIR}/${ETCD_KEY_FILE} \ + -e FELIX_IGNORELOOSERPF=true \ + -v ${ETCD_CERT_DIR}:${ETCD_CONTAINER_CERT_DIR} \ + -v /var/log/calico:/var/log/calico \ + -v /run/docker/plugins:/run/docker/plugins \ + -v /lib/modules:/lib/modules \ + -v /var/run/calico:/var/run/calico \ + -v /var/run/docker.sock:/var/run/docker.sock \ + ${CALICO_DOCKER_IMAGE} + +# Need FELIX_IGNORELOOSERPF for DC/OS, see https://github.com/projectcalico/calicoctl/issues/1082 +# Need /var/run/docker.sock to connect to host Docker socket from within container + +ExecStop=-/usr/bin/docker stop calico-node + +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target +EOF + +#### calico node (Docker container) systemd unit file +sudo tee /etc/calico/dcos-calico-node-open.service <<-'EOF' +[Unit] +Description=calico-node +After=docker.service +Requires=docker.service + +[Service] +EnvironmentFile=/etc/calico/calico.env +ExecStartPre=-/usr/bin/docker rm -f calico-node +ExecStart=/usr/bin/docker run --net=host --privileged \ + --name=calico-node \ + -e NODENAME=${CALICO_NODENAME} \ + -e IP=${CALICO_IP} \ + -e IP6=${CALICO_IP6} \ + -e CALICO_NETWORKING_BACKEND=${CALICO_NETWORKING_BACKEND} \ + -e AS=${CALICO_AS} \ + -e NO_DEFAULT_POOLS=${CALICO_NO_DEFAULT_POOLS} \ + -e CALICO_LIBNETWORK_ENABLED=${CALICO_LIBNETWORK_ENABLED} \ + -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS_OPEN} \ + -e FELIX_IGNORELOOSERPF=true \ + -v /var/log/calico:/var/log/calico \ + -v /run/docker/plugins:/run/docker/plugins \ + -v /lib/modules:/lib/modules \ + -v /var/run/calico:/var/run/calico \ + -v /var/run/docker.sock:/var/run/docker.sock \ + ${CALICO_DOCKER_IMAGE} + +# Need FELIX_IGNORELOOSERPF for DC/OS, see https://github.com/projectcalico/calicoctl/issues/1082 +# Need /var/run/docker.sock to connect to host Docker socket from within container + +ExecStop=-/usr/bin/docker stop calico-node + +Restart=always +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target +EOF + + +#### calico node (Docker container) systemd timer file +sudo tee /etc/calico/dcos-calico-node.timer <<-'EOF' +[Unit] +Description=Ensure Calico Node is running + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +EOF + + +#### calico node (Docker container) systemd timer file +sudo tee /etc/calico/dcos-calico-node-open.timer <<-'EOF' +[Unit] +Description=Ensure Calico Node is running + +[Timer] +OnBootSec=1min +OnUnitActiveSec=1min +EOF diff --git a/calico-secure/v05/1d-prereqs-conf.sh b/calico-secure/v05/1d-prereqs-conf.sh new file mode 100644 index 0000000..2196445 --- /dev/null +++ b/calico-secure/v05/1d-prereqs-conf.sh @@ -0,0 +1,75 @@ +source env.export + + +sudo mkdir -p ${CALICO_CNI_PLUGIN_DIR} +sudo mkdir -p ${CALICO_CNI_CONF_DIR} + + +#### Docker daemon config; Specifies cluster-store and storage driver. If storage-driver is configured in docker systemd unit, it must be removed from here. +sudo tee /etc/docker/daemon.json <<-'EOF' +{ + "storage-driver": "overlay", + "cluster-store": "etcd://127.0.0.1:ETCD_LISTEN_PORT", + "cluster-store-opts": { + "kv.cacertfile": "DOCKER_CLUSTER_CERTS_DIR/dcos-ca.crt", + "kv.certfile": "DOCKER_CLUSTER_CERTS_DIR/docker-etcd.crt", + "kv.keyfile": "DOCKER_CLUSTER_CERTS_DIR/docker-etcd.key" + } +} +EOF +sudo sed -i "s|DOCKER_CLUSTER_CERTS_DIR|${DOCKER_CLUSTER_CERTS_DIR}|g" /etc/docker/daemon.json +sudo sed -i "s|ETCD_LISTEN_PORT|${ETCD_LISTEN_PORT}|g" /etc/docker/daemon.json + + +#### Mesos CNI Config +sudo tee ${CALICO_CNI_CONF_DIR}/${CALICO_CNI_CONF_FILE} <<-'EOF' +{ + "name": "calico", + "cniVersion": "0.1.0", + "type": "calico", + "ipam": { + "type": "calico-ipam" + }, + "etcd_endpoints": "https://127.0.0.1:ETCD_LISTEN_PORT", + "etcd_ca_cert_file": "CALICO_CNI_CERTS_DIR/dcos-ca.crt", + "etcd_key_file": "CALICO_CNI_CERTS_DIR/calico.key", + "etcd_cert_file": "CALICO_CNI_CERTS_DIR/calico.crt" +} +EOF +sudo sed -i "s|CALICO_CNI_CERTS_DIR|${CALICO_CNI_CERTS_DIR}|g" ${CALICO_CNI_CONF_DIR}/${CALICO_CNI_CONF_FILE} +sudo sed -i "s|ETCD_LISTEN_PORT|${ETCD_LISTEN_PORT}|g" ${CALICO_CNI_CONF_DIR}/${CALICO_CNI_CONF_FILE} + + +#### calicoctl config (config for command line tool) +sudo tee /etc/calico/calicoctl.cfg <<-'EOF' +apiVersion: v1 +kind: calicoApiConfig +metadata: +spec: + etcdEndpoints: https://127.0.0.1:ETCD_LISTEN_PORT + etcdKeyFile: CALICO_CALICOCTL_CERTS_DIR/calico.key + etcdCertFile: CALICO_CALICOCTL_CERTS_DIR/calico.crt + etcdCACertFile: CALICO_CALICOCTL_CERTS_DIR/dcos-ca.crt +EOF +sudo sed -i "s|CALICO_CALICOCTL_CERTS_DIR|${CALICO_CALICOCTL_CERTS_DIR}|g" /etc/calico/calicoctl.cfg +sudo sed -i "s|ETCD_LISTEN_PORT|${ETCD_LISTEN_PORT}|g" /etc/calico/calicoctl.cfg + + +#### Calico Pool Config (realistically, this is only used once on one node, but it's good to have for reference purposes) +sudo tee /etc/calico/ippool.json <<-'EOF' + { + "kind": "ipPool", + "apiVersion": "v1", + "metadata": { + "cidr": "CALICO_CIDR" + }, + "spec": { + "nat-outgoing": true, + "ipip": { + "enabled": true, + "mode": "cross-subnet" + } + } + } +EOF +sudo sed -i "s|CALICO_CIDR|${CALICO_CIDR}|g" /etc/calico/ippool.json diff --git a/calico-secure/v05/1do-prereqs-conf.sh b/calico-secure/v05/1do-prereqs-conf.sh new file mode 100644 index 0000000..b65bb4b --- /dev/null +++ b/calico-secure/v05/1do-prereqs-conf.sh @@ -0,0 +1,64 @@ +source env.export + + +sudo mkdir -p ${CALICO_CNI_PLUGIN_DIR} +sudo mkdir -p ${CALICO_CNI_CONF_DIR} + + +#### Docker daemon config; Specifies cluster-store and storage driver. If storage-driver is configured in docker systemd unit, it must be removed from here. +sudo tee /etc/docker/daemon.json <<-'EOF' +{ + "storage-driver": "overlay", + "cluster-store": "etcd://127.0.0.1:ETCD_LISTEN_PORT" +} +EOF +sudo sed -i "s|DOCKER_CLUSTER_CERTS_DIR|${DOCKER_CLUSTER_CERTS_DIR}|g" /etc/docker/daemon.json +sudo sed -i "s|ETCD_LISTEN_PORT|${ETCD_LISTEN_PORT}|g" /etc/docker/daemon.json + + +#### Mesos CNI Config +sudo tee ${CALICO_CNI_CONF_DIR}/${CALICO_CNI_CONF_FILE} <<-'EOF' +{ + "name": "calico", + "cniVersion": "0.1.0", + "type": "calico", + "ipam": { + "type": "calico-ipam" + }, + "etcd_endpoints": "http://127.0.0.1:ETCD_LISTEN_PORT" +} +EOF +sudo sed -i "s|CALICO_CNI_CERTS_DIR|${CALICO_CNI_CERTS_DIR}|g" ${CALICO_CNI_CONF_DIR}/${CALICO_CNI_CONF_FILE} +sudo sed -i "s|ETCD_LISTEN_PORT|${ETCD_LISTEN_PORT}|g" ${CALICO_CNI_CONF_DIR}/${CALICO_CNI_CONF_FILE} + + +#### calicoctl config (config for command line tool) +sudo tee /etc/calico/calicoctl.cfg <<-'EOF' +apiVersion: v1 +kind: calicoApiConfig +metadata: +spec: + etcdEndpoints: http://127.0.0.1:ETCD_LISTEN_PORT +EOF +sudo sed -i "s|CALICO_CALICOCTL_CERTS_DIR|${CALICO_CALICOCTL_CERTS_DIR}|g" /etc/calico/calicoctl.cfg +sudo sed -i "s|ETCD_LISTEN_PORT|${ETCD_LISTEN_PORT}|g" /etc/calico/calicoctl.cfg + + +#### Calico Pool Config (realistically, this is only used once on one node, but it's good to have for reference purposes) +sudo tee /etc/calico/ippool.json <<-'EOF' + { + "kind": "ipPool", + "apiVersion": "v1", + "metadata": { + "cidr": "CALICO_CIDR" + }, + "spec": { + "nat-outgoing": true, + "ipip": { + "enabled": true, + "mode": "cross-subnet" + } + } + } +EOF +sudo sed -i "s|CALICO_CIDR|${CALICO_CIDR}|g" /etc/calico/ippool.json diff --git a/calico-secure/v05/2-package-install.sh b/calico-secure/v05/2-package-install.sh new file mode 100644 index 0000000..d946ebd --- /dev/null +++ b/calico-secure/v05/2-package-install.sh @@ -0,0 +1,26 @@ +source env.export + + +sudo mkdir -p ${ETCD_ROOT_DIR} +sudo mkdir -p ${ETCD_DATA_DIR} + + +## Download and extract etcd package +curl -LO https://github.com/coreos/etcd/releases/download/v3.3.5/etcd-v3.3.5-linux-amd64.tar.gz +sudo tar -xzvf etcd-v3.3.5-linux-amd64.tar.gz -C ${ETCD_ROOT_DIR} --strip-components=1 + + +## Download and install calico and calico-ipam CNI plugin binaries +sudo curl -L https://github.com/projectcalico/cni-plugin/releases/download/v1.11.5/calico -o ${CALICO_CNI_PLUGIN_DIR}/calico +sudo curl -L https://github.com/projectcalico/cni-plugin/releases/download/v1.11.5/calico-ipam -o ${CALICO_CNI_PLUGIN_DIR}/calico-ipam +sudo chmod +x ${CALICO_CNI_PLUGIN_DIR}/calico +sudo chmod +x ${CALICO_CNI_PLUGIN_DIR}/calico-ipam + + +## Download calicoctl +sudo curl -L https://github.com/projectcalico/calicoctl/releases/download/v1.6.4/calicoctl -o /usr/bin/calicoctl +sudo chmod +x /usr/bin/calicoctl + + +## Download Docker image for Calico node +sudo docker pull ${CALICO_NODE_IMAGE} diff --git a/calico-secure/v05/3-enable-masters.sh b/calico-secure/v05/3-enable-masters.sh new file mode 100644 index 0000000..991a4b0 --- /dev/null +++ b/calico-secure/v05/3-enable-masters.sh @@ -0,0 +1,46 @@ +source env.export +# Unused, but specified for consistency + +####### etcd +# Install, enable, and start etcd systemd unit (will hang until it's run on all masters) +sudo cp /etc/etcd/dcos-etcd.service /etc/systemd/system/dcos-etcd.service + +sudo systemctl daemon-reload +sudo systemctl enable dcos-etcd.service +sudo systemctl restart dcos-etcd.service + +# Validate it's running (etcd must be running on all masters prior to this working) +sudo ETCDCTL_API=2 /opt/etcd/etcdctl \ + --endpoints https://localhost:${ETCD_LISTEN_PORT} \ + --key-file /etc/etcd/certs/etcd.key \ + --cert-file /etc/etcd/certs/etcd.crt \ + --ca-file /etc/etcd/certs/dcos-ca.crt \ + cluster-health + + +####### Docker cluster store +# Get docker to pick up the new config +# !!! If this fails, you may have to remove the 'overlay' line from /etc/docker/daemon.json - it doesn't like redundant configurations +if [[ $(systemctl cat docker | grep 'storage-driver=overlay' | wc -l) -eq 1 ]]; then + sudo sed -i "/storage-driver/d" /etc/docker/daemon.json +fi +sudo systemctl restart docker + +# Validate +sudo docker info | grep -i cluster + + +####### Calico node (used so that masters can reach container IPs) +sudo cp /etc/calico/dcos-calico-node.service /etc/systemd/system/dcos-calico-node.service +sudo cp /etc/calico/dcos-calico-node.timer /etc/systemd/system/dcos-calico-node.timer + +sudo systemctl daemon-reload +sudo systemctl enable dcos-calico-node.service +sudo systemctl restart dcos-calico-node.service + +sudo systemctl enable dcos-calico-node.timer +sudo systemctl restart dcos-calico-node.timer + +# Check status +sleep 5 +sudo calicoctl node status diff --git a/calico-secure/v05/3o-enable-masters.sh b/calico-secure/v05/3o-enable-masters.sh new file mode 100644 index 0000000..dcef1f8 --- /dev/null +++ b/calico-secure/v05/3o-enable-masters.sh @@ -0,0 +1,43 @@ +source env.export +# Unused, but specified for consistency + +####### etcd +# Install, enable, and start etcd systemd unit (will hang until it's run on all masters) +sudo cp /etc/etcd/dcos-etcd-open.service /etc/systemd/system/dcos-etcd.service + +sudo systemctl daemon-reload +sudo systemctl enable dcos-etcd.service +sudo systemctl restart dcos-etcd.service + +# Validate it's running (etcd must be running on all masters prior to this working) +sudo ETCDCTL_API=2 /opt/etcd/etcdctl \ + --endpoints http://localhost:${ETCD_LISTEN_PORT} \ + cluster-health + + +####### Docker cluster store +# Get docker to pick up the new config +# !!! If this fails, you may have to remove the 'overlay' line from /etc/docker/daemon.json - it doesn't like redundant configurations +if [[ $(systemctl cat docker | grep 'storage-driver=overlay' | wc -l) -eq 1 ]]; then + sudo sed -i "/storage-driver/d" /etc/docker/daemon.json +fi +sudo systemctl restart docker + +# Validate +sudo docker info | grep -i cluster + + +####### Calico node (used so that masters can reach container IPs) +sudo cp /etc/calico/dcos-calico-node-open.service /etc/systemd/system/dcos-calico-node.service +sudo cp /etc/calico/dcos-calico-node-open.timer /etc/systemd/system/dcos-calico-node.timer + +sudo systemctl daemon-reload +sudo systemctl enable dcos-calico-node.service +sudo systemctl restart dcos-calico-node.service + +sudo systemctl enable dcos-calico-node.timer +sudo systemctl restart dcos-calico-node.timer + +# Check status +sleep 5 +sudo calicoctl node status diff --git a/calico-secure/v05/4-set-up-pool.sh b/calico-secure/v05/4-set-up-pool.sh new file mode 100644 index 0000000..520ad31 --- /dev/null +++ b/calico-secure/v05/4-set-up-pool.sh @@ -0,0 +1,12 @@ +source env.export +# Unused, but specified for consistency + +# This should only be run on one master +sudo calicoctl get ipps -o json | sudo tee /etc/calico/ippool-backup.json +sudo calicoctl delete ipps 192.168.0.0/16 +sudo calicoctl apply -f /etc/calico/ippool.json + +sudo calicoctl get ipps -o json + +### Set up Docker network +docker network create --driver calico --ipam-driver calico-ipam calico diff --git a/calico-secure/v05/5-enable-agents.sh b/calico-secure/v05/5-enable-agents.sh new file mode 100644 index 0000000..ea2b90c --- /dev/null +++ b/calico-secure/v05/5-enable-agents.sh @@ -0,0 +1,82 @@ +source env.export + + +####### etcd +# Install, enable, and start etcd systemd unit (can be run independently on all agents) +sudo cp /etc/etcd/dcos-etcd-proxy.service /etc/systemd/system/dcos-etcd-proxy.service + +sudo systemctl daemon-reload +sudo systemctl enable dcos-etcd-proxy.service +sudo systemctl restart dcos-etcd-proxy.service + +# Validate it's running (etcd must be running on all masters prior to this working) +sudo ETCDCTL_API=2 /opt/etcd/etcdctl \ + --endpoints https://localhost:${ETCD_LISTEN_PORT} \ + --key-file /etc/etcd/certs/etcd.key \ + --cert-file /etc/etcd/certs/etcd.crt \ + --ca-file /etc/etcd/certs/dcos-ca.crt \ + cluster-health + + +####### Docker cluster store +# Get docker to pick up the new config +# !!! If this fails, you may have to remove the 'overlay' line from /etc/docker/daemon.json - it doesn't like redundant configurations +if [[ $(systemctl cat docker | grep 'storage-driver=overlay' | wc -l) -eq 1 ]]; then + sudo sed -i "/storage-driver/d" /etc/docker/daemon.json +fi +sudo systemctl restart docker + +# Validate +sudo docker info | grep -i cluster + + +####### Calico node +sudo cp /etc/calico/dcos-calico-node.service /etc/systemd/system/dcos-calico-node.service +sudo cp /etc/calico/dcos-calico-node.timer /etc/systemd/system/dcos-calico-node.timer + +sudo systemctl daemon-reload +sudo systemctl enable dcos-calico-node.service +sudo systemctl restart dcos-calico-node.service + +sudo systemctl enable dcos-calico-node.timer +sudo systemctl restart dcos-calico-node.timer + +# Check status +sleep 5 +sudo calicoctl node status + + +####### Set up CNI +## Plugin dir +# Copy setting from /opt/mesosphere/etc/mesos-slave-common to /var/lib/dcos/mesos-slave-common +# Append custom CALICO_CNI_PLUGIN_DIR +# Running this multiple times will create duplicate and ugly but not harmful settings +grep MESOS_NETWORK_CNI_PLUGINS_DIR /opt/mesosphere/etc/mesos-slave-common | sudo tee -a /var/lib/dcos/mesos-slave-common +sudo sed -i '/MESOS_NETWORK_CNI_PLUGINS_DIR/s|$|:CALICO_CNI_PLUGIN_DIR|g' /var/lib/dcos/mesos-slave-common +sudo sed -i "s|CALICO_CNI_PLUGIN_DIR|${CALICO_CNI_PLUGIN_DIR}|g" /var/lib/dcos/mesos-slave-common + +## Plugin conf +sudo mkdir -p /etc/systemd/system/dcos-mesos-slave.service.d +# We can do both dcos-mesos-slave and dcos-mesos-slave-common on all nodes, safely; only the relevant one will be used by the corresponding systemd unit +# We use a systemd override to copy the conf from custom location into default MESOS_NETWORK_CNI_CONFIG_DIR +sudo tee /etc/systemd/system/dcos-mesos-slave.service.d/dcos-calico-conf-override.conf <<-'EOF' +[Service] +ExecStartPre=/bin/cp CALICO_CNI_CONF_DIR/CALICO_CNI_CONF_FILE /opt/mesosphere/etc/dcos/network/cni/ +EOF + +sudo sed -i "s|CALICO_CNI_CONF_DIR|${CALICO_CNI_CONF_DIR}|g" /etc/systemd/system/dcos-mesos-slave.service.d/dcos-calico-conf-override.conf +sudo sed -i "s|CALICO_CNI_CONF_FILE|${CALICO_CNI_CONF_FILE}|g" /etc/systemd/system/dcos-mesos-slave.service.d/dcos-calico-conf-override.conf + +sudo mkdir -p /etc/systemd/system/dcos-mesos-slave-public.service.d +# /etc/systemd/system/dcos-mesos-slave.service.d/override.conf +sudo tee /etc/systemd/system/dcos-mesos-slave-public.service.d/dcos-calico-conf-override.conf <<-'EOF' +[Service] +ExecStartPre=/bin/cp CALICO_CNI_CONF_DIR/CALICO_CNI_CONF_FILE /opt/mesosphere/etc/dcos/network/cni/ +EOF + +sudo sed -i "s|CALICO_CNI_CONF_DIR|${CALICO_CNI_CONF_DIR}|g" /etc/systemd/system/dcos-mesos-slave-public.service.d/dcos-calico-conf-override.conf +sudo sed -i "s|CALICO_CNI_CONF_FILE|${CALICO_CNI_CONF_FILE}|g" /etc/systemd/system/dcos-mesos-slave-public.service.d/dcos-calico-conf-override.conf + +## Restart +sudo systemctl daemon-reload +sudo systemctl restart dcos-mesos-slave* diff --git a/calico-secure/v05/5o-enable-agents.sh b/calico-secure/v05/5o-enable-agents.sh new file mode 100644 index 0000000..f7e65ca --- /dev/null +++ b/calico-secure/v05/5o-enable-agents.sh @@ -0,0 +1,82 @@ +source env.export + + +####### etcd +# Install, enable, and start etcd systemd unit (can be run independently on all agents) +sudo cp /etc/etcd/dcos-etcd-proxy-open.service /etc/systemd/system/dcos-etcd-proxy.service + +sudo systemctl daemon-reload +sudo systemctl enable dcos-etcd-proxy.service +sudo systemctl restart dcos-etcd-proxy.service + +# Validate it's running (etcd must be running on all masters prior to this working) +sudo ETCDCTL_API=2 /opt/etcd/etcdctl \ + --endpoints http://localhost:${ETCD_LISTEN_PORT} \ + cluster-health + + + + + +####### Docker cluster store +# Get docker to pick up the new config +# !!! If this fails, you may have to remove the 'overlay' line from /etc/docker/daemon.json - it doesn't like redundant configurations +if [[ $(systemctl cat docker | grep 'storage-driver=overlay' | wc -l) -eq 1 ]]; then + sudo sed -i "/storage-driver/d" /etc/docker/daemon.json +fi +sudo systemctl restart docker + +# Validate +sudo docker info | grep -i cluster + + +####### Calico node +sudo cp /etc/calico/dcos-calico-node-open.service /etc/systemd/system/dcos-calico-node.service +sudo cp /etc/calico/dcos-calico-node-open.timer /etc/systemd/system/dcos-calico-node.timer + +sudo systemctl daemon-reload +sudo systemctl enable dcos-calico-node.service +sudo systemctl restart dcos-calico-node.service + +sudo systemctl enable dcos-calico-node.timer +sudo systemctl restart dcos-calico-node.timer + +# Check status +sleep 5 +sudo calicoctl node status + + +####### Set up CNI +## Plugin dir +# Copy setting from /opt/mesosphere/etc/mesos-slave-common to /var/lib/dcos/mesos-slave-common +# Append custom CALICO_CNI_PLUGIN_DIR +# Running this multiple times will create duplicate and ugly but not harmful settings +grep MESOS_NETWORK_CNI_PLUGINS_DIR /opt/mesosphere/etc/mesos-slave-common | sudo tee -a /var/lib/dcos/mesos-slave-common +sudo sed -i '/MESOS_NETWORK_CNI_PLUGINS_DIR/s|$|:CALICO_CNI_PLUGIN_DIR|g' /var/lib/dcos/mesos-slave-common +sudo sed -i "s|CALICO_CNI_PLUGIN_DIR|${CALICO_CNI_PLUGIN_DIR}|g" /var/lib/dcos/mesos-slave-common + +## Plugin conf +sudo mkdir -p /etc/systemd/system/dcos-mesos-slave.service.d +# We can do both dcos-mesos-slave and dcos-mesos-slave-common on all nodes, safely; only the relevant one will be used by the corresponding systemd unit +# We use a systemd override to copy the conf from custom location into default MESOS_NETWORK_CNI_CONFIG_DIR +sudo tee /etc/systemd/system/dcos-mesos-slave.service.d/dcos-calico-conf-override.conf <<-'EOF' +[Service] +ExecStartPre=/bin/cp CALICO_CNI_CONF_DIR/CALICO_CNI_CONF_FILE /opt/mesosphere/etc/dcos/network/cni/ +EOF + +sudo sed -i "s|CALICO_CNI_CONF_DIR|${CALICO_CNI_CONF_DIR}|g" /etc/systemd/system/dcos-mesos-slave.service.d/dcos-calico-conf-override.conf +sudo sed -i "s|CALICO_CNI_CONF_FILE|${CALICO_CNI_CONF_FILE}|g" /etc/systemd/system/dcos-mesos-slave.service.d/dcos-calico-conf-override.conf + +sudo mkdir -p /etc/systemd/system/dcos-mesos-slave-public.service.d +# /etc/systemd/system/dcos-mesos-slave.service.d/override.conf +sudo tee /etc/systemd/system/dcos-mesos-slave-public.service.d/dcos-calico-conf-override.conf <<-'EOF' +[Service] +ExecStartPre=/bin/cp CALICO_CNI_CONF_DIR/CALICO_CNI_CONF_FILE /opt/mesosphere/etc/dcos/network/cni/ +EOF + +sudo sed -i "s|CALICO_CNI_CONF_DIR|${CALICO_CNI_CONF_DIR}|g" /etc/systemd/system/dcos-mesos-slave-public.service.d/dcos-calico-conf-override.conf +sudo sed -i "s|CALICO_CNI_CONF_FILE|${CALICO_CNI_CONF_FILE}|g" /etc/systemd/system/dcos-mesos-slave-public.service.d/dcos-calico-conf-override.conf + +## Restart +sudo systemctl daemon-reload +sudo systemctl restart dcos-mesos-slave* diff --git a/calico-secure/v05/6-kubernetes.sh b/calico-secure/v05/6-kubernetes.sh new file mode 100644 index 0000000..aa2a5d0 --- /dev/null +++ b/calico-secure/v05/6-kubernetes.sh @@ -0,0 +1,100 @@ +source env.export + + +## Scriptlet used to generate certs using DC/OS CA +tee bootstrap-certs.py <<-'EOF' +#!/opt/mesosphere/bin/python + +import sys +sys.path.append('/opt/mesosphere/lib/python3.6/site-packages') + +from dcos_internal_utils import bootstrap + +if len(sys.argv) == 1: + print("Usage: ./bootstrap-certs.py | ./bootstrap-certs.py etcd /var/lib/dcos/etcd/certs") + sys.exit(1) + +b = bootstrap.Bootstrapper(bootstrap.parse_args()) +b.read_agent_secrets() + +cn = sys.argv[1] +location = sys.argv[2] + +keyfile = location + '/' + cn + '.key' +crtfile = location + '/' + cn + '.crt' + +b.ensure_key_certificate(cn, keyfile, crtfile, service_account='dcos_bootstrap_agent') +EOF +chmod +x bootstrap-certs.py + +sudo mkdir -p /etc/calico/certs/kubernetes +sudo /home/centos/bootstrap-certs.py kubernetes-etcd /etc/calico/certs/kubernetes +sudo /home/centos/bootstrap-certs.py kubernetes-client /etc/calico/certs/kubernetes +sudo curl -Lk https://master.mesos/ca/dcos-ca.crt -o /etc/calico/certs/kubernetes/dcos-ca.crt + + +sudo tee ${CALICO_CNI_CONF_DIR}/${KUBERNETES_CNI_CONF_FILE} <<-'EOF' + { + "cniVersion": "0.3.0", + "name": "kube-cni", + "plugins": [ + { + "type": "calico", + "etcd_endpoints": "https://localhost:62379", + "etcd_ca_cert_file": "/etc/calico/certs/kubernetes/dcos-ca.crt", + "etcd_key_file": "/etc/calico/certs/kubernetes/kubernetes-etcd.key", + "etcd_cert_file": "/etc/calico/certs/kubernetes/kubernetes-etcd.crt", + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s", + "k8s_api_root": "https://apiserver.kubernetes.l4lb.thisdcos.directory:6443", + "k8s_client_certificate": "/etc/calico/certs/kubernetes/kubernetes-client.crt", + "k8s_client_key": "/etc/calico/certs/kubernetes/kubernetes-client.key", + "k8s_certificate_authority": "/etc/calico/certs/kubernetes/dcos-ca.crt" + } + }, + { + "type": "portmap", + "capabilities": { "portMappings": true}, + "snat": true + } + ] + } +EOF + +sudo /bin/cp ${CALICO_CNI_CONF_DIR}/${KUBERNETES_CNI_CONF_FILE} /opt/mesosphere/etc/dcos/network/cni/ +# Do not actually need to restart, as this isn't used by Mesos, only by K8s +# sudo systemctl restart dcos-mesos-slave* + +# We do need it to persist across upgrades, though + +# grep MESOS_NETWORK_CNI_PLUGINS_DIR /opt/mesosphere/etc/mesos-slave-common | sudo tee -a /var/lib/dcos/mesos-slave-common +# sudo sed -i '/MESOS_NETWORK_CNI_PLUGINS_DIR/s|$|:CALICO_CNI_PLUGIN_DIR|g' /var/lib/dcos/mesos-slave-common +# sudo sed -i "s|CALICO_CNI_PLUGIN_DIR|${CALICO_CNI_PLUGIN_DIR}|g" /var/lib/dcos/mesos-slave-common + +## Plugin conf +sudo mkdir -p /etc/systemd/system/dcos-mesos-slave.service.d +# We can do both dcos-mesos-slave and dcos-mesos-slave-common on all nodes, safely; only the relevant one will be used by the corresponding systemd unit +# We use a systemd override to copy the conf from custom location into default MESOS_NETWORK_CNI_CONFIG_DIR +sudo tee /etc/systemd/system/dcos-mesos-slave.service.d/kubernetes-calico-conf-override.conf <<-'EOF' +[Service] +ExecStartPre=/bin/cp CALICO_CNI_CONF_DIR/KUBERNETES_CNI_CONF_FILE /opt/mesosphere/etc/dcos/network/cni/ +EOF + +sudo sed -i "s|CALICO_CNI_CONF_DIR|${CALICO_CNI_CONF_DIR}|g" /etc/systemd/system/dcos-mesos-slave.service.d/kubernetes-calico-conf-override.conf +sudo sed -i "s|KUBERNETES_CNI_CONF_FILE|${KUBERNETES_CNI_CONF_FILE}|g" /etc/systemd/system/dcos-mesos-slave.service.d/kubernetes-calico-conf-override.conf + +sudo mkdir -p /etc/systemd/system/dcos-mesos-slave-public.service.d +# /etc/systemd/system/dcos-mesos-slave.service.d/override.conf +sudo tee /etc/systemd/system/dcos-mesos-slave-public.service.d/kubernetes-calico-conf-override.conf <<-'EOF' +[Service] +ExecStartPre=/bin/cp CALICO_CNI_CONF_DIR/KUBERNETES_CNI_CONF_FILE /opt/mesosphere/etc/dcos/network/cni/ +EOF + +sudo sed -i "s|CALICO_CNI_CONF_DIR|${CALICO_CNI_CONF_DIR}|g" /etc/systemd/system/dcos-mesos-slave-public.service.d/kubernetes-calico-conf-override.conf +sudo sed -i "s|KUBERNETES_CNI_CONF_FILE|${KUBERNETES_CNI_CONF_FILE}|g" /etc/systemd/system/dcos-mesos-slave-public.service.d/kubernetes-calico-conf-override.conf + +# Need to reload, but don't restart services cause we already manually copied stuff +sudo systemctl daemon-reload \ No newline at end of file diff --git a/calico-secure/v05/6o-kubernetes.sh b/calico-secure/v05/6o-kubernetes.sh new file mode 100644 index 0000000..538e9ae --- /dev/null +++ b/calico-secure/v05/6o-kubernetes.sh @@ -0,0 +1,97 @@ +source env.export + + +## Scriptlet used to generate certs using DC/OS CA +tee bootstrap-certs.py <<-'EOF' +#!/opt/mesosphere/bin/python + +import sys +sys.path.append('/opt/mesosphere/lib/python3.6/site-packages') + +from dcos_internal_utils import bootstrap + +if len(sys.argv) == 1: + print("Usage: ./bootstrap-certs.py | ./bootstrap-certs.py etcd /var/lib/dcos/etcd/certs") + sys.exit(1) + +b = bootstrap.Bootstrapper(bootstrap.parse_args()) +b.read_agent_secrets() + +cn = sys.argv[1] +location = sys.argv[2] + +keyfile = location + '/' + cn + '.key' +crtfile = location + '/' + cn + '.crt' + +b.ensure_key_certificate(cn, keyfile, crtfile, service_account='dcos_bootstrap_agent') +EOF +chmod +x bootstrap-certs.py + +sudo mkdir -p /etc/calico/certs/kubernetes +# sudo /home/centos/bootstrap-certs.py kubernetes-etcd /etc/calico/certs/kubernetes +sudo /home/centos/bootstrap-certs.py kubernetes-client /etc/calico/certs/kubernetes +sudo curl -Lk https://master.mesos/ca/dcos-ca.crt -o /etc/calico/certs/kubernetes/dcos-ca.crt + + +sudo tee ${CALICO_CNI_CONF_DIR}/${KUBERNETES_CNI_CONF_FILE} <<-'EOF' + { + "cniVersion": "0.3.0", + "name": "kube-cni", + "plugins": [ + { + "type": "calico", + "etcd_endpoints": "http://localhost:62379", + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s", + "k8s_api_root": "https://apiserver.kubernetes.l4lb.thisdcos.directory:6443", + "k8s_client_certificate": "/etc/calico/certs/kubernetes/kubernetes-client.crt", + "k8s_client_key": "/etc/calico/certs/kubernetes/kubernetes-client.key", + "k8s_certificate_authority": "/etc/calico/certs/kubernetes/dcos-ca.crt" + } + }, + { + "type": "portmap", + "capabilities": { "portMappings": true}, + "snat": true + } + ] + } +EOF + +sudo /bin/cp ${CALICO_CNI_CONF_DIR}/${KUBERNETES_CNI_CONF_FILE} /opt/mesosphere/etc/dcos/network/cni/ +# Do not actually need to restart, as this isn't used by Mesos, only by K8s +# sudo systemctl restart dcos-mesos-slave* + +# We do need it to persist across upgrades, though + +# grep MESOS_NETWORK_CNI_PLUGINS_DIR /opt/mesosphere/etc/mesos-slave-common | sudo tee -a /var/lib/dcos/mesos-slave-common +# sudo sed -i '/MESOS_NETWORK_CNI_PLUGINS_DIR/s|$|:CALICO_CNI_PLUGIN_DIR|g' /var/lib/dcos/mesos-slave-common +# sudo sed -i "s|CALICO_CNI_PLUGIN_DIR|${CALICO_CNI_PLUGIN_DIR}|g" /var/lib/dcos/mesos-slave-common + +## Plugin conf +sudo mkdir -p /etc/systemd/system/dcos-mesos-slave.service.d +# We can do both dcos-mesos-slave and dcos-mesos-slave-common on all nodes, safely; only the relevant one will be used by the corresponding systemd unit +# We use a systemd override to copy the conf from custom location into default MESOS_NETWORK_CNI_CONFIG_DIR +sudo tee /etc/systemd/system/dcos-mesos-slave.service.d/kubernetes-calico-conf-override.conf <<-'EOF' +[Service] +ExecStartPre=/bin/cp CALICO_CNI_CONF_DIR/KUBERNETES_CNI_CONF_FILE /opt/mesosphere/etc/dcos/network/cni/ +EOF + +sudo sed -i "s|CALICO_CNI_CONF_DIR|${CALICO_CNI_CONF_DIR}|g" /etc/systemd/system/dcos-mesos-slave.service.d/kubernetes-calico-conf-override.conf +sudo sed -i "s|KUBERNETES_CNI_CONF_FILE|${KUBERNETES_CNI_CONF_FILE}|g" /etc/systemd/system/dcos-mesos-slave.service.d/kubernetes-calico-conf-override.conf + +sudo mkdir -p /etc/systemd/system/dcos-mesos-slave-public.service.d +# /etc/systemd/system/dcos-mesos-slave.service.d/override.conf +sudo tee /etc/systemd/system/dcos-mesos-slave-public.service.d/kubernetes-calico-conf-override.conf <<-'EOF' +[Service] +ExecStartPre=/bin/cp CALICO_CNI_CONF_DIR/KUBERNETES_CNI_CONF_FILE /opt/mesosphere/etc/dcos/network/cni/ +EOF + +sudo sed -i "s|CALICO_CNI_CONF_DIR|${CALICO_CNI_CONF_DIR}|g" /etc/systemd/system/dcos-mesos-slave-public.service.d/kubernetes-calico-conf-override.conf +sudo sed -i "s|KUBERNETES_CNI_CONF_FILE|${KUBERNETES_CNI_CONF_FILE}|g" /etc/systemd/system/dcos-mesos-slave-public.service.d/kubernetes-calico-conf-override.conf + +# Need to reload, but don't restart services cause we already manually copied stuff +sudo systemctl daemon-reload \ No newline at end of file diff --git a/calico-secure/v05/7-kubernetes-bootstrap.sh b/calico-secure/v05/7-kubernetes-bootstrap.sh new file mode 100644 index 0000000..0003bcd --- /dev/null +++ b/calico-secure/v05/7-kubernetes-bootstrap.sh @@ -0,0 +1,74 @@ +dcos package install dcos-enterprise-cli --cli --yes +dcos security org service-accounts keypair kubernetes-private-key.pem kubernetes-public-key.pem +dcos security org service-accounts create -p kubernetes-public-key.pem -d 'Kubernetes service account' kubernetes +dcos security secrets create-sa-secret --strict kubernetes-private-key.pem kubernetes kubernetes/sa +dcos security org groups add_user superusers kubernetes +dcos package install kubernetes --cli --yes + +curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.9.7/bin/linux/amd64/kubectl + +chmod +x kubectl + +sudo mv kubectl /usr/local/bin/ + +#### + +dcos kubernetes kubeconfig + +dcos security cluster ca newcert --cn "calico-kube-controller" --host "localhost" --json > calico-kube-controller.json +cat calico-kube-controller.json | python -c 'import sys,json;j=sys.stdin.read();print(json.loads(j)["certificate"])' | grep -v '^$' > calico-kube-controller.crt +cat calico-kube-controller.json | python -c 'import sys,json;j=sys.stdin.read();print(json.loads(j)["private_key"])' | grep -v '^$' > calico-kube-controller.key +rm calico-kube-controller.json +dcos security cluster ca cacert > dcos-ca.crt + +kubectl create secret generic calico-etcd-certs \ + --namespace=kube-system \ + --from-file=./calico-kube-controller.crt \ + --from-file=./calico-kube-controller.key \ + --from-file=./dcos-ca.crt + +tee policy_controller.yaml <<-'EOF' +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-kube-controller + namespace: kube-system + labels: + k8s-app: calico-kube-controller +spec: + replicas: 1 + template: + metadata: + name: calico-kube-controller + namespace: kube-system + labels: + k8s-app: calico-kube-controller + spec: + hostNetwork: true + containers: + - name: calico-kube-controller + # Make sure to pin this to your desired version - 0.3.0 doesn't support tls 1.2 + # image: calico/kube-policy-controller:v0.3.0 + image: quay.io/calico/kube-controllers:v1.0.4 + env: + - name: ETCD_ENDPOINTS + value: "https://localhost:62379" + - name: CONFIGURE_ETC_HOSTS + value: "true" + - name: ETCD_CA_CERT_FILE + value: "/calico-secrets/dcos-ca.crt" + - name: ETCD_KEY_FILE + value: "/calico-secrets/calico-kube-controller.key" + - name: ETCD_CERT_FILE + value: "/calico-secrets/calico-kube-controller.crt" + volumeMounts: + - name: etcd-certs + mountPath: /calico-secrets + readOnly: true + volumes: + - name: etcd-certs + secret: + secretName: calico-etcd-certs +EOF + +kubectl apply -f policy_controller.yaml diff --git a/calico-secure/v05/7o-kubernetes-bootstrap.sh b/calico-secure/v05/7o-kubernetes-bootstrap.sh new file mode 100644 index 0000000..47a5d4a --- /dev/null +++ b/calico-secure/v05/7o-kubernetes-bootstrap.sh @@ -0,0 +1,47 @@ +dcos package install dcos-enterprise-cli --cli --yes +dcos security org service-accounts keypair kubernetes-private-key.pem kubernetes-public-key.pem +dcos security org service-accounts create -p kubernetes-public-key.pem -d 'Kubernetes service account' kubernetes +dcos security secrets create-sa-secret --strict kubernetes-private-key.pem kubernetes kubernetes/sa +dcos security org groups add_user superusers kubernetes +dcos package install kubernetes --cli --yes + +curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.9.7/bin/linux/amd64/kubectl + +chmod +x kubectl + +sudo mv kubectl /usr/local/bin/ + +#### + +dcos kubernetes kubeconfig + +tee policy_controller.yaml <<-'EOF' +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-kube-controller + namespace: kube-system + labels: + k8s-app: calico-kube-controller +spec: + replicas: 1 + template: + metadata: + name: calico-kube-controller + namespace: kube-system + labels: + k8s-app: calico-kube-controller + spec: + hostNetwork: true + containers: + - name: calico-kube-controller + # Make sure to pin this to your desired version. + image: calico/kube-policy-controller:v0.3.0 + env: + - name: ETCD_ENDPOINTS + value: "http://localhost:62379" + - name: CONFIGURE_ETC_HOSTS + value: "true" +EOF + +kubectl apply -f policy_controller.yaml \ No newline at end of file diff --git a/calico-secure/v05/README.md b/calico-secure/v05/README.md new file mode 100644 index 0000000..5436923 --- /dev/null +++ b/calico-secure/v05/README.md @@ -0,0 +1,28 @@ +# Notes +This entire process is meant to be relatively automatable - you can basically put each file into a shell file and run it via ansible/chef/automation tool of choice. Overall, most of these can be run multiple times, but not all of them (specifically, restarting docker and the various sytemd units shouldn't be run multiple times). Do your own testing. + +Depending on whether your docker config has `overlay` configured in the systemd unit or elsewhere, you'll have to consider removing the `overlay` line from /etc/docker/daemon.json - the command to do this is commented out in this script. If you're deploying on Mesosphere-provided AMIs, you may have to uncomment this command (so it gets automa†ically removed) from both enable-masters and enable-agents. + + This sets up a default pool of 172.16.0.0/16 (which supports something like 65k containers). You want to change this to meet the following requirements: +-Size the subnet accordingly +-Use a subnet that doesn't overlap with your environment. +This is configured in an environment variable that gets propagated to /etc/calico/ippool.json - if you need to change it, change it in the original env variable, or change it directly in the file prior to setting up the ip pool. + +# Process +0: Stand up your DC/OS cluster. This whole process assumes DC/OS enterprise with permissive or strict (in order to use the DC/OS CA) + +1a: Create env.export on all nodes. This will set up environment variables, and is used by other scripts + +1b: Generate certs on all nodes + +1c: Generate systemd units (and accompanying env files) on all nodes + +1d: Generate conf files on all nodes + +2: Run the package install on all nodes. This downloads all binaries and packages + +3: On all masters, run the enable-masters commands. This starts all processes. Note that depending on whether your docker config has `overlay` configured in the systemd unit or elsewhere, you'll have to consider removing the `overlay` line from /etc/docker/daemon.json + +4: On one master, run the commands from set-up-pool. This creates a cluster-wide Calico IP pool, used for all containers. + +5: On all agents (private and public), run the enable-agents commands. This starts all processes. \ No newline at end of file diff --git a/calico-secure/v05/calico-secure-v0.4.tgz b/calico-secure/v05/calico-secure-v0.4.tgz new file mode 100644 index 0000000..28134ac Binary files /dev/null and b/calico-secure/v05/calico-secure-v0.4.tgz differ diff --git a/calico-secure/v05/env.export b/calico-secure/v05/env.export new file mode 100644 index 0000000..9aaed7f --- /dev/null +++ b/calico-secure/v05/env.export @@ -0,0 +1,29 @@ +# This will be customer-specific, so it's at the top +export CALICO_CIDR=172.16.0.0/16 + +# 2379 and 2380 are within the DC/OS service port range, and are used by the etcd included with Calico. +export ETCD_LISTEN_PORT=62379 +export ETCD_TRANSPORT_PORT=62380 + +## Env variables +export MASTER_LIST_NOPORT=$(curl -sS master.mesos:8181/exhibitor/v1/cluster/status | python -c 'import sys,json;j=json.loads(sys.stdin.read());print(",".join([y["hostname"]+"=https://"+y["hostname"]+":ETCD_TRANSPORT_PORT" for y in j]))') +export MASTER_LIST=$(echo $MASTER_LIST_NOPORT | sed "s|ETCD_TRANSPORT_PORT|${ETCD_TRANSPORT_PORT}|") + +export ETCD_ROOT_DIR=/opt/etcd +export ETCD_DATA_DIR=/var/etcd/data +export ETCD_TLS_CERT=etcd.crt +export ETCD_TLS_KEY=etcd.key +export ETCD_CA_CERT=dcos-ca.crt +export LOCAL_HOSTNAME=$(/opt/mesosphere/bin/detect_ip) +export INITIAL_CLUSTER=${MASTER_LIST} + +export CALICO_CNI_PLUGIN_DIR=/opt/calico/plugins +export CALICO_CNI_CONF_DIR=/etc/calico/cni + +export CALICO_NODE_IMAGE=quay.io/calico/node:v2.6.9 + +export ETCD_CERTS_DIR=/etc/etcd/certs +export DOCKER_CLUSTER_CERTS_DIR=/etc/docker/cluster/certs +export CALICO_NODE_CERTS_DIR=/etc/calico/certs/node +export CALICO_CALICOCTL_CERTS_DIR=/etc/calico/certs/calicoctl +export CALICO_CNI_CERTS_DIR=/etc/calico/certs/cni diff --git a/calico-secure/v05/x-kubernetes-notes.sh b/calico-secure/v05/x-kubernetes-notes.sh new file mode 100644 index 0000000..275b790 --- /dev/null +++ b/calico-secure/v05/x-kubernetes-notes.sh @@ -0,0 +1,113 @@ +dcos package install dcos-enterprise-cli --cli --yes +dcos security org service-accounts keypair kubernetes-private-key.pem kubernetes-public-key.pem +dcos security org service-accounts create -p kubernetes-public-key.pem -d 'Kubernetes service account' kubernetes +dcos security secrets create-sa-secret --strict kubernetes-private-key.pem kubernetes kubernetes/sa +dcos security org groups add_user superusers kubernetes +dcos package install kubernetes --cli --yes + +curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.9.7/bin/linux/amd64/kubectl + +chmod +x kubectl + +sudo mv kubectl /usr/local/bin/ + +dcos kubernetes kubeconfig + +# Insecure install +bash 1a-env.export.sh ; bash 1c-prereqs-systemd.sh ; bash 1do-prereqs-conf.sh; bash 2-package-install.sh ; bash 3o-enable-masters.sh + +bash 1a-env.export.sh ; bash 1c-prereqs-systemd.sh ; bash 1do-prereqs-conf.sh; bash 2-package-install.sh ; bash 5o-enable-agents.sh +bash 6o-kubernetes.sh + +# Secure install +# master +bash 1a-env.export.sh; bash 1b-prereqs-certs.sh; bash 1c-prereqs-systemd.sh; bash 1d-prereqs-conf.sh +bash 2-package-install.sh +bash 3-enable-masters.sh + +# slave +bash 1a-env.export.sh; bash 1b-prereqs-certs.sh; bash 1c-prereqs-systemd.sh; bash 1d-prereqs-conf.sh +bash 2-package-install.sh + + +sudo mkdir -p /etc/calico/certs/kubernetes +sudo /home/centos/bootstrap-certs.py kubernetes-etcd /etc/calico/certs/kubernetes +sudo /home/centos/bootstrap-certs.py kubernetes-client /etc/calico/certs/kubernetes +sudo curl -Lk https://master.mesos/ca/dcos-ca.crt -o /etc/calico/certs/kubernetes/dcos-ca.crt + + + + +sudo tee /opt/mesosphere/etc/dcos/network/cni/cni.conflist <<-'EOF' + { + "cniVersion": "0.3.0", + "name": "kube-cni", + "plugins": [ + { + "type": "calico", + "etcd_endpoints": "http://localhost:62379", + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s", + "k8s_api_root": "https://apiserver.kubernetes.l4lb.thisdcos.directory:6443", + "k8s_client_certificate": "/etc/calico/certs/kubernetes/kubernetes-client.crt", + "k8s_client_key": /etc/calico/certs/kubernetes/kubernetes-client.key", + "k8s_certificate_authority": "/etc/calico/certs/kubernetes/dcos-ca.crt" + } + }, + { + "type": "portmap", + "capabilities": { "portMappings": true}, + "snat": true + } + ] + } +EOF + +sudo systemctl restart dcos-mesos-slave* + + +tee policy_controller.yaml <<-'EOF' +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: calico-kube-controller + namespace: kube-system + labels: + k8s-app: calico-kube-controller +spec: + replicas: 1 + template: + metadata: + name: calico-kube-controller + namespace: kube-system + labels: + k8s-app: calico-kube-controller + spec: + hostNetwork: true + containers: + - name: calico-kube-controller + # Make sure to pin this to your desired version. + image: calico/kube-policy-controller:v0.3.0 + env: + - name: ETCD_ENDPOINTS + value: "http://localhost:62379" + - name: CONFIGURE_ETC_HOSTS + value: "true" +EOF + +kubectl apply -f policy_controller.yaml + +sudo rm /etc/systemd/system/dcos-mesos-slave*/override.conf +sudo rm /opt/mesosphere/etc/dcos/network/cni/calico.conf +sudo systemctl daemon-reload; sudo systemctl restart dcos-mesos-slave* + +# Re-add +sudo cp /etc/calico/cni/calico.conf /opt/mesosphere/etc/dcos/network/cni/ +sudo systemctl restart dcos-mesos-slave* + +# re-remove +sudo rm /opt/mesosphere/etc/dcos/network/cni/calico.conf +sudo systemctl restart dcos-mesos-slave* \ No newline at end of file