Workaround k3s not loading images in agent directory when in airgapped environment

README.md

@@ -25,19 +25,25 @@ Production deploments are intended to run on a satellite with an emphasis on red
     # Or specify the architecture to download a different architecture
     /var/spacedev/scripts/stage_spacefx.sh --architecture arm64
 
+    # Clean out /var/spacedev of unnecessary logs and artifacts
+    sudo rm /var/spacedev/logs/* -rf
+    sudo rm /var/spacedev/tmp/* -rf
+    sudo find /var/spacedev/certs -type f ! -name '*.json' -delete
+
     # Create a clean output directory
     sudo mkdir -p ./output && sudo rm -rf ./output/*
-    sudo tar -czf ./output/msft_azure_orbital_space_sdk.tar.gz -C /var/spacedev .
+    sudo tar -czf ./output/msft_azure_orbital_framework.tar.gz -C /var/spacedev .
+    sudo sha256sum ./output/msft_azure_orbital_framework.tar.gz | awk '{print $1}' | sudo tee ./output/msft_azure_orbital_framework.tar.gz.sha256
     ```
 
-1.  Copy the `./output/msft_azure_orbital_space_sdk.tar.gz` to the target hardware / satellite / host
+1.  Copy the `./output/msft_azure_orbital_framework.tar.gz` to the target hardware / satellite / host
 
 1.  Deploy the Microsoft Azure Orbital Space SDK
     ```bash
     # Extract the Microsoft Azure Orbital Space SDK to /var/spacedev
     sudo mkdir -p /var/spacedev
     sudo chown -R "${USER:-$(id -un)}" /var/spacedev
-    sudo tar -xzvf msft_azure_orbital_space_sdk.tar.gz -C /var/spacedev
+    sudo tar -xzvf msft_azure_orbital_framework.tar.gz -C /var/spacedev
 
     # Deploy the Microsoft Azure Orbital Space SDK
     /var/spacedev/scripts/deploy_spacefx.sh

env/spacefx.env

@@ -10,4 +10,4 @@ VER_K3S=v1.29.3+k3s1
 VER_JQ=1.7.1
 VER_YQ=4.44.2
 VER_REGCTL=v0.5.7
-KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml

modules/m_15_directories.sh

@@ -25,6 +25,14 @@ function _setup_initial_directories() {
     create_directory "${SPACEFX_DIR}/tmp"
     create_directory "${SPACEFX_DIR}/tmp/yamls"
     create_directory "${SPACEFX_DIR}/xfer"
+
+    if [[ ! -L "${SPACEFX_DIR}/scripts/deploy/deploy_spacefx.sh" ]]; then
+        ln -s "${SPACEFX_DIR}/scripts/deploy_spacefx.sh" "${SPACEFX_DIR}/scripts/deploy/deploy_spacefx.sh"
+    fi
+
+    if [[ ! -L "${SPACEFX_DIR}/scripts/stage/stage_spacefx.sh" ]]; then
+        ln -s "${SPACEFX_DIR}/scripts/stage_spacefx.sh" "${SPACEFX_DIR}/scripts/stage/stage_spacefx.sh"
+    fi
 }
 
 ############################################################

scripts/big_red_button.sh

@@ -122,7 +122,27 @@ function remove_k3s() {
     fi
 
     info_log "...k3s found.  Uninstalling..."
-    [[ -f "/usr/local/bin/k3s-uninstall.sh" ]] && run_a_script "/usr/local/bin/k3s-uninstall.sh"
+    if [[ -f "/usr/local/bin/k3s-uninstall.sh" ]]; then
+        run_a_script "/usr/local/bin/k3s-uninstall.sh" k3s_uninstall_pid --background
+        debug_log "...wating for k3s to finish uninstalling (pid $k3s_uninstall_pid)..."
+        wait $((k3s_uninstall_pid))
+    fi
+
+    info_log "Cleaning \$PATH array of any k3s references"
+
+    #Loop through the PATH array and remove any paths that contain 'k3s'
+    IFS=':' read -r -a path_array <<< "$PATH"
+
+    cleaned_paths=()
+    for path in "${path_array[@]}"; do
+        if [[ "$path" != *k3s* ]]; then
+            cleaned_paths+=("$path")
+        fi
+    done
+
+    # Rebuild the PATH array and export it back out
+    cleaned_path=$(IFS=:; echo "${cleaned_paths[*]}")
+    export PATH=$cleaned_path
 
     info_log "...k3s successfully uninstalled"
 
@@ -159,9 +179,14 @@ function prune_docker() {
 function prune_registry() {
     info_log "START: ${FUNCNAME[0]}"
 
+    is_cmd_available "pgrep" HAS_PGREP
 
     info_log "Stopping registry processes (if still running)"
-    run_a_script "pgrep '^registry'" pids --ignore_error
+    if [[ "${HAS_PGREP}" == true ]]; then
+        run_a_script "pgrep '^registry'" pids --ignore_error
+    else
+        run_a_script "ps aux | grep '^registry' | grep -v grep | awk '{print \$2}'" pids --ignore_error
+    fi
 
     for pid in $pids; do
         debug_log "...terminating process id '${pid}'"
@@ -172,18 +197,50 @@ function prune_registry() {
 
     info_log "Stopping pypiserver processes (if still running)"
 
-    run_a_script "pgrep '^pypiserver'" pids --ignore_error
+    if [[ "${HAS_PGREP}" == true ]]; then
+        run_a_script "pgrep '^pypi-server'" pids --ignore_error
+    else
+        run_a_script "ps aux | grep '^pypi-server' | grep -v grep | awk '{print \$2}'" pids --ignore_error
+    fi
 
     for pid in $pids; do
         debug_log "...terminating process id '${pid}'"
         run_a_script "kill -9 ${pid}" --disable_log --ignore_error
     done
 
+
     info_log "...successfully stopped pypiserver processes."
 
     info_log "END: ${FUNCNAME[0]}"
 }
 
+############################################################
+# Remove apps that we installed as part of setup
+############################################################
+function remove_app(){
+    local app=""
+
+    while [[ "$#" -gt 0 ]]; do
+        case $1 in
+            --app)
+                shift
+                app=$1
+                ;;
+            *) echo "Unknown parameter '$1'"; show_help ;;
+        esac
+        shift
+    done
+
+    run_a_script "which -a ${app}" app_paths --ignore_error
+    for app_path in $app_paths; do
+        if [[ -f "$app_path" ]]; then
+            debug_log "Removing ${app} at $app_path"
+            run_a_script "sudo rm -f $app_path"
+            debug_log "...successfull removed old version of ${app} at $app_path"
+        fi
+    done
+}
+
 ############################################################
 # Remove k3s data directory if its been changed
 ############################################################
@@ -214,17 +271,25 @@ function main() {
     show_header
 
     check_and_disable_k3s
-
     stop_all_docker_containers
     remove_k3s
     prune_docker
     prune_registry
+
     remove_k3s_data_dir
 
     info_log "Removing '${SPACEFX_DIR:?}'..."
     run_a_script "rm -rf ${SPACEFX_DIR:?}"
     info_log "...successfully removed '${SPACEFX_DIR:?}'"
 
+    remove_app --app "yq"
+    remove_app --app "jq"
+    remove_app --app "regctl"
+    remove_app --app "cfssl"
+    remove_app --app "cfssljson"
+    remove_app --app "helm"
+
+
     info_log "------------------------------------------"
     info_log "END: ${SCRIPT_NAME}"
 }

scripts/coresvc_registry.sh

@@ -65,14 +65,20 @@ function check_prerequisites(){
 
     is_cmd_available "docker" HAS_DOCKER
     is_cmd_available "kubectl" HAS_K3S
+    is_cmd_available "pgrep" HAS_PGREP
 
     if [[ "${HAS_K3S}" == true ]]; then
         # if we have kubectl, then check if we have k3s
         is_cmd_available "k3s" HAS_K3S
 
         if [[ "${HAS_K3S}" == true ]]; then
             # We have k3s, so we need to check if it's running
-            run_a_script "pgrep \"k3s\"" k3s_status --ignore_error
+            if [[ "${HAS_PGREP}" == true ]]; then
+                run_a_script "pgrep \"k3s\"" k3s_status --ignore_error
+            else
+                run_a_script "ps | grep \"k3s server\"" k3s_status --ignore_error
+            fi
+
 
             if [[ -z "${k3s_status}" ]]; then
                 # k3s is installed but not running
@@ -92,6 +98,10 @@ function check_prerequisites(){
         DESTINATION_HOST="k3s"
     fi
 
+    if [[ "${HAS_DOCKER}" == false ]] && [[ "${HAS_K3S}" == false ]]; then
+        exit_with_error "No suitable environment found (HAS_DOCKER = 'false'.  HAS_K#s = 'false').  Please install either Docker or K3s"
+    fi
+
     [[ ! -d "${SPACEFX_DIR}/registry/data" ]] && create_directory "${SPACEFX_DIR}/registry/data"
     [[ ! -d "${SPACEFX_DIR}/registry/pypiserver" ]] && create_directory "${SPACEFX_DIR}/registry/pypiserver"
     [[ ! -d "${SPACEFX_DIR}/certs/registry" ]] && create_directory "${SPACEFX_DIR}/certs/registry"
@@ -192,7 +202,11 @@ function stop_registry(){
     fi
 
     info_log "Stopping registry processes (if still running)"
-    run_a_script "pgrep '^registry'" pids --ignore_error
+    if [[ "${HAS_PGREP}" == true ]]; then
+        run_a_script "pgrep '^registry'" pids --ignore_error
+    else
+        run_a_script "ps aux | grep '^registry' | grep -v grep | awk '{print \$2}'" pids --ignore_error
+    fi
 
     for pid in $pids; do
         debug_log "...terminating process id '${pid}'"
@@ -203,7 +217,11 @@ function stop_registry(){
 
     info_log "Stopping pypiserver processes (if still running)"
 
-    run_a_script "pgrep '^pypiserver'" pids --ignore_error
+    if [[ "${HAS_PGREP}" == true ]]; then
+        run_a_script "pgrep '^pypi-server'" pids --ignore_error
+    else
+        run_a_script "ps aux | grep '^pypi-server' | grep -v grep | awk '{print \$2}'" pids --ignore_error
+    fi
 
     for pid in $pids; do
         debug_log "...terminating process id '${pid}'"

scripts/deploy/deploy_k3s.sh

@@ -76,16 +76,16 @@ SPACEFX_UPDATE_END"
 
     _check_for_file "${SPACEFX_DIR}/bin/${ARCHITECTURE}/k3s/${VER_K3S}/k3s"
     _check_for_file "${SPACEFX_DIR}/bin/${ARCHITECTURE}/k3s/${VER_K3S}/k3s_install.sh"
-    _check_for_file "${SPACEFX_DIR}/images/${ARCHITECTURE}/k3s-airgap-images-${ARCHITECTURE}.tar.gz"
+    _check_for_file "${SPACEFX_DIR}/images/${ARCHITECTURE}/k3s-airgap-images-${ARCHITECTURE}.tar"
 
     info_log "...copying files to destinations..."
 
 
     [[ ! -d "/usr/local/bin" ]] && create_directory "/usr/local/bin"
-    [[ ! -f "/usr/local/bin/k3s" ]] && run_a_script "cp ${SPACEFX_DIR}/bin/${ARCHITECTURE}/k3s/${VER_K3S}/k3s /usr/local/bin/k3s"
+    run_a_script "cp ${SPACEFX_DIR}/bin/${ARCHITECTURE}/k3s/${VER_K3S}/k3s /usr/local/bin/k3s"
 
     [[ ! -d "/var/lib/rancher/k3s/agent/images" ]] && create_directory "/var/lib/rancher/k3s/agent/images"
-    [[ ! -f "/var/lib/rancher/k3s/agent/images/k3s-airgap-images-${ARCHITECTURE}.tar.gz" ]] && run_a_script "cp ${SPACEFX_DIR}/images/${ARCHITECTURE}/k3s-airgap-images-${ARCHITECTURE}.tar.gz /var/lib/rancher/k3s/agent/images/k3s-airgap-images-${ARCHITECTURE}.tar.gz"
+    run_a_script "cp ${SPACEFX_DIR}/images/${ARCHITECTURE}/k3s-airgap-images-${ARCHITECTURE}.tar /var/lib/rancher/k3s/agent/images/k3s-airgap-images-${ARCHITECTURE}.tar"
 
 
     export INSTALL_K3S_SKIP_DOWNLOAD=true
@@ -104,6 +104,112 @@ SPACEFX_UPDATE_END"
     info_log "FINISHED: ${FUNCNAME[0]}"
 }
 
+############################################################
+# Check if the images need to be loaded into k3s
+############################################################
+function load_images_to_k3s(){
+    info_log "START: ${FUNCNAME[0]}"
+
+    info_log "Validating images are loaded for k3s..."
+
+    if [[ ! -f "/etc/systemd/system/k3s.service" ]]; then
+        info_log "/etc/systemd/system/k3s.service not found.  Nothing to do."
+        info_log "FINISHED: ${FUNCNAME[0]}"
+        return
+    fi
+
+    run_a_script "cat /etc/systemd/system/k3s.service" k3s_service_file
+    if [[ "$k3s_service_file" == *"--docker"* ]]; then
+        info_log "...docker detected.  Validating images via docker..."
+        load_images_to_k3s_docker
+    else
+        info_log "...docker not detected.  Validating images via ctr..."
+        load_images_to_k3s_ctr
+    fi
+
+    info_log "Validated images are loaded"
+
+    info_log "FINISHED: ${FUNCNAME[0]}"
+}
+
+
+############################################################
+# Check if the images need to be loaded into k3s (via docker)
+############################################################
+function load_images_to_k3s_ctr(){
+    info_log "START: ${FUNCNAME[0]}"
+
+    start_time=$(date +%s)
+
+
+    is_cmd_available "ctr" has_ctr_cmd
+    while [[ "${has_ctr_cmd}" == "false" ]]; do
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [[ $elapsed_time -ge $MAX_WAIT_SECS ]]; then
+            exit_with_error "Timed out waiting for k3s to come online."
+        fi
+
+        info_log "...ctr not available yet.  Rechecking in 5 seconds..."
+        sleep 5
+        is_cmd_available "ctr" has_ctr_cmd
+    done
+
+    info_log "ctr is available.  Checking if images are needed..."
+    k3s_images=("klipper-helm" "klipper-lb" "local-path-provisioner" "mirrored-coredns-coredns" "mirrored-library-busybox" "mirrored-library-traefik" "mirrored-metrics-server" "mirrored-pause")
+
+    run_a_script "ctr images list" ctr_images
+
+    needs_images="false"
+
+    for k3s_image in "${k3s_images[@]}"; do
+        if [[ "$ctr_images" != *"$k3s_image"* ]]; then
+            needs_images="true"
+        fi
+    done
+
+    if [[ "${needs_images}" == "true" ]]; then
+        info_log "Detected missing mirrored images.  Loading from '${SPACEFX_DIR}/images/${ARCHITECTURE}/k3s-airgap-images-${ARCHITECTURE}.tar'..."
+        run_a_script "ctr images import ${SPACEFX_DIR}/images/${ARCHITECTURE}/k3s-airgap-images-${ARCHITECTURE}.tar"
+        info_log "Images successfully imported"
+    else
+        info_log "All k3s images are already loaded.  Nothing to do."
+    fi
+
+
+    info_log "FINISHED: ${FUNCNAME[0]}"
+}
+
+############################################################
+# Check if the images need to be loaded into k3s (via docker)
+############################################################
+function load_images_to_k3s_docker(){
+    info_log "START: ${FUNCNAME[0]}"
+
+    info_log "Checking if images are needed (docker)..."
+    k3s_images=("klipper-helm" "klipper-lb" "local-path-provisioner" "mirrored-coredns-coredns" "mirrored-library-busybox" "mirrored-library-traefik" "mirrored-metrics-server" "mirrored-pause")
+
+    run_a_script "docker images" ctr_images
+
+    needs_images="false"
+
+    for k3s_image in "${k3s_images[@]}"; do
+        if [[ "$ctr_images" != *"$k3s_image"* ]]; then
+            needs_images="true"
+        fi
+    done
+
+    if [[ "${needs_images}" == "true" ]]; then
+        info_log "Detected missing k3s images.  Loading from '${SPACEFX_DIR}/images/${ARCHITECTURE}/k3s-airgap-images-${ARCHITECTURE}.tar'..."
+        run_a_script "docker load --input ${SPACEFX_DIR}/images/${ARCHITECTURE}/k3s-airgap-images-${ARCHITECTURE}.tar"
+        info_log "Images successfully imported"
+    else
+        info_log "All k3s images are already loaded.  Nothing to do."
+    fi
+
+
+    info_log "FINISHED: ${FUNCNAME[0]}"
+}
 
 ############################################################
 # Wait for k3s to finish deploying by checking for running pods
@@ -161,6 +267,7 @@ function wait_for_k3s_to_finish_initializing(){
 function main() {
 
     deploy_k3s_cluster
+    load_images_to_k3s
     wait_for_k3s_to_finish_initializing
 
     info_log "------------------------------------------"