From 1d1245fb1b796780bb19b78c642c63445faf9947 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Wed, 31 May 2023 14:53:04 +0200
Subject: [PATCH 01/13] adding license download and in addition ftp and scp as
 protocols for file downloads (also startup-configs)

---
 clab/config.go      |  42 +++++++-
 go.mod              |   1 -
 go.sum              |   2 -
 types/topo_paths.go |   4 +-
 utils/file.go       | 229 +++++++++++++++++++++++++++++++++++++++++++-
 5 files changed, 266 insertions(+), 12 deletions(-)

diff --git a/clab/config.go b/clab/config.go
index b49b0f972..014d73580 100644
--- a/clab/config.go
+++ b/clab/config.go
@@ -226,10 +226,11 @@ func (c *CLab) createNodeCfg(nodeName string, nodeDef *types.NodeDefinition, idx
 	nodeCfg.EnforceStartupConfig = c.Config.Topology.GetNodeEnforceStartupConfig(nodeCfg.ShortName)
 	nodeCfg.SuppressStartupConfig = c.Config.Topology.GetNodeSuppressStartupConfig(nodeCfg.ShortName)
 
-	// initialize license field
-	p := c.Config.Topology.GetNodeLicense(nodeCfg.ShortName)
-	// resolve the lic path to an abs path
-	nodeCfg.License = utils.ResolvePath(p, c.TopoPaths.TopologyFileDir())
+	// process startup-config
+	err = c.processNodeLicense(nodeCfg)
+	if err != nil {
+		return nil, err
+	}
 
 	// initialize bind mounts
 	binds, err := c.Config.Topology.GetNodeBinds(nodeName)
@@ -257,9 +258,14 @@ func (c *CLab) createNodeCfg(nodeName string, nodeDef *types.NodeDefinition, idx
 // It handles remote files, local files and embedded configs.
 // Returns an absolute path to the startup-config file.
 func (c *CLab) processStartupConfig(nodeCfg *types.NodeConfig) error {
+	var err error
 	// process startup-config
 	p := c.Config.Topology.GetNodeStartupConfig(nodeCfg.ShortName)
 
+	if p == "" {
+		return nil
+	}
+
 	// embedded config is a config that is defined as a multi-line string in the topology file
 	// it contains at least one newline
 	isEmbeddedConfig := strings.Count(p, "\n") >= 1
@@ -301,12 +307,40 @@ func (c *CLab) processStartupConfig(nodeCfg *types.NodeConfig) error {
 			p = absDestFile
 		}
 	}
+
+	p, err = utils.ProcessDownloadableAndEmbeddedFile(nodeCfg.ShortName, p, "embedded.partial.cfg", c.TopoPaths)
+	if err != nil {
+		return err
+	}
+
 	// resolve the startup config path to an abs path
 	nodeCfg.StartupConfig = utils.ResolvePath(p, c.TopoPaths.TopologyFileDir())
 
 	return nil
 }
 
+// processStartupConfig processes the raw path of the startup-config as it is defined in the topology file.
+// It handles remote files, local files and embedded configs.
+// Returns an absolute path to the startup-config file.
+func (c *CLab) processNodeLicense(nodeCfg *types.NodeConfig) error {
+	var err error
+	// process startup-config
+	p := c.Config.Topology.GetNodeLicense(nodeCfg.ShortName)
+	if p == "" {
+		return nil
+	}
+
+	p, err = utils.ProcessDownloadableAndEmbeddedFile(nodeCfg.ShortName, p, "embedded.lic", c.TopoPaths)
+	if err != nil {
+		return err
+	}
+
+	// resolve the startup config path to an abs path
+	nodeCfg.License = utils.ResolvePath(p, c.TopoPaths.TopologyFileDir())
+
+	return nil
+}
+
 // CheckTopologyDefinition runs topology checks and returns any errors found.
 // This function runs after topology file is parsed and all nodes/links are initialized.
 func (c *CLab) CheckTopologyDefinition(ctx context.Context) error {
diff --git a/go.mod b/go.mod
index b8f870a48..f4f6468bf 100644
--- a/go.mod
+++ b/go.mod
@@ -38,7 +38,6 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/spf13/cobra v1.8.0
-	github.com/steiler/acls v0.1.1
 	github.com/stretchr/testify v1.8.4
 	github.com/tklauser/numcpus v0.7.0
 	github.com/vishvananda/netlink v1.2.1-beta.2
diff --git a/go.sum b/go.sum
index 8b5c3fd20..c7c9cea50 100644
--- a/go.sum
+++ b/go.sum
@@ -2717,8 +2717,6 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
-github.com/steiler/acls v0.1.1 h1:ghWAs51+psOLScMhaG6+23RCqXegtwnUqienhYT+l5g=
-github.com/steiler/acls v0.1.1/go.mod h1:Tpbo8Kj/6mQlon8pDVqh9PIsyvYbrLbCiXD9cUrpmGI=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
diff --git a/types/topo_paths.go b/types/topo_paths.go
index ee6cd96ee..d1c93ef6f 100644
--- a/types/topo_paths.go
+++ b/types/topo_paths.go
@@ -183,9 +183,9 @@ func (*TopoPaths) ClabTmpDir() string {
 	return clabTmpDir
 }
 
-// StartupConfigDownloadFileAbsPath returns the absolute path to the startup-config file
+// DownloadFileTmpAbsPath returns the absolute path to a file
 // when it is downloaded from a remote location to the clab temp directory.
-func (t *TopoPaths) StartupConfigDownloadFileAbsPath(node, postfix string) string {
+func (t *TopoPaths) DownloadFileTmpAbsPath(node string, postfix string) string {
 	return filepath.Join(t.ClabTmpDir(), fmt.Sprintf("%s-%s-%s", t.topoName, node, postfix))
 }
 
diff --git a/utils/file.go b/utils/file.go
index 36ab520b7..5653fe376 100644
--- a/utils/file.go
+++ b/utils/file.go
@@ -6,11 +6,14 @@ package utils
 
 import (
 	"bufio"
+	"bytes"
+	"context"
 	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
 	"io/fs"
+	"io/ioutil"
 	"math"
 	"mime"
 	"net/http"
@@ -21,10 +24,15 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/steiler/acls"
 
+	"github.com/bramvdbogaerde/go-scp"
+	"github.com/bramvdbogaerde/go-scp/auth"
+	"github.com/jlaffaye/ftp"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/crypto/ssh"
 )
 
 var (
@@ -60,7 +68,7 @@ func DirExists(filename string) bool {
 // mode is the desired target file permissions, e.g. "0644".
 func CopyFile(src, dst string, mode os.FileMode) (err error) {
 	var sfi os.FileInfo
-	if !IsHttpURL(src, false) {
+	if !IsDownloadableUri(src) {
 		sfi, err = os.Stat(src)
 		if err != nil {
 			return err
@@ -117,6 +125,18 @@ func IsHttpURL(s string, allowSchemaless bool) bool {
 	return err == nil && u.Host != ""
 }
 
+func IsFtpUri(s string) bool {
+	return strings.HasPrefix(s, "ftp://")
+}
+
+func IsScpUri(s string) bool {
+	return strings.HasPrefix(s, "scp://")
+}
+
+func IsDownloadableUri(s string) bool {
+	return IsHttpURL(s, false) || IsFtpUri(s) || IsScpUri(s)
+}
+
 // CopyFileContents copies the contents of the file named src to the file named
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents
@@ -125,7 +145,8 @@ func IsHttpURL(s string, allowSchemaless bool) bool {
 func CopyFileContents(src, dst string, mode os.FileMode) (err error) {
 	var in io.ReadCloser
 
-	if IsHttpURL(src, false) {
+	switch {
+	case IsHttpURL(src, false):
 		client := NewHTTPClient()
 
 		// download using client
@@ -135,7 +156,17 @@ func CopyFileContents(src, dst string, mode os.FileMode) (err error) {
 		}
 
 		in = resp.Body
-	} else {
+	case IsFtpUri(src):
+		in, err = processFtpUri(src)
+		if err != nil {
+			return err
+		}
+	case IsScpUri(src):
+		in, err = processScpUri(src)
+		if err != nil {
+			return err
+		}
+	default:
 		in, err = os.Open(src)
 		if err != nil {
 			return err
@@ -176,6 +207,142 @@ func CopyFileContents(src, dst string, mode os.FileMode) (err error) {
 	return err
 }
 
+func processScpUri(src string) (io.ReadCloser, error) {
+	// parse the scp url
+	u, err := url.Parse(src)
+	if err != nil {
+		return nil, err
+	}
+
+	// check username provided
+	if u.User == nil {
+		return nil, fmt.Errorf("no username provided for scp connection")
+	}
+	// try loading ssh agent keys
+	clientConfig, _ := auth.SshAgent(u.User.Username(), ssh.InsecureIgnoreHostKey())
+
+	// if CLAB_SSH_KEY is set we use the key referenced here
+	keyPath := os.Getenv("CLAB_SSH_KEY")
+	keyPassphrase := os.Getenv("CLAB_SSH_KEY_PASSPHRASE")
+	if keyPath != "" {
+		if !FileExists(keyPath) {
+			return nil, fmt.Errorf("keyfile %q does not exist", keyPath)
+		}
+		privateKey, err := ioutil.ReadFile(keyPath)
+		if err != nil {
+			return nil, err
+		}
+		var signer ssh.Signer
+		if keyPassphrase != "" {
+			// if keyPassphrase is set, use the withPassphrase method
+			signer, err = ssh.ParsePrivateKeyWithPassphrase(privateKey, []byte(keyPassphrase))
+		} else {
+			// otherwise use the basic ParsePrivateKey
+			signer, err = ssh.ParsePrivateKey(privateKey)
+		}
+		if err != nil {
+			return nil, err
+		}
+		clientConfig.Auth = append(clientConfig.Auth, ssh.PublicKeys(signer))
+	}
+
+	// if a password is set, use the password
+	// and make it the first item in the AuthMethods
+	pw, hasPW := u.User.Password()
+	if hasPW {
+		clientConfig.Auth = append([]ssh.AuthMethod{ssh.Password(pw)}, clientConfig.Auth...)
+	}
+
+	// set username in scp client config
+	clientConfig.User = u.User.Username()
+
+	// normalize host[host and port] portion
+	u.Host, _ = strings.CutSuffix(u.Host, ":")
+
+	// set port if not set
+	hostname := u.Hostname()
+	port := "22"
+	if u.Port() != "" {
+		port = u.Port()
+	}
+
+	// Create a new SCP client
+	client := scp.NewClient(fmt.Sprintf("%s:%s", hostname, port), &clientConfig)
+
+	// Connect to the remote server
+	err = client.Connect()
+	if err != nil {
+		return nil, fmt.Errorf("couldn't establish a connection to the remote server %w", err)
+	}
+	// create a temp file to store the downloaded content in
+	f, err := os.CreateTemp(os.TempDir(), "scp-")
+	if err != nil {
+		return nil, err
+	}
+	// copy the file content from remote to local
+	err = client.CopyFromRemote(context.Background(), f, u.Path)
+	if err != nil {
+		return nil, err
+	}
+	// reset the read/write pointer to the beginning of the file
+	_, err = f.Seek(0, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func processFtpUri(src string) (io.ReadCloser, error) {
+	// parse the ftp url
+	u, err := url.Parse(src)
+	if err != nil {
+		return nil, err
+	}
+
+	// set port if not set
+	hostname := u.Hostname()
+	port := "21"
+	if u.Port() != "" {
+		port = u.Port()
+	}
+
+	// establish connection
+	c, err := ftp.Dial(fmt.Sprintf("%s:%s", hostname, port), ftp.DialWithTimeout(5*time.Second))
+	if err != nil {
+		return nil, err
+	}
+
+	// is user is provided perform a login
+	if u.User != nil {
+		pw, _ := u.User.Password()
+		err = c.Login(u.User.Username(), pw)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// retrieve the file
+	r, err := c.Retr(u.Path)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	// read the data
+	buf, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := c.Quit(); err != nil {
+		return nil, err
+	}
+
+	// return the data
+	return io.NopCloser(bytes.NewBuffer(buf)), nil
+}
+
 // CreateFile writes content to a file by path `file`.
 func CreateFile(file, content string) (err error) {
 	var f *os.File
@@ -469,3 +636,59 @@ func recursiveChown(path string, uid, gid int) error {
 		return err
 	})
 }
+
+type DownloadFilesInterface interface {
+	ClabTmpDir() string
+	DownloadFileTmpAbsPath(nodeName string, filenamePostfix string) string
+}
+
+func ProcessDownloadableAndEmbeddedFile(nodename string, fileRef string, filenamePostfix string, paths DownloadFilesInterface) (string, error) {
+	var result string
+	// embedded config is a config that is defined as a multi-line string in the topology file
+	// it contains at least one newline
+	isEmbeddedConfig := strings.Count(fileRef, "\n") >= 1
+	// downloadable config starts with http(s)://
+	isDownloadableConfig := IsDownloadableUri(fileRef)
+
+	if isEmbeddedConfig || isDownloadableConfig {
+		// both embedded and downloadable configs are require clab tmp dir to be created
+		tmpLoc := paths.ClabTmpDir()
+		CreateDirectory(tmpLoc, 0755)
+
+		switch {
+		case isEmbeddedConfig:
+			log.Debugf("%q of node %q is an embedded blob", fileRef, nodename)
+			// for embedded config we create a file with the name embedded.partial.cfg
+			// as embedded configs are meant to be partial configs
+			absDestFile := paths.DownloadFileTmpAbsPath(
+				nodename, filenamePostfix)
+
+			err := CreateFile(absDestFile, fileRef)
+			if err != nil {
+				return "", err
+			}
+
+			result = absDestFile
+
+		case isDownloadableConfig:
+			log.Debugf("Node %q startup-config is a downloadable config %q", nodename, fileRef)
+			// get file name from an URL
+			fname := FilenameForURL(fileRef)
+
+			// Deduce the absolute destination filename for the downloaded content
+			absDestFile := paths.DownloadFileTmpAbsPath(nodename, fname)
+
+			log.Debugf("Fetching %q for node %q storing at %q", fileRef, nodename, absDestFile)
+			// download the file to tmp location
+			err := CopyFileContents(fileRef, absDestFile, 0755)
+			if err != nil {
+				return "", err
+			}
+
+			// adjust the nodeconfig by pointing startup-config to the local downloaded file
+			result = absDestFile
+		}
+		return result, nil
+	}
+	return fileRef, nil
+}

From 08bba855f13601db40d6950a2b34837cec66ac2f Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Wed, 31 May 2023 15:40:05 +0200
Subject: [PATCH 02/13] ssh hostkey warnings

---
 utils/file.go | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/utils/file.go b/utils/file.go
index 5653fe376..16b9cda1f 100644
--- a/utils/file.go
+++ b/utils/file.go
@@ -16,6 +16,7 @@ import (
 	"io/ioutil"
 	"math"
 	"mime"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -33,6 +34,7 @@ import (
 	"github.com/jlaffaye/ftp"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
 )
 
 var (
@@ -218,8 +220,9 @@ func processScpUri(src string) (io.ReadCloser, error) {
 	if u.User == nil {
 		return nil, fmt.Errorf("no username provided for scp connection")
 	}
+
 	// try loading ssh agent keys
-	clientConfig, _ := auth.SshAgent(u.User.Username(), ssh.InsecureIgnoreHostKey())
+	clientConfig, _ := auth.SshAgent(u.User.Username(), KnownHostsLogWarningCallBack) // skipcq: GSC-G106
 
 	// if CLAB_SSH_KEY is set we use the key referenced here
 	keyPath := os.Getenv("CLAB_SSH_KEY")
@@ -692,3 +695,24 @@ func ProcessDownloadableAndEmbeddedFile(nodename string, fileRef string, filenam
 	}
 	return fileRef, nil
 }
+
+// KnownHostsLogWarningCallBack is a KnownHostsCallback implementation that relies on the
+// implementation of the crypto package of golang. However, it will not error out if the
+// Hostkey is unknown, but issue a warning in the logs, if the host is unknown.
+func KnownHostsLogWarningCallBack(hostname string, remote net.Addr, key ssh.PublicKey) error {
+	// rely on the knownhosts implementation of the crypto package
+	// cretae an instance of it
+	khFile := ResolvePath("~/.ssh/known_hosts", "")
+	origKHCB, err := knownhosts.New(khFile)
+	if err != nil {
+		return err
+	}
+
+	// delegate the call
+	err = origKHCB(hostname, remote, key)
+	if err != nil {
+		// But if an error crops up, make it a warning and continue
+		log.Warnf("error while performing host key validation based on %q for hostname %q (%v). continuing anyways", khFile, hostname, err)
+	}
+	return nil
+}

From 99cf40e712d375330ef06eb72de8b93fbcf911e5 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Wed, 31 May 2023 15:50:23 +0200
Subject: [PATCH 03/13] remove ioutils

---
 go.mod        | 3 +++
 go.sum        | 7 +++++++
 utils/file.go | 5 ++---
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index f4f6468bf..033a6fea5 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/a8m/envsubst v1.4.2
 	github.com/awalterschulze/gographviz v2.0.3+incompatible
+	github.com/bramvdbogaerde/go-scp v1.2.1
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/containernetworking/plugins v1.4.0
 	github.com/containers/common v0.57.2
@@ -24,6 +25,7 @@ require (
 	github.com/h2non/gock v1.2.0
 	github.com/hairyhenderson/gomplate/v3 v3.11.6
 	github.com/hashicorp/go-version v1.6.0
+	github.com/jlaffaye/ftp v0.2.0
 	github.com/joho/godotenv v1.5.1
 	github.com/jsimonetti/rtnetlink v1.4.0
 	github.com/kellerza/template v0.0.6
@@ -38,6 +40,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/spf13/cobra v1.8.0
+	github.com/steiler/acls v0.1.1
 	github.com/stretchr/testify v1.8.4
 	github.com/tklauser/numcpus v0.7.0
 	github.com/vishvananda/netlink v1.2.1-beta.2
diff --git a/go.sum b/go.sum
index c7c9cea50..46fcbd8d2 100644
--- a/go.sum
+++ b/go.sum
@@ -919,6 +919,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dR
 github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
+github.com/bramvdbogaerde/go-scp v1.2.1 h1:BKTqrqXiQYovrDlfuVFaEGz0r4Ou6EED8L7jCXw6Buw=
+github.com/bramvdbogaerde/go-scp v1.2.1/go.mod h1:s4ZldBoRAOgUg8IrRP2Urmq5qqd2yPXQTPshACY8vQ0=
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
@@ -2039,6 +2041,8 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
 github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
+github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@@ -2717,6 +2721,8 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
+github.com/steiler/acls v0.1.1 h1:ghWAs51+psOLScMhaG6+23RCqXegtwnUqienhYT+l5g=
+github.com/steiler/acls v0.1.1/go.mod h1:Tpbo8Kj/6mQlon8pDVqh9PIsyvYbrLbCiXD9cUrpmGI=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
@@ -3031,6 +3037,7 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
diff --git a/utils/file.go b/utils/file.go
index 16b9cda1f..dd8a09e1d 100644
--- a/utils/file.go
+++ b/utils/file.go
@@ -13,7 +13,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"io/ioutil"
 	"math"
 	"mime"
 	"net"
@@ -231,7 +230,7 @@ func processScpUri(src string) (io.ReadCloser, error) {
 		if !FileExists(keyPath) {
 			return nil, fmt.Errorf("keyfile %q does not exist", keyPath)
 		}
-		privateKey, err := ioutil.ReadFile(keyPath)
+		privateKey, err := os.ReadFile(keyPath)
 		if err != nil {
 			return nil, err
 		}
@@ -333,7 +332,7 @@ func processFtpUri(src string) (io.ReadCloser, error) {
 	defer r.Close()
 
 	// read the data
-	buf, err := ioutil.ReadAll(r)
+	buf, err := io.ReadAll(r)
 	if err != nil {
 		return nil, err
 	}

From a77d7a961256e48a7109aae659c7b2a974bb5eaa Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Wed, 31 May 2023 15:52:33 +0200
Subject: [PATCH 04/13] revert libpod back to podman

---
 clab/clab.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/clab/clab.go b/clab/clab.go
index ecf5e55a7..2cd2f7b58 100644
--- a/clab/clab.go
+++ b/clab/clab.go
@@ -148,7 +148,7 @@ func (c *CLab) ProcessTopoPath(path string) (string, error) {
 
 	switch {
 	case path == "-" || path == "stdin":
-		file, err = readFromStdin(c.TopoPaths.ClabTmpDir())
+		file, err = c.readFromStdin()
 		if err != nil {
 			return "", err
 		}
@@ -215,8 +215,10 @@ func FindTopoFileByPath(path string) (string, error) {
 // readFromStdin reads the topology file from stdin
 // creates a temp file with topology contents
 // and returns a path to the temp file.
-func readFromStdin(tempDir string) (string, error) {
-	tmpFile, err := os.CreateTemp(tempDir, "topo-*.clab.yml")
+func (c *CLab) readFromStdin() (string, error) {
+	utils.CreateDirectory(c.TopoPaths.ClabTmpDir(), 0755)
+
+	tmpFile, err := os.CreateTemp(c.TopoPaths.ClabTmpDir(), "topo-*.clab.yml")
 	if err != nil {
 		return "", err
 	}

From 7a32c6d00eb8d927238301cd7fa0adbc9551634f Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Fri, 2 Jun 2023 09:50:19 +0200
Subject: [PATCH 05/13] improve HostKeyCallback

---
 utils/file.go | 52 +++++++++++++++++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/utils/file.go b/utils/file.go
index dd8a09e1d..cd09429b7 100644
--- a/utils/file.go
+++ b/utils/file.go
@@ -220,8 +220,10 @@ func processScpUri(src string) (io.ReadCloser, error) {
 		return nil, fmt.Errorf("no username provided for scp connection")
 	}
 
+	knownHostsPath := ResolvePath("~/.ssh/known_hosts", "")
+
 	// try loading ssh agent keys
-	clientConfig, _ := auth.SshAgent(u.User.Username(), KnownHostsLogWarningCallBack) // skipcq: GSC-G106
+	clientConfig, _ := auth.SshAgent(u.User.Username(), getCustomHostKeyCallback(knownHostsPath, "/tmp/foobar")) // skipcq: GSC-G106
 
 	// if CLAB_SSH_KEY is set we use the key referenced here
 	keyPath := os.Getenv("CLAB_SSH_KEY")
@@ -695,23 +697,41 @@ func ProcessDownloadableAndEmbeddedFile(nodename string, fileRef string, filenam
 	return fileRef, nil
 }
 
-// KnownHostsLogWarningCallBack is a KnownHostsCallback implementation that relies on the
-// implementation of the crypto package of golang. However, it will not error out if the
-// Hostkey is unknown, but issue a warning in the logs, if the host is unknown.
-func KnownHostsLogWarningCallBack(hostname string, remote net.Addr, key ssh.PublicKey) error {
-	// rely on the knownhosts implementation of the crypto package
-	// cretae an instance of it
-	khFile := ResolvePath("~/.ssh/known_hosts", "")
-	origKHCB, err := knownhosts.New(khFile)
-	if err != nil {
-		return err
+// getCustomHostKeyCallback returns a custom ssh.HostKeyCallback.
+// it will never block the connection, but issue a log.Warn if the
+// host_key cannot be found (due to absense of the entry or
+// the file being missing)
+func getCustomHostKeyCallback(knownHostsFiles ...string) ssh.HostKeyCallback {
+	var usefiles []string
+	// check
+	for _, file := range knownHostsFiles {
+		if !FileExists(file) {
+			log.Debugf("known_hosts file %s does not exist.", file)
+			continue
+		}
+		usefiles = append(usefiles, file)
 	}
 
-	// delegate the call
-	err = origKHCB(hostname, remote, key)
+	// load the known_hosts file retrieving an ssh.HostKeyCallback
+	knownHostsFileCallback, err := knownhosts.New(usefiles...)
 	if err != nil {
-		// But if an error crops up, make it a warning and continue
-		log.Warnf("error while performing host key validation based on %q for hostname %q (%v). continuing anyways", khFile, hostname, err)
+		log.Debugf("error loading known_hosts files %q", strings.Join(knownHostsFiles, ", "))
+		// this is an always failing knownHosts checker.
+		// it will make sure that the log message of the custom function further down
+		// is consistently logged. Meaning if file can't be loaded or entry does not exist.
+		knownHostsFileCallback = func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+			return fmt.Errorf("error loading known_hosts files %v", err)
+		}
+	}
+
+	// defien the custom ssh.HostKeyCallback function.
+	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		// delegate the call
+		err = knownHostsFileCallback(hostname, remote, key)
+		if err != nil {
+			// But if an error crops up, make it a warning and continue
+			log.Warnf("error performing host key validation based on %q for hostname %q (%v). continuing anyways", strings.Join(knownHostsFiles, ", "), hostname, err)
+		}
+		return nil
 	}
-	return nil
 }

From d3542b0a9f1e32151cc521b5531119aee4880fef Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Mon, 9 Oct 2023 10:31:14 +0200
Subject: [PATCH 06/13] adjust comment

---
 clab/config.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clab/config.go b/clab/config.go
index 014d73580..1c07d7c2e 100644
--- a/clab/config.go
+++ b/clab/config.go
@@ -226,7 +226,7 @@ func (c *CLab) createNodeCfg(nodeName string, nodeDef *types.NodeDefinition, idx
 	nodeCfg.EnforceStartupConfig = c.Config.Topology.GetNodeEnforceStartupConfig(nodeCfg.ShortName)
 	nodeCfg.SuppressStartupConfig = c.Config.Topology.GetNodeSuppressStartupConfig(nodeCfg.ShortName)
 
-	// process startup-config
+	// process NodeLicense
 	err = c.processNodeLicense(nodeCfg)
 	if err != nil {
 		return nil, err

From b616aa2e24ed891f04c88b5f2acaa21493359c46 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Wed, 25 Oct 2023 09:51:52 +0200
Subject: [PATCH 07/13] please deepsource

---
 utils/file.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/utils/file.go b/utils/file.go
index cd09429b7..cfeab1adf 100644
--- a/utils/file.go
+++ b/utils/file.go
@@ -223,7 +223,7 @@ func processScpUri(src string) (io.ReadCloser, error) {
 	knownHostsPath := ResolvePath("~/.ssh/known_hosts", "")
 
 	// try loading ssh agent keys
-	clientConfig, _ := auth.SshAgent(u.User.Username(), getCustomHostKeyCallback(knownHostsPath, "/tmp/foobar")) // skipcq: GSC-G106
+	clientConfig, _ := auth.SshAgent(u.User.Username(), getCustomHostKeyCallback(knownHostsPath)) // skipcq: GSC-G106
 
 	// if CLAB_SSH_KEY is set we use the key referenced here
 	keyPath := os.Getenv("CLAB_SSH_KEY")
@@ -271,7 +271,7 @@ func processScpUri(src string) (io.ReadCloser, error) {
 	}
 
 	// Create a new SCP client
-	client := scp.NewClient(fmt.Sprintf("%s:%s", hostname, port), &clientConfig)
+	client := scp.NewClient(net.JoinHostPort(hostname, port), &clientConfig)
 
 	// Connect to the remote server
 	err = client.Connect()
@@ -312,7 +312,7 @@ func processFtpUri(src string) (io.ReadCloser, error) {
 	}
 
 	// establish connection
-	c, err := ftp.Dial(fmt.Sprintf("%s:%s", hostname, port), ftp.DialWithTimeout(5*time.Second))
+	c, err := ftp.Dial(net.JoinHostPort(hostname, port), ftp.DialWithTimeout(5*time.Second))
 	if err != nil {
 		return nil, err
 	}

From 200d5e7273e6412b56ef06ca06abff0ea8681277 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Wed, 8 Nov 2023 08:51:50 +0100
Subject: [PATCH 08/13] fix create tempdir

---
 clab/clab.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/clab/clab.go b/clab/clab.go
index 2cd2f7b58..614e086ac 100644
--- a/clab/clab.go
+++ b/clab/clab.go
@@ -231,8 +231,10 @@ func (c *CLab) readFromStdin() (string, error) {
 	return tmpFile.Name(), nil
 }
 
-func downloadTopoFile(url, tempDir string) (string, error) {
-	tmpFile, err := os.CreateTemp(tempDir, "topo-*.clab.yml")
+func (c *CLab) downloadTopoFile(url string) (string, error) {
+	utils.CreateDirectory(c.TopoPaths.ClabTmpDir(), 0755)
+
+	tmpFile, err := os.CreateTemp(c.TopoPaths.ClabTmpDir(), "topo-*.clab.yml")
 	if err != nil {
 		return "", err
 	}

From 18df7a4a1e821c72799ccba1e907750c9974c757 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Wed, 8 Nov 2023 16:49:30 +0100
Subject: [PATCH 09/13] tests

---
 tests/09-download/01-download.robot      | 59 ++++++++++++++++++++++++
 tests/09-download/01-downloads.clab.yaml |  9 ++++
 tests/09-download/Dockerfile             |  9 ++++
 tests/09-download/docker-entrypoint.sh   |  5 ++
 utils/file.go                            | 22 +++++++--
 5 files changed, 100 insertions(+), 4 deletions(-)
 create mode 100644 tests/09-download/01-download.robot
 create mode 100644 tests/09-download/01-downloads.clab.yaml
 create mode 100644 tests/09-download/Dockerfile
 create mode 100755 tests/09-download/docker-entrypoint.sh

diff --git a/tests/09-download/01-download.robot b/tests/09-download/01-download.robot
new file mode 100644
index 000000000..3bc4ba8dd
--- /dev/null
+++ b/tests/09-download/01-download.robot
@@ -0,0 +1,59 @@
+*** Settings ***
+Library             OperatingSystem
+Library             String
+Library             Process
+Resource            ../common.robot
+Suite Setup         Setup
+Suite Teardown      Teardown
+
+
+*** Variables ***
+${lab-file}                 01-downloads.clab.yaml
+${lab-name}                 01-downloads
+${idrsa}                    ${CURDIR}/id_rsa
+${serverhelpername}         server_helper
+${lic_text}                 this is the fake license
+# runtime command to execute tasks in a container
+# defaults to docker exec. Will be rewritten to containerd `ctr` if needed in "Define runtime exec" test
+${runtime-cli-exec-cmd}     sudo docker exec
+
+
+*** Test Cases ***
+
+Deploy helper container
+    Run    sudo docker run -d --name ${serverhelpername} dlc
+    Run    sudo docker exec ${serverhelpername} mkdir -p /root/.ssh/
+    Run    cat ${idrsa}.pub | sudo docker exec -i ${serverhelpername} tee /root/.ssh/authorized_keys -
+    Run    echo "${lic_text}" | sudo docker exec -i ${serverhelpername} tee /root/lic.txt -
+    ${rc}    ${server_ip} =    Run And Return Rc And Output
+    ...    sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${serverhelpername}
+    Log    ${server_ip}
+    Set Suite Variable    ${server_ip}    ${server_ip}
+    Set Environment Variable     CLAB_SSH_KEY    ${idrsa}
+    Set Environment Variable     SERVER_IP    ${server_ip}
+
+Deploy ${lab-name} lab
+    Log    ${CURDIR}
+    ${rc}    ${output} =    Run And Return Rc And Output
+    ...    cat ${CURDIR}/${lab-file}| envsubst | sudo -E ${CLAB_BIN} --runtime ${runtime} deploy -t -
+    Log    ${output}
+    Should Be Equal As Integers    ${rc}    0
+    # save output to be used in next steps
+    Set Suite Variable    ${deploy-output}    ${output}
+
+
+Check license
+    ${rc}    ${output} =   Run And Return Rc And Output     cat /tmp/.clab/${lab-name}-l1-lic.txt
+    Should Contain    ${output}    ${lic_text}
+
+
+*** Keywords ***
+Teardown
+    Run    sudo -E ${CLAB_BIN} --runtime ${runtime} destroy -t ${CURDIR}/01-downloads.clab.yaml --cleanup
+    Run    rm ${CURDIR}/id_rsa*
+    Run    sudo docker stop ${serverhelpername}
+    Run    sudo docker rm ${serverhelpername}
+    Run    ssh-keygen -f "~/.ssh/known_hosts" -R ${server_ip}
+
+Setup
+    Run    ssh-keygen -t ssh-rsa -f ${idrsa} -N ""
\ No newline at end of file
diff --git a/tests/09-download/01-downloads.clab.yaml b/tests/09-download/01-downloads.clab.yaml
new file mode 100644
index 000000000..31c6e48e2
--- /dev/null
+++ b/tests/09-download/01-downloads.clab.yaml
@@ -0,0 +1,9 @@
+name: 01-downloads
+
+topology:
+  nodes:
+    l1:
+      kind: linux
+      image: alpine:edge
+      license: scp://root@${SERVER_IP}/root/lic.txt
+      cmd: sleep infinity
\ No newline at end of file
diff --git a/tests/09-download/Dockerfile b/tests/09-download/Dockerfile
new file mode 100644
index 000000000..16f9311f6
--- /dev/null
+++ b/tests/09-download/Dockerfile
@@ -0,0 +1,9 @@
+FROM alpine:edge
+
+RUN apk add openssh py3-twisted && \
+    ssh-keygen -A  && \
+    echo "done"
+
+ADD docker-entrypoint.sh /usr/local/bin
+
+CMD ["docker-entrypoint.sh"]
diff --git a/tests/09-download/docker-entrypoint.sh b/tests/09-download/docker-entrypoint.sh
new file mode 100755
index 000000000..f8f984745
--- /dev/null
+++ b/tests/09-download/docker-entrypoint.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+# start the ssh daemon
+twistd -n ftp -p 21 -r /root/ &
+/usr/sbin/sshd &
+sleep infinity
\ No newline at end of file
diff --git a/utils/file.go b/utils/file.go
index cfeab1adf..97a72c2f9 100644
--- a/utils/file.go
+++ b/utils/file.go
@@ -29,10 +29,10 @@ import (
 	"github.com/steiler/acls"
 
 	"github.com/bramvdbogaerde/go-scp"
-	"github.com/bramvdbogaerde/go-scp/auth"
 	"github.com/jlaffaye/ftp"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/crypto/ssh/knownhosts"
 )
 
@@ -222,8 +222,22 @@ func processScpUri(src string) (io.ReadCloser, error) {
 
 	knownHostsPath := ResolvePath("~/.ssh/known_hosts", "")
 
-	// try loading ssh agent keys
-	clientConfig, _ := auth.SshAgent(u.User.Username(), getCustomHostKeyCallback(knownHostsPath)) // skipcq: GSC-G106
+	clientConfig := ssh.ClientConfig{
+		User:            u.User.Username(),
+		Auth:            []ssh.AuthMethod{},
+		HostKeyCallback: getCustomHostKeyCallback(knownHostsPath),
+	}
+
+	// if we have an ssh agent running use it.
+	if socket := os.Getenv("SSH_AUTH_SOCK"); socket != "" {
+		conn, err := net.Dial("unix", socket)
+		if err != nil {
+			log.Error(err)
+		} else {
+			agentClient := agent.NewClient(conn)
+			clientConfig.Auth = append(clientConfig.Auth, ssh.PublicKeysCallback(agentClient.Signers))
+		}
+	}
 
 	// if CLAB_SSH_KEY is set we use the key referenced here
 	keyPath := os.Getenv("CLAB_SSH_KEY")
@@ -247,7 +261,7 @@ func processScpUri(src string) (io.ReadCloser, error) {
 		if err != nil {
 			return nil, err
 		}
-		clientConfig.Auth = append(clientConfig.Auth, ssh.PublicKeys(signer))
+		clientConfig.Auth = append([]ssh.AuthMethod{ssh.PublicKeys(signer)}, clientConfig.Auth...)
 	}
 
 	// if a password is set, use the password

From a61ccf5af4fac08fa9a7824785fcec5e560f8076 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Thu, 9 Nov 2023 15:35:13 +0100
Subject: [PATCH 10/13] test ftp, scp, http

---
 tests/09-download/01-download.robot      | 62 ++++++++++++++++--------
 tests/09-download/01-downloads.clab.yaml | 12 ++++-
 tests/09-download/Dockerfile             |  9 ----
 tests/09-download/docker-entrypoint.sh   |  5 --
 utils/file.go                            |  6 +--
 5 files changed, 57 insertions(+), 37 deletions(-)
 delete mode 100644 tests/09-download/Dockerfile
 delete mode 100755 tests/09-download/docker-entrypoint.sh

diff --git a/tests/09-download/01-download.robot b/tests/09-download/01-download.robot
index 3bc4ba8dd..1e8a1b01e 100644
--- a/tests/09-download/01-download.robot
+++ b/tests/09-download/01-download.robot
@@ -11,7 +11,9 @@ Suite Teardown      Teardown
 ${lab-file}                 01-downloads.clab.yaml
 ${lab-name}                 01-downloads
 ${idrsa}                    ${CURDIR}/id_rsa
-${serverhelpername}         server_helper
+${scpservername}                scp_server
+${ftpservername}                ftp_server
+${httpservername}                http_server
 ${lic_text}                 this is the fake license
 # runtime command to execute tasks in a container
 # defaults to docker exec. Will be rewritten to containerd `ctr` if needed in "Define runtime exec" test
@@ -19,41 +21,63 @@ ${runtime-cli-exec-cmd}     sudo docker exec
 
 
 *** Test Cases ***
-
-Deploy helper container
-    Run    sudo docker run -d --name ${serverhelpername} dlc
-    Run    sudo docker exec ${serverhelpername} mkdir -p /root/.ssh/
-    Run    cat ${idrsa}.pub | sudo docker exec -i ${serverhelpername} tee /root/.ssh/authorized_keys -
-    Run    echo "${lic_text}" | sudo docker exec -i ${serverhelpername} tee /root/lic.txt -
-    ${rc}    ${server_ip} =    Run And Return Rc And Output
-    ...    sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${serverhelpername}
-    Log    ${server_ip}
-    Set Suite Variable    ${server_ip}    ${server_ip}
+Deploy helper container - SCP
+    ${rc}    ${sshpubkey} =    Run And Return Rc And Output   sudo cat ${idrsa}.pub
+    Run    sudo docker run -d --name=${scpservername} -e USER_NAME=user -e PUBLIC_KEY=\"${sshpubkey}\" --restart unless-stopped lscr.io/linuxserver/openssh-server:latest
+    #Run    cat ${idrsa}.pub | sudo docker exec -i ${scpservername} tee /root/.ssh/authorized_keys -
+    Run    echo "${lic_text} scp" | sudo docker exec -i ${scpservername} tee /config/lic.txt -
+    ${rc}    ${scp_server_ip} =    Run And Return Rc And Output
+    ...    sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${scpservername}
+    Log    ${scp_server_ip}
+    Set Suite Variable    ${scp_server_ip}    ${scp_server_ip}
     Set Environment Variable     CLAB_SSH_KEY    ${idrsa}
-    Set Environment Variable     SERVER_IP    ${server_ip}
+    Set Environment Variable     SCP_SERVER_IP    ${scp_server_ip}
+
+Deploy helper container - FTP
+    Run    docker run -d --name ${ftpservername} lhauspie/vsftpd-alpine
+    ${rc}    ${ftp_server_ip} =    Run And Return Rc And Output
+    ...    sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${ftpservername}
+    Log    ${ftp_server_ip}
+    Set Suite Variable    ${ftp_server_ip}    ${ftp_server_ip}
+    Set Environment Variable     FTP_SERVER_IP    ${ftp_server_ip}
+    Run    sudo docker exec -i ${ftpservername} mkdir -p /home/vsftpd/user/
+    Run    echo "${lic_text} ftp" | sudo docker exec -i ${ftpservername} tee /home/vsftpd/user/lic.txt -
+
+Deploy helper container - HTTP
+    Run    docker run -d --name ${httpservername} httpd:2.4
+    Run    echo "${lic_text} http" | sudo docker exec -i ${httpservername} tee /usr/local/apache2/htdocs/lic.txt -
+    ${rc}    ${http_server_ip} =    Run And Return Rc And Output
+    ...    sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${httpservername}
+    Log    ${http_server_ip}
+    Set Suite Variable    ${http_server_ip}    ${http_server_ip}
+    Set Environment Variable     HTTP_SERVER_IP    ${http_server_ip}
+
 
 Deploy ${lab-name} lab
     Log    ${CURDIR}
     ${rc}    ${output} =    Run And Return Rc And Output
-    ...    cat ${CURDIR}/${lab-file}| envsubst | sudo -E ${CLAB_BIN} --runtime ${runtime} deploy -t -
+    ...    cat ${CURDIR}/${lab-file}| envsubst | tee ${CURDIR}/rendered.clab.yml | sudo -E ${CLAB_BIN} --runtime ${runtime} deploy -t -
     Log    ${output}
     Should Be Equal As Integers    ${rc}    0
     # save output to be used in next steps
     Set Suite Variable    ${deploy-output}    ${output}
 
 
-Check license
+Check licenses
     ${rc}    ${output} =   Run And Return Rc And Output     cat /tmp/.clab/${lab-name}-l1-lic.txt
-    Should Contain    ${output}    ${lic_text}
-
+    Should Contain    ${output}    ${lic_text} scp
+    ${rc}    ${output} =   Run And Return Rc And Output     cat /tmp/.clab/${lab-name}-l2-lic.txt
+    Should Contain    ${output}    ${lic_text} ftp
+    ${rc}    ${output} =   Run And Return Rc And Output     cat /tmp/.clab/${lab-name}-l3-lic.txt
+    Should Contain    ${output}    ${lic_text} http
 
 *** Keywords ***
 Teardown
     Run    sudo -E ${CLAB_BIN} --runtime ${runtime} destroy -t ${CURDIR}/01-downloads.clab.yaml --cleanup
     Run    rm ${CURDIR}/id_rsa*
-    Run    sudo docker stop ${serverhelpername}
-    Run    sudo docker rm ${serverhelpername}
-    Run    ssh-keygen -f "~/.ssh/known_hosts" -R ${server_ip}
+    Run    sudo docker rm -f ${scpservername} ${ftpservername} ${httpservername}
+    Run    ssh-keygen -f "~/.ssh/known_hosts" -R ${scp_server_ip}
+    Run    sudo rm -rf /tmp/.clab/${lab-name}-*
 
 Setup
     Run    ssh-keygen -t ssh-rsa -f ${idrsa} -N ""
\ No newline at end of file
diff --git a/tests/09-download/01-downloads.clab.yaml b/tests/09-download/01-downloads.clab.yaml
index 31c6e48e2..6d276e54b 100644
--- a/tests/09-download/01-downloads.clab.yaml
+++ b/tests/09-download/01-downloads.clab.yaml
@@ -5,5 +5,15 @@ topology:
     l1:
       kind: linux
       image: alpine:edge
-      license: scp://root@${SERVER_IP}/root/lic.txt
+      license: scp://user@${SCP_SERVER_IP}:2222/config/lic.txt
+      cmd: sleep infinity
+    l2:
+      kind: linux
+      image: alpine:edge
+      license: ftp://user:pass@${FTP_SERVER_IP}/lic.txt
+      cmd: sleep infinity
+    l3:
+      kind: linux
+      image: alpine:edge
+      license: http://${HTTP_SERVER_IP}/lic.txt
       cmd: sleep infinity
\ No newline at end of file
diff --git a/tests/09-download/Dockerfile b/tests/09-download/Dockerfile
deleted file mode 100644
index 16f9311f6..000000000
--- a/tests/09-download/Dockerfile
+++ /dev/null
@@ -1,9 +0,0 @@
-FROM alpine:edge
-
-RUN apk add openssh py3-twisted && \
-    ssh-keygen -A  && \
-    echo "done"
-
-ADD docker-entrypoint.sh /usr/local/bin
-
-CMD ["docker-entrypoint.sh"]
diff --git a/tests/09-download/docker-entrypoint.sh b/tests/09-download/docker-entrypoint.sh
deleted file mode 100755
index f8f984745..000000000
--- a/tests/09-download/docker-entrypoint.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-# start the ssh daemon
-twistd -n ftp -p 21 -r /root/ &
-/usr/sbin/sshd &
-sleep infinity
\ No newline at end of file
diff --git a/utils/file.go b/utils/file.go
index 97a72c2f9..1f2c559db 100644
--- a/utils/file.go
+++ b/utils/file.go
@@ -160,17 +160,17 @@ func CopyFileContents(src, dst string, mode os.FileMode) (err error) {
 	case IsFtpUri(src):
 		in, err = processFtpUri(src)
 		if err != nil {
-			return err
+			return fmt.Errorf("failure retrieving file %s: %v", src, err)
 		}
 	case IsScpUri(src):
 		in, err = processScpUri(src)
 		if err != nil {
-			return err
+			return fmt.Errorf("failure retrieving file %s: %v", src, err)
 		}
 	default:
 		in, err = os.Open(src)
 		if err != nil {
-			return err
+			return fmt.Errorf("failure retrieving file %s: %v", src, err)
 		}
 	}
 	defer in.Close() // skipcq: GO-S2307

From 84b58218773a0ed34ad78068e5a9cd040bc43099 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Fri, 10 Nov 2023 10:21:51 +0100
Subject: [PATCH 11/13] enable downloads cicd tests

---
 .github/workflows/cicd.yml           |  7 ++++
 .github/workflows/download-tests.yml | 63 ++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 .github/workflows/download-tests.yml

diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
index 7b9e5b50e..34a43c8b2 100644
--- a/.github/workflows/cicd.yml
+++ b/.github/workflows/cicd.yml
@@ -314,6 +314,13 @@ jobs:
     with:
       py_ver: ${{ needs.file-changes.outputs.py_ver }}
 
+  download-tests:
+    uses: ./.github/workflows/download-tests.yml
+    needs:
+      - unit-test
+      - staticcheck
+      - build-containerlab
+
   # a job that downloads coverage artifact and uses codecov to upload it
   coverage:
     runs-on: ubuntu-22.04
diff --git a/.github/workflows/download-tests.yml b/.github/workflows/download-tests.yml
new file mode 100644
index 000000000..b96c24a23
--- /dev/null
+++ b/.github/workflows/download-tests.yml
@@ -0,0 +1,63 @@
+name: vxlan-test
+
+"on":
+  workflow_call:
+
+jobs:
+  vxlan-tests:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        runtime:
+          - "docker"
+        test-suite:
+          - "*.robot"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: containerlab
+
+      - name: Move containerlab to usr/bin
+        run: sudo mv ./containerlab /usr/bin/containerlab && sudo chmod a+x /usr/bin/containerlab
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: "tests/requirements.txt"
+
+      - name: Install robotframework
+        run: |
+          pip install -r tests/requirements.txt
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run tests
+        run: |
+          bash ./tests/rf-run.sh ${{ matrix.runtime }} ./tests/09-download/${{ matrix.test-suite }}
+
+      # upload test reports as a zip file
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: 08-${{ matrix.runtime }}-vxlan-log
+          path: ./tests/out/*.html
+
+      # upload coverage report from unit tests, as they are then
+      # merged with e2e tests coverage
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: coverage
+          path: /tmp/clab-tests/coverage/*
+          retention-days: 7

From 1f5f55242196a32716a993386182f5a4a4d1a795 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Fri, 10 Nov 2023 10:28:41 +0100
Subject: [PATCH 12/13] fix test names

---
 .github/workflows/download-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/download-tests.yml b/.github/workflows/download-tests.yml
index b96c24a23..67b0ba6ab 100644
--- a/.github/workflows/download-tests.yml
+++ b/.github/workflows/download-tests.yml
@@ -1,10 +1,10 @@
-name: vxlan-test
+name: download-test
 
 "on":
   workflow_call:
 
 jobs:
-  vxlan-tests:
+  download-tests:
     runs-on: ubuntu-22.04
     strategy:
       matrix:

From 93bf471359fde62ccd13b158b3002ba9f6d38808 Mon Sep 17 00:00:00 2001
From: steiler <github@vahlenkamp.net>
Date: Tue, 23 Jan 2024 15:16:01 +0100
Subject: [PATCH 13/13] update

---
 clab/clab.go        | 2 +-
 types/topo_paths.go | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/clab/clab.go b/clab/clab.go
index 614e086ac..01faf083a 100644
--- a/clab/clab.go
+++ b/clab/clab.go
@@ -154,7 +154,7 @@ func (c *CLab) ProcessTopoPath(path string) (string, error) {
 		}
 	// if the path is not a local file and a URL, download the file and store it in the tmp dir
 	case !utils.FileOrDirExists(path) && utils.IsHttpURL(path, true):
-		file, err = downloadTopoFile(path, c.TopoPaths.ClabTmpDir())
+		file, err = c.downloadTopoFile(path)
 		if err != nil {
 			return "", err
 		}
diff --git a/types/topo_paths.go b/types/topo_paths.go
index d1c93ef6f..9d9cb8042 100644
--- a/types/topo_paths.go
+++ b/types/topo_paths.go
@@ -183,6 +183,12 @@ func (*TopoPaths) ClabTmpDir() string {
 	return clabTmpDir
 }
 
+// StartupConfigDownloadFileAbsPath returns the absolute path to the startup-config file
+// when it is downloaded from a remote location to the clab temp directory.
+func (t *TopoPaths) StartupConfigDownloadFileAbsPath(node, postfix string) string {
+	return filepath.Join(t.ClabTmpDir(), fmt.Sprintf("%s-%s-%s", t.topoName, node, postfix))
+}
+
 // DownloadFileTmpAbsPath returns the absolute path to a file
 // when it is downloaded from a remote location to the clab temp directory.
 func (t *TopoPaths) DownloadFileTmpAbsPath(node string, postfix string) string {