Make sure we include file support for blobs. (#113)

In the localfile client augment cp so it can take file:// style buckets to do local copies on machines (remotely).
Snowflake-Labs · May 9, 2022 · cc3a0be · cc3a0be
1 parent 37e385b
commit cc3a0be
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 3 deletions.
diff --git a/cmd/sansshell-server/main.go b/cmd/sansshell-server/main.go
@@ -33,6 +33,7 @@ import (
 	"github.com/go-logr/stdr"
 
 	_ "gocloud.dev/blob/azureblob" // Pull in Azure blob support
+	_ "gocloud.dev/blob/fileblob"  // Pull in file blob support
 	_ "gocloud.dev/blob/gcsblob"   // Pull in GCS blob support
 	_ "gocloud.dev/blob/s3blob"    // Pull in S3 blob support
 

diff --git a/services/localfile/client/client.go b/services/localfile/client/client.go
@@ -798,11 +798,14 @@ type cpCmd struct {
 }
 
 func (*cpCmd) Name() string     { return "cp" }
-func (*cpCmd) Synopsis() string { return "Copy a file onto a remote machine." }
+func (*cpCmd) Synopsis() string { return "Copy a file on/onto a remote machine." }
 func (*cpCmd) Usage() string {
 	return `cp [--bucket=XXX] [--overwrite] --uid=X|username=Y --gid=X|group=Y --mode=X [--immutable] <source> <remote destination>
-  Copy the source file (which can be local or a URL such as s3://bucket/source) to the target(s)
+  Copy the source file (which can be local or a URL such as --bucket=s3://bucket <source> or --bucket=file://directory <source>) to the target(s)
   placing it into the remote destination.
+
+NOTE: Using file:// means the file must be in that location on each remote target in turn as no data is transferred in that case. Also make
+sure to use a fully formed directory. i.e. copying /etc/hosts would be --bucket=file:///etc hosts <destination>
 `
 }
 
@@ -821,6 +824,7 @@ var validOutputPrefixes = []string{
 	"s3://",
 	"azblob://",
 	"gs://",
+	"file://",
 }
 
 func (p *cpCmd) Execute(ctx context.Context, f *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {

diff --git a/testing/integrate.sh b/testing/integrate.sh
@@ -736,7 +736,7 @@ check_perms_mode ${LOGS}/cp-hosts
 
 # Skip if on github
 if [ -z "${ON_GITHUB}" ]; then
-  echo cp test with bucket syntax
+  echo cp test with s3 bucket syntax
   aws s3 cp ${LOGS}/hosts s3://${USER}-dev/hosts
   run_a_test false 0 file cp --overwrite --uid=${EXPECTED_NEW_UID} --gid=${EXPECTED_NEW_GID} --mode=${EXPECTED_NEW_MODE} --bucket=s3://${USER}-dev?region=us-west-2 hosts ${LOGS}/cp-hosts
   check_perms_mode ${LOGS}/cp-hosts
@@ -745,6 +745,11 @@ else
   echo "Skipping cp with s3 on Github"
 fi
 
+# Always test file syntax
+echo "Now we use file:// format"
+run_a_test false 0 file cp --overwrite --uid=${EXPECTED_NEW_UID} --gid=${EXPECTED_NEW_GID} --mode=${EXPECTED_NEW_MODE} --bucket=file:///etc hosts ${LOGS}/cp-hosts
+check_perms_mode ${LOGS}/cp-hosts
+
 # Trying without --overwrite to validate
 echo "This can emit an error message about overwrite"
 ${SANSSH_NOPROXY} ${SINGLE_TARGET} --outputs=${LOGS}/1.cp-no-overwrite file cp --uid=${EXPECTED_NEW_UID} --gid=${EXPECTED_NEW_GID} --mode=${EXPECTED_NEW_MODE} ${LOGS}/hosts ${LOGS}/cp-hosts