Skip to content

Commit

Permalink
Nydusify localfs support
Browse files Browse the repository at this point in the history 
Signed-off-by: Mike Hotan <[email protected]>
  • Loading branch information
@mhotan
mhotan committed Oct 16, 2024
1 parent da2510b commit 3fc325c
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 90 deletions.
4 changes: 2 additions & 2 deletions contrib/nydusify/cmd/nydusify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func getBackendConfig(c *cli.Context, prefix string, required bool) (string, str
return "", "", nil
}

possibleBackendTypes := []string{"oss", "s3"}
possibleBackendTypes := []string{"oss", "s3", "localfs"}
if !isPossibleValue(possibleBackendTypes, backendType) {
return "", "", fmt.Errorf("--%sbackend-type should be one of %v", prefix, possibleBackendTypes)
}
Expand All @@ -89,7 +89,7 @@ func getBackendConfig(c *cli.Context, prefix string, required bool) (string, str
)
if err != nil {
return "", "", err
} else if (backendType == "oss" || backendType == "s3") && strings.TrimSpace(backendConfig) == "" {
} else if (backendType == "oss" || backendType == "s3" || backendType == "localfs") && strings.TrimSpace(backendConfig) == "" {
return "", "", errors.Errorf("backend configuration is empty, please specify option '--%sbackend-config'", prefix)
}

Expand Down
193 changes: 107 additions & 86 deletions contrib/nydusify/cmd/nydusify_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"testing"

Expand Down Expand Up @@ -80,100 +81,120 @@ func TestParseBackendConfig(t *testing.T) {
}

func TestGetBackendConfig(t *testing.T) {
app := &cli.App{
Flags: []cli.Flag{
&cli.StringFlag{
Name: "prefixbackend-type",
Value: "",
},
&cli.StringFlag{
Name: "prefixbackend-config",
Value: "",
},
&cli.StringFlag{
Name: "prefixbackend-config-file",
Value: "",
},
},
}
ctx := cli.NewContext(app, nil, nil)

backendType, backendConfig, err := getBackendConfig(ctx, "prefix", false)
require.NoError(t, err)
require.Empty(t, backendType)
require.Empty(t, backendConfig)

backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.Error(t, err)
require.Contains(t, err.Error(), "backend type is empty, please specify option")
require.Empty(t, backendType)
require.Empty(t, backendConfig)

flagSet := flag.NewFlagSet("test1", flag.PanicOnError)
flagSet.String("prefixbackend-type", "errType", "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.Error(t, err)
require.Contains(t, err.Error(), "backend-type should be one of")
require.Empty(t, backendType)
require.Empty(t, backendConfig)

flagSet = flag.NewFlagSet("test2", flag.PanicOnError)
flagSet.String("prefixbackend-type", "oss", "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.Error(t, err)
require.Contains(t, err.Error(), "backend configuration is empty, please specify option")
require.Empty(t, backendType)
require.Empty(t, backendConfig)

configJSON := `
tests := []struct {
backendType string
backendConfig string
}{
{
backendType: "oss",
backendConfig: `
{
"bucket_name": "test",
"endpoint": "region.oss.com",
"access_key_id": "testAK",
"access_key_secret": "testSK",
"meta_prefix": "meta",
"blob_prefix": "blob"
}`
require.True(t, json.Valid([]byte(configJSON)))

flagSet = flag.NewFlagSet("test3", flag.PanicOnError)
flagSet.String("prefixbackend-type", "oss", "")
flagSet.String("prefixbackend-config", configJSON, "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.NoError(t, err)
require.Equal(t, "oss", backendType)
require.Equal(t, configJSON, backendConfig)

file, err := os.CreateTemp("", "nydusify-backend-config-test.json")
require.NoError(t, err)
defer os.RemoveAll(file.Name())

_, err = file.WriteString(configJSON)
require.NoError(t, err)
file.Sync()

flagSet = flag.NewFlagSet("test4", flag.PanicOnError)
flagSet.String("prefixbackend-type", "oss", "")
flagSet.String("prefixbackend-config-file", file.Name(), "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.NoError(t, err)
require.Equal(t, "oss", backendType)
require.Equal(t, configJSON, backendConfig)
}`,
},
{
backendType: "localfs",
backendConfig: `
{
"dir": "/path/to/blobs"
}`,
},
}

flagSet = flag.NewFlagSet("test5", flag.PanicOnError)
flagSet.String("prefixbackend-type", "oss", "")
flagSet.String("prefixbackend-config", configJSON, "")
flagSet.String("prefixbackend-config-file", file.Name(), "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.Error(t, err)
require.Contains(t, err.Error(), "--backend-config conflicts with --backend-config-file")
require.Empty(t, backendType)
require.Empty(t, backendConfig)
for _, test := range tests {
t.Run(fmt.Sprintf("backend config %s", test.backendType), func(t *testing.T) {
app := &cli.App{
Flags: []cli.Flag{
&cli.StringFlag{
Name: "prefixbackend-type",
Value: "",
},
&cli.StringFlag{
Name: "prefixbackend-config",
Value: "",
},
&cli.StringFlag{
Name: "prefixbackend-config-file",
Value: "",
},
},
}
ctx := cli.NewContext(app, nil, nil)

backendType, backendConfig, err := getBackendConfig(ctx, "prefix", false)
require.NoError(t, err)
require.Empty(t, backendType)
require.Empty(t, backendConfig)

backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.Error(t, err)
require.Contains(t, err.Error(), "backend type is empty, please specify option")
require.Empty(t, backendType)
require.Empty(t, backendConfig)

flagSet := flag.NewFlagSet("test1", flag.PanicOnError)
flagSet.String("prefixbackend-type", "errType", "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.Error(t, err)
require.Contains(t, err.Error(), "backend-type should be one of")
require.Empty(t, backendType)
require.Empty(t, backendConfig)

flagSet = flag.NewFlagSet("test2", flag.PanicOnError)
flagSet.String("prefixbackend-type", test.backendType, "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.Error(t, err)
require.Contains(t, err.Error(), "backend configuration is empty, please specify option")
require.Empty(t, backendType)
require.Empty(t, backendConfig)

require.True(t, json.Valid([]byte(test.backendConfig)))

flagSet = flag.NewFlagSet("test3", flag.PanicOnError)
flagSet.String("prefixbackend-type", test.backendType, "")
flagSet.String("prefixbackend-config", test.backendConfig, "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.NoError(t, err)
require.Equal(t, test.backendType, backendType)
require.Equal(t, test.backendConfig, backendConfig)

file, err := os.CreateTemp("", "nydusify-backend-config-test.json")
require.NoError(t, err)
defer os.RemoveAll(file.Name())

_, err = file.WriteString(test.backendConfig)
require.NoError(t, err)
file.Sync()

flagSet = flag.NewFlagSet("test4", flag.PanicOnError)
flagSet.String("prefixbackend-type", test.backendType, "")
flagSet.String("prefixbackend-config-file", file.Name(), "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.NoError(t, err)
require.Equal(t, test.backendType, backendType)
require.Equal(t, test.backendConfig, backendConfig)

flagSet = flag.NewFlagSet("test5", flag.PanicOnError)
flagSet.String("prefixbackend-type", test.backendType, "")
flagSet.String("prefixbackend-config", test.backendConfig, "")
flagSet.String("prefixbackend-config-file", file.Name(), "")
ctx = cli.NewContext(app, flagSet, nil)
backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true)
require.Error(t, err)
require.Contains(t, err.Error(), "--backend-config conflicts with --backend-config-file")
require.Empty(t, backendType)
require.Empty(t, backendConfig)
})
}
}

func TestGetTargetReference(t *testing.T) {
Expand Down
20 changes: 18 additions & 2 deletions docs/nydusify.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,22 @@ nydusify convert \
--backend-config-file /path/to/backend-config.json
```

### localfs

``` shell
cat /path/to/backend-config.json
{
"dir": "/path/to/blobs"
}

nydusify convert \
--source myregistry/repo:tag \
--target myregistry/repo:tag-nydus \
--backend-config-file /path/to/backend-config.json \
```

Note: Image manifest is still published to target registry (`myregistry`). Blob files are published to localfs.

## Push Nydus Image to storage backend with subcommand pack

### OSS
Expand Down Expand Up @@ -176,7 +192,7 @@ nydusify check \

## Mount the nydus image as a filesystem

The nydusify mount command can mount a nydus image stored in the backend as a filesystem. Now the supported backend types include Registry (default backend), s3 and oss.
The nydusify mount command can mount a nydus image stored in the backend as a filesystem. Now the supported backend types include Registry (default backend), s3, oss, and localfs.

When using Registry as the backend, you don't need to specify the `--backend-type` .

Expand Down Expand Up @@ -252,7 +268,7 @@ See `nydusify convert/check/mount --help`

## Use Nydusify as a package

```
```
See `contrib/nydusify/examples/converter/main.go`
```

Expand Down

0 comments on commit 3fc325c

Please sign in to comment.