From 80c2abe0eed3eed4de35716df2e6f2c148c15b6b Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 21 Nov 2024 12:18:43 -0500 Subject: [PATCH] dynamic host volumes: monitor readiness from CLI When creating a dynamic host volumes, set up an optional monitor that waits for the node to fingerprint the volume as healthy. Ref: https://github.com/hashicorp/nomad/pull/24479 --- command/volume_create.go | 22 +++- command/volume_create_host.go | 151 ++++++++++++++++++++++++++-- command/volume_create_host_test.go | 2 +- command/volume_status_host.go | 64 +++++++----- demo/hostvolume/example-host-volume | 2 +- 5 files changed, 208 insertions(+), 33 deletions(-) diff --git a/command/volume_create.go b/command/volume_create.go index 258e37b1f47..f566cfa8f2d 100644 --- a/command/volume_create.go +++ b/command/volume_create.go @@ -31,7 +31,22 @@ Usage: nomad volume create [options] General Options: - ` + generalOptionsUsage(usageOptsDefault) + ` + generalOptionsUsage(usageOptsDefault) + ` + +Create Options: + + -detach + Return immediately instead of entering monitor mode for dynamic host + volumes. After creating a volume, the volume ID will be printed to the + screen, which can be used to examine the volume using the volume status + command. If -detach is omitted or false, the command will monitor the state + of the volume until it is ready to be scheduled. + + -verbose + Display full information when monitoring volume state. Used for dynamic host + volumes only. + +` return strings.TrimSpace(helpText) } @@ -51,7 +66,10 @@ func (c *VolumeCreateCommand) Synopsis() string { func (c *VolumeCreateCommand) Name() string { return "volume create" } func (c *VolumeCreateCommand) Run(args []string) int { + var detach, verbose bool flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.BoolVar(&detach, "detach", false, "detach from monitor") + flags.BoolVar(&verbose, "verbose", false, "display full volume IDs") flags.Usage = func() { c.Ui.Output(c.Help()) } if err := flags.Parse(args); err != nil { @@ -102,7 +120,7 @@ func (c *VolumeCreateCommand) Run(args []string) int { case "csi": return c.csiCreate(client, ast) case "host": - return c.hostVolumeCreate(client, ast) + return c.hostVolumeCreate(client, ast, detach, verbose) default: c.Ui.Error(fmt.Sprintf("Error unknown volume type: %s", volType)) return 1 diff --git a/command/volume_create_host.go b/command/volume_create_host.go index 32205610740..8947244aba0 100644 --- a/command/volume_create_host.go +++ b/command/volume_create_host.go @@ -4,17 +4,23 @@ package command import ( + "context" "fmt" "strconv" + "time" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/go-glint" + "github.com/mitchellh/go-glint/components" "github.com/mitchellh/mapstructure" ) -func (c *VolumeCreateCommand) hostVolumeCreate(client *api.Client, ast *ast.File) int { +func (c *VolumeCreateCommand) hostVolumeCreate( + client *api.Client, ast *ast.File, detach, verbose bool) int { + vol, err := decodeHostVolume(ast) if err != nil { c.Ui.Error(fmt.Sprintf("Error decoding the volume definition: %s", err)) @@ -29,17 +35,150 @@ func (c *VolumeCreateCommand) hostVolumeCreate(client *api.Client, ast *ast.File c.Ui.Error(fmt.Sprintf("Error creating volume: %s", err)) return 1 } + + var volID string + var lastIndex uint64 + + // note: the command only ever returns 1 volume from the API for _, vol := range vols { - // note: the command only ever returns 1 volume from the API - c.Ui.Output(fmt.Sprintf( - "Created host volume %s with ID %s", vol.Name, vol.ID)) + if detach || vol.State == api.HostVolumeStateReady { + c.Ui.Output(fmt.Sprintf( + "Created host volume %s with ID %s", vol.Name, vol.ID)) + return 0 + } else { + c.Ui.Output(fmt.Sprintf( + "==> Created host volume %s with ID %s", vol.Name, vol.ID)) + volID = vol.ID + lastIndex = vol.ModifyIndex + break + } } - // TODO(1.10.0): monitor so we can report when the node has fingerprinted - + err = c.monitorHostVolume(client, volID, lastIndex, verbose) + if err != nil { + c.Ui.Error(fmt.Sprintf("==> %s: %v", formatTime(time.Now()), err.Error())) + return 1 + } return 0 } +func (c *VolumeCreateCommand) monitorHostVolume(client *api.Client, id string, lastIndex uint64, verbose bool) error { + length := shortId + if verbose { + length = fullId + } + + opts := formatOpts{ + verbose: verbose, + short: !verbose, + length: length, + } + + if isStdoutTerminal() { + return c.ttyMonitor(client, id, lastIndex, opts) + } else { + return c.nottyMonitor(client, id, lastIndex, opts) + } +} + +func (c *VolumeCreateCommand) ttyMonitor(client *api.Client, id string, lastIndex uint64, opts formatOpts) error { + + gUi := glint.New() + spinner := glint.Layout( + components.Spinner(), + glint.Text(fmt.Sprintf(" Monitoring volume %q in progress...", limit(id, opts.length))), + ).Row().MarginLeft(2) + refreshRate := 100 * time.Millisecond + + gUi.SetRefreshRate(refreshRate) + gUi.Set(spinner) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go gUi.Render(ctx) + + qOpts := &api.QueryOptions{ + AllowStale: true, + WaitIndex: lastIndex, + WaitTime: time.Second * 5, + } + + var statusComponent *glint.LayoutComponent + var endSpinner *glint.LayoutComponent + +DONE: + for { + vol, meta, err := client.HostVolumes().Get(id, qOpts) + if err != nil { + return err + } + str, err := formatHostVolume(vol, opts) + if err != nil { + // should never happen b/c we don't pass json/template via opts here + return err + } + statusComponent = glint.Layout( + glint.Text(""), + glint.Text(formatTime(time.Now())), + glint.Text(c.Colorize().Color(str)), + ).MarginLeft(4) + + statusComponent = glint.Layout(statusComponent) + gUi.Set(spinner, statusComponent) + + endSpinner = glint.Layout( + components.Spinner(), + glint.Text(fmt.Sprintf(" Host volume %q %s", limit(id, opts.length), vol.State)), + ).Row().MarginLeft(2) + + switch vol.State { + case api.HostVolumeStateReady: + endSpinner = glint.Layout( + glint.Text(fmt.Sprintf("✓ Host volume %q %s", limit(id, opts.length), vol.State)), + ).Row().MarginLeft(2) + break DONE + + case api.HostVolumeStateDeleted: + endSpinner = glint.Layout( + glint.Text(fmt.Sprintf("! Host volume %q %s", limit(id, opts.length), vol.State)), + ).Row().MarginLeft(2) + break DONE + + default: + qOpts.WaitIndex = meta.LastIndex + continue + } + + } + + // Render one final time with completion message + gUi.Set(endSpinner, statusComponent, glint.Text("")) + gUi.RenderFrame() + return nil +} + +func (c *VolumeCreateCommand) nottyMonitor(client *api.Client, id string, lastIndex uint64, opts formatOpts) error { + + c.Ui.Info(fmt.Sprintf("==> %s: Monitoring volume %q...", + formatTime(time.Now()), limit(id, opts.length))) + + for { + vol, _, err := client.HostVolumes().Get(id, &api.QueryOptions{ + WaitIndex: lastIndex, + WaitTime: time.Second * 5, + }) + if err != nil { + return err + } + if vol.State == api.HostVolumeStateReady { + c.Ui.Info(fmt.Sprintf("==> %s: Volume %q ready", + formatTime(time.Now()), limit(vol.Name, opts.length))) + return nil + } + } +} + func decodeHostVolume(input *ast.File) (*api.HostVolume, error) { var err error vol := &api.HostVolume{} diff --git a/command/volume_create_host_test.go b/command/volume_create_host_test.go index ce713b20d5d..4ef92dc02a8 100644 --- a/command/volume_create_host_test.go +++ b/command/volume_create_host_test.go @@ -66,7 +66,7 @@ parameters { _, err = file.WriteString(hclTestFile) must.NoError(t, err) - args := []string{"-address", url, file.Name()} + args := []string{"-address", url, "-detach", file.Name()} code := cmd.Run(args) must.Eq(t, 0, code, must.Sprintf("got error: %s", ui.ErrorWriter.String())) diff --git a/command/volume_status_host.go b/command/volume_status_host.go index 7878afc55ce..ebe035ddb87 100644 --- a/command/volume_status_host.go +++ b/command/volume_status_host.go @@ -22,6 +22,14 @@ func (c *VolumeStatusCommand) hostVolumeStatus(client *api.Client, id, nodeID, n return 1 } + opts := formatOpts{ + verbose: c.verbose, + short: c.short, + length: c.length, + json: c.json, + template: c.template, + } + // get a host volume that matches the given prefix or a list of all matches // if an exact match is not found. note we can't use the shared getByPrefix // helper here because the List API doesn't match the required signature @@ -32,7 +40,7 @@ func (c *VolumeStatusCommand) hostVolumeStatus(client *api.Client, id, nodeID, n return 1 } if len(possible) > 0 { - out, err := c.formatHostVolumes(possible) + out, err := formatHostVolumes(possible, opts) if err != nil { c.Ui.Error(fmt.Sprintf("Error formatting: %s", err)) return 1 @@ -47,12 +55,12 @@ func (c *VolumeStatusCommand) hostVolumeStatus(client *api.Client, id, nodeID, n return 1 } - str, err := c.formatHostVolume(vol) + str, err := formatHostVolume(vol, opts) if err != nil { c.Ui.Error(fmt.Sprintf("Error formatting volume: %s", err)) return 1 } - c.Ui.Output(str) + c.Ui.Output(c.Colorize().Color(str)) return 0 } @@ -66,13 +74,20 @@ func (c *VolumeStatusCommand) listHostVolumes(client *api.Client, nodeID, nodePo return 1 } - str, err := c.formatHostVolumes(vols) + opts := formatOpts{ + verbose: c.verbose, + short: c.short, + length: c.length, + json: c.json, + template: c.template, + } + + str, err := formatHostVolumes(vols, opts) if err != nil { c.Ui.Error(fmt.Sprintf("Error formatting volumes: %s", err)) return 1 } - c.Ui.Output(str) - + c.Ui.Output(c.Colorize().Color(str)) return 0 } @@ -108,9 +123,9 @@ func (c *VolumeStatusCommand) getByPrefix(client *api.Client, prefix string) (*a } } -func (c *VolumeStatusCommand) formatHostVolume(vol *api.HostVolume) (string, error) { - if c.json || len(c.template) > 0 { - out, err := Format(c.json, c.template, vol) +func formatHostVolume(vol *api.HostVolume, opts formatOpts) (string, error) { + if opts.json || len(opts.template) > 0 { + out, err := Format(opts.json, opts.template, vol) if err != nil { return "", fmt.Errorf("format error: %v", err) } @@ -130,48 +145,51 @@ func (c *VolumeStatusCommand) formatHostVolume(vol *api.HostVolume) (string, err } // Exit early - if c.short { + if opts.short { return formatKV(output), nil } full := []string{formatKV(output)} // Format the allocs - banner := c.Colorize().Color("\n[bold]Allocations[reset]") - allocs := formatAllocListStubs(vol.Allocations, c.verbose, c.length) + banner := "\n[bold]Allocations[reset]" + allocs := formatAllocListStubs(vol.Allocations, opts.verbose, opts.length) full = append(full, banner) full = append(full, allocs) return strings.Join(full, "\n"), nil } -func (c *VolumeStatusCommand) formatHostVolumes(vols []*api.HostVolumeStub) (string, error) { +// TODO: we could make a bunch more formatters into shared functions using this +type formatOpts struct { + verbose bool + short bool + length int + json bool + template string +} + +func formatHostVolumes(vols []*api.HostVolumeStub, opts formatOpts) (string, error) { // Sort the output by volume ID sort.Slice(vols, func(i, j int) bool { return vols[i].ID < vols[j].ID }) - if c.json || len(c.template) > 0 { - out, err := Format(c.json, c.template, vols) + if opts.json || len(opts.template) > 0 { + out, err := Format(opts.json, opts.template, vols) if err != nil { return "", fmt.Errorf("format error: %v", err) } return out, nil } - // Truncate the id unless full length is requested - length := shortId - if c.verbose { - length = fullId - } - rows := make([]string, len(vols)+1) rows[0] = "ID|Name|Namespace|Plugin ID|Node ID|Node Pool|State" for i, v := range vols { rows[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s", - limit(v.ID, length), + limit(v.ID, opts.length), v.Name, v.Namespace, v.PluginID, - limit(v.NodeID, length), + limit(v.NodeID, opts.length), v.NodePool, v.State, ) diff --git a/demo/hostvolume/example-host-volume b/demo/hostvolume/example-host-volume index ae0f7711326..0d726cd3c01 100755 --- a/demo/hostvolume/example-host-volume +++ b/demo/hostvolume/example-host-volume @@ -94,7 +94,7 @@ case "$1" in create_volume "$host_path" "$CAPACITY_MIN_BYTES" # output what Nomad expects bytes="$(stat --format='%s' "$host_path.ext4")" - printf '{"path": "%s", "bytes": %s}' "$host_path", "$bytes" + printf '{"path": "%s", "bytes": %s}' "$host_path" "$bytes" ;; "delete") delete_volume "$host_path" ;;