Skip to content
This repository has been archived by the owner on Aug 28, 2024. It is now read-only.

Commit

Permalink
Remove versioning requirement for Models and Datasets (#254)
Browse files Browse the repository at this point in the history 
Fixes #250
  • Loading branch information
@nstogner
nstogner authored Oct 13, 2023
1 parent 506af5e commit 9958742
Show file tree
Hide file tree
Showing 6 changed files with 106 additions and 160 deletions.
18 changes: 14 additions & 4 deletions internal/cli/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,17 @@ func runCommand() *cobra.Command {
namespace string
filename string
kubeconfig string
increment bool
replace bool
}

run := func(cmd *cobra.Command, args []string) error {
defer tui.LogFile.Close()

if flags.increment && flags.replace {
return fmt.Errorf("flags: --increment (-i) and --replace (-r): not compatible")
}

kubeconfigNamespace, restConfig, err := utils.BuildConfigFromFlags("", flags.kubeconfig)
if err != nil {
return fmt.Errorf("rest config: %w", err)
Expand Down Expand Up @@ -51,8 +57,10 @@ func runCommand() *cobra.Command {
Contextual: kubeconfigNamespace,
Specified: flags.namespace,
},
Client: client,
K8s: clientset,
Increment: flags.increment,
Replace: flags.replace,
Client: client,
K8s: clientset,
}).New())
if _, err := tui.P.Run(); err != nil {
return err
Expand Down Expand Up @@ -86,9 +94,11 @@ func runCommand() *cobra.Command {
if defaultKubeconfig == "" {
defaultKubeconfig = clientcmd.RecommendedHomeFile
}
cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "path to Kubernetes Kubeconfig file")
cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "namespace of Notebook")
cmd.Flags().StringVarP(&flags.kubeconfig, "kubeconfig", "", defaultKubeconfig, "path to kubernetes kubeconfig file")
cmd.Flags().StringVarP(&flags.namespace, "namespace", "n", "", "kubernetes namespace")
cmd.Flags().StringVarP(&flags.filename, "filename", "f", "", "manifest file")
cmd.Flags().BoolVarP(&flags.increment, "increment", "i", false, "increment the name")
cmd.Flags().BoolVarP(&flags.replace, "replace", "r", false, "replace if already exists")

return cmd
}
88 changes: 50 additions & 38 deletions internal/tui/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ import (
"log"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"time"

tea "github.com/charmbracelet/bubbletea"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"

apiv1 "github.com/substratusai/substratus/api/v1"
Expand Down Expand Up @@ -155,50 +155,52 @@ type createdWithUploadMsg struct {
client.Object
}

func createWithUploadCmd(ctx context.Context, res *client.Resource, obj client.Object, tarball *client.Tarball) tea.Cmd {
func createWithUploadCmd(ctx context.Context, res *client.Resource, obj client.Object, tarball *client.Tarball, increment, replace bool) tea.Cmd {
return func() tea.Msg {
if err := specifyUpload(obj, tarball); err != nil {
return fmt.Errorf("specifying upload: %w", err)
}

lowerKind := strings.ToLower(obj.GetObjectKind().GroupVersionKind().Kind)
if obj.GetLabels() == nil {
obj.SetLabels(map[string]string{})
}
obj.GetLabels()[lowerKind] = obj.GetName()

list, err := res.List(obj.GetNamespace(), obj.GetObjectKind().GroupVersionKind().Version, &metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(map[string]string{
lowerKind: obj.GetName(),
}).String(),
})
if err != nil {
return fmt.Errorf("listing: %w", err)
}

var version int
switch list := list.(type) {
case *apiv1.ModelList:
version, err = nextModelVersion(list)
if increment {
list, err := res.List(obj.GetNamespace(), obj.GetObjectKind().GroupVersionKind().Version, &metav1.ListOptions{})
if err != nil {
return fmt.Errorf("next model version: %w", err)
return fmt.Errorf("listing: %w", err)
}
case *apiv1.DatasetList:
version, err = nextDatasetVersion(list)
if err != nil {
return fmt.Errorf("next dataset version: %w", err)

var version int
switch list := list.(type) {
case *apiv1.ModelList:
version, err = nextModelVersion(list, obj.GetName())
if err != nil {
return fmt.Errorf("next model version: %w", err)
}
case *apiv1.DatasetList:
version, err = nextDatasetVersion(list, obj.GetName())
if err != nil {
return fmt.Errorf("next dataset version: %w", err)
}
default:
return fmt.Errorf("unrecognized list type: %T", list)
}
default:
return fmt.Errorf("unrecognized list type: %T", list)
}

log.Printf("Next version: %v", version)
log.Printf("Next version: %v", version)

obj.SetName(fmt.Sprintf("%v-%v", obj.GetName(), version))
}

obj.SetName(fmt.Sprintf("%v.v%v", obj.GetName(), version))
obj.GetLabels()["version"] = fmt.Sprintf("%v", version)
if _, err := res.Create(obj.GetNamespace(), true, obj); err != nil {
return fmt.Errorf("creating: %w", err)
if replace && apierrors.IsAlreadyExists(err) {
if _, err := res.Delete(obj.GetNamespace(), obj.GetName()); err != nil {
return fmt.Errorf("replacing: delete: %w", err)
}
if _, err := res.Create(obj.GetNamespace(), true, obj); err != nil {
return fmt.Errorf("replacing: creating: %w", err)
}
} else {
return fmt.Errorf("creating: %w", err)
}
}

return createdWithUploadMsg{Object: obj}
}
}
Expand All @@ -222,10 +224,15 @@ func waitReadyCmd(ctx context.Context, res *client.Resource, obj client.Object)
}
}

func nextModelVersion(list *apiv1.ModelList) (int, error) {
func nextModelVersion(list *apiv1.ModelList, name string) (int, error) {
var highestVersion int
re := regexp.MustCompile(name + `-(\d+)`)
for _, item := range list.Items {
v, err := strconv.Atoi(item.GetLabels()["version"])
match := re.FindStringSubmatch(item.Name)
if len(match) != 2 {
continue
}
v, err := strconv.Atoi(match[1])
if err != nil {
return 0, fmt.Errorf("version label to int: %w", err)
}
Expand All @@ -237,10 +244,15 @@ func nextModelVersion(list *apiv1.ModelList) (int, error) {
return highestVersion + 1, nil
}

func nextDatasetVersion(list *apiv1.DatasetList) (int, error) {
func nextDatasetVersion(list *apiv1.DatasetList, name string) (int, error) {
var highestVersion int
re := regexp.MustCompile(name + `-(\d+)`)
for _, item := range list.Items {
v, err := strconv.Atoi(item.GetLabels()["version"])
match := re.FindStringSubmatch(item.Name)
if len(match) != 2 {
continue
}
v, err := strconv.Atoi(match[1])
if err != nil {
return 0, fmt.Errorf("version label to int: %w", err)
}
Expand Down
117 changes: 16 additions & 101 deletions internal/tui/get.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,12 @@ import (
"context"
"fmt"
"log"
"math"
"slices"
"sort"
"strings"

"github.com/charmbracelet/bubbles/spinner"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"

Expand Down Expand Up @@ -134,119 +131,37 @@ func (m GetModel) View() (v string) {
scopeResource, scopeName := splitScope(m.Scope)

var total int
for _, resource := range []struct {
plural string
versioned bool
}{
{plural: "notebooks", versioned: false},
{plural: "datasets", versioned: true},
{plural: "models", versioned: true},
{plural: "servers", versioned: false},
for _, resource := range []string{
"notebooks",
"datasets",
"models",
"servers",
} {
if len(m.objects[resource.plural]) == 0 {
if len(m.objects[resource]) == 0 {
continue
}

if scopeResource == "" {
v += resource.plural + "/" + "\n"
v += resource + "/" + "\n"
}

var names []string
for name := range m.objects[resource.plural] {
for name := range m.objects[resource] {
names = append(names, name)
total++
}
sort.Strings(names)

if !resource.versioned {
for _, name := range names {
o := m.objects[resource.plural][name]
for _, name := range names {
o := m.objects[resource][name]

var indicator string
if o.GetStatusReady() {
indicator = checkMark.String()
} else {
indicator = o.spinner.View()
}
v += "" + indicator + " " + name + "\n"
}
} else {
type objectVersions struct {
unversionedName string
versions []listedObject
}

var groups []objectVersions

var lastUnversionedName string

const longestName = 30
for _, name := range names {
o := m.objects[resource.plural][name]
lowerKind := strings.TrimSuffix(resource.plural, "s")
unversionedName := o.GetLabels()[lowerKind]

if unversionedName != lastUnversionedName {
groups = append(groups, objectVersions{
unversionedName: unversionedName,
versions: []listedObject{o},
})
} else {
groups[len(groups)-1].versions = append(groups[len(groups)-1].versions, o)
}

lastUnversionedName = unversionedName
//if n := len(name); n > longestName {
// longestName = n + 6
//}
var indicator string
if o.GetStatusReady() {
indicator = checkMark.String()
} else {
indicator = o.spinner.View()
}

for gi, g := range groups {
type versionDisplay struct {
indicator string
version string
}
var displayVersions []versionDisplay
for _, o := range g.versions {
version := o.GetLabels()["version"]

var indicator string
if o.GetStatusReady() {
indicator = checkMark.String()
} else if c := meta.FindStatusCondition(*o.GetConditions(), apiv1.ConditionComplete); c != nil && c.Reason == apiv1.ReasonJobFailed {
indicator = xMark.String()
} else {
indicator = o.spinner.View()
}
displayVersions = append(displayVersions, versionDisplay{
indicator: indicator,
version: version,
})
}

// Latest first
slices.Reverse(displayVersions)

var otherVersions []string
for _, other := range displayVersions[1:] {
otherVersions = append(otherVersions, fmt.Sprintf("%v.v%v", other.indicator, other.version))
}

primary := displayVersions[0].indicator + " " +
g.unversionedName + ".v" +
displayVersions[0].version

verWidth := int(math.Min(float64(60), float64(m.Style.GetWidth()-m.Style.GetHorizontalMargins()-longestName-18)))
v += lipgloss.JoinHorizontal(
lipgloss.Top,
lipgloss.NewStyle().Width(longestName).MarginLeft(0).MarginRight(2).Align(lipgloss.Left).Render(primary),
lipgloss.NewStyle().Width(verWidth).MarginRight(4).Align(lipgloss.Right).Render(strings.Join(otherVersions, " ")),
)
if gi < len(groups) {
v += "\n"
}
}

v += "" + indicator + " " + name + "\n"
}
v += "\n"
}
Expand Down
6 changes: 4 additions & 2 deletions internal/tui/readiness.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func (m readinessModel) View() (v string) {

if m.waiting == inProgress {
kind := m.Object.GetObjectKind().GroupVersionKind().Kind
v += fmt.Sprintf("%v:\n", kind)
v += fmt.Sprintf("%v (%v):\n", kind, m.Object.GetName())

if w, ok := m.Object.(interface {
GetConditions() *[]metav1.Condition
Expand All @@ -93,7 +93,9 @@ func (m readinessModel) View() (v string) {
v += "\n"
}
}

} else if m.waiting == completed {
kind := m.Object.GetObjectKind().GroupVersionKind().Kind
v += fmt.Sprintf("%v (%v): Ready\n", kind, m.Object.GetName())
}

return v
Expand Down
Loading

0 comments on commit 9958742

Please sign in to comment.