From a4073141da9327aaf07c59bac597f3993fdbfade Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 4 Jul 2024 17:54:11 +0200 Subject: [PATCH] osbuildprogress: switch to `chegggaaa/pb/v3` for progress Switch the progress output to `chegggaaa/pb/v3` and tweak the progress. Still open questions about the message output and error handling. --- bib/go.mod | 1 + bib/go.sum | 9 +++ bib/internal/osbuildprogress/progress.go | 92 ++++++++++++------------ 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/bib/go.mod b/bib/go.mod index 67ed248ec..f60a4a8b4 100644 --- a/bib/go.mod +++ b/bib/go.mod @@ -25,6 +25,7 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/cheggaaa/pb v1.0.29 // indirect github.com/containerd/cgroups/v3 v3.0.3 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect diff --git a/bib/go.sum b/bib/go.sum index 8c5a002ea..d7e3b4130 100644 --- a/bib/go.sum +++ b/bib/go.sum @@ -27,6 +27,8 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk= github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -78,6 +80,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -186,11 +189,15 @@ github.com/letsencrypt/boulder v0.0.0-20230907030200-6d76a0f91e1e h1:RLTpX495BXT github.com/letsencrypt/boulder v0.0.0-20230907030200-6d76a0f91e1e/go.mod h1:EAuqr9VFWxBi9nD5jc/EA2MT1RFty9288TF6zdtYoCU= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -373,7 +380,9 @@ golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/bib/internal/osbuildprogress/progress.go b/bib/internal/osbuildprogress/progress.go index 6988598c3..6d49881a9 100644 --- a/bib/internal/osbuildprogress/progress.go +++ b/bib/internal/osbuildprogress/progress.go @@ -10,6 +10,8 @@ import ( "os/exec" "strings" "time" + + "github.com/cheggaaa/pb/v3" ) type OsbuildJsonProgress struct { @@ -27,14 +29,14 @@ type OsbuildJsonProgress struct { } `json:"context"` Progress struct { Name string `json:"name"` - Total int `json:"total"` - Done int `json:"done"` + Total int64 `json:"total"` + Done int64 `json:"done"` // XXX: there are currently only two levels but it should be // deeper nested in theory SubProgress struct { Name string `json:"name"` - Total int `json:"total"` - Done int `json:"done"` + Total int64 `json:"total"` + Done int64 `json:"done"` // XXX: in theory this could be more nested but it's not } `json:"progress"` @@ -43,7 +45,7 @@ type OsbuildJsonProgress struct { Message string `json:"message"` } -func scanJsonSeq(r io.Reader, ch chan OsbuildJsonProgress) { +func scanJsonSeq(r io.Reader, ch chan OsbuildJsonProgress, errCh chan error) { var progress OsbuildJsonProgress scanner := bufio.NewScanner(r) @@ -53,33 +55,48 @@ func scanJsonSeq(r io.Reader, ch chan OsbuildJsonProgress) { line = bytes.Trim(line, "\x1e") if err := json.Unmarshal(line, &progress); err != nil { // XXX: provide an invalid lines chan - fmt.Fprintf(os.Stderr, "json decode err for line %q: %v\n", line, err) + errCh <- err continue } ch <- progress } if err := scanner.Err(); err != nil && err != io.EOF { - // log error here + errCh <- err } } func AttachProgress(r io.Reader, w io.Writer) { var progress OsbuildJsonProgress - spinner := []string{"|", "/", "-", "\\"} - i := 0 ch := make(chan OsbuildJsonProgress) - go scanJsonSeq(r, ch) - - mainProgress := "unknown" - subProgress := "" - message := "-" + errCh := make(chan error) + go scanJsonSeq(r, ch, errCh) + + lastMessage := "-" + + spinnerPb := pb.New(0) + spinnerPb.SetTemplate(`Building [{{ (cycle . "|" "/" "-" "\\") }}]`) + mainPb := pb.New(0) + progressBarTmplFmt := `[{{ counters . }}] %s: {{ string . "prefix" }} {{ bar .}} {{ percent . }}` + mainPb.SetTemplateString(fmt.Sprintf(progressBarTmplFmt, "step")) + subPb := pb.New(0) + subPb.SetTemplateString(fmt.Sprintf(progressBarTmplFmt, "module")) + msgPb := pb.New(0) + msgPb.SetTemplate(`last msg: {{ string . "msg" }}`) + + pool, err := pb.StartPool(spinnerPb, mainPb, subPb, msgPb) + if err != nil { + fmt.Fprintf(os.Stderr, "progress failed: %v\n", err) + return + } contextMap := map[string]string{} - fmt.Fprintf(w, "\n") for { select { + case err := <-errCh: + fmt.Fprintf(os.Stderr, "error: %v", err) + break case progress = <-ch: id := progress.Context.Pipeline.ID pipelineName := contextMap[id] @@ -96,54 +113,38 @@ func AttachProgress(r io.Reader, w io.Writer) { } if progress.Progress.Total > 0 { - mainProgress = fmt.Sprintf("step: %v [%v/%v]", pipelineName, progress.Progress.Done+1, progress.Progress.Total+1) + mainPb.SetTotal(progress.Progress.Total + 1) + mainPb.SetCurrent(progress.Progress.Done + 1) + mainPb.Set("prefix", pipelineName) } // XXX: use context instead of name here too if progress.Progress.SubProgress.Total > 0 { - subProgress = fmt.Sprintf("%v [%v/%v]", stageName, progress.Progress.SubProgress.Done+1, progress.Progress.SubProgress.Total+1) + subPb.SetTotal(progress.Progress.SubProgress.Total + 1) + subPb.SetCurrent(progress.Progress.SubProgress.Done + 1) + subPb.Set("prefix", strings.TrimPrefix(stageName, "org.osbuild.")) } // todo: make message more structured in osbuild? // message from the stages themselfs are very noisy // best not to show to the user (only for failures) if progress.Context.Origin == "osbuild.monitor" { - message = progress.Message + lastMessage = progress.Message } - //message = strings.TrimSpace(strings.SplitN(progress.Message, "\n", 2)[0]) - // todo: fix in osbuild? /* - l := strings.SplitN(message, ":", 2) + // todo: fix in osbuild? + lastMessage = strings.TrimSpace(strings.SplitN(progress.Message, "\n", 2)[0]) + l := strings.SplitN(lastMessage, ":", 2) if len(l) > 1 { - message = strings.TrimSpace(l[1]) + lastMessage = strings.TrimSpace(l[1]) } */ - if len(message) > 60 { - message = message[:60] + "..." - } + msgPb.Set("msg", lastMessage) + case <-time.After(200 * time.Millisecond): // nothing } - - // XXX: use real progressbar *or* use helper to get terminal - // size for proper length checks etc - // - // poor man progress, we need multiple progress bars and - // a message that keeps getting updated (or maybe not the - // message) - fmt.Fprintf(w, "\x1b[2KBuilding [%s]\n", spinner[i]) - fmt.Fprintf(w, "\x1b[2Kstep : %s\n", mainProgress) - if subProgress != "" { - fmt.Fprintf(w, "\x1b[2Kmodule : %s\n", strings.TrimPrefix(subProgress, "org.osbuild.")) - } - fmt.Fprintf(w, "\x1b[3Kmessage: %s\n", message) - if subProgress != "" { - fmt.Fprintf(w, "\x1b[%dA", 4) - } else { - fmt.Fprintf(w, "\x1b[%dA", 3) - } - // spin - i = (i + 1) % len(spinner) } + pool.Stop() } // XXX: merge back into images/pkg/osbuild/osbuild-exec.go(?) @@ -172,5 +173,6 @@ func RunOSBuild(manifest []byte, store, outputDirectory string, exports, extraEn if err := cmd.Start(); err != nil { return fmt.Errorf("error starting osbuild: %v", err) } + // XXX: add WaitGroup return cmd.Wait() }