Skip to content

Commit

Permalink
osbuildprogress: switch to chegggaaa/pb/v3 for progress
Browse files Browse the repository at this point in the history 
Switch the progress output to `chegggaaa/pb/v3` and tweak the
progress. Still open questions about the message output and
error handling.
  • Loading branch information
@mvo5
mvo5 committed Jul 4, 2024
1 parent 1cb509f commit a407314
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 45 deletions.
1 change: 1 addition & 0 deletions bib/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ require (
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/cheggaaa/pb v1.0.29 // indirect
github.com/containerd/cgroups/v3 v3.0.3 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
Expand Down
9 changes: 9 additions & 0 deletions bib/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk=
github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
Expand Down Expand Up @@ -78,6 +80,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
Expand Down Expand Up @@ -186,11 +189,15 @@ github.com/letsencrypt/boulder v0.0.0-20230907030200-6d76a0f91e1e h1:RLTpX495BXT
github.com/letsencrypt/boulder v0.0.0-20230907030200-6d76a0f91e1e/go.mod h1:EAuqr9VFWxBi9nD5jc/EA2MT1RFty9288TF6zdtYoCU=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
Expand Down Expand Up @@ -373,7 +380,9 @@ golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
92 changes: 47 additions & 45 deletions bib/internal/osbuildprogress/progress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"os/exec"
"strings"
"time"

"github.com/cheggaaa/pb/v3"
)

type OsbuildJsonProgress struct {
Expand All @@ -27,14 +29,14 @@ type OsbuildJsonProgress struct {
} `json:"context"`
Progress struct {
Name string `json:"name"`
Total int `json:"total"`
Done int `json:"done"`
Total int64 `json:"total"`
Done int64 `json:"done"`
// XXX: there are currently only two levels but it should be
// deeper nested in theory
SubProgress struct {
Name string `json:"name"`
Total int `json:"total"`
Done int `json:"done"`
Total int64 `json:"total"`
Done int64 `json:"done"`
// XXX: in theory this could be more nested but it's not

} `json:"progress"`
Expand All @@ -43,7 +45,7 @@ type OsbuildJsonProgress struct {
Message string `json:"message"`
}

func scanJsonSeq(r io.Reader, ch chan OsbuildJsonProgress) {
func scanJsonSeq(r io.Reader, ch chan OsbuildJsonProgress, errCh chan error) {
var progress OsbuildJsonProgress

scanner := bufio.NewScanner(r)
Expand All @@ -53,33 +55,48 @@ func scanJsonSeq(r io.Reader, ch chan OsbuildJsonProgress) {
line = bytes.Trim(line, "\x1e")
if err := json.Unmarshal(line, &progress); err != nil {
// XXX: provide an invalid lines chan
fmt.Fprintf(os.Stderr, "json decode err for line %q: %v\n", line, err)
errCh <- err
continue
}
ch <- progress
}
if err := scanner.Err(); err != nil && err != io.EOF {
// log error here
errCh <- err
}
}

func AttachProgress(r io.Reader, w io.Writer) {
var progress OsbuildJsonProgress
spinner := []string{"|", "/", "-", "\\"}
i := 0

ch := make(chan OsbuildJsonProgress)
go scanJsonSeq(r, ch)

mainProgress := "unknown"
subProgress := ""
message := "-"
errCh := make(chan error)
go scanJsonSeq(r, ch, errCh)

lastMessage := "-"

spinnerPb := pb.New(0)
spinnerPb.SetTemplate(`Building [{{ (cycle . "|" "/" "-" "\\") }}]`)
mainPb := pb.New(0)
progressBarTmplFmt := `[{{ counters . }}] %s: {{ string . "prefix" }} {{ bar .}} {{ percent . }}`
mainPb.SetTemplateString(fmt.Sprintf(progressBarTmplFmt, "step"))
subPb := pb.New(0)
subPb.SetTemplateString(fmt.Sprintf(progressBarTmplFmt, "module"))
msgPb := pb.New(0)
msgPb.SetTemplate(`last msg: {{ string . "msg" }}`)

pool, err := pb.StartPool(spinnerPb, mainPb, subPb, msgPb)

Check failure on line 87 in bib/internal/osbuildprogress/progress.go

View workflow job for this annotation

GitHub Actions / ⌨ Lint & unittests 

SA4006: this value of `pool` is never used (staticcheck)
if err != nil {
fmt.Fprintf(os.Stderr, "progress failed: %v\n", err)
return
}

contextMap := map[string]string{}

fmt.Fprintf(w, "\n")
for {
select {
case err := <-errCh:
fmt.Fprintf(os.Stderr, "error: %v", err)
break
case progress = <-ch:
id := progress.Context.Pipeline.ID
pipelineName := contextMap[id]
Expand All @@ -96,54 +113,38 @@ func AttachProgress(r io.Reader, w io.Writer) {
}

if progress.Progress.Total > 0 {
mainProgress = fmt.Sprintf("step: %v [%v/%v]", pipelineName, progress.Progress.Done+1, progress.Progress.Total+1)
mainPb.SetTotal(progress.Progress.Total + 1)
mainPb.SetCurrent(progress.Progress.Done + 1)
mainPb.Set("prefix", pipelineName)
}
// XXX: use context instead of name here too
if progress.Progress.SubProgress.Total > 0 {
subProgress = fmt.Sprintf("%v [%v/%v]", stageName, progress.Progress.SubProgress.Done+1, progress.Progress.SubProgress.Total+1)
subPb.SetTotal(progress.Progress.SubProgress.Total + 1)
subPb.SetCurrent(progress.Progress.SubProgress.Done + 1)
subPb.Set("prefix", strings.TrimPrefix(stageName, "org.osbuild."))
}

// todo: make message more structured in osbuild?
// message from the stages themselfs are very noisy
// best not to show to the user (only for failures)
if progress.Context.Origin == "osbuild.monitor" {
message = progress.Message
lastMessage = progress.Message
}
//message = strings.TrimSpace(strings.SplitN(progress.Message, "\n", 2)[0])
// todo: fix in osbuild?
/*
l := strings.SplitN(message, ":", 2)
// todo: fix in osbuild?
lastMessage = strings.TrimSpace(strings.SplitN(progress.Message, "\n", 2)[0])
l := strings.SplitN(lastMessage, ":", 2)
if len(l) > 1 {
message = strings.TrimSpace(l[1])
lastMessage = strings.TrimSpace(l[1])
}
*/
if len(message) > 60 {
message = message[:60] + "..."
}
msgPb.Set("msg", lastMessage)

case <-time.After(200 * time.Millisecond):
// nothing
}

// XXX: use real progressbar *or* use helper to get terminal
// size for proper length checks etc
//
// poor man progress, we need multiple progress bars and
// a message that keeps getting updated (or maybe not the
// message)
fmt.Fprintf(w, "\x1b[2KBuilding [%s]\n", spinner[i])
fmt.Fprintf(w, "\x1b[2Kstep : %s\n", mainProgress)
if subProgress != "" {
fmt.Fprintf(w, "\x1b[2Kmodule : %s\n", strings.TrimPrefix(subProgress, "org.osbuild."))
}
fmt.Fprintf(w, "\x1b[3Kmessage: %s\n", message)
if subProgress != "" {
fmt.Fprintf(w, "\x1b[%dA", 4)
} else {
fmt.Fprintf(w, "\x1b[%dA", 3)
}
// spin
i = (i + 1) % len(spinner)
}
pool.Stop()

Check failure on line 147 in bib/internal/osbuildprogress/progress.go

View workflow job for this annotation

GitHub Actions / ⌨ Lint & unittests 

Error return value of `pool.Stop` is not checked (errcheck)
}

// XXX: merge back into images/pkg/osbuild/osbuild-exec.go(?)
Expand Down Expand Up @@ -172,5 +173,6 @@ func RunOSBuild(manifest []byte, store, outputDirectory string, exports, extraEn
if err := cmd.Start(); err != nil {
return fmt.Errorf("error starting osbuild: %v", err)
}
// XXX: add WaitGroup
return cmd.Wait()
}

0 comments on commit a407314

Please sign in to comment.