Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: attach all logs on chart failure #150

Merged
merged 2 commits into from
Dec 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 43 additions & 32 deletions internal/cmd/local/local/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package local

import (
"context"
"errors"
"fmt"
"net/http"
"os"
Expand Down Expand Up @@ -253,7 +254,9 @@ func (c *Command) Install(ctx context.Context, opts *InstallOpts) error {
}); err != nil {
// if trace.SpanError isn't called here, the logs attached
// in the diagnoseAirbyteChartFailure method are lost
return trace.SpanError(span, c.diagnoseAirbyteChartFailure(ctx, err))
err = c.diagnoseAirbyteChartFailure(ctx, err)
err = fmt.Errorf("unable to install airbyte chart: %w", err)
return trace.SpanError(span, err)
}

nginxValues, err := helm.BuildNginxValues(c.portHTTP)
Expand Down Expand Up @@ -318,46 +321,54 @@ func (c *Command) Install(ctx context.Context, opts *InstallOpts) error {
}

func (c *Command) diagnoseAirbyteChartFailure(ctx context.Context, chartErr error) error {
if podList, err := c.k8s.PodList(ctx, common.AirbyteNamespace); err == nil {
var errors []string
for _, pod := range podList.Items {
pterm.Debug.Println(fmt.Sprintf("looking at %s\n %s(%s)", pod.Name, pod.Status.Phase, pod.Status.Reason))
if pod.Status.Phase != corev1.PodFailed {
continue
}

msg := "unknown"
if errors.Is(ctx.Err(), context.Canceled) {
return chartErr
}

podList, err := c.k8s.PodList(ctx, common.AirbyteNamespace)
if err != nil {
return chartErr
}

logs, err := c.k8s.LogsGet(ctx, common.AirbyteNamespace, pod.Name)
if err != nil {
msg = "unknown: failed to get pod logs."
}

preview := logs
if len(preview) > 50 {
preview = preview[:50]
}
pterm.Debug.Println("found logs: ", preview)
var failedPods []string
for _, pod := range podList.Items {
if pod.Status.Phase == corev1.PodFailed {
failedPods = append(failedPods, pod.Name)
}
}

trace.AttachLog(fmt.Sprintf("%s.log", pod.Name), logs)
// If none of the pods failed, don't bother looking at logs.
if len(failedPods) == 0 {
return chartErr
}

m, err := getLastLogError(strings.NewReader(logs))
if err != nil {
msg = "unknown: failed to find error log."
}
if m != "" {
msg = m
}
for _, pod := range podList.Items {
// skip pods that aren't part of the platform release (e.g. job pods)
if !strings.HasPrefix(pod.Name, common.AirbyteChartRelease) {
continue
}
pterm.Debug.Printfln("looking at %s\n %s(%s)", pod.Name, pod.Status.Phase, pod.Status.Reason)

errors = append(errors, fmt.Sprintf("pod %s: %s", pod.Name, msg))
logs, err := c.k8s.LogsGet(ctx, common.AirbyteNamespace, pod.Name)
if err != nil {
pterm.Debug.Printfln("failed to get pod logs: %s", err)
continue
}

if errors != nil {
return fmt.Errorf("unable to install airbyte chart:\n%s", strings.Join(errors, "\n"))
preview := logs
if len(preview) > 50 {
preview = preview[:50]
}
pterm.Debug.Println("found logs: ", preview)

trace.AttachLog(fmt.Sprintf("%s.log", pod.Name), logs)
}

if len(failedPods) == 1 && failedPods[0] == common.AirbyteBootloaderPodName {
return localerr.ErrBootloaderFailed
}

return fmt.Errorf("unable to install airbyte chart: %w", chartErr)
return chartErr
}

func (c *Command) handleIngress(ctx context.Context, hosts []string) error {
Expand Down
2 changes: 1 addition & 1 deletion internal/cmd/local/local/install_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ func TestCommand_InstallError(t *testing.T) {
t.Fatal(err)
}
err = c.Install(context.Background(), installOpts)
expect := "unable to install airbyte chart:\npod test-pod-1: unknown"
expect := "unable to install airbyte chart: unable to install helm: test error"
if expect != err.Error() {
t.Errorf("expected %q but got %q", expect, err)
}
Expand Down
18 changes: 0 additions & 18 deletions internal/cmd/local/local/log_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,21 +59,3 @@ func (j *logScanner) Scan() bool {
func (j *logScanner) Err() error {
return j.scanner.Err()
}

func getLastLogError(r io.Reader) (string, error) {
var lines []logLine
s := newLogScanner(r)
for s.Scan() {
lines = append(lines, s.line)
}
if s.Err() != nil {
return "", s.Err()
}

for i := len(lines) - 1; i >= 0; i-- {
if lines[i].level == "ERROR" {
return lines[i].msg, nil
}
}
return "", nil
}
11 changes: 0 additions & 11 deletions internal/cmd/local/local/log_utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,3 @@ func TestJavaLogScanner(t *testing.T) {
expectLogLine("INFO", "i.m.r.Micronaut(lambda$start$0):118 - Embedded Application shutting down")
expectLogLine("INFO", "2024-09-12T15:56:33.125352208Z Thread-4 INFO Loading mask data from '/seed/specs_secrets_mask.yaml")
}

func TestLastErrorLog(t *testing.T) {
l, err := getLastLogError(strings.NewReader(testLogs))
if err != nil {
t.Errorf("unexpected error %s", err)
}
expect := "Caused by: io.airbyte.db.check.DatabaseCheckException: Unable to connect to the database."
if l != expect {
t.Errorf("expected %q but got %q", expect, l)
}
}
5 changes: 5 additions & 0 deletions internal/cmd/local/localerr/localerr.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,9 @@ IP addresses won't work. Ports won't work (e.g. example:8000). URLs won't work (
By default, abctl will allow access from any hostname or IP, so you might not need the --host flag.`,
}

ErrBootloaderFailed = &LocalError{
msg: "bootloader failed",
help: "The bootloader failed to its initialization checks or migrations. Try running again with --verbose to see the full bootloader logs.",
}
)