-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
kind/feat: surface artifacts through sidecar container
- Loading branch information
1 parent
4fe52d4
commit 8efcd61
Showing
16 changed files
with
752 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| /* | ||
| Copyright 2024 The Tekton Authors | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "flag" | ||
| "log" | ||
| "os" | ||
| "strings" | ||
|
|
||
| "github.com/tektoncd/pipeline/internal/sidecarlogartifacts" | ||
| "github.com/tektoncd/pipeline/pkg/pod" | ||
| ) | ||
|
|
||
| func main() { | ||
| var stepNames string | ||
| flag.StringVar(&stepNames, "step-names", "", "comma separated step names to expect from the steps running in the pod. eg. foo,bar,baz") | ||
| flag.Parse() | ||
| if stepNames == "" { | ||
| log.Fatal("step-names were not provided") | ||
| } | ||
| names := strings.Split(stepNames, ",") | ||
| artifacts, err := sidecarlogartifacts.LookForArtifacts(names, pod.RunDir) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| err = json.NewEncoder(os.Stdout).Encode(artifacts) | ||
|
|
||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| /* | ||
| Copyright 2024 The Tekton Authors | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package sidecarlogartifacts | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| "github.com/tektoncd/pipeline/pkg/apis/pipeline" | ||
| v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/client-go/kubernetes" | ||
| ) | ||
|
|
||
| // for testing | ||
| var stepDir = pipeline.StepsDir | ||
|
|
||
| func fileExists(filename string) (bool, error) { | ||
| info, err := os.Stat(filename) | ||
| if os.IsNotExist(err) { | ||
| return false, nil | ||
| } else if err != nil { | ||
| return false, fmt.Errorf("error checking for file existence %w", err) | ||
| } | ||
| return !info.IsDir(), nil | ||
| } | ||
|
|
||
| func waitForStepsToFinish(runDir string) error { | ||
| steps := make(map[string]bool) | ||
| files, err := os.ReadDir(runDir) | ||
| if err != nil { | ||
| return fmt.Errorf("error parsing the run dir %w", err) | ||
| } | ||
| for _, file := range files { | ||
| steps[filepath.Join(runDir, file.Name(), "out")] = true | ||
| } | ||
| for len(steps) > 0 { | ||
| for stepFile := range steps { | ||
| // check if there is a post file without error | ||
| time.Sleep(200 * time.Millisecond) | ||
| exists, err := fileExists(stepFile) | ||
| if err != nil { | ||
| return fmt.Errorf("error checking for out file's existence %w", err) | ||
| } | ||
| if exists { | ||
| delete(steps, stepFile) | ||
| continue | ||
| } | ||
|
|
||
| // This is the same as results sidecar, it checks if there is a post file with error | ||
| // if err is nil then either the out.err file does not exist or it does and there was no issue | ||
| // in either case, existence of out.err marks that the step errored and the following steps will | ||
| // not run. We want the function to break out with nil error in that case so that | ||
| // the existing results can be logged. | ||
| if exists, err = fileExists(stepFile + ".err"); exists || err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func parseArtifacts(fileContent []byte) (v1.Artifacts, error) { | ||
| var as v1.Artifacts | ||
| if err := json.Unmarshal(fileContent, &as); err != nil { | ||
| return as, fmt.Errorf("invalid artifacts : %w", err) | ||
| } | ||
| return as, nil | ||
| } | ||
|
|
||
| func extractArtifactsFromFile(filename string) (v1.Artifacts, error) { | ||
| b, err := os.ReadFile(filename) | ||
| if err != nil { | ||
| return v1.Artifacts{}, err | ||
| } | ||
| return parseArtifacts(b) | ||
| } | ||
|
|
||
| type SidecarArtifacts map[string]v1.Artifacts | ||
|
|
||
| // GetArtifactsFromSidecarLogs retrieves artifacts from sidecar logs in a Kubernetes pod. | ||
| // It returns a SidecarArtifacts map (step name -> v1.Artifacts) and an error. If the pod is in the | ||
| // Pending phase, an empty map is returned without error. | ||
| func GetArtifactsFromSidecarLogs(ctx context.Context, clientset kubernetes.Interface, namespace string, name string, container string, podPhase corev1.PodPhase) (SidecarArtifacts, error) { | ||
| sidecarArtifacts := SidecarArtifacts{} | ||
| if podPhase == corev1.PodPending { | ||
| return sidecarArtifacts, nil | ||
| } | ||
| podLogOpts := corev1.PodLogOptions{Container: container} | ||
| req := clientset.CoreV1().Pods(namespace).GetLogs(name, &podLogOpts) | ||
| stream, err := req.Stream(ctx) | ||
| if err != nil { | ||
| return sidecarArtifacts, err | ||
| } | ||
| err = json.NewDecoder(stream).Decode(&sidecarArtifacts) | ||
| if err != nil { | ||
| return sidecarArtifacts, err | ||
| } | ||
|
|
||
| return sidecarArtifacts, nil | ||
| } | ||
|
|
||
| // LookForArtifacts searches for provenance.json files in the specified run directory | ||
| // and extracts artifacts from them. | ||
| func LookForArtifacts(names []string, runDir string) (SidecarArtifacts, error) { | ||
| err := waitForStepsToFinish(runDir) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| artifacts := SidecarArtifacts{} | ||
| for _, name := range names { | ||
| p := filepath.Join(stepDir, name, "artifacts", "provenance.json") | ||
| if exist, err := fileExists(p); err != nil { | ||
| return artifacts, err | ||
| } else if !exist { | ||
| continue | ||
| } | ||
| subRes, err := extractArtifactsFromFile(p) | ||
| if err != nil { | ||
| return SidecarArtifacts{}, err | ||
| } | ||
| artifacts[name] = v1.Artifacts{Inputs: subRes.Inputs, Outputs: subRes.Outputs} | ||
| } | ||
| return artifacts, nil | ||
| } |
Oops, something went wrong.