diff --git a/go.mod b/go.mod index addf917682..a31f6a8e14 100644 --- a/go.mod +++ b/go.mod @@ -12,11 +12,11 @@ require ( github.com/checkpoint-restore/checkpointctl v1.2.1 github.com/checkpoint-restore/go-criu/v7 v7.2.0 github.com/containernetworking/plugins v1.5.1 - github.com/containers/buildah v1.37.0 - github.com/containers/common v0.60.1-0.20240920125326-ff6611ae40ad + github.com/containers/buildah v1.37.3 + github.com/containers/common v0.60.2 github.com/containers/conmon v2.0.20+incompatible github.com/containers/gvisor-tap-vsock v0.7.5 - github.com/containers/image/v5 v5.32.1-0.20240806084436-e3e9287ca8e6 + github.com/containers/image/v5 v5.32.2 github.com/containers/libhvee v0.7.1 github.com/containers/ocicrypt v1.2.0 github.com/containers/psgo v1.9.0 diff --git a/go.sum b/go.sum index 0fbc30cce9..c51abd55ce 100644 --- a/go.sum +++ b/go.sum @@ -79,16 +79,16 @@ github.com/containernetworking/cni v1.2.3 h1:hhOcjNVUQTnzdRJ6alC5XF+wd9mfGIUaj8F github.com/containernetworking/cni v1.2.3/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M= github.com/containernetworking/plugins v1.5.1 h1:T5ji+LPYjjgW0QM+KyrigZbLsZ8jaX+E5J/EcKOE4gQ= github.com/containernetworking/plugins v1.5.1/go.mod h1:MIQfgMayGuHYs0XdNudf31cLLAC+i242hNm6KuDGqCM= -github.com/containers/buildah v1.37.0 h1:jvHwu1vIwIqnHyOSg9eef9Apdpry+5oWLrm43gdf8Rk= -github.com/containers/buildah v1.37.0/go.mod h1:MKd79tkluMf6vtH06SedhBQK5OB7E0pFVIuiTTw3dJk= -github.com/containers/common v0.60.1-0.20240920125326-ff6611ae40ad h1:Ida4yFcnk+xGPynWR267zGGUddWTfpAVMSzo6PhjPFQ= -github.com/containers/common v0.60.1-0.20240920125326-ff6611ae40ad/go.mod h1:UjxkwBehRqlASg/duCPlXbsc2hu5y+iYwUt+8/N4w+8= +github.com/containers/buildah v1.37.3 h1:nSmbdBqaRMjvTtwVuOKZGT2jefaUKsZXbgpH9b4HzIs= +github.com/containers/buildah v1.37.3/go.mod h1:alFCM3X0xfhE6ZjsFQkUlOMyKzOnbv9FL9fe1Ho48PA= +github.com/containers/common v0.60.2 h1:utcwp2YkO8c0mNlwRxsxfOiqfj157FRrBjxgjR6f+7o= +github.com/containers/common v0.60.2/go.mod h1:I0upBi1qJX3QmzGbUOBN1LVP6RvkKhd3qQpZbQT+Q54= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/gvisor-tap-vsock v0.7.5 h1:bTy4u3DOmmUPwurL6me2rsgfypAFDhyeJleUcQmBR/E= github.com/containers/gvisor-tap-vsock v0.7.5/go.mod h1:GW9jOqAEEGdaS20XwTYdm6KCYDHIulOE/yEEOabkoE4= -github.com/containers/image/v5 v5.32.1-0.20240806084436-e3e9287ca8e6 h1:nXEEUAo8l2HLlMBy+LsHju2AikpA30jvlTSHbnjJXVw= -github.com/containers/image/v5 v5.32.1-0.20240806084436-e3e9287ca8e6/go.mod h1:r//zsX8SjmVH0F87d+gakcgR4W5HTFGSgSLB4sufW6A= +github.com/containers/image/v5 v5.32.2 h1:SzNE2Y6sf9b1GJoC8qjCuMBXwQrACFp4p0RK15+4gmQ= +github.com/containers/image/v5 v5.32.2/go.mod h1:v1l73VeMugfj/QtKI+jhYbwnwFCFnNGckvbST3rQ5Hk= github.com/containers/libhvee v0.7.1 h1:dWGF5GLq9DZvXo3P8aDp3cNieL5eCaSell4UmeA/jY4= github.com/containers/libhvee v0.7.1/go.mod h1:fRKB3AyIqHMvq6xaeYhTpckM2cdoq0oecolyoiuLP7M= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= diff --git a/vendor/github.com/containers/buildah/.packit.yaml b/vendor/github.com/containers/buildah/.packit.yaml index b465fec67d..b96a160535 100644 --- a/vendor/github.com/containers/buildah/.packit.yaml +++ b/vendor/github.com/containers/buildah/.packit.yaml @@ -14,6 +14,8 @@ packages: specfile_path: rpm/buildah.spec buildah-rhel: specfile_path: rpm/buildah.spec + buildah-eln: + specfile_path: rpm/buildah.spec srpm_build_deps: - make @@ -26,8 +28,21 @@ jobs: failure_comment: message: "Ephemeral COPR build failed. @containers/packit-build please check." targets: - fedora-all-x86_64: {} - fedora-all-aarch64: {} + - fedora-development-x86_64 + - fedora-development-aarch64 + - fedora-latest-x86_64 + - fedora-latest-aarch64 + - fedora-latest-stable-x86_64 + - fedora-latest-stable-aarch64 + - fedora-40-x86_64 + - fedora-40-aarch64 + enable_net: true + + - job: copr_build + trigger: pull_request + packages: [buildah-eln] + notifications: *copr_build_failure_notification + targets: fedora-eln-x86_64: additional_repos: - "https://kojipkgs.fedoraproject.org/repos/eln-build/latest/x86_64/" @@ -73,7 +88,7 @@ jobs: trigger: release packages: [buildah-fedora] update_release: false - dist_git_branches: + dist_git_branches: &fedora_targets - fedora-all # Sync to CentOS Stream @@ -84,12 +99,13 @@ jobs: dist_git_branches: - c10s + # Fedora Koji build - job: koji_build trigger: commit - dist_git_branches: - - fedora-all - - - job: bodhi_update - trigger: commit - dist_git_branches: - - fedora-branched # rawhide updates are created automatically + sidetag_group: podman-releases + # Dependents are not rpm dependencies, but the package whose bodhi update + # should include this package. + # Ref: https://packit.dev/docs/fedora-releases-guide/releasing-multiple-packages + dependents: + - podman + dist_git_branches: *fedora_targets diff --git a/vendor/github.com/containers/buildah/CHANGELOG.md b/vendor/github.com/containers/buildah/CHANGELOG.md index 0886d41906..bbe234f058 100644 --- a/vendor/github.com/containers/buildah/CHANGELOG.md +++ b/vendor/github.com/containers/buildah/CHANGELOG.md @@ -2,6 +2,23 @@ # Changelog +## v1.37.3 (2024-09-20) + + Do not error on trying to write IMA xattr as rootless + imagebuildah.StageExecutor: clean up volumes/volumeCache + `manifest add --artifact`: handle multiple values + Packit: split out ELN jobs and reuse fedora downstream targets + Packit: Enable sidetags for bodhi updates + Use Epoch: 2 and respect the epoch in dependencies. + +## v1.37.2 (2024-08-20) + + [release-1.37] Bump c/common to v0.60.2, c/image to v5.32.2 + +## v1.37.1 (2024-08-12) + + [release-1.37] Bump c/common v0.60.1, c/image v5.32.1 + ## vv1.37.0 (2024-07-26) Bump c/storage, c/image, c/common for v1.37.0 diff --git a/vendor/github.com/containers/buildah/changelog.txt b/vendor/github.com/containers/buildah/changelog.txt index 3d861f4663..50a226daac 100644 --- a/vendor/github.com/containers/buildah/changelog.txt +++ b/vendor/github.com/containers/buildah/changelog.txt @@ -1,3 +1,17 @@ +- Changelog for v1.37.3 (2024-09-20) + * Do not error on trying to write IMA xattr as rootless + * imagebuildah.StageExecutor: clean up volumes/volumeCache + * `manifest add --artifact`: handle multiple values + * Packit: split out ELN jobs and reuse fedora downstream targets + * Packit: Enable sidetags for bodhi updates + * Use Epoch: 2 and respect the epoch in dependencies. + +- Changelog for v1.37.2 (2024-08-20) + * [release-1.37] Bump c/common to v0.60.2, c/image to v5.32.2 + +- Changelog for v1.37.1 (2024-08-12) + * [release-1.37] Bump c/common v0.60.1, c/image v5.32.1 + - Changelog for vv1.37.0 (2024-07-26) * Bump c/storage, c/image, c/common for v1.37.0 * "build with basename resolving user arg" tests: correct ARG use diff --git a/vendor/github.com/containers/buildah/copier/xattrs.go b/vendor/github.com/containers/buildah/copier/xattrs.go index f5b2e731fd..c0d93ac15c 100644 --- a/vendor/github.com/containers/buildah/copier/xattrs.go +++ b/vendor/github.com/containers/buildah/copier/xattrs.go @@ -10,15 +10,18 @@ import ( "strings" "syscall" + "github.com/containers/storage/pkg/unshare" + "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) const ( xattrsSupported = true + imaXattr = "security.ima" ) var ( - relevantAttributes = []string{"security.capability", "security.ima", "user.*"} // the attributes that we preserve - we discard others + relevantAttributes = []string{"security.capability", imaXattr, "user.*"} // the attributes that we preserve - we discard others initialXattrListSize = 64 * 1024 initialXattrValueSize = 64 * 1024 ) @@ -93,7 +96,11 @@ func Lsetxattrs(path string, xattrs map[string]string) error { for attribute, value := range xattrs { if isRelevantXattr(attribute) { if err := unix.Lsetxattr(path, attribute, []byte(value), 0); err != nil { - return fmt.Errorf("setting value of extended attribute %q on %q: %w", attribute, path, err) + if unshare.IsRootless() && attribute == imaXattr { + logrus.Warnf("Unable to set %q xattr on %q: %v", attribute, path, err) + } else { + return fmt.Errorf("setting value of extended attribute %q on %q: %w", attribute, path, err) + } } } } diff --git a/vendor/github.com/containers/buildah/define/types.go b/vendor/github.com/containers/buildah/define/types.go index 6659a8161d..0fd7b5d921 100644 --- a/vendor/github.com/containers/buildah/define/types.go +++ b/vendor/github.com/containers/buildah/define/types.go @@ -29,7 +29,7 @@ const ( // identify working containers. Package = "buildah" // Version for the Package. Also used by .packit.sh for Packit builds. - Version = "1.37.0" + Version = "1.37.3" // DefaultRuntime if containers.conf fails. DefaultRuntime = "runc" diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index ca7374750e..9dc5d7bf56 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -143,7 +143,7 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B } contents, err = os.Open(dfile) if err != nil { - return "", nil, err + return "", nil, fmt.Errorf("reading build instructions: %w", err) } dinfo, err = contents.Stat() if err != nil { diff --git a/vendor/github.com/containers/buildah/imagebuildah/executor.go b/vendor/github.com/containers/buildah/imagebuildah/executor.go index 6579fd1712..0fcc66ac21 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/executor.go +++ b/vendor/github.com/containers/buildah/imagebuildah/executor.go @@ -222,7 +222,7 @@ func newExecutor(logger *logrus.Logger, logPrefix string, store storage.Store, o } else { rusageLogFile, err = os.OpenFile(options.RusageLogFile, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - return nil, err + return nil, fmt.Errorf("creating file to store rusage logs: %w", err) } } } diff --git a/vendor/github.com/containers/buildah/imagebuildah/stage_executor.go b/vendor/github.com/containers/buildah/imagebuildah/stage_executor.go index b8a01befef..34e836ad2f 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/stage_executor.go +++ b/vendor/github.com/containers/buildah/imagebuildah/stage_executor.go @@ -67,9 +67,9 @@ type StageExecutor struct { name string builder *buildah.Builder preserved int - volumes imagebuilder.VolumeSet - volumeCache map[string]string - volumeCacheInfo map[string]os.FileInfo + volumes imagebuilder.VolumeSet // list of directories which are volumes + volumeCache map[string]string // mapping from volume directories to cache archives (used by vfs method) + volumeCacheInfo map[string]os.FileInfo // mapping from volume directories to perms/datestamps to reset after restoring mountPoint string output string containerIDs []string @@ -92,7 +92,7 @@ type StageExecutor struct { // writeable while the RUN instruction is being handled, even if any changes // made within the directory are ultimately discarded. func (s *StageExecutor) Preserve(path string) error { - logrus.Debugf("PRESERVE %q in %q", path, s.builder.ContainerID) + logrus.Debugf("PRESERVE %q in %q (already preserving %v)", path, s.builder.ContainerID, s.volumes) // Try and resolve the symlink (if one exists) // Set archivedPath and path based on whether a symlink is found or not @@ -111,71 +111,61 @@ func (s *StageExecutor) Preserve(path string) error { return fmt.Errorf("evaluating path %q: %w", path, err) } + // Whether or not we're caching and restoring the contents of this + // directory, we need to ensure it exists now. const createdDirPerms = os.FileMode(0o755) - if s.executor.compatVolumes != types.OptionalBoolTrue { - logrus.Debugf("ensuring volume path %q exists", path) + st, err := os.Stat(archivedPath) + if errors.Is(err, os.ErrNotExist) { + // Yup, we do have to create it. That means it's not in any + // cached copy of the path that covers it, so we have to + // invalidate such cached copy. + logrus.Debugf("have to create volume %q", path) createdDirPerms := createdDirPerms if err := copier.Mkdir(s.mountPoint, archivedPath, copier.MkdirOptions{ChmodNew: &createdDirPerms}); err != nil { return fmt.Errorf("ensuring volume path exists: %w", err) } - logrus.Debugf("not doing volume save-and-restore of %q in %q", path, s.builder.ContainerID) - return nil + if err := s.volumeCacheInvalidate(path); err != nil { + return fmt.Errorf("ensuring volume path %q is preserved: %w", filepath.Join(s.mountPoint, path), err) + } + if st, err = os.Stat(archivedPath); err != nil { + return fmt.Errorf("checking on just-created volume path: %w", err) + } + } + if err != nil { + return fmt.Errorf("reading info cache for volume at %q: %w", path, err) } if s.volumes.Covers(path) { // This path is a subdirectory of a volume path that we're - // already preserving, so there's nothing new to be done except - // ensure that it exists. - st, err := os.Stat(archivedPath) - if errors.Is(err, os.ErrNotExist) { - // We do have to create it. That means it's not in any - // cached copy of the path that covers it, so we have - // to invalidate such cached copy. - logrus.Debugf("have to create volume %q", path) - createdDirPerms := createdDirPerms - if err := copier.Mkdir(s.mountPoint, filepath.Join(s.mountPoint, path), copier.MkdirOptions{ChmodNew: &createdDirPerms}); err != nil { - return fmt.Errorf("ensuring volume path exists: %w", err) - } - if err := s.volumeCacheInvalidate(path); err != nil { - return fmt.Errorf("ensuring volume path %q is preserved: %w", filepath.Join(s.mountPoint, path), err) - } - if st, err = os.Stat(archivedPath); err != nil { - return fmt.Errorf("checking on just-created volume path: %w", err) - } - } + // already preserving, so there's nothing new to be done now + // that we've ensured that it exists. s.volumeCacheInfo[path] = st return nil } - // Figure out where the cache for this volume would be stored. - s.preserved++ - cacheDir, err := s.executor.store.ContainerDirectory(s.builder.ContainerID) - if err != nil { - return fmt.Errorf("unable to locate temporary directory for container") - } - cacheFile := filepath.Join(cacheDir, fmt.Sprintf("volume%d.tar", s.preserved)) - - // Save info about the top level of the location that we'll be archiving. - st, err := os.Stat(archivedPath) - if errors.Is(err, os.ErrNotExist) { - logrus.Debugf("have to create volume %q", path) - createdDirPerms := os.FileMode(0o755) - if err = copier.Mkdir(s.mountPoint, archivedPath, copier.MkdirOptions{ChmodNew: &createdDirPerms}); err != nil { - return fmt.Errorf("ensuring volume path exists: %w", err) - } - st, err = os.Stat(archivedPath) - } - if err != nil { - logrus.Debugf("error reading info about %q: %v", archivedPath, err) - return err - } - s.volumeCacheInfo[path] = st + // Add the new volume path to the ones that we're tracking. if !s.volumes.Add(path) { // This path is not a subdirectory of a volume path that we're // already preserving, so adding it to the list should have // worked. return fmt.Errorf("adding %q to the volume cache", path) } + s.volumeCacheInfo[path] = st + + // If we're not doing save/restore, we're done, since volumeCache + // should be empty. + if s.executor.compatVolumes != types.OptionalBoolTrue { + logrus.Debugf("not doing volume save-and-restore of %q in %q", path, s.builder.ContainerID) + return nil + } + + // Decide where the cache for this volume will be stored. + s.preserved++ + cacheDir, err := s.executor.store.ContainerDirectory(s.builder.ContainerID) + if err != nil { + return fmt.Errorf("unable to locate temporary directory for container") + } + cacheFile := filepath.Join(cacheDir, fmt.Sprintf("volume%d.tar", s.preserved)) s.volumeCache[path] = cacheFile // Now prune cache files for volumes that are newly supplanted by this one. @@ -206,7 +196,7 @@ func (s *StageExecutor) Preserve(path string) error { if errors.Is(err, os.ErrNotExist) { continue } - return err + return fmt.Errorf("removing cache of %q: %w", archivedPath, err) } delete(s.volumeCache, cachedPath) } @@ -256,16 +246,12 @@ func (s *StageExecutor) volumeCacheSaveVFS() (mounts []specs.Mount, err error) { continue } if !errors.Is(err, os.ErrNotExist) { - return nil, err - } - createdDirPerms := os.FileMode(0755) - if err := copier.Mkdir(s.mountPoint, archivedPath, copier.MkdirOptions{ChmodNew: &createdDirPerms}); err != nil { - return nil, fmt.Errorf("ensuring volume path exists: %w", err) + return nil, fmt.Errorf("checking for presence of a cached copy of %q at %q: %w", cachedPath, cacheFile, err) } logrus.Debugf("caching contents of volume %q in %q", archivedPath, cacheFile) cache, err := os.Create(cacheFile) if err != nil { - return nil, err + return nil, fmt.Errorf("creating cache for volume %q: %w", archivedPath, err) } defer cache.Close() rc, err := chrootarchive.Tar(archivedPath, nil, s.mountPoint) @@ -298,16 +284,12 @@ func (s *StageExecutor) volumeCacheRestoreVFS() (err error) { logrus.Debugf("restoring contents of volume %q from %q", archivedPath, cacheFile) cache, err := os.Open(cacheFile) if err != nil { - return err + return fmt.Errorf("restoring contents of volume %q: %w", archivedPath, err) } defer cache.Close() if err := copier.Remove(s.mountPoint, archivedPath, copier.RemoveOptions{All: true}); err != nil { return err } - createdDirPerms := os.FileMode(0o755) - if err := copier.Mkdir(s.mountPoint, archivedPath, copier.MkdirOptions{ChmodNew: &createdDirPerms}); err != nil { - return err - } err = chrootarchive.Untar(cache, archivedPath, nil) if err != nil { return fmt.Errorf("extracting archive at %q: %w", archivedPath, err) @@ -334,13 +316,11 @@ func (s *StageExecutor) volumeCacheRestoreVFS() (err error) { } // Save the contents of each of the executor's list of volumes for which we -// don't already have a cache file. +// don't already have a cache file. For overlay, we "save" and "restore" by +// using it as a lower for an overlay mount in the same location, and then +// discarding the upper. func (s *StageExecutor) volumeCacheSaveOverlay() (mounts []specs.Mount, err error) { for cachedPath := range s.volumeCache { - err = copier.Mkdir(s.mountPoint, filepath.Join(s.mountPoint, cachedPath), copier.MkdirOptions{}) - if err != nil { - return nil, fmt.Errorf("ensuring volume exists: %w", err) - } volumePath := filepath.Join(s.mountPoint, cachedPath) mount := specs.Mount{ Source: volumePath, @@ -1079,6 +1059,7 @@ func (s *StageExecutor) prepare(ctx context.Context, from string, initializeIBCo s.mountPoint = mountPoint s.builder = builder // Now that the rootfs is mounted, set up handling of volumes from the base image. + s.volumes = make([]string, 0, len(s.volumes)) s.volumeCache = make(map[string]string) s.volumeCacheInfo = make(map[string]os.FileInfo) for _, v := range builder.Volumes() { diff --git a/vendor/github.com/containers/common/libimage/copier.go b/vendor/github.com/containers/common/libimage/copier.go index 5151343ed6..1f4f925fe5 100644 --- a/vendor/github.com/containers/common/libimage/copier.go +++ b/vendor/github.com/containers/common/libimage/copier.go @@ -60,13 +60,6 @@ type CopyOptions struct { CertDirPath string // Force layer compression when copying to a `dir` transport destination. DirForceCompress bool - - // ImageListSelection is one of CopySystemImage, CopyAllImages, or - // CopySpecificImages, to control whether, when the source reference is a list, - // copy.Image() copies only an image which matches the current runtime - // environment, or all images which match the supplied reference, or only - // specific images from the source reference. - ImageListSelection copy.ImageListSelection // Allow contacting registries over HTTP, or HTTPS with failed TLS // verification. Note that this does not affect other TLS connections. InsecureSkipTLSVerify types.OptionalBool @@ -213,17 +206,13 @@ func getDockerAuthConfig(name, passwd, creds, idToken string) (*types.DockerAuth } } -// NewCopier is a simple, exported wrapper for newCopier -func NewCopier(options *CopyOptions, sc *types.SystemContext) (*copier, error) { - return newCopier(options, sc) -} - // newCopier creates a copier. Note that fields in options *may* overwrite the // counterparts of the specified system context. Please make sure to call // `(*copier).close()`. -func newCopier(options *CopyOptions, sc *types.SystemContext) (*copier, error) { +func (r *Runtime) newCopier(options *CopyOptions) (*copier, error) { c := copier{extendTimeoutSocket: options.extendTimeoutSocket} - c.systemContext = sc + c.systemContext = r.systemContextCopy() + if options.SourceLookupReferenceFunc != nil { c.sourceLookup = options.SourceLookupReferenceFunc } @@ -311,7 +300,6 @@ func newCopier(options *CopyOptions, sc *types.SystemContext) (*copier, error) { c.imageCopyOptions.ProgressInterval = time.Second } - c.imageCopyOptions.ImageListSelection = options.ImageListSelection c.imageCopyOptions.ForceCompressionFormat = options.ForceCompressionFormat c.imageCopyOptions.ForceManifestMIMEType = options.ManifestMIMEType c.imageCopyOptions.SourceCtx = c.systemContext @@ -337,22 +325,14 @@ func newCopier(options *CopyOptions, sc *types.SystemContext) (*copier, error) { return &c, nil } -// newCopier creates a copier. Note that fields in options *may* overwrite the -// counterparts of the specified system context. Please make sure to call -// `(*copier).close()`. -func (r *Runtime) newCopier(options *CopyOptions) (*copier, error) { - sc := r.systemContextCopy() - return newCopier(options, sc) -} - -// Close open resources. -func (c *copier) Close() error { +// close open resources. +func (c *copier) close() error { return c.policyContext.Destroy() } -// Copy the source to the destination. Returns the bytes of the copied +// copy the source to the destination. Returns the bytes of the copied // manifest which may be used for digest computation. -func (c *copier) Copy(ctx context.Context, source, destination types.ImageReference) ([]byte, error) { +func (c *copier) copy(ctx context.Context, source, destination types.ImageReference) ([]byte, error) { logrus.Debugf("Copying source image %s to destination image %s", source.StringWithinTransport(), destination.StringWithinTransport()) // Avoid running out of time when running inside a systemd unit by diff --git a/vendor/github.com/containers/common/libimage/disk_usage.go b/vendor/github.com/containers/common/libimage/disk_usage.go index 7b7b56ad7d..6264b25ec9 100644 --- a/vendor/github.com/containers/common/libimage/disk_usage.go +++ b/vendor/github.com/containers/common/libimage/disk_usage.go @@ -31,12 +31,12 @@ type ImageDiskUsage struct { // storage. Note that a single image may yield multiple usage reports, one for // each repository tag. func (r *Runtime) DiskUsage(ctx context.Context) ([]ImageDiskUsage, int64, error) { - images, layers, err := r.getImagesAndLayers() + images, err := r.ListImages(ctx, nil, nil) if err != nil { return nil, -1, err } - layerTree, err := r.newLayerTreeFromData(images, layers) + layerTree, err := r.layerTree(ctx, images) if err != nil { return nil, -1, err } diff --git a/vendor/github.com/containers/common/libimage/filters.go b/vendor/github.com/containers/common/libimage/filters.go index aa11d54f53..2465d370d2 100644 --- a/vendor/github.com/containers/common/libimage/filters.go +++ b/vendor/github.com/containers/common/libimage/filters.go @@ -19,22 +19,19 @@ import ( // filterFunc is a prototype for a positive image filter. Returning `true` // indicates that the image matches the criteria. -type filterFunc func(*Image, *layerTree) (bool, error) - -type compiledFilters map[string][]filterFunc +type filterFunc func(*Image) (bool, error) // Apply the specified filters. All filters of each key must apply. -// tree must be provided if compileImageFilters indicated it is necessary. -func (i *Image) applyFilters(ctx context.Context, filters compiledFilters, tree *layerTree) (bool, error) { +func (i *Image) applyFilters(ctx context.Context, filters map[string][]filterFunc) (bool, error) { for key := range filters { for _, filter := range filters[key] { - matches, err := filter(i, tree) + matches, err := filter(i) if err != nil { // Some images may have been corrupted in the // meantime, so do an extra check and make the // error non-fatal (see containers/podman/issues/12582). if errCorrupted := i.isCorrupted(ctx, ""); errCorrupted != nil { - logrus.Error(errCorrupted.Error()) + logrus.Errorf(errCorrupted.Error()) return false, nil } return false, err @@ -50,11 +47,18 @@ func (i *Image) applyFilters(ctx context.Context, filters compiledFilters, tree // filterImages returns a slice of images which are passing all specified // filters. -// tree must be provided if compileImageFilters indicated it is necessary. -func (r *Runtime) filterImages(ctx context.Context, images []*Image, filters compiledFilters, tree *layerTree) ([]*Image, error) { +func (r *Runtime) filterImages(ctx context.Context, images []*Image, options *ListImagesOptions) ([]*Image, error) { + if len(options.Filters) == 0 || len(images) == 0 { + return images, nil + } + + filters, err := r.compileImageFilters(ctx, options) + if err != nil { + return nil, err + } result := []*Image{} for i := range images { - match, err := images[i].applyFilters(ctx, filters, tree) + match, err := images[i].applyFilters(ctx, filters) if err != nil { return nil, err } @@ -69,19 +73,25 @@ func (r *Runtime) filterImages(ctx context.Context, images []*Image, filters com // required format is `key=value` with the following supported keys: // // after, since, before, containers, dangling, id, label, readonly, reference, intermediate -// -// compileImageFilters returns: compiled filters, if LayerTree is needed, error -func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOptions) (compiledFilters, bool, error) { +func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOptions) (map[string][]filterFunc, error) { logrus.Tracef("Parsing image filters %s", options.Filters) - if len(options.Filters) == 0 { - return nil, false, nil + + var tree *layerTree + getTree := func() (*layerTree, error) { + if tree == nil { + t, err := r.layerTree(ctx, nil) + if err != nil { + return nil, err + } + tree = t + } + return tree, nil } filterInvalidValue := `invalid image filter %q: must be in the format "filter=value or filter!=value"` var wantedReferenceMatches, unwantedReferenceMatches []string - filters := compiledFilters{} - needsLayerTree := false + filters := map[string][]filterFunc{} duplicate := map[string]string{} for _, f := range options.Filters { var key, value string @@ -93,7 +103,7 @@ func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOp } else { split = strings.SplitN(f, "=", 2) if len(split) != 2 { - return nil, false, fmt.Errorf(filterInvalidValue, f) + return nil, fmt.Errorf(filterInvalidValue, f) } } @@ -103,7 +113,7 @@ func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOp case "after", "since": img, err := r.time(key, value) if err != nil { - return nil, false, err + return nil, err } key = "since" filter = filterAfter(img.Created()) @@ -111,23 +121,27 @@ func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOp case "before": img, err := r.time(key, value) if err != nil { - return nil, false, err + return nil, err } filter = filterBefore(img.Created()) case "containers": if err := r.containers(duplicate, key, value, options.IsExternalContainerFunc); err != nil { - return nil, false, err + return nil, err } filter = filterContainers(value, options.IsExternalContainerFunc) case "dangling": dangling, err := r.bool(duplicate, key, value) if err != nil { - return nil, false, err + return nil, err + } + t, err := getTree() + if err != nil { + return nil, err } - needsLayerTree = true - filter = filterDangling(ctx, dangling) + + filter = filterDangling(ctx, dangling, t) case "id": filter = filterID(value) @@ -135,31 +149,35 @@ func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOp case "digest": f, err := filterDigest(value) if err != nil { - return nil, false, err + return nil, err } filter = f case "intermediate": intermediate, err := r.bool(duplicate, key, value) if err != nil { - return nil, false, err + return nil, err + } + t, err := getTree() + if err != nil { + return nil, err } - needsLayerTree = true - filter = filterIntermediate(ctx, intermediate) + + filter = filterIntermediate(ctx, intermediate, t) case "label": filter = filterLabel(ctx, value) case "readonly": readOnly, err := r.bool(duplicate, key, value) if err != nil { - return nil, false, err + return nil, err } filter = filterReadOnly(readOnly) case "manifest": manifest, err := r.bool(duplicate, key, value) if err != nil { - return nil, false, err + return nil, err } filter = filterManifest(ctx, manifest) @@ -174,12 +192,12 @@ func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOp case "until": until, err := r.until(value) if err != nil { - return nil, false, err + return nil, err } filter = filterBefore(until) default: - return nil, false, fmt.Errorf(filterInvalidValue, key) + return nil, fmt.Errorf(filterInvalidValue, key) } if negate { filter = negateFilter(filter) @@ -192,12 +210,12 @@ func (r *Runtime) compileImageFilters(ctx context.Context, options *ListImagesOp filter := filterReferences(r, wantedReferenceMatches, unwantedReferenceMatches) filters["reference"] = append(filters["reference"], filter) - return filters, needsLayerTree, nil + return filters, nil } func negateFilter(f filterFunc) filterFunc { - return func(img *Image, tree *layerTree) (bool, error) { - b, err := f(img, tree) + return func(img *Image) (bool, error) { + b, err := f(img) return !b, err } } @@ -254,7 +272,7 @@ func (r *Runtime) bool(duplicate map[string]string, key, value string) (bool, er // filterManifest filters whether or not the image is a manifest list func filterManifest(ctx context.Context, value bool) filterFunc { - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { isManifestList, err := img.IsManifestList(ctx) if err != nil { return false, err @@ -266,7 +284,7 @@ func filterManifest(ctx context.Context, value bool) filterFunc { // filterReferences creates a reference filter for matching the specified wantedReferenceMatches value (OR logic) // and for matching the unwantedReferenceMatches values (AND logic) func filterReferences(r *Runtime, wantedReferenceMatches, unwantedReferenceMatches []string) filterFunc { - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { // Empty reference filters, return true if len(wantedReferenceMatches) == 0 && len(unwantedReferenceMatches) == 0 { return true, nil @@ -358,7 +376,7 @@ func imageMatchesReferenceFilter(r *Runtime, img *Image, value string) (bool, er // filterLabel creates a label for matching the specified value. func filterLabel(ctx context.Context, value string) filterFunc { - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { labels, err := img.Labels(ctx) if err != nil { return false, err @@ -369,28 +387,28 @@ func filterLabel(ctx context.Context, value string) filterFunc { // filterAfter creates an after filter for matching the specified value. func filterAfter(value time.Time) filterFunc { - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { return img.Created().After(value), nil } } // filterBefore creates a before filter for matching the specified value. func filterBefore(value time.Time) filterFunc { - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { return img.Created().Before(value), nil } } // filterReadOnly creates a readonly filter for matching the specified value. func filterReadOnly(value bool) filterFunc { - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { return img.IsReadOnly() == value, nil } } // filterContainers creates a container filter for matching the specified value. func filterContainers(value string, fn IsExternalContainerFunc) filterFunc { - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { ctrs, err := img.Containers() if err != nil { return false, err @@ -415,8 +433,8 @@ func filterContainers(value string, fn IsExternalContainerFunc) filterFunc { } // filterDangling creates a dangling filter for matching the specified value. -func filterDangling(ctx context.Context, value bool) filterFunc { - return func(img *Image, tree *layerTree) (bool, error) { +func filterDangling(ctx context.Context, value bool, tree *layerTree) filterFunc { + return func(img *Image) (bool, error) { isDangling, err := img.isDangling(ctx, tree) if err != nil { return false, err @@ -427,7 +445,7 @@ func filterDangling(ctx context.Context, value bool) filterFunc { // filterID creates an image-ID filter for matching the specified value. func filterID(value string) filterFunc { - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { return strings.HasPrefix(img.ID(), value), nil } } @@ -437,7 +455,7 @@ func filterDigest(value string) (filterFunc, error) { if !strings.HasPrefix(value, "sha256:") { return nil, fmt.Errorf("invalid value %q for digest filter", value) } - return func(img *Image, _ *layerTree) (bool, error) { + return func(img *Image) (bool, error) { return img.containsDigestPrefix(value), nil }, nil } @@ -445,8 +463,8 @@ func filterDigest(value string) (filterFunc, error) { // filterIntermediate creates an intermediate filter for images. An image is // considered to be an intermediate image if it is dangling (i.e., no tags) and // has no children (i.e., no other image depends on it). -func filterIntermediate(ctx context.Context, value bool) filterFunc { - return func(img *Image, tree *layerTree) (bool, error) { +func filterIntermediate(ctx context.Context, value bool, tree *layerTree) filterFunc { + return func(img *Image) (bool, error) { isIntermediate, err := img.isIntermediate(ctx, tree) if err != nil { return false, err diff --git a/vendor/github.com/containers/common/libimage/history.go b/vendor/github.com/containers/common/libimage/history.go index 845f4088e2..56f84e37aa 100644 --- a/vendor/github.com/containers/common/libimage/history.go +++ b/vendor/github.com/containers/common/libimage/history.go @@ -25,7 +25,7 @@ func (i *Image) History(ctx context.Context) ([]ImageHistory, error) { return nil, err } - layerTree, err := i.runtime.newFreshLayerTree() + layerTree, err := i.runtime.layerTree(ctx, nil) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/common/libimage/image.go b/vendor/github.com/containers/common/libimage/image.go index a971b6c67e..9cc77cdb25 100644 --- a/vendor/github.com/containers/common/libimage/image.go +++ b/vendor/github.com/containers/common/libimage/image.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/containerd/platforms" + "github.com/containerd/containerd/platforms" "github.com/containers/common/libimage/platform" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/manifest" @@ -257,7 +257,7 @@ func (i *Image) TopLayer() string { // Parent returns the parent image or nil if there is none func (i *Image) Parent(ctx context.Context) (*Image, error) { - tree, err := i.runtime.newFreshLayerTree() + tree, err := i.runtime.layerTree(ctx, nil) if err != nil { return nil, err } @@ -291,7 +291,7 @@ func (i *Image) Children(ctx context.Context) ([]*Image, error) { // created for this invocation only. func (i *Image) getChildren(ctx context.Context, all bool, tree *layerTree) ([]*Image, error) { if tree == nil { - t, err := i.runtime.newFreshLayerTree() + t, err := i.runtime.layerTree(ctx, nil) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/common/libimage/image_tree.go b/vendor/github.com/containers/common/libimage/image_tree.go index 9628a319f1..8b9d1f4c47 100644 --- a/vendor/github.com/containers/common/libimage/image_tree.go +++ b/vendor/github.com/containers/common/libimage/image_tree.go @@ -3,6 +3,7 @@ package libimage import ( + "context" "fmt" "strings" @@ -37,7 +38,7 @@ func (i *Image) Tree(traverseChildren bool) (string, error) { fmt.Fprintf(sb, "No Image Layers") } - layerTree, err := i.runtime.newFreshLayerTree() + layerTree, err := i.runtime.layerTree(context.Background(), nil) if err != nil { return "", err } diff --git a/vendor/github.com/containers/common/libimage/import.go b/vendor/github.com/containers/common/libimage/import.go index a03f288533..552c48eae0 100644 --- a/vendor/github.com/containers/common/libimage/import.go +++ b/vendor/github.com/containers/common/libimage/import.go @@ -108,9 +108,9 @@ func (r *Runtime) Import(ctx context.Context, path string, options *ImportOption if err != nil { return "", err } - defer c.Close() + defer c.close() - if _, err := c.Copy(ctx, srcRef, destRef); err != nil { + if _, err := c.copy(ctx, srcRef, destRef); err != nil { return "", err } diff --git a/vendor/github.com/containers/common/libimage/layer_tree.go b/vendor/github.com/containers/common/libimage/layer_tree.go index aeb1d45bb3..9af120e252 100644 --- a/vendor/github.com/containers/common/libimage/layer_tree.go +++ b/vendor/github.com/containers/common/libimage/layer_tree.go @@ -89,18 +89,21 @@ func (l *layerNode) repoTags() ([]string, error) { return orderedTags, nil } -// newFreshLayerTree extracts a layerTree from consistent layers and images in the local storage. -func (r *Runtime) newFreshLayerTree() (*layerTree, error) { - images, layers, err := r.getImagesAndLayers() +// layerTree extracts a layerTree from the layers in the local storage and +// relates them to the specified images. +func (r *Runtime) layerTree(ctx context.Context, images []*Image) (*layerTree, error) { + layers, err := r.store.Layers() if err != nil { return nil, err } - return r.newLayerTreeFromData(images, layers) -} -// newLayerTreeFromData extracts a layerTree from the given the layers and images. -// The caller is responsible for (layers, images) being consistent. -func (r *Runtime) newLayerTreeFromData(images []*Image, layers []storage.Layer) (*layerTree, error) { + if images == nil { + images, err = r.ListImages(ctx, nil, nil) + if err != nil { + return nil, err + } + } + tree := layerTree{ nodes: make(map[string]*layerNode), ociCache: make(map[string]*ociv1.Image), diff --git a/vendor/github.com/containers/common/libimage/manifest_list.go b/vendor/github.com/containers/common/libimage/manifest_list.go index a23315da3c..1a6004c01a 100644 --- a/vendor/github.com/containers/common/libimage/manifest_list.go +++ b/vendor/github.com/containers/common/libimage/manifest_list.go @@ -7,28 +7,20 @@ import ( "errors" "fmt" "maps" - "os" - "path/filepath" "slices" "time" "github.com/containers/common/libimage/define" "github.com/containers/common/libimage/manifests" - manifesterrors "github.com/containers/common/pkg/manifests" - "github.com/containers/common/pkg/supplemented" imageCopy "github.com/containers/image/v5/copy" "github.com/containers/image/v5/docker" "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/oci/layout" - "github.com/containers/image/v5/signature" "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/types" "github.com/containers/storage" structcopier "github.com/jinzhu/copier" "github.com/opencontainers/go-digest" - imgspec "github.com/opencontainers/image-spec/specs-go" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sirupsen/logrus" ) // NOTE: the abstractions and APIs here are a first step to further merge @@ -109,157 +101,8 @@ func (r *Runtime) lookupManifestList(name string) (*Image, manifests.List, error return image, list, nil } -// ConvertToManifestList converts the image into a manifest list if it is not -// already also a list. An error is returned if the conversion fails. -func (i *Image) ConvertToManifestList(ctx context.Context) (*ManifestList, error) { - // If we don't need to do anything, don't do anything. - if list, err := i.ToManifestList(); err == nil || !errors.Is(err, ErrNotAManifestList) { - return list, err - } - - // Determine which type we prefer for the new manifest list or image index. - _, imageManifestType, err := i.Manifest(ctx) - if err != nil { - return nil, fmt.Errorf("reading the image's manifest: %w", err) - } - var preferredListType string - switch imageManifestType { - case manifest.DockerV2Schema2MediaType, - manifest.DockerV2Schema1SignedMediaType, - manifest.DockerV2Schema1MediaType, - manifest.DockerV2ListMediaType: - preferredListType = manifest.DockerV2ListMediaType - case imgspecv1.MediaTypeImageManifest, imgspecv1.MediaTypeImageIndex: - preferredListType = imgspecv1.MediaTypeImageIndex - default: - preferredListType = "" - } - - // Create a list and add the image's manifest to it. Use OCI format - // for now. If we need to convert it to Docker format, we'll do that - // while copying it. - list := manifests.Create() - if _, err := list.Add(ctx, &i.runtime.systemContext, i.storageReference, false); err != nil { - return nil, fmt.Errorf("generating new image index: %w", err) - } - listBytes, err := list.Serialize(imgspecv1.MediaTypeImageIndex) - if err != nil { - return nil, fmt.Errorf("serializing image index: %w", err) - } - listDigest, err := manifest.Digest(listBytes) - if err != nil { - return nil, fmt.Errorf("digesting image index: %w", err) - } - - // Build an OCI layout containing the image index as the only item. - tmp, err := os.MkdirTemp("", "") - if err != nil { - return nil, fmt.Errorf("serializing initial list: %w", err) - } - defer os.RemoveAll(tmp) - - // Drop our image index in there. - if err := os.Mkdir(filepath.Join(tmp, imgspecv1.ImageBlobsDir), 0o755); err != nil { - return nil, fmt.Errorf("creating directory for blobs: %w", err) - } - if err := os.Mkdir(filepath.Join(tmp, imgspecv1.ImageBlobsDir, listDigest.Algorithm().String()), 0o755); err != nil { - return nil, fmt.Errorf("creating directory for %s blobs: %w", listDigest.Algorithm().String(), err) - } - listFile := filepath.Join(tmp, imgspecv1.ImageBlobsDir, listDigest.Algorithm().String(), listDigest.Encoded()) - if err := os.WriteFile(listFile, listBytes, 0o644); err != nil { - return nil, fmt.Errorf("writing image index for OCI layout: %w", err) - } - - // Build the index for the layout. - index := imgspecv1.Index{ - Versioned: imgspec.Versioned{ - SchemaVersion: 2, - }, - MediaType: imgspecv1.MediaTypeImageIndex, - Manifests: []imgspecv1.Descriptor{{ - MediaType: imgspecv1.MediaTypeImageIndex, - Digest: listDigest, - Size: int64(len(listBytes)), - }}, - } - indexBytes, err := json.Marshal(&index) - if err != nil { - return nil, fmt.Errorf("encoding image index for OCI layout: %w", err) - } - - // Write the index for the layout. - indexFile := filepath.Join(tmp, imgspecv1.ImageIndexFile) - if err := os.WriteFile(indexFile, indexBytes, 0o644); err != nil { - return nil, fmt.Errorf("writing top-level index for OCI layout: %w", err) - } - - // Write the "why yes, this is an OCI layout" file. - layoutFile := filepath.Join(tmp, imgspecv1.ImageLayoutFile) - layoutBytes, err := json.Marshal(imgspecv1.ImageLayout{Version: imgspecv1.ImageLayoutVersion}) - if err != nil { - return nil, fmt.Errorf("encoding image layout structure for OCI layout: %w", err) - } - if err := os.WriteFile(layoutFile, layoutBytes, 0o644); err != nil { - return nil, fmt.Errorf("writing oci-layout file: %w", err) - } - - // Build an OCI layout reference to use as a source. - tmpRef, err := layout.NewReference(tmp, "") - if err != nil { - return nil, fmt.Errorf("creating reference to directory: %w", err) - } - bundle := supplemented.Reference(tmpRef, []types.ImageReference{i.storageReference}, imageCopy.CopySystemImage, nil) - - // Build a policy that ensures we don't prevent ourselves from reading - // this reference. - signaturePolicy, err := signature.DefaultPolicy(&i.runtime.systemContext) - if err != nil { - return nil, fmt.Errorf("obtaining default signature policy: %w", err) - } - acceptAnything := signature.PolicyTransportScopes{ - "": []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}, - } - signaturePolicy.Transports[i.storageReference.Transport().Name()] = acceptAnything - signaturePolicy.Transports[tmpRef.Transport().Name()] = acceptAnything - policyContext, err := signature.NewPolicyContext(signaturePolicy) - if err != nil { - return nil, fmt.Errorf("creating new signature policy context: %w", err) - } - defer func() { - if err2 := policyContext.Destroy(); err2 != nil { - logrus.Errorf("Destroying signature policy context: %v", err2) - } - }() - - // Copy from the OCI layout into the same image record, so that it gets - // both its own manifest and the image index. - copyOptions := imageCopy.Options{ - ForceManifestMIMEType: imageManifestType, - } - if _, err := imageCopy.Image(ctx, policyContext, i.storageReference, bundle, ©Options); err != nil { - return nil, fmt.Errorf("writing updates to image: %w", err) - } - - // Now explicitly write the list's manifest to the image as its "main" - // manifest. - if _, err := list.SaveToImage(i.runtime.store, i.ID(), i.storageImage.Names, preferredListType); err != nil { - return nil, fmt.Errorf("saving image index: %w", err) - } - - // Reload the record. - if err = i.reload(); err != nil { - return nil, fmt.Errorf("reloading image record: %w", err) - } - mList, err := i.runtime.LookupManifestList(i.storageImage.ID) - if err != nil { - return nil, fmt.Errorf("looking up new manifest list: %w", err) - } - - return mList, nil -} - // ToManifestList converts the image into a manifest list. An error is thrown -// if the image is not a manifest list. +// if the image is no manifest list. func (i *Image) ToManifestList() (*ManifestList, error) { list, err := i.getManifestList() if err != nil { @@ -296,7 +139,7 @@ func (m *ManifestList) LookupInstance(ctx context.Context, architecture, os, var return nil, err } - allImages, err := m.image.runtime.ListImages(ctx, nil) + allImages, err := m.image.runtime.ListImages(ctx, nil, nil) if err != nil { return nil, err } @@ -351,9 +194,6 @@ func (m *ManifestList) reload() error { // getManifestList is a helper to obtain a manifest list func (i *Image) getManifestList() (manifests.List, error) { _, list, err := manifests.LoadFromImage(i.runtime.store, i.ID()) - if errors.Is(err, manifesterrors.ErrManifestTypeNotSupported) { - err = fmt.Errorf("%s: %w", err.Error(), ErrNotAManifestList) - } return list, err } @@ -796,7 +636,7 @@ func (m *ManifestList) Push(ctx context.Context, destination string, options *Ma if err != nil { return "", err } - defer copier.Close() + defer copier.close() pushOptions := manifests.PushOptions{ AddCompression: options.AddCompression, diff --git a/vendor/github.com/containers/common/libimage/manifests/manifests.go b/vendor/github.com/containers/common/libimage/manifests/manifests.go index 50c32569ac..a1625bb1bf 100644 --- a/vendor/github.com/containers/common/libimage/manifests/manifests.go +++ b/vendor/github.com/containers/common/libimage/manifests/manifests.go @@ -342,7 +342,8 @@ func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, in } } // write the index that refers to this one artifact image - indexFile := filepath.Join(tmp, v1.ImageIndexFile) + tag := "latest" + indexFile := filepath.Join(tmp, "index.json") index := v1.Index{ Versioned: imgspec.Versioned{ SchemaVersion: 2, @@ -352,6 +353,9 @@ func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, in MediaType: v1.MediaTypeImageManifest, Digest: artifactManifestDigest, Size: int64(len(contents)), + Annotations: map[string]string{ + v1.AnnotationRefName: tag, + }, }}, } indexBytes, err := json.Marshal(&index) @@ -362,16 +366,12 @@ func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, in return nil, fmt.Errorf("writing image index for OCI layout: %w", err) } // write the layout file - layoutFile := filepath.Join(tmp, v1.ImageLayoutFile) - layoutBytes, err := json.Marshal(v1.ImageLayout{Version: v1.ImageLayoutVersion}) - if err != nil { - return nil, fmt.Errorf("encoding image layout for OCI layout: %w", err) - } - if err := os.WriteFile(layoutFile, layoutBytes, 0o644); err != nil { + layoutFile := filepath.Join(tmp, "oci-layout") + if err := os.WriteFile(layoutFile, []byte(`{"imageLayoutVersion": "1.0.0"}`), 0o644); err != nil { return nil, fmt.Errorf("writing oci-layout file: %w", err) } // build the reference to this artifact image's oci layout - ref, err := ocilayout.NewReference(tmp, "") + ref, err := ocilayout.NewReference(tmp, tag) if err != nil { return nil, fmt.Errorf("creating ImageReference for artifact with files %q: %w", symlinkedFiles, err) } @@ -676,14 +676,14 @@ func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.Imag // This should provide for all of the ways to construct a manifest outlined in // https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage -// - no blobs ￫ set ManifestArtifactType -// - blobs, no configuration ￫ set ManifestArtifactType and possibly LayerMediaType, and provide file names -// - blobs and configuration ￫ set ManifestArtifactType, possibly LayerMediaType, and ConfigDescriptor, and provide file names +// * no blobs ￫ set ManifestArtifactType +// * blobs, no configuration ￫ set ManifestArtifactType and possibly LayerMediaType, and provide file names +// * blobs and configuration ￫ set ManifestArtifactType, possibly LayerMediaType, and ConfigDescriptor, and provide file names // // The older style of describing artifacts: -// - leave ManifestArtifactType blank -// - specify a zero-length application/vnd.oci.image.config.v1+json config blob -// - set LayerMediaType to a custom type +// * leave ManifestArtifactType blank +// * specify a zero-length application/vnd.oci.image.config.v1+json config blob +// * set LayerMediaType to a custom type // // When reading data produced elsewhere, note that newer tooling will produce // manifests with ArtifactType set. If the manifest's ArtifactType is not set, diff --git a/vendor/github.com/containers/common/libimage/platform/platform.go b/vendor/github.com/containers/common/libimage/platform/platform.go index 54c3b8deb4..2272cee757 100644 --- a/vendor/github.com/containers/common/libimage/platform/platform.go +++ b/vendor/github.com/containers/common/libimage/platform/platform.go @@ -4,7 +4,7 @@ import ( "fmt" "runtime" - "github.com/containerd/platforms" + "github.com/containerd/containerd/platforms" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" ) diff --git a/vendor/github.com/containers/common/libimage/pull.go b/vendor/github.com/containers/common/libimage/pull.go index c4ad5df0c7..3db1b2992b 100644 --- a/vendor/github.com/containers/common/libimage/pull.go +++ b/vendor/github.com/containers/common/libimage/pull.go @@ -235,7 +235,7 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference, if err != nil { return nil, err } - defer c.Close() + defer c.close() // Figure out a name for the storage destination. var storageName, imageName string @@ -321,7 +321,7 @@ func (r *Runtime) copyFromDefault(ctx context.Context, ref types.ImageReference, return nil, fmt.Errorf("parsing %q: %w", storageName, err) } - _, err = c.Copy(ctx, ref, destRef) + _, err = c.copy(ctx, ref, destRef) return []string{imageName}, err } @@ -391,7 +391,7 @@ func (r *Runtime) copyFromDockerArchiveReaderReference(ctx context.Context, read if err != nil { return nil, err } - defer c.Close() + defer c.close() // Get a slice of storage references we can copy. references, destNames, err := r.storageReferencesReferencesFromArchiveReader(ctx, readerRef, reader) @@ -401,7 +401,7 @@ func (r *Runtime) copyFromDockerArchiveReaderReference(ctx context.Context, read // Now copy all of the images. Use readerRef for performance. for _, destRef := range references { - if _, err := c.Copy(ctx, readerRef, destRef); err != nil { + if _, err := c.copy(ctx, readerRef, destRef); err != nil { return nil, err } } @@ -640,7 +640,7 @@ func (r *Runtime) copySingleImageFromRegistry(ctx context.Context, imageName str if err != nil { return nil, err } - defer c.Close() + defer c.close() var pullErrors []error for _, candidate := range resolved.PullCandidates { @@ -678,7 +678,7 @@ func (r *Runtime) copySingleImageFromRegistry(ctx context.Context, imageName str } } var manifestBytes []byte - if manifestBytes, err = c.Copy(ctx, srcRef, destRef); err != nil { + if manifestBytes, err = c.copy(ctx, srcRef, destRef); err != nil { logrus.Debugf("Error pulling candidate %s: %v", candidateString, err) pullErrors = append(pullErrors, err) continue diff --git a/vendor/github.com/containers/common/libimage/push.go b/vendor/github.com/containers/common/libimage/push.go index 5db6cfbcfe..f89b8fc070 100644 --- a/vendor/github.com/containers/common/libimage/push.go +++ b/vendor/github.com/containers/common/libimage/push.go @@ -114,7 +114,7 @@ func (r *Runtime) Push(ctx context.Context, source, destination string, options return nil, err } - defer c.Close() + defer c.close() - return c.Copy(ctx, srcRef, destRef) + return c.copy(ctx, srcRef, destRef) } diff --git a/vendor/github.com/containers/common/libimage/runtime.go b/vendor/github.com/containers/common/libimage/runtime.go index f7c9b2bf97..1baf41d5d2 100644 --- a/vendor/github.com/containers/common/libimage/runtime.go +++ b/vendor/github.com/containers/common/libimage/runtime.go @@ -161,23 +161,6 @@ func (r *Runtime) storageToImage(storageImage *storage.Image, ref types.ImageRef } } -// getImagesAndLayers obtains consistent slices of Image and storage.Layer -func (r *Runtime) getImagesAndLayers() ([]*Image, []storage.Layer, error) { - snapshot, err := r.store.MultiList( - storage.MultiListOptions{ - Images: true, - Layers: true, - }) - if err != nil { - return nil, nil, err - } - images := []*Image{} - for i := range snapshot.Images { - images = append(images, r.storageToImage(&snapshot.Images[i], nil)) - } - return images, snapshot.Layers, nil -} - // Exists returns true if the specified image exists in the local containers // storage. Note that it may return false if an image corrupted. func (r *Runtime) Exists(name string) (bool, error) { @@ -496,7 +479,7 @@ func (r *Runtime) lookupImageInDigestsAndRepoTags(name string, possiblyUnqualifi return nil, "", fmt.Errorf("%s: %w (could not cast to tagged)", originalName, storage.ErrImageUnknown) } - allImages, err := r.ListImages(context.Background(), nil) + allImages, err := r.ListImages(context.Background(), nil, nil) if err != nil { return nil, "", err } @@ -584,46 +567,39 @@ type ListImagesOptions struct { SetListData bool } -// ListImagesByNames lists the images in the local container storage by specified names -// The name lookups use the LookupImage semantics. -func (r *Runtime) ListImagesByNames(names []string) ([]*Image, error) { - images := []*Image{} - for _, name := range names { - image, _, err := r.LookupImage(name, nil) - if err != nil { - return nil, err - } - images = append(images, image) - } - return images, nil -} - -// ListImages lists the images in the local container storage and filter the images by ListImagesOptions -func (r *Runtime) ListImages(ctx context.Context, options *ListImagesOptions) ([]*Image, error) { +// ListImages lists images in the local container storage. If names are +// specified, only images with the specified names are looked up and filtered. +func (r *Runtime) ListImages(ctx context.Context, names []string, options *ListImagesOptions) ([]*Image, error) { if options == nil { options = &ListImagesOptions{} } - filters, needsLayerTree, err := r.compileImageFilters(ctx, options) - if err != nil { - return nil, err - } - - if options.SetListData { - needsLayerTree = true + var images []*Image + if len(names) > 0 { + for _, name := range names { + image, _, err := r.LookupImage(name, nil) + if err != nil { + return nil, err + } + images = append(images, image) + } + } else { + storageImages, err := r.store.Images() + if err != nil { + return nil, err + } + for i := range storageImages { + images = append(images, r.storageToImage(&storageImages[i], nil)) + } } - snapshot, err := r.store.MultiList( - storage.MultiListOptions{ - Images: true, - Layers: needsLayerTree, - }) + filtered, err := r.filterImages(ctx, images, options) if err != nil { return nil, err } - images := []*Image{} - for i := range snapshot.Images { - images = append(images, r.storageToImage(&snapshot.Images[i], nil)) + + if !options.SetListData { + return filtered, nil } // If explicitly requested by the user, pre-compute and cache the @@ -632,23 +608,11 @@ func (r *Runtime) ListImages(ctx context.Context, options *ListImagesOptions) ([ // as the layer tree will computed once for all instead of once for // each individual image (see containers/podman/issues/17828). - var tree *layerTree - if needsLayerTree { - tree, err = r.newLayerTreeFromData(images, snapshot.Layers) - if err != nil { - return nil, err - } - } - - filtered, err := r.filterImages(ctx, images, filters, tree) + tree, err := r.layerTree(ctx, images) if err != nil { return nil, err } - if !options.SetListData { - return filtered, nil - } - for i := range filtered { isDangling, err := filtered[i].isDangling(ctx, tree) if err != nil { @@ -789,7 +753,7 @@ func (r *Runtime) RemoveImages(ctx context.Context, names []string, options *Rem IsExternalContainerFunc: options.IsExternalContainerFunc, Filters: options.Filters, } - filteredImages, err := r.ListImages(ctx, options) + filteredImages, err := r.ListImages(ctx, nil, options) if err != nil { appendError(err) return nil, rmErrors diff --git a/vendor/github.com/containers/common/libimage/save.go b/vendor/github.com/containers/common/libimage/save.go index 46529d10f3..62cad3288d 100644 --- a/vendor/github.com/containers/common/libimage/save.go +++ b/vendor/github.com/containers/common/libimage/save.go @@ -123,9 +123,9 @@ func (r *Runtime) saveSingleImage(ctx context.Context, name, format, path string if err != nil { return err } - defer c.Close() + defer c.close() - _, err = c.Copy(ctx, srcRef, destRef) + _, err = c.copy(ctx, srcRef, destRef) return err } @@ -208,7 +208,7 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st if err != nil { return err } - defer c.Close() + defer c.close() destRef, err := writer.NewReference(nil) if err != nil { @@ -220,7 +220,7 @@ func (r *Runtime) saveDockerArchive(ctx context.Context, names []string, path st return err } - if _, err := c.Copy(ctx, srcRef, destRef); err != nil { + if _, err := c.copy(ctx, srcRef, destRef); err != nil { return err } } diff --git a/vendor/github.com/containers/common/libimage/search.go b/vendor/github.com/containers/common/libimage/search.go index 6ab016b3b5..b26ad80d26 100644 --- a/vendor/github.com/containers/common/libimage/search.go +++ b/vendor/github.com/containers/common/libimage/search.go @@ -215,7 +215,7 @@ func (r *Runtime) searchImageInRegistry(ctx context.Context, term, registry stri } paramsArr := []SearchResult{} - for i := range limit { + for i := 0; i < limit; i++ { // Check whether query matches filters if !(filterMatchesAutomatedFilter(&options.Filter, results[i]) && filterMatchesOfficialFilter(&options.Filter, results[i]) && filterMatchesStarFilter(&options.Filter, results[i])) { continue @@ -275,7 +275,7 @@ func searchRepositoryTags(ctx context.Context, sys *types.SystemContext, registr } } paramsArr := []SearchResult{} - for i := range limit { + for i := 0; i < limit; i++ { params := SearchResult{ Name: imageRef.DockerReference().Name(), Tag: tags[i], diff --git a/vendor/github.com/containers/common/libnetwork/cni/run.go b/vendor/github.com/containers/common/libnetwork/cni/run.go index 68a42945c0..337a27b8ef 100644 --- a/vendor/github.com/containers/common/libnetwork/cni/run.go +++ b/vendor/github.com/containers/common/libnetwork/cni/run.go @@ -62,6 +62,7 @@ func (n *cniNetwork) Setup(namespacePath string, options types.SetupOptions) (ma } for name, netOpts := range options.Networks { + netOpts := netOpts network := n.networks[name] rt := getRuntimeConfig(namespacePath, options.ContainerName, options.ContainerID, name, ports, &netOpts) @@ -236,6 +237,7 @@ func (n *cniNetwork) teardown(namespacePath string, options types.TeardownOption var multiErr *multierror.Error teardown := func() error { for name, netOpts := range options.Networks { + netOpts := netOpts rt := getRuntimeConfig(namespacePath, options.ContainerName, options.ContainerID, name, ports, &netOpts) cniConfList, newRt, err := getCachedNetworkConfig(n.cniConf, name, rt) diff --git a/vendor/github.com/containers/common/libnetwork/etchosts/hosts.go b/vendor/github.com/containers/common/libnetwork/etchosts/hosts.go index 8a7cc534e6..7ca62137b2 100644 --- a/vendor/github.com/containers/common/libnetwork/etchosts/hosts.go +++ b/vendor/github.com/containers/common/libnetwork/etchosts/hosts.go @@ -229,10 +229,9 @@ func checkIfEntryExists(current HostEntry, entries HostEntries) bool { return false } -// parseExtraHosts converts a slice of "name1;name2;name3:ip" string to entries. -// Each entry can contain one or more hostnames separated by semicolons and an IP address separated by a colon. -// Because podman and buildah both store the extra hosts in this format, -// we convert it here instead of having to do this on the caller side. +// parseExtraHosts converts a slice of "name:ip" string to entries. +// Because podman and buildah both store the extra hosts in this format +// we convert it here instead of having to this on the caller side. func parseExtraHosts(extraHosts []string, hostContainersInternalIP string) (HostEntries, error) { entries := make(HostEntries, 0, len(extraHosts)) for _, entry := range extraHosts { @@ -253,8 +252,7 @@ func parseExtraHosts(extraHosts []string, hostContainersInternalIP string) (Host } ip = hostContainersInternalIP } - names := strings.Split(values[0], ";") - e := HostEntry{IP: ip, Names: names} + e := HostEntry{IP: ip, Names: []string{values[0]}} entries = append(entries, e) } return entries, nil diff --git a/vendor/github.com/containers/common/libnetwork/etchosts/ip.go b/vendor/github.com/containers/common/libnetwork/etchosts/ip.go index 588fbff352..909fcf67d2 100644 --- a/vendor/github.com/containers/common/libnetwork/etchosts/ip.go +++ b/vendor/github.com/containers/common/libnetwork/etchosts/ip.go @@ -10,27 +10,17 @@ import ( "github.com/containers/storage/pkg/unshare" ) -// HostContainersInternalOptions contains the options for GetHostContainersInternalIP() -type HostContainersInternalOptions struct { - // Conf is the containers.Conf, must not be nil - Conf *config.Config - // NetStatus is the network status for the container, - // if this is set networkInterface must not be nil - NetStatus map[string]types.StatusBlock - // NetworkInterface of the current runtime - NetworkInterface types.ContainerNetwork - // Exclude are then ips that should not be returned, this is - // useful to prevent returning the same ip as in the container. - Exclude []net.IP - // PreferIP is a ip that should be used if set but it has a - // lower priority than the containers.conf config option. - // This is used for the pasta --map-guest-addr ip. - PreferIP string +// GetHostContainersInternalIP returns the host.containers.internal ip +// if netStatus is not nil then networkInterface also must be non nil otherwise this function panics +func GetHostContainersInternalIP(conf *config.Config, netStatus map[string]types.StatusBlock, networkInterface types.ContainerNetwork) string { + return GetHostContainersInternalIPExcluding(conf, netStatus, networkInterface, nil) } -// GetHostContainersInternalIP returns the host.containers.internal ip -func GetHostContainersInternalIP(opts HostContainersInternalOptions) string { - switch opts.Conf.Containers.HostContainersInternalIP { +// GetHostContainersInternalIPExcluding returns the host.containers.internal ip +// Exclude are ips that should not be returned, this is useful to prevent returning the same ip as in the container. +// if netStatus is not nil then networkInterface also must be non nil otherwise this function panics +func GetHostContainersInternalIPExcluding(conf *config.Config, netStatus map[string]types.StatusBlock, networkInterface types.ContainerNetwork, exclude []net.IP) string { + switch conf.Containers.HostContainersInternalIP { case "": // if empty (default) we will automatically choose one below // if machine using gvproxy we let the gvproxy dns server handle the dns name so do not add it @@ -40,22 +30,16 @@ func GetHostContainersInternalIP(opts HostContainersInternalOptions) string { case "none": return "" default: - return opts.Conf.Containers.HostContainersInternalIP - } - - // caller has a specific ip it prefers - if opts.PreferIP != "" { - return opts.PreferIP + return conf.Containers.HostContainersInternalIP } - ip := "" // Only use the bridge ip when root, as rootless the interfaces are created // inside the special netns and not the host so we cannot use them. if unshare.IsRootless() { - return util.GetLocalIPExcluding(opts.Exclude) + return util.GetLocalIPExcluding(exclude) } - for net, status := range opts.NetStatus { - network, err := opts.NetworkInterface.NetworkInspect(net) + for net, status := range netStatus { + network, err := networkInterface.NetworkInspect(net) // only add the host entry for bridge networks // ip/macvlan gateway is normally not on the host if err != nil || network.Driver != types.BridgeNetworkDriver { @@ -76,19 +60,7 @@ func GetHostContainersInternalIP(opts HostContainersInternalOptions) string { if ip != "" { return ip } - return util.GetLocalIPExcluding(opts.Exclude) -} - -// GetHostContainersInternalIPExcluding returns the host.containers.internal ip -// Exclude are ips that should not be returned, this is useful to prevent returning the same ip as in the container. -// if netStatus is not nil then networkInterface also must be non nil otherwise this function panics -func GetHostContainersInternalIPExcluding(conf *config.Config, netStatus map[string]types.StatusBlock, networkInterface types.ContainerNetwork, exclude []net.IP) string { - return GetHostContainersInternalIP(HostContainersInternalOptions{ - Conf: conf, - NetStatus: netStatus, - NetworkInterface: networkInterface, - Exclude: exclude, - }) + return util.GetLocalIPExcluding(exclude) } // GetNetworkHostEntries returns HostEntries for all ips in the network status diff --git a/vendor/github.com/containers/common/libnetwork/internal/rootlessnetns/netns_linux.go b/vendor/github.com/containers/common/libnetwork/internal/rootlessnetns/netns_linux.go index 84bc4f621a..f8b9b76dfd 100644 --- a/vendor/github.com/containers/common/libnetwork/internal/rootlessnetns/netns_linux.go +++ b/vendor/github.com/containers/common/libnetwork/internal/rootlessnetns/netns_linux.go @@ -1,7 +1,6 @@ package rootlessnetns import ( - "encoding/json" "errors" "fmt" "io/fs" @@ -35,9 +34,6 @@ const ( // refCountFile file name for the ref count file refCountFile = "ref-count" - // infoCacheFile file name for the cache file used to store the rootless netns info - infoCacheFile = "info.json" - // rootlessNetNsConnPidFile is the name of the rootless netns slirp4netns/pasta pid file rootlessNetNsConnPidFile = "rootless-netns-conn.pid" @@ -58,9 +54,11 @@ type Netns struct { // config contains containers.conf options. config *config.Config - // info contain information about ip addresses used in the netns. - // A caller can get this info via Info(). - info *types.RootlessNetnsInfo + // ipAddresses used in the netns, this is needed to store + // the netns ips that are used by pasta. This is then handed + // back to the caller via IPAddresses() which then can make + // sure to not use them for host.containers.internal. + ipAddresses []net.IP } type rootlessNetnsError struct { @@ -117,9 +115,6 @@ func (n *Netns) getOrCreateNetns() (ns.NetNS, bool, error) { // quick check if pasta/slirp4netns are still running err := unix.Kill(pid, 0) if err == nil { - if err := n.deserializeInfo(); err != nil { - return nil, false, wrapError("deserialize info", err) - } // All good, return the netns. return nsRef, false, nil } @@ -232,15 +227,6 @@ func (n *Netns) setupPasta(nsPath string) error { return wrapError("create resolv.conf", err) } - n.info = &types.RootlessNetnsInfo{ - IPAddresses: res.IPAddresses, - DnsForwardIps: res.DNSForwardIPs, - MapGuestIps: res.MapGuestAddrIPs, - } - if err := n.serializeInfo(); err != nil { - return wrapError("serialize info", err) - } - return nil } @@ -275,12 +261,6 @@ func (n *Netns) setupSlirp4netns(nsPath string) error { if err != nil { return wrapError("determine default slirp4netns DNS address", err) } - nameservers := []string{resolveIP.String()} - - netnsIP, err := slirp4netns.GetIP(res.Subnet) - if err != nil { - return wrapError("determine default slirp4netns ip address", err) - } if err := resolvconf.New(&resolvconf.Params{ Path: n.getPath(resolvConfName), @@ -290,19 +270,10 @@ func (n *Netns) setupSlirp4netns(nsPath string) error { }, IPv6Enabled: res.IPv6, KeepHostServers: true, - Nameservers: nameservers, + Nameservers: []string{resolveIP.String()}, }); err != nil { return wrapError("create resolv.conf", err) } - - n.info = &types.RootlessNetnsInfo{ - IPAddresses: []net.IP{*netnsIP}, - DnsForwardIps: nameservers, - } - if err := n.serializeInfo(); err != nil { - return wrapError("serialize info", err) - } - return nil } @@ -570,6 +541,20 @@ func (n *Netns) runInner(toRun func() error, cleanup bool) (err error) { if err := toRun(); err != nil { return err } + + // get the current active addresses in the netns, and store them + addrs, err := net.InterfaceAddrs() + if err != nil { + return err + } + ips := make([]net.IP, 0, len(addrs)) + for _, addr := range addrs { + // make sure to skip localhost and other special addresses + if ipnet, ok := addr.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() { + ips = append(ips, ipnet.IP) + } + } + n.ipAddresses = ips return nil }) } @@ -645,7 +630,9 @@ func (n *Netns) Run(lock *lockfile.LockFile, toRun func() error) error { // IPAddresses returns the currently used ip addresses in the netns // These should then not be assigned for the host.containers.internal entry. func (n *Netns) Info() *types.RootlessNetnsInfo { - return n.info + return &types.RootlessNetnsInfo{ + IPAddresses: n.ipAddresses, + } } func refCount(dir string, inc int) (int, error) { @@ -684,26 +671,3 @@ func readPidFile(path string) (int, error) { } return strconv.Atoi(strings.TrimSpace(string(b))) } - -func (n *Netns) serializeInfo() error { - f, err := os.Create(filepath.Join(n.dir, infoCacheFile)) - if err != nil { - return err - } - return json.NewEncoder(f).Encode(n.info) -} - -func (n *Netns) deserializeInfo() error { - f, err := os.Open(filepath.Join(n.dir, infoCacheFile)) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil - } - return err - } - defer f.Close() - if n.info == nil { - n.info = new(types.RootlessNetnsInfo) - } - return json.NewDecoder(f).Decode(n.info) -} diff --git a/vendor/github.com/containers/common/libnetwork/internal/util/util.go b/vendor/github.com/containers/common/libnetwork/internal/util/util.go index 260e74d06e..96c21e76f6 100644 --- a/vendor/github.com/containers/common/libnetwork/internal/util/util.go +++ b/vendor/github.com/containers/common/libnetwork/internal/util/util.go @@ -112,7 +112,7 @@ func GetFreeIPv4NetworkSubnet(usedNetworks []*net.IPNet, subnetPools []config.Su // GetFreeIPv6NetworkSubnet returns a unused ipv6 subnet func GetFreeIPv6NetworkSubnet(usedNetworks []*net.IPNet) (*types.Subnet, error) { // FIXME: Is 10000 fine as limit? We should prevent an endless loop. - for range 10000 { + for i := 0; i < 10000; i++ { // RFC4193: Choose the ipv6 subnet random and NOT sequentially. network, err := getRandomIPv6Subnet() if err != nil { diff --git a/vendor/github.com/containers/common/libnetwork/internal/util/validate.go b/vendor/github.com/containers/common/libnetwork/internal/util/validate.go index ddc778a5fe..55995440e4 100644 --- a/vendor/github.com/containers/common/libnetwork/internal/util/validate.go +++ b/vendor/github.com/containers/common/libnetwork/internal/util/validate.go @@ -129,6 +129,7 @@ func ValidateSetupOptions(n NetUtil, namespacePath string, options types.SetupOp return errors.New("must specify at least one network") } for name, netOpts := range options.Networks { + netOpts := netOpts network, err := n.Network(name) if err != nil { return err diff --git a/vendor/github.com/containers/common/libnetwork/netavark/config.go b/vendor/github.com/containers/common/libnetwork/netavark/config.go index 8b43a787e4..842c7e3122 100644 --- a/vendor/github.com/containers/common/libnetwork/netavark/config.go +++ b/vendor/github.com/containers/common/libnetwork/netavark/config.go @@ -126,7 +126,7 @@ func (n *netavarkNetwork) networkCreate(newNetwork *types.Network, defaultNet bo // generate random network ID var i int - for i = range 1000 { + for i = 0; i < 1000; i++ { id := stringid.GenerateNonCryptoID() if _, err := n.getNetwork(id); err != nil { newNetwork.ID = id diff --git a/vendor/github.com/containers/common/libnetwork/network/interface_cni.go b/vendor/github.com/containers/common/libnetwork/network/interface_cni.go index 3de3a3045e..0a8bc44724 100644 --- a/vendor/github.com/containers/common/libnetwork/network/interface_cni.go +++ b/vendor/github.com/containers/common/libnetwork/network/interface_cni.go @@ -1,4 +1,6 @@ //go:build (linux || freebsd) && cni +// +build linux freebsd +// +build cni package network diff --git a/vendor/github.com/containers/common/libnetwork/network/interface_cni_unsupported.go b/vendor/github.com/containers/common/libnetwork/network/interface_cni_unsupported.go index d0645dc506..2f4bb83718 100644 --- a/vendor/github.com/containers/common/libnetwork/network/interface_cni_unsupported.go +++ b/vendor/github.com/containers/common/libnetwork/network/interface_cni_unsupported.go @@ -1,4 +1,6 @@ //go:build (linux || freebsd) && !cni +// +build linux freebsd +// +build !cni package network diff --git a/vendor/github.com/containers/common/libnetwork/pasta/pasta_linux.go b/vendor/github.com/containers/common/libnetwork/pasta/pasta_linux.go index 10b667eb6c..fd68f87f27 100644 --- a/vendor/github.com/containers/common/libnetwork/pasta/pasta_linux.go +++ b/vendor/github.com/containers/common/libnetwork/pasta/pasta_linux.go @@ -26,16 +26,11 @@ import ( ) const ( - dnsForwardOpt = "--dns-forward" - mapGuestAddrOpt = "--map-guest-addr" + dnsForwardOpt = "--dns-forward" // dnsForwardIpv4 static ip used as nameserver address inside the netns, // given this is a "link local" ip it should be very unlikely that it causes conflicts - dnsForwardIpv4 = "169.254.1.1" - - // mapGuestAddrIpv4 static ip used as forwarder address inside the netns to reach the host, - // given this is a "link local" ip it should be very unlikely that it causes conflicts - mapGuestAddrIpv4 = "169.254.1.2" + dnsForwardIpv4 = "169.254.0.1" ) type SetupOptions struct { @@ -50,61 +45,45 @@ type SetupOptions struct { ExtraOptions []string } -// Setup2 alias for Setup() -func Setup2(opts *SetupOptions) (*SetupResult, error) { - return Setup(opts) +func Setup(opts *SetupOptions) error { + _, err := Setup2(opts) + return err } -// Setup start the pasta process for the given netns. +// Setup2 start the pasta process for the given netns. // The pasta binary is looked up in the HelperBinariesDir and $PATH. // Note that there is no need for any special cleanup logic, the pasta // process will automatically exit when the netns path is deleted. -func Setup(opts *SetupOptions) (*SetupResult, error) { +func Setup2(opts *SetupOptions) (*SetupResult, error) { path, err := opts.Config.FindHelperBinary(BinaryName, true) if err != nil { return nil, fmt.Errorf("could not find pasta, the network namespace can't be configured: %w", err) } - cmdArgs, dnsForwardIPs, mapGuestAddrIPs, err := createPastaArgs(opts) + cmdArgs, dnsForwardIPs, err := createPastaArgs(opts) if err != nil { return nil, err } logrus.Debugf("pasta arguments: %s", strings.Join(cmdArgs, " ")) - for { - // pasta forks once ready, and quits once we delete the target namespace - out, err := exec.Command(path, cmdArgs...).CombinedOutput() - if err != nil { - exitErr := &exec.ExitError{} - if errors.As(err, &exitErr) { - // special backwards compat check, --map-guest-addr was added in pasta version 20240814 so we - // cannot hard require it yet. Once we are confident that the update is most distros we can remove it. - if exitErr.ExitCode() == 1 && - strings.Contains(string(out), "unrecognized option '"+mapGuestAddrOpt) && - len(mapGuestAddrIPs) == 1 && mapGuestAddrIPs[0] == mapGuestAddrIpv4 { - // we did add the default --map-guest-addr option, if users set something different we want - // to get to the error below. We have to unset mapGuestAddrIPs here to avoid a infinite loop. - mapGuestAddrIPs = nil - // Trim off last two args which are --map-guest-addr 169.254.1.2. - cmdArgs = cmdArgs[:len(cmdArgs)-2] - continue - } - return nil, fmt.Errorf("pasta failed with exit code %d:\n%s", - exitErr.ExitCode(), string(out)) - } - return nil, fmt.Errorf("failed to start pasta: %w", err) + // pasta forks once ready, and quits once we delete the target namespace + out, err := exec.Command(path, cmdArgs...).CombinedOutput() + if err != nil { + exitErr := &exec.ExitError{} + if errors.As(err, &exitErr) { + return nil, fmt.Errorf("pasta failed with exit code %d:\n%s", + exitErr.ExitCode(), string(out)) } + return nil, fmt.Errorf("failed to start pasta: %w", err) + } - if len(out) > 0 { - // TODO: This should be warning but as of August 2024 pasta still prints - // things with --quiet that we do not care about. In podman CI I still see - // "Couldn't get any nameserver address" so until this is fixed we cannot - // enable it. For now info is fine and we can bump it up later, it is only a - // nice to have. - logrus.Infof("pasta logged warnings: %q", strings.TrimSpace(string(out))) - } - break + if len(out) > 0 { + // TODO: This should be warning but right now pasta still prints + // things with --quiet that we do not care about. + // For now info is fine and we can bump it up later, it is only a + // nice to have. + logrus.Infof("pasta logged warnings: %q", string(out)) } var ipv4, ipv6 bool @@ -133,27 +112,19 @@ func Setup(opts *SetupOptions) (*SetupResult, error) { } result.IPv6 = ipv6 - result.DNSForwardIPs = filterIpFamily(dnsForwardIPs, ipv4, ipv6) - result.MapGuestAddrIPs = filterIpFamily(mapGuestAddrIPs, ipv4, ipv6) - - return result, nil -} - -func filterIpFamily(ips []string, ipv4, ipv6 bool) []string { - var result []string - for _, ip := range ips { + for _, ip := range dnsForwardIPs { ipp := net.ParseIP(ip) - // add the ip only if the address family matches + // add the namesever ip only if the address family matches if ipv4 && util.IsIPv4(ipp) || ipv6 && util.IsIPv6(ipp) { - result = append(result, ip) + result.DNSForwardIPs = append(result.DNSForwardIPs, ip) } } - return result + + return result, nil } -// createPastaArgs creates the pasta arguments, it returns the args to be passed to pasta(1) -// and as second arg the dns forward ips used. As third arg the map guest addr ips used. -func createPastaArgs(opts *SetupOptions) ([]string, []string, []string, error) { +// createPastaArgs creates the pasta arguments, it returns the args to be passed to pasta(1) and as second arg the dns forward ips used. +func createPastaArgs(opts *SetupOptions) ([]string, []string, error) { noTCPInitPorts := true noUDPInitPorts := true noTCPNamespacePorts := true @@ -178,7 +149,6 @@ func createPastaArgs(opts *SetupOptions) ([]string, []string, []string, error) { }) var dnsForwardIPs []string - var mapGuestAddrIPs []string for i, opt := range cmdArgs { switch opt { case "-t", "--tcp-ports": @@ -196,10 +166,6 @@ func createPastaArgs(opts *SetupOptions) ([]string, []string, []string, error) { if len(cmdArgs) > i+1 { dnsForwardIPs = append(dnsForwardIPs, cmdArgs[i+1]) } - case mapGuestAddrOpt: - if len(cmdArgs) > i+1 { - mapGuestAddrIPs = append(mapGuestAddrIPs, cmdArgs[i+1]) - } } } @@ -220,7 +186,7 @@ func createPastaArgs(opts *SetupOptions) ([]string, []string, []string, error) { noUDPInitPorts = false cmdArgs = append(cmdArgs, "-u") default: - return nil, nil, nil, fmt.Errorf("can't forward protocol: %s", protocol) + return nil, nil, fmt.Errorf("can't forward protocol: %s", protocol) } arg := fmt.Sprintf("%s%d-%d:%d-%d", addr, @@ -260,13 +226,5 @@ func createPastaArgs(opts *SetupOptions) ([]string, []string, []string, error) { cmdArgs = append(cmdArgs, "--netns", opts.Netns) - // do this as last arg so we can easily trim them off in the error case when we have an older version - if len(mapGuestAddrIPs) == 0 { - // the user did not request custom --map-guest-addr so add our own so that we can use this - // for our own host.containers.internal host entry. - cmdArgs = append(cmdArgs, mapGuestAddrOpt, mapGuestAddrIpv4) - mapGuestAddrIPs = append(mapGuestAddrIPs, mapGuestAddrIpv4) - } - - return cmdArgs, dnsForwardIPs, mapGuestAddrIPs, nil + return cmdArgs, dnsForwardIPs, nil } diff --git a/vendor/github.com/containers/common/libnetwork/pasta/types.go b/vendor/github.com/containers/common/libnetwork/pasta/types.go index f570afc3e5..b601e51695 100644 --- a/vendor/github.com/containers/common/libnetwork/pasta/types.go +++ b/vendor/github.com/containers/common/libnetwork/pasta/types.go @@ -10,9 +10,6 @@ type SetupResult struct { // DNSForwardIP is the ip used in --dns-forward, it should be added as first // entry to resolv.conf in the container. DNSForwardIPs []string - // MapGuestIps are the ips used for the --map-guest-addr option which - // we can use for the host.containers.internal entry. - MapGuestAddrIPs []string // IPv6 says whenever pasta run with ipv6 support IPv6 bool } diff --git a/vendor/github.com/containers/common/libnetwork/slirp4netns/slirp4netns.go b/vendor/github.com/containers/common/libnetwork/slirp4netns/slirp4netns.go index 9721750abf..6f713431c8 100644 --- a/vendor/github.com/containers/common/libnetwork/slirp4netns/slirp4netns.go +++ b/vendor/github.com/containers/common/libnetwork/slirp4netns/slirp4netns.go @@ -645,7 +645,7 @@ func setupRootlessPortMappingViaSlirp(ports []types.PortMapping, cmd *exec.Cmd, if hostIP == "" { hostIP = "0.0.0.0" } - for i := range port.Range { + for i := uint16(0); i < port.Range; i++ { if err := openSlirp4netnsPort(apiSocket, protocol, hostIP, port.HostPort+i, port.ContainerPort+i); err != nil { return err } diff --git a/vendor/github.com/containers/common/libnetwork/types/network.go b/vendor/github.com/containers/common/libnetwork/types/network.go index 77c76bf787..9741103f5b 100644 --- a/vendor/github.com/containers/common/libnetwork/types/network.go +++ b/vendor/github.com/containers/common/libnetwork/types/network.go @@ -342,10 +342,6 @@ type TeardownOptions struct { type RootlessNetnsInfo struct { // IPAddresses used in the netns, must not be used for host.containers.internal IPAddresses []net.IP - // DnsForwardIps ips used in resolv.conf - DnsForwardIps []string - // MapGuestIps should be used for the host.containers.internal entry when set - MapGuestIps []string } // FilterFunc can be passed to NetworkList to filter the networks. diff --git a/vendor/github.com/containers/common/pkg/cgroups/cgroups_linux.go b/vendor/github.com/containers/common/pkg/cgroups/cgroups_linux.go index d2c610fb15..717685789b 100644 --- a/vendor/github.com/containers/common/pkg/cgroups/cgroups_linux.go +++ b/vendor/github.com/containers/common/pkg/cgroups/cgroups_linux.go @@ -95,8 +95,7 @@ func init() { func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) { if cgroup2 { controllers := []controller{} - controllersFile := filepath.Join(cgroupRoot, "cgroup.controllers") - + controllersFile := cgroupRoot + "/cgroup.controllers" // rootless cgroupv2: check available controllers for current user, systemd or servicescope will inherit if unshare.IsRootless() { userSlice, err := getCgroupPathForCurrentProcess() @@ -105,7 +104,7 @@ func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) } // userSlice already contains '/' so not adding here basePath := cgroupRoot + userSlice - controllersFile = filepath.Join(basePath, "cgroup.controllers") + controllersFile = basePath + "/cgroup.controllers" } controllersFileBytes, err := os.ReadFile(controllersFile) if err != nil { @@ -598,7 +597,7 @@ func createCgroupv2Path(path string) (deferredError error) { if !strings.HasPrefix(path, cgroupRoot+"/") { return fmt.Errorf("invalid cgroup path %s", path) } - content, err := os.ReadFile(filepath.Join(cgroupRoot, "cgroup.controllers")) + content, err := os.ReadFile(cgroupRoot + "/cgroup.controllers") if err != nil { return err } @@ -626,43 +625,8 @@ func createCgroupv2Path(path string) (deferredError error) { // We enable the controllers for all the path components except the last one. It is not allowed to add // PIDs if there are already enabled controllers. if i < len(elements[3:])-1 { - subtreeControl := filepath.Join(current, "cgroup.subtree_control") - if err := os.WriteFile(subtreeControl, res, 0o755); err != nil { - // The kernel returns ENOENT either if the file itself is missing, or a controller - if errors.Is(err, os.ErrNotExist) { - if err2 := fileutils.Exists(subtreeControl); err2 != nil { - // If the file itself is missing, return the original error. - return err - } - repeatAttempts := 1000 - for repeatAttempts > 0 { - // store the controllers that failed to be enabled, so we can retry them - newCtrs := [][]byte{} - for _, ctr := range ctrs { - // Try to enable each controller individually, at least we can give a better error message if any fails. - if err := os.WriteFile(subtreeControl, []byte(fmt.Sprintf("+%s\n", ctr)), 0o755); err != nil { - // The kernel can return EBUSY when a process was moved to a sub-cgroup - // and the controllers are enabled in its parent cgroup. Retry a few times when - // it happens. - if errors.Is(err, unix.EBUSY) { - newCtrs = append(newCtrs, ctr) - } else { - return fmt.Errorf("enabling controller %s: %w", ctr, err) - } - } - } - if len(newCtrs) == 0 { - err = nil - break - } - ctrs = newCtrs - repeatAttempts-- - time.Sleep(time.Millisecond) - } - if err != nil { - return err - } - } + if err := os.WriteFile(filepath.Join(current, "cgroup.subtree_control"), res, 0o755); err != nil { + return err } } } @@ -822,12 +786,6 @@ func UserOwnsCurrentSystemdCgroup() (bool, error) { continue } - // If we are on a cgroup v2 system and there are cgroup v1 controllers - // mounted, ignore them when the current process is at the root cgroup. - if cgroup2 && parts[1] != "" && parts[2] == "/" { - continue - } - var cgroupPath string if cgroup2 { diff --git a/vendor/github.com/containers/common/pkg/cgroups/systemd_linux.go b/vendor/github.com/containers/common/pkg/cgroups/systemd_linux.go index b1529410f9..ead630e755 100644 --- a/vendor/github.com/containers/common/pkg/cgroups/systemd_linux.go +++ b/vendor/github.com/containers/common/pkg/cgroups/systemd_linux.go @@ -23,7 +23,7 @@ func systemdCreate(resources *configs.Resources, path string, c *systemdDbus.Con slice = strings.TrimSuffix(slice, "/") var lastError error - for i := range 2 { + for i := 0; i < 2; i++ { properties := []systemdDbus.Property{ systemdDbus.PropDescription("cgroup " + name), systemdDbus.PropWants(slice), diff --git a/vendor/github.com/containers/common/pkg/config/config_bsd.go b/vendor/github.com/containers/common/pkg/config/config_bsd.go index 9f3513a676..e378a07224 100644 --- a/vendor/github.com/containers/common/pkg/config/config_bsd.go +++ b/vendor/github.com/containers/common/pkg/config/config_bsd.go @@ -1,4 +1,4 @@ -//go:build freebsd || netbsd || openbsd +//go:build (freebsd || netbsd || openbsd) package config diff --git a/vendor/github.com/containers/common/pkg/config/default_bsd.go b/vendor/github.com/containers/common/pkg/config/default_bsd.go index 2e87dc8b0f..c619e1783f 100644 --- a/vendor/github.com/containers/common/pkg/config/default_bsd.go +++ b/vendor/github.com/containers/common/pkg/config/default_bsd.go @@ -1,4 +1,4 @@ -//go:build freebsd || netbsd || openbsd +//go:build (freebsd || netbsd || openbsd) package config diff --git a/vendor/github.com/containers/common/pkg/config/default_linux.go b/vendor/github.com/containers/common/pkg/config/default_linux.go index 64a98384e7..9e2ae4796b 100644 --- a/vendor/github.com/containers/common/pkg/config/default_linux.go +++ b/vendor/github.com/containers/common/pkg/config/default_linux.go @@ -28,9 +28,9 @@ func getDefaultProcessLimits() []string { dat, err := os.ReadFile("/proc/sys/kernel/pid_max") if err == nil { val := strings.TrimSuffix(string(dat), "\n") - maxLimit, err := strconv.ParseUint(val, 10, 64) + max, err := strconv.ParseUint(val, 10, 64) if err == nil { - rlim = unix.Rlimit{Cur: maxLimit, Max: maxLimit} + rlim = unix.Rlimit{Cur: max, Max: max} } } defaultLimits := []string{} diff --git a/vendor/github.com/containers/common/pkg/config/default_windows.go b/vendor/github.com/containers/common/pkg/config/default_windows.go index 610aa122bf..70627cbda8 100644 --- a/vendor/github.com/containers/common/pkg/config/default_windows.go +++ b/vendor/github.com/containers/common/pkg/config/default_windows.go @@ -42,17 +42,11 @@ func getLibpodTmpDir() string { } // getDefaultMachineVolumes returns default mounted volumes (possibly with env vars, which will be expanded) -// It is executed only if the machine provider is Hyper-V and it mimics WSL -// behavior where the host %USERPROFILE% drive (e.g. C:\) is automatically -// mounted in the guest under /mnt/ (e.g. /mnt/c/) func getDefaultMachineVolumes() []string { hd := homedir.Get() vol := filepath.VolumeName(hd) hostMnt := filepath.ToSlash(strings.TrimPrefix(hd, vol)) - return []string{ - fmt.Sprintf("%s:%s", hd, hostMnt), - fmt.Sprintf("%s:%s", vol+"\\", "/mnt/"+strings.ToLower(vol[0:1])), - } + return []string{fmt.Sprintf("%s:%s", hd, hostMnt)} } func getDefaultComposeProviders() []string { diff --git a/vendor/github.com/containers/common/pkg/hooks/exec/runtimeconfigfilter.go b/vendor/github.com/containers/common/pkg/hooks/exec/runtimeconfigfilter.go index 3027112609..ac17ac64da 100644 --- a/vendor/github.com/containers/common/pkg/hooks/exec/runtimeconfigfilter.go +++ b/vendor/github.com/containers/common/pkg/hooks/exec/runtimeconfigfilter.go @@ -56,6 +56,7 @@ func RuntimeConfigFilterWithOptions(ctx context.Context, options RuntimeConfigFi return nil, err } for i, hook := range options.Hooks { + hook := hook var stdout bytes.Buffer hookErr, err = RunWithOptions(ctx, RunOptions{Hook: &hook, Dir: options.Dir, State: data, Stdout: &stdout, PostKillTimeout: options.PostKillTimeout}) if err != nil { diff --git a/vendor/github.com/containers/common/pkg/netns/netns_linux.go b/vendor/github.com/containers/common/pkg/netns/netns_linux.go index 5461b05f75..bbcedc0f6c 100644 --- a/vendor/github.com/containers/common/pkg/netns/netns_linux.go +++ b/vendor/github.com/containers/common/pkg/netns/netns_linux.go @@ -40,8 +40,6 @@ import ( // threadNsPath is the /proc path to the current netns handle for the current thread const threadNsPath = "/proc/thread-self/ns/net" -var errNoFreeName = errors.New("failed to find free netns path name") - // GetNSRunDir returns the dir of where to create the netNS. When running // rootless, it needs to be at a location writable by user. func GetNSRunDir() (string, error) { @@ -62,26 +60,14 @@ func NewNSAtPath(nsPath string) (ns.NetNS, error) { // NewNS creates a new persistent (bind-mounted) network namespace and returns // an object representing that namespace, without switching to it. func NewNS() (ns.NetNS, error) { - nsRunDir, err := GetNSRunDir() - if err != nil { - return nil, err - } - - // Create the directory for mounting network namespaces - // This needs to be a shared mountpoint in case it is mounted in to - // other namespaces (containers) - err = makeNetnsDir(nsRunDir) - if err != nil { - return nil, err - } - - for range 10000 { - nsName, err := getRandomNetnsName() + for i := 0; i < 10000; i++ { + b := make([]byte, 16) + _, err := rand.Reader.Read(b) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to generate random netns name: %v", err) } - nsPath := path.Join(nsRunDir, nsName) - ns, err := newNSPath(nsPath) + nsName := fmt.Sprintf("netns-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]) + ns, err := NewNSWithName(nsName) if err == nil { return ns, nil } @@ -91,128 +77,62 @@ func NewNS() (ns.NetNS, error) { } return nil, err } - return nil, errNoFreeName + return nil, errors.New("failed to find free netns path name") } -// NewNSFrom creates a persistent (bind-mounted) network namespace from the -// given netns path, i.e. /proc//ns/net, and returns the new full path to -// the bind mounted file in the netns run dir. -func NewNSFrom(fromNetns string) (string, error) { +// NewNSWithName creates a new persistent (bind-mounted) network namespace and returns +// an object representing that namespace, without switching to it. +func NewNSWithName(name string) (ns.NetNS, error) { nsRunDir, err := GetNSRunDir() if err != nil { - return "", err + return nil, err } - err = makeNetnsDir(nsRunDir) + // Create the directory for mounting network namespaces + // This needs to be a shared mountpoint in case it is mounted in to + // other namespaces (containers) + err = os.MkdirAll(nsRunDir, 0o755) if err != nil { - return "", err + return nil, err } - for range 10000 { - nsName, err := getRandomNetnsName() - if err != nil { - return "", err + // Remount the namespace directory shared. This will fail if it is not + // already a mountpoint, so bind-mount it on to itself to "upgrade" it + // to a mountpoint. + err = unix.Mount("", nsRunDir, "none", unix.MS_SHARED|unix.MS_REC, "") + if err != nil { + if err != unix.EINVAL { + return nil, fmt.Errorf("mount --make-rshared %s failed: %q", nsRunDir, err) } - nsPath := filepath.Join(nsRunDir, nsName) - // create an empty file to use as at the mount point - err = createNetnsFile(nsPath) + // Recursively remount /run/netns on itself. The recursive flag is + // so that any existing netns bindmounts are carried over. + err = unix.Mount(nsRunDir, nsRunDir, "none", unix.MS_BIND|unix.MS_REC, "") if err != nil { - // retry when the name already exists - if errors.Is(err, os.ErrExist) { - continue - } - return "", err + return nil, fmt.Errorf("mount --rbind %s %s failed: %q", nsRunDir, nsRunDir, err) } - err = unix.Mount(fromNetns, nsPath, "none", unix.MS_BIND|unix.MS_SHARED|unix.MS_REC, "") + // Now we can make it shared + err = unix.Mount("", nsRunDir, "none", unix.MS_SHARED|unix.MS_REC, "") if err != nil { - // Do not leak the ns on errors - _ = os.RemoveAll(nsPath) - return "", fmt.Errorf("failed to bind mount ns at %s: %v", nsPath, err) + return nil, fmt.Errorf("mount --make-rshared %s failed: %q", nsRunDir, err) } - return nsPath, nil - } - - return "", errNoFreeName -} - -func getRandomNetnsName() (string, error) { - b := make([]byte, 16) - _, err := rand.Reader.Read(b) - if err != nil { - return "", fmt.Errorf("failed to generate random netns name: %v", err) - } - return fmt.Sprintf("netns-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]), nil -} - -func makeNetnsDir(nsRunDir string) error { - err := os.MkdirAll(nsRunDir, 0o755) - if err != nil { - return err - } - // Important, the bind mount setup is racy if two process try to set it up in parallel. - // This can have very bad consequences because we end up with two duplicated mounts - // for the netns file that then might have a different parent mounts. - // Also because as root netns dir is also created by ip netns we should not race against them. - // Use a lock on the netns dir like they do, compare the iproute2 ip netns add code. - // https://github.com/iproute2/iproute2/blob/8b9d9ea42759c91d950356ca43930a975d0c352b/ip/ipnetns.c#L806-L815 - - dirFD, err := unix.Open(nsRunDir, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) - if err != nil { - return &os.PathError{Op: "open", Path: nsRunDir, Err: err} - } - // closing the fd will also unlock so we do not have to call flock(fd,LOCK_UN) - defer unix.Close(dirFD) - - err = unix.Flock(dirFD, unix.LOCK_EX) - if err != nil { - return fmt.Errorf("failed to lock %s dir: %w", nsRunDir, err) - } - - // Remount the namespace directory shared. This will fail with EINVAL - // if it is not already a mountpoint, so bind-mount it on to itself - // to "upgrade" it to a mountpoint. - err = unix.Mount("", nsRunDir, "none", unix.MS_SHARED|unix.MS_REC, "") - if err == nil { - return nil - } - if err != unix.EINVAL { - return fmt.Errorf("mount --make-rshared %s failed: %q", nsRunDir, err) - } - - // Recursively remount /run/netns on itself. The recursive flag is - // so that any existing netns bindmounts are carried over. - err = unix.Mount(nsRunDir, nsRunDir, "none", unix.MS_BIND|unix.MS_REC, "") - if err != nil { - return fmt.Errorf("mount --rbind %s %s failed: %q", nsRunDir, nsRunDir, err) - } - - // Now we can make it shared - err = unix.Mount("", nsRunDir, "none", unix.MS_SHARED|unix.MS_REC, "") - if err != nil { - return fmt.Errorf("mount --make-rshared %s failed: %q", nsRunDir, err) } - return nil -} - -// createNetnsFile created the file with O_EXCL to ensure there are no conflicts with others -// Callers should check for ErrExist and loop over it to find a free file. -func createNetnsFile(path string) error { - mountPointFd, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) - if err != nil { - return err - } - return mountPointFd.Close() + nsPath := path.Join(nsRunDir, name) + return newNSPath(nsPath) } func newNSPath(nsPath string) (ns.NetNS, error) { - // create an empty file to use as at the mount point - err := createNetnsFile(nsPath) + // create an empty file at the mount point + mountPointFd, err := os.OpenFile(nsPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { return nil, err } + if err := mountPointFd.Close(); err != nil { + return nil, err + } + // Ensure the mount point is cleaned up on errors; if the namespace // was successfully mounted this will have no effect because the file // is in-use diff --git a/vendor/github.com/containers/common/pkg/report/camelcase/camelcase.go b/vendor/github.com/containers/common/pkg/report/camelcase/camelcase.go index adafb1c26c..830482ff0c 100644 --- a/vendor/github.com/containers/common/pkg/report/camelcase/camelcase.go +++ b/vendor/github.com/containers/common/pkg/report/camelcase/camelcase.go @@ -73,7 +73,7 @@ func Split(src string) (entries []string) { } // handle upper case -> lower case sequences, e.g. // "PDFL", "oader" -> "PDF", "Loader" - for i := range len(runes) - 1 { + for i := 0; i < len(runes)-1; i++ { if unicode.IsUpper(runes[i][0]) && unicode.IsLower(runes[i+1][0]) { runes[i+1] = append([]rune{runes[i][len(runes[i])-1]}, runes[i+1]...) runes[i] = runes[i][:len(runes[i])-1] diff --git a/vendor/github.com/containers/common/pkg/report/template.go b/vendor/github.com/containers/common/pkg/report/template.go index 607bb7ff2b..0f7d5e5bfe 100644 --- a/vendor/github.com/containers/common/pkg/report/template.go +++ b/vendor/github.com/containers/common/pkg/report/template.go @@ -101,7 +101,7 @@ func Headers(object any, overrides map[string]string) []map[string]string { // Column header will be field name upper-cased. headers := make(map[string]string, value.NumField()) - for i := range value.Type().NumField() { + for i := 0; i < value.Type().NumField(); i++ { field := value.Type().Field(i) // Recurse to find field names from promoted structs if field.Type.Kind() == reflect.Struct && field.Anonymous { diff --git a/vendor/github.com/containers/common/pkg/systemd/systemd_linux.go b/vendor/github.com/containers/common/pkg/systemd/systemd_linux.go index ea61ecaa18..c1d8ed72e9 100644 --- a/vendor/github.com/containers/common/pkg/systemd/systemd_linux.go +++ b/vendor/github.com/containers/common/pkg/systemd/systemd_linux.go @@ -74,7 +74,7 @@ func MoveRootlessNetnsSlirpProcessToUserSlice(pid int) error { func MovePauseProcessToScope(pausePidPath string) { var err error - for range 10 { + for i := 0; i < 10; i++ { randBytes := make([]byte, 4) _, err = rand.Read(randBytes) if err != nil { diff --git a/vendor/github.com/containers/common/version/version.go b/vendor/github.com/containers/common/version/version.go index c73ec15cd4..4867ade0bf 100644 --- a/vendor/github.com/containers/common/version/version.go +++ b/vendor/github.com/containers/common/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "0.61.0-dev" +const Version = "0.60.2" diff --git a/vendor/github.com/containers/image/v5/copy/compression.go b/vendor/github.com/containers/image/v5/copy/compression.go index 081c49312f..fb5e1b174e 100644 --- a/vendor/github.com/containers/image/v5/copy/compression.go +++ b/vendor/github.com/containers/image/v5/copy/compression.go @@ -52,6 +52,16 @@ func blobPipelineDetectCompressionStep(stream *sourceStream, srcInfo types.BlobI } stream.reader = reader + if decompressor != nil && format.Name() == compressiontypes.ZstdAlgorithmName { + tocDigest, err := chunkedToc.GetTOCDigest(srcInfo.Annotations) + if err != nil { + return bpDetectCompressionStepData{}, err + } + if tocDigest != nil { + format = compression.ZstdChunked + } + + } res := bpDetectCompressionStepData{ isCompressed: decompressor != nil, format: format, @@ -71,13 +81,14 @@ func blobPipelineDetectCompressionStep(stream *sourceStream, srcInfo types.BlobI // bpCompressionStepData contains data that the copy pipeline needs about the compression step. type bpCompressionStepData struct { - operation bpcOperation // What we are actually doing - uploadedOperation types.LayerCompression // Operation to use for updating the blob metadata (matching the end state, not necessarily what we do) - uploadedAlgorithm *compressiontypes.Algorithm // An algorithm parameter for the compressionOperation edits. - uploadedAnnotations map[string]string // Compression-related annotations that should be set on the uploaded blob. WARNING: This is only set after the srcStream.reader is fully consumed. - srcCompressorBaseVariantName string // Compressor base variant name to record in the blob info cache for the source blob. - uploadedCompressorName string // Compressor name to record in the blob info cache for the uploaded blob. - closers []io.Closer // Objects to close after the upload is done, if any. + operation bpcOperation // What we are actually doing + uploadedOperation types.LayerCompression // Operation to use for updating the blob metadata (matching the end state, not necessarily what we do) + uploadedAlgorithm *compressiontypes.Algorithm // An algorithm parameter for the compressionOperation edits. + uploadedAnnotations map[string]string // Compression-related annotations that should be set on the uploaded blob. WARNING: This is only set after the srcStream.reader is fully consumed. + srcCompressorBaseVariantName string // Compressor base variant name to record in the blob info cache for the source blob. + uploadedCompressorBaseVariantName string // Compressor base variant name to record in the blob info cache for the uploaded blob. + uploadedCompressorSpecificVariantName string // Compressor specific variant name to record in the blob info cache for the uploaded blob. + closers []io.Closer // Objects to close after the upload is done, if any. } type bpcOperation int @@ -129,11 +140,12 @@ func (ic *imageCopier) bpcPreserveEncrypted(stream *sourceStream, _ bpDetectComp // We can’t do anything with an encrypted blob unless decrypted. logrus.Debugf("Using original blob without modification for encrypted blob") return &bpCompressionStepData{ - operation: bpcOpPreserveOpaque, - uploadedOperation: types.PreserveOriginal, - uploadedAlgorithm: nil, - srcCompressorBaseVariantName: internalblobinfocache.UnknownCompression, - uploadedCompressorName: internalblobinfocache.UnknownCompression, + operation: bpcOpPreserveOpaque, + uploadedOperation: types.PreserveOriginal, + uploadedAlgorithm: nil, + srcCompressorBaseVariantName: internalblobinfocache.UnknownCompression, + uploadedCompressorBaseVariantName: internalblobinfocache.UnknownCompression, + uploadedCompressorSpecificVariantName: internalblobinfocache.UnknownCompression, }, nil } return nil, nil @@ -157,14 +169,19 @@ func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bp Digest: "", Size: -1, } + specificVariantName := uploadedAlgorithm.Name() + if specificVariantName == uploadedAlgorithm.BaseVariantName() { + specificVariantName = internalblobinfocache.UnknownCompression + } return &bpCompressionStepData{ - operation: bpcOpCompressUncompressed, - uploadedOperation: types.Compress, - uploadedAlgorithm: uploadedAlgorithm, - uploadedAnnotations: annotations, - srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName, - uploadedCompressorName: uploadedAlgorithm.Name(), - closers: []io.Closer{reader}, + operation: bpcOpCompressUncompressed, + uploadedOperation: types.Compress, + uploadedAlgorithm: uploadedAlgorithm, + uploadedAnnotations: annotations, + srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName, + uploadedCompressorBaseVariantName: uploadedAlgorithm.BaseVariantName(), + uploadedCompressorSpecificVariantName: specificVariantName, + closers: []io.Closer{reader}, }, nil } return nil, nil @@ -197,15 +214,20 @@ func (ic *imageCopier) bpcRecompressCompressed(stream *sourceStream, detected bp Digest: "", Size: -1, } + specificVariantName := ic.compressionFormat.Name() + if specificVariantName == ic.compressionFormat.BaseVariantName() { + specificVariantName = internalblobinfocache.UnknownCompression + } succeeded = true return &bpCompressionStepData{ - operation: bpcOpRecompressCompressed, - uploadedOperation: types.PreserveOriginal, - uploadedAlgorithm: ic.compressionFormat, - uploadedAnnotations: annotations, - srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName, - uploadedCompressorName: ic.compressionFormat.Name(), - closers: []io.Closer{decompressed, recompressed}, + operation: bpcOpRecompressCompressed, + uploadedOperation: types.PreserveOriginal, + uploadedAlgorithm: ic.compressionFormat, + uploadedAnnotations: annotations, + srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName, + uploadedCompressorBaseVariantName: ic.compressionFormat.BaseVariantName(), + uploadedCompressorSpecificVariantName: specificVariantName, + closers: []io.Closer{decompressed, recompressed}, }, nil } return nil, nil @@ -226,12 +248,13 @@ func (ic *imageCopier) bpcDecompressCompressed(stream *sourceStream, detected bp Size: -1, } return &bpCompressionStepData{ - operation: bpcOpDecompressCompressed, - uploadedOperation: types.Decompress, - uploadedAlgorithm: nil, - srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName, - uploadedCompressorName: internalblobinfocache.Uncompressed, - closers: []io.Closer{s}, + operation: bpcOpDecompressCompressed, + uploadedOperation: types.Decompress, + uploadedAlgorithm: nil, + srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName, + uploadedCompressorBaseVariantName: internalblobinfocache.Uncompressed, + uploadedCompressorSpecificVariantName: internalblobinfocache.UnknownCompression, + closers: []io.Closer{s}, }, nil } return nil, nil @@ -276,7 +299,8 @@ func (ic *imageCopier) bpcPreserveOriginal(_ *sourceStream, detected bpDetectCom // We only record the base variant of the format on upload; we didn’t do anything with // the TOC, we don’t know whether it matches the blob digest, so we don’t want to trigger // reuse of any kind between the blob digest and the TOC digest. - uploadedCompressorName: detected.srcCompressorBaseVariantName, + uploadedCompressorBaseVariantName: detected.srcCompressorBaseVariantName, + uploadedCompressorSpecificVariantName: internalblobinfocache.UnknownCompression, } } @@ -336,24 +360,16 @@ func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInf return fmt.Errorf("Internal error: Unexpected d.operation value %#v", d.operation) } } - if d.srcCompressorBaseVariantName == "" || d.uploadedCompressorName == "" { - return fmt.Errorf("internal error: missing compressor names (src base: %q, uploaded: %q)", - d.srcCompressorBaseVariantName, d.uploadedCompressorName) + if d.srcCompressorBaseVariantName == "" || d.uploadedCompressorBaseVariantName == "" || d.uploadedCompressorSpecificVariantName == "" { + return fmt.Errorf("internal error: missing compressor names (src base: %q, uploaded base: %q, uploaded specific: %q)", + d.srcCompressorBaseVariantName, d.uploadedCompressorBaseVariantName, d.uploadedCompressorSpecificVariantName) } - if d.uploadedCompressorName != internalblobinfocache.UnknownCompression { - if d.uploadedCompressorName != compressiontypes.ZstdChunkedAlgorithmName { - // HACK: Don’t record zstd:chunked algorithms. - // There is already a similar hack in internal/imagedestination/impl/helpers.CandidateMatchesTryReusingBlobOptions, - // and that one prevents reusing zstd:chunked blobs, so recording the algorithm here would be mostly harmless. - // - // We skip that here anyway to work around the inability of blobPipelineDetectCompressionStep to differentiate - // between zstd and zstd:chunked; so we could, in varying situations over time, call RecordDigestCompressorName - // with the same digest and both ZstdAlgorithmName and ZstdChunkedAlgorithmName , which causes warnings about - // inconsistent data to be logged. - c.blobInfoCache.RecordDigestCompressorData(uploadedInfo.Digest, internalblobinfocache.DigestCompressorData{ - BaseVariantCompressor: d.uploadedCompressorName, - }) - } + if d.uploadedCompressorBaseVariantName != internalblobinfocache.UnknownCompression { + c.blobInfoCache.RecordDigestCompressorData(uploadedInfo.Digest, internalblobinfocache.DigestCompressorData{ + BaseVariantCompressor: d.uploadedCompressorBaseVariantName, + SpecificVariantCompressor: d.uploadedCompressorSpecificVariantName, + SpecificVariantAnnotations: d.uploadedAnnotations, + }) } if srcInfo.Digest != "" && srcInfo.Digest != uploadedInfo.Digest && d.srcCompressorBaseVariantName != internalblobinfocache.UnknownCompression { @@ -361,7 +377,9 @@ func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInf // blob as is, or perhaps decompressed it; either way we don’t trust the TOC digest, // so record neither the variant name, nor the TOC digest. c.blobInfoCache.RecordDigestCompressorData(srcInfo.Digest, internalblobinfocache.DigestCompressorData{ - BaseVariantCompressor: d.srcCompressorBaseVariantName, + BaseVariantCompressor: d.srcCompressorBaseVariantName, + SpecificVariantCompressor: internalblobinfocache.UnknownCompression, + SpecificVariantAnnotations: nil, }) } return nil diff --git a/vendor/github.com/containers/image/v5/copy/single.go b/vendor/github.com/containers/image/v5/copy/single.go index 714dc81368..ba414a22d6 100644 --- a/vendor/github.com/containers/image/v5/copy/single.go +++ b/vendor/github.com/containers/image/v5/copy/single.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "maps" "reflect" "slices" "strings" @@ -162,7 +163,7 @@ func (c *copier) copySingleImage(ctx context.Context, unparsedImage *image.Unpar if format == nil { format = defaultCompressionFormat } - if format.Name() == compression.ZstdChunked.Name() { + if format.Name() == compressiontypes.ZstdChunkedAlgorithmName { if ic.requireCompressionFormatMatch { return copySingleImageResult{}, errors.New("explicitly requested to combine zstd:chunked with encryption, which is not beneficial; use plain zstd instead") } @@ -888,21 +889,33 @@ func updatedBlobInfoFromReuse(inputInfo types.BlobInfo, reusedBlob private.Reuse // Handling of compression, encryption, and the related MIME types and the like are all the responsibility // of the generic code in this package. res := types.BlobInfo{ - Digest: reusedBlob.Digest, - Size: reusedBlob.Size, - URLs: nil, // This _must_ be cleared if Digest changes; clear it in other cases as well, to preserve previous behavior. - Annotations: inputInfo.Annotations, // FIXME: This should remove zstd:chunked annotations (but those annotations being left with incorrect values should not break pulls) - MediaType: inputInfo.MediaType, // Mostly irrelevant, MediaType is updated based on Compression*/CryptoOperation. + Digest: reusedBlob.Digest, + Size: reusedBlob.Size, + URLs: nil, // This _must_ be cleared if Digest changes; clear it in other cases as well, to preserve previous behavior. + // FIXME: This should remove zstd:chunked annotations IF the original was chunked and the new one isn’t + // (but those annotations being left with incorrect values should not break pulls). + Annotations: maps.Clone(inputInfo.Annotations), + MediaType: inputInfo.MediaType, // Mostly irrelevant, MediaType is updated based on Compression*/CryptoOperation. CompressionOperation: reusedBlob.CompressionOperation, CompressionAlgorithm: reusedBlob.CompressionAlgorithm, CryptoOperation: inputInfo.CryptoOperation, // Expected to be unset anyway. } // The transport is only expected to fill CompressionOperation and CompressionAlgorithm - // if the blob was substituted; otherwise, fill it in based + // if the blob was substituted; otherwise, it is optional, and if not set, fill it in based // on what we know from the srcInfos we were given. if reusedBlob.Digest == inputInfo.Digest { - res.CompressionOperation = inputInfo.CompressionOperation - res.CompressionAlgorithm = inputInfo.CompressionAlgorithm + if res.CompressionOperation == types.PreserveOriginal { + res.CompressionOperation = inputInfo.CompressionOperation + } + if res.CompressionAlgorithm == nil { + res.CompressionAlgorithm = inputInfo.CompressionAlgorithm + } + } + if len(reusedBlob.CompressionAnnotations) != 0 { + if res.Annotations == nil { + res.Annotations = map[string]string{} + } + maps.Copy(res.Annotations, reusedBlob.CompressionAnnotations) } return res } diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go index 7f7a74bd37..ed3d4a2c0b 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go @@ -332,6 +332,7 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, return false, private.ReusedBlob{}, errors.New("Can not check for a blob with unknown digest") } + originalCandidateKnownToBeMissing := false if impl.OriginalCandidateMatchesTryReusingBlobOptions(options) { // First, check whether the blob happens to already exist at the destination. haveBlob, reusedInfo, err := d.tryReusingExactBlob(ctx, info, options.Cache) @@ -341,9 +342,17 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, if haveBlob { return true, reusedInfo, nil } + originalCandidateKnownToBeMissing = true } else { logrus.Debugf("Ignoring exact blob match, compression %s does not match required %s or MIME types %#v", optionalCompressionName(options.OriginalCompression), optionalCompressionName(options.RequiredCompression), options.PossibleManifestFormats) + // We can get here with a blob detected to be zstd when the user wants a zstd:chunked. + // In that case we keep originalCandiateKnownToBeMissing = false, so that if we find + // a BIC entry for this blob, we do use that entry and return a zstd:chunked entry + // with the BIC’s annotations. + // This is not quite correct, it only works if the BIC also contains an acceptable _location_. + // Ideally, we could look up just the compression algorithm/annotations for info.digest, + // and use it even if no location candidate exists and the original dandidate is present. } // Then try reusing blobs from other locations. @@ -387,7 +396,8 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, // for it in the current repo. candidateRepo = reference.TrimNamed(d.ref.ref) } - if candidateRepo.Name() == d.ref.ref.Name() && candidate.Digest == info.Digest { + if originalCandidateKnownToBeMissing && + candidateRepo.Name() == d.ref.ref.Name() && candidate.Digest == info.Digest { logrus.Debug("... Already tried the primary destination") continue } @@ -427,10 +437,12 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, options.Cache.RecordKnownLocation(d.ref.Transport(), bicTransportScope(d.ref), candidate.Digest, newBICLocationReference(d.ref)) return true, private.ReusedBlob{ - Digest: candidate.Digest, - Size: size, - CompressionOperation: candidate.CompressionOperation, - CompressionAlgorithm: candidate.CompressionAlgorithm}, nil + Digest: candidate.Digest, + Size: size, + CompressionOperation: candidate.CompressionOperation, + CompressionAlgorithm: candidate.CompressionAlgorithm, + CompressionAnnotations: candidate.CompressionAnnotations, + }, nil } return false, private.ReusedBlob{}, nil diff --git a/vendor/github.com/containers/image/v5/internal/blobinfocache/types.go b/vendor/github.com/containers/image/v5/internal/blobinfocache/types.go index 276c8073e3..acf82ee639 100644 --- a/vendor/github.com/containers/image/v5/internal/blobinfocache/types.go +++ b/vendor/github.com/containers/image/v5/internal/blobinfocache/types.go @@ -37,8 +37,11 @@ type BlobInfoCache2 interface { // RecordDigestCompressorData records data for the blob with the specified digest. // WARNING: Only call this with LOCALLY VERIFIED data: - // - don’t record a compressor for a digest just because some remote author claims so - // (e.g. because a manifest says so); + // - don’t record a compressor for a digest just because some remote author claims so + // (e.g. because a manifest says so); + // - don’t record the non-base variant or annotations if we are not _sure_ that the base variant + // and the blob’s digest match the non-base variant’s annotations (e.g. because we saw them + // in a manifest) // otherwise the cache could be poisoned and cause us to make incorrect edits to type // information in a manifest. RecordDigestCompressorData(anyDigest digest.Digest, data DigestCompressorData) @@ -52,6 +55,9 @@ type BlobInfoCache2 interface { // (This is worded generically, but basically targeted at the zstd / zstd:chunked situation.) type DigestCompressorData struct { BaseVariantCompressor string // A compressor’s base variant name, or Uncompressed or UnknownCompression. + // The following fields are only valid if the base variant is neither Uncompressed nor UnknownCompression: + SpecificVariantCompressor string // A non-base variant compressor (or UnknownCompression if the true format is just the base variant) + SpecificVariantAnnotations map[string]string // Annotations required to benefit from the base variant. } // CandidateLocations2Options are used in CandidateLocations2. @@ -66,9 +72,10 @@ type CandidateLocations2Options struct { // BICReplacementCandidate2 is an item returned by BlobInfoCache2.CandidateLocations2. type BICReplacementCandidate2 struct { - Digest digest.Digest - CompressionOperation types.LayerCompression // Either types.Decompress for uncompressed, or types.Compress for compressed - CompressionAlgorithm *compressiontypes.Algorithm // An algorithm when the candidate is compressed, or nil when it is uncompressed - UnknownLocation bool // is true when `Location` for this blob is not set - Location types.BICLocationReference // not set if UnknownLocation is set to `true` + Digest digest.Digest + CompressionOperation types.LayerCompression // Either types.Decompress for uncompressed, or types.Compress for compressed + CompressionAlgorithm *compressiontypes.Algorithm // An algorithm when the candidate is compressed, or nil when it is uncompressed + CompressionAnnotations map[string]string // If necessary, annotations necessary to use CompressionAlgorithm + UnknownLocation bool // is true when `Location` for this blob is not set + Location types.BICLocationReference // not set if UnknownLocation is set to `true` } diff --git a/vendor/github.com/containers/image/v5/internal/imagedestination/wrapper.go b/vendor/github.com/containers/image/v5/internal/imagedestination/wrapper.go index cdd3c5e5d0..f5a38541ae 100644 --- a/vendor/github.com/containers/image/v5/internal/imagedestination/wrapper.go +++ b/vendor/github.com/containers/image/v5/internal/imagedestination/wrapper.go @@ -76,6 +76,9 @@ func (w *wrapped) TryReusingBlobWithOptions(ctx context.Context, info types.Blob Size: blob.Size, CompressionOperation: blob.CompressionOperation, CompressionAlgorithm: blob.CompressionAlgorithm, + // CompressionAnnotations could be set to blob.Annotations, but that may contain unrelated + // annotations, and we didn’t use the blob.Annotations field previously, so we’ll + // continue not using it. }, nil } diff --git a/vendor/github.com/containers/image/v5/internal/manifest/manifest.go b/vendor/github.com/containers/image/v5/internal/manifest/manifest.go index ee0ddc772a..3fb52104a6 100644 --- a/vendor/github.com/containers/image/v5/internal/manifest/manifest.go +++ b/vendor/github.com/containers/image/v5/internal/manifest/manifest.go @@ -205,11 +205,6 @@ type ReuseConditions struct { // (which can be nil to represent uncompressed or unknown) matches reuseConditions. func CandidateCompressionMatchesReuseConditions(c ReuseConditions, candidateCompression *compressiontypes.Algorithm) bool { if c.RequiredCompression != nil { - if c.RequiredCompression.Name() == compressiontypes.ZstdChunkedAlgorithmName { - // HACK: Never match when the caller asks for zstd:chunked, because we don’t record the annotations required to use the chunked blobs. - // The caller must re-compress to build those annotations. - return false - } if candidateCompression == nil || (c.RequiredCompression.Name() != candidateCompression.Name() && c.RequiredCompression.Name() != candidateCompression.BaseVariantName()) { return false diff --git a/vendor/github.com/containers/image/v5/internal/private/private.go b/vendor/github.com/containers/image/v5/internal/private/private.go index 63fb9326de..d81ea6703e 100644 --- a/vendor/github.com/containers/image/v5/internal/private/private.go +++ b/vendor/github.com/containers/image/v5/internal/private/private.go @@ -134,9 +134,14 @@ type ReusedBlob struct { Size int64 // Must be provided // The following compression fields should be set when the reuse substitutes // a differently-compressed blob. + // They may be set also to change from a base variant to a specific variant of an algorithm. CompressionOperation types.LayerCompression // Compress/Decompress, matching the reused blob; PreserveOriginal if N/A CompressionAlgorithm *compression.Algorithm // Algorithm if compressed, nil if decompressed or N/A + // Annotations that should be added, for CompressionAlgorithm. Note that they might need to be + // added even if the digest doesn’t change (if we found the annotations in a cache). + CompressionAnnotations map[string]string + MatchedByTOCDigest bool // Whether the layer was reused/matched by TOC digest. Used only for UI purposes. } diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/internal/prioritize/prioritize.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/internal/prioritize/prioritize.go index 03548209f9..40ed09bed5 100644 --- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/internal/prioritize/prioritize.go +++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/internal/prioritize/prioritize.go @@ -28,9 +28,10 @@ const replacementUnknownLocationAttempts = 2 // CandidateTemplate is a subset of BICReplacementCandidate2 with data related to a specific digest, // which can be later combined with information about a location. type CandidateTemplate struct { - digest digest.Digest - compressionOperation types.LayerCompression // Either types.Decompress for uncompressed, or types.Compress for compressed - compressionAlgorithm *compression.Algorithm // An algorithm when the candidate is compressed, or nil when it is uncompressed + digest digest.Digest + compressionOperation types.LayerCompression // Either types.Decompress for uncompressed, or types.Compress for compressed + compressionAlgorithm *compression.Algorithm // An algorithm when the candidate is compressed, or nil when it is uncompressed + compressionAnnotations map[string]string // If necessary, annotations necessary to use compressionAlgorithm } // CandidateTemplateWithCompression returns a CandidateTemplate if a blob with data is acceptable @@ -40,7 +41,7 @@ type CandidateTemplate struct { // if not nil, the call is assumed to be CandidateLocations2. func CandidateTemplateWithCompression(v2Options *blobinfocache.CandidateLocations2Options, digest digest.Digest, data blobinfocache.DigestCompressorData) *CandidateTemplate { if v2Options == nil { - return &CandidateTemplate{ // Anything goes. The compressionOperation, compressionAlgorithm values are not used. + return &CandidateTemplate{ // Anything goes. The compressionOperation, compressionAlgorithm and compressionAnnotations values are not used. digest: digest, } } @@ -60,14 +61,40 @@ func CandidateTemplateWithCompression(v2Options *blobinfocache.CandidateLocation return nil } return &CandidateTemplate{ - digest: digest, - compressionOperation: types.Decompress, - compressionAlgorithm: nil, + digest: digest, + compressionOperation: types.Decompress, + compressionAlgorithm: nil, + compressionAnnotations: nil, } case blobinfocache.UnknownCompression: logrus.Debugf("Ignoring BlobInfoCache record of digest %q with unknown compression", digest.String()) return nil // Not allowed with CandidateLocations2 default: + // See if we can use the specific variant, first. + if data.SpecificVariantCompressor != blobinfocache.UnknownCompression { + algo, err := compression.AlgorithmByName(data.SpecificVariantCompressor) + if err != nil { + logrus.Debugf("Not considering unrecognized specific compression variant %q for BlobInfoCache record of digest %q: %v", + data.SpecificVariantCompressor, digest.String(), err) + } else { + if !manifest.CandidateCompressionMatchesReuseConditions(manifest.ReuseConditions{ + PossibleManifestFormats: v2Options.PossibleManifestFormats, + RequiredCompression: v2Options.RequiredCompression, + }, &algo) { + logrus.Debugf("Ignoring specific compression variant %q for BlobInfoCache record of digest %q, it does not match required %s or MIME types %#v", + data.SpecificVariantCompressor, digest.String(), requiredCompression, v2Options.PossibleManifestFormats) + } else { + return &CandidateTemplate{ + digest: digest, + compressionOperation: types.Compress, + compressionAlgorithm: &algo, + compressionAnnotations: data.SpecificVariantAnnotations, + } + } + } + } + + // Try the base variant. algo, err := compression.AlgorithmByName(data.BaseVariantCompressor) if err != nil { logrus.Debugf("Ignoring BlobInfoCache record of digest %q with unrecognized compression %q: %v", @@ -83,9 +110,10 @@ func CandidateTemplateWithCompression(v2Options *blobinfocache.CandidateLocation return nil } return &CandidateTemplate{ - digest: digest, - compressionOperation: types.Compress, - compressionAlgorithm: &algo, + digest: digest, + compressionOperation: types.Compress, + compressionAlgorithm: &algo, + compressionAnnotations: nil, } } } @@ -100,11 +128,12 @@ type CandidateWithTime struct { func (template CandidateTemplate) CandidateWithLocation(location types.BICLocationReference, lastSeen time.Time) CandidateWithTime { return CandidateWithTime{ candidate: blobinfocache.BICReplacementCandidate2{ - Digest: template.digest, - CompressionOperation: template.compressionOperation, - CompressionAlgorithm: template.compressionAlgorithm, - UnknownLocation: false, - Location: location, + Digest: template.digest, + CompressionOperation: template.compressionOperation, + CompressionAlgorithm: template.compressionAlgorithm, + CompressionAnnotations: template.compressionAnnotations, + UnknownLocation: false, + Location: location, }, lastSeen: lastSeen, } @@ -114,11 +143,12 @@ func (template CandidateTemplate) CandidateWithLocation(location types.BICLocati func (template CandidateTemplate) CandidateWithUnknownLocation() CandidateWithTime { return CandidateWithTime{ candidate: blobinfocache.BICReplacementCandidate2{ - Digest: template.digest, - CompressionOperation: template.compressionOperation, - CompressionAlgorithm: template.compressionAlgorithm, - UnknownLocation: true, - Location: types.BICLocationReference{Opaque: ""}, + Digest: template.digest, + CompressionOperation: template.compressionOperation, + CompressionAlgorithm: template.compressionAlgorithm, + CompressionAnnotations: template.compressionAnnotations, + UnknownLocation: true, + Location: types.BICLocationReference{Opaque: ""}, }, lastSeen: time.Time{}, } diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go index 067c6b7e11..9d4125d664 100644 --- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go +++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go @@ -28,7 +28,7 @@ type cache struct { uncompressedDigestsByTOC map[digest.Digest]digest.Digest digestsByUncompressed map[digest.Digest]*set.Set[digest.Digest] // stores a set of digests for each uncompressed digest knownLocations map[locationKey]map[types.BICLocationReference]time.Time // stores last known existence time for each location reference - compressors map[digest.Digest]string // stores a compressor name, or blobinfocache.Uncompressed (not blobinfocache.UnknownCompression), for each digest + compressors map[digest.Digest]blobinfocache.DigestCompressorData // stores compression data for each digest; BaseVariantCompressor != UnknownCompression } // New returns a BlobInfoCache implementation which is in-memory only. @@ -49,7 +49,7 @@ func new2() *cache { uncompressedDigestsByTOC: map[digest.Digest]digest.Digest{}, digestsByUncompressed: map[digest.Digest]*set.Set[digest.Digest]{}, knownLocations: map[locationKey]map[types.BICLocationReference]time.Time{}, - compressors: map[digest.Digest]string{}, + compressors: map[digest.Digest]blobinfocache.DigestCompressorData{}, } } @@ -148,20 +148,36 @@ func (mem *cache) RecordKnownLocation(transport types.ImageTransport, scope type // WARNING: Only call this with LOCALLY VERIFIED data: // - don’t record a compressor for a digest just because some remote author claims so // (e.g. because a manifest says so); +// - don’t record the non-base variant or annotations if we are not _sure_ that the base variant +// and the blob’s digest match the non-base variant’s annotations (e.g. because we saw them +// in a manifest) // // otherwise the cache could be poisoned and cause us to make incorrect edits to type // information in a manifest. func (mem *cache) RecordDigestCompressorData(anyDigest digest.Digest, data blobinfocache.DigestCompressorData) { mem.mutex.Lock() defer mem.mutex.Unlock() - if previous, ok := mem.compressors[anyDigest]; ok && previous != data.BaseVariantCompressor { - logrus.Warnf("Compressor for blob with digest %s previously recorded as %s, now %s", anyDigest, previous, data.BaseVariantCompressor) + if previous, ok := mem.compressors[anyDigest]; ok { + if previous.BaseVariantCompressor != data.BaseVariantCompressor { + logrus.Warnf("Base compressor for blob with digest %s previously recorded as %s, now %s", anyDigest, previous.BaseVariantCompressor, data.BaseVariantCompressor) + } else if previous.SpecificVariantCompressor != blobinfocache.UnknownCompression && data.SpecificVariantCompressor != blobinfocache.UnknownCompression && + previous.SpecificVariantCompressor != data.SpecificVariantCompressor { + logrus.Warnf("Specific compressor for blob with digest %s previously recorded as %s, now %s", anyDigest, previous.SpecificVariantCompressor, data.SpecificVariantCompressor) + } + // We don’t check SpecificVariantAnnotations for equality, it’s possible that their generation is not deterministic. + + // Preserve specific variant information if the incoming data does not have it. + if data.BaseVariantCompressor != blobinfocache.UnknownCompression && data.SpecificVariantCompressor == blobinfocache.UnknownCompression && + previous.SpecificVariantCompressor != blobinfocache.UnknownCompression { + data.SpecificVariantCompressor = previous.SpecificVariantCompressor + data.SpecificVariantAnnotations = previous.SpecificVariantAnnotations + } } if data.BaseVariantCompressor == blobinfocache.UnknownCompression { delete(mem.compressors, anyDigest) return } - mem.compressors[anyDigest] = data.BaseVariantCompressor + mem.compressors[anyDigest] = data } // appendReplacementCandidates creates prioritize.CandidateWithTime values for digest in memory @@ -171,13 +187,15 @@ func (mem *cache) RecordDigestCompressorData(anyDigest digest.Digest, data blobi // with unknown compression. func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, v2Options *blobinfocache.CandidateLocations2Options) []prioritize.CandidateWithTime { - compressorName := blobinfocache.UnknownCompression + compressionData := blobinfocache.DigestCompressorData{ + BaseVariantCompressor: blobinfocache.UnknownCompression, + SpecificVariantCompressor: blobinfocache.UnknownCompression, + SpecificVariantAnnotations: nil, + } if v, ok := mem.compressors[digest]; ok { - compressorName = v + compressionData = v } - template := prioritize.CandidateTemplateWithCompression(v2Options, digest, blobinfocache.DigestCompressorData{ - BaseVariantCompressor: compressorName, - }) + template := prioritize.CandidateTemplateWithCompression(v2Options, digest, compressionData) if template == nil { return candidates } diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/sqlite/sqlite.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/sqlite/sqlite.go index 8d2bf72898..1a79310239 100644 --- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/sqlite/sqlite.go +++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/sqlite/sqlite.go @@ -3,6 +3,7 @@ package sqlite import ( "database/sql" + "encoding/json" "errors" "fmt" "sync" @@ -303,6 +304,16 @@ func ensureDBHasCurrentSchema(db *sql.DB) error { `uncompressedDigest TEXT NOT NULL )`, }, + { + "DigestSpecificVariantCompressors", // If changing the schema incompatibly, merge this with DigestCompressors. + `CREATE TABLE IF NOT EXISTS DigestSpecificVariantCompressors(` + + // index implied by PRIMARY KEY + `digest TEXT PRIMARY KEY NOT NULL,` + + // The compressor is not `UnknownCompression`. + `specificVariantCompressor TEXT NOT NULL, + specificVariantAnnotations BLOB NOT NULL + )`, + }, } _, err := dbTransaction(db, func(tx *sql.Tx) (void, error) { @@ -461,6 +472,9 @@ func (sqc *cache) RecordKnownLocation(transport types.ImageTransport, scope type // WARNING: Only call this with LOCALLY VERIFIED data: // - don’t record a compressor for a digest just because some remote author claims so // (e.g. because a manifest says so); +// - don’t record the non-base variant or annotations if we are not _sure_ that the base variant +// and the blob’s digest match the non-base variant’s annotations (e.g. because we saw them +// in a manifest) // // otherwise the cache could be poisoned and cause us to make incorrect edits to type // information in a manifest. @@ -468,21 +482,46 @@ func (sqc *cache) RecordDigestCompressorData(anyDigest digest.Digest, data blobi _, _ = transaction(sqc, func(tx *sql.Tx) (void, error) { previous, gotPrevious, err := querySingleValue[string](tx, "SELECT compressor FROM DigestCompressors WHERE digest = ?", anyDigest.String()) if err != nil { - return void{}, fmt.Errorf("looking for compressor of for %q", anyDigest) + return void{}, fmt.Errorf("looking for compressor of %q", anyDigest) } + warned := false if gotPrevious && previous != data.BaseVariantCompressor { logrus.Warnf("Compressor for blob with digest %s previously recorded as %s, now %s", anyDigest, previous, data.BaseVariantCompressor) + warned = true } if data.BaseVariantCompressor == blobinfocache.UnknownCompression { if _, err := tx.Exec("DELETE FROM DigestCompressors WHERE digest = ?", anyDigest.String()); err != nil { return void{}, fmt.Errorf("deleting compressor for digest %q: %w", anyDigest, err) } + if _, err := tx.Exec("DELETE FROM DigestSpecificVariantCompressors WHERE digest = ?", anyDigest.String()); err != nil { + return void{}, fmt.Errorf("deleting specific variant compressor for digest %q: %w", anyDigest, err) + } } else { if _, err := tx.Exec("INSERT OR REPLACE INTO DigestCompressors(digest, compressor) VALUES (?, ?)", anyDigest.String(), data.BaseVariantCompressor); err != nil { return void{}, fmt.Errorf("recording compressor %q for %q: %w", data.BaseVariantCompressor, anyDigest, err) } } + + if data.SpecificVariantCompressor != blobinfocache.UnknownCompression { + if !warned { // Don’t warn twice about the same digest + prevSVC, found, err := querySingleValue[string](tx, "SELECT specificVariantCompressor FROM DigestSpecificVariantCompressors WHERE digest = ?", anyDigest.String()) + if err != nil { + return void{}, fmt.Errorf("looking for specific variant compressor of %q", anyDigest) + } + if found && data.SpecificVariantCompressor != prevSVC { + logrus.Warnf("Specific compressor for blob with digest %s previously recorded as %s, now %s", anyDigest, prevSVC, data.SpecificVariantCompressor) + } + } + annotations, err := json.Marshal(data.SpecificVariantAnnotations) + if err != nil { + return void{}, err + } + if _, err := tx.Exec("INSERT OR REPLACE INTO DigestSpecificVariantCompressors(digest, specificVariantCompressor, specificVariantAnnotations) VALUES (?, ?, ?)", + anyDigest.String(), data.SpecificVariantCompressor, annotations); err != nil { + return void{}, fmt.Errorf("recording specific variant compressor %q/%q for %q: %w", data.SpecificVariantCompressor, annotations, anyDigest, err) + } + } return void{}, nil }) // FIXME? Log error (but throttle the log volume on repeated accesses)? } @@ -493,19 +532,32 @@ func (sqc *cache) RecordDigestCompressorData(anyDigest digest.Digest, data blobi // with unknown compression. func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, tx *sql.Tx, transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, v2Options *blobinfocache.CandidateLocations2Options) ([]prioritize.CandidateWithTime, error) { - compressorName := blobinfocache.UnknownCompression + compressionData := blobinfocache.DigestCompressorData{ + BaseVariantCompressor: blobinfocache.UnknownCompression, + SpecificVariantCompressor: blobinfocache.UnknownCompression, + SpecificVariantAnnotations: nil, + } if v2Options != nil { - compressor, found, err := querySingleValue[string](tx, "SELECT compressor FROM DigestCompressors WHERE digest = ?", digest.String()) - if err != nil { - return nil, fmt.Errorf("scanning compressorName: %w", err) - } - if found { - compressorName = compressor + var baseVariantCompressor string + var specificVariantCompressor sql.NullString + var annotationBytes []byte + switch err := tx.QueryRow("SELECT compressor, specificVariantCompressor, specificVariantAnnotations "+ + "FROM DigestCompressors LEFT JOIN DigestSpecificVariantCompressors USING (digest) WHERE digest = ?", digest.String()). + Scan(&baseVariantCompressor, &specificVariantCompressor, &annotationBytes); { + case errors.Is(err, sql.ErrNoRows): // Do nothing + case err != nil: + return nil, fmt.Errorf("scanning compressor data: %w", err) + default: + compressionData.BaseVariantCompressor = baseVariantCompressor + if specificVariantCompressor.Valid && annotationBytes != nil { + compressionData.SpecificVariantCompressor = specificVariantCompressor.String + if err := json.Unmarshal(annotationBytes, &compressionData.SpecificVariantAnnotations); err != nil { + return nil, err + } + } } } - template := prioritize.CandidateTemplateWithCompression(v2Options, digest, blobinfocache.DigestCompressorData{ - BaseVariantCompressor: compressorName, - }) + template := prioritize.CandidateTemplateWithCompression(v2Options, digest, compressionData) if template == nil { return candidates, nil } @@ -561,40 +613,41 @@ func (sqc *cache) candidateLocations(transport types.ImageTransport, scope types if err != nil { return nil, err } - - // FIXME? We could integrate this with appendReplacementCandidates into a single join instead of N+1 queries. - // (In the extreme, we could turn _everything_ this function does into a single query. - // And going even further, even DestructivelyPrioritizeReplacementCandidates could be turned into SQL.) - // For now, we prioritize simplicity, and sharing both code and implementation structure with the other cache implementations. - rows, err := tx.Query("SELECT anyDigest FROM DigestUncompressedPairs WHERE uncompressedDigest = ?", uncompressedDigest.String()) - if err != nil { - return nil, fmt.Errorf("querying for other digests: %w", err) - } - defer rows.Close() - for rows.Next() { - var otherDigestString string - if err := rows.Scan(&otherDigestString); err != nil { - return nil, fmt.Errorf("scanning other digest: %w", err) - } - otherDigest, err := digest.Parse(otherDigestString) + if uncompressedDigest != "" { + // FIXME? We could integrate this with appendReplacementCandidates into a single join instead of N+1 queries. + // (In the extreme, we could turn _everything_ this function does into a single query. + // And going even further, even DestructivelyPrioritizeReplacementCandidates could be turned into SQL.) + // For now, we prioritize simplicity, and sharing both code and implementation structure with the other cache implementations. + rows, err := tx.Query("SELECT anyDigest FROM DigestUncompressedPairs WHERE uncompressedDigest = ?", uncompressedDigest.String()) if err != nil { - return nil, err + return nil, fmt.Errorf("querying for other digests: %w", err) } - if otherDigest != primaryDigest && otherDigest != uncompressedDigest { - res, err = sqc.appendReplacementCandidates(res, tx, transport, scope, otherDigest, v2Options) + defer rows.Close() + for rows.Next() { + var otherDigestString string + if err := rows.Scan(&otherDigestString); err != nil { + return nil, fmt.Errorf("scanning other digest: %w", err) + } + otherDigest, err := digest.Parse(otherDigestString) if err != nil { return nil, err } + if otherDigest != primaryDigest && otherDigest != uncompressedDigest { + res, err = sqc.appendReplacementCandidates(res, tx, transport, scope, otherDigest, v2Options) + if err != nil { + return nil, err + } + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating through other digests: %w", err) } - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating through other digests: %w", err) - } - if uncompressedDigest != primaryDigest { - res, err = sqc.appendReplacementCandidates(res, tx, transport, scope, uncompressedDigest, v2Options) - if err != nil { - return nil, err + if uncompressedDigest != primaryDigest { + res, err = sqc.appendReplacementCandidates(res, tx, transport, scope, uncompressedDigest, v2Options) + if err != nil { + return nil, err + } } } } diff --git a/vendor/github.com/containers/image/v5/signature/fulcio_cert.go b/vendor/github.com/containers/image/v5/signature/fulcio_cert.go index 4e99864226..31dfdd3422 100644 --- a/vendor/github.com/containers/image/v5/signature/fulcio_cert.go +++ b/vendor/github.com/containers/image/v5/signature/fulcio_cert.go @@ -195,10 +195,10 @@ func (f *fulcioTrustRoot) verifyFulcioCertificateAtTime(relevantTime time.Time, return untrustedCertificate.PublicKey, nil } -func verifyRekorFulcio(rekorPublicKey *ecdsa.PublicKey, fulcioTrustRoot *fulcioTrustRoot, untrustedRekorSET []byte, +func verifyRekorFulcio(rekorPublicKeys []*ecdsa.PublicKey, fulcioTrustRoot *fulcioTrustRoot, untrustedRekorSET []byte, untrustedCertificateBytes []byte, untrustedIntermediateChainBytes []byte, untrustedBase64Signature string, untrustedPayloadBytes []byte) (crypto.PublicKey, error) { - rekorSETTime, err := internal.VerifyRekorSET(rekorPublicKey, untrustedRekorSET, untrustedCertificateBytes, + rekorSETTime, err := internal.VerifyRekorSET(rekorPublicKeys, untrustedRekorSET, untrustedCertificateBytes, untrustedBase64Signature, untrustedPayloadBytes) if err != nil { return nil, err diff --git a/vendor/github.com/containers/image/v5/signature/internal/rekor_set.go b/vendor/github.com/containers/image/v5/signature/internal/rekor_set.go index e79c91cf99..604c21c089 100644 --- a/vendor/github.com/containers/image/v5/signature/internal/rekor_set.go +++ b/vendor/github.com/containers/image/v5/signature/internal/rekor_set.go @@ -113,7 +113,7 @@ func (p UntrustedRekorPayload) MarshalJSON() ([]byte, error) { // VerifyRekorSET verifies that unverifiedRekorSET is correctly signed by publicKey and matches the rest of the data. // Returns bundle upload time on success. -func VerifyRekorSET(publicKey *ecdsa.PublicKey, unverifiedRekorSET []byte, unverifiedKeyOrCertBytes []byte, unverifiedBase64Signature string, unverifiedPayloadBytes []byte) (time.Time, error) { +func VerifyRekorSET(publicKeys []*ecdsa.PublicKey, unverifiedRekorSET []byte, unverifiedKeyOrCertBytes []byte, unverifiedBase64Signature string, unverifiedPayloadBytes []byte) (time.Time, error) { // FIXME: Should the publicKey parameter hard-code ecdsa? // == Parse SET bytes @@ -130,7 +130,14 @@ func VerifyRekorSET(publicKey *ecdsa.PublicKey, unverifiedRekorSET []byte, unver return time.Time{}, NewInvalidSignatureError(fmt.Sprintf("canonicalizing Rekor SET JSON: %v", err)) } untrustedSETPayloadHash := sha256.Sum256(untrustedSETPayloadCanonicalBytes) - if !ecdsa.VerifyASN1(publicKey, untrustedSETPayloadHash[:], untrustedSET.UntrustedSignedEntryTimestamp) { + publicKeymatched := false + for _, pk := range publicKeys { + if ecdsa.VerifyASN1(pk, untrustedSETPayloadHash[:], untrustedSET.UntrustedSignedEntryTimestamp) { + publicKeymatched = true + break + } + } + if !publicKeymatched { return time.Time{}, NewInvalidSignatureError("cryptographic signature verification of Rekor SET failed") } diff --git a/vendor/github.com/containers/image/v5/signature/internal/sigstore_payload.go b/vendor/github.com/containers/image/v5/signature/internal/sigstore_payload.go index a2609c954b..8630b08e30 100644 --- a/vendor/github.com/containers/image/v5/signature/internal/sigstore_payload.go +++ b/vendor/github.com/containers/image/v5/signature/internal/sigstore_payload.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" "github.com/containers/image/v5/version" @@ -171,24 +172,62 @@ type SigstorePayloadAcceptanceRules struct { ValidateSignedDockerManifestDigest func(digest.Digest) error } -// VerifySigstorePayload verifies unverifiedBase64Signature of unverifiedPayload was correctly created by publicKey, and that its principal components +// verifySigstorePayloadBlobSignature verifies unverifiedSignature of unverifiedPayload was correctly created +// by any of the public keys in publicKeys. +// +// This is an internal implementation detail of VerifySigstorePayload and should have no other callers. +// It is INSUFFICIENT alone to consider the signature acceptable. +func verifySigstorePayloadBlobSignature(publicKeys []crypto.PublicKey, unverifiedPayload, unverifiedSignature []byte) error { + if len(publicKeys) == 0 { + return errors.New("Need at least one public key to verify the sigstore payload, but got 0") + } + + verifiers := make([]sigstoreSignature.Verifier, 0, len(publicKeys)) + for _, key := range publicKeys { + // Failing to load a verifier indicates that something is really, really + // invalid about the public key; prefer to fail even if the signature might be + // valid with other keys, so that users fix their fallback keys before they need them. + // For that reason, we even initialize all verifiers before trying to validate the signature + // with any key. + verifier, err := sigstoreSignature.LoadVerifier(key, sigstoreHarcodedHashAlgorithm) + if err != nil { + return err + } + verifiers = append(verifiers, verifier) + } + + var failures []string + for _, verifier := range verifiers { + // github.com/sigstore/cosign/pkg/cosign.verifyOCISignature uses signatureoptions.WithContext(), + // which seems to be not used by anything. So we don’t bother. + err := verifier.VerifySignature(bytes.NewReader(unverifiedSignature), bytes.NewReader(unverifiedPayload)) + if err == nil { + return nil + } + + failures = append(failures, err.Error()) + } + + if len(failures) == 0 { + // Coverage: We have checked there is at least one public key, any success causes an early return, + // and any failure adds an entry to failures => there must be at least one error. + return fmt.Errorf("Internal error: signature verification failed but no errors have been recorded") + } + return NewInvalidSignatureError("cryptographic signature verification failed: " + strings.Join(failures, ", ")) +} + +// VerifySigstorePayload verifies unverifiedBase64Signature of unverifiedPayload was correctly created by any of the public keys in publicKeys, and that its principal components // match expected values, both as specified by rules, and returns it. // We return an *UntrustedSigstorePayload, although nothing actually uses it, // just to double-check against stupid typos. -func VerifySigstorePayload(publicKey crypto.PublicKey, unverifiedPayload []byte, unverifiedBase64Signature string, rules SigstorePayloadAcceptanceRules) (*UntrustedSigstorePayload, error) { - verifier, err := sigstoreSignature.LoadVerifier(publicKey, sigstoreHarcodedHashAlgorithm) - if err != nil { - return nil, fmt.Errorf("creating verifier: %w", err) - } - +func VerifySigstorePayload(publicKeys []crypto.PublicKey, unverifiedPayload []byte, unverifiedBase64Signature string, rules SigstorePayloadAcceptanceRules) (*UntrustedSigstorePayload, error) { unverifiedSignature, err := base64.StdEncoding.DecodeString(unverifiedBase64Signature) if err != nil { return nil, NewInvalidSignatureError(fmt.Sprintf("base64 decoding: %v", err)) } - // github.com/sigstore/cosign/pkg/cosign.verifyOCISignature uses signatureoptions.WithContext(), - // which seems to be not used by anything. So we don’t bother. - if err := verifier.VerifySignature(bytes.NewReader(unverifiedSignature), bytes.NewReader(unverifiedPayload)); err != nil { - return nil, NewInvalidSignatureError(fmt.Sprintf("cryptographic signature verification failed: %v", err)) + + if err := verifySigstorePayloadBlobSignature(publicKeys, unverifiedPayload, unverifiedSignature); err != nil { + return nil, err } var unmatchedPayload UntrustedSigstorePayload diff --git a/vendor/github.com/containers/image/v5/signature/policy_config_sigstore.go b/vendor/github.com/containers/image/v5/signature/policy_config_sigstore.go index beb5d0673e..965901e187 100644 --- a/vendor/github.com/containers/image/v5/signature/policy_config_sigstore.go +++ b/vendor/github.com/containers/image/v5/signature/policy_config_sigstore.go @@ -2,7 +2,6 @@ package signature import ( "encoding/json" - "errors" "fmt" "github.com/containers/image/v5/signature/internal" @@ -15,29 +14,57 @@ type PRSigstoreSignedOption func(*prSigstoreSigned) error func PRSigstoreSignedWithKeyPath(keyPath string) PRSigstoreSignedOption { return func(pr *prSigstoreSigned) error { if pr.KeyPath != "" { - return errors.New(`"keyPath" already specified`) + return InvalidPolicyFormatError(`"keyPath" already specified`) } pr.KeyPath = keyPath return nil } } +// PRSigstoreSignedWithKeyPaths specifies a value for the "keyPaths" field when calling NewPRSigstoreSigned. +func PRSigstoreSignedWithKeyPaths(keyPaths []string) PRSigstoreSignedOption { + return func(pr *prSigstoreSigned) error { + if pr.KeyPaths != nil { + return InvalidPolicyFormatError(`"keyPaths" already specified`) + } + if len(keyPaths) == 0 { + return InvalidPolicyFormatError(`"keyPaths" contains no entries`) + } + pr.KeyPaths = keyPaths + return nil + } +} + // PRSigstoreSignedWithKeyData specifies a value for the "keyData" field when calling NewPRSigstoreSigned. func PRSigstoreSignedWithKeyData(keyData []byte) PRSigstoreSignedOption { return func(pr *prSigstoreSigned) error { if pr.KeyData != nil { - return errors.New(`"keyData" already specified`) + return InvalidPolicyFormatError(`"keyData" already specified`) } pr.KeyData = keyData return nil } } +// PRSigstoreSignedWithKeyDatas specifies a value for the "keyDatas" field when calling NewPRSigstoreSigned. +func PRSigstoreSignedWithKeyDatas(keyDatas [][]byte) PRSigstoreSignedOption { + return func(pr *prSigstoreSigned) error { + if pr.KeyDatas != nil { + return InvalidPolicyFormatError(`"keyDatas" already specified`) + } + if len(keyDatas) == 0 { + return InvalidPolicyFormatError(`"keyDatas" contains no entries`) + } + pr.KeyDatas = keyDatas + return nil + } +} + // PRSigstoreSignedWithFulcio specifies a value for the "fulcio" field when calling NewPRSigstoreSigned. func PRSigstoreSignedWithFulcio(fulcio PRSigstoreSignedFulcio) PRSigstoreSignedOption { return func(pr *prSigstoreSigned) error { if pr.Fulcio != nil { - return errors.New(`"fulcio" already specified`) + return InvalidPolicyFormatError(`"fulcio" already specified`) } pr.Fulcio = fulcio return nil @@ -48,29 +75,57 @@ func PRSigstoreSignedWithFulcio(fulcio PRSigstoreSignedFulcio) PRSigstoreSignedO func PRSigstoreSignedWithRekorPublicKeyPath(rekorPublicKeyPath string) PRSigstoreSignedOption { return func(pr *prSigstoreSigned) error { if pr.RekorPublicKeyPath != "" { - return errors.New(`"rekorPublicKeyPath" already specified`) + return InvalidPolicyFormatError(`"rekorPublicKeyPath" already specified`) } pr.RekorPublicKeyPath = rekorPublicKeyPath return nil } } +// PRSigstoreSignedWithRekorPublicKeyPaths specifies a value for the rRekorPublickeyPaths" field when calling NewPRSigstoreSigned. +func PRSigstoreSignedWithRekorPublicKeyPaths(rekorPublickeyPaths []string) PRSigstoreSignedOption { + return func(pr *prSigstoreSigned) error { + if pr.RekorPublicKeyPaths != nil { + return InvalidPolicyFormatError(`"rekorPublickeyPaths" already specified`) + } + if len(rekorPublickeyPaths) == 0 { + return InvalidPolicyFormatError(`"rekorPublickeyPaths" contains no entries`) + } + pr.RekorPublicKeyPaths = rekorPublickeyPaths + return nil + } +} + // PRSigstoreSignedWithRekorPublicKeyData specifies a value for the "rekorPublicKeyData" field when calling NewPRSigstoreSigned. func PRSigstoreSignedWithRekorPublicKeyData(rekorPublicKeyData []byte) PRSigstoreSignedOption { return func(pr *prSigstoreSigned) error { if pr.RekorPublicKeyData != nil { - return errors.New(`"rekorPublicKeyData" already specified`) + return InvalidPolicyFormatError(`"rekorPublicKeyData" already specified`) } pr.RekorPublicKeyData = rekorPublicKeyData return nil } } +// PRSigstoreSignedWithRekorPublicKeyDatas specifies a value for the "rekorPublickeyDatas" field when calling NewPRSigstoreSigned. +func PRSigstoreSignedWithRekorPublicKeyDatas(rekorPublickeyDatas [][]byte) PRSigstoreSignedOption { + return func(pr *prSigstoreSigned) error { + if pr.RekorPublicKeyDatas != nil { + return InvalidPolicyFormatError(`"rekorPublickeyDatas" already specified`) + } + if len(rekorPublickeyDatas) == 0 { + return InvalidPolicyFormatError(`"rekorPublickeyDatas" contains no entries`) + } + pr.RekorPublicKeyDatas = rekorPublickeyDatas + return nil + } +} + // PRSigstoreSignedWithSignedIdentity specifies a value for the "signedIdentity" field when calling NewPRSigstoreSigned. func PRSigstoreSignedWithSignedIdentity(signedIdentity PolicyReferenceMatch) PRSigstoreSignedOption { return func(pr *prSigstoreSigned) error { if pr.SignedIdentity != nil { - return errors.New(`"signedIdentity" already specified`) + return InvalidPolicyFormatError(`"signedIdentity" already specified`) } pr.SignedIdentity = signedIdentity return nil @@ -92,21 +147,40 @@ func newPRSigstoreSigned(options ...PRSigstoreSignedOption) (*prSigstoreSigned, if res.KeyPath != "" { keySources++ } + if res.KeyPaths != nil { + keySources++ + } if res.KeyData != nil { keySources++ } + if res.KeyDatas != nil { + keySources++ + } if res.Fulcio != nil { keySources++ } if keySources != 1 { - return nil, InvalidPolicyFormatError("exactly one of keyPath, keyData and fulcio must be specified") + return nil, InvalidPolicyFormatError("exactly one of keyPath, keyPaths, keyData, keyDatas and fulcio must be specified") } - if res.RekorPublicKeyPath != "" && res.RekorPublicKeyData != nil { - return nil, InvalidPolicyFormatError("rekorPublickeyType and rekorPublickeyData cannot be used simultaneously") + rekorSources := 0 + if res.RekorPublicKeyPath != "" { + rekorSources++ + } + if res.RekorPublicKeyPaths != nil { + rekorSources++ + } + if res.RekorPublicKeyData != nil { + rekorSources++ + } + if res.RekorPublicKeyDatas != nil { + rekorSources++ } - if res.Fulcio != nil && res.RekorPublicKeyPath == "" && res.RekorPublicKeyData == nil { - return nil, InvalidPolicyFormatError("At least one of RekorPublickeyPath and RekorPublickeyData must be specified if fulcio is used") + if rekorSources > 1 { + return nil, InvalidPolicyFormatError("at most one of rekorPublickeyPath, rekorPublicKeyPaths, rekorPublickeyData and rekorPublicKeyDatas can be used simultaneously") + } + if res.Fulcio != nil && rekorSources == 0 { + return nil, InvalidPolicyFormatError("At least one of rekorPublickeyPath, rekorPublicKeyPaths, rekorPublickeyData and rekorPublicKeyDatas must be specified if fulcio is used") } if res.SignedIdentity == nil { @@ -144,7 +218,8 @@ var _ json.Unmarshaler = (*prSigstoreSigned)(nil) func (pr *prSigstoreSigned) UnmarshalJSON(data []byte) error { *pr = prSigstoreSigned{} var tmp prSigstoreSigned - var gotKeyPath, gotKeyData, gotFulcio, gotRekorPublicKeyPath, gotRekorPublicKeyData bool + var gotKeyPath, gotKeyPaths, gotKeyData, gotKeyDatas, gotFulcio bool + var gotRekorPublicKeyPath, gotRekorPublicKeyPaths, gotRekorPublicKeyData, gotRekorPublicKeyDatas bool var fulcio prSigstoreSignedFulcio var signedIdentity json.RawMessage if err := internal.ParanoidUnmarshalJSONObject(data, func(key string) any { @@ -154,18 +229,30 @@ func (pr *prSigstoreSigned) UnmarshalJSON(data []byte) error { case "keyPath": gotKeyPath = true return &tmp.KeyPath + case "keyPaths": + gotKeyPaths = true + return &tmp.KeyPaths case "keyData": gotKeyData = true return &tmp.KeyData + case "keyDatas": + gotKeyDatas = true + return &tmp.KeyDatas case "fulcio": gotFulcio = true return &fulcio case "rekorPublicKeyPath": gotRekorPublicKeyPath = true return &tmp.RekorPublicKeyPath + case "rekorPublicKeyPaths": + gotRekorPublicKeyPaths = true + return &tmp.RekorPublicKeyPaths case "rekorPublicKeyData": gotRekorPublicKeyData = true return &tmp.RekorPublicKeyData + case "rekorPublicKeyDatas": + gotRekorPublicKeyDatas = true + return &tmp.RekorPublicKeyDatas case "signedIdentity": return &signedIdentity default: @@ -192,18 +279,30 @@ func (pr *prSigstoreSigned) UnmarshalJSON(data []byte) error { if gotKeyPath { opts = append(opts, PRSigstoreSignedWithKeyPath(tmp.KeyPath)) } + if gotKeyPaths { + opts = append(opts, PRSigstoreSignedWithKeyPaths(tmp.KeyPaths)) + } if gotKeyData { opts = append(opts, PRSigstoreSignedWithKeyData(tmp.KeyData)) } + if gotKeyDatas { + opts = append(opts, PRSigstoreSignedWithKeyDatas(tmp.KeyDatas)) + } if gotFulcio { opts = append(opts, PRSigstoreSignedWithFulcio(&fulcio)) } if gotRekorPublicKeyPath { opts = append(opts, PRSigstoreSignedWithRekorPublicKeyPath(tmp.RekorPublicKeyPath)) } + if gotRekorPublicKeyPaths { + opts = append(opts, PRSigstoreSignedWithRekorPublicKeyPaths(tmp.RekorPublicKeyPaths)) + } if gotRekorPublicKeyData { opts = append(opts, PRSigstoreSignedWithRekorPublicKeyData(tmp.RekorPublicKeyData)) } + if gotRekorPublicKeyDatas { + opts = append(opts, PRSigstoreSignedWithRekorPublicKeyDatas(tmp.RekorPublicKeyDatas)) + } opts = append(opts, PRSigstoreSignedWithSignedIdentity(tmp.SignedIdentity)) res, err := newPRSigstoreSigned(opts...) @@ -221,7 +320,7 @@ type PRSigstoreSignedFulcioOption func(*prSigstoreSignedFulcio) error func PRSigstoreSignedFulcioWithCAPath(caPath string) PRSigstoreSignedFulcioOption { return func(f *prSigstoreSignedFulcio) error { if f.CAPath != "" { - return errors.New(`"caPath" already specified`) + return InvalidPolicyFormatError(`"caPath" already specified`) } f.CAPath = caPath return nil @@ -232,7 +331,7 @@ func PRSigstoreSignedFulcioWithCAPath(caPath string) PRSigstoreSignedFulcioOptio func PRSigstoreSignedFulcioWithCAData(caData []byte) PRSigstoreSignedFulcioOption { return func(f *prSigstoreSignedFulcio) error { if f.CAData != nil { - return errors.New(`"caData" already specified`) + return InvalidPolicyFormatError(`"caData" already specified`) } f.CAData = caData return nil @@ -243,7 +342,7 @@ func PRSigstoreSignedFulcioWithCAData(caData []byte) PRSigstoreSignedFulcioOptio func PRSigstoreSignedFulcioWithOIDCIssuer(oidcIssuer string) PRSigstoreSignedFulcioOption { return func(f *prSigstoreSignedFulcio) error { if f.OIDCIssuer != "" { - return errors.New(`"oidcIssuer" already specified`) + return InvalidPolicyFormatError(`"oidcIssuer" already specified`) } f.OIDCIssuer = oidcIssuer return nil @@ -254,7 +353,7 @@ func PRSigstoreSignedFulcioWithOIDCIssuer(oidcIssuer string) PRSigstoreSignedFul func PRSigstoreSignedFulcioWithSubjectEmail(subjectEmail string) PRSigstoreSignedFulcioOption { return func(f *prSigstoreSignedFulcio) error { if f.SubjectEmail != "" { - return errors.New(`"subjectEmail" already specified`) + return InvalidPolicyFormatError(`"subjectEmail" already specified`) } f.SubjectEmail = subjectEmail return nil diff --git a/vendor/github.com/containers/image/v5/signature/policy_eval_signedby.go b/vendor/github.com/containers/image/v5/signature/policy_eval_signedby.go index 896ca5a60d..e5c9329185 100644 --- a/vendor/github.com/containers/image/v5/signature/policy_eval_signedby.go +++ b/vendor/github.com/containers/image/v5/signature/policy_eval_signedby.go @@ -6,7 +6,6 @@ import ( "context" "errors" "fmt" - "os" "slices" "github.com/containers/image/v5/internal/multierr" @@ -27,33 +26,18 @@ func (pr *prSignedBy) isSignatureAuthorAccepted(ctx context.Context, image priva } // FIXME: move this to per-context initialization - var data [][]byte - keySources := 0 - if pr.KeyPath != "" { - keySources++ - d, err := os.ReadFile(pr.KeyPath) - if err != nil { - return sarRejected, nil, err - } - data = [][]byte{d} - } - if pr.KeyPaths != nil { - keySources++ - data = [][]byte{} - for _, path := range pr.KeyPaths { - d, err := os.ReadFile(path) - if err != nil { - return sarRejected, nil, err - } - data = append(data, d) - } - } - if pr.KeyData != nil { - keySources++ - data = [][]byte{pr.KeyData} + const notOneSourceErrorText = `Internal inconsistency: not exactly one of "keyPath", "keyPaths" and "keyData" specified` + data, err := loadBytesFromConfigSources(configBytesSources{ + inconsistencyErrorMessage: notOneSourceErrorText, + path: pr.KeyPath, + paths: pr.KeyPaths, + data: pr.KeyData, + }) + if err != nil { + return sarRejected, nil, err } - if keySources != 1 { - return sarRejected, nil, errors.New(`Internal inconsistency: not exactly one of "keyPath", "keyPaths" and "keyData" specified`) + if data == nil { + return sarRejected, nil, errors.New(notOneSourceErrorText) } // FIXME: move this to per-context initialization diff --git a/vendor/github.com/containers/image/v5/signature/policy_eval_sigstore.go b/vendor/github.com/containers/image/v5/signature/policy_eval_sigstore.go index 4851650778..9c553771cb 100644 --- a/vendor/github.com/containers/image/v5/signature/policy_eval_sigstore.go +++ b/vendor/github.com/containers/image/v5/signature/policy_eval_sigstore.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "os" + "strings" "github.com/containers/image/v5/internal/multierr" "github.com/containers/image/v5/internal/private" @@ -20,37 +21,69 @@ import ( "github.com/sigstore/sigstore/pkg/cryptoutils" ) -// loadBytesFromDataOrPath ensures there is at most one of ${prefix}Data and ${prefix}Path set, +// configBytesSources contains configuration fields which may result in one or more []byte values +type configBytesSources struct { + inconsistencyErrorMessage string // Error to return if more than one source is set + path string // …Path: a path to a file containing the data, or "" + paths []string // …Paths: paths to files containing the data, or nil + data []byte // …Data: a single instance ofhe raw data, or nil + datas [][]byte // …Datas: the raw data, or nil // codespell:ignore datas +} + +// loadBytesFromConfigSources ensures at most one of the sources in src is set, // and returns the referenced data, or nil if neither is set. -func loadBytesFromDataOrPath(prefix string, data []byte, path string) ([]byte, error) { - switch { - case data != nil && path != "": - return nil, fmt.Errorf(`Internal inconsistency: both "%sPath" and "%sData" specified`, prefix, prefix) - case path != "": - d, err := os.ReadFile(path) +func loadBytesFromConfigSources(src configBytesSources) ([][]byte, error) { + sources := 0 + var data [][]byte // = nil + if src.path != "" { + sources++ + d, err := os.ReadFile(src.path) if err != nil { return nil, err } - return d, nil - case data != nil: - return data, nil - default: // Nothing - return nil, nil + data = [][]byte{d} + } + if src.paths != nil { + sources++ + data = [][]byte{} + for _, path := range src.paths { + d, err := os.ReadFile(path) + if err != nil { + return nil, err + } + data = append(data, d) + } } + if src.data != nil { + sources++ + data = [][]byte{src.data} + } + if src.datas != nil { // codespell:ignore datas + sources++ + data = src.datas // codespell:ignore datas + } + if sources > 1 { + return nil, errors.New(src.inconsistencyErrorMessage) + } + return data, nil } // prepareTrustRoot creates a fulcioTrustRoot from the input data. // (This also prevents external implementations of this interface, ensuring that prSigstoreSignedFulcio is the only one.) func (f *prSigstoreSignedFulcio) prepareTrustRoot() (*fulcioTrustRoot, error) { - caCertBytes, err := loadBytesFromDataOrPath("fulcioCA", f.CAData, f.CAPath) + caCertPEMs, err := loadBytesFromConfigSources(configBytesSources{ + inconsistencyErrorMessage: `Internal inconsistency: both "caPath" and "caData" specified`, + path: f.CAPath, + data: f.CAData, + }) if err != nil { return nil, err } - if caCertBytes == nil { - return nil, errors.New(`Internal inconsistency: Fulcio specified with neither "caPath" nor "caData"`) + if len(caCertPEMs) != 1 { + return nil, errors.New(`Internal inconsistency: Fulcio specified with not exactly one of "caPath" nor "caData"`) } certs := x509.NewCertPool() - if ok := certs.AppendCertsFromPEM(caCertBytes); !ok { + if ok := certs.AppendCertsFromPEM(caCertPEMs[0]); !ok { return nil, errors.New("error loading Fulcio CA certificates") } fulcio := fulcioTrustRoot{ @@ -66,24 +99,35 @@ func (f *prSigstoreSignedFulcio) prepareTrustRoot() (*fulcioTrustRoot, error) { // sigstoreSignedTrustRoot contains an already parsed version of the prSigstoreSigned policy type sigstoreSignedTrustRoot struct { - publicKey crypto.PublicKey - fulcio *fulcioTrustRoot - rekorPublicKey *ecdsa.PublicKey + publicKeys []crypto.PublicKey + fulcio *fulcioTrustRoot + rekorPublicKeys []*ecdsa.PublicKey } func (pr *prSigstoreSigned) prepareTrustRoot() (*sigstoreSignedTrustRoot, error) { res := sigstoreSignedTrustRoot{} - publicKeyPEM, err := loadBytesFromDataOrPath("key", pr.KeyData, pr.KeyPath) + publicKeyPEMs, err := loadBytesFromConfigSources(configBytesSources{ + inconsistencyErrorMessage: `Internal inconsistency: more than one of "keyPath", "keyPaths", "keyData", "keyDatas" specified`, + path: pr.KeyPath, + paths: pr.KeyPaths, + data: pr.KeyData, + datas: pr.KeyDatas, // codespell:ignore datas + }) if err != nil { return nil, err } - if publicKeyPEM != nil { - pk, err := cryptoutils.UnmarshalPEMToPublicKey(publicKeyPEM) - if err != nil { - return nil, fmt.Errorf("parsing public key: %w", err) + if publicKeyPEMs != nil { + for index, keyData := range publicKeyPEMs { + pk, err := cryptoutils.UnmarshalPEMToPublicKey(keyData) + if err != nil { + return nil, fmt.Errorf("parsing public key %d: %w", index+1, err) + } + res.publicKeys = append(res.publicKeys, pk) + } + if len(res.publicKeys) == 0 { + return nil, errors.New(`Internal inconsistency: "keyPath", "keyPaths", "keyData" and "keyDatas" produced no public keys`) } - res.publicKey = pk } if pr.Fulcio != nil { @@ -94,21 +138,32 @@ func (pr *prSigstoreSigned) prepareTrustRoot() (*sigstoreSignedTrustRoot, error) res.fulcio = f } - rekorPublicKeyPEM, err := loadBytesFromDataOrPath("rekorPublicKey", pr.RekorPublicKeyData, pr.RekorPublicKeyPath) + rekorPublicKeyPEMs, err := loadBytesFromConfigSources(configBytesSources{ + inconsistencyErrorMessage: `Internal inconsistency: both "rekorPublicKeyPath" and "rekorPublicKeyData" specified`, + path: pr.RekorPublicKeyPath, + paths: pr.RekorPublicKeyPaths, + data: pr.RekorPublicKeyData, + datas: pr.RekorPublicKeyDatas, // codespell:ignore datas + }) if err != nil { return nil, err } - if rekorPublicKeyPEM != nil { - pk, err := cryptoutils.UnmarshalPEMToPublicKey(rekorPublicKeyPEM) - if err != nil { - return nil, fmt.Errorf("parsing Rekor public key: %w", err) - } - pkECDSA, ok := pk.(*ecdsa.PublicKey) - if !ok { - return nil, fmt.Errorf("Rekor public key is not using ECDSA") + if rekorPublicKeyPEMs != nil { + for index, pem := range rekorPublicKeyPEMs { + pk, err := cryptoutils.UnmarshalPEMToPublicKey(pem) + if err != nil { + return nil, fmt.Errorf("parsing Rekor public key %d: %w", index+1, err) + } + pkECDSA, ok := pk.(*ecdsa.PublicKey) + if !ok { + return nil, fmt.Errorf("Rekor public key %d is not using ECDSA", index+1) + } + res.rekorPublicKeys = append(res.rekorPublicKeys, pkECDSA) + } + if len(res.rekorPublicKeys) == 0 { + return nil, errors.New(`Internal inconsistency: "rekorPublicKeyPath", "rekorPublicKeyPaths", "rekorPublicKeyData" and "rekorPublicKeyDatas" produced no public keys`) } - res.rekorPublicKey = pkECDSA } return &res, nil @@ -134,37 +189,51 @@ func (pr *prSigstoreSigned) isSignatureAccepted(ctx context.Context, image priva } untrustedPayload := sig.UntrustedPayload() - var publicKey crypto.PublicKey + var publicKeys []crypto.PublicKey switch { - case trustRoot.publicKey != nil && trustRoot.fulcio != nil: // newPRSigstoreSigned rejects such combinations. + case trustRoot.publicKeys != nil && trustRoot.fulcio != nil: // newPRSigstoreSigned rejects such combinations. return sarRejected, errors.New("Internal inconsistency: Both a public key and Fulcio CA specified") - case trustRoot.publicKey == nil && trustRoot.fulcio == nil: // newPRSigstoreSigned rejects such combinations. + case trustRoot.publicKeys == nil && trustRoot.fulcio == nil: // newPRSigstoreSigned rejects such combinations. return sarRejected, errors.New("Internal inconsistency: Neither a public key nor a Fulcio CA specified") - case trustRoot.publicKey != nil: - if trustRoot.rekorPublicKey != nil { + case trustRoot.publicKeys != nil: + if trustRoot.rekorPublicKeys != nil { untrustedSET, ok := untrustedAnnotations[signature.SigstoreSETAnnotationKey] if !ok { // For user convenience; passing an empty []byte to VerifyRekorSet should work. return sarRejected, fmt.Errorf("missing %s annotation", signature.SigstoreSETAnnotationKey) } - // We could use publicKeyPEM directly, but let’s re-marshal to avoid inconsistencies. - // FIXME: We could just generate DER instead of the full PEM text - recreatedPublicKeyPEM, err := cryptoutils.MarshalPublicKeyToPEM(trustRoot.publicKey) - if err != nil { - // Coverage: The key was loaded from a PEM format, so it’s unclear how this could fail. - // (PEM is not essential, MarshalPublicKeyToPEM can only fail if marshaling to ASN1.DER fails.) - return sarRejected, fmt.Errorf("re-marshaling public key to PEM: %w", err) + var rekorFailures []string + for _, candidatePublicKey := range trustRoot.publicKeys { + // We could use publicKeyPEM directly, but let’s re-marshal to avoid inconsistencies. + // FIXME: We could just generate DER instead of the full PEM text + recreatedPublicKeyPEM, err := cryptoutils.MarshalPublicKeyToPEM(candidatePublicKey) + if err != nil { + // Coverage: The key was loaded from a PEM format, so it’s unclear how this could fail. + // (PEM is not essential, MarshalPublicKeyToPEM can only fail if marshaling to ASN1.DER fails.) + return sarRejected, fmt.Errorf("re-marshaling public key to PEM: %w", err) + } + // We don’t care about the Rekor timestamp, just about log presence. + _, err = internal.VerifyRekorSET(trustRoot.rekorPublicKeys, []byte(untrustedSET), recreatedPublicKeyPEM, untrustedBase64Signature, untrustedPayload) + if err == nil { + publicKeys = append(publicKeys, candidatePublicKey) + break // The SET can only accept one public key entry, so if we found one, the rest either doesn’t match or is a duplicate + } + rekorFailures = append(rekorFailures, err.Error()) } - // We don’t care about the Rekor timestamp, just about log presence. - if _, err := internal.VerifyRekorSET(trustRoot.rekorPublicKey, []byte(untrustedSET), recreatedPublicKeyPEM, untrustedBase64Signature, untrustedPayload); err != nil { - return sarRejected, err + if len(publicKeys) == 0 { + if len(rekorFailures) == 0 { + // Coverage: We have ensured that len(trustRoot.publicKeys) != 0, when nothing succeeds, there must be at least one failure. + return sarRejected, errors.New(`Internal inconsistency: Rekor SET did not match any key but we have no failures.`) + } + return sarRejected, internal.NewInvalidSignatureError(fmt.Sprintf("No public key verified against the RekorSET: %s", strings.Join(rekorFailures, ", "))) } + } else { + publicKeys = trustRoot.publicKeys } - publicKey = trustRoot.publicKey case trustRoot.fulcio != nil: - if trustRoot.rekorPublicKey == nil { // newPRSigstoreSigned rejects such combinations. + if trustRoot.rekorPublicKeys == nil { // newPRSigstoreSigned rejects such combinations. return sarRejected, errors.New("Internal inconsistency: Fulcio CA specified without a Rekor public key") } untrustedSET, ok := untrustedAnnotations[signature.SigstoreSETAnnotationKey] @@ -179,19 +248,20 @@ func (pr *prSigstoreSigned) isSignatureAccepted(ctx context.Context, image priva if untrustedIntermediateChain, ok := untrustedAnnotations[signature.SigstoreIntermediateCertificateChainAnnotationKey]; ok { untrustedIntermediateChainBytes = []byte(untrustedIntermediateChain) } - pk, err := verifyRekorFulcio(trustRoot.rekorPublicKey, trustRoot.fulcio, + pk, err := verifyRekorFulcio(trustRoot.rekorPublicKeys, trustRoot.fulcio, []byte(untrustedSET), []byte(untrustedCert), untrustedIntermediateChainBytes, untrustedBase64Signature, untrustedPayload) if err != nil { return sarRejected, err } - publicKey = pk + publicKeys = []crypto.PublicKey{pk} } - if publicKey == nil { - // Coverage: This should never happen, we have already excluded the possibility in the switch above. + if len(publicKeys) == 0 { + // Coverage: This should never happen, we ensured that trustRoot.publicKeys is non-empty if set, + // and we have already excluded the possibility in the switch above. return sarRejected, fmt.Errorf("Internal inconsistency: publicKey not set before verifying sigstore payload") } - signature, err := internal.VerifySigstorePayload(publicKey, untrustedPayload, untrustedBase64Signature, internal.SigstorePayloadAcceptanceRules{ + signature, err := internal.VerifySigstorePayload(publicKeys, untrustedPayload, untrustedBase64Signature, internal.SigstorePayloadAcceptanceRules{ ValidateSignedDockerReference: func(ref string) error { if !pr.SignedIdentity.matchesDockerReference(image, ref) { return PolicyRequirementError(fmt.Sprintf("Signature for identity %q is not accepted", ref)) diff --git a/vendor/github.com/containers/image/v5/signature/policy_types.go b/vendor/github.com/containers/image/v5/signature/policy_types.go index 96e91a0a9c..32aa1c0ad4 100644 --- a/vendor/github.com/containers/image/v5/signature/policy_types.go +++ b/vendor/github.com/containers/image/v5/signature/policy_types.go @@ -74,7 +74,7 @@ type prSignedBy struct { // KeyPath is a pathname to a local file containing the trusted key(s). Exactly one of KeyPath, KeyPaths and KeyData must be specified. KeyPath string `json:"keyPath,omitempty"` - // KeyPaths if a set of pathnames to local files containing the trusted key(s). Exactly one of KeyPath, KeyPaths and KeyData must be specified. + // KeyPaths is a set of pathnames to local files containing the trusted key(s). Exactly one of KeyPath, KeyPaths and KeyData must be specified. KeyPaths []string `json:"keyPaths,omitempty"` // KeyData contains the trusted key(s), base64-encoded. Exactly one of KeyPath, KeyPaths and KeyData must be specified. KeyData []byte `json:"keyData,omitempty"` @@ -111,24 +111,35 @@ type prSignedBaseLayer struct { type prSigstoreSigned struct { prCommon - // KeyPath is a pathname to a local file containing the trusted key. Exactly one of KeyPath, KeyData, Fulcio must be specified. + // KeyPath is a pathname to a local file containing the trusted key. Exactly one of KeyPath, KeyPaths, KeyData, KeyDatas and Fulcio must be specified. KeyPath string `json:"keyPath,omitempty"` - // KeyData contains the trusted key, base64-encoded. Exactly one of KeyPath, KeyData, Fulcio must be specified. + // KeyPaths is a set of pathnames to local files containing the trusted key(s). Exactly one of KeyPath, KeyPaths, KeyData, KeyDatas and Fulcio must be specified. + KeyPaths []string `json:"keyPaths,omitempty"` + // KeyData contains the trusted key, base64-encoded. Exactly one of KeyPath, KeyPaths, KeyData, KeyDatas and Fulcio must be specified. KeyData []byte `json:"keyData,omitempty"` - // FIXME: Multiple public keys? + // KeyDatas is a set of trusted keys, base64-encoded. Exactly one of KeyPath, KeyPaths, KeyData, KeyDatas and Fulcio must be specified. + KeyDatas [][]byte `json:"keyDatas,omitempty"` - // Fulcio specifies which Fulcio-generated certificates are accepted. Exactly one of KeyPath, KeyData, Fulcio must be specified. + // Fulcio specifies which Fulcio-generated certificates are accepted. Exactly one of KeyPath, KeyPaths, KeyData, KeyDatas and Fulcio must be specified. // If Fulcio is specified, one of RekorPublicKeyPath or RekorPublicKeyData must be specified as well. Fulcio PRSigstoreSignedFulcio `json:"fulcio,omitempty"` // RekorPublicKeyPath is a pathname to local file containing a public key of a Rekor server which must record acceptable signatures. - // If Fulcio is used, one of RekorPublicKeyPath or RekorPublicKeyData must be specified as well; otherwise it is optional - // (and Rekor inclusion is not required if a Rekor public key is not specified). + // If Fulcio is used, one of RekorPublicKeyPath, RekorPublicKeyPaths, RekorPublicKeyData and RekorPublicKeyDatas must be specified as well; + // otherwise it is optional (and Rekor inclusion is not required if a Rekor public key is not specified). RekorPublicKeyPath string `json:"rekorPublicKeyPath,omitempty"` + // RekorPublicKeyPaths is a set of pathnames to local files, each containing a public key of a Rekor server. One of the keys must record acceptable signatures. + // If Fulcio is used, one of RekorPublicKeyPath, RekorPublicKeyPaths, RekorPublicKeyData and RekorPublicKeyDatas must be specified as well; + // otherwise it is optional (and Rekor inclusion is not required if a Rekor public key is not specified). + RekorPublicKeyPaths []string `json:"rekorPublicKeyPaths,omitempty"` // RekorPublicKeyPath contain a base64-encoded public key of a Rekor server which must record acceptable signatures. - // If Fulcio is used, one of RekorPublicKeyPath or RekorPublicKeyData must be specified as well; otherwise it is optional - // (and Rekor inclusion is not required if a Rekor public key is not specified). + // If Fulcio is used, one of RekorPublicKeyPath, RekorPublicKeyPaths, RekorPublicKeyData and RekorPublicKeyDatas must be specified as well; + // otherwise it is optional (and Rekor inclusion is not required if a Rekor public key is not specified). RekorPublicKeyData []byte `json:"rekorPublicKeyData,omitempty"` + // RekorPublicKeyDatas each contain a base64-encoded public key of a Rekor server. One of the keys must record acceptable signatures. + // If Fulcio is used, one of RekorPublicKeyPath, RekorPublicKeyPaths, RekorPublicKeyData and RekorPublicKeyDatas must be specified as well; + // otherwise it is optional (and Rekor inclusion is not required if a Rekor public key is not specified). + RekorPublicKeyDatas [][]byte `json:"rekorPublicKeyDatas,omitempty"` // SignedIdentity specifies what image identity the signature must be claiming about the image. // Defaults to "matchRepoDigestOrExact" if not specified. diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index d98e04b9a5..64e4687259 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -6,12 +6,12 @@ const ( // VersionMajor is for an API incompatible changes VersionMajor = 5 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 33 + VersionMinor = 32 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 0 + VersionPatch = 2 // VersionDev indicates development branch. Releases will be empty string. - VersionDev = "-dev" + VersionDev = "" ) // Version is the specification version that the package types support. diff --git a/vendor/modules.txt b/vendor/modules.txt index bf5999fee4..a2dffe3085 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -141,7 +141,7 @@ github.com/containernetworking/cni/pkg/version # github.com/containernetworking/plugins v1.5.1 ## explicit; go 1.20 github.com/containernetworking/plugins/pkg/ns -# github.com/containers/buildah v1.37.0 +# github.com/containers/buildah v1.37.3 ## explicit; go 1.21.0 github.com/containers/buildah github.com/containers/buildah/bind @@ -171,8 +171,8 @@ github.com/containers/buildah/pkg/sshagent github.com/containers/buildah/pkg/util github.com/containers/buildah/pkg/volumes github.com/containers/buildah/util -# github.com/containers/common v0.60.1-0.20240920125326-ff6611ae40ad -## explicit; go 1.22.0 +# github.com/containers/common v0.60.2 +## explicit; go 1.21.0 github.com/containers/common/internal github.com/containers/common/internal/attributedstring github.com/containers/common/libimage @@ -244,7 +244,7 @@ github.com/containers/conmon/runner/config # github.com/containers/gvisor-tap-vsock v0.7.5 ## explicit; go 1.21 github.com/containers/gvisor-tap-vsock/pkg/types -# github.com/containers/image/v5 v5.32.1-0.20240806084436-e3e9287ca8e6 +# github.com/containers/image/v5 v5.32.2 ## explicit; go 1.21.0 github.com/containers/image/v5/copy github.com/containers/image/v5/directory