From 5d9043bbddd7755fa5f13776250e7f1a4836eedf Mon Sep 17 00:00:00 2001 From: Jacob Blain Christen Date: Thu, 11 Mar 2021 15:40:12 -0700 Subject: [PATCH] Mirror repository rewrites (v1.1) Support CRI configuration to allow for request-time rewrite rules applicable only to the repository portion of resource paths when pulling images. Because the rewrites are applied at request time, images themselves will not be "rewritten" -- images as stored by CRI (and the underlying containerd facility) will continue to present as normal. As an example, if you use the following config for your containerd: ```toml [plugins] [plugins."io.containerd.grpc.v1.cri"] [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] endpoint = ["https://registry-1.docker.io/v2"] [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io".rewrite] "^library/(.*)" = "my-org/$1" ``` And then subsequently invoke `crictl pull alpine:3.13` it will pull content from `docker.io/my-org/alpine:3.13` but still show up as `docker.io/library/alpine:3.13` in the `crictl images` listing. This commit has been reworked from the original implementation. Rewites are now done when resolving instead of when building the request, so that auth token scopes stored in the context properly reflect the rewritten repository path. For the original implementation, see 06c4ea9baec2b278b8172a789bf601168292f645. Ref: https://github.com/k3s-io/k3s/issues/11191#issuecomment-2455525773 Signed-off-by: Jacob Blain Christen Co-authored-by: Brad Davidson Signed-off-by: Brad Davidson --- pkg/cri/config/config.go | 17 +++++++++++ pkg/cri/server/image_pull.go | 19 +++++++++++++ remotes/docker/fetcher.go | 55 ++++++++++++++++++++++-------------- remotes/docker/pusher.go | 16 ++++++----- remotes/docker/registry.go | 1 + remotes/docker/resolver.go | 42 +++++++++++++++++++-------- 6 files changed, 111 insertions(+), 39 deletions(-) diff --git a/pkg/cri/config/config.go b/pkg/cri/config/config.go index 2200e2778fae..48430cfa3e76 100644 --- a/pkg/cri/config/config.go +++ b/pkg/cri/config/config.go @@ -199,6 +199,23 @@ type Mirror struct { // with host specified. // The scheme, host and path from the endpoint URL will be used. Endpoints []string `toml:"endpoint" json:"endpoint"` + + // Rewrites are repository rewrite rules for a namespace. When fetching image resources + // from an endpoint and a key matches the repository via regular expression matching + // it will be replaced with the corresponding value from the map in the resource request. + // + // This example configures CRI to pull docker.io/library/* images from docker.io/my-org/*: + // + // [plugins] + // [plugins."io.containerd.grpc.v1.cri"] + // [plugins."io.containerd.grpc.v1.cri".registry] + // [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + // [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + // endpoint = ["https://registry-1.docker.io/v2"] + // [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io".rewrite] + // "^library/(.*)" = "my-org/$1" + // + Rewrites map[string]string `toml:"rewrite" json:"rewrite"` } // AuthConfig contains the config related to authentication to a specific registry diff --git a/pkg/cri/server/image_pull.go b/pkg/cri/server/image_pull.go index 6b321515bd70..1b5457074d3f 100644 --- a/pkg/cri/server/image_pull.go +++ b/pkg/cri/server/image_pull.go @@ -449,6 +449,10 @@ func (c *criService) registryHosts(ctx context.Context, auth *runtime.AuthConfig if err != nil { return nil, fmt.Errorf("get registry endpoints: %w", err) } + rewrites, err := c.registryRewrites(host) + if err != nil { + return nil, fmt.Errorf("get registry rewrites: %w", err) + } for _, e := range endpoints { u, err := url.Parse(e) if err != nil { @@ -503,6 +507,7 @@ func (c *criService) registryHosts(ctx context.Context, auth *runtime.AuthConfig Scheme: u.Scheme, Path: u.Path, Capabilities: docker.HostCapabilityResolve | docker.HostCapabilityPull, + Rewrites: rewrites, }) } return registries, nil @@ -565,6 +570,20 @@ func (c *criService) registryEndpoints(host string) ([]string, error) { return append(endpoints, defaultScheme(defaultHost)+"://"+defaultHost), nil } +func (c *criService) registryRewrites(host string) (map[string]string, error) { + var rewrites map[string]string + _, ok := c.config.Registry.Mirrors[host] + if ok { + rewrites = c.config.Registry.Mirrors[host].Rewrites + } else { + rewrites = c.config.Registry.Mirrors["*"].Rewrites + } + if rewrites == nil { + rewrites = map[string]string{} + } + return rewrites, nil +} + // newTransport returns a new HTTP transport used to pull image. // TODO(random-liu): Create a library and share this code with `ctr`. func newTransport() *http.Transport { diff --git a/remotes/docker/fetcher.go b/remotes/docker/fetcher.go index c4c401ad1d56..f5dc2d4e9bb9 100644 --- a/remotes/docker/fetcher.go +++ b/remotes/docker/fetcher.go @@ -46,14 +46,14 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R return nil, fmt.Errorf("no pull hosts: %w", errdefs.ErrNotFound) } - ctx, err := ContextWithRepositoryScope(ctx, r.refspec, false) - if err != nil { - return nil, err - } - return newHTTPReadSeeker(desc.Size, func(offset int64) (io.ReadCloser, error) { // firstly try fetch via external urls for _, us := range desc.URLs { + ctx, err := ContextWithRepositoryScope(ctx, r.refspec, false) + if err != nil { + return nil, err + } + u, err := url.Parse(us) if err != nil { log.G(ctx).WithError(err).Debugf("failed to parse %q", us) @@ -101,8 +101,14 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R var firstErr error for _, host := range r.hosts { - req := r.request(host, http.MethodGet, "manifests", desc.Digest.String()) - if err := req.addNamespace(r.refspec.Hostname()); err != nil { + base := r.withRewritesFromHost(host) + ctx, err := ContextWithRepositoryScope(ctx, base.refspec, false) + if err != nil { + return nil, err + } + + req := base.request(host, http.MethodGet, "manifests", desc.Digest.String()) + if err := req.addNamespace(base.refspec.Hostname()); err != nil { return nil, err } @@ -124,8 +130,14 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R // Finally use blobs endpoints var firstErr error for _, host := range r.hosts { - req := r.request(host, http.MethodGet, "blobs", desc.Digest.String()) - if err := req.addNamespace(r.refspec.Hostname()); err != nil { + base := r.withRewritesFromHost(host) + ctx, err := ContextWithRepositoryScope(ctx, base.refspec, false) + if err != nil { + return nil, err + } + + req := base.request(host, http.MethodGet, "blobs", desc.Digest.String()) + if err := req.addNamespace(base.refspec.Hostname()); err != nil { return nil, err } @@ -153,8 +165,14 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R } func (r dockerFetcher) createGetReq(ctx context.Context, host RegistryHost, ps ...string) (*request, int64, error) { - headReq := r.request(host, http.MethodHead, ps...) - if err := headReq.addNamespace(r.refspec.Hostname()); err != nil { + base := r.withRewritesFromHost(host) + ctx, err := ContextWithRepositoryScope(ctx, base.refspec, false) + if err != nil { + return nil, 0, err + } + + headReq := base.request(host, http.MethodHead, ps...) + if err := headReq.addNamespace(base.refspec.Hostname()); err != nil { return nil, 0, err } @@ -169,8 +187,8 @@ func (r dockerFetcher) createGetReq(ctx context.Context, host RegistryHost, ps . return nil, 0, fmt.Errorf("unexpected HEAD status code %v: %s", headReq.String(), headResp.Status) } - getReq := r.request(host, http.MethodGet, ps...) - if err := getReq.addNamespace(r.refspec.Hostname()); err != nil { + getReq := base.request(host, http.MethodGet, ps...) + if err := getReq.addNamespace(base.refspec.Hostname()); err != nil { return nil, 0, err } return getReq, headResp.ContentLength, nil @@ -185,15 +203,10 @@ func (r dockerFetcher) FetchByDigest(ctx context.Context, dgst digest.Digest) (i return nil, desc, fmt.Errorf("no pull hosts: %w", errdefs.ErrNotFound) } - ctx, err := ContextWithRepositoryScope(ctx, r.refspec, false) - if err != nil { - return nil, desc, err - } - var ( - getReq *request - sz int64 - firstErr error + getReq *request + sz int64 + err, firstErr error ) for _, host := range r.hosts { diff --git a/remotes/docker/pusher.go b/remotes/docker/pusher.go index f97ab144e8b7..c8c6ae4b2c78 100644 --- a/remotes/docker/pusher.go +++ b/remotes/docker/pusher.go @@ -73,10 +73,6 @@ func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, ref str l.Lock(ref) defer l.Unlock(ref) } - ctx, err := ContextWithRepositoryScope(ctx, p.refspec, true) - if err != nil { - return nil, err - } status, err := p.tracker.GetStatus(ref) if err == nil { if status.Committed && status.Offset == status.Total { @@ -104,6 +100,12 @@ func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, ref str host = hosts[0] ) + base := p.withRewritesFromHost(host) + ctx, err = ContextWithRepositoryScope(ctx, base.refspec, true) + if err != nil { + return nil, err + } + switch desc.MediaType { case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex: @@ -113,7 +115,7 @@ func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, ref str existCheck = []string{"blobs", desc.Digest.String()} } - req := p.request(host, http.MethodHead, existCheck...) + req := base.request(host, http.MethodHead, existCheck...) req.header.Set("Accept", strings.Join([]string{desc.MediaType, `*/*`}, ", ")) log.G(ctx).WithField("url", req.String()).Debugf("checking and pushing to") @@ -163,11 +165,11 @@ func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, ref str if isManifest { putPath := getManifestPath(p.object, desc.Digest) - req = p.request(host, http.MethodPut, putPath...) + req = base.request(host, http.MethodPut, putPath...) req.header.Add("Content-Type", desc.MediaType) } else { // Start upload request - req = p.request(host, http.MethodPost, "blobs", "uploads/") + req = base.request(host, http.MethodPost, "blobs", "uploads/") mountedFrom := "" var resp *http.Response diff --git a/remotes/docker/registry.go b/remotes/docker/registry.go index 98cafcd069e6..5c01a4b27974 100644 --- a/remotes/docker/registry.go +++ b/remotes/docker/registry.go @@ -74,6 +74,7 @@ type RegistryHost struct { Path string Capabilities HostCapabilities Header http.Header + Rewrites map[string]string } func (h RegistryHost) isProxy(refhost string) bool { diff --git a/remotes/docker/resolver.go b/remotes/docker/resolver.go index fe8a4399b102..a2d9ee0338de 100644 --- a/remotes/docker/resolver.go +++ b/remotes/docker/resolver.go @@ -26,6 +26,7 @@ import ( "net/http" "net/url" "path" + "regexp" "strings" "github.com/containerd/log" @@ -237,15 +238,14 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp if err != nil { return "", ocispec.Descriptor{}, err } - refspec := base.refspec - if refspec.Object == "" { + if base.refspec.Object == "" { return "", ocispec.Descriptor{}, reference.ErrObjectRequired } var ( firstErr error paths [][]string - dgst = refspec.Digest() + dgst = base.refspec.Digest() caps = HostCapabilityPull ) @@ -263,7 +263,7 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp paths = append(paths, []string{"blobs", dgst.String()}) } else { // Add - paths = append(paths, []string{"manifests", refspec.Object}) + paths = append(paths, []string{"manifests", base.refspec.Object}) caps |= HostCapabilityResolve } @@ -272,15 +272,14 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp return "", ocispec.Descriptor{}, fmt.Errorf("no resolve hosts: %w", errdefs.ErrNotFound) } - ctx, err = ContextWithRepositoryScope(ctx, refspec, false) - if err != nil { - return "", ocispec.Descriptor{}, err - } - for _, u := range paths { for _, host := range hosts { ctx := log.WithLogger(ctx, log.G(ctx).WithField("host", host.Host)) - + base := base.withRewritesFromHost(host) + ctx, err = ContextWithRepositoryScope(ctx, base.refspec, false) + if err != nil { + return "", ocispec.Descriptor{}, err + } req := base.request(host, http.MethodHead, u...) if err := req.addNamespace(base.refspec.Hostname()); err != nil { return "", ocispec.Descriptor{}, err @@ -481,7 +480,6 @@ func (r *dockerBase) request(host RegistryHost, method string, ps ...string) *re if header == nil { header = http.Header{} } - for key, value := range host.Header { header[key] = append(header[key], value...) } @@ -499,6 +497,28 @@ func (r *dockerBase) request(host RegistryHost, method string, ps ...string) *re } } +func (r *dockerBase) withRewritesFromHost(host RegistryHost) *dockerBase { + for pattern, replace := range host.Rewrites { + exp, err := regexp.Compile(pattern) + if err != nil { + log.L.WithError(err).Warnf("Failed to compile rewrite, `%s`, for %s", pattern, host.Host) + continue + } + if rr := exp.ReplaceAllString(r.repository, replace); rr != r.repository { + log.L.Debugf("Rewrote repository for %s: %s => %s", r.refspec, r.repository, rr) + return &dockerBase{ + refspec: reference.Spec{ + Locator: r.refspec.Hostname() + "/" + rr, + Object: r.refspec.Object, + }, + repository: rr, + header: r.header, + } + } + } + return r +} + func (r *request) authorize(ctx context.Context, req *http.Request) error { // Check if has header for host if r.host.Authorizer != nil {