Support always proxying requests for Docker rules

manifest/Scarf/Manifest.hs

@@ -29,6 +29,7 @@ import Data.Aeson.Encode.Pretty (encodePretty)
 import Data.Bifunctor (second)
 import Data.ByteString.Lazy (ByteString)
 import Data.HashMap.Strict qualified as HashMap
+import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import Data.Text qualified as Text
 import Scarf.Gateway.ImagePattern qualified as ImagePattern
@@ -74,7 +75,8 @@ data ManifestRule
         -- | e.g. @"library/hello-world"@
         manifestRuleRepositoryName :: !Text,
         -- | e.g. @docker.io@, @ghcr.io@
-        manifestRuleBackendRegistry :: !Text
+        manifestRuleBackendRegistry :: !Text,
+        manifestRuleAlwaysProxy :: !(Maybe Bool)
       }
   | ManifestDockerRuleV2
       { -- | Owner of the package
@@ -86,7 +88,8 @@ data ManifestRule
         -- | Id of the corresponding rule in the database
         manifestRuleId :: !Text,
         -- | e.g. @docker.io@
-        manifestRuleBackendRegistry :: !Text
+        manifestRuleBackendRegistry :: !Text,
+        manifestRuleAlwaysProxy :: !(Maybe Bool)
       }
   | ManifestFileRuleV1
       { -- | Package name
@@ -204,13 +207,15 @@ instance FromJSON ManifestRule where
           <*> o .: "domain"
           <*> o .: "repository-name"
           <*> o .: "registry"
+          <*> o .:? "always-proxy"
       "docker-v2" ->
         ManifestDockerRuleV2
           <$> o .:? "owner" .!= ""
           <*> o .: "domain"
           <*> o .: "pattern"
           <*> o .: "rule-id"
           <*> o .: "registry"
+          <*> o .:? "always-proxy"
       "file-v1" ->
         ManifestFileRuleV1
           <$> o .:? "package-name" .!= ""
@@ -260,7 +265,8 @@ instance ToJSON ManifestRule where
           "package-id" .= manifestRulePackageId,
           "domain" .= manifestRuleDomain,
           "repository-name" .= manifestRuleRepositoryName,
-          "registry" .= manifestRuleBackendRegistry
+          "registry" .= manifestRuleBackendRegistry,
+          "always-proxy" .= manifestRuleAlwaysProxy
         ]
   toJSON ManifestDockerRuleV2 {..} =
     object $
@@ -270,7 +276,8 @@ instance ToJSON ManifestRule where
           "domain" .= manifestRuleDomain,
           "registry" .= manifestRuleBackendRegistry,
           "pattern" .= manifestRulePattern,
-          "rule-id" .= manifestRuleId
+          "rule-id" .= manifestRuleId,
+          "always-proxy" .= manifestRuleAlwaysProxy
         ]
   toJSON ManifestFileRuleV1 {..} =
     object $
@@ -357,11 +364,13 @@ manifestRuleToRule manifestRule = case manifestRule of
       manifestRulePackageId
       (Text.splitOn "/" manifestRuleRepositoryName)
       manifestRuleBackendRegistry
+      (fromMaybe False manifestRuleAlwaysProxy)
   ManifestDockerRuleV2 {..} ->
     newDockerRuleV2
       manifestRuleId
       (ImagePattern.toText manifestRulePattern)
       manifestRuleBackendRegistry
+      (fromMaybe False manifestRuleAlwaysProxy)
   ManifestFileRuleV1 {..} ->
     newFlatfileRule
       manifestRulePackageId

src/Scarf/Gateway/Rule.hs

@@ -288,24 +288,44 @@ sanitizeDockerhubDomain domain = case domain of
   "registry.hub.docker.com" -> "registry-1.docker.io"
   domain -> domain
 
-newDockerRuleV1 :: Text -> [Text] -> Text -> Rule
-newDockerRuleV1 package image backendDomain =
+newDockerRuleV1 ::
+  -- Package identifier
+  Text ->
+  -- | Docker image components. e.g. ["library", "hello-world"].
+  [Text] ->
+  -- | Docker registry requests are redirected or proxied to.
+  Text ->
+  -- | Flag indicating whether to always proxy and never redirect.
+  Bool ->
+  Rule
+newDockerRuleV1 package image backendDomain alwaysProxy =
   RuleDockerV1
     DockerRuleV1
       { ruleImages = HashMap.singleton image package,
-        ruleBackendRegistry = Text.encodeUtf8 (sanitizeDockerhubDomain backendDomain)
+        ruleBackendRegistry = Text.encodeUtf8 (sanitizeDockerhubDomain backendDomain),
+        ruleAlwaysProxy = alwaysProxy
       }
 
-newDockerRuleV2 :: Text -> Text -> Text -> Rule
-newDockerRuleV2 ruleId textPattern backendDomain =
+newDockerRuleV2 ::
+  -- | Collection identifier
+  Text ->
+  -- | Pattern matching images to redirect or proxy
+  Text ->
+  -- | Docker registry requests are redirected or proxied to.
+  Text ->
+  -- | Flag indicating whether to always proxy and never redirect.
+  Bool ->
+  Rule
+newDockerRuleV2 ruleId textPattern backendDomain alwaysProxy =
   RuleDockerV2
     DockerRuleV2
       { ruleImagePattern =
           case ImagePattern.fromText textPattern of
             Nothing -> error "Invalid Pattern"
             Just p -> p,
         ruleRuleId = ruleId,
-        ruleBackendRegistry = Text.encodeUtf8 (sanitizeDockerhubDomain backendDomain)
+        ruleBackendRegistry = Text.encodeUtf8 (sanitizeDockerhubDomain backendDomain),
+        ruleAlwaysProxy = alwaysProxy
       }
 
 -- | Unsafely parses the templates and returns a flatfile rule.

src/Scarf/Gateway/Rule/Docker.hs

@@ -30,7 +30,9 @@ data DockerRuleV1 = DockerRuleV1
     -- respective package ids.
     ruleImages :: !(HashMap [Text] Text),
     -- | Domain of the registry to redirect (or proxy) to.
-    ruleBackendRegistry :: !ByteString
+    ruleBackendRegistry :: !ByteString,
+    -- | Flag indicating whether to always proxy and never redirect.
+    ruleAlwaysProxy :: !Bool
   }
   deriving (Eq, Show)
 
@@ -41,7 +43,9 @@ data DockerRuleV2 = DockerRuleV2
     -- | Id of the rule in the backend.
     ruleRuleId :: !Text,
     -- | Domain of the registry to redirect (or proxy) to.
-    ruleBackendRegistry :: !ByteString
+    ruleBackendRegistry :: !ByteString,
+    -- | Flag indicating whether to always proxy and never redirect.
+    ruleAlwaysProxy :: !Bool
   }
   deriving (Eq, Show)
 
@@ -62,24 +66,26 @@ shouldAlwaysProxy domain =
 type DockerImageMatcher = [Text] -> Maybe (Either Text Text)
 
 matchDockerRuleV1 :: (MonadMatch m) => DockerRuleV1 -> Request -> ResponseBuilder response -> m (Maybe response)
-matchDockerRuleV1 DockerRuleV1 {ruleImages, ruleBackendRegistry} =
+matchDockerRuleV1 DockerRuleV1 {ruleImages, ruleBackendRegistry, ruleAlwaysProxy} =
   matchDocker
     ( \image ->
         case HashMap.lookup image ruleImages of
           Just packageId -> Just (Left packageId)
           Nothing -> Nothing
     )
     ruleBackendRegistry
+    ruleAlwaysProxy
 
 matchDockerRuleV2 :: (MonadMatch m) => DockerRuleV2 -> Request -> ResponseBuilder response -> m (Maybe response)
-matchDockerRuleV2 DockerRuleV2 {ruleImagePattern, ruleRuleId, ruleBackendRegistry} =
+matchDockerRuleV2 DockerRuleV2 {ruleImagePattern, ruleRuleId, ruleBackendRegistry, ruleAlwaysProxy} =
   matchDocker
     ( \image ->
         if ImagePattern.match ruleImagePattern image
           then Just (Right ruleRuleId)
           else Nothing
     )
     ruleBackendRegistry
+    ruleAlwaysProxy
 
 type MonadMatch m = m ~ IO
 
@@ -95,14 +101,16 @@ matchDocker ::
   DockerImageMatcher ->
   -- | Backend registry
   ByteString ->
+  -- | Flag indicating whether to always proxy and never redirect.
+  Bool ->
   Request ->
   ResponseBuilder response ->
   m (Maybe response)
-matchDocker matchImage backendRegistry Request {requestWai = request} responseBuilder@ResponseBuilder {..}
+matchDocker matchImage backendRegistry alwaysProxy Request {requestWai = request} responseBuilder@ResponseBuilder {..}
   | "/v2/_catalog" <- Wai.rawPathInfo request =
       pure $ Just (notFound emptyCapture)
   | "/v2/" <- Wai.rawPathInfo request =
-      pure $ Just $ redirectOrProxy request backendRegistry emptyCapture responseBuilder
+      pure $ Just $ redirectOrProxy request backendRegistry alwaysProxy emptyCapture responseBuilder
   | ("v2" : image, _manifestsOrBlobsOrTags : reference : _) <-
       break
         (\x -> x == "manifests" || x == "blobs" || x == "tags")
@@ -116,7 +124,7 @@ matchDocker matchImage backendRegistry Request {requestWai = request} responseBu
                 dockerCapturePackage = either Just (const Nothing) result,
                 dockerCaptureAutoCreate = either (const Nothing) Just result
               }
-       in pure $ Just $ redirectOrProxy request backendRegistry capture responseBuilder
+       in pure $ Just $ redirectOrProxy request backendRegistry alwaysProxy capture responseBuilder
   | otherwise =
       pure Nothing
   where
@@ -134,10 +142,15 @@ matchDocker matchImage backendRegistry Request {requestWai = request} responseBu
 redirectOrProxy ::
   Wai.Request ->
   ByteString ->
+  -- | Flag indicating whether to always proxy and never redirect.
+  Bool ->
   RuleCapture ->
   ResponseBuilder response ->
   response
-redirectOrProxy request domain !capture ResponseBuilder {..}
+redirectOrProxy request domain alwaysProxy !capture ResponseBuilder {..}
+  | alwaysProxy =
+      -- Proxy request unconditionally
+      proxyTo (const capture) domain
   | shouldRedirectDockerRequest request =
       -- As with the Host header we have to respect the proxy protocol
       -- when redirecting.

test/Scarf/Gateway/Test.hs

@@ -230,7 +230,7 @@ test_gateway_docker_rule_v1 =
     "Gateway, Docker Pull"
     ( newGatewayConfig
         [ ( "test.scarf.sh",
-            [ newDockerRuleV1 "package-1" ["library", "hello-world"] "docker.io"
+            [ newDockerRuleV1 "package-1" ["library", "hello-world"] "docker.io" False
             ]
           )
         ]
@@ -292,7 +292,7 @@ test_gateway_docker_rule_v1_proxied =
     "Gateway, Docker Pull with proxied requests"
     ( newGatewayConfig
         [ ( "test.scarf.sh",
-            [ newDockerRuleV1 "package-1" ["library", "hello-world"] "docker.io"
+            [ newDockerRuleV1 "package-1" ["library", "hello-world"] "docker.io" False
             ]
           )
         ]
@@ -354,7 +354,7 @@ test_gateway_docker_rule_v2 =
     "Gateway, Docker Pull (with auto package creation)"
     ( newGatewayConfig
         [ ( "test.scarf.sh",
-            [ newDockerRuleV2 "test-rule-1" "library/*" "docker.io"
+            [ newDockerRuleV2 "test-rule-1" "library/*" "docker.io" False
             ]
           )
         ]

test/golden/docker-always-proxy-2.output.yaml

@@ -0,0 +1,14 @@
+capture: |-
+  Just
+    DockerCapture
+      { dockerCaptureImage = [ "library" , "proxy" ]
+      , dockerCaptureReference = "latest"
+      , dockerCaptureBackendRegistry = "ghcr.io"
+      , dockerCapturePackage =
+          Just "8717953c-3452-4ef7-9a14-b64dc19163b4"
+      , dockerCaptureAutoCreate = Nothing
+      }
+headers:
+  Location: https://ghcr.io/v2/library/proxy/manifests/latest
+  X-This-Request-Was-Proxied: '1'
+status: 307

test/golden/docker-always-proxy-2.yaml

@@ -0,0 +1,13 @@
+# A manifest download with a user-agent that is usually redirected
+path: /v2/library/proxy/manifests/latest
+headers: 
+  User-Agent: Docker-Client
+  Host: cr.test.io
+manifest:
+  rules:
+    - type:  docker-v1
+      repository-name: library/proxy
+      domain: cr.test.io
+      registry: ghcr.io
+      package-id: "8717953c-3452-4ef7-9a14-b64dc19163b4"
+      always-proxy: true

test/golden/docker-always-proxy.output.yaml

@@ -0,0 +1,14 @@
+capture: |-
+  Just
+    DockerCapture
+      { dockerCaptureImage = [ "library" , "proxy" ]
+      , dockerCaptureReference = "latest"
+      , dockerCaptureBackendRegistry = "ghcr.io"
+      , dockerCapturePackage =
+          Just "8717953c-3452-4ef7-9a14-b64dc19163b4"
+      , dockerCaptureAutoCreate = Nothing
+      }
+headers:
+  Location: https://ghcr.io/v2/library/proxy/manifests/latest
+  X-This-Request-Was-Proxied: '1'
+status: 307

test/golden/docker-always-proxy.yaml

@@ -0,0 +1,11 @@
+path: /v2/library/proxy/manifests/latest
+headers: 
+  Host: cr.test.io
+manifest:
+  rules:
+    - type:  docker-v1
+      repository-name: library/proxy
+      domain: cr.test.io
+      registry: ghcr.io
+      package-id: "8717953c-3452-4ef7-9a14-b64dc19163b4"
+      always-proxy: true