Switch test-suite to use pretty-show and yaml for the output capture …

…for easier diffs

scarf-gateway.cabal

@@ -207,6 +207,7 @@ test-suite scarf-gateway-test
     , internal
     , manifest
     , network-uri
+    , pretty-show
     , scarf-gateway
     , tasty
     , tasty-golden

src/Scarf/Gateway/Rule.hs

@@ -237,10 +237,10 @@ data RedirectOrProxy
   | -- | Respond with bytes directly.
     RespondBytes
       !RuleCapture
+      -- | Headers we might want sent back
       !ResponseHeaders
-      -- ^ Headers we might want sent back
+      -- | Response body
       !LBS.ByteString
-      -- ^ Response body
   | -- | Respond OK with no data
     RespondOk
       !RuleCapture

src/Scarf/Gateway/Rule/Capture.hs

@@ -14,7 +14,6 @@ import Data.Aeson (Value)
 import Data.ByteString (ByteString)
 import Data.HashMap.Strict (HashMap)
 import Data.Text (Text)
-import Data.Text qualified as Text
 import Data.Time (UTCTime)
 import Data.UUID (UUID)
 import Data.UUID.V4 qualified as UUID
@@ -78,7 +77,7 @@ newRequestId = RequestId <$> UUID.nextRandom
 -- scarf-server.
 --
 -- TODO Switch to e.g. thrift encoding using Schema
-newtype CapturedRequest = CapturedRequest Text
+newtype CapturedRequest = CapturedRequest (Maybe RuleCapture)
   deriving newtype (Show)
 
 -- | Captures all the relevant information for further processing.
@@ -102,4 +101,4 @@ captureRequest ::
   Maybe RuleCapture ->
   CapturedRequest
 captureRequest _time _requestId _request _responseStatus mcapture =
-  CapturedRequest $ Text.pack $ maybe "" show mcapture
+  CapturedRequest $ mcapture

src/Scarf/Lib/Tracing.hs

@@ -183,7 +183,8 @@ withRootTracer manager msamplingRate action = do
           not (null pushEndpoint) = do
             let options =
                   zipkinOptions manager endpoint
-                    & zoEndpoint .~ pushEndpoint
+                    & zoEndpoint
+                    .~ pushEndpoint
 
                 close zipkin =
                   closeZipkin zipkin `catch` \(_ :: BlockedIndefinitelyOnSTM) ->
@@ -207,7 +208,8 @@ withRootTracer manager msamplingRate action = do
           Just port <- readMaybe (unpack port') = do
             let options =
                   jaegerAgentOptions serviceName
-                    & jaoAddr .~ UDPAddr (unpack hostname) port
+                    & jaoAddr
+                    .~ UDPAddr (unpack hostname) port
             withJaegerAgent options $ \agent -> do
               let batchOpts =
                     batchOptions (traverse_ (jaegerAgentReporter agent))

test/Scarf/Gateway/Golden.hs

@@ -11,29 +11,24 @@ import Data.Aeson
   ( FromJSON (..),
     ToJSON (..),
     Value (Null),
-    decode,
-    encode,
     object,
     withObject,
     (.!=),
     (.:),
     (.:?),
     (.=),
   )
-import Data.Aeson.Encode.Pretty (confCompare, defConfig, encodePretty')
-import Data.Aeson.Encoding (dict, int, pair, pairs, text, unsafeToEncoding)
 import Data.Bifunctor (Bifunctor (bimap), first)
 import Data.ByteString qualified as ByteString
-import Data.ByteString.Builder (stringUtf8)
+import Data.ByteString.Lazy qualified
 import Data.CaseInsensitive qualified as CaseInsensitive
 import Data.HashMap.Strict qualified as HashMap
 import Data.IORef (newIORef, readIORef, writeIORef)
 import Data.List (sort)
 import Data.Map.Strict qualified as Map
-import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import Data.Text.Encoding (decodeUtf8, encodeUtf8)
-import Data.Yaml (decodeFileThrow)
+import Data.Yaml (decodeFileThrow, encode)
 import Network.Wai
   ( Request
       ( rawQueryString,
@@ -75,10 +70,11 @@ import Scarf.Gateway.Rule.Capture
   )
 import Scarf.Lib.Tracing (nullTracer)
 import Scarf.Manifest (Manifest, manifestToRules)
-import System.FilePath (replaceExtension)
+import System.FilePath (replaceExtension, takeExtensions)
 import System.IO.Unsafe (unsafeInterleaveIO)
 import Test.Tasty (TestTree)
 import Test.Tasty.Golden (findByExtension, goldenVsStringDiff)
+import Text.Show.Pretty qualified
 
 -- | Input values for Golden tests.
 data Input = Input
@@ -141,7 +137,15 @@ test_gateway_golden = goldenTests "test/golden"
 
 goldenTests :: FilePath -> IO [TestTree]
 goldenTests testsDirectory = do
-  inputFiles <- findByExtension [".yaml"] testsDirectory
+  inputFiles' <- findByExtension [".yaml"] testsDirectory
+
+  -- The glob above also includes the .output.yaml files. Ensure
+  -- we only take the input files.
+  let inputFiles =
+        filter
+          (\file -> takeExtensions file == ".yaml")
+          inputFiles'
+
   when (null inputFiles) $
     error "No golden tests!"
   flip foldMap inputFiles $ \inputFile -> do
@@ -198,32 +202,27 @@ goldenTests testsDirectory = do
             }
       (request, capture) <- takeMVar chan
 
-      -- We want to have clear and deterministic diffs. For that we have to run the produced
-      -- JSON through aeson-pretty.
-      let json :: Value
-          json =
-            fromMaybe (error "impossible") $
-              decode $
-                encode $
-                  Output
-                    { outputQueryString =
-                        decodeUtf8 (rawQueryString request),
-                      outputStatus =
-                        fromEnum simpleStatus,
-                      outputHeaders =
-                        fmap (bimap (decodeUtf8 . CaseInsensitive.original) decodeUtf8) simpleHeaders,
-                      outputCapture =
-                        capture
-                    }
-      pure (encodePretty' (defConfig {confCompare = compare}) json, simpleBody)
+      let output =
+            Output
+              { outputQueryString =
+                  decodeUtf8 (rawQueryString request),
+                outputStatus =
+                  fromEnum simpleStatus,
+                outputHeaders =
+                  fmap (bimap (decodeUtf8 . CaseInsensitive.original) decodeUtf8) simpleHeaders,
+                outputCapture =
+                  capture
+              }
+
+      pure (Data.ByteString.Lazy.fromStrict (Data.Yaml.encode output), simpleBody)
 
     let compareFiles = \ref new -> ["diff", "-u", ref, new]
 
     pure
       [ goldenVsStringDiff
           inputFile
           compareFiles
-          (replaceExtension inputFile "output.json")
+          (replaceExtension inputFile "output.yaml")
           (pure (fst testResult)),
         goldenVsStringDiff
           (inputFile <> " body")
@@ -256,21 +255,12 @@ instance FromJSON Input where
       <*> o .: "manifest"
 
 instance ToJSON Output where
-  toJSON =
-    error "Unimplemented due to CapturedRequest not having a ToJSON instance for performance reasons"
-
-  toEncoding Output {..} =
-    pairs $
-      pair "status" (int outputStatus)
-        <> pair
-          "headers"
-          (dict text text Map.foldrWithKey (Map.fromList outputHeaders))
-        <> pair
-          "capture"
-          ( unsafeToEncoding
-              ( stringUtf8 $ show outputCapture
-              )
-          )
-        <> if outputQueryString /= ""
-          then pair "query" (text outputQueryString)
-          else mempty
+  toJSON Output {..} =
+    object $
+      [ "status" .= outputStatus,
+        "headers" .= Map.fromList outputHeaders,
+        "capture" .= Text.Show.Pretty.ppShow outputCapture
+      ]
+        <> [ "query" .= outputQueryString
+             | outputQueryString /= ""
+           ]

test/golden/auto-creation-rules-1.output.yaml

@@ -0,0 +1,13 @@
+capture: |-
+  Just
+    DockerCapture
+      { dockerCaptureImage = [ "library" , "proxy" ]
+      , dockerCaptureReference = "latest"
+      , dockerCaptureBackendRegistry = "ghcr.io"
+      , dockerCapturePackage = Nothing
+      , dockerCaptureAutoCreate = Just "rule-123"
+      }
+headers:
+  Location: https://ghcr.io/v2/library/proxy/manifests/latest
+  X-This-Request-Was-Proxied: '1'
+status: 307

test/golden/auto-creation-rules-2.output.yaml

@@ -0,0 +1,14 @@
+capture: |-
+  Just
+    DockerCapture
+      { dockerCaptureImage = [ "library" , "hello-world" ]
+      , dockerCaptureReference = "latest"
+      , dockerCaptureBackendRegistry = "ghcr.io"
+      , dockerCapturePackage =
+          Just "8717953c-3452-4ef7-9a14-b64dc19163b4"
+      , dockerCaptureAutoCreate = Nothing
+      }
+headers:
+  Location: https://ghcr.io/v2/library/hello-world/manifests/latest
+  X-This-Request-Was-Proxied: '1'
+status: 307

test/golden/auto-creation-rules-2.yaml

@@ -0,0 +1,15 @@
+path: /v2/library/hello-world/manifests/latest
+headers: 
+  Host: cr.test.io
+manifest:
+  rules:
+    - type:  docker-v1
+      repository-name: library/hello-world
+      domain: cr.test.io
+      registry: ghcr.io
+      package-id: "8717953c-3452-4ef7-9a14-b64dc19163b4"
+    - type:  docker-v2
+      domain: cr.test.io
+      registry: ghcr.io
+      rule-id: rule-123
+      pattern: "library/hello-world"

test/golden/docker-test-1.output.yaml

@@ -0,0 +1,14 @@
+capture: |-
+  Just
+    DockerCapture
+      { dockerCaptureImage = [ "library" , "proxy" ]
+      , dockerCaptureReference = "latest"
+      , dockerCaptureBackendRegistry = "ghcr.io"
+      , dockerCapturePackage =
+          Just "8717953c-3452-4ef7-9a14-b64dc19163b4"
+      , dockerCaptureAutoCreate = Nothing
+      }
+headers:
+  Location: https://ghcr.io/v2/library/proxy/manifests/latest
+  X-This-Request-Was-Proxied: '1'
+status: 307

test/golden/file-package-with-multiple-variables-and-no-outgoing-url.output.yaml

@@ -0,0 +1,11 @@
+capture: |-
+  Just
+    FlatfileCapture
+      { fileAbsoluteUrl = Nothing
+      , fileVariables =
+          fromList
+            [ ( "somevar" , "very-nice" ) , ( "somevar2" , "really-nice" ) ]
+      , filePackage = "21c24cd1-73fa-4970-8a6a-bc570e55b91e"
+      }
+headers: {}
+status: 200

test/golden/file-package-with-no-outgoing-url-and-no-match.output.yaml

@@ -0,0 +1,3 @@
+capture: Nothing
+headers: {}
+status: 404

test/golden/file-package-with-no-outgoing-url-with-variables-and-no-match.output.yaml

@@ -0,0 +1,3 @@
+capture: Nothing
+headers: {}
+status: 404

test/golden/file-package-with-no-outgoing-url.output.yaml

@@ -0,0 +1,9 @@
+capture: |-
+  Just
+    FlatfileCapture
+      { fileAbsoluteUrl = Nothing
+      , fileVariables = fromList []
+      , filePackage = "a3941b62-12ff-4c39-a263-4342495f4b8b"
+      }
+headers: {}
+status: 200

test/golden/file-package-with-variables-1.output.yaml

@@ -0,0 +1,15 @@
+capture: |-
+  Just
+    FlatfileCapture
+      { fileAbsoluteUrl =
+          Just
+            "https://datausa.io/api/data?drilldowns=Nation&measures=Population"
+      , fileVariables =
+          fromList
+            [ ( "drilldowns" , "Nation" ) , ( "measures" , "Population" ) ]
+      , filePackage = "a1b331fa-1539-49e5-bdc4-dc8a48e586d1"
+      }
+headers:
+  Location: https://datausa.io/api/data?drilldowns=Nation&measures=Population
+query: ?drilldowns=Nation&measures=Population
+status: 302

test/golden/file-package-with-variables-2.output.yaml

@@ -0,0 +1,15 @@
+capture: |-
+  Just
+    FlatfileCapture
+      { fileAbsoluteUrl =
+          Just
+            "https://datausa.io/api/data?drilldowns=Nation&measures=Population"
+      , fileVariables =
+          fromList
+            [ ( "drilldowns" , "Nation" ) , ( "measures" , "Population" ) ]
+      , filePackage = "a1b331fa-1539-49e5-bdc4-dc8a48e586d1"
+      }
+headers:
+  Location: https://datausa.io/api/data?drilldowns=Nation&measures=Population
+query: ?measures=Population
+status: 302