Merge branch 'main' into Aggregation-updates

common/internal_model/src/main/scala/weco/catalogue/internal_model/image/Image.scala

@@ -92,21 +92,15 @@ object ImageFsm {
 }
 
 case class InferredData(
-  // We split the feature vector so that it can fit into
-  // ES's dense vector type (max length 2048)
-  features1: List[Float],
-  features2: List[Float],
-  reducedFeatures: List[Float],
+  features: List[Float],
   paletteEmbedding: List[Float],
   averageColorHex: Option[String],
   aspectRatio: Option[Float]
 )
 
 object InferredData {
   def empty: InferredData = InferredData(
-    features1 = Nil,
-    features2 = Nil,
-    reducedFeatures = Nil,
+    features = Nil,
     paletteEmbedding = Nil,
     averageColorHex = None,
     aspectRatio = None

common/internal_model/src/test/scala/weco/catalogue/internal_model/generators/ImageGenerators.scala

@@ -171,13 +171,9 @@ trait ImageGenerators
 
   def createInferredData: InferredData = {
     val features = randomUnitLengthVector(4096)
-    val (features1, features2) = features.splitAt(features.size / 2)
-    val reducedFeatures = randomUnitLengthVector(1024)
     val paletteEmbedding = randomUnitLengthVector(1000)
     InferredData(
-      features1 = features1.toList,
-      features2 = features2.toList,
-      reducedFeatures = reducedFeatures.toList,
+      features = features.toList,
       paletteEmbedding = paletteEmbedding.toList,
       averageColorHex = Some(randomHexString),
       aspectRatio = inferredDataAspectRatio

index_config/mappings.images_indexed.2024-11-14.json

@@ -155,29 +155,17 @@
     },
     "vectorValues": {
       "properties": {
-        "features1": {
+        "features": {
           "type": "dense_vector",
-          "dims": 2048,
+          "dims": 4096,
           "index": true,
-          "similarity": "cosine"
-        },
-        "features2": {
-          "type": "dense_vector",
-          "dims": 2048,
-          "index": true,
-          "similarity": "cosine"
+          "similarity": "dot_product"
         },
         "paletteEmbedding": {
           "type": "dense_vector",
           "dims": 1000,
           "index": true,
           "similarity": "dot_product"
-        },
-        "reducedFeatures": {
-          "type": "dense_vector",
-          "dims": 1024,
-          "index": true,
-          "similarity": "dot_product"
         }
       }
     },

pipeline/inferrer/feature_inferrer/app/main.py

@@ -15,36 +15,13 @@
 logger.info("Starting API")
 app = FastAPI(
     title="Feature vector encoder",
-    description=(
-        "Takes an image url and returns the image's feature vector, "
-        "and a reduced 1024-dimensional form of the vector"
-    ),
+    description=("Takes an image url and returns the image's feature vector"),
 )
 logger.info("API started, awaiting requests")
 
 
-def feature_reducer(vectors: np.ndarray) -> np.ndarray:
-    """
-    return the first 1024 elements of a set of vectors, normalised
-    to unit length. Normalisation is done to ensure that the vectors
-    can be compared using dot_product similarity, rather than cosine.
-
-    N.B. This is a placeholder for a more sophisticated dimensionality
-    reduction technique
-    """
-    sliced = vectors[:, :1024]
-    normalised_vectors = sliced / np.linalg.norm(sliced, axis=1, keepdims=True)
-    return normalised_vectors
-
-
-def batch_infer_features(images):
-    vectors = extract_features(images)
-    reduced = feature_reducer(vectors)
-    return [{"vector": v, "reduced_vector": r} for v, r in zip(vectors, reduced)]
-
-
 batch_inferrer_queue = BatchExecutionQueue(
-    batch_infer_features, batch_size=16, timeout=0.250
+    extract_features, batch_size=16, timeout=0.250
 )
 
 
@@ -58,12 +35,11 @@ async def main(query_url: str):
         raise HTTPException(status_code=404, detail=error_string)
 
     features = await batch_inferrer_queue.execute(image)
+    normalised_features = features / np.linalg.norm(features, axis=0, keepdims=True)
+
     logger.info(f"extracted features from url: {query_url}")
 
-    return {
-        "features_b64": base64.b64encode(features["vector"]),
-        "reduced_features_b64": base64.b64encode(features["reduced_vector"]),
-    }
+    return {"features_b64": base64.b64encode(normalised_features)}
 
 
 @app.get("/healthcheck")

pipeline/inferrer/inference_manager/src/main/scala/weco/pipeline/inference_manager/adapters/FeatureVectorInferrerAdapter.scala

@@ -41,19 +41,13 @@ class FeatureVectorInferrerAdapter(val host: String, port: Int)
   ): InferredData =
     inferrerResponse match {
       case FeatureVectorInferrerResponse(
-            features_b64,
-            reduced_features_b64
+            features_b64
           ) =>
         val features = decodeBase64ToFloatList(features_b64)
-        val reducedFeatures = decodeBase64ToFloatList(
-          reduced_features_b64
-        )
+
         if (features.size == 4096) {
-          val (features1, features2) = features.splitAt(features.size / 2)
           inferredData.copy(
-            features1 = features1,
-            features2 = features2,
-            reducedFeatures = reducedFeatures
+            features = features
           )
         } else inferredData
     }

pipeline/inferrer/inference_manager/src/main/scala/weco/pipeline/inference_manager/models/FeatureVectorInferrerResponse.scala

@@ -4,6 +4,5 @@ import weco.pipeline.inference_manager.adapters.InferrerResponse
 
 // The type of the response from the inferrer, for Circe's decoding
 case class FeatureVectorInferrerResponse(
-  features_b64: String,
-  reduced_features_b64: String
+  features_b64: String
 ) extends InferrerResponse

pipeline/inferrer/inference_manager/src/test/scala/weco/pipeline/inference_manager/adapters/FeatureVectorInferrerAdapterTest.scala

@@ -43,16 +43,11 @@ class FeatureVectorInferrerAdapterTest
     it("augments InferredData with the data from the inferrer response") {
       val features = (0 until 4096).map(_ / 4096f).toList
       val featuresB64 = Encoding.toLittleEndianBase64(features)
-      val reducedFeatures = (0 until 1024).map(_ / 1024f).toList
-      val reducedFeaturesB64 = Encoding.toLittleEndianBase64(reducedFeatures)
       val response = FeatureVectorInferrerResponse(
-        features_b64 = featuresB64,
-        reduced_features_b64 = reducedFeaturesB64
+        features_b64 = featuresB64
       )
       val inferredData = adapter.augment(InferredData.empty, response)
-      inferredData.features1 should be(features.slice(0, 2048))
-      inferredData.features2 should be(features.slice(2048, 4096))
-      inferredData.reducedFeatures should be(reducedFeatures)
+      inferredData.features should be(features)
     }
   }
 }

pipeline/inferrer/inference_manager/src/test/scala/weco/pipeline/inference_manager/fixtures/Responses.scala

@@ -16,9 +16,6 @@ object Responses {
     s"""{
       "features_b64": "${Encoding.toLittleEndianBase64(
         randomFeatureVector(seed)
-      )}",
-      "reduced_features_b64": "${Encoding.toLittleEndianBase64(
-        randomFeatureVector(seed).slice(0, 1024)
       )}"
      }""".stripMargin
   )

pipeline/inferrer/inference_manager/src/test/scala/weco/pipeline/inference_manager/integration/ManagerInferrerIntegrationTest.scala

@@ -77,21 +77,16 @@ class ManagerInferrerIntegrationTest
               id should be(image.state.canonicalId)
               inside(inferredData) {
                 case InferredData(
-                      features1,
-                      features2,
-                      reducedFeatures,
+                      features,
                       paletteEmbedding,
                       Some(averageColorHex),
                       aspectRatio
                     ) =>
                   // Note for future explorers: if a vector is the wrong length,
                   // make sure that the inferrer is encoding a list of single precision
                   // floats (ie float32/double) as that's what we're expecting to decode!
-                  features1 should have length 2048
-                  features2 should have length 2048
-                  forAll(features1 ++ features2) { _.isNaN shouldBe false }
-                  reducedFeatures should have length 1024
-                  forAll(reducedFeatures) { _.isNaN shouldBe false }
+                  features should have length 4096
+                  forAll(features) { _.isNaN shouldBe false }
                   paletteEmbedding should have length 1000
                   averageColorHex should have length 7
                   aspectRatio should not be empty

pipeline/inferrer/inference_manager/src/test/scala/weco/pipeline/inference_manager/services/InferenceManagerWorkerServiceTest.scala

@@ -97,20 +97,14 @@ class InferenceManagerWorkerServiceTest
                   val seed = id.hashCode
                   inside(inferredData) {
                     case InferredData(
-                          features1,
-                          features2,
-                          reducedFeatures,
+                          features,
                           paletteEmbedding,
                           Some(averageColorHex),
                           aspectRatio
                         ) =>
                       val featureVector =
                         Responses.randomFeatureVector(seed)
-                      features1 should be(featureVector.slice(0, 2048))
-                      features2 should be(featureVector.slice(2048, 4096))
-                      reducedFeatures should be(
-                        featureVector.slice(0, 1024)
-                      )
+                      features should be(featureVector)
                       paletteEmbedding should be(
                         Responses.randomPaletteVector(seed)
                       )
@@ -156,16 +150,12 @@ class InferenceManagerWorkerServiceTest
                 case ImageState.Augmented(_, _, inferredData) =>
                   inside(inferredData) {
                     case InferredData(
-                          features1,
-                          features2,
-                          reducedFeatures,
+                          features,
                           paletteEmbedding,
                           averageColorHex,
                           aspectRatio
                         ) =>
-                      features1 should have length 2048
-                      features2 should have length 2048
-                      reducedFeatures should have length 1024
+                      features should have length 4096
                       paletteEmbedding should have length 1000
                       averageColorHex.get should have length 7
                       aspectRatio should not be empty