partition dataspace analytics results (finos#3252)

gs-jp1 · Nov 18, 2024 · e9c5379 · e9c5379
1 parent c0ac835
commit e9c5379
Show file tree

Hide file tree

Showing 7 changed files with 352 additions and 35 deletions.
diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/pom.xml
@@ -131,6 +131,10 @@
         <!-- ENGINE -->
 
         <!-- JACKSON -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>

diff --git a/...ava/org/finos/legend/engine/generation/DataSpaceAnalyticsArtifactGenerationExtension.java b/...ava/org/finos/legend/engine/generation/DataSpaceAnalyticsArtifactGenerationExtension.java
@@ -17,15 +17,22 @@
 import static org.finos.legend.engine.language.pure.compiler.toPureGraph.HelperModelBuilder.getElementFullPath;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.Collections;
+
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
+import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.list.MutableList;
 import org.finos.legend.engine.generation.analytics.DataSpaceAnalyticsHelper;
 import org.finos.legend.engine.generation.analytics.model.DataSpaceAnalysisResult;
+import org.finos.legend.engine.generation.analytics.partition.MappingAnalysisCoveragePartition;
+import org.finos.legend.engine.generation.analytics.partition.MappingModelCoveragePartition;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
 import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact;
 import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension;
+import org.finos.legend.engine.protocol.analytics.model.MappingModelCoverageAnalysisResult;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpace;
 import org.finos.legend.engine.shared.core.operational.Assert;
@@ -36,6 +43,11 @@
 public class DataSpaceAnalyticsArtifactGenerationExtension implements ArtifactGenerationExtension
 {
     public final String ROOT_PATH = "dataSpace-analytics";
+    private boolean returnLightGraph = false;
+    private final String MAIN_ANALYTICS_FILE = "AnalyticsResult.json";
+    private final String MAPPING_MODEL_PREFIX = "MappingModel_";
+    private final String MAPPING_ANALYSIS_FILE_PREFIX = "MappingAnalysis_";
+    private final String ANALYTICS_FORMAT = "json";
 
     public static ObjectMapper objectMapper = DataSpaceAnalyticsHelper.getNewObjectMapper();
 
@@ -51,6 +63,11 @@ public String getKey()
         return ROOT_PATH;
     }
 
+    public void setReturnLightGraph(boolean returnLightGraph)
+    {
+        this.returnLightGraph = returnLightGraph;
+    }
+
     @Override
     public boolean canGenerate(PackageableElement element)
     {
@@ -60,22 +77,78 @@ public boolean canGenerate(PackageableElement element)
     @Override
     public List<Artifact> generate(PackageableElement element, PureModel pureModel, PureModelContextData data, String clientVersion)
     {
-        String fileName = "AnalyticsResult.json";
+
         String dataSpacePath = getElementFullPath(element, pureModel.getExecutionSupport());
         Assert.assertTrue(this.canGenerate(element), () -> "DataSpace analytics only supports dataSpace elements");
         Root_meta_pure_metamodel_dataSpace_DataSpace dataSpace = (Root_meta_pure_metamodel_dataSpace_DataSpace) element;
         org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement dataSpaceProtocol = data.getElements().stream().filter(el -> dataSpacePath.equals(el.getPath())).findFirst().orElse(null);
         Assert.assertTrue(dataSpaceProtocol instanceof DataSpace, () -> "Can't find data space '" + dataSpacePath + "'");
-        DataSpaceAnalysisResult result = DataSpaceAnalyticsHelper.analyzeDataSpace(dataSpace, pureModel, (DataSpace) dataSpaceProtocol, data, clientVersion);
+        DataSpaceAnalysisResult result = DataSpaceAnalyticsHelper.analyzeDataSpace(dataSpace, pureModel, (DataSpace) dataSpaceProtocol, data, clientVersion, this.returnLightGraph);
+
+        // Partition
+        List<MappingModelCoveragePartition> mappingModelPartitions = Lists.mutable.empty();
+        List<MappingAnalysisCoveragePartition> mappingAnalysisPartitions = Lists.mutable.empty();
+        partitionDataSpaceAnalyticsResult(result,mappingModelPartitions, mappingAnalysisPartitions);
         try
         {
+            List<Artifact> artifacts = Lists.mutable.empty();
             String stringResult = objectMapper.writeValueAsString(result);
-            Artifact output = new Artifact(stringResult, fileName, "json");
-            return Collections.singletonList(output);
+            artifacts.add(new Artifact(stringResult, MAIN_ANALYTICS_FILE, ANALYTICS_FORMAT));
+            if (mappingModelPartitions != null && !mappingModelPartitions.isEmpty())
+            {
+                for (int i = 0; i < mappingModelPartitions.size(); i++)
+                {
+                    MappingModelCoveragePartition modelPartition = mappingModelPartitions.get(i);
+                    String fileName = MAPPING_MODEL_PREFIX + i + "." + ANALYTICS_FORMAT;
+                    artifacts.add(new Artifact(objectMapper.writeValueAsString(modelPartition), fileName, "json"));
+                }
+            }
+            if (mappingAnalysisPartitions != null && !mappingAnalysisPartitions.isEmpty())
+            {
+                for (int i = 0; i < mappingAnalysisPartitions.size(); i++)
+                {
+                    MappingAnalysisCoveragePartition analysisPartition = mappingAnalysisPartitions.get(i);
+                    String fileName = MAPPING_ANALYSIS_FILE_PREFIX + i + "." + ANALYTICS_FORMAT;
+                    artifacts.add(new Artifact(objectMapper.writeValueAsString(analysisPartition), fileName, ANALYTICS_FORMAT));
+                }
+            }
+            return artifacts;
         }
         catch (Exception exception)
         {
             throw new EngineException("Can't serialize data space analysis result", exception);
         }
     }
+
+
+    private void partitionDataSpaceAnalyticsResult(DataSpaceAnalysisResult result, List<MappingModelCoveragePartition> mappingModelPartitions, List<MappingAnalysisCoveragePartition> mappingAnalysisPartitions)
+    {
+        Map<String, MappingModelCoverageAnalysisResult> mappingToMappingCoverageResult = result.mappingToMappingCoverageResult;
+        if (mappingToMappingCoverageResult != null && !mappingToMappingCoverageResult.isEmpty())
+        {
+            Set<String> mappingsToRemove = new HashSet<>();
+            mappingToMappingCoverageResult.forEach((mapping, mappingCoverageResult) ->
+            {
+                if (mappingCoverageResult.model != null)
+                {
+                    MappingModelCoveragePartition modelPartition = new MappingModelCoveragePartition();
+                    modelPartition.mapping = mapping;
+                    modelPartition.model = mappingCoverageResult.model;
+                    mappingModelPartitions.add(modelPartition);
+                    // remove model from coverage result
+                    mappingCoverageResult.model = null;
+                }
+                MappingAnalysisCoveragePartition mappingCoverageModelPartition = new MappingAnalysisCoveragePartition();
+                mappingCoverageModelPartition.mapping = mapping;
+                mappingCoverageModelPartition.analysisResult = mappingCoverageResult;
+                mappingAnalysisPartitions.add(mappingCoverageModelPartition);
+                mappingsToRemove.add(mapping);
+            });
+            mappingsToRemove.forEach(mappingToMappingCoverageResult::remove);
+        }
+    }
+
+
+
+
 }
diff --git a/.../src/main/java/org/finos/legend/engine/generation/analytics/DataSpaceAnalyticsHelper.java b/.../src/main/java/org/finos/legend/engine/generation/analytics/DataSpaceAnalyticsHelper.java
@@ -301,12 +301,6 @@ private static MappingModelCoverageAnalysisResult buildMappingModelCoverageAnaly
         return null;
     }
 
-    public static DataSpaceAnalysisResult analyzeDataSpace(Root_meta_pure_metamodel_dataSpace_DataSpace dataSpace, PureModel pureModel, DataSpace dataSpaceProtocol, PureModelContextData pureModelContextData, String clientVersion)
-    {
-        boolean forDisableLightGraph = false;
-        boolean isDataspaceInDev = dataSpace._stereotypes().anySatisfy(stereotype -> stereotype._profile()._name().equals("devStatus") && stereotype._profile()._p_stereotypes().anySatisfy(s -> s._value().equals("inProgress")));
-        return analyzeDataSpace(dataSpace, pureModel, dataSpaceProtocol, pureModelContextData, clientVersion, Lists.mutable.withAll(ServiceLoader.load(PlanGeneratorExtension.class)), EntitlementServiceExtensionLoader.extensions(), forDisableLightGraph && !isDataspaceInDev);
-    }
 
     public static DataSpaceAnalysisResult analyzeDataSpaceCoverage(Root_meta_pure_metamodel_dataSpace_DataSpace dataSpace, PureModel pureModel, DataSpace dataSpaceProtocol, PureModelContextData pureModelContextData, String clientVersion, MutableList<PlanGeneratorExtension> generatorExtensions, List<EntitlementServiceExtension> entitlementServiceExtensions, boolean returnLightGraph)
     {
@@ -404,6 +398,13 @@ else if (!result.mappingToMappingCoverageResult.containsKey(excResult.mapping))
         return result;
     }
 
+
+    public static DataSpaceAnalysisResult analyzeDataSpace(Root_meta_pure_metamodel_dataSpace_DataSpace dataSpace, PureModel pureModel, DataSpace dataSpaceProtocol, PureModelContextData pureModelContextData, String clientVersion, boolean returnLightGraph)
+    {
+        boolean isDataSpaceInDev = dataSpace._stereotypes().anySatisfy(stereotype -> stereotype._profile()._name().equals("devStatus") && stereotype._profile()._p_stereotypes().anySatisfy(s -> s._value().equals("inProgress")));
+        return analyzeDataSpace(dataSpace, pureModel, dataSpaceProtocol, pureModelContextData, clientVersion, Lists.mutable.withAll(ServiceLoader.load(PlanGeneratorExtension.class)), EntitlementServiceExtensionLoader.extensions(), returnLightGraph && !isDataSpaceInDev);
+    }
+
     public static DataSpaceAnalysisResult analyzeDataSpace(Root_meta_pure_metamodel_dataSpace_DataSpace dataSpace, PureModel pureModel, DataSpace dataSpaceProtocol, PureModelContextData pureModelContextData, String clientVersion, MutableList<PlanGeneratorExtension> generatorExtensions, List<EntitlementServiceExtension> entitlementServiceExtensions, boolean returnLightGraph)
     {
         Root_meta_pure_metamodel_dataSpace_analytics_DataSpaceAnalysisResult analysisResult = core_data_space_analytics_analytics.Root_meta_pure_metamodel_dataSpace_analytics_analyzeDataSpace_DataSpace_1__PackageableRuntime_MANY__Boolean_1__DataSpaceAnalysisResult_1_(
@@ -566,4 +567,5 @@ else if (elementDoc instanceof Root_meta_pure_metamodel_dataSpace_analytics_Data
 
         return result;
     }
+
 }
diff --git a/.../finos/legend/engine/generation/analytics/partition/MappingAnalysisCoveragePartition.java b/.../finos/legend/engine/generation/analytics/partition/MappingAnalysisCoveragePartition.java
@@ -0,0 +1,25 @@
+// Copyright 2022 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.engine.generation.analytics.partition;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.finos.legend.engine.protocol.analytics.model.MappingModelCoverageAnalysisResult;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type")
+public class MappingAnalysisCoveragePartition
+{
+    public String mapping;
+    public MappingModelCoverageAnalysisResult analysisResult;
+}
diff --git a/...org/finos/legend/engine/generation/analytics/partition/MappingModelCoveragePartition.java b/...org/finos/legend/engine/generation/analytics/partition/MappingModelCoveragePartition.java
@@ -0,0 +1,25 @@
+// Copyright 2022 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.engine.generation.analytics.partition;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type")
+public class MappingModelCoveragePartition
+{
+    public String mapping;
+    public PureModelContextData model;
+}