v2.0.0

.github/workflows/dev_build.yml

@@ -38,12 +38,11 @@ jobs:
         path: gcm-taskit
         ref: dev
 
-    - name: Get GCM and Taskit Version
+    - name: Get GCM Version
       if: ${{ endsWith(env.gcm_taskit_version, 'SNAPSHOT') }}
       run: |
         echo "gcm_version=v$(mvn help:evaluate -Dexpression=gcm.version -q -DforceStdout --file gcm-taskit/pom.xml)" >> "$GITHUB_ENV"
-        echo "taskit_version=v$(mvn help:evaluate -Dexpression=taskit.version -q -DforceStdout --file gcm-taskit/pom.xml)" >> "$GITHUB_ENV"
-
+
     - name: Checkout GCM
       if: ${{ endsWith(env.gcm_version, 'SNAPSHOT') }}
       uses: actions/checkout@v4
@@ -52,18 +51,23 @@ jobs:
         path: gcm
         ref: dev
 
+    - name: Get Taskit Version
+      if: ${{ endsWith(env.gcm_version, 'SNAPSHOT') }}
+      run: |
+        echo "taskit_version=v$(mvn help:evaluate -Dexpression=taskit.version -q -DforceStdout --file gcm/pom.xml)" >> "$GITHUB_ENV"
+
     - name: Checkout Taskit
       if: ${{ endsWith(env.taskit_version, 'SNAPSHOT') }}
       uses: actions/checkout@v4
       with:
         repository: HHS/ASPR-ms-taskit
-        path: taskit-core
+        path: taskit
         ref: dev
 
     - name: Get Util Version
-      if: ${{ endsWith(env.gcm_version, 'SNAPSHOT') }}
+      if: ${{ endsWith(env.taskit_version, 'SNAPSHOT') }}
       run: |
-        echo "util_version=v$(mvn help:evaluate -Dexpression=util.version -q -DforceStdout --file gcm/pom.xml)" >> "$GITHUB_ENV"
+        echo "util_version=v$(mvn help:evaluate -Dexpression=util.version -q -DforceStdout --file taskit/pom.xml)" >> "$GITHUB_ENV"
 
     - name: Checkout Util
       if: ${{ endsWith(env.util_version, 'SNAPSHOT') }}
@@ -77,14 +81,14 @@ jobs:
       if: ${{ endsWith(env.util_version, 'SNAPSHOT') }}
       run: mvn clean install -DskipTests --file util/pom.xml
 
+    - name: Build Taskit
+      if: ${{ endsWith(env.taskit_version, 'SNAPSHOT') }}
+      run: mvn clean install -DskipTests --file taskit/pom.xml
+
     - name: Build GCM
       if: ${{ endsWith(env.gcm_version, 'SNAPSHOT') }}
       run: mvn clean install -DskipTests --file gcm/pom.xml
 
-    - name: Build Taskit
-      if: ${{ endsWith(env.taskit_version, 'SNAPSHOT') }}
-      run: mvn clean install -DskipTests --file taskit-core/pom.xml
-
     - name: Build GCM Taskit
       if: ${{ endsWith(env.gcm_taskit_version, 'SNAPSHOT') }}
       run: mvn clean install -DskipTests --file gcm-taskit/pom.xml

.github/workflows/dev_pr_build.yml

@@ -38,12 +38,11 @@ jobs:
         path: gcm-taskit
         ref: dev
 
-    - name: Get GCM and Taskit Version
+    - name: Get GCM Version
       if: ${{ endsWith(env.gcm_taskit_version, 'SNAPSHOT') }}
       run: |
         echo "gcm_version=v$(mvn help:evaluate -Dexpression=gcm.version -q -DforceStdout --file gcm-taskit/pom.xml)" >> "$GITHUB_ENV"
-        echo "taskit_version=v$(mvn help:evaluate -Dexpression=taskit.version -q -DforceStdout --file gcm-taskit/pom.xml)" >> "$GITHUB_ENV"
-
+
     - name: Checkout GCM
       if: ${{ endsWith(env.gcm_version, 'SNAPSHOT') }}
       uses: actions/checkout@v4
@@ -52,18 +51,23 @@ jobs:
         path: gcm
         ref: dev
 
+    - name: Get Taskit Version
+      if: ${{ endsWith(env.gcm_version, 'SNAPSHOT') }}
+      run: |
+        echo "taskit_version=v$(mvn help:evaluate -Dexpression=taskit.version -q -DforceStdout --file gcm/pom.xml)" >> "$GITHUB_ENV"
+
     - name: Checkout Taskit
       if: ${{ endsWith(env.taskit_version, 'SNAPSHOT') }}
       uses: actions/checkout@v4
       with:
         repository: HHS/ASPR-ms-taskit
-        path: taskit-core
+        path: taskit
         ref: dev
 
     - name: Get Util Version
-      if: ${{ endsWith(env.gcm_version, 'SNAPSHOT') }}
+      if: ${{ endsWith(env.taskit_version, 'SNAPSHOT') }}
       run: |
-        echo "util_version=v$(mvn help:evaluate -Dexpression=util.version -q -DforceStdout --file gcm/pom.xml)" >> "$GITHUB_ENV"
+        echo "util_version=v$(mvn help:evaluate -Dexpression=util.version -q -DforceStdout --file taskit/pom.xml)" >> "$GITHUB_ENV"
 
     - name: Checkout Util
       if: ${{ endsWith(env.util_version, 'SNAPSHOT') }}
@@ -77,14 +81,14 @@ jobs:
       if: ${{ endsWith(env.util_version, 'SNAPSHOT') }}
       run: mvn clean install -DskipTests --file util/pom.xml
 
+    - name: Build Taskit
+      if: ${{ endsWith(env.taskit_version, 'SNAPSHOT') }}
+      run: mvn clean install -DskipTests --file taskit/pom.xml
+
     - name: Build GCM
       if: ${{ endsWith(env.gcm_version, 'SNAPSHOT') }}
       run: mvn clean install -DskipTests --file gcm/pom.xml
 
-    - name: Build Taskit
-      if: ${{ endsWith(env.taskit_version, 'SNAPSHOT') }}
-      run: mvn clean install -DskipTests --file taskit-core/pom.xml
-
     - name: Build GCM Taskit
       if: ${{ endsWith(env.gcm_taskit_version, 'SNAPSHOT') }}
       run: mvn clean install -DskipTests --file gcm-taskit/pom.xml

README.md

@@ -5,13 +5,13 @@
 [![GitHub Workflow Status (with event)][build-shield]][build-url]
 
 # GCM Pipeline
-A Pipeline to read heterogenous raw data files and convert them into GCM Plugin Datas for use with [GCM](https://github.com/HHS/ASPR-8).
+A Pipeline to read heterogenous raw data files and translate them into GCM PluginDatas for use with [GCM](https://github.com/HHS/ASPR-8).
 As of v1.0.0, this project is in Maven Central
 
 ## License
 Distributed under the GPLv3 License. See [LICENSE](LICENSE) for more information.
 
-Please read the [HHS vulnerability discloure](https://www.hhs.gov/vulnerability-disclosure-policy/index.html).
+Please read the [HHS vulnerability disclosure](https://www.hhs.gov/vulnerability-disclosure-policy/index.html).
 
 ## Usage 
 To use this project in your project, simply add the following dependency to your `dependencies` section of your pom.xml file.

pom.xml

@@ -9,7 +9,7 @@
     <packaging>jar</packaging>
 
     <name>General Computation Model: Pipeline</name>
-    <description>A pipeline to convert heterogenous raw data into data files that can be utilized by the GCM Simulation via GCM Taskit</description>
+    <description>A pipeline to translate heterogenous raw data into data files that can be utilized by the GCM Simulation via GCM Taskit</description>
     <url>https://github.com/HHS/ASPR-ms-gcm-pipeline</url>
 
     <!-- License Information -->
@@ -49,7 +49,7 @@
 
     <!-- Properties -->
     <properties>
-        <revision>1.1.2</revision>
+        <revision>2.0.0</revision>
 
         <!-- basic project properties -->
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -64,26 +64,25 @@
 
         <!-- plugin versions -->
         <flatten-maven-plugin.version>1.6.0</flatten-maven-plugin.version>
-        <maven-surefire-plugin.version>3.3.0</maven-surefire-plugin.version>
-        <maven-failsafe-plugin.version>3.3.0</maven-failsafe-plugin.version>
+        <maven-surefire-plugin.version>3.4.0</maven-surefire-plugin.version>
+        <maven-failsafe-plugin.version>3.4.0</maven-failsafe-plugin.version>
         <maven-source-plugin.version>3.3.1</maven-source-plugin.version>
-        <maven-javadoc-plugin.version>3.7.0</maven-javadoc-plugin.version>
-        <maven-gpg-plugin.version>3.2.4</maven-gpg-plugin.version>
-        <central-publishing-maven-plugin.version>0.4.0</central-publishing-maven-plugin.version>
+        <maven-javadoc-plugin.version>3.8.0</maven-javadoc-plugin.version>
+        <maven-gpg-plugin.version>3.2.5</maven-gpg-plugin.version>
+        <central-publishing-maven-plugin.version>0.5.0</central-publishing-maven-plugin.version>
 
         <!-- dependency versions -->
-        <gcm-taskit.version>2.8.5</gcm-taskit.version>
+        <gcm-taskit.version>3.0.0</gcm-taskit.version>
         <protoc.version>3.25.2</protoc.version>
         <jacoco-maven-plugin.version>0.8.12</jacoco-maven-plugin.version>
-        <junit-jupiter-engine.version>5.10.2</junit-jupiter-engine.version>
+        <junit-jupiter-engine.version>5.11.0</junit-jupiter-engine.version>
     </properties>
 
     <dependencies>
         <dependency>
             <groupId>gov.hhs.aspr.ms.gcm.taskit</groupId>
             <artifactId>protobuf</artifactId>
             <version>${gcm-taskit.version}</version>
-            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.jacoco</groupId>
@@ -106,6 +105,11 @@
     </dependencies>
 
     <build>
+        <resources>
+            <resource>
+                <directory>src/main/proto</directory>
+            </resource>
+        </resources>
         <plugins>
             <plugin>
                 <groupId>kr.motd.maven</groupId>

src/main/java/gov/hhs/aspr/ms/gcm/pipeline/ATypedPipeline.java

@@ -0,0 +1,16 @@
+package gov.hhs.aspr.ms.gcm.pipeline;
+
+import java.nio.file.Path;
+
+import gov.hhs.aspr.ms.taskit.core.engine.TaskitEngineManager;
+
+public abstract class ATypedPipeline<T> implements IPipeline {
+
+    protected final TaskitEngineManager taskitEngineManager;
+
+    public ATypedPipeline(TaskitEngineManager taskitEngineManager) {
+        this.taskitEngineManager = taskitEngineManager;
+    }
+
+    public abstract ATypedPipeline<T> using(T pipelineInput, Path inputDirectory, Path outputDirectory);
+}

src/main/java/gov/hhs/aspr/ms/gcm/pipeline/pipelines/ExperimentParameterDataPipeline.java

@@ -0,0 +1,120 @@
+package gov.hhs.aspr.ms.gcm.pipeline.pipelines;
+
+import java.nio.file.Path;
+
+import gov.hhs.aspr.ms.gcm.pipeline.ATypedPipeline;
+import gov.hhs.aspr.ms.gcm.pipeline.input.ExperimentParameterPipelineInput;
+import gov.hhs.aspr.ms.gcm.pipeline.support.GCMPipelineFileHeaders;
+import gov.hhs.aspr.ms.gcm.simulation.nucleus.ExperimentParameterData;
+import gov.hhs.aspr.ms.gcm.taskit.protobuf.nucleus.input.ExperimentParameterDataInput;
+import gov.hhs.aspr.ms.taskit.core.engine.TaskitEngineManager;
+import gov.hhs.aspr.ms.taskit.protobuf.engine.ProtobufTaskitEngineId;
+import gov.hhs.aspr.ms.util.errors.ContractException;
+import gov.hhs.aspr.ms.util.readers.TextTableReader;
+import gov.hhs.aspr.ms.util.resourcehelper.ResourceError;
+import gov.hhs.aspr.ms.util.resourcehelper.ResourceHelper;
+
+/**
+ * A utility for constructing the Experiment Parameter Data file from the
+ * experimentParametersFile, and explicitScenarioIdsFile
+ */
+public class ExperimentParameterDataPipeline extends ATypedPipeline<ExperimentParameterPipelineInput> {
+
+    /**
+     * Given a {@link ExperimentParameterPipelineInput}, an input directory path and
+     * an output directory path, return a {@link ExperimentParameterDataPipeline}
+     * 
+     * @throws ContractException
+     *                           <ul>
+     *                           <li>{@link ResourceError#UNKNOWN_FILE} if any of
+     *                           the input paths are not valid</li>
+     *                           <li>{@link ResourceError#FILE_PATH_IS_DIRECTORY} if
+     *                           any of the input paths or output path point to a
+     *                           directory</li>
+     *                           </ul>
+     */
+    public ExperimentParameterDataPipeline using(ExperimentParameterPipelineInput pipelineInput,
+            Path inputDirectory, Path outputDirectory) {
+
+        this.experimentParametersFilePath = ResourceHelper
+                .validateFile(inputDirectory.resolve(pipelineInput.getExperimentParametersFile()));
+        this.explicitScenarioIdsFilePath = ResourceHelper
+                .validateFile(inputDirectory.resolve(pipelineInput.getExplicitScenarioIdsFile()));
+
+        this.experimentParameterDataFilePath = ResourceHelper
+                .validateFilePath(outputDirectory.resolve(pipelineInput.getExperimentParameterDataFile()));
+
+        return this;
+    }
+
+    private Path experimentParametersFilePath;
+    private Path explicitScenarioIdsFilePath;
+
+    private Path experimentParameterDataFilePath;
+
+    public ExperimentParameterDataPipeline(TaskitEngineManager taskitEngineManager) {
+        super(taskitEngineManager);
+    }
+
+    /**
+     * Executes the pipeline with the given input files
+     * <p>
+     * writes out the resulting experimentParameter file to the path provided
+     */
+    public void execute() {
+        ExperimentParameterData.Builder builder = ExperimentParameterData.builder();
+
+        System.out.println("ExperimentParameterData Pipeline::loading experiment parameters");
+        loadExperimentParametersFile(builder);
+
+        System.out.println("ExperimentParameterData Pipeline::loading explicit scenario ids");
+        loadExplicitScenarioIdsFile(builder);
+
+        System.out.println("ExperimentParameterData Pipeline::translating experiment parameter data");
+        ExperimentParameterDataInput translatedObject = this.taskitEngineManager.translateObject(builder.build(),
+                ProtobufTaskitEngineId.JSON_ENGINE_ID);
+        // null out the data
+        builder = null;
+
+        System.out.println("ExperimentParameterData Pipeline::writing experiment parameter data");
+        this.taskitEngineManager.write(this.experimentParameterDataFilePath, translatedObject,
+                ProtobufTaskitEngineId.JSON_ENGINE_ID);
+        // null out the data
+        translatedObject = null;
+
+        System.out.println("ExperimentParameterData Pipeline::done");
+    }
+
+    private void loadExperimentParametersFile(ExperimentParameterData.Builder builder) {
+        TextTableReader.read(",", GCMPipelineFileHeaders.EXPERIMENT_PARAMETERS, experimentParametersFilePath,
+                (values) -> {
+                    switch (values[0]) {
+                        case "thread_count":
+                            builder.setThreadCount(Integer.parseInt(values[1]));
+                            break;
+                        case "state_recording_is_scheduled":
+                            builder.setRecordState(Boolean.parseBoolean(values[1]));
+                            break;
+                        case "simulation_halt_time":
+                            builder.setSimulationHaltTime(Double.parseDouble(values[1]));
+                            break;
+                        case "halt_on_exception":
+                            builder.setHaltOnException(Boolean.parseBoolean(values[1]));
+                            break;
+                        case "experiment_progress_log_path":
+                            builder.setExperimentProgressLog(Path.of(values[1]));
+                            break;
+                        case "continue_from_progress_log":
+                            builder.setContinueFromProgressLog(Boolean.parseBoolean(values[1]));
+                            break;
+                    }
+                });
+    }
+
+    private void loadExplicitScenarioIdsFile(ExperimentParameterData.Builder builder) {
+        TextTableReader.read(",", GCMPipelineFileHeaders.EXPLICIT_SCENARIO_IDS, explicitScenarioIdsFilePath,
+                (values) -> {
+                    builder.addExplicitScenarioId(Integer.parseInt(values[0]));
+                });
+    }
+}