chore(git): resolve branch conflict

queenbee/cli.py

@@ -66,6 +66,7 @@ def run():
 import urllib.parse
 from pkg_resources import iter_entry_points
 from queenbee.schema.workflow import Workflow
+from queenbee.schema.arguments import WorkflowInputs
 
 # TODO: Comment out and use for logging once we are adding commands to this file.
 # import logging
@@ -78,13 +79,30 @@ class Context():
     def parse_workflow(filepath):
         try:
             wf = Workflow.from_file(filepath)
-            wf.validate_all()
             return wf
         except AssertionError as e:
             raise click.UsageError(e)
         except Exception as e:
             raise click.ClickException(e)
 
+    @staticmethod
+    def parse_and_hydrate_workflow(wf_filepath: str, inputs_filepath: str = None):
+        try:
+            wf = Workflow.from_file(wf_filepath)
+            inputs = None
+
+            if inputs_filepath is not None:
+                inputs = WorkflowInputs.from_file(inputs_filepath)
+
+            return Workflow.parse_obj(
+                wf.hydrate_workflow_templates(inputs=inputs)
+            )
+
+        except AssertionError as e:
+            raise click.UsageError(e)
+        except Exception as e:
+            raise click.ClickException(e)
+
 
 @with_plugins(iter_entry_points('queenbee.plugins'))
 @click.group()
@@ -143,5 +161,38 @@ def validate(ctx, file, display):
         webbrowser.open(url)
 
 
+@main.command('hydrate')
+@click.option('-f', '--file', help='path to the workflow file to validate', required=True)
+@click.option('-i', '--inputs', help='path to the inputs file inject into the workflow', required=False)
+@click.option('-o', '--output', help='path to file to save the hydrated workflow to', required=False)
+@click.option('-d', '--display', help='boolean flag to display the workflow in your browser', default=False, type=bool, is_flag=True)
+@click.pass_context
+def hydrate(ctx, file, inputs, output, display):
+    """Hydrate a workflow and optionally add inputs"""
+
+    wf = ctx.obj.parse_and_hydrate_workflow(file, inputs)
+
+    dot = wf.to_diagraph(filename=file.split('.')[0])
+
+#     click.echo("""
+#                        _   _
+#                       ( | / )
+#                     \\\\ \\|/,'_
+#                     (")(_)()))=-
+# Valid Workflow!        <\\\\
+
+#     """)
+
+    if output:
+        wf.to_yaml(output)
+    else:
+        print(wf.yaml())
+
+    if display:
+        query = urllib.parse.quote(dot.pipe(format='xdot'))
+        url = 'https://dreampuf.github.io/GraphvizOnline/#{}'.format(query)
+        webbrowser.open(url)
+
+
 if __name__ == "__main__":
     main()

queenbee/schema/arguments.py

@@ -297,3 +297,25 @@ class WorkflowArguments(Arguments):
         description='Optional user data as a dictionary. User data is for user reference'
         ' only and will not be used in the execution of the workflow.'
     )
+
+
+class WorkflowInputs(BaseModel):
+    """Inputs to inject into a workflow before hydration
+    
+    """
+
+    artifacts: Dict[str, dict] = Field(
+        None,
+        description='A dictionary containing the workflow input argument name as a key and a dictonary of key value pairs to update'
+    )
+
+    parameters: Dict[str, dict] = Field(
+        None,
+        description='A dictionary containing the workflow input argument name as a key and a dictonary of key value pairs to update'
+    )
+
+    user_data: Dict = Field(
+        None,
+        description='Optional user data as a dictionary. User data is for user reference'
+        ' only and will not be used in the execution of the workflow.'
+    )

queenbee/schema/artifact_location.py

@@ -9,8 +9,9 @@
     3. S3: An S3 bucket
 """
 from queenbee.schema.qutil import BaseModel
+import queenbee.schema.variable as qbvar
 from pydantic import Field, constr
-from typing import Dict
+from typing import Dict, List
 from enum import Enum
 
 
@@ -38,8 +39,31 @@ class ArtifactLocation(BaseModel):
         description='The root path to the artifacts.'
     )
 
+    @staticmethod
+    def _referenced_values(values) -> Dict[str, List[str]]:
+        """Get referenced variables if any"""
+        ref_values = {}
+
+        if not values:
+            return ref_values
 
-class InputFolderLocation(BaseModel):
+        for value in values:
+            if value is None:
+                continue
+            ref_var = qbvar.get_ref_variable(value)
+            if ref_var:
+                ref_values[value] = ref_var
+
+        return ref_values
+
+    @property
+    def referenced_values(self) -> Dict[str, List[str]]:
+        values = [self.root]
+
+        return self._referenced_values(values)
+
+
+class InputFolderLocation(ArtifactLocation):
     """Input Folder Location
 
     This is a folder that the workflow can use to pull input artifacts from.
@@ -59,8 +83,15 @@ class InputFolderLocation(BaseModel):
             Will be ignored when running on the Pollination platform."
     )
 
+    @property
+    def referenced_values(self) -> Dict[str, List[str]]:
+        values = [self.root]
+
+        return self._referenced_values(values)
+
+
 
-class RunFolderLocation(BaseModel):
+class RunFolderLocation(ArtifactLocation):
     """Run Folder Location
 
     This is the folder a workflow will use as it's root path when running a simulation.
@@ -82,8 +113,13 @@ class RunFolderLocation(BaseModel):
             Will be ignored when running on the Pollination platform."
     )
 
+    @property
+    def referenced_values(self) -> Dict[str, List[str]]:
+        values = [self.root]
 
-class HTTPLocation(BaseModel):
+        return self._referenced_values(values)
+
+class HTTPLocation(ArtifactLocation):
     """HTTPLocation
 
     A web HTTP to an FTP server or an API for example.
@@ -102,7 +138,7 @@ class HTTPLocation(BaseModel):
     )
 
     headers: Dict[str, str] = Field(
-        None,
+        {},
         description="An object with Key Value pairs of HTTP headers"
     )
 
@@ -111,11 +147,20 @@ class HTTPLocation(BaseModel):
         description="The HTTP verb to use when making the request."
     )
 
+    @property
+    def referenced_values(self) -> Dict[str, List[str]]:
+        values = [self.root, self.verb]
+
+        for k, v in self.headers:
+            values.append(v)
+
+        return self._referenced_values(values)
+
     class Config:
         use_enum_value = True
 
 
-class S3Location(BaseModel):
+class S3Location(ArtifactLocation):
     """S3Location
 
     An S3 bucket
@@ -148,3 +193,9 @@ class S3Location(BaseModel):
         description="Path to the file holding the AccessKey and SecretAccessKey to "
         "authenticate to the bucket"
     )
+
+    @property
+    def referenced_values(self) -> Dict[str, List[str]]:
+        values = [self.root, self.endpoint, self.bucket, self.credentials_path]
+
+        return self._referenced_values(values)

queenbee/schema/qutil.py

@@ -24,7 +24,7 @@ class BaseModel(PydanticBaseModel):
 
     def yaml(self, exclude_unset=False):
         return yaml.dump(
-            self.dict(exclude_unset=exclude_unset),
+            json.loads(self.json(exclude_unset=exclude_unset)),
             default_flow_style=False
         )