add workflow cli

.argo/build.yaml

@@ -0,0 +1,115 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  namespace: argo
+spec:
+  entrypoint: main
+  serviceAccountName: argo-server
+  templates:
+    - name: main
+      steps:
+        - - name: publish-pkg-container-1
+            templateRef:
+              name: cwft-kaniko
+              template: build-push-https
+              clusterScope: true
+            arguments:
+              parameters:
+                - name: mode
+                  value: '{{workflow.parameters.mode}}'
+                - name: imageName
+                  value: '{{workflow.parameters.imageName1}}'
+                - name: containerFilePath
+                  value: '{{workflow.parameters.containerFilePath1}}'
+                - name: appName
+                  value: '{{workflow.parameters.appName}}'
+                - name: branch
+                  value: '{{workflow.parameters.branch}}'
+                - name: shortSha
+                  value: '{{workflow.parameters.shortSha}}'
+                - name: containerRegistryURL1
+                  value: '{{workflow.parameters.containerRegistryURL1}}'
+                - name: containerRegistryURL2
+                  value: '{{workflow.parameters.containerRegistryURL2}}'
+                - name: gitUrlNoProtocol
+                  value: '{{workflow.parameters.gitUrlNoProtocol}}'
+                - name: memoryRequest
+                  value: "5Gi" # override default: 512Mi
+                - name: cpuRequest
+                  value: "3" # override default: 500m
+                - name: memoryLimit
+                  value: "16Gi" # override default: 2Gi
+                - name: cpuLimit
+                  value: "8" # override default: 2
+                - name: storageRequest
+                  value: "35Gi" # override default: 15Gi
+          - name: publish-gpu-container-2
+            templateRef:
+              name: cwft-kaniko
+              template: build-push-https
+              clusterScope: true
+            arguments:
+              parameters:
+                - name: mode
+                  value: '{{workflow.parameters.mode}}'
+                - name: imageName
+                  value: '{{workflow.parameters.imageName2}}'
+                - name: containerFilePath
+                  value: '{{workflow.parameters.containerFilePath2}}'
+                - name: appName
+                  value: '{{workflow.parameters.appName}}'
+                - name: branch
+                  value: '{{workflow.parameters.branch}}'
+                - name: shortSha
+                  value: '{{workflow.parameters.shortSha}}'
+                - name: containerRegistryURL1
+                  value: '{{workflow.parameters.containerRegistryURL1}}'
+                - name: containerRegistryURL2
+                  value: '{{workflow.parameters.containerRegistryURL2}}'
+                - name: gitUrlNoProtocol
+                  value: '{{workflow.parameters.gitUrlNoProtocol}}'
+                - name: memoryRequest
+                  value: "7Gi" # override default: 512Mi
+                - name: cpuRequest
+                  value: "2" # override default: 500m
+                - name: memoryLimit
+                  value: "32Gi" # override default: 2Gi
+                - name: cpuLimit
+                  value: "8" # override default: 2
+                - name: storageRequest
+                  value: "50Gi" # override default: 15Gi
+          # - name: publish-conda-container-3
+          #   templateRef:
+          #     name: cwft-kaniko
+          #     template: build-push-https
+          #     clusterScope: true
+          #   arguments:
+          #     parameters:
+          #       - name: mode
+          #         value: '{{workflow.parameters.mode}}'
+          #       - name: imageName
+          #         value: '{{workflow.parameters.imageName3}}'
+          #       - name: containerFilePath
+          #         value: '{{workflow.parameters.containerFilePath3}}'
+          #       - name: appName
+          #         value: '{{workflow.parameters.appName}}'
+          #       - name: branch
+          #         value: '{{workflow.parameters.branch}}'
+          #       - name: shortSha
+          #         value: '{{workflow.parameters.shortSha}}'
+          #       - name: containerRegistryURL1
+          #         value: '{{workflow.parameters.containerRegistryURL1}}'
+          #       - name: containerRegistryURL2
+          #         value: '{{workflow.parameters.containerRegistryURL2}}'
+          #       - name: gitUrlNoProtocol
+          #         value: '{{workflow.parameters.gitUrlNoProtocol}}'
+          #       - name: memoryRequest
+          #         value: "7Gi" # override default: 512Mi
+          #       - name: cpuRequest
+          #         value: "2" # override default: 500m
+          #       - name: memoryLimit
+          #         value: "32Gi" # override default: 2Gi
+          #       - name: cpuLimit
+          #         value: "8" # override default: 2
+          #       - name: storageRequest
+          #         value: "50Gi" # override default: 15Gi

.devcontainer/devcontainer.Dockerfile

@@ -0,0 +1,4 @@
+FROM ghcr.io/cachix/devenv:latest
+
+RUN echo 'extra-substituters = https://devenv.cachix.org' >> /etc/nix/nix.conf && \
+    echo 'extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=' >> /etc/nix/nix.conf

.devcontainer/devcontainer.json

@@ -0,0 +1,35 @@
+{
+  "name": "test-devenv",
+  "build": {
+    "dockerfile": "./devcontainer.Dockerfile",
+    "context": ".."
+  },
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        // "mkhl.direnv",
+        // vim extension breaks
+        // during installation
+        // install via `recommends`
+        // "[email protected]",
+        "pmndrs.pmndrs",
+        "jnoortheen.nix-ide",
+        "tamasfe.even-better-toml",
+        "donjayamanne.python-extension-pack",
+        "charliermarsh.ruff",
+        "redhat.vscode-yaml",
+        "ms-kubernetes-tools.vscode-kubernetes-tools",
+        "ms-vsliveshare.vsli",
+        "eamodio.gitlens",
+        "GitHub.vscode-pull-request-github",
+        "github.vscode-github-actions",
+        "ms-azuretools.vscode-docker",
+        "ms-toolsai.jupyter",
+        "njzy.stats-bar",
+        "vscode-icons-team.vscode-icons"
+      ]
+    }
+  },
+  "overrideCommand": false,
+  "forwardPorts": [53593]
+}

.envrc

@@ -0,0 +1,15 @@
+# if ! has nix_direnv_version || ! nix_direnv_version 2.2.1; then
+#   source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.2.1/direnvrc" "sha256-zelF0vLbEl5uaqrfIzbgNzJWGmLzCmYAkInj/LNxvKs="
+# fi
+
+# if ! use flake . --impure --accept-flake-config
+# then
+#   echo "nix flake could not be built; update flake.nix and run direnv allow/reload" >&2
+# fi
+
+# printf "\nrun \`nix flake update --impure && nix flake check --impure\` to update the flake lockfile."
+# printf "\nuse \`direnv revoke\` to unload the dev environment or \`direnv allow\` to reload it.\n\n"
+
+# not necessary if using nix flake
+# devenv devshell with dotenv enabled
+dotenv

.example.env

@@ -0,0 +1,58 @@
+# Used as admin.clientSecretEnvVar with the unhashed version of
+# configuration.auth.internal.clientSecretHash
+# from the flyte-binary helm chart values.yaml file
+#   authType: ClientSecret
+#   clientId: flytepropeller
+#   clientSecretEnvVar: FLYTE_OAUTH_CLIENT_SECRET
+
+## nix
+CACHIX_CACHE_NAME=dnadiffusion
+CACHIX_AUTH_TOKEN=e1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111
+
+## Compute
+GCP_PROJECT_ID=project-111111
+GCP_STORAGE_SCOPES=https://www.googleapis.com/auth/devstorage.read_write
+GCP_STORAGE_CONTAINER=project-111111-flyte-meta-cluster
+GCP_ARTIFACT_REGISTRY_PATH=us.gcr.io/project/path
+
+## GitHub
+GH_ORG=pinellolab
+GH_REPO_NAME=DNA-Diffusion
+GH_REPO_NAME_SLUG=dna-diffusion
+GH_REPO=pinellolab/DNA-Diffusion
+
+## Flyte
+FLYTE_CLUSTER_ENDPOINT=dns:///cluster.net
+FLYTE_OAUTH_CLIENT_SECRET=O0000000000000000000000000000000
+FLYTECTL_CONFIG_TEMPLATE=.flyte/config-browser.yaml
+FLYTECTL_CONFIG=.flyte/config.yaml # required CI, may override constants.py
+
+## Python
+WORKFLOW_IMAGE="ghcr.io/pinellolab/dna-diffusion" # tag computed from git
+LOG_LEVEL=INFO
+
+## Makefile
+WORKFLOW_PROJECT=dnadiffusion
+WORKFLOW_DOMAIN=development
+WORKFLOW_PACKAGE_PATH=src
+WORKFLOW_IMPORT_PATH=dnadiffusion.workflows.example
+WORKFLOW_NAME=wf
+# WORKFLOW_IMPORT_PATH=dnadiffusion.workflows.lrwine
+# WORKFLOW_NAME=training_workflow
+# WORKFLOW_REGISTRATION_MODE=prod
+# WORKFLOW_REGISTRATION_MODE=dev
+# WORKFLOW_VERSION="repo-branch-shortsha" # override auto version
+# WORKFLOW_FILE=workflows/example.py
+# WORKFLOW_FILE_WORKFLOW_ARGS='{"C": 0.1, "max_iter": 1000}'
+GLOBAL_IMAGE_TAG=latest
+WORKFLOW_OUTPUT_FORMAT=dot
+
+## Local development
+# Toggle to use local dev cluster
+# these will override the values above
+#
+LOCAL_CONTAINER_REGISTRY=localhost:30000
+ACTIVE_DOCKERFILE=containers/pkg.Dockerfile
+# FLYTECTL_CONFIG=.flyte/config-local.yaml
+# WORKFLOW_IMAGE=localhost:30000/dnadiffusion
+# KUBECONFIG=/path/to/user/.flyte/sandbox/kubeconfig

.flyte/config-browser.yaml

@@ -0,0 +1,17 @@
+admin:
+  endpoint: ${FLYTE_CLUSTER_ENDPOINT}
+  # authenticate in browser via oauth2-proxy
+  authType: Pkce
+
+logger:
+  show-source: true
+  level: 0
+storage:
+  type: stow
+  stow:
+    kind: google
+    config:
+      json: ""
+      project_id: ${GCP_PROJECT_ID}
+      scopes: ${GCP_STORAGE_SCOPES}
+  container: ${GCP_STORAGE_CONTAINER}

.flyte/config-local.yaml

@@ -0,0 +1,8 @@
+admin:
+  endpoint: localhost:30080
+  insecure: true
+storage:
+  connection:
+    endpoint: http://localhost:30002
+    access-key: minio
+    secret-key: miniostorage

.flyte/config-template.yaml

@@ -0,0 +1,25 @@
+admin:
+  endpoint: ${FLYTE_CLUSTER_ENDPOINT}
+  # authenticate in browser via oauth2-proxy
+  # authType: Pkce
+
+  # programmatic auth for CI
+  # where EnvVar is the unhashed version of
+  # configuration.auth.internal.clientSecretHash
+  # from the helm chart values file
+  authType: ClientSecret
+  clientId: flytepropeller
+  clientSecretEnvVar: FLYTE_OAUTH_CLIENT_SECRET
+  insecure: false
+logger:
+  show-source: true
+  level: 0
+storage:
+  type: stow
+  stow:
+    kind: google
+    config:
+      json: ""
+      project_id: ${GCP_PROJECT_ID}
+      scopes: ${GCP_STORAGE_SCOPES}
+  container: ${GCP_STORAGE_CONTAINER}

.github/SECURITY.md

@@ -0,0 +1,3 @@
+# security
+
+To report a potential security vulnerability in tctsti, please follow the [instructions to create a draft security advisory](https://github.com/pinellolab/DNA-Diffusion/security/advisories/new).

.github/actions/setup_environment/action.yml

@@ -0,0 +1,59 @@
+name: 'Setup Environment'
+description: 'Shared steps to setup the environment for CI jobs'
+
+inputs:
+  python_version:
+    description: 'Python version to setup'
+    required: true
+  debug_enabled:
+    description: "Run with tmate.io debugging enabled"
+    required: true
+    default: "false"
+  gpu_enabled:
+    description: "Do not downgrade to CPU libraries"
+    required: true
+    default: "false"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      id: setup-python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python_version }}
+    - name: Install system dependencies
+      shell: bash
+      run: |
+        echo "install system dependencies"
+        # sudo apt-get update
+        # sudo apt-get install -y tree
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.7.1
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        installer-parallel: true
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v3
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+    - name: Install dependencies
+      shell: bash
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: poetry install --with docs,lint,test,workflows,bioinformatics --no-interaction --no-root
+    - name: Install project
+      shell: bash
+      run: poetry install --with docs,lint,test,workflows,bioinformatics --no-interaction
+    - name: Downgrade to torch CPU
+      if: ${{ inputs.gpu_enabled != 'true' }}
+      shell: bash
+      run: |
+        source $VENV
+        poe torch-cpu
+    - name: "Setup tmate debug session"
+      uses: mxschmitt/action-tmate@v3
+      if: ${{ inputs.debug_enabled == 'true' }}

.github/codecov.yml

@@ -0,0 +1,15 @@
+codecov:
+  branch: main
+
+comment: false
+github_checks: false
+
+coverage:
+  status:
+    project:
+      default:
+        threshold: 10%
+
+    patch: off
+
+    changes: false

.github/workflows/build.yml → .github/deprecated/build.yml

@@ -2,10 +2,10 @@ name: Build
 
 on:
   workflow_dispatch:
-  push:
-    branches:
-      - main
-  pull_request:
+  # push:
+  #   branches:
+  #     - main
+  # pull_request:
 
 jobs:
   test: