Merge branch 'master' into dan-ss-molecular-struct-properties

.github/workflows/CI-full.yml

@@ -98,8 +98,8 @@ jobs:
       shell: bash
       run: |
         cd docker/build
-        sudo docker login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} ghcr.io
-        docker login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} ghcr.io
+        echo "${{ secrets.GITHUB_TOKEN }}" | sudo docker login ghcr.io -u ${{ github.actor }} --password-stdin
+        echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
         ./build.sh all ${{ env.VCELL_REPO_NAMESPACE }} ${{ env.VCELL_TAG }}
 
     - name: tag as latest and push to registry

.github/workflows/NightlyBMDB_CLI.yml

@@ -5,12 +5,12 @@ on:
   workflow_dispatch:
 
 env:
-  python-version: "3.9"
+  python-version: "3.10"
   poetry-version: "1.2.1"
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout Our Code
         uses: actions/checkout@v4

.github/workflows/bsts_local.yml

@@ -28,12 +28,12 @@ jobs:
 
       - name: Install Perl # for BioNetGen
         run: |
-          sudo apt-get update -y
-          sudo apt-get install -y --no-install-recommends perl
+          sudo apt update -y
+          sudo apt install -y --no-install-recommends perl
 
       - name: Install XPP
         run: |
-          sudo apt-get install -y --no-install-recommends xppaut
+          sudo apt install -y --no-install-recommends xppaut
 
       - name: Install Go (for Singularity)
         uses: actions/setup-go@v3
@@ -47,7 +47,7 @@ jobs:
           OS: linux
           ARCH: amd64
         run: |
-          sudo apt-get install -y --no-install-recommends \
+          sudo apt install -y --no-install-recommends \
             build-essential \
             libseccomp-dev \
             pkg-config \
@@ -91,7 +91,7 @@ jobs:
       - name: Login into Docker registries
         run: |
           # VCell Docker registry (GHCR)
-          docker login ghcr.io --username ${{ secrets.GHCR_USERNAME }} --password ${{ secrets.GHCR_TOKEN }}
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
       - id: validateCommitSimulator
         name: Validate and commit simulator

.github/workflows/ci_cd.yml

@@ -10,7 +10,7 @@ on:
     types: [published, edited]
 
 env:
-  python-version: "3.9"
+  python-version: "3.10"
 
 jobs:
   CI-Build:
@@ -107,7 +107,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install hdf5 tools needed for testing
-        run: sudo apt-get -y update && sudo apt-get -y install hdf5-tools
+        run: sudo apt -y update && sudo apt -y install hdf5-tools
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -267,7 +267,7 @@ jobs:
           REVISION=$(git rev-parse HEAD)
           CREATED=$(date --rfc-3339=seconds | sed 's/ /T/')
           sed -i 's/_VC_VERSION_TAG_/${{steps.version.outputs.tag}}/g' biosimulators.json
-          echo ${{ secrets.ACTION_TOKEN }} | docker login ghcr.io -u ${{ secrets.ACTION_USER }} --password-stdin
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
           docker build \
             --no-cache \
             --file Dockerfile \

.github/workflows/create_bs_singularity.yml

@@ -8,12 +8,12 @@ on:
         default: 3ca1093da599a0ff4229b1130efe7a3924ff2b4c
 
 env:
-  python-version: "3.9"
+  python-version: "3.10"
 
 jobs:
   CD:
     name: CD
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v4
@@ -64,7 +64,7 @@ jobs:
           REVISION=$(git rev-parse HEAD)
           CREATED=$(date --rfc-3339=seconds | sed 's/ /T/')
           sed -i 's/_VC_VERSION_TAG_/${VCELL_SHA}/g' biosimulators.json
-          echo ${{ secrets.ACTION_TOKEN }} | docker login ghcr.io -u ${{ secrets.ACTION_USER }} --password-stdin
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
           docker build \
             --no-cache \
             --file Dockerfile \
@@ -77,7 +77,7 @@ jobs:
             .
           docker push --all-tags ghcr.io/virtualcell/biosimulators_vcell
           singularity build biosimulators_vcell_${VCELL_SHA}.img docker:ghcr.io/virtualcell/biosimulators_vcell:${VCELL_SHA}
-          singularity remote login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} oras://ghcr.io
+          singularity remote login -u "${{ secrets.GITHUB_TOKEN }}" -p ${{ github.actor }} oras://ghcr.io
           singularity push -U biosimulators_vcell_${VCELL_SHA}.img oras://ghcr.io/virtualcell/biosimulators_vcell_singularity:${VCELL_SHA}
 
       - name: Setup tmate session

.github/workflows/nightly-model-testing.yml

@@ -6,12 +6,12 @@ on:
   workflow_dispatch:
 
 env:
-  python-version: "3.9"
+  python-version: "3.10"
   poetry-version: "1.2.1"
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout Our Code
         uses: actions/checkout@v4

.github/workflows/site_deploy.yml

@@ -87,7 +87,7 @@ jobs:
       if: ${{ github.event.inputs.server_only != 'true' }}
       run: |
         cd docker/swarm
-        sudo docker login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} ghcr.io
+        echo "${{ secrets.GITHUB_TOKEN }}" | sudo docker login ghcr.io -u ${{ github.actor }} --password-stdin
         sudo docker pull $VCELL_REPO_NAMESPACE/vcell-clientgen:$VCELL_TAG
         ./generate_installers.sh ./${VCELL_CONFIG_FILE_NAME}
     - name: upload generated installers
@@ -197,7 +197,7 @@ jobs:
       run: |
         set -ux
         cd docker/swarm
-        ssh -t ${{ secrets.CD_FULL_USER }}@${VCELL_MANAGER_NODE} sudo docker login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} ghcr.io
+        ssh -t ${{ secrets.CD_FULL_USER }}@${VCELL_MANAGER_NODE} sudo docker login -u ${{ github.actor }} -p "${{ secrets.GITHUB_TOKEN }}" ghcr.io
         if ${{ github.event.inputs.server_only != 'true' }}; then
           # build and install the client installers, and the web help (kubernetes cluster deployments are separate)
           ./deploy-action-kubernetes.sh \

docker/build/Dockerfile-admin-dev

@@ -10,17 +10,17 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM debian:buster-slim
+FROM debian:bullseye-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
 # now we have Java 17
-RUN apt-get -y update && \
-    apt-get install -y curl dnsutils apt-utils libfreetype6 fontconfig fonts-dejavu openssh-client && \
+RUN apt -y update && \
+    apt install -y curl dnsutils apt-utils libfreetype6 fontconfig fonts-dejavu openssh-client && \
     mkdir -p /usr/local/app/lib 
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
 RUN unlink /etc/localtime || true
 RUN ln -s /usr/share/zoneinfo/America/New_York /etc/localtime
 

docker/build/Dockerfile-api-dev

@@ -10,18 +10,18 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM python:3.9.7-slim
+FROM python:3.10.15-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
 # now we have Java 17 and Python 3.9
 
-RUN apt-get -y update && \
-    apt-get install -y --no-install-recommends curl dnsutils apt-utils libfreetype6 fontconfig fonts-dejavu && \
+RUN apt -y update && \
+    apt install -y --no-install-recommends curl dnsutils apt-utils libfreetype6 fontconfig fonts-dejavu && \
     mkdir -p /usr/local/app/lib 
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
 RUN unlink /etc/localtime || true
 RUN ln -s /usr/share/zoneinfo/America/New_York /etc/localtime
 

docker/build/Dockerfile-batch-dev

@@ -1,11 +1,11 @@
-FROM ghcr.io/virtualcell/vcell-solvers:v0.8.1.3
+FROM ghcr.io/virtualcell/vcell-solvers:v0.8.2
 
-RUN apt-get -y update && \
-    apt-get install -y  curl && \
-	apt-get install -y wget gdebi-core apt-utils libfreetype6 fontconfig fonts-dejavu && \
+RUN apt -y update && \
+    apt install -y  curl && \
+	apt install -y wget gdebi-core apt-utils libfreetype6 fontconfig fonts-dejavu && \
     pip3 install thrift
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
 RUN unlink /etc/localtime || true
 RUN ln -s /usr/share/zoneinfo/America/New_York /etc/localtime
 

docker/build/Dockerfile-clientgen-dev

@@ -10,14 +10,14 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM debian:buster-slim
+FROM debian:bullseye-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
 RUN mkdir -p /usr/local/app && \
-	apt-get -y update && \
-    apt-get -y install wget apt-utils libfreetype6 fontconfig fonts-dejavu
+	apt -y update && \
+    apt -y install wget apt-utils libfreetype6 fontconfig fonts-dejavu
 
 RUN mkdir /installer && cd /installer && \
     wget --quiet -O install4j_unix_10_0_5.tar.gz \

docker/build/Dockerfile-data-dev

@@ -10,21 +10,21 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM python:3.9.7-slim
+FROM python:3.10.15-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
 # now we have Java 17 and Python 3.9
 
-RUN apt-get -y update && \
-	apt-get -y install openssh-client screen apt-utils libfreetype6 fontconfig fonts-dejavu && \
+RUN apt -y update && \
+	apt -y install openssh-client screen apt-utils libfreetype6 fontconfig fonts-dejavu && \
     mkdir -p /usr/local/app
 
 RUN python3 -m pip install poetry &&  poetry config cache-dir "/poetry/.cache"
 ENV PATH="/root/.poetry/bin:/root/.local/bin:$PATH"
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
 RUN unlink /etc/localtime || true
 RUN ln -s /usr/share/zoneinfo/America/New_York /etc/localtime
 

docker/build/Dockerfile-db-dev

@@ -10,16 +10,16 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM debian:buster-slim
+FROM debian:bullseye-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
 RUN mkdir -p /usr/local/app && \
-	apt-get -y update && \
-    apt-get -y install screen apt-utils libfreetype6 fontconfig fonts-dejavu
+	apt -y update && \
+    apt -y install screen apt-utils libfreetype6 fontconfig fonts-dejavu
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
 RUN unlink /etc/localtime || true
 RUN ln -s /usr/share/zoneinfo/America/New_York /etc/localtime
 

docker/build/Dockerfile-sched-dev

@@ -10,17 +10,17 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM debian:buster-slim
+FROM debian:bullseye-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
 
 RUN mkdir -p /usr/local/app && \
-	apt-get -y update && \
-    apt-get -y install openssh-client curl screen apt-utils libfreetype6 fontconfig fonts-dejavu
+	apt -y update && \
+    apt -y install openssh-client curl screen apt-utils libfreetype6 fontconfig fonts-dejavu
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
 RUN unlink /etc/localtime || true
 RUN ln -s /usr/share/zoneinfo/America/New_York /etc/localtime
 
@@ -68,7 +68,11 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \
 	maxOdeJobsPerUser="max-ode-jobs-per-user-not-set" \
 	vcell_ssh_cmd_cmdtimeout="cmdSrvcSshCmdTimeoutMS-not-set" \
     vcell_ssh_cmd_restoretimeout="cmdSrvcSshCmdRestoreTimeoutFactor-not-set" \
-	maxPdeJobsPerUser="max-pde-jobs-per-user-not-set"
+	maxPdeJobsPerUser="max-pde-jobs-per-user-not-set" \
+	htcMinMemoryMB="htc-min-memory-not-set" \
+	htcMaxMemoryMB="htc-max-memory-not-set" \
+	htcPowerUserMemoryFloorMB="htc-power-user-memory-floor-not-set" \
+    htcPowerUserMemoryMaxMB="htc-power-user-memory-max-not-set"
 
 ENV dbpswdfile=/run/secrets/dbpswd \
     jmspswdfile=/run/secrets/jmspswd \
@@ -119,4 +123,8 @@ ENTRYPOINT java \
 	-Dvcell.server.maxPdeJobsPerUser=${maxPdeJobsPerUser} \
 	-Dvcell.ssh.cmd.cmdtimeout=${vcell_ssh_cmd_cmdtimeout} \
 	-Dvcell.ssh.cmd.restoretimeout=${vcell_ssh_cmd_restoretimeout} \
-	-cp "./lib/*" cbit.vcell.message.server.dispatcher.SimulationDispatcher
+	-Dvcell.htc.memory.min.mb=${htcMinMemoryMB} \
+	-Dvcell.htc.memory.max.mb=${htcMaxMemoryMB} \
+	-Dvcell.htc.memory.pu.floor.mb=${htcPowerUserMemoryFloorMB} \
+    -Dvcell.htc.memory.pu.max.mb=${htcPowerUserMemoryMaxMB} \
+	-cp "./lib/*" cbit.vcell.message.server.dispatcher.SimulationDispatcherMain

docker/build/Dockerfile-submit-dev

@@ -10,16 +10,16 @@ RUN $JAVA_HOME/bin/jlink \
          --output /javaruntime
 
 # Define base image and copy in jlink created minimal Java 17 environment
-FROM debian:buster-slim
+FROM debian:bullseye-slim
 ENV JAVA_HOME=/opt/java/openjdk
 ENV PATH "${JAVA_HOME}/bin:${PATH}"
 COPY --from=jre-build /javaruntime $JAVA_HOME
 
 RUN mkdir -p /usr/local/app && \
-	apt-get -y update && \
-    apt-get -y install openssh-client screen apt-utils libfreetype6 fontconfig fonts-dejavu
+	apt -y update && \
+    apt -y install openssh-client screen apt-utils libfreetype6 fontconfig fonts-dejavu
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
 RUN unlink /etc/localtime || true
 RUN ln -s /usr/share/zoneinfo/America/New_York /etc/localtime
 
@@ -91,7 +91,11 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \
     vcell_ssh_cmd_cmdtimeout="cmdSrvcSshCmdTimeoutMS-not-set" \
     vcell_ssh_cmd_restoretimeout="cmdSrvcSshCmdRestoreTimeoutFactor-not-set" \
     simdatadir_archive_external="simdatadir_archive_external-not-set" \
-    simdatadir_archive_internal="simdatadir_archive_internal-not-set"
+    simdatadir_archive_internal="simdatadir_archive_internal-not-set" \
+    htcMinMemoryMB="htc-min-memory-not-set" \
+    htcMaxMemoryMB="htc-max-memory-not-set" \
+    htcPowerUserMemoryFloorMB="htc-power-user-memory-floor-not-set" \
+    htcPowerUserMemoryMaxMB="htc-power-user-memory-max-not-set"
 
 ENV jmspswdfile=/run/secrets/jmspswd \
 	jmsrestpswdfile=/run/secrets/jmsrestpswd \
@@ -170,4 +174,8 @@ ENTRYPOINT java \
 	-Dvcell.simdatadir.archive.external=${simdatadir_archive_external} \
 	-Dvcell.ssh.cmd.cmdtimeout=${vcell_ssh_cmd_cmdtimeout} \
 	-Dvcell.ssh.cmd.restoretimeout=${vcell_ssh_cmd_restoretimeout} \
+	-Dvcell.htc.memory.min.mb=${htcMinMemoryMB} \
+    -Dvcell.htc.memory.max.mb=${htcMaxMemoryMB} \
+    -Dvcell.htc.memory.pu.floor.mb=${htcPowerUserMemoryFloorMB} \
+    -Dvcell.htc.memory.pu.max.mb=${htcPowerUserMemoryMaxMB} \
 	-cp "./lib/*" cbit.vcell.message.server.batch.sim.HtcSimulationWorker

docker/build/build.sh

@@ -130,6 +130,8 @@ build_webapp() {
   if [[ $? -ne 0 ]]; then echo "failed to build prod"; exit 1; fi
   build_webapp_common island
   if [[ $? -ne 0 ]]; then echo "failed to build island"; exit 1; fi
+  build_webapp_common remote
+  if [[ $? -ne 0 ]]; then echo "failed to build remote"; exit 1; fi
 }
 
 build_batch() {

docker/swarm/README_DockerSwarmConfig.md

@@ -25,8 +25,8 @@ install jq (utility for json processing)
 install poetry (for building/installing vcell-admin)
 
 ```bash
-sudo apt-get install -y jq
-sudo apt-get install -y python3-poetry
+sudo apt install -y jq
+sudo apt install -y python3-poetry
 ```
 
 ## Install singularity [more info linux](https://singularity.lbl.gov/install-linux)

docker/swarm/README_zeke_stack_on_linux.md

@@ -11,7 +11,7 @@
    2. 
        ```bash
        pushd ../build
-       ./build.sh --skip-maven --skip-singularity --skip-sudo all localhost:5000/virtualcell dev_zeke
+       ./build.sh --skip-maven --skip-sudo all localhost:5000/virtualcell dev_zeke
        popd
       ```