DIALS 3.18.2

.azure-pipelines/ci-build.yml

@@ -9,9 +9,9 @@ jobs:
       modules: modules
   - template: ./conda-linux.yml
     parameters:
-      distribution: centos
-      version: [7]
-      modules: modules
+      distribution: rockylinux
+      version: [8]
+      modules: ${{ parameters.modules }}
       template: ./unix-conda-build.yml
   - template: ./conda-osx.yml
     parameters:

.azure-pipelines/download-source.yml

@@ -13,11 +13,6 @@ jobs:
 
   steps:
 
-  - task: DownloadSecureFile@1
-    name: ssh_config
-    inputs:
-      secureFile: ssh.config
-
   # download sources
   - checkout: self
     path: ./modules/cctbx_project
@@ -29,7 +24,6 @@ jobs:
 
   - script: |
       cd $(Pipeline.Workspace)
-      export SVN_SSH="ssh -F $(ssh_config.secureFilePath)"
       python bootstrap.py hot update --builder=cctbx --python=38 ${{ parameters.flags }}
     displayName: Download sources
 

.azure-pipelines/full-build.yml

@@ -18,12 +18,12 @@ jobs:
       version: [9]
       modules: ${{ parameters.modules }}
       template: ./unix-conda-build.yml
-  - template: ./conda-linux.yml
-    parameters:
-      distribution: centos
-      version: [7]
-      modules: ${{ parameters.modules }}
-      template: ./unix-conda-build.yml
+  # - template: ./conda-linux.yml
+  #   parameters:
+  #     distribution: centos
+  #     version: [7]
+  #     modules: ${{ parameters.modules }}
+  #     template: ./unix-conda-build.yml
   - template: ./conda-linux.yml
     parameters:
       distribution: rockylinux
@@ -63,6 +63,6 @@ jobs:
   - template: ./conda-osx.yml
     parameters:
       vmImage: [13]
-      xcode_version: [15, 0, 1]
+      xcode_version: [15, 2]
       modules: ${{ parameters.modules }}
       template: ./unix-conda-build.yml

.azure-pipelines/py2_syntax_exceptions.txt

@@ -2,7 +2,10 @@ crys3d/hklviewer/webbrowser_messenger_py3.py
 crys3d/hklviewer/xtriage_runner.py
 crys3d/regression/tst_websocket.py
 libtbx/auto_build/conda_build/create_custom_bin.py
+libtbx/resource_monitor.py
 serialtbx/util/energy_scan_notch_finder.py
+xfel/ui/components/xfel_gui_init.py
+xfel/ui/components/ebeam_plotter.py
 xfel/ui/db/dxtbx_db.py
 xfel/euxfel/agipd_from_defs.py
 xfel/euxfel/agipd_cxigeom2nexus.py
@@ -13,3 +16,7 @@ xfel/ui/db/job.py
 xfel/command_line/cxi_mpi_submit.py
 xfel/command_line/fee_calibration.py
 xfel/util/drift.py
+xfel/merging/application/errors/error_modifier_mm24.py
+xfel/util/preference.py
+cctbx/maptbx/qscore.py
+cctbx/maptbx/tst_qscore.py

.azure-pipelines/syntax.yml

@@ -18,18 +18,52 @@ jobs:
         PYTHON_VERSION: 3.9
       python3.10:
         PYTHON_VERSION: 3.10
+      python3.11:
+        PYTHON_VERSION: 3.11
+      python3.12:
+        PYTHON_VERSION: 3.12
+      python3.13:
+        PYTHON_VERSION: 3.13
 
   steps:
 
+  - task: InstallSSHKey@0
+    inputs:
+      knownHostsEntry: boa.lbl.gov ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAnPOtqyYASs6hc3fsmUQn92ovSozJsMsJyrMWjppUERX1tD4Fh0D7YaNhLvYpKi5mYVREo+tsKoOatCcqKGHCONWpzx7JHXUzfIBsy6nCeygRyyW4yuyoCuYFvOYLkaM4PrN/7XaZEnp1ux6/ZcbRxBDNK4f42svJUV39OX33tRuQWpP+O85paJr1ePHVz6U2n14a+3fRN3DBMti1dJuKOThU4djYEBQimx54pnW71eYORN2oUz+N/4RHAcomtxZpcUMl3Qadw8hD4s1XM6fzJ0Que7KefnnrPOgAEtZxKl9j09aaE70Oh+ie5Y6ckwddJ/4qZB5m2tBaEi3xuy0TSQ==
+      sshPublicKey: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA7XgVgdU9GmZuo5yVyW/htrAuxe7ypuq7IowWwfUC0WZw9SPzQ8kOGj63plF3ytx/qpgvUlS1AiywO20rZ83gHmRdAiuvt9laH09KlWEUhIhAQTzesqBG5aUT0MKo01yxijoRl5HC8w/MdOanc0h01e6IxnZvgG0N30RB1i4xVPCtR/VBmXSp4R0T+Q/gJjfQssuBxHVMrrute4V3uUHek58FL2QgUB0+zq6ipETjuCJxTSyYVAFtCYDPYs+0lAYYkWrXALCr9kX9s22jYtkyv5kNw6mEW8nhtA7GbTdJLv4M6/WYtcvQV8TDcNCfltOfl81U3gcZ9zhQDKGVoNaJEw== [email protected]
+      sshKeySecureFile: id_rsa
+    displayName: Download SSH key
+
   - checkout: self
     path: ./modules/cctbx_project
 
+  - script: |
+      set -xe
+      ssh-keyscan -t rsa gitlab.com >> ~/.ssh/known_hosts
+      cd $(Pipeline.Workspace)/modules/cctbx_project
+      git checkout master
+      git remote add gitlab [email protected]:cctbx/cctbx_project.git
+      git fetch gitlab
+      git push -u gitlab master
+    displayName: Update GitLab mirror
+    condition: eq(variables['PYTHON_VERSION'], '3.9')
+    continueOnError: true
+
   - script: echo "##vso[task.prependpath]$CONDA/bin"
     displayName: Add conda to PATH
 
   - script: |
       conda create -y -c conda-forge -n py$(PYTHON_VERSION) python=$(PYTHON_VERSION) six future scons
     displayName: Create conda environment
+    condition: ne(variables['PYTHON_VERSION'], '3.13')
+
+  - script: |
+      conda create -y -c conda-forge -c conda-forge/label/python_rc -n py$(PYTHON_VERSION) python=$(PYTHON_VERSION) six future setuptools
+      conda activate py$(PYTHON_VERSION)
+      cd $(Pipeline.Workspace)/modules
+      git clone -b 4.8.1 https://github.com/SCons/scons.git
+    displayName: Create conda environment
+    condition: eq(variables['PYTHON_VERSION'], '3.13')
 
   - script: |
       set -xe
@@ -38,7 +72,7 @@ jobs:
         echo Removing $f
         rm $f; done
     displayName: Remove Python 3 only files for Python 2 syntax check
-    condition: eq(variables['PYTHON_VERSION'], '2.7')
+    condition: or(eq(variables['PYTHON_VERSION'], '2.7'), eq(variables['PYTHON_VERSION'], '3.7'))
 
   - script: |
       source activate py$(PYTHON_VERSION)

.azure-pipelines/templates/download-data-cache.yml

@@ -1,8 +1,8 @@
-# Template for updating data cache artifacts
+# Template for downloading data cache artifacts
 #   - chem_data
+#     - geostd
 #   - phenix_regression
 #   - phenix_examples
-# A ssh_config secure file is needed
 
 steps:
 - task: InstallSSHKey@0
@@ -11,7 +11,9 @@ steps:
     sshPublicKey: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA7XgVgdU9GmZuo5yVyW/htrAuxe7ypuq7IowWwfUC0WZw9SPzQ8kOGj63plF3ytx/qpgvUlS1AiywO20rZ83gHmRdAiuvt9laH09KlWEUhIhAQTzesqBG5aUT0MKo01yxijoRl5HC8w/MdOanc0h01e6IxnZvgG0N30RB1i4xVPCtR/VBmXSp4R0T+Q/gJjfQssuBxHVMrrute4V3uUHek58FL2QgUB0+zq6ipETjuCJxTSyYVAFtCYDPYs+0lAYYkWrXALCr9kX9s22jYtkyv5kNw6mEW8nhtA7GbTdJLv4M6/WYtcvQV8TDcNCfltOfl81U3gcZ9zhQDKGVoNaJEw== [email protected]
     sshKeySecureFile: id_rsa
   displayName: Download SSH key
+  continueOnError: true
 
+# chem_data
 - task: DownloadPipelineArtifact@2
   inputs:
     source: 'specific'
@@ -22,15 +24,34 @@ steps:
   displayName: Download chem_data
 
 - script: |
-    export SVN_SSH="ssh -F $(ssh_config.secureFilePath)"
-    if [ ! -d "$(Pipeline.Workspace)/modules/chem_data" ]; then
-      svn co svn+ssh://[email protected]/chem_data/trunk chem_data
-    fi
     cd $(Pipeline.Workspace)/modules/chem_data
-    svn update
+    git reset --hard origin/main
+    git lfs install --local
+    git pull --rebase
+    git lfs pull
   displayName: Update chem_data
   continueOnError: true
 
+# geostd (in chem_data)
+- task: DownloadPipelineArtifact@2
+  inputs:
+    source: 'specific'
+    project: '$(resources.pipeline.data_cache.projectID)'
+    pipeline: '$(resources.pipeline.data_cache.pipelineID)'
+    allowPartiallySucceededBuilds: true
+    artifact: 'geostd'
+    path: $(Pipeline.Workspace)/modules/chem_data/geostd
+  displayName: Download geostd
+  continueOnError: true
+
+- script: |
+    cd $(Pipeline.Workspace)/modules/chem_data/geostd
+    git reset --hard origin/master
+    git pull --rebase
+  displayName: Update geostd
+  continueOnError: true
+
+# phenix_regression
 - task: DownloadPipelineArtifact@2
   inputs:
     source: 'specific'
@@ -49,6 +70,7 @@ steps:
   displayName: Update phenix_regression
   continueOnError: true
 
+# phenix_examples
 - task: DownloadPipelineArtifact@2
   inputs:
     source: 'specific'

.azure-pipelines/templates/update-data-cache.yml

@@ -3,6 +3,7 @@
 # is required to prevent Azure Pipelines from deleting the .git directory.
 # when creating the artifact.
 #   - miniforge
+#   - mambaforge
 #   - chem_data
 #   - geostd
 #   - phenix_regression
@@ -26,11 +27,6 @@ jobs:
       sshKeySecureFile: id_rsa
     displayName: Download SSH key
 
-  - task: DownloadSecureFile@1
-    name: ssh_config
-    inputs:
-      secureFile: ssh.config
-
   # miniforge
   - script: |
       mkdir -p $(System.DefaultWorkingDirectory)/miniforge
@@ -66,15 +62,18 @@ jobs:
     continueOnError: true
 
   - script: |
-      export SVN_SSH="ssh -F $(ssh_config.secureFilePath)"
-      if [ ! -d "$(System.DefaultWorkingDirectory)/chem_data/.svn" ]; then
+      if [ ! -d "$(System.DefaultWorkingDirectory)/chem_data/.git" ]; then
         rm -fr $(System.DefaultWorkingDirectory)/chem_data
         cd $(System.DefaultWorkingDirectory)
-        svn co svn+ssh://[email protected]/chem_data/trunk chem_data
+        git clone https://gitlab.com/phenix_project/chem_data.git
+        git lfs install --local
+        git lfs pull
       fi
       cd $(System.DefaultWorkingDirectory)/chem_data
-      svn relocate svn+ssh://[email protected]/chem_data/trunk
-      svn update
+      git reset --hard origin/main
+      git lfs install --local
+      git pull --rebase
+      git lfs pull
     displayName: Update chem_data
     continueOnError: true
 

.azure-pipelines/unix-conda-build.yml

@@ -84,7 +84,7 @@ steps:
 - script: |
     set -xe
     source $(Pipeline.Workspace)/miniforge/etc/profile.d/conda.sh
-    mamba update --all -y -n base
+    conda update --all -y -n base
   displayName: Update miniforge
 
 # create conda environment

.azure-pipelines/xfel/azure-pipelines-xfel-branch.yml

@@ -11,6 +11,7 @@ trigger:
     exclude:
       - master
       - main
+      - releases/*
 
 resources:
   pipelines:

.azure-pipelines/xfel/conda-linux.yml

@@ -49,15 +49,17 @@ jobs:
   # https://github.com/ApexAI/performance_test/blob/master/azure-pipelines.yml#L9-L17
 
   # centos setup
+  # https://serverfault.com/questions/1161816/mirrorlist-centos-org-no-longer-resolve
   - script: |
       set -xe
       /tmp/docker exec -t -u 0 ci-container \
-      sh -c "sed -i 's/mirrorlist=http/#mirrorlist=http/g' /etc/yum.repos.d/CentOS-Base.repo"
+      sh -c "sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo"
       /tmp/docker exec -t -u 0 ci-container \
-      sh -c "sed -i 's/#baseurl=http:\/\/mirror/baseurl=http:\/\/vault/g' /etc/yum.repos.d/CentOS-Base.repo"
-      cat /etc/yum.repos.d/CentOS-Base.repo
-    displayName: Modify yum repositories for CentOS 6
-    condition: eq('${{ parameters.version[0] }}', 6)
+      sh -c "sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo"
+      /tmp/docker exec -t -u 0 ci-container \
+      sh -c "sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo"
+    displayName: Modify yum repositories for CentOS 7
+    condition: eq('${{ parameters.distribution }}', 'centos')
 
   - script: |
       /tmp/docker exec -t -u 0 ci-container \
@@ -71,15 +73,11 @@ jobs:
     displayName: Install dependencies for CentOS
     condition: eq('${{ parameters.distribution }}', 'centos')
 
-  - script: |
-      sudo yum install -y python-argparse
-    displayName: Install argparse for CentOS 6
-    condition: eq('${{ parameters.version[0] }}', 6)
-
   - bash: |
       echo "##vso[task.setvariable variable=LC_ALL]C.utf8"
     displayName: Setup locale for CentOS 8
-    condition: eq('${{ parameters.version[0] }}', 8)
+    condition: or(eq('${{ parameters.version[0] }}', 8),
+                  eq('${{ parameters.version[0] }}', 9))
 
   # https://lists.openhpc.community/g/users/topic/openmpi_and_shared_memory/16489081?p=,,,20,0,0,0::recentpostdate%2Fsticky,,,20,2,0,16489081
   - script: |

.azure-pipelines/xfel/conda-osx.yml

@@ -92,10 +92,11 @@ jobs:
     failOnStderr: false
 
   - script: |
-      rm -f $(Pipeline.Workspace)/build/lib/scitbx_array_family_flex_ext.so
-    displayName: Fix broken cache on macOS 10.15 and 11
-    condition: or(and(eq('${{ parameters.vmImage[0] }}', 10), eq('${{ parameters.vmImage[1] }}', 15)),
-                  eq('${{ parameters.vmImage[0] }}', 11))
+      set -xe
+      for f in cctbx_geometry_restraints_ext.so scitbx_array_family_flex_ext.so ; do
+        rm -f $(Pipeline.Workspace)/build/lib/${f}
+      done
+    displayName: Fix broken cache on macOS
     continueOnError: true
     failOnStderr: false
 

.azure-pipelines/xfel/download-source.yml

@@ -13,12 +13,28 @@ jobs:
   - checkout: self
     path: ./modules/cctbx_project
 
+  # add conda to path
+  # https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/anaconda?view=azure-devops&tabs=ubuntu-16-04#add-conda-to-your-system-path
+  - bash: echo "##vso[task.prependpath]$CONDA/bin"
+    displayName: Add conda to PATH
+
+  # create environment with earlier version of git
+  # https://github.com/git-lfs/git-lfs/issues/5749
+  - script: |
+      set -xe
+      conda create -n git -c conda-forge -y git=2.44
+    displayName: Create environment with git
+    retryCountOnTaskFailure: 3
+
   - script: |
       cd $(Pipeline.Workspace)
       ln -s modules/cctbx_project/libtbx/auto_build/bootstrap.py
     displayName: Link bootstrap.py
 
   - script: |
+      set -xe
+      source $(CONDA)/etc/profile.d/conda.sh
+      conda activate git
       cd $(Pipeline.Workspace)
       python bootstrap.py hot update --builder=xfel --python=39 --no-boost-src
       cd modules
@@ -61,7 +77,9 @@ jobs:
   # preserve permissions and delete extra files
   - script: |
       cd $(Pipeline.Workspace)
+      mv modules/kokkos/.git modules/kokkos/.git.tmp
       rm -fr modules/*/.git/*
+      mv modules/kokkos/.git.tmp modules/kokkos/.git
       rm -fr modules/*/.svn/*
       rm -fr modules/*.tar
       tar -cf modules.tar modules

.azure-pipelines/xfel/unix-conda-build.yml

@@ -56,8 +56,9 @@ steps:
     source $(Pipeline.Workspace)/miniforge/etc/profile.d/conda.sh
     conda create -y -n $(PYTHON_VERSION) --file $(Pipeline.Workspace)/modules/dials/.conda-envs/$(OS).txt python=$(PYTHON_FULL_VERSION)
     conda install -y -c conda-forge --no-deps -n $(PYTHON_VERSION) junit-xml
-    conda install -y -c conda-forge cmake pandas distro -n $(PYTHON_VERSION)
+    conda install -y -c conda-forge cmake make pandas distro -n $(PYTHON_VERSION)
   displayName: Create conda environment
+  retryCountOnTaskFailure: 3
 
 - script: |
     set -xe
@@ -95,6 +96,7 @@ steps:
       libtbx.configure lunus
       make
     fi
+    rm -fr modules/kokkos/.git/*
   displayName: Configure and Build
 
 # test