Merge branch 'alan-turing-institute:develop' into platform-path

.github/scripts/update_dbeaver_drivers.py

@@ -4,30 +4,69 @@
 from natsort import natsorted
 import requests
 
-output = {}
-remote_page = requests.get("https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-output["mssql_jdbc"] = natsorted([v for v in root.xpath("//a[contains(text(), 'jre8/')]/@href") if v != "../"])[-1].replace("/", "")
-
-remote_page = requests.get("https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-output["pgjdbc"] = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])[-1].replace("/", "")
-
-remote_page = requests.get("https://repo1.maven.org/maven2/org/postgresql/postgresql/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-output["postgresql"] = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])[-1].replace("/", "")
-
-remote_page = requests.get("https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-postgis_jdbc_versions = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])
-
-remote_page = requests.get("https://repo1.maven.org/maven2/net/postgis/postgis-geometry/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-postgis_geometry_versions = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])
-
-postgis = natsorted(set(postgis_jdbc_versions).intersection(set(postgis_geometry_versions)))[-1].replace("/", "")
-output["postgis_geometry"] = postgis
-output["postgis_jdbc"] = postgis
+
+def get_latest_version(url, search_text):
+    """
+    Get latest version number of a database driver from the Maven repository.
+
+    Fetches the HTML page at the given URL, then converts it to an lxml tree.
+    Numeric strings are then extracted.
+    Note that mostly numeric strings for some drivers contain non-numeric text,
+    as different driver types exist for those drivers, even where the version number is the same.
+    The largest (latest) version number of the driver is then returned.
+
+    Parameters
+    ----------
+    url : str
+        The URL of the Maven repository containing the driver
+    search_text : str
+        Text to search for in the repository, to distinguish the driver from other files
+
+    Returns
+    -------
+    list
+        The latest available version number of the driver
+    """
+
+    remote_page = requests.get(url, allow_redirects=True)
+    root = html.fromstring(remote_page.content)
+    return natsorted([v for v in root.xpath("//a[contains(text(), '" + search_text + "')]/@href") if v != "../"])[-1].replace("/", "")
+
+
+drivers = [
+    {
+        'name': "mssql_jdbc",
+        'url': "https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/",
+        'search_text': "jre8/"
+    },
+    {
+        'name': "pgjdbc",
+        'url': "https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/",
+        'search_text': "/"
+    },
+    {
+        'name': "postgresql",
+        'url': "https://repo1.maven.org/maven2/org/postgresql/postgresql/",
+        'search_text': "/"
+    },
+    {
+        'name': "postgis_geometry",
+        'url': "https://repo1.maven.org/maven2/net/postgis/postgis-geometry/",
+        'search_text': "/"
+    },
+    {
+        'name': "postgis_jdbc",
+        'url': "https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/",
+        'search_text': "/"
+    },
+    {
+        'name': "waffle_jna",
+        'url': "https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/",
+        'search_text': "/"
+    }
+]
+
+output = {driver['name']: get_latest_version(driver['url'], driver['search_text']) for driver in drivers}
 
 with open("deployment/secure_research_desktop/packages/dbeaver-driver-versions.json", "w") as f_out:
     f_out.writelines(json.dumps(output, indent=4, sort_keys=True))

deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml

@@ -208,7 +208,7 @@ runcmd:
   - if [ "$(which azuredatastudio)" = "" ]; then echo "Could not install Azure Data Studio!"; exit 1; else echo "... successfully installed Azure Data Studio"; fi
   # Install drivers from maven
   - echo "Installing DBeaver drivers..."
-  - DBEAVER_DRIVER_DIR="/usr/share/dbeaver/drivers/maven/maven-central"
+  - DBEAVER_DRIVER_DIR="/usr/share/dbeaver-ce/drivers/maven/maven-central"
   # Note that the filenames specified here have to be kept synchronised with the names in the dbeaver_drivers_config.xml file.
   # Adding new drivers therefore involves changing both this file and the XML file.
   - mkdir -p ${DBEAVER_DRIVER_DIR}/com.microsoft.sqlserver/
@@ -224,9 +224,12 @@ runcmd:
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/{{dbeaver.drivers.postgis_jdbc}}/postgis-jdbc-{{dbeaver.drivers.postgis_jdbc}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.jar -P ${DBEAVER_DRIVER_DIR}/net.postgis/
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/
-  - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver/drivers-config.xml
-  - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver/drivers-config.xml" >> /usr/share/dbeaver/dbeaver.ini
-  - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver/dbeaver.ini
+  - mkdir -p ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.jar -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.pom -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver-ce/drivers-config.xml
+  - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver-ce/drivers-config.xml" >> /usr/share/dbeaver-ce/dbeaver.ini
+  - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver-ce/dbeaver.ini
   - ls -alh ${DBEAVER_DRIVER_DIR}/*
   - echo "... successfully installed DBeaver drivers";
   # Install RStudio

deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml

@@ -211,7 +211,7 @@ runcmd:
   - if [ "$(which azuredatastudio)" = "" ]; then echo "Could not install Azure Data Studio!"; exit 1; else echo "... successfully installed Azure Data Studio"; fi
   # Install drivers from maven
   - echo "Installing DBeaver drivers..."
-  - DBEAVER_DRIVER_DIR="/usr/share/dbeaver/drivers/maven/maven-central"
+  - DBEAVER_DRIVER_DIR="/usr/share/dbeaver-ce/drivers/maven/maven-central"
   # Note that the filenames specified here have to be kept synchronised with the names in the dbeaver_drivers_config.xml file.
   # Adding new drivers therefore involves changing both this file and the XML file.
   - mkdir -p ${DBEAVER_DRIVER_DIR}/com.microsoft.sqlserver/
@@ -227,9 +227,12 @@ runcmd:
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/{{dbeaver.drivers.postgis_jdbc}}/postgis-jdbc-{{dbeaver.drivers.postgis_jdbc}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.jar -P ${DBEAVER_DRIVER_DIR}/net.postgis/
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/
-  - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver/drivers-config.xml
-  - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver/drivers-config.xml" >> /usr/share/dbeaver/dbeaver.ini
-  - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver/dbeaver.ini
+  - mkdir -p ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.jar -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.pom -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver-ce/drivers-config.xml
+  - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver-ce/drivers-config.xml" >> /usr/share/dbeaver-ce/dbeaver.ini
+  - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver-ce/dbeaver.ini
   - ls -alh ${DBEAVER_DRIVER_DIR}/*
   - echo "... successfully installed DBeaver drivers";
   # Install RStudio

deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml

@@ -217,7 +217,7 @@ runcmd:
   - if [ "$(which azuredatastudio)" = "" ]; then echo "Could not install Azure Data Studio!"; exit 1; else echo "... successfully installed Azure Data Studio"; fi
   # Install drivers from maven
   - echo "Installing DBeaver drivers..."
-  - DBEAVER_DRIVER_DIR="/usr/share/dbeaver/drivers/maven/maven-central"
+  - DBEAVER_DRIVER_DIR="/usr/share/dbeaver-ce/drivers/maven/maven-central"
   # Note that the filenames specified here have to be kept synchronised with the names in the dbeaver_drivers_config.xml file.
   # Adding new drivers therefore involves changing both this file and the XML file.
   - mkdir -p ${DBEAVER_DRIVER_DIR}/com.microsoft.sqlserver/
@@ -233,9 +233,12 @@ runcmd:
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/{{dbeaver.drivers.postgis_jdbc}}/postgis-jdbc-{{dbeaver.drivers.postgis_jdbc}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.jar -P ${DBEAVER_DRIVER_DIR}/net.postgis/
   - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/
-  - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver/drivers-config.xml
-  - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver/drivers-config.xml" >> /usr/share/dbeaver/dbeaver.ini
-  - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver/dbeaver.ini
+  - mkdir -p ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.jar -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.pom -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/
+  - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver-ce/drivers-config.xml
+  - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver-ce/drivers-config.xml" >> /usr/share/dbeaver-ce/dbeaver.ini
+  - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver-ce/dbeaver.ini
   - ls -alh ${DBEAVER_DRIVER_DIR}/*
   - echo "... successfully installed DBeaver drivers";
   # Install RStudio

deployment/secure_research_desktop/cloud_init/resources/dbeaver_drivers_config.mustache.xml

@@ -3,20 +3,23 @@
     <provider id="sqlserver">
         <driver id="microsoft" category="MS SQL Server" categories="sql" custom="false" embedded="false" name="SQL Server" class="com.microsoft.sqlserver.jdbc.SQLServerDriver" url="jdbc:sqlserver://{host}[:{port}][;databaseName={database}]" port="1433" description="Microsoft JDBC Driver for SQL Server (MSSQL)">
             <library type="jar" path="maven:/com.microsoft.sqlserver:mssql-jdbc:RELEASE" custom="false" version="{{dbeaver.drivers.mssql_jdbc}}">
-                <file id="com.microsoft.sqlserver:mssql-jdbc" version="{{dbeaver.drivers.mssql_jdbc}}" path="/usr/share/dbeaver/drivers/maven/maven-central/com.microsoft.sqlserver/mssql-jdbc-{{dbeaver.drivers.mssql_jdbc}}.jar"/>
+                <file id="com.microsoft.sqlserver:mssql-jdbc" version="{{dbeaver.drivers.mssql_jdbc}}" path="/usr/share/dbeaver-ce/drivers/maven/maven-central/com.microsoft.sqlserver/mssql-jdbc-{{dbeaver.drivers.mssql_jdbc}}.jar"/>
             </library>
         </driver>
     </provider>
     <provider id="postgresql">
         <driver id="postgres-jdbc" categories="sql" custom="false" embedded="false" name="PostgreSQL" class="org.postgresql.Driver" url="jdbc:postgresql://{host}[:{port}]/[{database}]" port="5432" description="PostgreSQL standard driver">
-            <library type="jar" path="maven:/org.postgresql:postgresql:RELEASE" custom="false" version="{{dbeaver.drivers.postgresl}}">
-                <file id="org.postgresql:postgresql" version="{{dbeaver.drivers.postgresl}}" path="/usr/share/dbeaver/drivers/maven/maven-central/org.postgresql/postgresql-{{dbeaver.drivers.postgresl}}.jar"/>
+            <library type="jar" path="maven:/org.postgresql:postgresql:RELEASE" custom="false" version="{{dbeaver.drivers.postgresql}}">
+                <file id="org.postgresql:postgresql" version="{{dbeaver.drivers.postgresql}}" path="/usr/share/dbeaver-ce/drivers/maven/maven-central/org.postgresql/postgresql-{{dbeaver.drivers.postgresql}}.jar"/>
             </library>
             <library type="jar" path="maven:/net.postgis:postgis-jdbc:RELEASE" custom="false" version="{{dbeaver.drivers.postgis_jdbc}}">
-                <file id="net.postgis:postgis-jdbc" version="{{dbeaver.drivers.postgis_jdbc}}" path="/usr/share/dbeaver/drivers/maven/maven-central/net.postgis/postgis-jdbc-{{dbeaver.drivers.postgis_jdbc}}.jar"/>
+                <file id="net.postgis:postgis-jdbc" version="{{dbeaver.drivers.postgis_jdbc}}" path="/usr/share/dbeaver-ce/drivers/maven/maven-central/net.postgis/postgis-jdbc-{{dbeaver.drivers.postgis_jdbc}}.jar"/>
             </library>
             <library type="jar" path="maven:/net.postgis:postgis-geometry:RELEASE" custom="false" version="{{dbeaver.drivers.postgis_geometry}}">
-                <file id="net.postgis:postgis-geometry" version="{{dbeaver.drivers.postgis_geometry}}" path="/usr/share/dbeaver/drivers/maven/maven-central/net.postgis/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.jar"/>
+                <file id="net.postgis:postgis-geometry" version="{{dbeaver.drivers.postgis_geometry}}" path="/usr/share/dbeaver-ce/drivers/maven/maven-central/net.postgis/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.jar"/>
+            </library>
+            <library type="jar" path="maven:/com.github.waffle:waffle-jna:RELEASE" custom="false" version="{{dbeaver.drivers.waffle_jna}}">
+                <file id="com.github.waffle:waffle-jna" version="{{dbeaver.drivers.waffle_jna}}" path="/usr/share/dbeaver-ce/drivers/maven/maven-central/com.github.waffle/waffle-jna-{{dbeaver.drivers.waffle_jna}}.jar"/>
             </library>
         </driver>
     </provider>

deployment/secure_research_desktop/packages/dbeaver-driver-versions.json

@@ -3,5 +3,6 @@
     "pgjdbc": "1.1.6",
     "postgis_geometry": "2023.1.0",
     "postgis_jdbc": "2023.1.0",
-    "postgresql": "42.7.1"
+    "postgresql": "42.7.1",
+    "waffle_jna": "3.3.0"
 }

docs/source/roles/researcher/user_guide.md

@@ -1002,6 +1002,18 @@ Our example user Ada Lovelace, working in the `sandbox` SRE on the `projects.tur
 Be sure to select `Kerberos authentication` so that your username and password will be passed through to the database
 ```
 
+````{note}
+After clicking finish, you may be prompted to download missing driver files.
+Drivers have already been provided on the SRD for Microsoft SQL databases.
+Clicking `Download` will make DBeaver use these pre-downloaded drivers without requiring internet access.
+Thus, even on SRDs with no external internet access (Tier 2 or above), click `Download`.
+Note that the prompt may appear multiple times.
+```{image} user_guide/db_dbeaver_mssql_download.png
+:alt: DBeaver driver download for Microsoft SQL
+:align: center
+```
+````
+
 #### PostgreSQL
 
 - Select `PostgreSQL` as the database type
@@ -1028,6 +1040,18 @@ If you are prompted for `Username` or `Password` when connecting, you can leave
 ```
 ````
 
+````{note}
+After clicking finish, you may be prompted to download missing driver files.
+Drivers have already been provided on the SRD for PostgreSQL databases.
+Clicking `Download` will make DBeaver use these pre-downloaded drivers without requiring internet access.
+Thus, even on SRDs with no external internet access (Tier 2 or above), click `Download`.
+Note that the prompt may appear multiple times.
+```{image} user_guide/db_dbeaver_pstgrs_download.png
+:alt: DBeaver driver download for Microsoft SQL
+:align: center
+```
+````
+
 ### {{snake}} Connecting using Python
 
 Database connections can be made using `pyodbc` or `psycopg2` depending on which database flavour is being used.

environment_configs/sre_bluet1guac_core_config.json

@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.253"],

environment_configs/sre_bluet2guac_core_config.json

@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.253"],

environment_configs/sre_bluet3guac_core_config.json

@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.240"],

environment_configs/sre_greent1guac_core_config.json

@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.253"],

environment_configs/sre_greent2guac_core_config.json

@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.253"],

environment_configs/sre_greent3guac_core_config.json

@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.240"],

tests/resources/sre_bluet1guac_full_config.json

@@ -1291,7 +1291,7 @@
       "rg": "RG_SHM_BLUE_SRE_T1GUAC_COMPUTE",
       "vmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
       },
       "vmSizeDefault": "Standard_D2s_v3"
     },

tests/resources/sre_bluet3guac_full_config.json

@@ -1291,7 +1291,7 @@
       "rg": "RG_SHM_BLUE_SRE_T3GUAC_COMPUTE",
       "vmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
       },
       "vmSizeDefault": "Standard_D2s_v3"
     },

tests/resources/sre_greent2guac_full_config.json

@@ -1344,7 +1344,7 @@
       "rg": "RG_SHM_GREEN_SRE_T2GUAC_COMPUTE",
       "vmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023082900"
+        "version": "20.04.2024011200"
       },
       "vmSizeDefault": "Standard_D2s_v3"
     },