diff --git a/.github/scripts/update_dbeaver_drivers.py b/.github/scripts/update_dbeaver_drivers.py index c39a20d909..696a501858 100644 --- a/.github/scripts/update_dbeaver_drivers.py +++ b/.github/scripts/update_dbeaver_drivers.py @@ -4,30 +4,69 @@ from natsort import natsorted import requests -output = {} -remote_page = requests.get("https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/", allow_redirects=True) -root = html.fromstring(remote_page.content) -output["mssql_jdbc"] = natsorted([v for v in root.xpath("//a[contains(text(), 'jre8/')]/@href") if v != "../"])[-1].replace("/", "") - -remote_page = requests.get("https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/", allow_redirects=True) -root = html.fromstring(remote_page.content) -output["pgjdbc"] = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])[-1].replace("/", "") - -remote_page = requests.get("https://repo1.maven.org/maven2/org/postgresql/postgresql/", allow_redirects=True) -root = html.fromstring(remote_page.content) -output["postgresql"] = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])[-1].replace("/", "") - -remote_page = requests.get("https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/", allow_redirects=True) -root = html.fromstring(remote_page.content) -postgis_jdbc_versions = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"]) - -remote_page = requests.get("https://repo1.maven.org/maven2/net/postgis/postgis-geometry/", allow_redirects=True) -root = html.fromstring(remote_page.content) -postgis_geometry_versions = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"]) - -postgis = natsorted(set(postgis_jdbc_versions).intersection(set(postgis_geometry_versions)))[-1].replace("/", "") -output["postgis_geometry"] = postgis -output["postgis_jdbc"] = postgis + +def get_latest_version(url, search_text): + """ + Get latest version number of a database driver from the Maven repository. + + Fetches the HTML page at the given URL, then converts it to an lxml tree. + Numeric strings are then extracted. + Note that mostly numeric strings for some drivers contain non-numeric text, + as different driver types exist for those drivers, even where the version number is the same. + The largest (latest) version number of the driver is then returned. + + Parameters + ---------- + url : str + The URL of the Maven repository containing the driver + search_text : str + Text to search for in the repository, to distinguish the driver from other files + + Returns + ------- + list + The latest available version number of the driver + """ + + remote_page = requests.get(url, allow_redirects=True) + root = html.fromstring(remote_page.content) + return natsorted([v for v in root.xpath("//a[contains(text(), '" + search_text + "')]/@href") if v != "../"])[-1].replace("/", "") + + +drivers = [ + { + 'name': "mssql_jdbc", + 'url': "https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/", + 'search_text': "jre8/" + }, + { + 'name': "pgjdbc", + 'url': "https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/", + 'search_text': "/" + }, + { + 'name': "postgresql", + 'url': "https://repo1.maven.org/maven2/org/postgresql/postgresql/", + 'search_text': "/" + }, + { + 'name': "postgis_geometry", + 'url': "https://repo1.maven.org/maven2/net/postgis/postgis-geometry/", + 'search_text': "/" + }, + { + 'name': "postgis_jdbc", + 'url': "https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/", + 'search_text': "/" + }, + { + 'name': "waffle_jna", + 'url': "https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/", + 'search_text': "/" + } +] + +output = {driver['name']: get_latest_version(driver['url'], driver['search_text']) for driver in drivers} with open("deployment/secure_research_desktop/packages/dbeaver-driver-versions.json", "w") as f_out: f_out.writelines(json.dumps(output, indent=4, sort_keys=True)) diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml index ce9b2a0630..129fa470ea 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml @@ -208,7 +208,7 @@ runcmd: - if [ "$(which azuredatastudio)" = "" ]; then echo "Could not install Azure Data Studio!"; exit 1; else echo "... successfully installed Azure Data Studio"; fi # Install drivers from maven - echo "Installing DBeaver drivers..." - - DBEAVER_DRIVER_DIR="/usr/share/dbeaver/drivers/maven/maven-central" + - DBEAVER_DRIVER_DIR="/usr/share/dbeaver-ce/drivers/maven/maven-central" # Note that the filenames specified here have to be kept synchronised with the names in the dbeaver_drivers_config.xml file. # Adding new drivers therefore involves changing both this file and the XML file. - mkdir -p ${DBEAVER_DRIVER_DIR}/com.microsoft.sqlserver/ @@ -224,9 +224,12 @@ runcmd: - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/{{dbeaver.drivers.postgis_jdbc}}/postgis-jdbc-{{dbeaver.drivers.postgis_jdbc}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.jar -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver/drivers-config.xml - - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver/drivers-config.xml" >> /usr/share/dbeaver/dbeaver.ini - - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver/dbeaver.ini + - mkdir -p ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.jar -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.pom -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver-ce/drivers-config.xml + - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver-ce/drivers-config.xml" >> /usr/share/dbeaver-ce/dbeaver.ini + - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver-ce/dbeaver.ini - ls -alh ${DBEAVER_DRIVER_DIR}/* - echo "... successfully installed DBeaver drivers"; # Install RStudio diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml index c1d5a53c2e..79a5389997 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml @@ -211,7 +211,7 @@ runcmd: - if [ "$(which azuredatastudio)" = "" ]; then echo "Could not install Azure Data Studio!"; exit 1; else echo "... successfully installed Azure Data Studio"; fi # Install drivers from maven - echo "Installing DBeaver drivers..." - - DBEAVER_DRIVER_DIR="/usr/share/dbeaver/drivers/maven/maven-central" + - DBEAVER_DRIVER_DIR="/usr/share/dbeaver-ce/drivers/maven/maven-central" # Note that the filenames specified here have to be kept synchronised with the names in the dbeaver_drivers_config.xml file. # Adding new drivers therefore involves changing both this file and the XML file. - mkdir -p ${DBEAVER_DRIVER_DIR}/com.microsoft.sqlserver/ @@ -227,9 +227,12 @@ runcmd: - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/{{dbeaver.drivers.postgis_jdbc}}/postgis-jdbc-{{dbeaver.drivers.postgis_jdbc}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.jar -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver/drivers-config.xml - - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver/drivers-config.xml" >> /usr/share/dbeaver/dbeaver.ini - - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver/dbeaver.ini + - mkdir -p ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.jar -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.pom -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver-ce/drivers-config.xml + - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver-ce/drivers-config.xml" >> /usr/share/dbeaver-ce/dbeaver.ini + - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver-ce/dbeaver.ini - ls -alh ${DBEAVER_DRIVER_DIR}/* - echo "... successfully installed DBeaver drivers"; # Install RStudio diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml index 5cfcc671e7..3cc36ee4f3 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml @@ -217,7 +217,7 @@ runcmd: - if [ "$(which azuredatastudio)" = "" ]; then echo "Could not install Azure Data Studio!"; exit 1; else echo "... successfully installed Azure Data Studio"; fi # Install drivers from maven - echo "Installing DBeaver drivers..." - - DBEAVER_DRIVER_DIR="/usr/share/dbeaver/drivers/maven/maven-central" + - DBEAVER_DRIVER_DIR="/usr/share/dbeaver-ce/drivers/maven/maven-central" # Note that the filenames specified here have to be kept synchronised with the names in the dbeaver_drivers_config.xml file. # Adding new drivers therefore involves changing both this file and the XML file. - mkdir -p ${DBEAVER_DRIVER_DIR}/com.microsoft.sqlserver/ @@ -233,9 +233,12 @@ runcmd: - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/{{dbeaver.drivers.postgis_jdbc}}/postgis-jdbc-{{dbeaver.drivers.postgis_jdbc}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.jar -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - wget -nv https://repo1.maven.org/maven2/net/postgis/postgis-geometry/{{dbeaver.drivers.postgis_geometry}}/postgis-geometry-{{dbeaver.drivers.postgis_geometry}}.pom -P ${DBEAVER_DRIVER_DIR}/net.postgis/ - - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver/drivers-config.xml - - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver/drivers-config.xml" >> /usr/share/dbeaver/dbeaver.ini - - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver/dbeaver.ini + - mkdir -p ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.jar -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - wget -nv https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/{{dbeaver.drivers.waffle_jna}}/waffle-jna-{{dbeaver.drivers.waffle_jna}}.pom -P ${DBEAVER_DRIVER_DIR}/com.github.waffle/ + - mv /opt/build/dbeaver_drivers_config.xml /usr/share/dbeaver-ce/drivers-config.xml + - echo "-Ddbeaver.drivers.configuration-file=/usr/share/dbeaver-ce/drivers-config.xml" >> /usr/share/dbeaver-ce/dbeaver.ini + - echo "-Djava.security.auth.login.config=/etc/jaas.conf" >> /usr/share/dbeaver-ce/dbeaver.ini - ls -alh ${DBEAVER_DRIVER_DIR}/* - echo "... successfully installed DBeaver drivers"; # Install RStudio diff --git a/deployment/secure_research_desktop/cloud_init/resources/dbeaver_drivers_config.mustache.xml b/deployment/secure_research_desktop/cloud_init/resources/dbeaver_drivers_config.mustache.xml index d1309329f7..7ce425aea4 100644 --- a/deployment/secure_research_desktop/cloud_init/resources/dbeaver_drivers_config.mustache.xml +++ b/deployment/secure_research_desktop/cloud_init/resources/dbeaver_drivers_config.mustache.xml @@ -3,20 +3,23 @@ - + - - + + - + - + + + + diff --git a/deployment/secure_research_desktop/packages/dbeaver-driver-versions.json b/deployment/secure_research_desktop/packages/dbeaver-driver-versions.json index 8485bbc2b1..a865bf9060 100644 --- a/deployment/secure_research_desktop/packages/dbeaver-driver-versions.json +++ b/deployment/secure_research_desktop/packages/dbeaver-driver-versions.json @@ -3,5 +3,6 @@ "pgjdbc": "1.1.6", "postgis_geometry": "2023.1.0", "postgis_jdbc": "2023.1.0", - "postgresql": "42.7.1" + "postgresql": "42.7.1", + "waffle_jna": "3.3.0" } \ No newline at end of file diff --git a/docs/source/roles/researcher/user_guide.md b/docs/source/roles/researcher/user_guide.md index 0daa143896..fe9a1af399 100644 --- a/docs/source/roles/researcher/user_guide.md +++ b/docs/source/roles/researcher/user_guide.md @@ -1002,6 +1002,18 @@ Our example user Ada Lovelace, working in the `sandbox` SRE on the `projects.tur Be sure to select `Kerberos authentication` so that your username and password will be passed through to the database ``` +````{note} +After clicking finish, you may be prompted to download missing driver files. +Drivers have already been provided on the SRD for Microsoft SQL databases. +Clicking `Download` will make DBeaver use these pre-downloaded drivers without requiring internet access. +Thus, even on SRDs with no external internet access (Tier 2 or above), click `Download`. +Note that the prompt may appear multiple times. +```{image} user_guide/db_dbeaver_mssql_download.png +:alt: DBeaver driver download for Microsoft SQL +:align: center +``` +```` + #### PostgreSQL - Select `PostgreSQL` as the database type @@ -1028,6 +1040,18 @@ If you are prompted for `Username` or `Password` when connecting, you can leave ``` ```` +````{note} +After clicking finish, you may be prompted to download missing driver files. +Drivers have already been provided on the SRD for PostgreSQL databases. +Clicking `Download` will make DBeaver use these pre-downloaded drivers without requiring internet access. +Thus, even on SRDs with no external internet access (Tier 2 or above), click `Download`. +Note that the prompt may appear multiple times. +```{image} user_guide/db_dbeaver_pstgrs_download.png +:alt: DBeaver driver download for Microsoft SQL +:align: center +``` +```` + ### {{snake}} Connecting using Python Database connections can be made using `pyodbc` or `psycopg2` depending on which database flavour is being used. diff --git a/docs/source/roles/researcher/user_guide/db_dbeaver_mssql_download.png b/docs/source/roles/researcher/user_guide/db_dbeaver_mssql_download.png new file mode 100644 index 0000000000..a2225657b1 Binary files /dev/null and b/docs/source/roles/researcher/user_guide/db_dbeaver_mssql_download.png differ diff --git a/docs/source/roles/researcher/user_guide/db_dbeaver_pstgrs_download.png b/docs/source/roles/researcher/user_guide/db_dbeaver_pstgrs_download.png new file mode 100644 index 0000000000..ecd00eadad Binary files /dev/null and b/docs/source/roles/researcher/user_guide/db_dbeaver_pstgrs_download.png differ diff --git a/environment_configs/sre_bluet1guac_core_config.json b/environment_configs/sre_bluet1guac_core_config.json index 365ae14a8c..f985b5e7a6 100644 --- a/environment_configs/sre_bluet1guac_core_config.json +++ b/environment_configs/sre_bluet1guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_bluet2guac_core_config.json b/environment_configs/sre_bluet2guac_core_config.json index 15d96d2cda..decc4758b1 100644 --- a/environment_configs/sre_bluet2guac_core_config.json +++ b/environment_configs/sre_bluet2guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_bluet3guac_core_config.json b/environment_configs/sre_bluet3guac_core_config.json index eff5d1e24c..2f7ba41cf1 100644 --- a/environment_configs/sre_bluet3guac_core_config.json +++ b/environment_configs/sre_bluet3guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.240"], diff --git a/environment_configs/sre_greent1guac_core_config.json b/environment_configs/sre_greent1guac_core_config.json index 10cad425c3..eef9a7f902 100644 --- a/environment_configs/sre_greent1guac_core_config.json +++ b/environment_configs/sre_greent1guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_greent2guac_core_config.json b/environment_configs/sre_greent2guac_core_config.json index 091a74e605..950131ee4d 100644 --- a/environment_configs/sre_greent2guac_core_config.json +++ b/environment_configs/sre_greent2guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_greent3guac_core_config.json b/environment_configs/sre_greent3guac_core_config.json index a3c0c4c57e..b4e905d9d4 100644 --- a/environment_configs/sre_greent3guac_core_config.json +++ b/environment_configs/sre_greent3guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.240"], diff --git a/tests/resources/sre_bluet1guac_full_config.json b/tests/resources/sre_bluet1guac_full_config.json index 8ee042161e..cf8466d39a 100644 --- a/tests/resources/sre_bluet1guac_full_config.json +++ b/tests/resources/sre_bluet1guac_full_config.json @@ -1291,7 +1291,7 @@ "rg": "RG_SHM_BLUE_SRE_T1GUAC_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "vmSizeDefault": "Standard_D2s_v3" }, diff --git a/tests/resources/sre_bluet3guac_full_config.json b/tests/resources/sre_bluet3guac_full_config.json index a2f21eb68a..88a42815bd 100644 --- a/tests/resources/sre_bluet3guac_full_config.json +++ b/tests/resources/sre_bluet3guac_full_config.json @@ -1291,7 +1291,7 @@ "rg": "RG_SHM_BLUE_SRE_T3GUAC_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "vmSizeDefault": "Standard_D2s_v3" }, diff --git a/tests/resources/sre_greent2guac_full_config.json b/tests/resources/sre_greent2guac_full_config.json index 165e24acda..7d336095ed 100644 --- a/tests/resources/sre_greent2guac_full_config.json +++ b/tests/resources/sre_greent2guac_full_config.json @@ -1344,7 +1344,7 @@ "rg": "RG_SHM_GREEN_SRE_T2GUAC_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023082900" + "version": "20.04.2024011200" }, "vmSizeDefault": "Standard_D2s_v3" },