From cd2db573b6c12c478ab8415ca1a225a3a7a74b74 Mon Sep 17 00:00:00 2001 From: Michael Skarbek Date: Tue, 23 Apr 2024 10:43:37 -0400 Subject: [PATCH] update trino and hive images for local dev (#4717) --- .../trino/etc/catalog/hive.properties | 7 ------- .../trino/etc/catalog/postgres.properties | 0 .../containers}/trino/etc/config.properties | 0 .../trino/etc}/hadoop-config/core-site.xml | 0 {deploy => dev/containers}/trino/etc/jvm.config | 7 ++++++- .../containers}/trino/etc/log.properties | 0 docker-compose.arm.yml | 4 ++-- docker-compose.yml | 16 ++++++++-------- 8 files changed, 16 insertions(+), 18 deletions(-) rename {deploy => dev/containers}/trino/etc/catalog/hive.properties (76%) rename {deploy => dev/containers}/trino/etc/catalog/postgres.properties (100%) rename {deploy => dev/containers}/trino/etc/config.properties (100%) rename {deploy/hadoop => dev/containers/trino/etc}/hadoop-config/core-site.xml (100%) rename {deploy => dev/containers}/trino/etc/jvm.config (70%) rename {deploy => dev/containers}/trino/etc/log.properties (100%) diff --git a/deploy/trino/etc/catalog/hive.properties b/dev/containers/trino/etc/catalog/hive.properties similarity index 76% rename from deploy/trino/etc/catalog/hive.properties rename to dev/containers/trino/etc/catalog/hive.properties index 92f98b0309..4d80a06a81 100644 --- a/deploy/trino/etc/catalog/hive.properties +++ b/dev/containers/trino/etc/catalog/hive.properties @@ -1,9 +1,4 @@ connector.name=hive -hive.allow-drop-table=true -hive.allow-rename-table=true -hive.allow-add-column=true -hive.allow-drop-column=true -hive.allow-rename-column=true hive.storage-format=Parquet hive.compression-codec=SNAPPY hive.hdfs.authentication.type=NONE @@ -20,5 +15,3 @@ hive.s3.aws-secret-key=${ENV:AWS_SECRET_ACCESS_KEY} hive.s3.endpoint=${ENV:S3_ENDPOINT} hive.s3.ssl.enabled=false hive.s3.path-style-access=true - -parquet.optimized-reader.enabled=false diff --git a/deploy/trino/etc/catalog/postgres.properties b/dev/containers/trino/etc/catalog/postgres.properties similarity index 100% rename from deploy/trino/etc/catalog/postgres.properties rename to dev/containers/trino/etc/catalog/postgres.properties diff --git a/deploy/trino/etc/config.properties b/dev/containers/trino/etc/config.properties similarity index 100% rename from deploy/trino/etc/config.properties rename to dev/containers/trino/etc/config.properties diff --git a/deploy/hadoop/hadoop-config/core-site.xml b/dev/containers/trino/etc/hadoop-config/core-site.xml similarity index 100% rename from deploy/hadoop/hadoop-config/core-site.xml rename to dev/containers/trino/etc/hadoop-config/core-site.xml diff --git a/deploy/trino/etc/jvm.config b/dev/containers/trino/etc/jvm.config similarity index 70% rename from deploy/trino/etc/jvm.config rename to dev/containers/trino/etc/jvm.config index bbb1c92ba0..f26cc56d85 100644 --- a/deploy/trino/etc/jvm.config +++ b/dev/containers/trino/etc/jvm.config @@ -1,7 +1,6 @@ -server -XX:+UseContainerSupport -XX:+UseG1GC --XX:+UseGCOverheadLimit -XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=75.0 -XX:+ExplicitGCInvokesConcurrent @@ -17,4 +16,10 @@ -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=10000 -Dcom.sun.management.jmxremote.rmi.port=10000 +-Dfile.encoding=UTF-8 -Djava.rmi.server.hostname=127.0.0.1 +# Reduce starvation of threads by GClocker, recommend to set about the number of cpu cores (JDK-8192647) +-XX:+UnlockDiagnosticVMOptions +-XX:GCLockerRetryAllocationCount=8 +# Allow loading dynamic agent used by JOL +-XX:+EnableDynamicAgentLoading diff --git a/deploy/trino/etc/log.properties b/dev/containers/trino/etc/log.properties similarity index 100% rename from deploy/trino/etc/log.properties rename to dev/containers/trino/etc/log.properties diff --git a/docker-compose.arm.yml b/docker-compose.arm.yml index 2c18a2edd7..4b98207136 100644 --- a/docker-compose.arm.yml +++ b/docker-compose.arm.yml @@ -4,7 +4,7 @@ version: '3' services: hive-metastore: - image: quay.io/mskarbek/ubi-hive:3.1.2-metastore-008 + image: quay.io/mskarbek/ubi-hive:3.1.3-metastore-024 trino: - image: quay.io/mskarbek/ubi-trino:411-001 + image: quay.io/mskarbek/ubi-trino:443-002 diff --git a/docker-compose.yml b/docker-compose.yml index 8df3a78f8e..6d0242bfc3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -614,7 +614,7 @@ services: hive-metastore: container_name: hive-metastore - image: quay.io/cloudservices/ubi-hive:3.1.2-metastore-008 + image: quay.io/cloudservices/ubi-hive:3.1.3-metastore-024 ports: - 9083:8000 environment: @@ -637,7 +637,7 @@ services: trino: container_name: trino - image: quay.io/cloudservices/ubi-trino:411-002 + image: quay.io/cloudservices/ubi-trino:443-002 user: root ports: - 8080:8080 @@ -656,12 +656,12 @@ services: - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} volumes: - - ./deploy/trino/etc/config.properties:/etc/trino/config.properties - - ./deploy/trino/etc/jvm.config:/etc/trino/jvm.config - - ./deploy/trino/etc/log.properties:/etc/trino/log.properties - - ./deploy/trino/etc/catalog/hive.properties:/etc/trino/catalog/hive.properties - - ./deploy/trino/etc/catalog/postgres.properties:/etc/trino/catalog/postgres.properties - - ./deploy/hadoop/hadoop-config/core-site.xml:/etc/trino/hadoop-config/core-site.xml + - ./dev/containers/trino/etc/config.properties:/etc/trino/config.properties + - ./dev/containers/trino/etc/jvm.config:/etc/trino/jvm.config + - ./dev/containers/trino/etc/log.properties:/etc/trino/log.properties + - ./dev/containers/trino/etc/catalog/hive.properties:/etc/trino/catalog/hive.properties + - ./dev/containers/trino/etc/catalog/postgres.properties:/etc/trino/catalog/postgres.properties + - ./dev/containers/trino/etc/hadoop-config/core-site.xml:/etc/trino/hadoop-config/core-site.xml - ./.trino/trino/trino-data:/data/trino/data - ./.trino/trino/trino-logs:/data/trino/logs links: