From edcffc0c3a3f8c81ddf2ce8f95b95ffa805b918e Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Mon, 28 Oct 2024 23:52:29 +0100 Subject: [PATCH 01/12] ci: try automatically re-running tests --- .circleci/config.yml | 21 +++++++++++++++++---- .circleci/doTests.sh | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 236eabffa..3c1695d23 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,17 +11,30 @@ jobs: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: root resource_class: large + parallelism: 4 parameters: plugin: type: string steps: - checkout + - run: mkdir ~/junit - run: echo $'\n[mysqld]\ncharacter_set_server=utf8mb4\nmax_connections=10000' >> /etc/mysql/mysql.cnf - run: apt-get update && apt-get -y -q install postgresql-9.5 postgresql-client-9.5 postgresql-contrib-9.5 sudo - run: echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/9.5/main/pg_hba.conf - run: echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf - run: sed -i 's/^#*\s*max_connections\s*=.*/max_connections = 10000/' /etc/postgresql/9.5/main/postgresql.conf - run: (cd .circleci/ && ./doTests.sh << parameters.plugin >>) + - run: + command: cp ~/supertokens-root/supertokens-core/build/test-results/test/*.xml ~/junit/ + when: always + - when: + condition: + not: + equal: [ << parameters.plugin >>, "sqlite" ] + steps: + - run: cp ~/supertokens-root/supertokens-<< parameters.plugin >>-plugin/build/test-results/test/*.xml ~/junit/ + - store_test_results: + path: ~/junit - slack/status mark-passed: @@ -45,7 +58,7 @@ workflows: tags: only: /dev-v[0-9]+(\.[0-9]+)*/ branches: - ignore: /.*/ + only: /test-cicd\/.*/ - test: plugin: mongodb name: test-mongodb @@ -55,7 +68,7 @@ workflows: tags: only: /dev-v[0-9]+(\.[0-9]+)*/ branches: - ignore: /.*/ + only: /test-cicd\/.*/ - test: plugin: postgresql name: test-postgresql @@ -65,7 +78,7 @@ workflows: tags: only: /dev-v[0-9]+(\.[0-9]+)*/ branches: - ignore: /.*/ + only: /test-cicd\/.*/ - test: plugin: mysql name: test-mysql @@ -75,7 +88,7 @@ workflows: tags: only: /dev-v[0-9]+(\.[0-9]+)*/ branches: - ignore: /.*/ + only: /test-cicd\/.*/ - mark-passed: context: - slack-notification diff --git a/.circleci/doTests.sh b/.circleci/doTests.sh index 3fb117cc5..dba0669be 100755 --- a/.circleci/doTests.sh +++ b/.circleci/doTests.sh @@ -162,6 +162,24 @@ do fi cd ../ echo $SUPERTOKENS_API_KEY > apiPassword + + # Get list of classnames of tests that should run on this node. + circleci tests glob "supertokens-*/src/test/**/*Test.java" | cut -c 1- | sed 's@/@.@g' | sed 's/.\{5\}$//' | + circleci tests run --command=">classnames.txt xargs echo" --verbose --split-by=timings --timings-type=classname + + echo "!!!!! CLASSNAMES" + cat classnames.txt + echo "----- CLASSNAMES" + + + #if this is a re-run and it is a parallel run that does not have tests to run, halt execution of this parallel run + [ -s classnames.txt ] || circleci-agent step halt + + GRADLE_ARGS=$(cat src/test/java/classnames.txt | awk '{for (i=1; i<=NF; i++) print "--tests",$i}') + echo "Prepared arguments for Gradle: $GRADLE_ARGS" + + # TODO: remove before merging, this is here to make testing the scripts quicker + sed -i -e "s/\.\/gradlew test/\.\/gradlew --tests \*Config\*/" startTestEnv ./startTestingEnv --cicd if [[ $? -ne 0 ]] From a8c201130eb946a1258c4f889987a37511817053 Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Mon, 28 Oct 2024 23:54:33 +0100 Subject: [PATCH 02/12] ci: try automatically re-running tests --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c1695d23..91e7f2a2c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,8 +25,8 @@ jobs: - run: sed -i 's/^#*\s*max_connections\s*=.*/max_connections = 10000/' /etc/postgresql/9.5/main/postgresql.conf - run: (cd .circleci/ && ./doTests.sh << parameters.plugin >>) - run: - command: cp ~/supertokens-root/supertokens-core/build/test-results/test/*.xml ~/junit/ - when: always + command: cp ~/supertokens-root/supertokens-core/build/test-results/test/*.xml ~/junit/ + when: always - when: condition: not: From d0131bd8e9de975b06c609dcd62001316583374c Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Mon, 28 Oct 2024 23:56:20 +0100 Subject: [PATCH 03/12] ci: try automatically re-running tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 91e7f2a2c..bcf49b4f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: steps: - run: cp ~/supertokens-root/supertokens-<< parameters.plugin >>-plugin/build/test-results/test/*.xml ~/junit/ - store_test_results: - path: ~/junit + path: ~/junit - slack/status mark-passed: From 173f1c75665d59125cd51d8131733778dba30cb4 Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 00:10:38 +0100 Subject: [PATCH 04/12] ci: try automatically re-running tests --- .circleci/doTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/doTests.sh b/.circleci/doTests.sh index dba0669be..cde56b142 100755 --- a/.circleci/doTests.sh +++ b/.circleci/doTests.sh @@ -179,7 +179,7 @@ do echo "Prepared arguments for Gradle: $GRADLE_ARGS" # TODO: remove before merging, this is here to make testing the scripts quicker - sed -i -e "s/\.\/gradlew test/\.\/gradlew --tests \*Config\*/" startTestEnv + sed -i -e "s/\.\/gradlew test/\.\/gradlew test --tests \*Config\*/" startTestEnv ./startTestingEnv --cicd if [[ $? -ne 0 ]] From c9a9cd7bd7ee6b5997af32a1871418e1a484a8dc Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 00:18:29 +0100 Subject: [PATCH 05/12] ci: try automatically re-running tests --- .circleci/doTests.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.circleci/doTests.sh b/.circleci/doTests.sh index cde56b142..761026309 100755 --- a/.circleci/doTests.sh +++ b/.circleci/doTests.sh @@ -164,18 +164,13 @@ do echo $SUPERTOKENS_API_KEY > apiPassword # Get list of classnames of tests that should run on this node. - circleci tests glob "supertokens-*/src/test/**/*Test.java" | cut -c 1- | sed 's@/@.@g' | sed 's/.\{5\}$//' | + circleci tests glob "supertokens-*/src/test/**/*.java" | cut -c 1- | sed 's@/@.@g' | sed 's/.\{5\}$//' | circleci tests run --command=">classnames.txt xargs echo" --verbose --split-by=timings --timings-type=classname - echo "!!!!! CLASSNAMES" - cat classnames.txt - echo "----- CLASSNAMES" - - #if this is a re-run and it is a parallel run that does not have tests to run, halt execution of this parallel run [ -s classnames.txt ] || circleci-agent step halt - GRADLE_ARGS=$(cat src/test/java/classnames.txt | awk '{for (i=1; i<=NF; i++) print "--tests",$i}') + GRADLE_ARGS=$(cat classnames.txt | awk '{for (i=1; i<=NF; i++) print "--tests",$i}') echo "Prepared arguments for Gradle: $GRADLE_ARGS" # TODO: remove before merging, this is here to make testing the scripts quicker From 8dc6806ff9bb6d2bdf1fdad5a9f243c0e440852b Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 00:22:09 +0100 Subject: [PATCH 06/12] ci: try automatically re-running tests --- .circleci/doTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/doTests.sh b/.circleci/doTests.sh index 761026309..0b5bf3eda 100755 --- a/.circleci/doTests.sh +++ b/.circleci/doTests.sh @@ -174,7 +174,7 @@ do echo "Prepared arguments for Gradle: $GRADLE_ARGS" # TODO: remove before merging, this is here to make testing the scripts quicker - sed -i -e "s/\.\/gradlew test/\.\/gradlew test --tests \*Config\*/" startTestEnv + sed -i -e "s/\.\/gradlew test/\.\/gradlew test $GRADLE_ARGS/" startTestEnv ./startTestingEnv --cicd if [[ $? -ne 0 ]] From 6d32f48114ed69da10da001acd1d9cebb7129bea Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 00:27:05 +0100 Subject: [PATCH 07/12] ci: try automatically re-running tests --- .circleci/doTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/doTests.sh b/.circleci/doTests.sh index 0b5bf3eda..cfc7f8966 100755 --- a/.circleci/doTests.sh +++ b/.circleci/doTests.sh @@ -174,7 +174,7 @@ do echo "Prepared arguments for Gradle: $GRADLE_ARGS" # TODO: remove before merging, this is here to make testing the scripts quicker - sed -i -e "s/\.\/gradlew test/\.\/gradlew test $GRADLE_ARGS/" startTestEnv + sed -i -e "s/\.\/gradlew test/\.\/gradlew test \$GRADLE_ARGS/" startTestEnv ./startTestingEnv --cicd if [[ $? -ne 0 ]] From 96bf20aa812c7980745423b32ff7f7a16350309c Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 00:39:54 +0100 Subject: [PATCH 08/12] ci: try automatically re-running tests --- .circleci/doTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/doTests.sh b/.circleci/doTests.sh index cfc7f8966..5ba4a50d7 100755 --- a/.circleci/doTests.sh +++ b/.circleci/doTests.sh @@ -175,7 +175,7 @@ do # TODO: remove before merging, this is here to make testing the scripts quicker sed -i -e "s/\.\/gradlew test/\.\/gradlew test \$GRADLE_ARGS/" startTestEnv - ./startTestingEnv --cicd + GRADLE_ARGS=$GRADLE_ARGS ./startTestingEnv --cicd if [[ $? -ne 0 ]] then From 130b779cba0b677847c4d12ba2b8d2185beab66f Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 00:49:58 +0100 Subject: [PATCH 09/12] ci: try automatically re-running tests --- .circleci/doTests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/doTests.sh b/.circleci/doTests.sh index 5ba4a50d7..607933f38 100755 --- a/.circleci/doTests.sh +++ b/.circleci/doTests.sh @@ -165,6 +165,7 @@ do # Get list of classnames of tests that should run on this node. circleci tests glob "supertokens-*/src/test/**/*.java" | cut -c 1- | sed 's@/@.@g' | sed 's/.\{5\}$//' | + sed 's/^.*io\.supertokens/io.supertokens/' | circleci tests run --command=">classnames.txt xargs echo" --verbose --split-by=timings --timings-type=classname #if this is a re-run and it is a parallel run that does not have tests to run, halt execution of this parallel run From 3ec386123623e89db9d05c308354bd038e3a026a Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 02:46:42 +0100 Subject: [PATCH 10/12] test: try and improve flaky tests --- .circleci/doTests.sh | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.circleci/doTests.sh b/.circleci/doTests.sh index 607933f38..6404af168 100755 --- a/.circleci/doTests.sh +++ b/.circleci/doTests.sh @@ -163,20 +163,7 @@ do cd ../ echo $SUPERTOKENS_API_KEY > apiPassword - # Get list of classnames of tests that should run on this node. - circleci tests glob "supertokens-*/src/test/**/*.java" | cut -c 1- | sed 's@/@.@g' | sed 's/.\{5\}$//' | - sed 's/^.*io\.supertokens/io.supertokens/' | - circleci tests run --command=">classnames.txt xargs echo" --verbose --split-by=timings --timings-type=classname - - #if this is a re-run and it is a parallel run that does not have tests to run, halt execution of this parallel run - [ -s classnames.txt ] || circleci-agent step halt - - GRADLE_ARGS=$(cat classnames.txt | awk '{for (i=1; i<=NF; i++) print "--tests",$i}') - echo "Prepared arguments for Gradle: $GRADLE_ARGS" - - # TODO: remove before merging, this is here to make testing the scripts quicker - sed -i -e "s/\.\/gradlew test/\.\/gradlew test \$GRADLE_ARGS/" startTestEnv - GRADLE_ARGS=$GRADLE_ARGS ./startTestingEnv --cicd + ./startTestingEnv --cicd if [[ $? -ne 0 ]] then From 4fa696827c9231c46a0ba2d2954788e4470d5cb1 Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 02:47:23 +0100 Subject: [PATCH 11/12] chore: changelog consistency with plugin --- CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8fb12a9e..b6b0589d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,7 +107,6 @@ CREATE TABLE IF NOT EXISTS oauth_clients ( FOREIGN KEY(app_id) REFERENCES apps(app_id) ON DELETE CASCADE ); - CREATE TABLE IF NOT EXISTS oauth_sessions ( gid VARCHAR(255), app_id VARCHAR(64) DEFAULT 'public', @@ -121,8 +120,8 @@ CREATE TABLE IF NOT EXISTS oauth_sessions ( FOREIGN KEY(app_id, client_id) REFERENCES oauth_clients(app_id, client_id) ON DELETE CASCADE ); -CREATE INDEX IF NOT EXISTS oauth_session_exp_index ON oauth_sessions(exp DESC); -CREATE INDEX IF NOT EXISTS oauth_session_external_refresh_token_index ON oauth_sessions(app_id, external_refresh_token DESC); +CREATE INDEX oauth_session_exp_index ON oauth_sessions(exp DESC); +CREATE INDEX oauth_session_external_refresh_token_index ON oauth_sessions(app_id, external_refresh_token DESC); CREATE TABLE oauth_m2m_tokens ( app_id VARCHAR(64) DEFAULT 'public', From d7fab0cd9283382eb311f463971013b30c3eb187 Mon Sep 17 00:00:00 2001 From: Mihaly Lengyel Date: Tue, 29 Oct 2024 02:48:11 +0100 Subject: [PATCH 12/12] feat: make deep-links work --- .../io/supertokens/oauth/Transformations.java | 81 +++++++++---------- 1 file changed, 36 insertions(+), 45 deletions(-) diff --git a/src/main/java/io/supertokens/oauth/Transformations.java b/src/main/java/io/supertokens/oauth/Transformations.java index a0157696c..0ffa38a7e 100644 --- a/src/main/java/io/supertokens/oauth/Transformations.java +++ b/src/main/java/io/supertokens/oauth/Transformations.java @@ -68,24 +68,23 @@ public static Map transformRequestHeadersForHydra(Map 1 && keyValue[1].startsWith("ory_")) { - updatedQuery.append(keyValue[0]).append("=").append(keyValue[1].replaceFirst("ory_", "st_")).append("&"); - } else { - updatedQuery.append(param).append("&"); - } + if (!redirectTo.contains("?")) { + return redirectTo; + } + + String query = redirectTo.split("\\?")[1]; + if (query != null) { + String[] queryParams = query.split("&"); + StringBuilder updatedQuery = new StringBuilder(); + for (String param : queryParams) { + String[] keyValue = param.split("="); + if (keyValue.length > 1 && keyValue[1].startsWith("ory_")) { + updatedQuery.append(keyValue[0]).append("=").append(keyValue[1].replaceFirst("ory_", "st_")).append("&"); + } else { + updatedQuery.append(param).append("&"); } - redirectTo = redirectTo.replace("?" + query, "?" + updatedQuery.toString().trim()); } - } catch (MalformedURLException e) { - throw new IllegalStateException(e); + redirectTo = redirectTo.replace("?" + query, "?" + updatedQuery.toString().trim()); } return redirectTo; @@ -153,37 +152,29 @@ private static String transformRedirectUrlFromHydra(Main main, AppIdentifier app if (!redirectTo.startsWith("/")) { redirectTo = transformQueryParamsInURLFromHydra(redirectTo); - try { - if (Utils.containsUrl(redirectTo, hydraInternalAddress, true)) { - try { - URL url = new URL(redirectTo); - String query = url.getQuery(); - Map urlQueryParams = new HashMap<>(); - if (query != null) { - String[] pairs = query.split("&"); - for (String pair : pairs) { - int idx = pair.indexOf("="); - urlQueryParams.put(pair.substring(0, idx), URLDecoder.decode(pair.substring(idx + 1), StandardCharsets.UTF_8)); - } - } - String error = urlQueryParams.getOrDefault("error", null); - String errorDescription = urlQueryParams.getOrDefault("error_description", null); - if (error != null) { - throw new OAuthAPIException(error, errorDescription, 400); - } - redirectTo = redirectTo.replace(hydraInternalAddress, "{apiDomain}"); - - // path to hydra starts with /oauth2 while on the SDK it would be /oauth - redirectTo = redirectTo.replace("oauth2/", "oauth/"); - - } catch (MalformedURLException e) { - throw new IllegalStateException(e); + // We do not use the containsURL util to compare these because redirectTo can be a deep link + // Also, we do not mind comparison to internal addresses being strict comparisons + if (redirectTo.startsWith(hydraInternalAddress)) { + String query = redirectTo.contains("?") ? redirectTo.split("\\?")[1] : null; + Map urlQueryParams = new HashMap<>(); + if (query != null) { + String[] pairs = query.split("&"); + for (String pair : pairs) { + int idx = pair.indexOf("="); + urlQueryParams.put(pair.substring(0, idx), URLDecoder.decode(pair.substring(idx + 1), StandardCharsets.UTF_8)); } - } else if (Utils.containsUrl(redirectTo, hydraBaseUrlForConsentAndLogin, true)) { - redirectTo = redirectTo.replace(hydraBaseUrlForConsentAndLogin, "{apiDomain}"); } - } catch (MalformedURLException e) { - throw new IllegalStateException(e); + String error = urlQueryParams.getOrDefault("error", null); + String errorDescription = urlQueryParams.getOrDefault("error_description", null); + if (error != null) { + throw new OAuthAPIException(error, errorDescription, 400); + } + redirectTo = redirectTo.replace(hydraInternalAddress, "{apiDomain}"); + + // path to hydra starts with /oauth2 while on the SDK it would be /oauth + redirectTo = redirectTo.replace("oauth2/", "oauth/"); + } else if (redirectTo.startsWith(hydraBaseUrlForConsentAndLogin)) { + redirectTo = redirectTo.replace(hydraBaseUrlForConsentAndLogin, "{apiDomain}"); } }