From c0eb8cd53a8590c7ea8d01594b8aa9550ccd5835 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Mon, 14 Aug 2023 16:56:26 +0800 Subject: [PATCH 1/3] queries: add analyze recent contributors query --- .../analyze-recent-contributors/params.json | 10 ++ .../analyze-recent-contributors/template.sql | 91 +++++++++++++++++++ .../events-increment-list/template.sql | 38 ++++---- .../recent-hot-collections/template.sql | 15 +-- 4 files changed, 130 insertions(+), 24 deletions(-) create mode 100644 configs/queries/analyze-recent-contributors/params.json create mode 100644 configs/queries/analyze-recent-contributors/template.sql diff --git a/configs/queries/analyze-recent-contributors/params.json b/configs/queries/analyze-recent-contributors/params.json new file mode 100644 index 00000000000..67b3eb47ed5 --- /dev/null +++ b/configs/queries/analyze-recent-contributors/params.json @@ -0,0 +1,10 @@ +{ + "cacheHours": 1, + "params": [ + { + "name": "repoId", + "replaces": "41986369", + "pattern": "^[1-9]\\d*$" + } + ] +} diff --git a/configs/queries/analyze-recent-contributors/template.sql b/configs/queries/analyze-recent-contributors/template.sql new file mode 100644 index 00000000000..8084284458c --- /dev/null +++ b/configs/queries/analyze-recent-contributors/template.sql @@ -0,0 +1,91 @@ +WITH RECURSIVE seq(idx, current_period_day, last_period_day) AS ( + SELECT + 1 AS idx, + CURRENT_DATE() AS current_period_day, + DATE_SUB(CURRENT_DATE(), INTERVAL 28 day) AS last_period_day + UNION ALL + SELECT + idx + 1 AS idx, + DATE_SUB(CURRENT_DATE(), INTERVAL idx day) AS current_period_day, + DATE_SUB(CURRENT_DATE(), INTERVAL idx + 28 day) AS last_period_day + FROM seq + WHERE idx < 28 +), group_by_day AS ( + SELECT + day_offset % 28 + 1 AS idx, + day_offset DIV 28 AS period, + day, + contributors + FROM ( + SELECT + (DATEDIFF(CURRENT_DATE(), day)) AS day_offset, + day, + contributors + FROM ( + SELECT + DATE_FORMAT(created_at, '%Y-%m-%d') AS day, + COUNT(DISTINCT actor_id) AS contributors + FROM + github_events ge + WHERE + repo_id = 41986369 + AND ( + (type = 'PullRequestEvent' AND action = 'opened') OR + (type = 'IssuesEvent' AND action = 'opened') OR + (type = 'IssueCommentEvent' AND action = 'created') OR + (type = 'PullRequestReviewEvent' AND action = 'created') OR + (type = 'PullRequestReviewCommentEvent' AND action = 'created') OR + (type = 'PushEvent' AND action = '') + ) + AND created_at > DATE_SUB(CURRENT_DATE(), INTERVAL 56 DAY) + GROUP BY day + ORDER BY day + ) sub + ) sub2 +), group_by_period AS ( + SELECT + (DATEDIFF(CURRENT_DATE(), created_at)) DIV 28 AS period, + COUNT(DISTINCT actor_id) AS contributors + FROM + github_events ge + WHERE + repo_id = 41986369 + AND ( + (type = 'PullRequestEvent' AND action = 'opened') OR + (type = 'IssuesEvent' AND action = 'opened') OR + (type = 'IssueCommentEvent' AND action = 'created') OR + (type = 'PullRequestReviewEvent' AND action = 'created') OR + (type = 'PullRequestReviewCommentEvent' AND action = 'created') OR + (type = 'PushEvent' AND action = '') + ) + AND created_at > DATE_SUB(CURRENT_DATE(), INTERVAL 56 DAY) + GROUP BY period + ORDER BY period +), last_28_days AS ( + SELECT idx, day, contributors + FROM group_by_day + WHERE period = 0 +), last_2nd_28_days AS ( + SELECT idx, day, contributors + FROM group_by_day + WHERE period = 1 +), last_28_days_total AS ( + SELECT contributors AS total FROM group_by_period WHERE period = 0 +), last_2nd_28_days_total AS ( + SELECT contributors AS total FROM group_by_period WHERE period = 1 +) +SELECT + s.idx AS idx, + s.current_period_day AS current_period_day, + IFNULL(cp.contributors, 0) AS current_period_day_contributors, + IFNULL(cpt.total, 0) AS current_period_contributors, + s.last_period_day AS last_period_day, + IFNULL(lp.contributors, 0) AS last_period_day_contributors, + IFNULL(lpt.total, 0) AS last_period_contributors +FROM seq s +LEFT JOIN last_28_days cp ON s.idx = cp.idx +LEFT JOIN last_2nd_28_days lp ON s.idx = lp.idx +JOIN last_28_days_total cpt +JOIN last_2nd_28_days_total lpt +ORDER BY idx +; \ No newline at end of file diff --git a/configs/queries/events-increment-list/template.sql b/configs/queries/events-increment-list/template.sql index 1520ee73bdf..52f08f72d61 100644 --- a/configs/queries/events-increment-list/template.sql +++ b/configs/queries/events-increment-list/template.sql @@ -1,18 +1,22 @@ -SELECT - /*+ MAX_EXECUTION_TIME(15000) */ - id, type, action, actor_id, actor_login, repo_id, repo_name, number, pr_merged, created_at -FROM github_events -WHERE - created_at BETWEEN ( - UTC_TIMESTAMP - INTERVAL 6 MINUTE - INTERVAL UNIX_TIMESTAMP(UTC_TIMESTAMP - INTERVAL 6 MINUTE) % 5 SECOND - ) AND ( - UTC_TIMESTAMP - INTERVAL 5 MINUTE - INTERVAL UNIX_TIMESTAMP(UTC_TIMESTAMP - INTERVAL 5 MINUTE) % 5 SECOND - ) - AND actor_login NOT LIKE '%[bot]' - AND actor_login NOT LIKE '%-bot' - AND type IN ( - 'WatchEvent', 'ForkEvent', 'IssuesEvent', 'PullRequestEvent', 'PushEvent', 'CreateEvent', 'ReleaseEvent', - 'PullRequestReviewCommentEvent', 'PullRequestReviewEvent', 'IssueCommentEvent' - ) +SELECT * +FROM ( + SELECT + /*+ READ_FROM_STORAGE(TIKV[github_events]), MAX_EXECUTION_TIME(15000) */ + id, type, action, actor_id, actor_login, repo_id, repo_name, number, pr_merged, created_at + FROM github_events + WHERE + created_at BETWEEN ( + UTC_TIMESTAMP - INTERVAL 6 MINUTE - INTERVAL UNIX_TIMESTAMP(UTC_TIMESTAMP - INTERVAL 6 MINUTE) % 5 SECOND + ) AND ( + UTC_TIMESTAMP - INTERVAL 5 MINUTE - INTERVAL UNIX_TIMESTAMP(UTC_TIMESTAMP - INTERVAL 5 MINUTE) % 5 SECOND + ) + AND actor_login NOT LIKE '%[bot]' + AND actor_login NOT LIKE '%-bot' + AND type IN ( + 'WatchEvent', 'ForkEvent', 'IssuesEvent', 'PullRequestEvent', 'PushEvent', 'CreateEvent', 'ReleaseEvent', + 'PullRequestReviewCommentEvent', 'PullRequestReviewEvent', 'IssueCommentEvent' + ) + LIMIT 50 +) sub ORDER BY created_at DESC -LIMIT 50; +; diff --git a/configs/queries/recent-hot-collections/template.sql b/configs/queries/recent-hot-collections/template.sql index 214a50bb5b7..0464f4730d2 100644 --- a/configs/queries/recent-hot-collections/template.sql +++ b/configs/queries/recent-hot-collections/template.sql @@ -1,13 +1,13 @@ WITH collectionsOrderByVisits AS ( SELECT - CAST(JSON_EXTRACT(request_params, '$.collectionId') AS SIGNED) AS collection_id, + CAST(JSON_EXTRACT(query, '$.collectionId') AS SIGNED) AS collection_id, COUNT(*) AS visits - FROM access_logs + FROM stats_api_requests WHERE - request_path LIKE '/q/collection-%' - AND requested_at > DATE_SUB(NOW(), INTERVAL 1 MONTH) - GROUP BY 1 - ORDER BY 2 DESC + path LIKE '/q/collection-%' + AND finished_at > DATE_SUB(NOW(), INTERVAL 1 MONTH) + GROUP BY collection_id + ORDER BY visits DESC ), top10collections AS ( SELECT c.id, c.name, cv.visits FROM collectionsOrderByVisits cv @@ -29,4 +29,5 @@ FROM ( FROM collection_items ci JOIN top10collections tc ON ci.collection_id = tc.id ) sub -WHERE `rank` <= 3; +WHERE `rank` <= 3 +ORDER BY visits DESC; From f569a8cf584b1bf9b26e9d4299703b77345a07b6 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Mon, 14 Aug 2023 17:01:20 +0800 Subject: [PATCH 2/3] fix workflow --- .github/workflows/verify-query.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/verify-query.yml b/.github/workflows/verify-query.yml index 421660cf7d6..529a6091e68 100644 --- a/.github/workflows/verify-query.yml +++ b/.github/workflows/verify-query.yml @@ -44,6 +44,9 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- + - name: Install OSSInsight Types Dependencies + run: pnpm --filter "@ossinsight/types" install && pnpm --filter "@ossinsight/types" build + - name: Install Dependencies run: pnpm i --frozen-lockfile --strict-peer-dependencies From 628c2047c033464f3340a9144b192602bb7ed76a Mon Sep 17 00:00:00 2001 From: Mini256 Date: Mon, 14 Aug 2023 17:05:01 +0800 Subject: [PATCH 3/3] fix workflow --- .github/workflows/verify-query.yml | 7 +++---- packages/api-server/package.json | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify-query.yml b/.github/workflows/verify-query.yml index 529a6091e68..6536841f0b4 100644 --- a/.github/workflows/verify-query.yml +++ b/.github/workflows/verify-query.yml @@ -4,11 +4,12 @@ on: pull_request: paths: - 'configs/queries/**' + - 'packages/types/**' defaults: run: shell: bash - working-directory: packages/api-server + working-directory: packages/types jobs: validate: @@ -40,12 +41,10 @@ jobs: name: Setup pnpm cache with: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('./pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm-store- - - name: Install OSSInsight Types Dependencies - run: pnpm --filter "@ossinsight/types" install && pnpm --filter "@ossinsight/types" build - name: Install Dependencies run: pnpm i --frozen-lockfile --strict-peer-dependencies diff --git a/packages/api-server/package.json b/packages/api-server/package.json index ad4330d74bd..82897fbe443 100644 --- a/packages/api-server/package.json +++ b/packages/api-server/package.json @@ -9,8 +9,6 @@ "test": "__tests__" }, "scripts": { - "gen:schema-ts": "json2ts ./schema/query.schema.json ./src/types/query.schema.d.ts", - "validate:query": "ajv -s ./schema/query.schema.json -r ./schema/openapi-2021-09-28.schema.json -c ajv-formats -d \"../../configs/queries/*/params.json\"", "build:ts": "tsc", "dev": "NODE_ENV=development fastify start -p ${API_SERVER_PORT:-3450} -P --ignore-watch=.ts$ -L dist/logger.js -l info -w dist/app.js", "build": "rm -rf dist && npm run build:ts",