Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add DatabaseReplayProvider #400

Merged
merged 74 commits into from
Nov 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
74 commits
Select commit Hold shift + click to select a range
c19bfa1
wip
vinceau Oct 13, 2023
196d712
add database replay provider wip
vinceau Oct 13, 2023
d8adc0c
still crashes unfortunately
vinceau Oct 13, 2023
b5ebcc5
add migration and fix adding replays
vinceau Oct 14, 2023
d349727
add database
vinceau Oct 26, 2023
6ae806f
add database syncing
vinceau Oct 26, 2023
1806783
fix migration not matching schema
vinceau Oct 26, 2023
399c416
fix stat not working
vinceau Oct 26, 2023
72b0787
it hecking works lets hecking go
vinceau Oct 27, 2023
897f1ff
reduce the number of fetched items for now
vinceau Oct 27, 2023
7b19a0a
add players
vinceau Oct 27, 2023
645c27a
move repositories into subfolder
vinceau Oct 27, 2023
37b2e97
fix isDev flag
vinceau Oct 27, 2023
303c41e
log when using in-memory db
vinceau Oct 27, 2023
00e039d
add mapping function
vinceau Oct 27, 2023
decb7a3
add consts and indicate load complete
vinceau Oct 27, 2023
1c9e1a4
fix incorrect player type
vinceau Oct 27, 2023
776c9fe
use partition
vinceau Oct 27, 2023
d1628e9
ignore invalid files
vinceau Oct 28, 2023
d5b1bbd
update log messages
vinceau Oct 28, 2023
da8fea4
reorder columns and add unique constraint
vinceau Oct 28, 2023
8219687
add date time inference to database provider
vinceau Oct 28, 2023
403fda6
check database before adding game
vinceau Oct 28, 2023
40ae8b0
add stats calculation to database replay provider
vinceau Oct 28, 2023
79a0458
allow empty database path
vinceau Oct 28, 2023
dfa59a9
add database tests
vinceau Oct 28, 2023
c48af20
add test for row count
vinceau Oct 28, 2023
caf2f62
add more tests
vinceau Oct 28, 2023
eb5ef9f
add test for players
vinceau Oct 28, 2023
6c9e74a
add test for unique player index
vinceau Oct 28, 2023
bac9c66
move partition to own file
vinceau Oct 28, 2023
987f395
move mocks into own file
vinceau Oct 28, 2023
a01f974
clean up database tests
vinceau Oct 28, 2023
caf01ca
allow size bytes to be undefined when inserting
vinceau Oct 28, 2023
961deff
update database name
vinceau Oct 28, 2023
5cd6ebe
clean up replay provider logic
vinceau Oct 28, 2023
44a837c
clean up replay providers
vinceau Oct 28, 2023
cf41f7a
rename file to snake case
vinceau Oct 28, 2023
893d3da
use dynamic imports
vinceau Oct 28, 2023
398512e
add proper migration handling
vinceau Oct 29, 2023
e19a18d
simplify migrations webpack conf
vinceau Oct 29, 2023
f7663c4
add kysely workers and fix broken tests
vinceau Oct 29, 2023
b285d33
add user id
vinceau Oct 29, 2023
8592e6e
add ranked info to games table
vinceau Oct 29, 2023
68992fe
remove default exports from migration
vinceau Oct 29, 2023
5141f25
clean up code
vinceau Oct 29, 2023
8922f11
infer start time before writing to db
vinceau Oct 30, 2023
1d54033
add set info to game table
vinceau Oct 30, 2023
07740e6
add ability to re-index database from scratch
vinceau Oct 30, 2023
95d094e
fix typo
vinceau Oct 30, 2023
ac79633
return all replays for now
vinceau Oct 30, 2023
7908269
update migration name format
vinceau Oct 30, 2023
863122c
update database path alias
vinceau Oct 30, 2023
a358fc7
remove test repository file
vinceau Oct 30, 2023
eeaa0d6
add total folder size calculation
vinceau Oct 30, 2023
dab056c
use promise.all
vinceau Oct 30, 2023
e6bf70c
fix migration schema
vinceau Oct 30, 2023
e66b4a6
refactor repositories
vinceau Oct 30, 2023
deed2d8
rename sequence index to sequence number
vinceau Oct 30, 2023
8c4c488
add missing drop index
vinceau Oct 30, 2023
713e5a6
move record generation outside of class
vinceau Oct 30, 2023
b6fde0b
no need to explicitly drop indexes
vinceau Oct 30, 2023
fb2b3e8
add retry logic for database creation
vinceau Nov 3, 2023
2ea6273
rename replay table to file
vinceau Nov 3, 2023
9603c2c
add readme to database folder
vinceau Nov 3, 2023
cbef7eb
chore(deps): move kysely to prod dependency
vinceau Nov 8, 2023
43141c3
optimize player fetching
vinceau Nov 8, 2023
e7f34c7
also index by player index
vinceau Nov 8, 2023
b5ccdcb
use forEach
vinceau Nov 8, 2023
ea3c5bb
optimise database calls
vinceau Nov 9, 2023
603858b
remove else condition
vinceau Nov 9, 2023
a04492d
return player records too on insertion
vinceau Nov 9, 2023
911267a
rename game record to game and file record
vinceau Nov 9, 2023
86cd796
go back to using a for-loop for speed
vinceau Nov 10, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions .erb/configs/webpack.config.migrations.prod.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/**
* Webpack config for Kysely database migrations
*/

import { fdir } from "fdir";
import path from "path";
import TerserPlugin from "terser-webpack-plugin";
import type { Configuration } from "webpack";
import { merge } from "webpack-merge";

import checkNodeEnv from "../scripts/check-node-env";
import baseConfig from "./webpack.config.base";
import webpackPaths from "./webpack.paths";

checkNodeEnv("production");

function resolveMigrations(): Record<string, string> {
const migrations: Record<string, string> = {};
// eslint-disable-next-line new-cap
const crawler = new fdir().glob("./**/*.ts").withFullPaths();
const files = crawler.crawl(path.join(webpackPaths.srcPath, "database", "migrations")).sync() as string[];
files.forEach((filename) => {
const basename = path.basename(filename, ".ts");
migrations[basename] = filename;
});
return migrations;
}

const configuration: Configuration = {
mode: "production",

target: "electron-main",

entry: resolveMigrations(),

output: {
path: path.join(webpackPaths.distPath, "migrations"),
filename: "[name].js",
},

optimization: {
minimizer: [
new TerserPlugin({
parallel: true,
}),
],
},
};

export default merge(baseConfig, configuration);
9 changes: 8 additions & 1 deletion electron-builder.json
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,14 @@
"buildResources": "assets",
"output": "release/build"
},
"extraResources": ["./assets/include/**"],
"extraResources": [
"./assets/include/**",
{
"from": "./release/app/dist/migrations",
"to": "./migrations",
"filter": ["**/*.js"]
}
],
"publish": {
"provider": "github",
"owner": "project-slippi",
Expand Down
9 changes: 6 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,23 @@
"description": "Launch Slippi Online, browse and watch saved replays",
"license": "GPL-3.0",
"scripts": {
"build": "concurrently \"yarn run build:main\" \"yarn run build:renderer\"",
"build": "concurrently \"yarn run build:main\" \"yarn run build:renderer\" \"yarn run build:migrations\"",
"build:main": "cross-env NODE_ENV=production TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.main.prod.ts",
"build:renderer": "cross-env NODE_ENV=production TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.renderer.prod.ts",
"build:migrations": "cross-env NODE_ENV=production TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.migrations.prod.ts",
"rebuild": "electron-rebuild --parallel --types prod,dev,optional --module-dir release/app",
"lint": "cross-env NODE_ENV=development eslint . --ext .js,.jsx,.ts,.tsx",
"lint:fix": "yarn run lint --fix",
"typecheck": "yarn tsc --noEmit",
"package": "ts-node ./.erb/scripts/clean.js dist && yarn run build && electron-builder build --publish never",
"clean": "ts-node ./.erb/scripts/clean.js dist",
"package": "yarn run clean && yarn run build && electron-builder build --publish never",
"postinstall": "patch-package && ts-node .erb/scripts/check-native-dep.js && electron-builder install-app-deps && cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.renderer.dev.dll.ts",
"dev": "yarn run start --env mock",
"start": "ts-node ./.erb/scripts/check-port-in-use.js && yarn run start:renderer",
"start:main": "cross-env NODE_ENV=development electronmon -r ts-node/register/transpile-only -r ./.erb/scripts/fix-esm.js ./src/main/main.ts",
"start:preload": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.preload.dev.ts",
"start:renderer": "cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack serve --config ./.erb/configs/webpack.config.renderer.dev.ts",
"test": "jest --verbose",
"test": "cross-env ELECTRON_RUN_AS_NODE=true electron ./node_modules/jest-cli/bin/jest.js --verbose",
"storybook": "start-storybook -p 6006",
"build-storybook": "build-storybook"
},
Expand Down Expand Up @@ -67,6 +69,7 @@
"@teamsupercell/typings-for-css-modules-loader": "^2.5.1",
"@testing-library/jest-dom": "^5.16.1",
"@testing-library/react": "^12.1.2",
"@types/better-sqlite3": "^7.6.5",
"@types/compare-func": "^1.3.0",
"@types/default-gateway": "^7.2.0",
"@types/jest": "^27.4.1",
Expand Down
3 changes: 3 additions & 0 deletions release/app/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
"postinstall": "yarn run electron-rebuild && yarn run link-modules"
},
"dependencies": {
"better-sqlite3": "^9.0.0",
"kysely": "^0.26.3",
"kysely-sqlite-worker": "^0.4.5",
"threads": "^1.6.5",
"websocket": "^1.0.34"
},
Expand Down
Loading
Loading