report migration

database/000_recreate_playerdata_dev.sql

@@ -0,0 +1,3 @@
+DROP DATABASE IF EXISTS playerdata_dev;
+CREATE DATABASE playerdata_dev;
+GRANT SELECT, INSERT, UPDATE, DELETE, EVENT, TRIGGER ON `playerdata\_dev`.* TO 'event_admin'@'localhost'; ALTER USER 'event_admin'@'localhost' ;

database/001_insert_apiUser.sql

@@ -0,0 +1,19 @@
+insert ignore into apiPermissions (id, permission) values 
+(1, 'verify_ban'),
+(2, 'verify_players'),
+(3, 'request_highscores')
+;
+
+insert ignore into apiUser (id, username, token, ratelimit, is_active) values 
+(1, 'verify_ban', 'verify_ban', -1, 1),
+(2, 'verify_players', 'verify_players', -1, 1),
+(3, 'request_highscores', 'request_highscores', -1, 1)
+;
+
+insert ignore into apiUserPerms (user_id, permission_id) VALUES
+(1,1),
+(1,2),
+(1,3),
+(2,2),
+(3,3)
+;

database/002_insert_players.sql

@@ -0,0 +1,50 @@
+/*
+    Insert players
+*/
+insert ignore into playerdata_dev.Labels 
+select * from playerdata.Labels order by id desc;
+update playerdata_dev.Labels set id=0  where label='Unknown';
+
+insert ignore into playerdata_dev.LabelJagex 
+select * from playerdata.LabelJagex;
+
+insert ignore into playerdata_dev.Players
+select p.* from playerdata.Labels lb
+join lateral (
+    select 
+        pl.*
+    from playerdata.Players pl
+    where lb.id = pl.label_id
+    limit 100
+) p on (1=1)
+;
+
+/*
+insert ignore into playerdata_dev.Players
+select * from playerdata.Players pl 
+where 1=1
+    and pl.label_id = 0 -- unkown player
+    and pl.id not in (select id from playerdata_dev.Players)
+limit 100
+;
+
+insert ignore into playerdata_dev.Players
+select * from playerdata.Players pl 
+where 1=1
+    and pl.label_id = 1 -- real player
+    and pl.id not in (select id from playerdata_dev.Players)
+limit 15000
+;
+*/
+
+/*
+    Encrypt player names
+*/
+update playerdata_dev.Players 
+set 
+    name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser.")), 
+    normalized_name=HEX(AES_ENCRYPT(normalized_name, "Victor_careers_onto_THE9_free0_endorser."))
+;
+/*
+    To decrypt the encryption: ES_DECRYPT(BINARY(UNHEX(setting_value)), 'key')
+*/

database/003_insert_playerdata_dev.sql

@@ -0,0 +1,17 @@
+/*
+    Generic data
+*/
+insert ignore into playerdata_dev.PredictionsFeedback
+select pr.* from playerdata.PredictionsFeedback pr
+join playerdata_dev.Players pl on pr.voter_id = pl.id or pr.subject_id=pl.id;
+
+insert ignore into playerdata_dev.reportLatest
+select rp.* from playerdata.reportLatest rp
+join playerdata_dev.Players pl on rp.reported_id = pl.id;
+
+insert ignore into playerdata_dev.Predictions
+select pr.* from playerdata.Predictions pr
+join playerdata_dev.Players pl on pr.id = pl.id;
+
+ALTER TABLE playerdata_dev.Predictions CHANGE `name` `name` VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL;
+update playerdata_dev.Predictions set name=HEX(AES_ENCRYPT(name, "Victor_careers_onto_THE9_free0_endorser."));

database/004_insert_playerHiscoreData.sql

@@ -0,0 +1,12 @@
+insert ignore into playerdata_dev.playerHiscoreData
+select phd.* from playerdata.playerHiscoreData phd
+join playerdata_dev.Players pl on phd.Player_id = pl.id;
+
+insert ignore into playerdata_dev.playerHiscoreDataLatest
+select phd.* from playerdata.playerHiscoreDataLatest phd
+join playerdata_dev.Players pl on phd.Player_id = pl.id;
+
+insert ignore into playerdata_dev.playerHiscoreDataXPChange
+select phd.* from playerdata.playerHiscoreDataXPChange phd 
+join playerdata_dev.Players pl on phd.Player_id = pl.id;
+

database/005_insert_reports.sql

@@ -0,0 +1,21 @@
+insert ignore into playerdata_dev.Reports
+select a.* from playerdata_dev.Players pl
+join lateral (
+    select 
+        plr.*
+    from playerdata.Reports plr
+    where plr.reportedid = pl.id
+    limit 100
+) a on (1=1)
+;
+
+insert ignore into playerdata_dev.Reports
+select a.* from playerdata_dev.Players pl
+join lateral (
+    select 
+        plr.*
+    from playerdata.Reports plr
+    where plr.reportingid = pl.id
+    limit 100
+) a on (1=1)
+;

database/create.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+sudo mysql -u root < /home/ubuntu/000_recreate_playerdata_dev.sql
+sudo mysqldump -u root --no-data --routines --triggers --events --databases playerdata| sudo mysql -u root playerdata_dev
+sudo mysql -u root playerdata_dev < /home/ubuntu/001_insert_apiUser.sql
+sudo mysql -u root playerdata_dev < /home/ubuntu/002_insert_players.sql
+sudo mysql -u root playerdata_dev < /home/ubuntu/003_insert_playerdata_dev.sql
+sudo mysql -u root playerdata_dev < /home/ubuntu/004_insert_playerHiscoreData.sql
+sudo mysql -u root playerdata_dev < /home/ubuntu/005_insert_reports.sql
+sudo mysqldump -u root --routines --events --databases playerdata_dev | sed 's/playerdata_dev/playerdata/g'|  gzip > 100_playerdata_dev.sql.gz
+
+# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReports.sql
+# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerReportsManual.sql
+# sudo mysql -u root playerdata_dev < /home/ubuntu/insert_playerLocationsDetail.sql

database/xx_insert_playerLocationsDetail.sql

@@ -0,0 +1,21 @@
+insert ignore into playerdata_dev.playerLocations
+select a.* from playerdata_dev.Players pl
+join lateral (
+    select 
+        plo.*
+    from playerdata.playerLocations plo
+    where plo.reported_id = pl.id
+    limit 100
+) a on (1=1)
+;
+
+insert ignore into playerdata_dev.playerLocationsDetail
+select a.* from playerdata_dev.Players pl
+join lateral (
+    select 
+        pld.*
+    from playerdata.playerLocationsDetail pld
+    where pld.reported_id = pl.id
+    limit 100
+) a on (1=1)
+;

database/xx_insert_playerReports.sql

@@ -0,0 +1,21 @@
+insert ignore into playerdata_dev.playerReports
+select a.* from playerdata_dev.Players pl
+join lateral (
+    select 
+        plr.*
+    from playerdata.playerReports plr
+    where plr.reported_id = pl.id
+    limit 100
+) a on (1=1)
+;
+
+insert ignore into playerdata_dev.playerReports
+select a.* from playerdata_dev.Players pl
+join lateral (
+    select 
+        plr.*
+    from playerdata.playerReports plr
+    where plr.reporting_id = pl.id
+    limit 100
+) a on (1=1)
+;

database/xx_insert_playerReportsManual.sql

@@ -0,0 +1,21 @@
+insert ignore into playerdata_dev.playerReportsManual
+select a.* from playerdata_dev.Players pl
+join lateral (
+    select 
+        prm.*
+    from playerdata.playerReportsManual prm
+    where prm.reporting_id = pl.id
+    limit 100
+) a on (1=1)
+;
+
+insert ignore into playerdata_dev.playerReportsManual
+select a.* from playerdata_dev.Players pl
+join lateral (
+    select 
+        prm.*
+    from playerdata.playerReportsManual prm
+    where prm.reported_id = pl.id
+    limit 100
+) a on (1=1)
+;

notes.md

@@ -0,0 +1,18 @@
+## setup
+```
+python -m venv .venv
+.venv\Scripts\activate
+python -m pip install --upgrade pip
+pip install -r requirements.txt
+```
+# for admin purposes saving & upgrading
+run this in cmd
+
+```
+venv\Scripts\activate
+call pip freeze > requirements.txt
+powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '==', '>=' } | Set-Content requirements.txt"
+call pip install -r requirements.txt --upgrade
+call pip freeze > requirements.txt
+powershell "(Get-Content requirements.txt) | ForEach-Object { $_ -replace '>=', '==' } | Set-Content requirements.txt"
+```

report_migration/migrate_report_data.py

@@ -20,14 +20,45 @@
     autoflush=False,
 )
 
+with open('report_data.sql','r') as x:
+    migration_query = x.read()
+
+
+with open('report_data_where_not_exists.sql','r') as x:
+    migration_query_where_not_exists = x.read()
+
+
+with open('reset_tables.sql','r') as x:
+    reset_tables_query = x.read()
+
+
+
 async def main():
+    await migrate_selection(1, 100_000)
+
+async def reset_tables():
     # get a session
     async with Session() as session:
         session: AsyncSession
-        # get a transaction
         async with session.begin():
-            # do something
-            pass
+            await session.execute(sqla.text(reset_tables_query+" COMMIT;"))
+async def migrate_selection(startId, endId):
+    # get a session
+    async with Session() as session:
+        session: AsyncSession
+        params = {"startId": startId, "endId": endId}
+        async with session.begin():
+            await session.execute(sqla.text(migration_query+" COMMIT;"), params=params)
+
+async def migrate_selection_where_not_exists(startId, endId):
+    # get a session
+    async with Session() as session:
+        session: AsyncSession
+        params = {"startId": startId, "endId": endId}
+        async with session.begin():
+            await session.execute(sqla.text(migration_query_where_not_exists+" COMMIT;"), params=params)
+
+
 
 if __name__ == "__main__":
     asyncio.run(main())

report_migration/migrate_test.py

@@ -0,0 +1,54 @@
+import asyncio
+import time
+from migrate_report_data import reset_tables, migrate_selection, migrate_selection_where_not_exists
+
+
+async def reset_and_time(func, *args, **kwargs):
+    """
+    Helper function to reset tables and measure execution time of a function.
+    """
+    await reset_tables()  # Reset tables before timing
+    start_time = time.time()
+    await func(*args, **kwargs)
+    end_time = time.time()
+    return end_time - start_time
+
+
+async def test_migrate_selection():
+    """
+    Test migrate_selection and measure average time for 10 calls.
+    """
+    print("Testing migrate_selection...")
+    total_time = 0
+    for i in range(10):
+        elapsed = await reset_and_time(migrate_selection, 1, 100_000)
+        print(f"Run {i + 1}: {elapsed:.4f} seconds")
+        total_time += elapsed
+    avg_time = total_time / 10
+    print(f"Average time for migrate_selection: {avg_time:.4f} seconds")
+
+
+async def test_migrate_selection_where_not_exists():
+    """
+    Test migrate_selection_where_not_exists and measure average time for 10 calls.
+    """
+    print("\nTesting migrate_selection_where_not_exists...")
+    total_time = 0
+    for i in range(10):
+        elapsed = await reset_and_time(migrate_selection_where_not_exists, 1, 100_000)
+        print(f"Run {i + 1}: {elapsed:.4f} seconds")
+        total_time += elapsed
+    avg_time = total_time / 10
+    print(f"Average time for migrate_selection_where_not_exists: {avg_time:.4f} seconds")
+
+
+async def main():
+    """
+    Main function to run all tests sequentially.
+    """
+    await test_migrate_selection()
+    await test_migrate_selection_where_not_exists()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())