Add infrastructure for database migration

src/strava_offline/sqlite.py

@@ -3,7 +3,9 @@
 import json
 from pathlib import Path
 import sqlite3
+from typing import Callable
 from typing import Iterator
+from typing import List
 from typing import Optional
 
 from . import config
@@ -13,37 +15,104 @@
 @contextmanager
 def database(config: config.DatabaseConfig) -> Iterator[sqlite3.Connection]:
     Path(config.strava_sqlite_database).parent.mkdir(parents=True, exist_ok=True)
-    db = sqlite3.connect(config.strava_sqlite_database)
+    db = sqlite3.connect(config.strava_sqlite_database, isolation_level=None)
     db.row_factory = sqlite3.Row
     try:
-        db.execute((
-            "CREATE TABLE IF NOT EXISTS bike"
-            "( id TEXT PRIMARY KEY"
-            ", json TEXT"
-            ", name TEXT"
-            ")"
-        ))
-        db.execute((
-            "CREATE TABLE IF NOT EXISTS activity"
-            "( id INTEGER PRIMARY KEY"
-            ", json TEXT"
-            ", upload_id TEXT"
-            ", name TEXT"
-            ", start_date TEXT"
-            ", moving_time INTEGER"
-            ", elapsed_time INTEGER"
-            ", distance REAL"
-            ", total_elevation_gain REAL"
-            ", gear_id TEXT"
-            ", type TEXT"
-            ", commute BOOLEAN"
-            ")"
-        ))
+        with db:  # transaction
+            db.execute("BEGIN")
+            migrations = schema_prepare_migrations(db)
+            schema_init(db)
+            schema_do_migrations(db, migrations)
         yield db
     finally:
         db.close()
 
 
+# Version of database schema. Bump this whenever one of the following is changed:
+#
+#  * schema_init
+#  * sync_bike
+#  * sync_activity
+#
+# The tables will be recreated using the stored json data and the new schema.
+SCHEMA_VERSION = 1
+
+
+def schema_init(db: sqlite3.Connection) -> None:
+    db.execute((
+        "CREATE TABLE IF NOT EXISTS bike"
+        "( id TEXT PRIMARY KEY"
+        ", json TEXT"
+        ", name TEXT"
+        ")"
+    ))
+    db.execute((
+        "CREATE TABLE IF NOT EXISTS activity"
+        "( id INTEGER PRIMARY KEY"
+        ", json TEXT"
+        ", upload_id TEXT"
+        ", name TEXT"
+        ", start_date TEXT"
+        ", moving_time INTEGER"
+        ", elapsed_time INTEGER"
+        ", distance REAL"
+        ", total_elevation_gain REAL"
+        ", gear_id TEXT"
+        ", type TEXT"
+        ", commute BOOLEAN"
+        ")"
+    ))
+
+
+def schema_prepare_migrations(db: sqlite3.Connection) -> List[Callable]:
+    migrations: List[Callable] = []
+
+    db_version = db.execute("PRAGMA user_version").fetchone()[0]
+    if db_version >= SCHEMA_VERSION:
+        return migrations
+
+    # migrate table "bike" by re-syncing entries from stored raw json replies
+    try:
+        db.execute("DROP TABLE IF EXISTS bike_old")
+        db.execute("ALTER TABLE bike RENAME TO bike_old")
+
+        def migrate_bike(db: sqlite3.Connection):
+            for row in db.execute("SELECT json FROM bike_old"):
+                sync_bike(json.loads(row['json']), db)
+            db.execute("DROP TABLE bike_old")
+
+        migrations.append(migrate_bike)
+    except sqlite3.DatabaseError:
+        pass
+
+    # migrate table "activity" by re-syncing entries from stored raw json replies
+    try:
+        db.execute("DROP TABLE IF EXISTS activity_old")
+        db.execute("ALTER TABLE activity RENAME TO activity_old")
+
+        def migrate_activity(db: sqlite3.Connection):
+            for row in db.execute("SELECT json FROM activity_old"):
+                sync_activity(json.loads(row['json']), db)
+            db.execute("DROP TABLE activity_old")
+
+        migrations.append(migrate_activity)
+    except sqlite3.DatabaseError:
+        pass
+
+    # update schema version
+    def migrate_version(db: sqlite3.Connection):
+        db.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
+
+    migrations.append(migrate_version)
+
+    return migrations
+
+
+def schema_do_migrations(db: sqlite3.Connection, migrations: List[Callable]) -> None:
+    for migration in migrations:
+        migration(db)
+
+
 def sync_bike(bike, db: sqlite3.Connection):
     db.execute(
         "INSERT OR REPLACE INTO bike(id, json, name) VALUES (?, ?, ?)",
@@ -53,6 +122,8 @@ def sync_bike(bike, db: sqlite3.Connection):
 
 def sync_bikes(strava, db: sqlite3.Connection):
     with db:  # transaction
+        db.execute("BEGIN")
+
         old_bikes = set(b['id'] for b in db.execute("SELECT id FROM bike"))
 
         for bike in strava.get_bikes():
@@ -103,6 +174,8 @@ def sync_activities(
     strava: StravaAPI, db: sqlite3.Connection, before: Optional[datetime] = None
 ):
     with db:  # transaction
+        db.execute("BEGIN")
+
         old_activities = set(a['id'] for a in db.execute("SELECT id FROM activity"))
 
         for activity in strava.get_activities(before=before):
@@ -120,6 +193,8 @@ def sync_activities_incremental(
     strava: StravaAPI, db: sqlite3.Connection, before: Optional[datetime] = None
 ):
     with db:  # transaction
+        db.execute("BEGIN")
+
         old_activities = set(a['id'] for a in db.execute("SELECT id FROM activity"))
 
         seen = 0

tests/cassettes/test_sqlite/test_migration_activities.yaml

@@ -0,0 +1 @@
+test_sync_activities.yaml

tests/cassettes/test_sqlite/test_migration_bikes.yaml

@@ -0,0 +1 @@
+test_sync_bikes.yaml

tests/test_sqlite.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+import sqlite3
 
 import pytest  # type: ignore [import]
 import pytz
@@ -123,3 +124,45 @@ def test_sync_activities(tmp_path):
             [1234567910],
             [1234567912],
         ]
+
+
+@pytest.mark.vcr
+def test_migration_bikes(tmp_path):
+    db_uri = "file:test_migration_bikes?mode=memory&cache=shared"
+    cfg = config.DatabaseConfig(strava_sqlite_database=db_uri)
+    db_keep_in_memory = sqlite3.connect(db_uri)
+
+    with sqlite.database(cfg) as db:
+        sqlite.sync_bikes(strava=strava(tmp_path), db=db)
+        db.execute("UPDATE bike SET name = 'xxx'")
+        db.execute("PRAGMA user_version = 0")
+        db.commit()
+
+    with sqlite.database(cfg) as db:
+        bikes = [row['name'] for row in db.execute(
+            "SELECT name FROM bike ORDER BY id LIMIT 1")]
+        assert bikes == ['bike1']
+
+    db_keep_in_memory.close()
+
+
+@pytest.mark.vcr
+def test_migration_activities(tmp_path):
+    db_uri = "file:test_migration_activities?mode=memory&cache=shared"
+    cfg = config.DatabaseConfig(strava_sqlite_database=db_uri)
+    db_keep_in_memory = sqlite3.connect(db_uri)
+
+    before = datetime.fromtimestamp(1610000000, tz=pytz.utc)
+
+    with sqlite.database(cfg) as db:
+        sqlite.sync_activities(strava=strava(tmp_path), db=db, before=before)
+        db.execute("UPDATE activity SET name = 'xxx'")
+        db.execute("PRAGMA user_version = 0")
+        db.commit()
+
+    with sqlite.database(cfg) as db:
+        activities = [row['name'] for row in db.execute(
+            "SELECT name FROM activity ORDER BY id LIMIT 1")]
+        assert activities == ['name1']
+
+    db_keep_in_memory.close()