Merge pull request #110 from schireson/dc/allow-null-backup-name-field

docs/source/config/table.md

@@ -47,6 +47,21 @@ tables:
   - '*.*'
 ```
 
+````{note}
+`name` **can** also be omitted entirely, with some caveats. The "name" field
+populates the `{table}` templated into queries and location paths (**both**
+of which default to including the `{table}` template value).
+
+Thus, if you omit the "name" field, you must have also provided a concrete "query"
+and "location" field.
+
+```yaml
+tables:
+  - query: select * from for_example_a_view
+    location: backups/public.for_example_a_view
+```
+````
+
 ### Globbing
 
 Using common globbing rules:

src/databudgie/adapter/base.py

@@ -173,6 +173,9 @@ def materialize_table_dependencies(
         tables = set()
         dependent_table_ops = []
         for table_op in table_ops:
+            if table_op.full_name is None:
+                continue
+
             tables.add(table_op.full_name)
 
             if not table_op.raw_conf.follow_foreign_keys:

src/databudgie/backup.py

@@ -85,7 +85,7 @@ def backup_ddl(
 
     for table_op in table_ops:
         schema_op = table_op.schema_op()
-        if schema_op.name in schemas:
+        if not schema_op or schema_op.name in schemas:
             continue
 
         if not table_op.raw_conf.ddl:
@@ -115,10 +115,10 @@ def backup_ddl(
         console.info("Finished backing up schema DDL")
 
         for table_op in table_ops:
-            if not table_op.raw_conf.ddl:
+            if not table_op.full_name or not table_op.raw_conf.ddl:
                 continue
 
-            progress.update(task, description=f"Backing up DDL: {table_op.full_name}")
+            progress.update(task, description=f"Backing up DDL: {table_op.pretty_name}")
             result = adapter.export_table_ddl(table_op.full_name)
 
             filename = storage.write_buffer(
@@ -128,7 +128,7 @@ def backup_ddl(
                 name=table_op.full_name,
             )
 
-            console.trace(f"Uploaded {table_op.full_name} to {filename}")
+            console.trace(f"Uploaded {table_op.pretty_name} to {filename}")
             table_names.append(table_op.full_name)
 
     console.info("Finished backing up DDL")
@@ -158,11 +158,11 @@ def backup_sequences(
         task = progress.add_task("Backing up sequence positions", total=len(table_ops))
 
         for table_op in table_ops:
-            progress.update(task, description=f"Backing up sequence position: {table_op.full_name}")
-
-            if not table_op.raw_conf.sequences:
+            if not table_op.full_name or not table_op.raw_conf.sequences:
                 continue
 
+            progress.update(task, description=f"Backing up sequence position: {table_op.pretty_name}")
+
             sequences = table_sequences.get(table_op.full_name)
             if not sequences:
                 continue
@@ -182,7 +182,7 @@ def backup_sequences(
                 name=table_op.full_name,
             )
 
-            console.trace(f"Wrote {table_op.full_name} sequences to {filename}")
+            console.trace(f"Wrote {table_op.pretty_name} sequences to {filename}")
 
     console.info("Finished backing up sequence positions")
 
@@ -198,7 +198,7 @@ def backup_tables(
         task = progress.add_task("Backing up tables", total=len(table_ops))
 
         for table_op in table_ops:
-            progress.update(task, description=f"Backing up table: {table_op.full_name}")
+            progress.update(task, description=f"Backing up table: {table_op.pretty_name}")
 
             if not table_op.raw_conf.data:
                 continue
@@ -233,7 +233,7 @@ def backup(
     path = table_op.location()
 
     if table_op.raw_conf.skip_if_exists and storage.path_exists(path):
-        console.trace(f"Skipping {table_op.full_name} due to `skip_if_exists`")
+        console.trace(f"Skipping {table_op.pretty_name} due to `skip_if_exists`")
         return
 
     buffer = adapter.export_query(table_op.query())
@@ -245,4 +245,4 @@ def backup(
         path, buffer, file_type=FileTypes.data, name=table_op.full_name, compression=compression
     )
 
-    console.trace(f"Uploaded {table_op.full_name} to {filename}")
+    console.trace(f"Uploaded {table_op.pretty_name} to {filename}")

src/databudgie/config.py

@@ -236,7 +236,7 @@ def from_collection(cls, collection: list | dict | None) -> dict[str, Connection
 
 @dataclass
 class BackupTableConfig(Config):
-    name: str
+    name: str | None = None
     location: str = "backups/{table}"
     query: str = "select * from {table}"
     compression: str | None = None

src/databudgie/restore.py

@@ -113,7 +113,7 @@ def restore_all_ddl(
 
     for table_op in table_ops:
         schema_op = table_op.schema_op()
-        if schema_op.name in schema_names:
+        if not schema_op or schema_op.name in schema_names:
             continue
 
         if not table_op.raw_conf.ddl:
@@ -199,7 +199,7 @@ def truncate_tables(table_ops: Sequence[TableOp], adapter: Adapter, console: Con
         for table_op in table_ops:
             data = table_op.raw_conf.data
             truncate = table_op.raw_conf.truncate
-            if not data or not truncate:
+            if not data or not truncate or table_op.full_name is None:
                 continue
 
             progress.update(task, description=f"[trace]Truncating {table_op.full_name}[/trace]", advance=1)
@@ -220,7 +220,7 @@ def restore_tables(
         task = progress.add_task("Restoring tables", total=len(table_ops))
 
         for table_op in table_ops:
-            if not table_op.raw_conf.data:
+            if not table_op.full_name or not table_op.raw_conf.data:
                 continue
 
             progress.update(task, description=f"Restoring table: {table_op.full_name}")
@@ -246,6 +246,8 @@ def restore(
     console: Console = default_console,
 ) -> None:
     """Restore a CSV file from S3 to the database."""
+    assert table_op.full_name
+
     # Force table_name to be fully qualified
     schema, table = parse_table(table_op.full_name)
     table_name = f"{schema}.{table}"

src/databudgie/table_op.py

@@ -41,9 +41,9 @@ class TableOp(Generic[T]):
     table matching that criteria.
     """
 
-    schema: str
-    table_name: str
-    full_name: str
+    schema: str | None
+    table_name: str | None
+    full_name: str | None
     raw_conf: T
 
     @classmethod
@@ -61,9 +61,20 @@ def query(self) -> str:
 
         return query.format(table=self.full_name)
 
-    def schema_op(self) -> SchemaOp:
+    def schema_op(self) -> SchemaOp | None:
+        if self.schema is None:
+            return None
+
         return SchemaOp(self.schema, self.raw_conf)
 
+    @property
+    def pretty_name(self) -> str:
+        if self.full_name:
+            return self.full_name
+
+        assert isinstance(self.raw_conf, BackupTableConfig)
+        return self.raw_conf.query
+
 
 def expand_table_ops(
     session: Session,
@@ -89,8 +100,13 @@ def expand_table_ops(
 
     # expand table globs into fully qualified mappings to the config.
     matching_tables: dict[str, list[T]] = {}
+    unnamed_tables: list[T] = []
     for table_conf in tables:
         pattern = table_conf.name
+        if pattern is None:
+            unnamed_tables.append(table_conf)
+            continue
+
         if "." not in pattern:
             pattern = f"{default_schema_name}.{pattern}"
 
@@ -128,4 +144,7 @@ def expand_table_ops(
             table_op = TableOp.from_name(table, raw_conf=table_conf)
             result.append(table_op)
 
+    for unnamed_table in unnamed_tables:
+        result.append(TableOp(None, None, None, unnamed_table))
+
     return result

tests/conftest.py

@@ -1,3 +1,5 @@
+import logging
+
 import boto3
 import pytest
 from freezegun import freeze_time
@@ -7,6 +9,8 @@
 from databudgie.config import RootConfig
 from tests.mockmodels.models import Base
 
+logging.basicConfig(level="INFO")
+
 
 @pytest.fixture(scope="session")
 def pmr_postgres_config():

tests/test_backup.py

@@ -177,6 +177,26 @@ def test_backup_failure(pg):
             assert console.call_count == 2
 
 
+def test_backup_unnamed_table(pg, mf, s3_resource):
+    """Validate unnamed table can be backed up."""
+    customer = mf.customer.new(external_id="cid_123")
+
+    config = RootConfig.from_dict(
+        {
+            "backup": {
+                "location": "s3://sample-bucket/databudgie/test/{table}",
+                "tables": [{"query": "select * from public.customer"}],
+                **s3_config,
+            },
+        }
+    )
+    backup_all(pg, config.backup)
+
+    _validate_backup_contents(
+        get_file_buffer("s3://sample/databudgie/test/None/2021-04-26T09:00:00.csv", s3_resource), [customer]
+    )
+
+
 def _validate_backup_contents(buffer, expected_contents: List[Customer]):
     """Validate the contents of a backup file. Columns from the file will be raw."""
 

tests/utils.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 from typing import List
 
+from botocore.exceptions import ClientError
 from mypy_boto3_s3.service_resource import S3ServiceResource
 
 from databudgie.s3 import is_s3_path, S3Location
@@ -49,7 +50,13 @@ def get_file_buffer(filename, s3_resource=None):
         assert s3_resource
         location = S3Location(filename)
         uploaded_object = s3_resource.Object("sample-bucket", location.key)
-        uploaded_object.download_fileobj(buffer)
+
+        try:
+            uploaded_object.download_fileobj(buffer)
+        except ClientError:
+            log.info(str(list(s3_resource.Bucket("sample-bucket").objects.all())))
+
+            raise
     else:
         try:
             with open(filename, "rb") as f: