From 1c5fb73a23f2e7ec5a90e20b6461aeeea62362e5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 13 May 2024 15:52:46 -0700 Subject: [PATCH 01/26] fixed repo name --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 0f04542cc..98fad5c10 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ API Reference Changelog --------- -For a list of all ``google-cloud-datastore`` releases: +For a list of all ``google-cloud-bigtable`` releases: .. toctree:: :maxdepth: 2 From 8cc204ff535d01c9f9690728742f20bd3a08b344 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 14 May 2024 14:13:31 -0700 Subject: [PATCH 02/26] added toc template --- docs/toc.yml | 375 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 docs/toc.yml diff --git a/docs/toc.yml b/docs/toc.yml new file mode 100644 index 000000000..4b28cc226 --- /dev/null +++ b/docs/toc.yml @@ -0,0 +1,375 @@ +- items: + - href: index.md + name: Overview + - href: changelog.md + name: Changelog + + - name: Standard Client + items: + - href: client-intro.md + name: Base for Everything + - name: Overview + items: + - href: summary_class.yml + name: Classes + - href: summary_method.yml + name: Methods + - href: summary_property.yml + name: Properties and Attributes + - href: data-api.md + name: Data API + - href: table-api.md + name: Table Admin API + - href: instance-api.md + name: Instance Admin API + + - href: app-profile.md + name: App Profile + - href: backup.md + name: Backup + - href: batcher.md + name: Mutations Batching + - href: client.md + name: Client + - href: cluster.md + name: Cluster + - href: column-family.md + name: Column Families + - href: encryption-info.md + name: Encryption Info + - href: instance.md + name: Instance + - href: multiprocessing.md + name: Multiprocessing + - href: row-data.md + name: Row Data + - href: row-filters.md + name: Bigtable Row Filters + - href: row-set.md + name: Row Set + - href: row.md + name: Bigtable Row + - href: table.md + name: Table + + + - name: Async Data Client + items: + - name: BigtableDataClientAsync + href: async_data_client.md + - name: Mutations Batcher + href: async_data_mutations_batcher.md + - name: ReadRowsQuery + href: async_data_read_rows_query.md + - name: Row + href: async_data_row.md + - name: Row Filters + href: async_data_row_filters.md + - name: Mutations + href: async_data_mutations.md + - name: Read Modify Write Rules + href: async_data_read_modify_write_rules.md + - name: Exceptions + href: async_data_exceptions.md + + + - items: + - items: + - name: Overview + uid: google.cloud.bigtable.app_profile + - name: AppProfile + uid: google.cloud.bigtable.app_profile.AppProfile + name: app_profile + uid: google.cloud.bigtable.app_profile + - items: + - name: Overview + uid: google.cloud.bigtable.backup + - name: Backup + uid: google.cloud.bigtable.backup.Backup + name: backup + uid: google.cloud.bigtable.backup + - items: + - name: Overview + uid: google.cloud.bigtable.batcher + - name: MutationsBatchError + uid: google.cloud.bigtable.batcher.MutationsBatchError + - name: MutationsBatcher + uid: google.cloud.bigtable.batcher.MutationsBatcher + name: batcher + uid: google.cloud.bigtable.batcher + - items: + - name: Overview + uid: google.cloud.bigtable.client + - name: Client + uid: google.cloud.bigtable.client.Client + name: client + uid: google.cloud.bigtable.client + - items: + - name: Overview + uid: google.cloud.bigtable.cluster + - name: Cluster + uid: google.cloud.bigtable.cluster.Cluster + name: cluster + uid: google.cloud.bigtable.cluster + - items: + - name: Overview + uid: google.cloud.bigtable.column_family + - name: ColumnFamily + uid: google.cloud.bigtable.column_family.ColumnFamily + - name: GCRuleIntersection + uid: google.cloud.bigtable.column_family.GCRuleIntersection + - name: GCRuleUnion + uid: google.cloud.bigtable.column_family.GCRuleUnion + - name: GarbageCollectionRule + uid: google.cloud.bigtable.column_family.GarbageCollectionRule + - name: MaxAgeGCRule + uid: google.cloud.bigtable.column_family.MaxAgeGCRule + - name: MaxVersionsGCRule + uid: google.cloud.bigtable.column_family.MaxVersionsGCRule + name: column_family + uid: google.cloud.bigtable.column_family + - items: + - name: Overview + uid: google.cloud.bigtable.data._async.client + - name: BigtableDataClientAsync + uid: google.cloud.bigtable.data._async.client.BigtableDataClientAsync + - name: TableAsync + uid: google.cloud.bigtable.data._async.client.TableAsync + name: data.client + uid: google.cloud.bigtable.data._async.client + - items: + - name: Overview + uid: google.cloud.bigtable.data._async.mutations_batcher + - name: MutationsBatcherAsync + uid: google.cloud.bigtable.data._async.mutations_batcher.MutationsBatcherAsync + name: mutations_batcher + uid: google.cloud.bigtable.data._async.mutations_batcher + - items: + - name: Overview + uid: google.cloud.bigtable.data.exceptions + - name: FailedMutationEntryError + uid: google.cloud.bigtable.data.exceptions.FailedMutationEntryError + - name: FailedQueryShardError + uid: google.cloud.bigtable.data.exceptions.FailedQueryShardError + - name: InvalidChunk + uid: google.cloud.bigtable.data.exceptions.InvalidChunk + - name: MutationsExceptionGroup + uid: google.cloud.bigtable.data.exceptions.MutationsExceptionGroup + - name: RetryExceptionGroup + uid: google.cloud.bigtable.data.exceptions.RetryExceptionGroup + - name: ShardedReadRowsExceptionGroup + uid: google.cloud.bigtable.data.exceptions.ShardedReadRowsExceptionGroup + name: exceptions + uid: google.cloud.bigtable.data.exceptions + - items: + - name: Overview + uid: google.cloud.bigtable.data.mutations + - name: DeleteAllFromFamily + uid: google.cloud.bigtable.data.mutations.DeleteAllFromFamily + - name: DeleteAllFromRow + uid: google.cloud.bigtable.data.mutations.DeleteAllFromRow + - name: DeleteRangeFromColumn + uid: google.cloud.bigtable.data.mutations.DeleteRangeFromColumn + - name: Mutation + uid: google.cloud.bigtable.data.mutations.Mutation + - name: SetCell + uid: google.cloud.bigtable.data.mutations.SetCell + name: mutations + uid: google.cloud.bigtable.data.mutations + - items: + - name: Overview + uid: google.cloud.bigtable.data.read_modify_write_rules + - name: AppendValueRule + uid: google.cloud.bigtable.data.read_modify_write_rules.AppendValueRule + - name: IncrementRule + uid: google.cloud.bigtable.data.read_modify_write_rules.IncrementRule + - name: ReadModifyWriteRule + uid: google.cloud.bigtable.data.read_modify_write_rules.ReadModifyWriteRule + name: read_modify_write_rules + uid: google.cloud.bigtable.data.read_modify_write_rules + - items: + - name: Overview + uid: google.cloud.bigtable.data.read_rows_query + - name: ReadRowsQuery + uid: google.cloud.bigtable.data.read_rows_query.ReadRowsQuery + - name: RowRange + uid: google.cloud.bigtable.data.read_rows_query.RowRange + name: read_rows_query + uid: google.cloud.bigtable.data.read_rows_query + - items: + - name: Overview + uid: google.cloud.bigtable.data.row + - name: Cell + uid: google.cloud.bigtable.data.row.Cell + - name: Row + uid: google.cloud.bigtable.data.row.Row + name: data.row + uid: google.cloud.bigtable.data.row + - items: + - name: Overview + uid: google.cloud.bigtable.data.row_filters + - name: ApplyLabelFilter + uid: google.cloud.bigtable.data.row_filters.ApplyLabelFilter + - name: BlockAllFilter + uid: google.cloud.bigtable.data.row_filters.BlockAllFilter + - name: CellsColumnLimitFilter + uid: google.cloud.bigtable.data.row_filters.CellsColumnLimitFilter + - name: CellsRowLimitFilter + uid: google.cloud.bigtable.data.row_filters.CellsRowLimitFilter + - name: CellsRowOffsetFilter + uid: google.cloud.bigtable.data.row_filters.CellsRowOffsetFilter + - name: ColumnQualifierRegexFilter + uid: google.cloud.bigtable.data.row_filters.ColumnQualifierRegexFilter + - name: ColumnRangeFilter + uid: google.cloud.bigtable.data.row_filters.ColumnRangeFilter + - name: ConditionalRowFilter + uid: google.cloud.bigtable.data.row_filters.ConditionalRowFilter + - name: FamilyNameRegexFilter + uid: google.cloud.bigtable.data.row_filters.FamilyNameRegexFilter + - name: LiteralValueFilter + uid: google.cloud.bigtable.data.row_filters.LiteralValueFilter + - name: PassAllFilter + uid: google.cloud.bigtable.data.row_filters.PassAllFilter + - name: RowFilter + uid: google.cloud.bigtable.data.row_filters.RowFilter + - name: RowFilterChain + uid: google.cloud.bigtable.data.row_filters.RowFilterChain + - name: RowFilterUnion + uid: google.cloud.bigtable.data.row_filters.RowFilterUnion + - name: RowKeyRegexFilter + uid: google.cloud.bigtable.data.row_filters.RowKeyRegexFilter + - name: RowSampleFilter + uid: google.cloud.bigtable.data.row_filters.RowSampleFilter + - name: SinkFilter + uid: google.cloud.bigtable.data.row_filters.SinkFilter + - name: StripValueTransformerFilter + uid: google.cloud.bigtable.data.row_filters.StripValueTransformerFilter + - name: TimestampRange + uid: google.cloud.bigtable.data.row_filters.TimestampRange + - name: TimestampRangeFilter + uid: google.cloud.bigtable.data.row_filters.TimestampRangeFilter + - name: ValueRangeFilter + uid: google.cloud.bigtable.data.row_filters.ValueRangeFilter + - name: ValueRegexFilter + uid: google.cloud.bigtable.data.row_filters.ValueRegexFilter + name: data.row_filters + uid: google.cloud.bigtable.data.row_filters + - items: + - name: Overview + uid: google.cloud.bigtable.encryption_info + - name: EncryptionInfo + uid: google.cloud.bigtable.encryption_info.EncryptionInfo + name: encryption_info + uid: google.cloud.bigtable.encryption_info + - items: + - name: Overview + uid: google.cloud.bigtable.instance + - name: Instance + uid: google.cloud.bigtable.instance.Instance + name: instance + uid: google.cloud.bigtable.instance + - items: + - name: Overview + uid: google.cloud.bigtable.row + - name: AppendRow + uid: google.cloud.bigtable.row.AppendRow + - name: Cell + uid: google.cloud.bigtable.row.Cell + - name: ConditionalRow + uid: google.cloud.bigtable.row.ConditionalRow + - name: DirectRow + uid: google.cloud.bigtable.row.DirectRow + - name: InvalidChunk + uid: google.cloud.bigtable.row.InvalidChunk + - name: PartialRowData + uid: google.cloud.bigtable.row.PartialRowData + - name: Row + uid: google.cloud.bigtable.row.Row + name: row + uid: google.cloud.bigtable.row + - items: + - name: Overview + uid: google.cloud.bigtable.row_data + - name: InvalidReadRowsResponse + uid: google.cloud.bigtable.row_data.InvalidReadRowsResponse + - name: InvalidRetryRequest + uid: google.cloud.bigtable.row_data.InvalidRetryRequest + - name: PartialCellData + uid: google.cloud.bigtable.row_data.PartialCellData + - name: PartialRowsData + uid: google.cloud.bigtable.row_data.PartialRowsData + name: row_data + uid: google.cloud.bigtable.row_data + - items: + - name: Overview + uid: google.cloud.bigtable.row_filters + - name: ApplyLabelFilter + uid: google.cloud.bigtable.row_filters.ApplyLabelFilter + - name: BlockAllFilter + uid: google.cloud.bigtable.row_filters.BlockAllFilter + - name: CellsColumnLimitFilter + uid: google.cloud.bigtable.row_filters.CellsColumnLimitFilter + - name: CellsRowLimitFilter + uid: google.cloud.bigtable.row_filters.CellsRowLimitFilter + - name: CellsRowOffsetFilter + uid: google.cloud.bigtable.row_filters.CellsRowOffsetFilter + - name: ColumnQualifierRegexFilter + uid: google.cloud.bigtable.row_filters.ColumnQualifierRegexFilter + - name: ColumnRangeFilter + uid: google.cloud.bigtable.row_filters.ColumnRangeFilter + - name: ConditionalRowFilter + uid: google.cloud.bigtable.row_filters.ConditionalRowFilter + - name: ExactValueFilter + uid: google.cloud.bigtable.row_filters.ExactValueFilter + - name: FamilyNameRegexFilter + uid: google.cloud.bigtable.row_filters.FamilyNameRegexFilter + - name: PassAllFilter + uid: google.cloud.bigtable.row_filters.PassAllFilter + - name: RowFilter + uid: google.cloud.bigtable.row_filters.RowFilter + - name: RowFilterChain + uid: google.cloud.bigtable.row_filters.RowFilterChain + - name: RowFilterUnion + uid: google.cloud.bigtable.row_filters.RowFilterUnion + - name: RowKeyRegexFilter + uid: google.cloud.bigtable.row_filters.RowKeyRegexFilter + - name: RowSampleFilter + uid: google.cloud.bigtable.row_filters.RowSampleFilter + - name: SinkFilter + uid: google.cloud.bigtable.row_filters.SinkFilter + - name: StripValueTransformerFilter + uid: google.cloud.bigtable.row_filters.StripValueTransformerFilter + - name: TimestampRange + uid: google.cloud.bigtable.row_filters.TimestampRange + - name: TimestampRangeFilter + uid: google.cloud.bigtable.row_filters.TimestampRangeFilter + - name: ValueRangeFilter + uid: google.cloud.bigtable.row_filters.ValueRangeFilter + - name: ValueRegexFilter + uid: google.cloud.bigtable.row_filters.ValueRegexFilter + name: row_filters + uid: google.cloud.bigtable.row_filters + - items: + - name: Overview + uid: google.cloud.bigtable.row_set + - name: RowRange + uid: google.cloud.bigtable.row_set.RowRange + - name: RowSet + uid: google.cloud.bigtable.row_set.RowSet + name: row_set + uid: google.cloud.bigtable.row_set + - items: + - name: Overview + uid: google.cloud.bigtable.table + - name: ClusterState + uid: google.cloud.bigtable.table.ClusterState + - name: Table + uid: google.cloud.bigtable.table.Table + - name: TableMismatchError + uid: google.cloud.bigtable.table.TableMismatchError + - name: TooManyMutationsError + uid: google.cloud.bigtable.table.TooManyMutationsError + name: table + uid: google.cloud.bigtable.table + name: Bigtable API Reference + name: google-cloud-bigtable From 8b433f195ea7defe33573feecac7dbe9190b6ca3 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 14 May 2024 14:36:12 -0700 Subject: [PATCH 03/26] used more standard namings --- docs/toc.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/toc.yml b/docs/toc.yml index 4b28cc226..d4028d787 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -3,19 +3,21 @@ name: Overview - href: changelog.md name: Changelog + - name: bigtable APIs + items: + - href: summary_overview.md + name: Overview + - href: summary_class.yml + name: Classes + - href: summary_method.yml + name: Methods + - href: summary_property.yml + name: Properties and Attributes - name: Standard Client items: - href: client-intro.md name: Base for Everything - - name: Overview - items: - - href: summary_class.yml - name: Classes - - href: summary_method.yml - name: Methods - - href: summary_property.yml - name: Properties and Attributes - href: data-api.md name: Data API - href: table-api.md @@ -371,5 +373,5 @@ uid: google.cloud.bigtable.table.TooManyMutationsError name: table uid: google.cloud.bigtable.table - name: Bigtable API Reference + name: Bigtable name: google-cloud-bigtable From a23173a45e44b64c37722e651406d31aee4c858f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 14 May 2024 14:48:46 -0700 Subject: [PATCH 04/26] moved API reference under standard client --- docs/async_data_usage.rst | 4 ++-- docs/index.rst | 12 +----------- docs/usage.rst | 8 ++++++-- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/docs/async_data_usage.rst b/docs/async_data_usage.rst index c436c5988..5ea4f79ac 100644 --- a/docs/async_data_usage.rst +++ b/docs/async_data_usage.rst @@ -1,5 +1,5 @@ -Using the Async Data Client -=========================== +Async Data Client +================= .. toctree:: :maxdepth: 2 diff --git a/docs/index.rst b/docs/index.rst index 98fad5c10..e34dba4e3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ .. include:: multiprocessing.rst -Using the API +Client Types ------------- .. toctree:: :maxdepth: 2 @@ -11,16 +11,6 @@ Using the API async_data_usage -API Reference -------------- -.. toctree:: - :maxdepth: 2 - - instance-api - table-api - data-api - - Changelog --------- diff --git a/docs/usage.rst b/docs/usage.rst index de0abac9c..dce49ea2d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,5 +1,5 @@ -Using the Sync Client -===================== +Standard Client +=============== .. toctree:: :maxdepth: 2 @@ -19,6 +19,10 @@ Using the Sync Client row-set batcher + instance-api + table-api + data-api + In the hierarchy of API concepts From a022366bf3604b41884f99f019d0836110c881d9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 14 May 2024 14:54:32 -0700 Subject: [PATCH 05/26] reordered section --- docs/usage.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index dce49ea2d..409eb5516 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -5,6 +5,11 @@ Standard Client :maxdepth: 2 client-intro + + instance-api + table-api + data-api + client cluster instance @@ -19,10 +24,6 @@ Standard Client row-set batcher - instance-api - table-api - data-api - In the hierarchy of API concepts From b3d8fe9150e8dc276096ad88ceb906eb8fe53e06 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 14 May 2024 15:02:45 -0700 Subject: [PATCH 06/26] broke table out separately --- docs/async_data_client.rst | 2 +- docs/async_data_usage.rst | 1 + docs/toc.yml | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/async_data_client.rst b/docs/async_data_client.rst index 7d2901de4..c5cc70740 100644 --- a/docs/async_data_client.rst +++ b/docs/async_data_client.rst @@ -1,6 +1,6 @@ Bigtable Data Client Async ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: google.cloud.bigtable.data._async.client +.. autoclass:: google.cloud.bigtable.data._async.client.BigtableDataClientAsync :members: :show-inheritance: diff --git a/docs/async_data_usage.rst b/docs/async_data_usage.rst index 5ea4f79ac..8843b506b 100644 --- a/docs/async_data_usage.rst +++ b/docs/async_data_usage.rst @@ -5,6 +5,7 @@ Async Data Client :maxdepth: 2 async_data_client + async_data_table async_data_mutations_batcher async_data_read_rows_query async_data_row diff --git a/docs/toc.yml b/docs/toc.yml index d4028d787..0df3b28a6 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -59,6 +59,8 @@ items: - name: BigtableDataClientAsync href: async_data_client.md + - name: TableAsync + href: async_data_table.md - name: Mutations Batcher href: async_data_mutations_batcher.md - name: ReadRowsQuery From 26212856a7251826109aed4a7368940ef099c62e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 14 May 2024 16:22:32 -0700 Subject: [PATCH 07/26] added table file --- docs/async_data_table.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/async_data_table.rst diff --git a/docs/async_data_table.rst b/docs/async_data_table.rst new file mode 100644 index 000000000..a977beb6a --- /dev/null +++ b/docs/async_data_table.rst @@ -0,0 +1,6 @@ +Table Async +~~~~~~~~~~~ + +.. autoclass:: google.cloud.bigtable.data._async.client.TableAsync + :members: + :show-inheritance: From 94ab7cc93cedf749315d82f0bf2ebe4cb14327eb Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 14 May 2024 16:33:50 -0700 Subject: [PATCH 08/26] moved files into directories --- docs/{ => async_data_client}/async_data_client.rst | 0 docs/{ => async_data_client}/async_data_exceptions.rst | 0 docs/{ => async_data_client}/async_data_mutations.rst | 0 docs/{ => async_data_client}/async_data_mutations_batcher.rst | 0 .../async_data_read_modify_write_rules.rst | 0 docs/{ => async_data_client}/async_data_read_rows_query.rst | 0 docs/{ => async_data_client}/async_data_row.rst | 0 docs/{ => async_data_client}/async_data_row_filters.rst | 0 docs/{ => async_data_client}/async_data_table.rst | 0 docs/{ => async_data_client}/async_data_usage.rst | 0 docs/index.rst | 4 ++-- docs/{ => standard_client}/app-profile.rst | 0 docs/{ => standard_client}/backup.rst | 0 docs/{ => standard_client}/batcher.rst | 0 docs/{ => standard_client}/client-intro.rst | 0 docs/{ => standard_client}/client.rst | 0 docs/{ => standard_client}/cluster.rst | 0 docs/{ => standard_client}/column-family.rst | 0 docs/{ => standard_client}/data-api.rst | 0 docs/{ => standard_client}/encryption-info.rst | 0 docs/{ => standard_client}/instance-api.rst | 0 docs/{ => standard_client}/instance.rst | 0 docs/{ => standard_client}/row-data.rst | 0 docs/{ => standard_client}/row-filters.rst | 0 docs/{ => standard_client}/row-set.rst | 0 docs/{ => standard_client}/row.rst | 0 docs/{ => standard_client}/snippets.py | 0 docs/{ => standard_client}/snippets_table.py | 0 docs/{ => standard_client}/table-api.rst | 0 docs/{ => standard_client}/table.rst | 0 docs/{ => standard_client}/usage.rst | 0 31 files changed, 2 insertions(+), 2 deletions(-) rename docs/{ => async_data_client}/async_data_client.rst (100%) rename docs/{ => async_data_client}/async_data_exceptions.rst (100%) rename docs/{ => async_data_client}/async_data_mutations.rst (100%) rename docs/{ => async_data_client}/async_data_mutations_batcher.rst (100%) rename docs/{ => async_data_client}/async_data_read_modify_write_rules.rst (100%) rename docs/{ => async_data_client}/async_data_read_rows_query.rst (100%) rename docs/{ => async_data_client}/async_data_row.rst (100%) rename docs/{ => async_data_client}/async_data_row_filters.rst (100%) rename docs/{ => async_data_client}/async_data_table.rst (100%) rename docs/{ => async_data_client}/async_data_usage.rst (100%) rename docs/{ => standard_client}/app-profile.rst (100%) rename docs/{ => standard_client}/backup.rst (100%) rename docs/{ => standard_client}/batcher.rst (100%) rename docs/{ => standard_client}/client-intro.rst (100%) rename docs/{ => standard_client}/client.rst (100%) rename docs/{ => standard_client}/cluster.rst (100%) rename docs/{ => standard_client}/column-family.rst (100%) rename docs/{ => standard_client}/data-api.rst (100%) rename docs/{ => standard_client}/encryption-info.rst (100%) rename docs/{ => standard_client}/instance-api.rst (100%) rename docs/{ => standard_client}/instance.rst (100%) rename docs/{ => standard_client}/row-data.rst (100%) rename docs/{ => standard_client}/row-filters.rst (100%) rename docs/{ => standard_client}/row-set.rst (100%) rename docs/{ => standard_client}/row.rst (100%) rename docs/{ => standard_client}/snippets.py (100%) rename docs/{ => standard_client}/snippets_table.py (100%) rename docs/{ => standard_client}/table-api.rst (100%) rename docs/{ => standard_client}/table.rst (100%) rename docs/{ => standard_client}/usage.rst (100%) diff --git a/docs/async_data_client.rst b/docs/async_data_client/async_data_client.rst similarity index 100% rename from docs/async_data_client.rst rename to docs/async_data_client/async_data_client.rst diff --git a/docs/async_data_exceptions.rst b/docs/async_data_client/async_data_exceptions.rst similarity index 100% rename from docs/async_data_exceptions.rst rename to docs/async_data_client/async_data_exceptions.rst diff --git a/docs/async_data_mutations.rst b/docs/async_data_client/async_data_mutations.rst similarity index 100% rename from docs/async_data_mutations.rst rename to docs/async_data_client/async_data_mutations.rst diff --git a/docs/async_data_mutations_batcher.rst b/docs/async_data_client/async_data_mutations_batcher.rst similarity index 100% rename from docs/async_data_mutations_batcher.rst rename to docs/async_data_client/async_data_mutations_batcher.rst diff --git a/docs/async_data_read_modify_write_rules.rst b/docs/async_data_client/async_data_read_modify_write_rules.rst similarity index 100% rename from docs/async_data_read_modify_write_rules.rst rename to docs/async_data_client/async_data_read_modify_write_rules.rst diff --git a/docs/async_data_read_rows_query.rst b/docs/async_data_client/async_data_read_rows_query.rst similarity index 100% rename from docs/async_data_read_rows_query.rst rename to docs/async_data_client/async_data_read_rows_query.rst diff --git a/docs/async_data_row.rst b/docs/async_data_client/async_data_row.rst similarity index 100% rename from docs/async_data_row.rst rename to docs/async_data_client/async_data_row.rst diff --git a/docs/async_data_row_filters.rst b/docs/async_data_client/async_data_row_filters.rst similarity index 100% rename from docs/async_data_row_filters.rst rename to docs/async_data_client/async_data_row_filters.rst diff --git a/docs/async_data_table.rst b/docs/async_data_client/async_data_table.rst similarity index 100% rename from docs/async_data_table.rst rename to docs/async_data_client/async_data_table.rst diff --git a/docs/async_data_usage.rst b/docs/async_data_client/async_data_usage.rst similarity index 100% rename from docs/async_data_usage.rst rename to docs/async_data_client/async_data_usage.rst diff --git a/docs/index.rst b/docs/index.rst index e34dba4e3..b2e48559f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,8 +7,8 @@ Client Types .. toctree:: :maxdepth: 2 - usage - async_data_usage + standard_client/usage + async_data_client/async_data_usage Changelog diff --git a/docs/app-profile.rst b/docs/standard_client/app-profile.rst similarity index 100% rename from docs/app-profile.rst rename to docs/standard_client/app-profile.rst diff --git a/docs/backup.rst b/docs/standard_client/backup.rst similarity index 100% rename from docs/backup.rst rename to docs/standard_client/backup.rst diff --git a/docs/batcher.rst b/docs/standard_client/batcher.rst similarity index 100% rename from docs/batcher.rst rename to docs/standard_client/batcher.rst diff --git a/docs/client-intro.rst b/docs/standard_client/client-intro.rst similarity index 100% rename from docs/client-intro.rst rename to docs/standard_client/client-intro.rst diff --git a/docs/client.rst b/docs/standard_client/client.rst similarity index 100% rename from docs/client.rst rename to docs/standard_client/client.rst diff --git a/docs/cluster.rst b/docs/standard_client/cluster.rst similarity index 100% rename from docs/cluster.rst rename to docs/standard_client/cluster.rst diff --git a/docs/column-family.rst b/docs/standard_client/column-family.rst similarity index 100% rename from docs/column-family.rst rename to docs/standard_client/column-family.rst diff --git a/docs/data-api.rst b/docs/standard_client/data-api.rst similarity index 100% rename from docs/data-api.rst rename to docs/standard_client/data-api.rst diff --git a/docs/encryption-info.rst b/docs/standard_client/encryption-info.rst similarity index 100% rename from docs/encryption-info.rst rename to docs/standard_client/encryption-info.rst diff --git a/docs/instance-api.rst b/docs/standard_client/instance-api.rst similarity index 100% rename from docs/instance-api.rst rename to docs/standard_client/instance-api.rst diff --git a/docs/instance.rst b/docs/standard_client/instance.rst similarity index 100% rename from docs/instance.rst rename to docs/standard_client/instance.rst diff --git a/docs/row-data.rst b/docs/standard_client/row-data.rst similarity index 100% rename from docs/row-data.rst rename to docs/standard_client/row-data.rst diff --git a/docs/row-filters.rst b/docs/standard_client/row-filters.rst similarity index 100% rename from docs/row-filters.rst rename to docs/standard_client/row-filters.rst diff --git a/docs/row-set.rst b/docs/standard_client/row-set.rst similarity index 100% rename from docs/row-set.rst rename to docs/standard_client/row-set.rst diff --git a/docs/row.rst b/docs/standard_client/row.rst similarity index 100% rename from docs/row.rst rename to docs/standard_client/row.rst diff --git a/docs/snippets.py b/docs/standard_client/snippets.py similarity index 100% rename from docs/snippets.py rename to docs/standard_client/snippets.py diff --git a/docs/snippets_table.py b/docs/standard_client/snippets_table.py similarity index 100% rename from docs/snippets_table.py rename to docs/standard_client/snippets_table.py diff --git a/docs/table-api.rst b/docs/standard_client/table-api.rst similarity index 100% rename from docs/table-api.rst rename to docs/standard_client/table-api.rst diff --git a/docs/table.rst b/docs/standard_client/table.rst similarity index 100% rename from docs/table.rst rename to docs/standard_client/table.rst diff --git a/docs/usage.rst b/docs/standard_client/usage.rst similarity index 100% rename from docs/usage.rst rename to docs/standard_client/usage.rst From 2a80311e6f914d9c81eac277b8af4a9b903eade1 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 14 May 2024 17:43:18 -0700 Subject: [PATCH 09/26] added rough patcher script --- docs/scripts/patch_devsite_toc.py | 79 +++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/scripts/patch_devsite_toc.py diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py new file mode 100644 index 000000000..017638601 --- /dev/null +++ b/docs/scripts/patch_devsite_toc.py @@ -0,0 +1,79 @@ +""" +python-bigtable is made up of multiple clients, which we want to categorize separately in +the devsite table of contents. + +To accomplish this, we will read in the generated toc file, remove the markdown content, +and replace it with a hand-written template +""" + + +import yaml +import os + +STANDARD_CLIENT_SECTION = [ + "client-intro.md", + "data-api.md", + "table-api.md", + "instance-api.md", + "app-profile.md", + "backup.md", + "batcher.md", + "client.md", + "cluster.md", + "column-family.md", + "encryption-info.md", + "instance.md", + "multiprocessing.md", + "row-data.md", + "row-filters.md", + "row-set.md", + "row.md", + "table.md", +] + +SECTIONS = [ + { + "prefix": "async_data_", + "title": "Async Data Client", + "items": [] + } +] + +# DESIRED_ROOT = ["Overview", "Changelog", "bigtable APIs", STANDARD_CLIENT_SECTION, ASYNC_CLIENT_SECTION, "Bigtable"] + +os.chdir(os.path.dirname(os.path.abspath(__file__))) + +toc_file_path = os.path.join( + os.getcwd(), os.pardir, "_build", "html", "docfx_yaml", "toc.yml" +) +root_toc = yaml.safe_load(open(toc_file_path, "r"))[0]["items"] + +# new_toc = [] +# for item in root_toc: +# if item.get("href", "").startswith(SECTIONS[0]["prefix"]): +# SECTIONS[0]["items"].append(item) +# else: +# new_toc.append(item) +# new_toc.append(SECTIONS[0]) + +client_docs_dir = "async_data_client" +found_files = [] +for file in os.listdir(os.path.join(os.getcwd(), os.pardir, client_docs_dir)): + file_path = os.path.join(os.getcwd(), os.pardir, client_docs_dir, file) + if file.endswith(".rst"): + base_name = file.replace(".rst", "") + title = open(file_path, "r").readline().replace("\n", "") + found_files.append( + { + "name": title, + "href": f"{base_name}.md", + } + ) +new_items = root_toc + [{"name": "Async Data Client", "items": found_files}] + +out_file_path = os.path.join( + os.getcwd(), os.pardir, "_build", "html", "docfx_yaml", "toc2.yml" +) +with open(out_file_path, "w") as f: + f.write(yaml.dump([{"items": new_items, "name": "google-cloud-bigtable"}])) +print("done") From d03ffe950a752a7b3b8c0f3e23dff1199c5c40a0 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 14:45:26 -0700 Subject: [PATCH 10/26] improve customzation script --- docs/scripts/patch_devsite_toc.py | 131 ++++++++++++++++-------------- noxfile.py | 17 ++++ owlbot.py | 30 +++++++ 3 files changed, 118 insertions(+), 60 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index 017638601..d230c858b 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -9,71 +9,82 @@ import yaml import os +import shutil -STANDARD_CLIENT_SECTION = [ - "client-intro.md", - "data-api.md", - "table-api.md", - "instance-api.md", - "app-profile.md", - "backup.md", - "batcher.md", - "client.md", - "cluster.md", - "column-family.md", - "encryption-info.md", - "instance.md", - "multiprocessing.md", - "row-data.md", - "row-filters.md", - "row-set.md", - "row.md", - "table.md", -] +# set workinf directory to /docs +os.chdir(f"{os.path.dirname(os.path.abspath(__file__))}/{os.pardir}") -SECTIONS = [ - { - "prefix": "async_data_", - "title": "Async Data Client", - "items": [] - } -] +class TocSection: + def __init__(self, dir_name, index_file_name): + """ + :param dir_name: name of the directory containing the rst files + :param index_file_name: name of an index file within dir_name. This file + will not be included in the table of contents, but provides an ordered + list of the other files which should be included + """ + self.dir_name = dir_name + index_file_path = os.path.join(dir_name, index_file_name) + with open(index_file_path, "r") as f: + self.title = f.readline().replace("\n", "") + in_toc = False + self.items = [] + for line in f: + if line.startswith(".. toctree::"): + in_toc = True + elif in_toc: + if not line.strip(): + # ignore empty lines + continue + elif line.startswith(" ") or line.startswith("\t"): + # keep track of the items in the table of contents + # ignore : directives + if ":" not in line: + file_name = line.strip() + # load the file to get the title + with open(f"{dir_name}/{file_name}.rst", "r") as f2: + file_title = f2.readline().replace("\n", "") + self.items.append({"name": file_title, "href": f"{file_name}.md"}) + else: + # if we are out of the indented block, we are done + break + in_toc = False -# DESIRED_ROOT = ["Overview", "Changelog", "bigtable APIs", STANDARD_CLIENT_SECTION, ASYNC_CLIENT_SECTION, "Bigtable"] + def to_dict(self): + """ + Convert the TocSection object to a dictionary that can be written to a yaml file + """ + return {"name": self.title, "items": self.items} -os.chdir(os.path.dirname(os.path.abspath(__file__))) + def copy_markdown(self): + """ + Copy markdown files from _build/markdown/dir_name to _build/html/docfx_yaml -toc_file_path = os.path.join( - os.getcwd(), os.pardir, "_build", "html", "docfx_yaml", "toc.yml" -) -root_toc = yaml.safe_load(open(toc_file_path, "r"))[0]["items"] + This is necessary because the markdown files in dub-directories + are not copied over by the docfx build by default + """ + for file in os.listdir("_build/markdown/" + self.dir_name): + shutil.copy( + f"_build/markdown/{self.dir_name}/{file}", + f"_build/html/docfx_yaml", + ) -# new_toc = [] -# for item in root_toc: -# if item.get("href", "").startswith(SECTIONS[0]["prefix"]): -# SECTIONS[0]["items"].append(item) -# else: -# new_toc.append(item) -# new_toc.append(SECTIONS[0]) +def add_sections(toc_file_path, section_list, output_file_path=None): + """ + Add new sections to the autogenerated docfx table of contents file -client_docs_dir = "async_data_client" -found_files = [] -for file in os.listdir(os.path.join(os.getcwd(), os.pardir, client_docs_dir)): - file_path = os.path.join(os.getcwd(), os.pardir, client_docs_dir, file) - if file.endswith(".rst"): - base_name = file.replace(".rst", "") - title = open(file_path, "r").readline().replace("\n", "") - found_files.append( - { - "name": title, - "href": f"{base_name}.md", - } - ) -new_items = root_toc + [{"name": "Async Data Client", "items": found_files}] + Takes in a list of TocSection objects, which should point to a directory of rst files + within the main /docs directory, which represents a self-contained section of content -out_file_path = os.path.join( - os.getcwd(), os.pardir, "_build", "html", "docfx_yaml", "toc2.yml" -) -with open(out_file_path, "w") as f: - f.write(yaml.dump([{"items": new_items, "name": "google-cloud-bigtable"}])) -print("done") + :param toc_file_path: path to the autogenerated toc file + :param section_list: list of TocSection objects to add + :param output_file_path: path to save the updated toc file. If None, save to the input file + """ + current_toc = yaml.safe_load(open(toc_file_path, "r")) + for section in section_list: + current_toc[0]["items"].insert(-1, section.to_dict()) + section.copy_markdown() + # save file + if output_file_path is None: + output_file_path = toc_file_path + with open(output_file_path, "w") as f: + yaml.dump(current_toc, f) diff --git a/noxfile.py b/noxfile.py index f175c66da..cc0503cfb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -425,6 +425,23 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) + # Customization: Add extra sections to the table of contents for the Standard vs Async clients + import sys + sys.path.append(os.path.abspath(os.path.join("docs", "scripts"))) + from docs.scripts import patch_devsite_toc + patch_devsite_toc.add_sections( + os.path.join("_build", "html", "docfx_yaml", "toc.yml"), + [ + patch_devsite_toc.TocSection( + dir_name="async_data_client", + index_file_name="async_data_usage.rst" + ), + patch_devsite_toc.TocSection( + dir_name="standard_client", + index_file_name="usage.rst" + ), + ], + ) @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) diff --git a/owlbot.py b/owlbot.py index cde9fce64..917e04e2c 100644 --- a/owlbot.py +++ b/owlbot.py @@ -213,6 +213,36 @@ def mypy(session): ) +# add customization to docfx +docfx_postprocess = """ + # Customization: Add extra sections to the table of contents for the Standard vs Async clients + import sys + sys.path.append(os.path.abspath(os.path.join("docs", "scripts"))) + from docs.scripts import patch_devsite_toc + patch_devsite_toc.add_sections( + os.path.join("_build", "html", "docfx_yaml", "toc.yml"), + [ + patch_devsite_toc.TocSection( + dir_name="async_data_client", + index_file_name="async_data_usage.rst" + ), + patch_devsite_toc.TocSection( + dir_name="standard_client", + index_file_name="usage.rst" + ), + ], + ) +""" + +place_before( + "noxfile.py", + "@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)\n" + "def prerelease_deps(session):", + docfx_postprocess, + escape="()" +) + + @nox.session(python=DEFAULT_PYTHON_VERSION) def lint_setup_py(session): ''', From 102fb127c650402182ff5000cb186eafbccd8c24 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 14:45:56 -0700 Subject: [PATCH 11/26] remove custom toc --- docs/toc.yml | 379 --------------------------------------------------- 1 file changed, 379 deletions(-) delete mode 100644 docs/toc.yml diff --git a/docs/toc.yml b/docs/toc.yml deleted file mode 100644 index 0df3b28a6..000000000 --- a/docs/toc.yml +++ /dev/null @@ -1,379 +0,0 @@ -- items: - - href: index.md - name: Overview - - href: changelog.md - name: Changelog - - name: bigtable APIs - items: - - href: summary_overview.md - name: Overview - - href: summary_class.yml - name: Classes - - href: summary_method.yml - name: Methods - - href: summary_property.yml - name: Properties and Attributes - - - name: Standard Client - items: - - href: client-intro.md - name: Base for Everything - - href: data-api.md - name: Data API - - href: table-api.md - name: Table Admin API - - href: instance-api.md - name: Instance Admin API - - - href: app-profile.md - name: App Profile - - href: backup.md - name: Backup - - href: batcher.md - name: Mutations Batching - - href: client.md - name: Client - - href: cluster.md - name: Cluster - - href: column-family.md - name: Column Families - - href: encryption-info.md - name: Encryption Info - - href: instance.md - name: Instance - - href: multiprocessing.md - name: Multiprocessing - - href: row-data.md - name: Row Data - - href: row-filters.md - name: Bigtable Row Filters - - href: row-set.md - name: Row Set - - href: row.md - name: Bigtable Row - - href: table.md - name: Table - - - - name: Async Data Client - items: - - name: BigtableDataClientAsync - href: async_data_client.md - - name: TableAsync - href: async_data_table.md - - name: Mutations Batcher - href: async_data_mutations_batcher.md - - name: ReadRowsQuery - href: async_data_read_rows_query.md - - name: Row - href: async_data_row.md - - name: Row Filters - href: async_data_row_filters.md - - name: Mutations - href: async_data_mutations.md - - name: Read Modify Write Rules - href: async_data_read_modify_write_rules.md - - name: Exceptions - href: async_data_exceptions.md - - - - items: - - items: - - name: Overview - uid: google.cloud.bigtable.app_profile - - name: AppProfile - uid: google.cloud.bigtable.app_profile.AppProfile - name: app_profile - uid: google.cloud.bigtable.app_profile - - items: - - name: Overview - uid: google.cloud.bigtable.backup - - name: Backup - uid: google.cloud.bigtable.backup.Backup - name: backup - uid: google.cloud.bigtable.backup - - items: - - name: Overview - uid: google.cloud.bigtable.batcher - - name: MutationsBatchError - uid: google.cloud.bigtable.batcher.MutationsBatchError - - name: MutationsBatcher - uid: google.cloud.bigtable.batcher.MutationsBatcher - name: batcher - uid: google.cloud.bigtable.batcher - - items: - - name: Overview - uid: google.cloud.bigtable.client - - name: Client - uid: google.cloud.bigtable.client.Client - name: client - uid: google.cloud.bigtable.client - - items: - - name: Overview - uid: google.cloud.bigtable.cluster - - name: Cluster - uid: google.cloud.bigtable.cluster.Cluster - name: cluster - uid: google.cloud.bigtable.cluster - - items: - - name: Overview - uid: google.cloud.bigtable.column_family - - name: ColumnFamily - uid: google.cloud.bigtable.column_family.ColumnFamily - - name: GCRuleIntersection - uid: google.cloud.bigtable.column_family.GCRuleIntersection - - name: GCRuleUnion - uid: google.cloud.bigtable.column_family.GCRuleUnion - - name: GarbageCollectionRule - uid: google.cloud.bigtable.column_family.GarbageCollectionRule - - name: MaxAgeGCRule - uid: google.cloud.bigtable.column_family.MaxAgeGCRule - - name: MaxVersionsGCRule - uid: google.cloud.bigtable.column_family.MaxVersionsGCRule - name: column_family - uid: google.cloud.bigtable.column_family - - items: - - name: Overview - uid: google.cloud.bigtable.data._async.client - - name: BigtableDataClientAsync - uid: google.cloud.bigtable.data._async.client.BigtableDataClientAsync - - name: TableAsync - uid: google.cloud.bigtable.data._async.client.TableAsync - name: data.client - uid: google.cloud.bigtable.data._async.client - - items: - - name: Overview - uid: google.cloud.bigtable.data._async.mutations_batcher - - name: MutationsBatcherAsync - uid: google.cloud.bigtable.data._async.mutations_batcher.MutationsBatcherAsync - name: mutations_batcher - uid: google.cloud.bigtable.data._async.mutations_batcher - - items: - - name: Overview - uid: google.cloud.bigtable.data.exceptions - - name: FailedMutationEntryError - uid: google.cloud.bigtable.data.exceptions.FailedMutationEntryError - - name: FailedQueryShardError - uid: google.cloud.bigtable.data.exceptions.FailedQueryShardError - - name: InvalidChunk - uid: google.cloud.bigtable.data.exceptions.InvalidChunk - - name: MutationsExceptionGroup - uid: google.cloud.bigtable.data.exceptions.MutationsExceptionGroup - - name: RetryExceptionGroup - uid: google.cloud.bigtable.data.exceptions.RetryExceptionGroup - - name: ShardedReadRowsExceptionGroup - uid: google.cloud.bigtable.data.exceptions.ShardedReadRowsExceptionGroup - name: exceptions - uid: google.cloud.bigtable.data.exceptions - - items: - - name: Overview - uid: google.cloud.bigtable.data.mutations - - name: DeleteAllFromFamily - uid: google.cloud.bigtable.data.mutations.DeleteAllFromFamily - - name: DeleteAllFromRow - uid: google.cloud.bigtable.data.mutations.DeleteAllFromRow - - name: DeleteRangeFromColumn - uid: google.cloud.bigtable.data.mutations.DeleteRangeFromColumn - - name: Mutation - uid: google.cloud.bigtable.data.mutations.Mutation - - name: SetCell - uid: google.cloud.bigtable.data.mutations.SetCell - name: mutations - uid: google.cloud.bigtable.data.mutations - - items: - - name: Overview - uid: google.cloud.bigtable.data.read_modify_write_rules - - name: AppendValueRule - uid: google.cloud.bigtable.data.read_modify_write_rules.AppendValueRule - - name: IncrementRule - uid: google.cloud.bigtable.data.read_modify_write_rules.IncrementRule - - name: ReadModifyWriteRule - uid: google.cloud.bigtable.data.read_modify_write_rules.ReadModifyWriteRule - name: read_modify_write_rules - uid: google.cloud.bigtable.data.read_modify_write_rules - - items: - - name: Overview - uid: google.cloud.bigtable.data.read_rows_query - - name: ReadRowsQuery - uid: google.cloud.bigtable.data.read_rows_query.ReadRowsQuery - - name: RowRange - uid: google.cloud.bigtable.data.read_rows_query.RowRange - name: read_rows_query - uid: google.cloud.bigtable.data.read_rows_query - - items: - - name: Overview - uid: google.cloud.bigtable.data.row - - name: Cell - uid: google.cloud.bigtable.data.row.Cell - - name: Row - uid: google.cloud.bigtable.data.row.Row - name: data.row - uid: google.cloud.bigtable.data.row - - items: - - name: Overview - uid: google.cloud.bigtable.data.row_filters - - name: ApplyLabelFilter - uid: google.cloud.bigtable.data.row_filters.ApplyLabelFilter - - name: BlockAllFilter - uid: google.cloud.bigtable.data.row_filters.BlockAllFilter - - name: CellsColumnLimitFilter - uid: google.cloud.bigtable.data.row_filters.CellsColumnLimitFilter - - name: CellsRowLimitFilter - uid: google.cloud.bigtable.data.row_filters.CellsRowLimitFilter - - name: CellsRowOffsetFilter - uid: google.cloud.bigtable.data.row_filters.CellsRowOffsetFilter - - name: ColumnQualifierRegexFilter - uid: google.cloud.bigtable.data.row_filters.ColumnQualifierRegexFilter - - name: ColumnRangeFilter - uid: google.cloud.bigtable.data.row_filters.ColumnRangeFilter - - name: ConditionalRowFilter - uid: google.cloud.bigtable.data.row_filters.ConditionalRowFilter - - name: FamilyNameRegexFilter - uid: google.cloud.bigtable.data.row_filters.FamilyNameRegexFilter - - name: LiteralValueFilter - uid: google.cloud.bigtable.data.row_filters.LiteralValueFilter - - name: PassAllFilter - uid: google.cloud.bigtable.data.row_filters.PassAllFilter - - name: RowFilter - uid: google.cloud.bigtable.data.row_filters.RowFilter - - name: RowFilterChain - uid: google.cloud.bigtable.data.row_filters.RowFilterChain - - name: RowFilterUnion - uid: google.cloud.bigtable.data.row_filters.RowFilterUnion - - name: RowKeyRegexFilter - uid: google.cloud.bigtable.data.row_filters.RowKeyRegexFilter - - name: RowSampleFilter - uid: google.cloud.bigtable.data.row_filters.RowSampleFilter - - name: SinkFilter - uid: google.cloud.bigtable.data.row_filters.SinkFilter - - name: StripValueTransformerFilter - uid: google.cloud.bigtable.data.row_filters.StripValueTransformerFilter - - name: TimestampRange - uid: google.cloud.bigtable.data.row_filters.TimestampRange - - name: TimestampRangeFilter - uid: google.cloud.bigtable.data.row_filters.TimestampRangeFilter - - name: ValueRangeFilter - uid: google.cloud.bigtable.data.row_filters.ValueRangeFilter - - name: ValueRegexFilter - uid: google.cloud.bigtable.data.row_filters.ValueRegexFilter - name: data.row_filters - uid: google.cloud.bigtable.data.row_filters - - items: - - name: Overview - uid: google.cloud.bigtable.encryption_info - - name: EncryptionInfo - uid: google.cloud.bigtable.encryption_info.EncryptionInfo - name: encryption_info - uid: google.cloud.bigtable.encryption_info - - items: - - name: Overview - uid: google.cloud.bigtable.instance - - name: Instance - uid: google.cloud.bigtable.instance.Instance - name: instance - uid: google.cloud.bigtable.instance - - items: - - name: Overview - uid: google.cloud.bigtable.row - - name: AppendRow - uid: google.cloud.bigtable.row.AppendRow - - name: Cell - uid: google.cloud.bigtable.row.Cell - - name: ConditionalRow - uid: google.cloud.bigtable.row.ConditionalRow - - name: DirectRow - uid: google.cloud.bigtable.row.DirectRow - - name: InvalidChunk - uid: google.cloud.bigtable.row.InvalidChunk - - name: PartialRowData - uid: google.cloud.bigtable.row.PartialRowData - - name: Row - uid: google.cloud.bigtable.row.Row - name: row - uid: google.cloud.bigtable.row - - items: - - name: Overview - uid: google.cloud.bigtable.row_data - - name: InvalidReadRowsResponse - uid: google.cloud.bigtable.row_data.InvalidReadRowsResponse - - name: InvalidRetryRequest - uid: google.cloud.bigtable.row_data.InvalidRetryRequest - - name: PartialCellData - uid: google.cloud.bigtable.row_data.PartialCellData - - name: PartialRowsData - uid: google.cloud.bigtable.row_data.PartialRowsData - name: row_data - uid: google.cloud.bigtable.row_data - - items: - - name: Overview - uid: google.cloud.bigtable.row_filters - - name: ApplyLabelFilter - uid: google.cloud.bigtable.row_filters.ApplyLabelFilter - - name: BlockAllFilter - uid: google.cloud.bigtable.row_filters.BlockAllFilter - - name: CellsColumnLimitFilter - uid: google.cloud.bigtable.row_filters.CellsColumnLimitFilter - - name: CellsRowLimitFilter - uid: google.cloud.bigtable.row_filters.CellsRowLimitFilter - - name: CellsRowOffsetFilter - uid: google.cloud.bigtable.row_filters.CellsRowOffsetFilter - - name: ColumnQualifierRegexFilter - uid: google.cloud.bigtable.row_filters.ColumnQualifierRegexFilter - - name: ColumnRangeFilter - uid: google.cloud.bigtable.row_filters.ColumnRangeFilter - - name: ConditionalRowFilter - uid: google.cloud.bigtable.row_filters.ConditionalRowFilter - - name: ExactValueFilter - uid: google.cloud.bigtable.row_filters.ExactValueFilter - - name: FamilyNameRegexFilter - uid: google.cloud.bigtable.row_filters.FamilyNameRegexFilter - - name: PassAllFilter - uid: google.cloud.bigtable.row_filters.PassAllFilter - - name: RowFilter - uid: google.cloud.bigtable.row_filters.RowFilter - - name: RowFilterChain - uid: google.cloud.bigtable.row_filters.RowFilterChain - - name: RowFilterUnion - uid: google.cloud.bigtable.row_filters.RowFilterUnion - - name: RowKeyRegexFilter - uid: google.cloud.bigtable.row_filters.RowKeyRegexFilter - - name: RowSampleFilter - uid: google.cloud.bigtable.row_filters.RowSampleFilter - - name: SinkFilter - uid: google.cloud.bigtable.row_filters.SinkFilter - - name: StripValueTransformerFilter - uid: google.cloud.bigtable.row_filters.StripValueTransformerFilter - - name: TimestampRange - uid: google.cloud.bigtable.row_filters.TimestampRange - - name: TimestampRangeFilter - uid: google.cloud.bigtable.row_filters.TimestampRangeFilter - - name: ValueRangeFilter - uid: google.cloud.bigtable.row_filters.ValueRangeFilter - - name: ValueRegexFilter - uid: google.cloud.bigtable.row_filters.ValueRegexFilter - name: row_filters - uid: google.cloud.bigtable.row_filters - - items: - - name: Overview - uid: google.cloud.bigtable.row_set - - name: RowRange - uid: google.cloud.bigtable.row_set.RowRange - - name: RowSet - uid: google.cloud.bigtable.row_set.RowSet - name: row_set - uid: google.cloud.bigtable.row_set - - items: - - name: Overview - uid: google.cloud.bigtable.table - - name: ClusterState - uid: google.cloud.bigtable.table.ClusterState - - name: Table - uid: google.cloud.bigtable.table.Table - - name: TableMismatchError - uid: google.cloud.bigtable.table.TableMismatchError - - name: TooManyMutationsError - uid: google.cloud.bigtable.table.TooManyMutationsError - name: table - uid: google.cloud.bigtable.table - name: Bigtable - name: google-cloud-bigtable From 972c617ef76a04f603ec50a5df07e5ed8a762375 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 15:20:02 -0700 Subject: [PATCH 12/26] added validation to catch errors --- docs/scripts/patch_devsite_toc.py | 27 +++++++++++++++++++++++++++ noxfile.py | 30 +++++++++++++++++++----------- owlbot.py | 30 +++++++++++++++++++----------- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index d230c858b..cfd700b73 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -23,6 +23,7 @@ def __init__(self, dir_name, index_file_name): list of the other files which should be included """ self.dir_name = dir_name + self.index_file_name = index_file_name index_file_path = os.path.join(dir_name, index_file_name) with open(index_file_path, "r") as f: self.title = f.readline().replace("\n", "") @@ -88,3 +89,29 @@ def add_sections(toc_file_path, section_list, output_file_path=None): output_file_path = toc_file_path with open(output_file_path, "w") as f: yaml.dump(current_toc, f) + + +def validate_toc(toc_file_path, expected_section_list, added_sections): + current_toc = yaml.safe_load(open(toc_file_path, "r")) + # make sure the set of sections matches what we expect + found_sections = [d["name"] for d in current_toc[0]["items"]] + assert found_sections == expected_section_list + # make sure each customs ection is in the toc + for section in added_sections: + assert section.title in found_sections + # make sure each rst file in each custom section dir is listed in the toc + for section in added_sections: + items_in_toc = [d["items"] for d in current_toc[0]["items"] if d["name"] == section.title and ".rst"][0] + items_in_dir = [f for f in os.listdir(section.dir_name) if f.endswith(".rst")] + # subtract 1 for index + assert len(items_in_toc) == len(items_in_dir) - 1 + for file in items_in_dir: + if file != section.index_file_name: + base_name, _ = os.path.splitext(file) + assert any(d["href"] == f"{base_name}.md" for d in items_in_toc) + # make sure the markdown files are present in the docfx_yaml directory + for section in added_sections: + items_in_toc = [d["items"] for d in current_toc[0]["items"] if d["name"] == section.title and ".rst"][0] + md_files = [d["href"] for d in items_in_toc] + for file in md_files: + assert os.path.exists(f"_build/html/docfx_yaml/{file}") diff --git a/noxfile.py b/noxfile.py index cc0503cfb..d62dbe000 100644 --- a/noxfile.py +++ b/noxfile.py @@ -429,18 +429,26 @@ def docfx(session): import sys sys.path.append(os.path.abspath(os.path.join("docs", "scripts"))) from docs.scripts import patch_devsite_toc - patch_devsite_toc.add_sections( - os.path.join("_build", "html", "docfx_yaml", "toc.yml"), - [ - patch_devsite_toc.TocSection( - dir_name="async_data_client", - index_file_name="async_data_usage.rst" - ), - patch_devsite_toc.TocSection( - dir_name="standard_client", - index_file_name="usage.rst" - ), + toc_path = os.path.join("_build", "html", "docfx_yaml", "toc.yml") + custom_sections = [ + patch_devsite_toc.TocSection( + dir_name="async_data_client", + index_file_name="async_data_usage.rst" + ), + patch_devsite_toc.TocSection( + dir_name="standard_client", + index_file_name="usage.rst" + ), + ] + patch_devsite_toc.add_sections(toc_path, custom_sections) + # run validation to make sure yaml is structured as we expect + patch_devsite_toc.validate_toc( + toc_file_path=toc_path, + expected_section_list=[ + "Overview", "bigtable APIs", "Changelog", "Multiprocessing", + "Async Data Client", "Standard Client", "Bigtable" ], + added_sections=custom_sections ) diff --git a/owlbot.py b/owlbot.py index 917e04e2c..8cbba8774 100644 --- a/owlbot.py +++ b/owlbot.py @@ -219,18 +219,26 @@ def mypy(session): import sys sys.path.append(os.path.abspath(os.path.join("docs", "scripts"))) from docs.scripts import patch_devsite_toc - patch_devsite_toc.add_sections( - os.path.join("_build", "html", "docfx_yaml", "toc.yml"), - [ - patch_devsite_toc.TocSection( - dir_name="async_data_client", - index_file_name="async_data_usage.rst" - ), - patch_devsite_toc.TocSection( - dir_name="standard_client", - index_file_name="usage.rst" - ), + toc_path = os.path.join("_build", "html", "docfx_yaml", "toc.yml") + custom_sections = [ + patch_devsite_toc.TocSection( + dir_name="async_data_client", + index_file_name="async_data_usage.rst" + ), + patch_devsite_toc.TocSection( + dir_name="standard_client", + index_file_name="usage.rst" + ), + ] + patch_devsite_toc.add_sections(toc_path, custom_sections) + # run validation to make sure yaml is structured as we expect + patch_devsite_toc.validate_toc( + toc_file_path=toc_path, + expected_section_list=[ + "Overview", "bigtable APIs", "Changelog", "Multiprocessing", + "Async Data Client", "Standard Client", "Bigtable" ], + added_sections=custom_sections ) """ From f5c191502e2460c0d7094157e7df265dd92a125c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 15:35:25 -0700 Subject: [PATCH 13/26] clean up script --- docs/scripts/patch_devsite_toc.py | 63 +++++++++++++++++++------------ 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index cfd700b73..cfe805261 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -1,9 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ python-bigtable is made up of multiple clients, which we want to categorize separately in the devsite table of contents. -To accomplish this, we will read in the generated toc file, remove the markdown content, -and replace it with a hand-written template +This script allows us to add separate sections to the autogenerated toc.yml file to +accomplish this. """ @@ -11,9 +24,32 @@ import os import shutil -# set workinf directory to /docs +# set working directory to /docs os.chdir(f"{os.path.dirname(os.path.abspath(__file__))}/{os.pardir}") + +def add_sections(toc_file_path, section_list, output_file_path=None): + """ + Add new sections to the autogenerated docfx table of contents file + + Takes in a list of TocSection objects, which should point to a directory of rst files + within the main /docs directory, which represents a self-contained section of content + + :param toc_file_path: path to the autogenerated toc file + :param section_list: list of TocSection objects to add + :param output_file_path: path to save the updated toc file. If None, save to the input file + """ + current_toc = yaml.safe_load(open(toc_file_path, "r")) + for section in section_list: + current_toc[0]["items"].insert(-1, section.to_dict()) + section.copy_markdown() + # save file + if output_file_path is None: + output_file_path = toc_file_path + with open(output_file_path, "w") as f: + yaml.dump(current_toc, f) + + class TocSection: def __init__(self, dir_name, index_file_name): """ @@ -69,27 +105,6 @@ def copy_markdown(self): f"_build/html/docfx_yaml", ) -def add_sections(toc_file_path, section_list, output_file_path=None): - """ - Add new sections to the autogenerated docfx table of contents file - - Takes in a list of TocSection objects, which should point to a directory of rst files - within the main /docs directory, which represents a self-contained section of content - - :param toc_file_path: path to the autogenerated toc file - :param section_list: list of TocSection objects to add - :param output_file_path: path to save the updated toc file. If None, save to the input file - """ - current_toc = yaml.safe_load(open(toc_file_path, "r")) - for section in section_list: - current_toc[0]["items"].insert(-1, section.to_dict()) - section.copy_markdown() - # save file - if output_file_path is None: - output_file_path = toc_file_path - with open(output_file_path, "w") as f: - yaml.dump(current_toc, f) - def validate_toc(toc_file_path, expected_section_list, added_sections): current_toc = yaml.safe_load(open(toc_file_path, "r")) From a8c6364e9525622d4a4c9dd0975d840f6636a5d7 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 15:36:06 -0700 Subject: [PATCH 14/26] ran blacken --- docs/scripts/patch_devsite_toc.py | 18 ++++++++++++++---- noxfile.py | 19 ++++++++++++------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index cfe805261..cbd904633 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -80,7 +80,9 @@ def __init__(self, dir_name, index_file_name): # load the file to get the title with open(f"{dir_name}/{file_name}.rst", "r") as f2: file_title = f2.readline().replace("\n", "") - self.items.append({"name": file_title, "href": f"{file_name}.md"}) + self.items.append( + {"name": file_title, "href": f"{file_name}.md"} + ) else: # if we are out of the indented block, we are done break @@ -96,7 +98,7 @@ def copy_markdown(self): """ Copy markdown files from _build/markdown/dir_name to _build/html/docfx_yaml - This is necessary because the markdown files in dub-directories + This is necessary because the markdown files in dub-directories are not copied over by the docfx build by default """ for file in os.listdir("_build/markdown/" + self.dir_name): @@ -116,7 +118,11 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): assert section.title in found_sections # make sure each rst file in each custom section dir is listed in the toc for section in added_sections: - items_in_toc = [d["items"] for d in current_toc[0]["items"] if d["name"] == section.title and ".rst"][0] + items_in_toc = [ + d["items"] + for d in current_toc[0]["items"] + if d["name"] == section.title and ".rst" + ][0] items_in_dir = [f for f in os.listdir(section.dir_name) if f.endswith(".rst")] # subtract 1 for index assert len(items_in_toc) == len(items_in_dir) - 1 @@ -126,7 +132,11 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): assert any(d["href"] == f"{base_name}.md" for d in items_in_toc) # make sure the markdown files are present in the docfx_yaml directory for section in added_sections: - items_in_toc = [d["items"] for d in current_toc[0]["items"] if d["name"] == section.title and ".rst"][0] + items_in_toc = [ + d["items"] + for d in current_toc[0]["items"] + if d["name"] == section.title and ".rst" + ][0] md_files = [d["href"] for d in items_in_toc] for file in md_files: assert os.path.exists(f"_build/html/docfx_yaml/{file}") diff --git a/noxfile.py b/noxfile.py index d62dbe000..5867703c8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -427,17 +427,17 @@ def docfx(session): ) # Customization: Add extra sections to the table of contents for the Standard vs Async clients import sys + sys.path.append(os.path.abspath(os.path.join("docs", "scripts"))) from docs.scripts import patch_devsite_toc + toc_path = os.path.join("_build", "html", "docfx_yaml", "toc.yml") custom_sections = [ patch_devsite_toc.TocSection( - dir_name="async_data_client", - index_file_name="async_data_usage.rst" + dir_name="async_data_client", index_file_name="async_data_usage.rst" ), patch_devsite_toc.TocSection( - dir_name="standard_client", - index_file_name="usage.rst" + dir_name="standard_client", index_file_name="usage.rst" ), ] patch_devsite_toc.add_sections(toc_path, custom_sections) @@ -445,10 +445,15 @@ def docfx(session): patch_devsite_toc.validate_toc( toc_file_path=toc_path, expected_section_list=[ - "Overview", "bigtable APIs", "Changelog", "Multiprocessing", - "Async Data Client", "Standard Client", "Bigtable" + "Overview", + "bigtable APIs", + "Changelog", + "Multiprocessing", + "Async Data Client", + "Standard Client", + "Bigtable", ], - added_sections=custom_sections + added_sections=custom_sections, ) From 93efb3ce9665929debb1bf3000d250568fcd787d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 15:40:54 -0700 Subject: [PATCH 15/26] changed import --- noxfile.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 5867703c8..4fd8a1c8a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -428,8 +428,9 @@ def docfx(session): # Customization: Add extra sections to the table of contents for the Standard vs Async clients import sys - sys.path.append(os.path.abspath(os.path.join("docs", "scripts"))) - from docs.scripts import patch_devsite_toc + # import local module at docs/scripts/patch_devsite_toc.py + sys.path.append(os.path.join("docs", "scripts")) + import patch_devsite_toc toc_path = os.path.join("_build", "html", "docfx_yaml", "toc.yml") custom_sections = [ From 9866b18b798d6802a01266713d24e5aa93cc410f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 15:48:38 -0700 Subject: [PATCH 16/26] added import --- docs/scripts/patch_devsite_toc.py | 1 + noxfile.py | 1 + owlbot.py | 7 +++++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index cbd904633..07670ccd4 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -61,6 +61,7 @@ def __init__(self, dir_name, index_file_name): self.dir_name = dir_name self.index_file_name = index_file_name index_file_path = os.path.join(dir_name, index_file_name) + # find set of files referenced by the index file with open(index_file_path, "r") as f: self.title = f.readline().replace("\n", "") in_toc = False diff --git a/noxfile.py b/noxfile.py index 4fd8a1c8a..e63a8ff4f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -428,6 +428,7 @@ def docfx(session): # Customization: Add extra sections to the table of contents for the Standard vs Async clients import sys + session.install("pyyaml") # import local module at docs/scripts/patch_devsite_toc.py sys.path.append(os.path.join("docs", "scripts")) import patch_devsite_toc diff --git a/owlbot.py b/owlbot.py index 8cbba8774..f3378aa7b 100644 --- a/owlbot.py +++ b/owlbot.py @@ -217,8 +217,11 @@ def mypy(session): docfx_postprocess = """ # Customization: Add extra sections to the table of contents for the Standard vs Async clients import sys - sys.path.append(os.path.abspath(os.path.join("docs", "scripts"))) - from docs.scripts import patch_devsite_toc + + session.install("pyyaml") + # import local module at docs/scripts/patch_devsite_toc.py + sys.path.append(os.path.join("docs", "scripts")) + import patch_devsite_toc toc_path = os.path.join("_build", "html", "docfx_yaml", "toc.yml") custom_sections = [ patch_devsite_toc.TocSection( From 27245066889946fc1f8a4132a269f0fcb779a16b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 16:01:11 -0700 Subject: [PATCH 17/26] moved logic from noxfile to script --- docs/scripts/patch_devsite_toc.py | 31 +++++++++++++++++++++++++++++++ noxfile.py | 31 +------------------------------ owlbot.py | 27 +-------------------------- 3 files changed, 33 insertions(+), 56 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index 07670ccd4..f15e2116f 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -41,6 +41,7 @@ def add_sections(toc_file_path, section_list, output_file_path=None): """ current_toc = yaml.safe_load(open(toc_file_path, "r")) for section in section_list: + print(f"Adding section {section.title}...") current_toc[0]["items"].insert(-1, section.to_dict()) section.copy_markdown() # save file @@ -141,3 +142,33 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): md_files = [d["href"] for d in items_in_toc] for file in md_files: assert os.path.exists(f"_build/html/docfx_yaml/{file}") + print("Toc validation passed") + +if __name__ == "__main__": + """ + Add secrtions for the async_data_client and standard_client directories + """ + toc_path = "_build/html/docfx_yaml/toc.yml" + custom_sections = [ + TocSection( + dir_name="async_data_client", index_file_name="async_data_usage.rst" + ), + TocSection( + dir_name="standard_client", index_file_name="usage.rst" + ), + ] + add_sections(toc_path, custom_sections) + # run validation to make sure yaml is structured as we expect + validate_toc( + toc_file_path=toc_path, + expected_section_list=[ + "Overview", + "bigtable APIs", + "Changelog", + "Multiprocessing", + "Async Data Client", + "Standard Client", + "Bigtable", + ], + added_sections=custom_sections, + ) diff --git a/noxfile.py b/noxfile.py index e63a8ff4f..fadd91a96 100644 --- a/noxfile.py +++ b/noxfile.py @@ -426,37 +426,8 @@ def docfx(session): os.path.join("docs", "_build", "html", ""), ) # Customization: Add extra sections to the table of contents for the Standard vs Async clients - import sys - session.install("pyyaml") - # import local module at docs/scripts/patch_devsite_toc.py - sys.path.append(os.path.join("docs", "scripts")) - import patch_devsite_toc - - toc_path = os.path.join("_build", "html", "docfx_yaml", "toc.yml") - custom_sections = [ - patch_devsite_toc.TocSection( - dir_name="async_data_client", index_file_name="async_data_usage.rst" - ), - patch_devsite_toc.TocSection( - dir_name="standard_client", index_file_name="usage.rst" - ), - ] - patch_devsite_toc.add_sections(toc_path, custom_sections) - # run validation to make sure yaml is structured as we expect - patch_devsite_toc.validate_toc( - toc_file_path=toc_path, - expected_section_list=[ - "Overview", - "bigtable APIs", - "Changelog", - "Multiprocessing", - "Async Data Client", - "Standard Client", - "Bigtable", - ], - added_sections=custom_sections, - ) + session.run("python", "docs/scripts/patch_devsite_toc.py") @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) diff --git a/owlbot.py b/owlbot.py index f3378aa7b..84ab8b587 100644 --- a/owlbot.py +++ b/owlbot.py @@ -216,33 +216,8 @@ def mypy(session): # add customization to docfx docfx_postprocess = """ # Customization: Add extra sections to the table of contents for the Standard vs Async clients - import sys - session.install("pyyaml") - # import local module at docs/scripts/patch_devsite_toc.py - sys.path.append(os.path.join("docs", "scripts")) - import patch_devsite_toc - toc_path = os.path.join("_build", "html", "docfx_yaml", "toc.yml") - custom_sections = [ - patch_devsite_toc.TocSection( - dir_name="async_data_client", - index_file_name="async_data_usage.rst" - ), - patch_devsite_toc.TocSection( - dir_name="standard_client", - index_file_name="usage.rst" - ), - ] - patch_devsite_toc.add_sections(toc_path, custom_sections) - # run validation to make sure yaml is structured as we expect - patch_devsite_toc.validate_toc( - toc_file_path=toc_path, - expected_section_list=[ - "Overview", "bigtable APIs", "Changelog", "Multiprocessing", - "Async Data Client", "Standard Client", "Bigtable" - ], - added_sections=custom_sections - ) + session.run("python", "docs/scripts/patch_devsite_toc.py") """ place_before( From 898fcafc715d56406ff1730549ccfacd127f800e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 15 May 2024 16:04:07 -0700 Subject: [PATCH 18/26] fixed lint --- docs/scripts/patch_devsite_toc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index f15e2116f..cd222fc0a 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -144,6 +144,7 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): assert os.path.exists(f"_build/html/docfx_yaml/{file}") print("Toc validation passed") + if __name__ == "__main__": """ Add secrtions for the async_data_client and standard_client directories @@ -153,9 +154,7 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): TocSection( dir_name="async_data_client", index_file_name="async_data_usage.rst" ), - TocSection( - dir_name="standard_client", index_file_name="usage.rst" - ), + TocSection(dir_name="standard_client", index_file_name="usage.rst"), ] add_sections(toc_path, custom_sections) # run validation to make sure yaml is structured as we expect From e5b40ec179093a49e0262f7e5c0ad9d13c11b37a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 May 2024 14:36:30 -0700 Subject: [PATCH 19/26] added step to remove Bigtable section --- docs/scripts/patch_devsite_toc.py | 33 ++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index cd222fc0a..11407f505 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -39,6 +39,9 @@ def add_sections(toc_file_path, section_list, output_file_path=None): :param section_list: list of TocSection objects to add :param output_file_path: path to save the updated toc file. If None, save to the input file """ + # remove any sections that are already in the toc + remove_sections(toc_file_path, [section.title for section in section_list]) + # add new sections current_toc = yaml.safe_load(open(toc_file_path, "r")) for section in section_list: print(f"Adding section {section.title}...") @@ -50,6 +53,25 @@ def add_sections(toc_file_path, section_list, output_file_path=None): with open(output_file_path, "w") as f: yaml.dump(current_toc, f) +def remove_sections(toc_file_path, section_list, output_file_path=None): + """ + Add new sections to the autogenerated docfx table of contents file + + Takes in a list of string section names to remove from the toc file + + :param toc_file_path: path to the autogenerated toc file + :param section_list: list of section names to remove + :param output_file_path: path to save the updated toc file. If None, save to the input file + """ + current_toc = yaml.safe_load(open(toc_file_path, "r")) + print(f"Removing sections {section_list}...") + new_items = [d for d in current_toc[0]["items"] if d["name"] not in section_list] + current_toc[0]["items"] = new_items + # save file + if output_file_path is None: + output_file_path = toc_file_path + with open(output_file_path, "w") as f: + yaml.dump(current_toc, f) class TocSection: def __init__(self, dir_name, index_file_name): @@ -146,17 +168,19 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): if __name__ == "__main__": - """ - Add secrtions for the async_data_client and standard_client directories - """ + # Add secrtions for the async_data_client and standard_client directories toc_path = "_build/html/docfx_yaml/toc.yml" custom_sections = [ TocSection( dir_name="async_data_client", index_file_name="async_data_usage.rst" ), - TocSection(dir_name="standard_client", index_file_name="usage.rst"), + TocSection( + dir_name="standard_client", index_file_name="usage.rst" + ), ] add_sections(toc_path, custom_sections) + # Remove the Bigtable section, since it has duplicated data + remove_sections(toc_path, ["Bigtable"]) # run validation to make sure yaml is structured as we expect validate_toc( toc_file_path=toc_path, @@ -167,7 +191,6 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): "Multiprocessing", "Async Data Client", "Standard Client", - "Bigtable", ], added_sections=custom_sections, ) From 94e1468d92d28c1769e83ad873d19f2238e95c05 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 May 2024 14:46:56 -0700 Subject: [PATCH 20/26] improved file docstring --- docs/scripts/patch_devsite_toc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index 11407f505..d71206f6c 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -python-bigtable is made up of multiple clients, which we want to categorize separately in -the devsite table of contents. +This script will run after ``nox -s docfx`` is run. docfx is the api doc format used by +google cloud. It is described here: https://github.com/googleapis/docuploader?tab=readme-ov-file#requirements-for-docfx-yaml-tarballs. -This script allows us to add separate sections to the autogenerated toc.yml file to -accomplish this. +One of the file used by docfx is toc.yml which is used to generate the table of contents sidebar. +This script will patch file to create subfolders for each of the clients """ From 9612c7a8b27754af17207fc1bda61649d4745def Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 May 2024 14:56:35 -0700 Subject: [PATCH 21/26] fixed typo --- docs/scripts/patch_devsite_toc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index d71206f6c..09c0d067a 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -122,7 +122,7 @@ def copy_markdown(self): """ Copy markdown files from _build/markdown/dir_name to _build/html/docfx_yaml - This is necessary because the markdown files in dub-directories + This is necessary because the markdown files in sub-directories are not copied over by the docfx build by default """ for file in os.listdir("_build/markdown/" + self.dir_name): From 23965bf0b32576550d1f5f68e6f8e2c4ab3f82c5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 May 2024 14:56:42 -0700 Subject: [PATCH 22/26] refactor --- docs/scripts/patch_devsite_toc.py | 47 ++++++++++++++++++------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index 09c0d067a..e7513346d 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -86,31 +86,38 @@ def __init__(self, dir_name, index_file_name): index_file_path = os.path.join(dir_name, index_file_name) # find set of files referenced by the index file with open(index_file_path, "r") as f: - self.title = f.readline().replace("\n", "") + self.title = f.readline().strip() in_toc = False self.items = [] for line in f: + # ignore empty lines + if not line.strip(): + continue if line.startswith(".. toctree::"): in_toc = True - elif in_toc: - if not line.strip(): - # ignore empty lines - continue - elif line.startswith(" ") or line.startswith("\t"): - # keep track of the items in the table of contents - # ignore : directives - if ":" not in line: - file_name = line.strip() - # load the file to get the title - with open(f"{dir_name}/{file_name}.rst", "r") as f2: - file_title = f2.readline().replace("\n", "") - self.items.append( - {"name": file_title, "href": f"{file_name}.md"} - ) - else: - # if we are out of the indented block, we are done - break - in_toc = False + continue + # ignore directives + if ":" in line: + continue + if not in_toc: + continue + # bail when toc indented block is done + if not line.startswith(" ") and not line.startswith("\t"): + break + # extract entries + self.items.append(self.extract_toc_entry(line.strip())) + + + def extract_toc_entry(self, file_name): + """ + Given the name of a file, extract the title and href for the toc entry, + and return as a dictionary + """ + # load the file to get the title + with open(f"{self.dir_name}/{file_name}.rst", "r") as f2: + file_title = f2.readline().strip() + return {"name": file_title, "href": f"{file_name}.md"} + def to_dict(self): """ From 9866009c07c527e43e81ecda0e8c231f2eb527c7 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 16 May 2024 15:02:33 -0700 Subject: [PATCH 23/26] fixed lint issue --- docs/scripts/patch_devsite_toc.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index e7513346d..95e73e4d3 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -53,6 +53,7 @@ def add_sections(toc_file_path, section_list, output_file_path=None): with open(output_file_path, "w") as f: yaml.dump(current_toc, f) + def remove_sections(toc_file_path, section_list, output_file_path=None): """ Add new sections to the autogenerated docfx table of contents file @@ -73,6 +74,7 @@ def remove_sections(toc_file_path, section_list, output_file_path=None): with open(output_file_path, "w") as f: yaml.dump(current_toc, f) + class TocSection: def __init__(self, dir_name, index_file_name): """ @@ -107,7 +109,6 @@ def __init__(self, dir_name, index_file_name): # extract entries self.items.append(self.extract_toc_entry(line.strip())) - def extract_toc_entry(self, file_name): """ Given the name of a file, extract the title and href for the toc entry, @@ -118,7 +119,6 @@ def extract_toc_entry(self, file_name): file_title = f2.readline().strip() return {"name": file_title, "href": f"{file_name}.md"} - def to_dict(self): """ Convert the TocSection object to a dictionary that can be written to a yaml file @@ -181,9 +181,7 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): TocSection( dir_name="async_data_client", index_file_name="async_data_usage.rst" ), - TocSection( - dir_name="standard_client", index_file_name="usage.rst" - ), + TocSection(dir_name="standard_client", index_file_name="usage.rst"), ] add_sections(toc_path, custom_sections) # Remove the Bigtable section, since it has duplicated data From 06ae3ae4fd3f5b8f8a9a2a2d710de302c51008d7 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 29 May 2024 12:20:31 -0700 Subject: [PATCH 24/26] fixed docstring --- docs/scripts/patch_devsite_toc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index 95e73e4d3..e8e3079ee 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -56,7 +56,7 @@ def add_sections(toc_file_path, section_list, output_file_path=None): def remove_sections(toc_file_path, section_list, output_file_path=None): """ - Add new sections to the autogenerated docfx table of contents file + Remove sections from the autogenerated docfx table of contents file Takes in a list of string section names to remove from the toc file From 4bfdb2d99dde181140acb5308c0eb8af4d63cff5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 29 May 2024 12:22:25 -0700 Subject: [PATCH 25/26] renamed standard client to classic client --- docs/{standard_client => classic_client}/app-profile.rst | 0 docs/{standard_client => classic_client}/backup.rst | 0 docs/{standard_client => classic_client}/batcher.rst | 0 docs/{standard_client => classic_client}/client-intro.rst | 0 docs/{standard_client => classic_client}/client.rst | 0 docs/{standard_client => classic_client}/cluster.rst | 0 docs/{standard_client => classic_client}/column-family.rst | 0 docs/{standard_client => classic_client}/data-api.rst | 0 .../{standard_client => classic_client}/encryption-info.rst | 0 docs/{standard_client => classic_client}/instance-api.rst | 0 docs/{standard_client => classic_client}/instance.rst | 0 docs/{standard_client => classic_client}/row-data.rst | 0 docs/{standard_client => classic_client}/row-filters.rst | 0 docs/{standard_client => classic_client}/row-set.rst | 0 docs/{standard_client => classic_client}/row.rst | 0 docs/{standard_client => classic_client}/snippets.py | 0 docs/{standard_client => classic_client}/snippets_table.py | 0 docs/{standard_client => classic_client}/table-api.rst | 0 docs/{standard_client => classic_client}/table.rst | 0 docs/{standard_client => classic_client}/usage.rst | 4 ++-- docs/scripts/patch_devsite_toc.py | 6 +++--- 21 files changed, 5 insertions(+), 5 deletions(-) rename docs/{standard_client => classic_client}/app-profile.rst (100%) rename docs/{standard_client => classic_client}/backup.rst (100%) rename docs/{standard_client => classic_client}/batcher.rst (100%) rename docs/{standard_client => classic_client}/client-intro.rst (100%) rename docs/{standard_client => classic_client}/client.rst (100%) rename docs/{standard_client => classic_client}/cluster.rst (100%) rename docs/{standard_client => classic_client}/column-family.rst (100%) rename docs/{standard_client => classic_client}/data-api.rst (100%) rename docs/{standard_client => classic_client}/encryption-info.rst (100%) rename docs/{standard_client => classic_client}/instance-api.rst (100%) rename docs/{standard_client => classic_client}/instance.rst (100%) rename docs/{standard_client => classic_client}/row-data.rst (100%) rename docs/{standard_client => classic_client}/row-filters.rst (100%) rename docs/{standard_client => classic_client}/row-set.rst (100%) rename docs/{standard_client => classic_client}/row.rst (100%) rename docs/{standard_client => classic_client}/snippets.py (100%) rename docs/{standard_client => classic_client}/snippets_table.py (100%) rename docs/{standard_client => classic_client}/table-api.rst (100%) rename docs/{standard_client => classic_client}/table.rst (100%) rename docs/{standard_client => classic_client}/usage.rst (96%) diff --git a/docs/standard_client/app-profile.rst b/docs/classic_client/app-profile.rst similarity index 100% rename from docs/standard_client/app-profile.rst rename to docs/classic_client/app-profile.rst diff --git a/docs/standard_client/backup.rst b/docs/classic_client/backup.rst similarity index 100% rename from docs/standard_client/backup.rst rename to docs/classic_client/backup.rst diff --git a/docs/standard_client/batcher.rst b/docs/classic_client/batcher.rst similarity index 100% rename from docs/standard_client/batcher.rst rename to docs/classic_client/batcher.rst diff --git a/docs/standard_client/client-intro.rst b/docs/classic_client/client-intro.rst similarity index 100% rename from docs/standard_client/client-intro.rst rename to docs/classic_client/client-intro.rst diff --git a/docs/standard_client/client.rst b/docs/classic_client/client.rst similarity index 100% rename from docs/standard_client/client.rst rename to docs/classic_client/client.rst diff --git a/docs/standard_client/cluster.rst b/docs/classic_client/cluster.rst similarity index 100% rename from docs/standard_client/cluster.rst rename to docs/classic_client/cluster.rst diff --git a/docs/standard_client/column-family.rst b/docs/classic_client/column-family.rst similarity index 100% rename from docs/standard_client/column-family.rst rename to docs/classic_client/column-family.rst diff --git a/docs/standard_client/data-api.rst b/docs/classic_client/data-api.rst similarity index 100% rename from docs/standard_client/data-api.rst rename to docs/classic_client/data-api.rst diff --git a/docs/standard_client/encryption-info.rst b/docs/classic_client/encryption-info.rst similarity index 100% rename from docs/standard_client/encryption-info.rst rename to docs/classic_client/encryption-info.rst diff --git a/docs/standard_client/instance-api.rst b/docs/classic_client/instance-api.rst similarity index 100% rename from docs/standard_client/instance-api.rst rename to docs/classic_client/instance-api.rst diff --git a/docs/standard_client/instance.rst b/docs/classic_client/instance.rst similarity index 100% rename from docs/standard_client/instance.rst rename to docs/classic_client/instance.rst diff --git a/docs/standard_client/row-data.rst b/docs/classic_client/row-data.rst similarity index 100% rename from docs/standard_client/row-data.rst rename to docs/classic_client/row-data.rst diff --git a/docs/standard_client/row-filters.rst b/docs/classic_client/row-filters.rst similarity index 100% rename from docs/standard_client/row-filters.rst rename to docs/classic_client/row-filters.rst diff --git a/docs/standard_client/row-set.rst b/docs/classic_client/row-set.rst similarity index 100% rename from docs/standard_client/row-set.rst rename to docs/classic_client/row-set.rst diff --git a/docs/standard_client/row.rst b/docs/classic_client/row.rst similarity index 100% rename from docs/standard_client/row.rst rename to docs/classic_client/row.rst diff --git a/docs/standard_client/snippets.py b/docs/classic_client/snippets.py similarity index 100% rename from docs/standard_client/snippets.py rename to docs/classic_client/snippets.py diff --git a/docs/standard_client/snippets_table.py b/docs/classic_client/snippets_table.py similarity index 100% rename from docs/standard_client/snippets_table.py rename to docs/classic_client/snippets_table.py diff --git a/docs/standard_client/table-api.rst b/docs/classic_client/table-api.rst similarity index 100% rename from docs/standard_client/table-api.rst rename to docs/classic_client/table-api.rst diff --git a/docs/standard_client/table.rst b/docs/classic_client/table.rst similarity index 100% rename from docs/standard_client/table.rst rename to docs/classic_client/table.rst diff --git a/docs/standard_client/usage.rst b/docs/classic_client/usage.rst similarity index 96% rename from docs/standard_client/usage.rst rename to docs/classic_client/usage.rst index 409eb5516..7a47f4d4a 100644 --- a/docs/standard_client/usage.rst +++ b/docs/classic_client/usage.rst @@ -1,5 +1,5 @@ -Standard Client -=============== +Classic Client +============== .. toctree:: :maxdepth: 2 diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py index e8e3079ee..6338128dd 100644 --- a/docs/scripts/patch_devsite_toc.py +++ b/docs/scripts/patch_devsite_toc.py @@ -175,13 +175,13 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): if __name__ == "__main__": - # Add secrtions for the async_data_client and standard_client directories + # Add secrtions for the async_data_client and classic_client directories toc_path = "_build/html/docfx_yaml/toc.yml" custom_sections = [ TocSection( dir_name="async_data_client", index_file_name="async_data_usage.rst" ), - TocSection(dir_name="standard_client", index_file_name="usage.rst"), + TocSection(dir_name="classic_client", index_file_name="usage.rst"), ] add_sections(toc_path, custom_sections) # Remove the Bigtable section, since it has duplicated data @@ -195,7 +195,7 @@ def validate_toc(toc_file_path, expected_section_list, added_sections): "Changelog", "Multiprocessing", "Async Data Client", - "Standard Client", + "Classic Client", ], added_sections=custom_sections, ) From 2ca10bb7243c934fdbf78b180947927e739c1109 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 29 May 2024 15:25:01 -0700 Subject: [PATCH 26/26] cleaned up remaining standard client references --- docs/index.rst | 2 +- noxfile.py | 2 +- owlbot.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b2e48559f..4204e981d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ Client Types .. toctree:: :maxdepth: 2 - standard_client/usage + classic_client/usage async_data_client/async_data_usage diff --git a/noxfile.py b/noxfile.py index fadd91a96..3ea12c187 100644 --- a/noxfile.py +++ b/noxfile.py @@ -425,7 +425,7 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) - # Customization: Add extra sections to the table of contents for the Standard vs Async clients + # Customization: Add extra sections to the table of contents for the Classic vs Async clients session.install("pyyaml") session.run("python", "docs/scripts/patch_devsite_toc.py") diff --git a/owlbot.py b/owlbot.py index 84ab8b587..170bc08d4 100644 --- a/owlbot.py +++ b/owlbot.py @@ -215,7 +215,7 @@ def mypy(session): # add customization to docfx docfx_postprocess = """ - # Customization: Add extra sections to the table of contents for the Standard vs Async clients + # Customization: Add extra sections to the table of contents for the Classic vs Async clients session.install("pyyaml") session.run("python", "docs/scripts/patch_devsite_toc.py") """