fix collection name collection

src/update_collection_metadata.py

@@ -3,17 +3,16 @@
 
 import github3
 import pendulum
+import utils
+from generate_block_metadata import update_block_metadata_for_collection
+from generate_flow_metadata import update_flow_metadata_for_collection
+from generate_worker_metadata import update_worker_metadata_for_package
 from prefect import flow, task
 from prefect.deployments import run_deployment
 from prefect.server.schemas.core import FlowRun
 from prefect.states import Completed, Failed, State
 from prefect.utilities.collections import listrepr
 
-import utils
-from generate_block_metadata import update_block_metadata_for_collection
-from generate_flow_metadata import update_flow_metadata_for_collection
-from generate_worker_metadata import update_worker_metadata_for_package
-
 UPDATE_ALL_DESCRIPTION = """
 The `update_all_collections` triggers many instances of `update_collection_metadata` in order to
 update the [`prefect-collection-registry`](https://github.com/PrefectHQ/prefect-collection-registry)
@@ -168,7 +167,7 @@ async def update_all_collections(
         for collection_name, needs_update in await asyncio.gather(
             *[
                 collection_needs_update(collection_name)
-                for collection_name in utils.get_collection_names()
+                for collection_name in await utils.get_collection_names()
             ]
         )
         if needs_update
@@ -201,9 +200,9 @@ async def update_all_collections(
 
 
 # if __name__ == "__main__":
-    # # ALL COLLECTIONS
-    # asyncio.run(update_all_collections())
+# # ALL COLLECTIONS
+# asyncio.run(update_all_collections())
 
-    # # MANUAL RUNS
-    # for collection in ["prefect-sqlalchemy"]:
-    #     update_collection_metadata(collection, "update-metadata-manually")
+# # MANUAL RUNS
+# for collection in ["prefect-sqlalchemy"]:
+#     update_collection_metadata(collection, "update-metadata-manually")

src/utils.py

@@ -1,3 +1,4 @@
+import asyncio
 import inspect
 import json
 from pkgutil import iter_modules
@@ -7,13 +8,16 @@
 import fastjsonschema
 import github3
 import httpx
+import yaml
 from prefect import Flow, task
 from prefect.blocks.core import Block
 from prefect.blocks.system import Secret
 from prefect.client.cloud import get_cloud_client
 from prefect.settings import PREFECT_API_URL
-from prefect.utilities.asyncutils import run_sync_in_worker_thread  # noqa
-from prefect.utilities.asyncutils import sync_compatible
+from prefect.utilities.asyncutils import (
+    run_sync_in_worker_thread,  # noqa
+    sync_compatible,
+)
 from prefect.utilities.importtools import load_module, to_qualified_name
 from typing_extensions import Literal
 
@@ -182,18 +186,36 @@ def submit_updates(
         )
 
 
-def get_collection_names():
+async def get_file_content(url):
+    async with httpx.AsyncClient() as client:
+        response = await client.get(url)
+        response.raise_for_status()
+        return response.text
+
+
+async def get_collection_names():
     repo_owner = "PrefectHQ"
     repo_name = "Prefect"
     path = "docs/integrations/catalog"
 
-    response = httpx.get(
-        url=f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents/{path}",
-        headers={"Accept": "application/vnd.github+json"},
-    )
+    async with httpx.AsyncClient() as client:
+        response = await client.get(
+            url=f"https://api.github.com/repos/{repo_owner}/{repo_name}/contents/{path}",
+            headers={"Accept": "application/vnd.github+json"},
+        )
+        response.raise_for_status()
+        files = response.json()
+
+    collections = []
+
+    async def process_file(file):
+        if file["type"] == "file" and file["name"].endswith(".yaml"):
+            file_content = await get_file_content(file["download_url"])
+            yaml_data = yaml.safe_load(file_content)
+            if yaml_data.get("author") == "Prefect":
+                collections.append(file["name"].replace(".yaml", ""))
 
-    response.raise_for_status()
-    collections = [file['name'] for file in response.json() if file['type'] == 'file' and file['name'] != "TEMPLATE.yaml"]
+    await asyncio.gather(*[process_file(file) for file in files])
     collections.append("prefect")
     return collections