diff --git a/collections/prefect-openmetadata/blocks/v0.3.0.json b/collections/prefect-openmetadata/blocks/v0.3.0.json new file mode 100644 index 00000000..67aa8059 --- /dev/null +++ b/collections/prefect-openmetadata/blocks/v0.3.0.json @@ -0,0 +1,3 @@ +{ + "prefect-openmetadata": {} +} \ No newline at end of file diff --git a/collections/prefect-openmetadata/flows/v0.3.0.json b/collections/prefect-openmetadata/flows/v0.3.0.json new file mode 100644 index 00000000..67aa8059 --- /dev/null +++ b/collections/prefect-openmetadata/flows/v0.3.0.json @@ -0,0 +1,3 @@ +{ + "prefect-openmetadata": {} +} \ No newline at end of file diff --git a/collections/prefect-openmetadata/workers/v0.3.0.json b/collections/prefect-openmetadata/workers/v0.3.0.json new file mode 100644 index 00000000..67aa8059 --- /dev/null +++ b/collections/prefect-openmetadata/workers/v0.3.0.json @@ -0,0 +1,3 @@ +{ + "prefect-openmetadata": {} +} \ No newline at end of file diff --git a/collections/prefect-sqlalchemy/blocks/v0.3.0.json b/collections/prefect-sqlalchemy/blocks/v0.3.0.json new file mode 100644 index 00000000..1cae7c6f --- /dev/null +++ b/collections/prefect-sqlalchemy/blocks/v0.3.0.json @@ -0,0 +1,279 @@ +{ + "prefect-sqlalchemy": { + "block_types": { + "database-credentials": { + "name": "Database Credentials", + "slug": "database-credentials", + "logo_url": "https://images.ctfassets.net/gm98wzqotmnx/3xLant5G70S4vJpmdWCYmr/8fdb19f15b97c3a07c3af3efde4d28fb/download.svg.png?h=250", + "documentation_url": "https://prefecthq.github.io/prefect-sqlalchemy/credentials/#prefect_sqlalchemy.credentials.DatabaseCredentials", + "description": "Block used to manage authentication with a database. This block is part of the prefect-sqlalchemy collection. Install prefect-sqlalchemy with `pip install prefect-sqlalchemy` to use this block.", + "code_example": "Load stored database credentials:\n```python\nfrom prefect_sqlalchemy import DatabaseCredentials\ndatabase_block = DatabaseCredentials.load(\"BLOCK_NAME\")\n```", + "block_schema": { + "checksum": "sha256:76d1ccbf0ab2038fea77e9689b91a7c8b6398e080e95d9303f65a93a4c03162e", + "fields": { + "title": "DatabaseCredentials", + "description": "Block used to manage authentication with a database.", + "type": "object", + "properties": { + "driver": { + "title": "Driver", + "description": "The driver name to use.", + "anyOf": [ + { + "$ref": "#/definitions/AsyncDriver" + }, + { + "$ref": "#/definitions/SyncDriver" + }, + { + "type": "string" + } + ] + }, + "username": { + "title": "Username", + "description": "The user name used to authenticate.", + "type": "string" + }, + "password": { + "title": "Password", + "description": "The password used to authenticate.", + "type": "string", + "writeOnly": true, + "format": "password" + }, + "database": { + "title": "Database", + "description": "The name of the database to use.", + "type": "string" + }, + "host": { + "title": "Host", + "description": "The host address of the database.", + "type": "string" + }, + "port": { + "title": "Port", + "description": "The port to connect to the database.", + "type": "string" + }, + "query": { + "title": "Query", + "description": "A dictionary of string keys to string values to be passed to the dialect and/or the DBAPI upon connect. To specify non-string parameters to a Python DBAPI directly, use connect_args.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "url": { + "title": "Url", + "description": "Manually create and provide a URL to create the engine, this is useful for external dialects, e.g. Snowflake, because some of the params, such as 'warehouse', is not directly supported in the vanilla `sqlalchemy.engine.URL.create` method; do not provide this alongside with other URL params as it will raise a `ValueError`.", + "minLength": 1, + "maxLength": 65536, + "format": "uri", + "type": "string" + }, + "connect_args": { + "title": "Connect Args", + "description": "The options which will be passed directly to the DBAPI's connect() method as additional keyword arguments.", + "type": "object" + } + }, + "block_type_slug": "database-credentials", + "secret_fields": [ + "password" + ], + "block_schema_references": {}, + "definitions": { + "AsyncDriver": { + "title": "AsyncDriver", + "description": "Known dialects with their corresponding async drivers.\n\nAttributes:\n POSTGRESQL_ASYNCPG (Enum): [postgresql+asyncpg](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.asyncpg)\n\n SQLITE_AIOSQLITE (Enum): [sqlite+aiosqlite](https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#module-sqlalchemy.dialects.sqlite.aiosqlite)\n\n MYSQL_ASYNCMY (Enum): [mysql+asyncmy](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.asyncmy)\n MYSQL_AIOMYSQL (Enum): [mysql+aiomysql](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.aiomysql)", + "enum": [ + "postgresql+asyncpg", + "sqlite+aiosqlite", + "mysql+asyncmy", + "mysql+aiomysql" + ] + }, + "SyncDriver": { + "title": "SyncDriver", + "description": "Known dialects with their corresponding sync drivers.\n\nAttributes:\n POSTGRESQL_PSYCOPG2 (Enum): [postgresql+psycopg2](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.psycopg2)\n POSTGRESQL_PG8000 (Enum): [postgresql+pg8000](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.pg8000)\n POSTGRESQL_PSYCOPG2CFFI (Enum): [postgresql+psycopg2cffi](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.psycopg2cffi)\n POSTGRESQL_PYPOSTGRESQL (Enum): [postgresql+pypostgresql](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.pypostgresql)\n POSTGRESQL_PYGRESQL (Enum): [postgresql+pygresql](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.pygresql)\n\n MYSQL_MYSQLDB (Enum): [mysql+mysqldb](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.mysqldb)\n MYSQL_PYMYSQL (Enum): [mysql+pymysql](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.pymysql)\n MYSQL_MYSQLCONNECTOR (Enum): [mysql+mysqlconnector](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.mysqlconnector)\n MYSQL_CYMYSQL (Enum): [mysql+cymysql](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.cymysql)\n MYSQL_OURSQL (Enum): [mysql+oursql](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.oursql)\n MYSQL_PYODBC (Enum): [mysql+pyodbc](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.pyodbc)\n\n SQLITE_PYSQLITE (Enum): [sqlite+pysqlite](https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#module-sqlalchemy.dialects.sqlite.pysqlite)\n SQLITE_PYSQLCIPHER (Enum): [sqlite+pysqlcipher](https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#module-sqlalchemy.dialects.sqlite.pysqlcipher)\n\n ORACLE_CX_ORACLE (Enum): [oracle+cx_oracle](https://docs.sqlalchemy.org/en/14/dialects/oracle.html#module-sqlalchemy.dialects.oracle.cx_oracle)\n\n MSSQL_PYODBC (Enum): [mssql+pyodbc](https://docs.sqlalchemy.org/en/14/dialects/mssql.html#module-sqlalchemy.dialects.mssql.pyodbc)\n MSSQL_MXODBC (Enum): [mssql+mxodbc](https://docs.sqlalchemy.org/en/14/dialects/mssql.html#module-sqlalchemy.dialects.mssql.mxodbc)\n MSSQL_PYMSSQL (Enum): [mssql+pymssql](https://docs.sqlalchemy.org/en/14/dialects/mssql.html#module-sqlalchemy.dialects.mssql.pymssql)", + "enum": [ + "postgresql+psycopg2", + "postgresql+pg8000", + "postgresql+psycopg2cffi", + "postgresql+pypostgresql", + "postgresql+pygresql", + "mysql+mysqldb", + "mysql+pymysql", + "mysql+mysqlconnector", + "mysql+cymysql", + "mysql+oursql", + "mysql+pyodbc", + "sqlite+pysqlite", + "sqlite+pysqlcipher", + "oracle+cx_oracle", + "mssql+pyodbc", + "mssql+mxodbc", + "mssql+pymssql" + ] + } + } + }, + "capabilities": [], + "version": "0.3.0" + } + }, + "sqlalchemy-connector": { + "name": "SQLAlchemy Connector", + "slug": "sqlalchemy-connector", + "logo_url": "https://images.ctfassets.net/gm98wzqotmnx/37TOcxeP9kfXffpKVRAHiJ/0f359112e79d0bd3dfe38c73c4fc6363/sqlalchemy.png?h=250", + "documentation_url": "https://prefecthq.github.io/prefect-sqlalchemy/database/#prefect_sqlalchemy.database.SqlAlchemyConnector", + "description": "Block used to manage authentication with a database.\n\nUpon instantiating, an engine is created and maintained for the life of\nthe object until the close method is called.\n\nIt is recommended to use this block as a context manager, which will automatically\nclose the engine and its connections when the context is exited.\n\nIt is also recommended that this block is loaded and consumed within a single task\nor flow because if the block is passed across separate tasks and flows,\nthe state of the block's connection and cursor could be lost. This block is part of the prefect-sqlalchemy collection. Install prefect-sqlalchemy with `pip install prefect-sqlalchemy` to use this block.", + "code_example": "Load stored database credentials and use in context manager:\n```python\nfrom prefect_sqlalchemy import SqlAlchemyConnector\n\ndatabase_block = SqlAlchemyConnector.load(\"BLOCK_NAME\")\nwith database_block:\n ...\n```\n\nCreate table named customers and insert values; then fetch the first 10 rows.\n```python\nfrom prefect_sqlalchemy import (\n SqlAlchemyConnector, SyncDriver, ConnectionComponents\n)\n\nwith SqlAlchemyConnector(\n connection_info=ConnectionComponents(\n driver=SyncDriver.SQLITE_PYSQLITE,\n database=\"prefect.db\"\n )\n) as database:\n database.execute(\n \"CREATE TABLE IF NOT EXISTS customers (name varchar, address varchar);\",\n )\n for i in range(1, 42):\n database.execute(\n \"INSERT INTO customers (name, address) VALUES (:name, :address);\",\n parameters={\"name\": \"Marvin\", \"address\": f\"Highway {i}\"},\n )\n results = database.fetch_many(\n \"SELECT * FROM customers WHERE name = :name;\",\n parameters={\"name\": \"Marvin\"},\n size=10\n )\nprint(results)\n```", + "block_schema": { + "checksum": "sha256:01e6c0bdaac125860811b201f5a5e98ffefd5f8a49f1398b6996aec362643acc", + "fields": { + "title": "SqlAlchemyConnector", + "description": "Block used to manage authentication with a database.\n\nUpon instantiating, an engine is created and maintained for the life of\nthe object until the close method is called.\n\nIt is recommended to use this block as a context manager, which will automatically\nclose the engine and its connections when the context is exited.\n\nIt is also recommended that this block is loaded and consumed within a single task\nor flow because if the block is passed across separate tasks and flows,\nthe state of the block's connection and cursor could be lost.", + "type": "object", + "properties": { + "connection_info": { + "title": "Connection Info", + "description": "SQLAlchemy URL to create the engine; either create from components or create from a string.", + "anyOf": [ + { + "$ref": "#/definitions/ConnectionComponents" + }, + { + "type": "string", + "minLength": 1, + "maxLength": 65536, + "format": "uri" + } + ] + }, + "connect_args": { + "title": "Additional Connection Arguments", + "description": "The options which will be passed directly to the DBAPI's connect() method as additional keyword arguments.", + "type": "object" + }, + "fetch_size": { + "title": "Fetch Size", + "description": "The number of rows to fetch at a time.", + "default": 1, + "type": "integer" + } + }, + "required": [ + "connection_info" + ], + "block_type_slug": "sqlalchemy-connector", + "secret_fields": [ + "connection_info.password" + ], + "block_schema_references": {}, + "definitions": { + "AsyncDriver": { + "title": "AsyncDriver", + "description": "Known dialects with their corresponding async drivers.\n\nAttributes:\n POSTGRESQL_ASYNCPG (Enum): [postgresql+asyncpg](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.asyncpg)\n\n SQLITE_AIOSQLITE (Enum): [sqlite+aiosqlite](https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#module-sqlalchemy.dialects.sqlite.aiosqlite)\n\n MYSQL_ASYNCMY (Enum): [mysql+asyncmy](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.asyncmy)\n MYSQL_AIOMYSQL (Enum): [mysql+aiomysql](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.aiomysql)", + "enum": [ + "postgresql+asyncpg", + "sqlite+aiosqlite", + "mysql+asyncmy", + "mysql+aiomysql" + ] + }, + "SyncDriver": { + "title": "SyncDriver", + "description": "Known dialects with their corresponding sync drivers.\n\nAttributes:\n POSTGRESQL_PSYCOPG2 (Enum): [postgresql+psycopg2](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.psycopg2)\n POSTGRESQL_PG8000 (Enum): [postgresql+pg8000](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.pg8000)\n POSTGRESQL_PSYCOPG2CFFI (Enum): [postgresql+psycopg2cffi](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.psycopg2cffi)\n POSTGRESQL_PYPOSTGRESQL (Enum): [postgresql+pypostgresql](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.pypostgresql)\n POSTGRESQL_PYGRESQL (Enum): [postgresql+pygresql](https://docs.sqlalchemy.org/en/14/dialects/postgresql.html#module-sqlalchemy.dialects.postgresql.pygresql)\n\n MYSQL_MYSQLDB (Enum): [mysql+mysqldb](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.mysqldb)\n MYSQL_PYMYSQL (Enum): [mysql+pymysql](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.pymysql)\n MYSQL_MYSQLCONNECTOR (Enum): [mysql+mysqlconnector](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.mysqlconnector)\n MYSQL_CYMYSQL (Enum): [mysql+cymysql](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.cymysql)\n MYSQL_OURSQL (Enum): [mysql+oursql](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.oursql)\n MYSQL_PYODBC (Enum): [mysql+pyodbc](https://docs.sqlalchemy.org/en/14/dialects/mysql.html#module-sqlalchemy.dialects.mysql.pyodbc)\n\n SQLITE_PYSQLITE (Enum): [sqlite+pysqlite](https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#module-sqlalchemy.dialects.sqlite.pysqlite)\n SQLITE_PYSQLCIPHER (Enum): [sqlite+pysqlcipher](https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#module-sqlalchemy.dialects.sqlite.pysqlcipher)\n\n ORACLE_CX_ORACLE (Enum): [oracle+cx_oracle](https://docs.sqlalchemy.org/en/14/dialects/oracle.html#module-sqlalchemy.dialects.oracle.cx_oracle)\n\n MSSQL_PYODBC (Enum): [mssql+pyodbc](https://docs.sqlalchemy.org/en/14/dialects/mssql.html#module-sqlalchemy.dialects.mssql.pyodbc)\n MSSQL_MXODBC (Enum): [mssql+mxodbc](https://docs.sqlalchemy.org/en/14/dialects/mssql.html#module-sqlalchemy.dialects.mssql.mxodbc)\n MSSQL_PYMSSQL (Enum): [mssql+pymssql](https://docs.sqlalchemy.org/en/14/dialects/mssql.html#module-sqlalchemy.dialects.mssql.pymssql)", + "enum": [ + "postgresql+psycopg2", + "postgresql+pg8000", + "postgresql+psycopg2cffi", + "postgresql+pypostgresql", + "postgresql+pygresql", + "mysql+mysqldb", + "mysql+pymysql", + "mysql+mysqlconnector", + "mysql+cymysql", + "mysql+oursql", + "mysql+pyodbc", + "sqlite+pysqlite", + "sqlite+pysqlcipher", + "oracle+cx_oracle", + "mssql+pyodbc", + "mssql+mxodbc", + "mssql+pymssql" + ] + }, + "ConnectionComponents": { + "title": "ConnectionComponents", + "description": "Parameters to use to create a SQLAlchemy engine URL.\n\nAttributes:\n driver: The driver name to use.\n database: The name of the database to use.\n username: The user name used to authenticate.\n password: The password used to authenticate.\n host: The host address of the database.\n port: The port to connect to the database.\n query: A dictionary of string keys to string values to be passed to the dialect\n and/or the DBAPI upon connect.", + "type": "object", + "properties": { + "driver": { + "title": "Driver", + "description": "The driver name to use.", + "anyOf": [ + { + "$ref": "#/definitions/AsyncDriver" + }, + { + "$ref": "#/definitions/SyncDriver" + }, + { + "type": "string" + } + ] + }, + "database": { + "title": "Database", + "description": "The name of the database to use.", + "type": "string" + }, + "username": { + "title": "Username", + "description": "The user name used to authenticate.", + "type": "string" + }, + "password": { + "title": "Password", + "description": "The password used to authenticate.", + "type": "string", + "writeOnly": true, + "format": "password" + }, + "host": { + "title": "Host", + "description": "The host address of the database.", + "type": "string" + }, + "port": { + "title": "Port", + "description": "The port to connect to the database.", + "type": "string" + }, + "query": { + "title": "Query", + "description": "A dictionary of string keys to string values to be passed to the dialect and/or the DBAPI upon connect. To specify non-string parameters to a Python DBAPI directly, use connect_args.", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "driver", + "database" + ] + } + } + }, + "capabilities": [], + "version": "0.3.0" + } + } + } + } +} \ No newline at end of file diff --git a/collections/prefect-sqlalchemy/flows/v0.3.0.json b/collections/prefect-sqlalchemy/flows/v0.3.0.json new file mode 100644 index 00000000..a1638a3e --- /dev/null +++ b/collections/prefect-sqlalchemy/flows/v0.3.0.json @@ -0,0 +1,3 @@ +{ + "prefect-sqlalchemy": {} +} \ No newline at end of file diff --git a/collections/prefect-sqlalchemy/workers/v0.3.0.json b/collections/prefect-sqlalchemy/workers/v0.3.0.json new file mode 100644 index 00000000..a1638a3e --- /dev/null +++ b/collections/prefect-sqlalchemy/workers/v0.3.0.json @@ -0,0 +1,3 @@ +{ + "prefect-sqlalchemy": {} +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 91d35814..e43f5d35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ classifiers = [ "Programming Language :: Python :: 3", ] dependencies = [ - "prefect>=2.8.2", + "prefect>=2.13.5", "fastjsonschema==2.16.2", "github3.py>=3.2.0", diff --git a/src/update_collection_metadata.py b/src/update_collection_metadata.py index d8c6c650..78b9dfef 100644 --- a/src/update_collection_metadata.py +++ b/src/update_collection_metadata.py @@ -199,6 +199,11 @@ async def update_all_collections( return Completed(message="All new releases have been recorded.") -if __name__ == "__main__": - import asyncio - asyncio.run(update_all_collections()) + +# if __name__ == "__main__": +## ALL COLLECTIONS +# asyncio.run(update_all_collections()) + +## MANUAL RUNS +# for collection in ["prefect-sqlalchemy"]: +# update_collection_metadata(collection, "update-metadata-manually") diff --git a/views/aggregate-block-metadata.json b/views/aggregate-block-metadata.json index ed3e519a..2ae78eb3 100644 --- a/views/aggregate-block-metadata.json +++ b/views/aggregate-block-metadata.json @@ -8426,7 +8426,7 @@ } }, "capabilities": [], - "version": "0.2.4" + "version": "0.3.0" } }, "sqlalchemy-connector": { @@ -8576,7 +8576,7 @@ } }, "capabilities": [], - "version": "0.2.4" + "version": "0.3.0" } } } @@ -8637,4 +8637,4 @@ } } } -} +} \ No newline at end of file