From bebee881006124cd725668314e40719246f051e1 Mon Sep 17 00:00:00 2001 From: Hyejin Yoon <0327jane@gmail.com> Date: Wed, 30 Aug 2023 09:27:38 +0900 Subject: [PATCH] feat(docs): move versioned_sidebars to static-assets (#8743) --- .gitignore | 1 + docs-website/download_historical_versions.py | 50 +- .../version-0.10.5-sidebars.json | 594 ------------------ 3 files changed, 35 insertions(+), 610 deletions(-) delete mode 100644 docs-website/versioned_sidebars/version-0.10.5-sidebars.json diff --git a/.gitignore b/.gitignore index b6edbccf711258..49ab5c475096c9 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ metadata-ingestion/generated/** # docs docs/generated/ docs-website/versioned_docs/ +docs-website/versioned_sidebars/ tmp* temp/** diff --git a/docs-website/download_historical_versions.py b/docs-website/download_historical_versions.py index a005445cb14970..83157edc1972cf 100644 --- a/docs-website/download_historical_versions.py +++ b/docs-website/download_historical_versions.py @@ -1,7 +1,7 @@ +import json import os import tarfile import urllib.request -import json repo_url = "https://api.github.com/repos/datahub-project/static-assets" @@ -16,30 +16,36 @@ def download_file(url, destination): f.write(chunk) -def fetch_tar_urls(repo_url, folder_path): +def fetch_urls(repo_url: str, folder_path: str, file_format: str): api_url = f"{repo_url}/contents/{folder_path}" response = urllib.request.urlopen(api_url) - data = response.read().decode('utf-8') - tar_urls = [ - file["download_url"] for file in json.loads(data) if file["name"].endswith(".tar.gz") + data = response.read().decode("utf-8") + urls = [ + file["download_url"] + for file in json.loads(data) + if file["name"].endswith(file_format) ] - print(tar_urls) - return tar_urls + print(urls) + return urls -def main(): - folder_path = "versioned_docs" - destination_dir = "versioned_docs" +def extract_tar_file(destination_path): + with tarfile.open(destination_path, "r:gz") as tar: + tar.extractall() + os.remove(destination_path) + + +def download_versioned_docs(folder_path: str, destination_dir: str, file_format: str): if not os.path.exists(destination_dir): os.makedirs(destination_dir) - tar_urls = fetch_tar_urls(repo_url, folder_path) + urls = fetch_urls(repo_url, folder_path, file_format) - for url in tar_urls: + for url in urls: filename = os.path.basename(url) destination_path = os.path.join(destination_dir, filename) - version = '.'.join(filename.split('.')[:3]) + version = ".".join(filename.split(".")[:3]) extracted_path = os.path.join(destination_dir, version) print("extracted_path", extracted_path) if os.path.exists(extracted_path): @@ -48,13 +54,25 @@ def main(): try: download_file(url, destination_path) print(f"Downloaded {filename} to {destination_dir}") - with tarfile.open(destination_path, "r:gz") as tar: - tar.extractall() - os.remove(destination_path) + if file_format == ".tar.gz": + extract_tar_file(destination_path) except urllib.error.URLError as e: print(f"Error while downloading {filename}: {e}") continue +def main(): + download_versioned_docs( + folder_path="versioned_docs", + destination_dir="versioned_docs", + file_format=".tar.gz", + ) + download_versioned_docs( + folder_path="versioned_sidebars", + destination_dir="versioned_sidebars", + file_format=".json", + ) + + if __name__ == "__main__": main() diff --git a/docs-website/versioned_sidebars/version-0.10.5-sidebars.json b/docs-website/versioned_sidebars/version-0.10.5-sidebars.json deleted file mode 100644 index 67179075fc994a..00000000000000 --- a/docs-website/versioned_sidebars/version-0.10.5-sidebars.json +++ /dev/null @@ -1,594 +0,0 @@ -{ - "overviewSidebar": [ - { - "label": "Getting Started", - "type": "category", - "collapsed": true, - "items": [ - { - "type": "doc", - "label": "Introduction", - "id": "docs/features" - }, - { - "type": "doc", - "label": "Quickstart", - "id": "docs/quickstart" - }, - { - "type": "link", - "label": "Demo", - "href": "https://demo.datahubproject.io/" - }, - "docs/what-is-datahub/datahub-concepts", - "docs/saas" - ] - }, - { - "Integrations": [ - { - "type": "doc", - "label": "Introduction", - "id": "metadata-ingestion/README" - }, - { - "Quickstart Guides": [ - { - "BigQuery": [ - "docs/quick-ingestion-guides/bigquery/overview", - "docs/quick-ingestion-guides/bigquery/setup", - "docs/quick-ingestion-guides/bigquery/configuration" - ] - }, - { - "Redshift": [ - "docs/quick-ingestion-guides/redshift/overview", - "docs/quick-ingestion-guides/redshift/setup", - "docs/quick-ingestion-guides/redshift/configuration" - ] - }, - { - "Snowflake": [ - "docs/quick-ingestion-guides/snowflake/overview", - "docs/quick-ingestion-guides/snowflake/setup", - "docs/quick-ingestion-guides/snowflake/configuration" - ] - }, - { - "Tableau": [ - "docs/quick-ingestion-guides/tableau/overview", - "docs/quick-ingestion-guides/tableau/setup", - "docs/quick-ingestion-guides/tableau/configuration" - ] - }, - { - "PowerBI": [ - "docs/quick-ingestion-guides/powerbi/overview", - "docs/quick-ingestion-guides/powerbi/setup", - "docs/quick-ingestion-guides/powerbi/configuration" - ] - } - ] - }, - { - "Sources": [ - { - "type": "doc", - "id": "docs/lineage/airflow", - "label": "Airflow" - }, - "metadata-integration/java/spark-lineage/README", - "metadata-ingestion/integration_docs/great-expectations", - "metadata-integration/java/datahub-protobuf/README", - { - "type": "autogenerated", - "dirName": "docs/generated/ingestion/sources" - } - ] - }, - { - "Sinks": [ - { - "type": "autogenerated", - "dirName": "metadata-ingestion/sink_docs" - } - ] - }, - { - "Transformers": [ - "metadata-ingestion/docs/transformer/intro", - "metadata-ingestion/docs/transformer/dataset_transformer" - ] - }, - { - "Advanced Guides": [ - { - "Scheduling Ingestion": [ - "metadata-ingestion/schedule_docs/intro", - "metadata-ingestion/schedule_docs/cron", - "metadata-ingestion/schedule_docs/airflow", - "metadata-ingestion/schedule_docs/kubernetes" - ] - }, - "docs/platform-instances", - "metadata-ingestion/docs/dev_guides/stateful", - "metadata-ingestion/docs/dev_guides/classification", - "metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source", - "metadata-ingestion/docs/dev_guides/sql_profiles" - ] - } - ] - }, - { - "Deployment": [ - "docs/deploy/aws", - "docs/deploy/gcp", - "docker/README", - "docs/deploy/kubernetes", - "docs/deploy/environment-vars", - { - "Authentication": [ - "docs/authentication/README", - "docs/authentication/concepts", - "docs/authentication/changing-default-credentials", - "docs/authentication/guides/add-users", - { - "Frontend Authentication": [ - "docs/authentication/guides/jaas", - { - "OIDC Authentication": [ - "docs/authentication/guides/sso/configure-oidc-react", - "docs/authentication/guides/sso/configure-oidc-react-google", - "docs/authentication/guides/sso/configure-oidc-react-okta", - "docs/authentication/guides/sso/configure-oidc-react-azure" - ] - } - ] - }, - "docs/authentication/introducing-metadata-service-authentication", - "docs/authentication/personal-access-tokens" - ] - }, - { - "Authorization": [ - "docs/authorization/README", - "docs/authorization/roles", - "docs/authorization/policies", - "docs/authorization/groups" - ] - }, - { - "Advanced Guides": [ - "docs/how/delete-metadata", - "docs/how/configuring-authorization-with-apache-ranger", - "docs/how/backup-datahub", - "docs/how/restore-indices", - "docs/advanced/db-retention", - "docs/advanced/monitoring", - "docs/how/extract-container-logs", - "docs/deploy/telemetry", - "docs/how/kafka-config", - "docs/deploy/confluent-cloud", - "docs/advanced/no-code-upgrade", - "docs/how/jattach-guide" - ] - }, - "docs/how/updating-datahub" - ] - }, - { - "API": [ - "docs/api/datahub-apis", - { - "GraphQL API": [ - { - "label": "Overview", - "type": "doc", - "id": "docs/api/graphql/overview" - }, - { - "Reference": [ - { - "type": "doc", - "label": "Queries", - "id": "graphql/queries" - }, - { - "type": "doc", - "label": "Mutations", - "id": "graphql/mutations" - }, - { - "type": "doc", - "label": "Objects", - "id": "graphql/objects" - }, - { - "type": "doc", - "label": "Inputs", - "id": "graphql/inputObjects" - }, - { - "type": "doc", - "label": "Interfaces", - "id": "graphql/interfaces" - }, - { - "type": "doc", - "label": "Unions", - "id": "graphql/unions" - }, - { - "type": "doc", - "label": "Enums", - "id": "graphql/enums" - }, - { - "type": "doc", - "label": "Scalars", - "id": "graphql/scalars" - } - ] - }, - { - "Guides": [ - { - "type": "doc", - "label": "How To Set Up GraphQL", - "id": "docs/api/graphql/how-to-set-up-graphql" - }, - { - "type": "doc", - "label": "Getting Started With GraphQL", - "id": "docs/api/graphql/getting-started" - }, - { - "type": "doc", - "label": "Access Token Management", - "id": "docs/api/graphql/token-management" - } - ] - } - ] - }, - { - "type": "doc", - "label": "OpenAPI", - "id": "docs/api/openapi/openapi-usage-guide" - }, - "docs/dev-guides/timeline", - { - "Rest.li API": [ - { - "type": "doc", - "label": "Rest.li API Guide", - "id": "docs/api/restli/restli-overview" - }, - { - "type": "doc", - "label": "Restore Indices", - "id": "docs/api/restli/restore-indices" - }, - { - "type": "doc", - "label": "Get Index Sizes", - "id": "docs/api/restli/get-index-sizes" - }, - { - "type": "doc", - "label": "Truncate Timeseries Aspect", - "id": "docs/api/restli/truncate-time-series-aspect" - }, - { - "type": "doc", - "label": "Get ElasticSearch Task Status Endpoint", - "id": "docs/api/restli/get-elastic-task-status" - }, - { - "type": "doc", - "label": "Evaluate Tests", - "id": "docs/api/restli/evaluate-tests" - }, - { - "type": "doc", - "label": "Aspect Versioning and Rest.li Modeling", - "id": "docs/advanced/aspect-versioning" - } - ] - }, - { - "Python SDK": [ - "metadata-ingestion/as-a-library", - { - "Python SDK Reference": [ - { - "type": "autogenerated", - "dirName": "python-sdk" - } - ] - } - ] - }, - "metadata-integration/java/as-a-library", - { - "API and SDK Guides": [ - "docs/advanced/patch", - "docs/api/tutorials/datasets", - "docs/api/tutorials/lineage", - "docs/api/tutorials/tags", - "docs/api/tutorials/terms", - "docs/api/tutorials/owners", - "docs/api/tutorials/domains", - "docs/api/tutorials/deprecation", - "docs/api/tutorials/descriptions", - "docs/api/tutorials/custom-properties", - "docs/api/tutorials/ml" - ] - }, - { - "type": "category", - "label": "DataHub CLI", - "link": { - "type": "doc", - "id": "docs/cli" - }, - "items": [ - "docs/datahub_lite" - ] - }, - { - "type": "category", - "label": "Datahub Actions", - "link": { - "type": "doc", - "id": "docs/act-on-metadata" - }, - "items": [ - "docs/actions/README", - "docs/actions/quickstart", - "docs/actions/concepts", - { - "Sources": [ - { - "type": "autogenerated", - "dirName": "docs/actions/sources" - } - ] - }, - { - "Events": [ - { - "type": "autogenerated", - "dirName": "docs/actions/events" - } - ] - }, - { - "Actions": [ - { - "type": "autogenerated", - "dirName": "docs/actions/actions" - } - ] - }, - { - "Guides": [ - { - "type": "autogenerated", - "dirName": "docs/actions/guides" - } - ] - } - ] - } - ] - }, - { - "Features": [ - "docs/ui-ingestion", - "docs/how/search", - "docs/schema-history", - "docs/domains", - "docs/dataproducts", - "docs/glossary/business-glossary", - "docs/tags", - "docs/ownership/ownership-types", - "docs/browse", - "docs/authorization/access-policies-guide", - "docs/features/dataset-usage-and-query-history", - "docs/posts", - "docs/sync-status", - "docs/lineage/lineage-feature-guide", - { - "type": "doc", - "id": "docs/tests/metadata-tests", - "className": "saasOnly" - }, - "docs/act-on-metadata/impact-analysis", - { - "Observability": [ - "docs/managed-datahub/observe/freshness-assertions" - ] - } - ] - }, - { - "Develop": [ - { - "DataHub Metadata Model": [ - "docs/modeling/metadata-model", - "docs/modeling/extending-the-metadata-model", - "docs/what/mxe", - { - "Entities": [ - { - "type": "autogenerated", - "dirName": "docs/generated/metamodel/entities" - } - ] - } - ] - }, - { - "Architecture": [ - "docs/architecture/architecture", - "docs/components", - "docs/architecture/metadata-ingestion", - "docs/architecture/metadata-serving", - "docs/architecture/docker-containers" - ] - }, - { - "Developing on DataHub": [ - "docs/developers", - "docs/docker/development", - "metadata-ingestion/developing", - "docs/api/graphql/graphql-endpoint-development", - { - "Modules": [ - "datahub-web-react/README", - "datahub-frontend/README", - "datahub-graphql-core/README", - "metadata-service/README", - "metadata-jobs/mae-consumer-job/README", - "metadata-jobs/mce-consumer-job/README" - ] - } - ] - }, - "docs/plugins", - { - "Troubleshooting": [ - "docs/troubleshooting/quickstart", - "docs/troubleshooting/build", - "docs/troubleshooting/general" - ] - }, - { - "Advanced": [ - "metadata-ingestion/docs/dev_guides/reporting_telemetry", - "docs/advanced/mcp-mcl", - "docker/datahub-upgrade/README", - "docs/advanced/no-code-modeling", - "datahub-web-react/src/app/analytics/README", - "docs/how/migrating-graph-service-implementation", - "docs/advanced/field-path-spec-v2", - "metadata-ingestion/adding-source", - "docs/how/add-custom-ingestion-source", - "docs/how/add-custom-data-platform", - "docs/advanced/browse-paths-upgrade", - "docs/browseV2/browse-paths-v2" - ] - } - ] - }, - { - "Community": [ - "docs/slack", - "docs/townhalls", - "docs/townhall-history", - "docs/CODE_OF_CONDUCT", - "docs/CONTRIBUTING", - "docs/links", - "docs/rfc" - ] - }, - { - "Managed DataHub": [ - "docs/managed-datahub/managed-datahub-overview", - "docs/managed-datahub/welcome-acryl", - { - "type": "doc", - "id": "docs/managed-datahub/saas-slack-setup", - "className": "saasOnly" - }, - { - "type": "doc", - "id": "docs/managed-datahub/approval-workflows", - "className": "saasOnly" - }, - { - "Metadata Ingestion With Acryl": [ - "docs/managed-datahub/metadata-ingestion-with-acryl/ingestion" - ] - }, - { - "DataHub API": [ - { - "type": "doc", - "id": "docs/managed-datahub/datahub-api/entity-events-api", - "className": "saasOnly" - }, - { - "GraphQL API": [ - "docs/managed-datahub/datahub-api/graphql-api/getting-started", - { - "type": "doc", - "id": "docs/managed-datahub/datahub-api/graphql-api/incidents-api-beta", - "className": "saasOnly" - } - ] - } - ] - }, - { - "Integrations": [ - { - "type": "doc", - "id": "docs/managed-datahub/integrations/aws-privatelink", - "className": "saasOnly" - }, - { - "type": "doc", - "id": "docs/managed-datahub/integrations/oidc-sso-integration", - "className": "saasOnly" - } - ] - }, - { - "Operator Guide": [ - { - "type": "doc", - "id": "docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor-on-aws", - "className": "saasOnly" - }, - { - "type": "doc", - "id": "docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge", - "className": "saasOnly" - } - ] - }, - { - "type": "doc", - "id": "docs/managed-datahub/chrome-extension", - "className": "saasOnly" - }, - { - "Managed DataHub Release History": [ - "docs/managed-datahub/release-notes/v_0_2_10", - "docs/managed-datahub/release-notes/v_0_2_9", - "docs/managed-datahub/release-notes/v_0_2_8", - "docs/managed-datahub/release-notes/v_0_2_7", - "docs/managed-datahub/release-notes/v_0_2_6", - "docs/managed-datahub/release-notes/v_0_2_5", - "docs/managed-datahub/release-notes/v_0_2_4", - "docs/managed-datahub/release-notes/v_0_2_3", - "docs/managed-datahub/release-notes/v_0_2_2", - "docs/managed-datahub/release-notes/v_0_2_1", - "docs/managed-datahub/release-notes/v_0_2_0", - "docs/managed-datahub/release-notes/v_0_1_73", - "docs/managed-datahub/release-notes/v_0_1_72", - "docs/managed-datahub/release-notes/v_0_1_70", - "docs/managed-datahub/release-notes/v_0_1_69" - ] - } - ] - }, - { - "Release History": [ - "releases" - ] - } - ] -}