From a1d3748993b07c3b6cbb1ab0cbcb618362671ee7 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Fri, 16 Sep 2022 11:46:34 -0400 Subject: [PATCH 1/3] codespace compatible --- .devcontainer.json | 94 +++++++++++++++++++++++++++++++++++++++++ .gitignore | 4 ++ .sqlfluff | 3 ++ .vscode/extensions.json | 16 +++++++ Dockerfile | 14 ++++++ profiles.yml | 29 +++++++++++++ requirements.txt | 3 ++ 7 files changed, 163 insertions(+) create mode 100644 .devcontainer.json create mode 100644 .sqlfluff create mode 100644 .vscode/extensions.json create mode 100644 Dockerfile create mode 100644 profiles.yml create mode 100644 requirements.txt diff --git a/.devcontainer.json b/.devcontainer.json new file mode 100644 index 0000000..faef12f --- /dev/null +++ b/.devcontainer.json @@ -0,0 +1,94 @@ +// Update the VARIANT arg in docker-compose.yml to pick a Python version: 3, 3.8, 3.7, 3.6 +{ + "name": "dbt", + "dockerFile" : "Dockerfile", + "settings": { + "terminal.integrated.defaultProfile.linux#": "/bin/sh", + "files.associations": { + "*.sql": "jinja-sql", + "*.sqlfluff": "ini", + "*.yml": "yaml", + "**/target/**": "" + }, + "sql.linter.executablePath": "/usr/local/bin/sqlfluff", + "sql.format.enable": true, + "sql.linter.run": "onType", + "files.exclude": { + "**/.git": true, + "**/.svn": true, + "**/.hg": true, + "**/CVS": true, + "**/.DS_Store": true, + "**/__pycache__": true + }, + "findrelated.workspaceRulesets": [ + { + "name": "sql", + "rules": [ + { + "pattern": "^(.*/)?models/(.*/)?(.+\\.sql)$", + "locators": [ + "**/compiled/**/$3" + ] + }, + { + "pattern": "^(.*/)?compiled/(.*/)?(.+\\.sql)$", + "locators": [ + "**/run/**/$3" + ] + }, + { + "pattern": "^(.*/)?run/(.*/)?(.+\\.sql)$", + "locators": [ + "**/models/**/$3" + ] + } + ] + } + ], + "findrelated.applyRulesets": [ + "sql" + ], + "findrelated.applyWorkspaceRulesets": [ + "sql" + ], + "workbench.editor.highlightModifiedTabs": true, + "workbench.editor.labelFormat": "medium", + "workbench.editor.revealIfOpen": true, + "editor.rulers": [ + 99 + ], + "yaml.schemas": { + "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/dbt_yml_files.json": [ + "/*.yml", + "!profiles.yml", + "!dbt_project.yml", + "!packages.yml", + "!selectors.yml", + "!profile_template.yml" + ], + "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/dbt_project.json": [ + "dbt_project.yml" + ], + "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/selectors.json": [ + "selectors.yml" + ], + "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/packages.json": [ + "packages.yml" + ] + } + }, + "extensions": [ + "bastienboutonnet.vscode-dbt", + "dorzey.vscode-sqlfluff", + "editorconfig.editorconfig", + "amodio.find-related", + "ms-azuretools.vscode-docker", + "ms-python.python", + "visualstudioexptteam.vscodeintellicode", + "samuelcolvin.jinjahtml", + "redhat.vscode-yaml" + ], + + "remoteUser": "vscode" +} diff --git a/.gitignore b/.gitignore index 49f147c..c2ab63c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ target/ dbt_packages/ logs/ +**/.DS_Store +.user.yml +venv/ +env/ \ No newline at end of file diff --git a/.sqlfluff b/.sqlfluff new file mode 100644 index 0000000..a186218 --- /dev/null +++ b/.sqlfluff @@ -0,0 +1,3 @@ +[sqlfluff] + +dialect = snowflake \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..2d4ec01 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,16 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. + // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp + + // List of extensions which should be recommended for users of this workspace. + "recommendations": [ + "innoverio.vscode-dbt-power-user", + "mechatroner.rainbow-csv", + "yzhang.markdown-all-in-one", + "redhat.vscode-yaml" + ], + // List of extensions recommended by VS Code that should not be recommended for users of this workspace. + "unwantedRecommendations": [ + + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..457517f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM mcr.microsoft.com/vscode/devcontainers/python:3.9 + +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +RUN if [ "$USER_GID" != "1000" ] || [ "$USER_UID" != "1000" ]; then groupmod --gid $USER_GID vscode && usermod --uid $USER_UID --gid $USER_GID vscode; fi + +RUN pwd +RUN ls +COPY requirements.txt /tmp/ +RUN pip3 install --upgrade pip +RUN pip3 install --requirement /tmp/requirements.txt + +ENV DBT_PROFILES_DIR=/workspaces/new-python-wrench-demo diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..8822b5e --- /dev/null +++ b/profiles.yml @@ -0,0 +1,29 @@ +config: + partial_parse: true +variables: + snow-dev: &snow-dev + type: snowflake + account: '{{ env_var("DBT_SNOW_DEV_ACCOUNT") }}' + warehouse: '{{ env_var("DBT_SNOW_DEV_WH") }}' # transforming + database: '{{ env_var("DBT_SNOW_DEV_DB") }}' # jaffle_shop + schema: '{{ env_var("DBT_SNOW_SCH") }}' # ajs + user: '{{ env_var("DBT_SNOW_USER") }}' # ANDERS.SWANSON + role: transformer + authenticator: externalbrowser + threads: 10 + + snow-prd: &snow-prd + type: snowflake + account: '{{ env_var("DBT_SNOW_PRD_ACCOUNT") }}' + warehouse: '{{ env_var("DBT_SNOW_PRD_WH") }}' # transforming + database: '{{ env_var("DBT_SNOW_PRD_DB") }}' # jaffle_shop + schema: '{{ env_var("DBT_SNOW_SCH") }}' # ajs + user: '{{ env_var("DBT_SNOW_USER") }}' # ANDERS.SWANSON + role: transformer + authenticator: externalbrowser + threads: 10 +python_wrench: + target: dev + outputs: + dev: *snow-dev + prd: *snow-prd diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5f65008 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +dbt-snowflake==1.3.0b2 +# extra features +sqlfluff~=1.2.1 \ No newline at end of file From c41591819957b02d09c4051b44f6b7b2d5ba2f7f Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Fri, 16 Sep 2022 11:59:30 -0400 Subject: [PATCH 2/3] add example env file --- .env.example | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3c12f9b --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +DBT_SNOW_DEV_ACCOUNT= +DBT_SNOW_DEV_WH= +DBT_SNOW_DEV_DB= +DBT_SNOW_SCH= +DBT_SNOW_USER= +DBT_SNOW_PRD_ACCOUNT= +DBT_SNOW_PRD_WH= +DBT_SNOW_PRD_DB= \ No newline at end of file From 9efe6eeb354d765f1419adeea755f38659c9884d Mon Sep 17 00:00:00 2001 From: Anders Date: Fri, 16 Sep 2022 16:24:30 +0000 Subject: [PATCH 3/3] refactor creds --- profiles.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/profiles.yml b/profiles.yml index 8822b5e..099ce43 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,27 +1,25 @@ config: partial_parse: true + variables: snow-dev: &snow-dev type: snowflake account: '{{ env_var("DBT_SNOW_DEV_ACCOUNT") }}' - warehouse: '{{ env_var("DBT_SNOW_DEV_WH") }}' # transforming - database: '{{ env_var("DBT_SNOW_DEV_DB") }}' # jaffle_shop - schema: '{{ env_var("DBT_SNOW_SCH") }}' # ajs - user: '{{ env_var("DBT_SNOW_USER") }}' # ANDERS.SWANSON + warehouse: '{{ env_var("DBT_SNOW_DEV_WH") }}' + database: '{{ env_var("DBT_SNOW_DEV_DB") }}' + schema: '{{ env_var("DBT_SNOW_SCH") }}' + user: '{{ env_var("DBT_SNOW_DEV_USER") }}' role: transformer authenticator: externalbrowser threads: 10 snow-prd: &snow-prd - type: snowflake + <<: *snow-dev account: '{{ env_var("DBT_SNOW_PRD_ACCOUNT") }}' - warehouse: '{{ env_var("DBT_SNOW_PRD_WH") }}' # transforming - database: '{{ env_var("DBT_SNOW_PRD_DB") }}' # jaffle_shop - schema: '{{ env_var("DBT_SNOW_SCH") }}' # ajs - user: '{{ env_var("DBT_SNOW_USER") }}' # ANDERS.SWANSON - role: transformer - authenticator: externalbrowser - threads: 10 + warehouse: '{{ env_var("DBT_SNOW_PRD_WH") }}' + database: '{{ env_var("DBT_SNOW_PRD_DB") }}' + schema: '{{ env_var("DBT_SNOW_SCH") }}' + user: '{{ env_var("DBT_SNOW_PRD_USER") }}' python_wrench: target: dev outputs: