From 79db885c2977a03572e5cff9326a8b97c03dac65 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Tue, 24 Sep 2024 15:22:25 -0400 Subject: [PATCH 1/4] Add '--inline-direct' parameter to 'dbt show'. --- core/dbt/cli/main.py | 29 +++++++++++++----- core/dbt/cli/params.py | 6 ++++ core/dbt/cli/requires.py | 47 ++++++++++++++++-------------- core/dbt/task/show.py | 27 +++++++++++++++++ tests/functional/show/test_show.py | 17 +++++++++++ 5 files changed, 96 insertions(+), 30 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index ca79d5eb073..489ac0a4955 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -8,12 +8,15 @@ from click.exceptions import Exit as ClickExit from click.exceptions import NoSuchOption, UsageError +from dbt.adapters.factory import register_adapter from dbt.artifacts.schemas.catalog import CatalogArtifact from dbt.artifacts.schemas.run import RunExecutionResult from dbt.cli import params as p from dbt.cli import requires from dbt.cli.exceptions import DbtInternalException, DbtUsageException +from dbt.cli.requires import setup_manifest from dbt.contracts.graph.manifest import Manifest +from dbt.mp_context import get_mp_context from dbt_common.events.base_types import EventMsg @@ -354,6 +357,7 @@ def compile(ctx, **kwargs): @p.select @p.selector @p.inline +@p.inline_direct @p.target_path @p.threads @p.vars @@ -362,17 +366,26 @@ def compile(ctx, **kwargs): @requires.profile @requires.project @requires.runtime_config -@requires.manifest def show(ctx, **kwargs): """Generates executable SQL for a named resource or inline query, runs that SQL, and returns a preview of the results. Does not materialize anything to the warehouse.""" - from dbt.task.show import ShowTask - - task = ShowTask( - ctx.obj["flags"], - ctx.obj["runtime_config"], - ctx.obj["manifest"], - ) + from dbt.task.show import ShowTask, ShowTaskDirect + + if ctx.obj["flags"].inline_direct: + # Issue the inline query directly, wit no templating. Does not require + # loading the mainfest. + register_adapter(ctx.obj["runtime_config"], get_mp_context()) + task = ShowTaskDirect( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ) + else: + setup_manifest(ctx) + task = ShowTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) results = task.run() success = task.interpret_results(results) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 425009d76ee..33f586248c5 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -487,6 +487,12 @@ help="Pass SQL inline to dbt compile and show", ) +inline_direct = click.option( + "--inline-direct", + envvar=None, + help="Pass SQL inline to dbt show. Do not load the entire project or apply templating.", +) + # `--select` and `--models` are analogous for most commands except `dbt list` for legacy reasons. # Most CLI arguments should use the combined `select` option that aliases `--models` to `--select`. # However, if you need to split out these separators (like `dbt ls`), use the `models` and `raw_select` options instead. diff --git a/core/dbt/cli/requires.py b/core/dbt/cli/requires.py index 0c0b1900827..058b16cdfaa 100644 --- a/core/dbt/cli/requires.py +++ b/core/dbt/cli/requires.py @@ -324,28 +324,7 @@ def wrapper(*args, **kwargs): ctx = args[0] assert isinstance(ctx, Context) - req_strs = ["profile", "project", "runtime_config"] - reqs = [ctx.obj.get(dep) for dep in req_strs] - - if None in reqs: - raise DbtProjectError("profile, project, and runtime_config required for manifest") - - runtime_config = ctx.obj["runtime_config"] - - # if a manifest has already been set on the context, don't overwrite it - if ctx.obj.get("manifest") is None: - ctx.obj["manifest"] = parse_manifest( - runtime_config, write_perf_info, write, ctx.obj["flags"].write_json - ) - else: - register_adapter(runtime_config, get_mp_context()) - adapter = get_adapter(runtime_config) - adapter.set_macro_context_generator(generate_runtime_macro_context) - adapter.set_macro_resolver(ctx.obj["manifest"]) - query_header_context = generate_query_header_context( - adapter.config, ctx.obj["manifest"] - ) - adapter.connections.set_query_header(query_header_context) + setup_manifest(ctx, write=write, write_perf_info=write_perf_info) return func(*args, **kwargs) return update_wrapper(wrapper, func) @@ -355,3 +334,27 @@ def wrapper(*args, **kwargs): if len(args0) == 0: return outer_wrapper return outer_wrapper(args0[0]) + + +def setup_manifest(ctx: Context, write: bool = True, write_perf_info: bool = False): + """Load the manifest and add it to the context.""" + req_strs = ["profile", "project", "runtime_config"] + reqs = [ctx.obj.get(dep) for dep in req_strs] + + if None in reqs: + raise DbtProjectError("profile, project, and runtime_config required for manifest") + + runtime_config = ctx.obj["runtime_config"] + + # if a manifest has already been set on the context, don't overwrite it + if ctx.obj.get("manifest") is None: + ctx.obj["manifest"] = parse_manifest( + runtime_config, write_perf_info, write, ctx.obj["flags"].write_json + ) + else: + register_adapter(runtime_config, get_mp_context()) + adapter = get_adapter(runtime_config) + adapter.set_macro_context_generator(generate_runtime_macro_context) # type: ignore[arg-type] + adapter.set_macro_resolver(ctx.obj["manifest"]) + query_header_context = generate_query_header_context(adapter.config, ctx.obj["manifest"]) # type: ignore[attr-defined] + adapter.connections.set_query_header(query_header_context) diff --git a/core/dbt/task/show.py b/core/dbt/task/show.py index 0fb6551bf94..f7784136ceb 100644 --- a/core/dbt/task/show.py +++ b/core/dbt/task/show.py @@ -2,10 +2,12 @@ import threading import time +from dbt.adapters.factory import get_adapter from dbt.artifacts.schemas.run import RunResult, RunStatus from dbt.context.providers import generate_runtime_model_context from dbt.contracts.graph.nodes import SeedNode from dbt.events.types import ShowNode +from dbt.task.base import ConfiguredTask from dbt.task.compile import CompileRunner, CompileTask from dbt.task.seed import SeedRunner from dbt_common.events.base_types import EventLevel @@ -117,3 +119,28 @@ def _handle_result(self, result) -> None: and (self.args.select or getattr(self.args, "inline", None)) ): self.node_results.append(result) + + +class ShowTaskDirect(ConfiguredTask): + def run(self): + adapter = get_adapter(self.config) + with adapter.connection_named("show", should_release_connection=False): + response, table = adapter.execute( + self.args.inline_direct, fetch=True, limit=self.args.limit + ) + + output = io.StringIO() + if self.args.output == "json": + table.to_json(path=output) + else: + table.print_table(output=output, max_rows=None) + + fire_event( + ShowNode( + node_name="direct-query", + preview=output.getvalue(), + is_inline=True, + output_format=self.args.output, + unique_id="direct-query", + ) + ) diff --git a/tests/functional/show/test_show.py b/tests/functional/show/test_show.py index b1aa16210b8..09c35035a1a 100644 --- a/tests/functional/show/test_show.py +++ b/tests/functional/show/test_show.py @@ -120,6 +120,23 @@ def test_inline_pass(self, project): assert "sample_bool" in log_output +class TestShowInlineDirect(ShowBase): + + def test_inline_direct_pass(self, project): + query = f"select * from {project.test_schema}.sample_seed" + (_, log_output) = run_dbt_and_capture(["show", "--inline-direct", query]) + assert "Previewing inline node" in log_output + assert "sample_num" in log_output + assert "sample_bool" in log_output + + # This is a bit of a hack. Unfortunately, the test teardown code + # expects that dbt loaded an adapter with a macro context the last + # time it was called. The '--inline-direct' parameter used on the + # previous run explicitly disables macros. So now we call 'dbt seed', + # which will load the adapter fully and satisfy the teardown code. + run_dbt(["seed"]) + + class TestShowInlineFail(ShowBase): def test_inline_fail(self, project): with pytest.raises(DbtException, match="Error parsing inline query"): From 12c3dd357d58dce8d609f82945435a16578c007d Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Tue, 24 Sep 2024 15:29:43 -0400 Subject: [PATCH 2/4] Add changelog entry. --- .changes/unreleased/Features-20240924-152922.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20240924-152922.yaml diff --git a/.changes/unreleased/Features-20240924-152922.yaml b/.changes/unreleased/Features-20240924-152922.yaml new file mode 100644 index 00000000000..095a82365ce --- /dev/null +++ b/.changes/unreleased/Features-20240924-152922.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Added the --inline-direct parameter to 'dbt show' +time: 2024-09-24T15:29:22.874496-04:00 +custom: + Author: aranke peterallenwebb + Issue: "10770" From 90c59bd9e8ec1c99f7c76939bba4ad849a2ce9c9 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Wed, 25 Sep 2024 11:51:04 +0100 Subject: [PATCH 3/4] Update core/dbt/cli/main.py Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- core/dbt/cli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 489ac0a4955..610163042c4 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -372,8 +372,8 @@ def show(ctx, **kwargs): from dbt.task.show import ShowTask, ShowTaskDirect if ctx.obj["flags"].inline_direct: - # Issue the inline query directly, wit no templating. Does not require - # loading the mainfest. + # Issue the inline query directly, with no templating. Does not require + # loading the manifest. register_adapter(ctx.obj["runtime_config"], get_mp_context()) task = ShowTaskDirect( ctx.obj["flags"], From 9b0de70f3983d541d5692fb8229c064ba04ffe73 Mon Sep 17 00:00:00 2001 From: Peter Allen Webb Date: Wed, 25 Sep 2024 10:40:20 -0400 Subject: [PATCH 4/4] Add test of failure for --inline-direct --- tests/functional/show/test_show.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/functional/show/test_show.py b/tests/functional/show/test_show.py index 09c35035a1a..7640338d103 100644 --- a/tests/functional/show/test_show.py +++ b/tests/functional/show/test_show.py @@ -120,6 +120,18 @@ def test_inline_pass(self, project): assert "sample_bool" in log_output +class TestShowInlineFail(ShowBase): + def test_inline_fail(self, project): + with pytest.raises(DbtException, match="Error parsing inline query"): + run_dbt(["show", "--inline", "select * from {{ ref('third_model') }}"]) + + +class TestShowInlineFailDB(ShowBase): + def test_inline_fail_database_error(self, project): + with pytest.raises(DbtRuntimeError, match="Database Error"): + run_dbt(["show", "--inline", "slect asdlkjfsld;j"]) + + class TestShowInlineDirect(ShowBase): def test_inline_direct_pass(self, project): @@ -137,16 +149,14 @@ def test_inline_direct_pass(self, project): run_dbt(["seed"]) -class TestShowInlineFail(ShowBase): - def test_inline_fail(self, project): - with pytest.raises(DbtException, match="Error parsing inline query"): - run_dbt(["show", "--inline", "select * from {{ ref('third_model') }}"]) - +class TestShowInlineDirectFail(ShowBase): -class TestShowInlineFailDB(ShowBase): def test_inline_fail_database_error(self, project): with pytest.raises(DbtRuntimeError, match="Database Error"): - run_dbt(["show", "--inline", "slect asdlkjfsld;j"]) + run_dbt(["show", "--inline-direct", "slect asdlkjfsld;j"]) + + # See prior test for explanation of why this is here + run_dbt(["seed"]) class TestShowEphemeral(ShowBase):