Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add '--inline-direct' parameter to 'dbt show'. #10770

Merged
merged 4 commits into from
Sep 25, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20240924-152922.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Added the --inline-direct parameter to 'dbt show'
time: 2024-09-24T15:29:22.874496-04:00
custom:
Author: aranke peterallenwebb
Issue: "10770"
29 changes: 21 additions & 8 deletions core/dbt/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,15 @@
from click.exceptions import Exit as ClickExit
from click.exceptions import NoSuchOption, UsageError

from dbt.adapters.factory import register_adapter
from dbt.artifacts.schemas.catalog import CatalogArtifact
from dbt.artifacts.schemas.run import RunExecutionResult
from dbt.cli import params as p
from dbt.cli import requires
from dbt.cli.exceptions import DbtInternalException, DbtUsageException
from dbt.cli.requires import setup_manifest
from dbt.contracts.graph.manifest import Manifest
from dbt.mp_context import get_mp_context
from dbt_common.events.base_types import EventMsg


Expand Down Expand Up @@ -354,6 +357,7 @@ def compile(ctx, **kwargs):
@p.select
@p.selector
@p.inline
@p.inline_direct
@p.target_path
@p.threads
@p.vars
Expand All @@ -362,17 +366,26 @@ def compile(ctx, **kwargs):
@requires.profile
@requires.project
@requires.runtime_config
@requires.manifest
def show(ctx, **kwargs):
"""Generates executable SQL for a named resource or inline query, runs that SQL, and returns a preview of the
results. Does not materialize anything to the warehouse."""
from dbt.task.show import ShowTask

task = ShowTask(
ctx.obj["flags"],
ctx.obj["runtime_config"],
ctx.obj["manifest"],
)
from dbt.task.show import ShowTask, ShowTaskDirect

if ctx.obj["flags"].inline_direct:
# Issue the inline query directly, with no templating. Does not require
# loading the manifest.
register_adapter(ctx.obj["runtime_config"], get_mp_context())
task = ShowTaskDirect(
ctx.obj["flags"],
ctx.obj["runtime_config"],
)
else:
setup_manifest(ctx)
task = ShowTask(
ctx.obj["flags"],
ctx.obj["runtime_config"],
ctx.obj["manifest"],
)

results = task.run()
success = task.interpret_results(results)
Expand Down
6 changes: 6 additions & 0 deletions core/dbt/cli/params.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,12 @@
help="Pass SQL inline to dbt compile and show",
)

inline_direct = click.option(
"--inline-direct",
envvar=None,
help="Pass SQL inline to dbt show. Do not load the entire project or apply templating.",
)

# `--select` and `--models` are analogous for most commands except `dbt list` for legacy reasons.
# Most CLI arguments should use the combined `select` option that aliases `--models` to `--select`.
# However, if you need to split out these separators (like `dbt ls`), use the `models` and `raw_select` options instead.
Expand Down
47 changes: 25 additions & 22 deletions core/dbt/cli/requires.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,28 +324,7 @@
ctx = args[0]
assert isinstance(ctx, Context)

req_strs = ["profile", "project", "runtime_config"]
reqs = [ctx.obj.get(dep) for dep in req_strs]

if None in reqs:
raise DbtProjectError("profile, project, and runtime_config required for manifest")

runtime_config = ctx.obj["runtime_config"]

# if a manifest has already been set on the context, don't overwrite it
if ctx.obj.get("manifest") is None:
ctx.obj["manifest"] = parse_manifest(
runtime_config, write_perf_info, write, ctx.obj["flags"].write_json
)
else:
register_adapter(runtime_config, get_mp_context())
adapter = get_adapter(runtime_config)
adapter.set_macro_context_generator(generate_runtime_macro_context)
adapter.set_macro_resolver(ctx.obj["manifest"])
query_header_context = generate_query_header_context(
adapter.config, ctx.obj["manifest"]
)
adapter.connections.set_query_header(query_header_context)
setup_manifest(ctx, write=write, write_perf_info=write_perf_info)
return func(*args, **kwargs)

return update_wrapper(wrapper, func)
Expand All @@ -355,3 +334,27 @@
if len(args0) == 0:
return outer_wrapper
return outer_wrapper(args0[0])


def setup_manifest(ctx: Context, write: bool = True, write_perf_info: bool = False):
"""Load the manifest and add it to the context."""
req_strs = ["profile", "project", "runtime_config"]
reqs = [ctx.obj.get(dep) for dep in req_strs]

if None in reqs:
raise DbtProjectError("profile, project, and runtime_config required for manifest")

Check warning on line 345 in core/dbt/cli/requires.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/cli/requires.py#L345

Added line #L345 was not covered by tests

runtime_config = ctx.obj["runtime_config"]

# if a manifest has already been set on the context, don't overwrite it
if ctx.obj.get("manifest") is None:
ctx.obj["manifest"] = parse_manifest(
runtime_config, write_perf_info, write, ctx.obj["flags"].write_json
)
else:
register_adapter(runtime_config, get_mp_context())
adapter = get_adapter(runtime_config)
adapter.set_macro_context_generator(generate_runtime_macro_context) # type: ignore[arg-type]
adapter.set_macro_resolver(ctx.obj["manifest"])
query_header_context = generate_query_header_context(adapter.config, ctx.obj["manifest"]) # type: ignore[attr-defined]
adapter.connections.set_query_header(query_header_context)
27 changes: 27 additions & 0 deletions core/dbt/task/show.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
import threading
import time

from dbt.adapters.factory import get_adapter
from dbt.artifacts.schemas.run import RunResult, RunStatus
from dbt.context.providers import generate_runtime_model_context
from dbt.contracts.graph.nodes import SeedNode
from dbt.events.types import ShowNode
from dbt.task.base import ConfiguredTask
from dbt.task.compile import CompileRunner, CompileTask
from dbt.task.seed import SeedRunner
from dbt_common.events.base_types import EventLevel
Expand Down Expand Up @@ -117,3 +119,28 @@
and (self.args.select or getattr(self.args, "inline", None))
):
self.node_results.append(result)


class ShowTaskDirect(ConfiguredTask):
def run(self):
adapter = get_adapter(self.config)
with adapter.connection_named("show", should_release_connection=False):
response, table = adapter.execute(
self.args.inline_direct, fetch=True, limit=self.args.limit
)

output = io.StringIO()
if self.args.output == "json":
table.to_json(path=output)

Check warning on line 134 in core/dbt/task/show.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/task/show.py#L134

Added line #L134 was not covered by tests
else:
table.print_table(output=output, max_rows=None)

fire_event(
ShowNode(
node_name="direct-query",
preview=output.getvalue(),
is_inline=True,
output_format=self.args.output,
unique_id="direct-query",
)
)
17 changes: 17 additions & 0 deletions tests/functional/show/test_show.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,23 @@ def test_inline_pass(self, project):
assert "sample_bool" in log_output


class TestShowInlineDirect(ShowBase):

def test_inline_direct_pass(self, project):
query = f"select * from {project.test_schema}.sample_seed"
(_, log_output) = run_dbt_and_capture(["show", "--inline-direct", query])
assert "Previewing inline node" in log_output
assert "sample_num" in log_output
assert "sample_bool" in log_output

# This is a bit of a hack. Unfortunately, the test teardown code
# expects that dbt loaded an adapter with a macro context the last
# time it was called. The '--inline-direct' parameter used on the
# previous run explicitly disables macros. So now we call 'dbt seed',
# which will load the adapter fully and satisfy the teardown code.
run_dbt(["seed"])


class TestShowInlineFail(ShowBase):
def test_inline_fail(self, project):
with pytest.raises(DbtException, match="Error parsing inline query"):
Expand Down
Loading