1.4.next

core/dbt/cli/main.py

@@ -86,6 +86,7 @@ def cli(ctx, **kwargs):
 @p.show
 @p.state
 @p.store_failures
+@p.submaterialization
 @p.target
 @p.target_path
 @p.threads
@@ -172,6 +173,7 @@ def docs_serve(ctx, **kwargs):
 @p.project_dir
 @p.selector
 @p.state
+@p.submaterialization
 @p.target
 @p.target_path
 @p.threads
@@ -190,6 +192,7 @@ def compile(ctx, **kwargs):
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.submaterialization
 @p.target
 @p.vars
 @p.version_check
@@ -242,6 +245,7 @@ def init(ctx, **kwargs):
 @p.resource_type
 @p.selector
 @p.state
+@p.submaterialization
 @p.target
 @p.vars
 def list(ctx, **kwargs):
@@ -258,6 +262,7 @@ def list(ctx, **kwargs):
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.submaterialization
 @p.target
 @p.target_path
 @p.threads
@@ -284,6 +289,7 @@ def parse(ctx, **kwargs):
 @p.project_dir
 @p.selector
 @p.state
+@p.submaterialization
 @p.target
 @p.target_path
 @p.threads

core/dbt/cli/params.py

@@ -304,6 +304,12 @@
     is_flag=True,
 )
 
+submaterialization = click.option(
+    "--submaterialization",
+    envvar=None,
+    help="Which submaterialization to select and/or run"
+)
+
 target = click.option(
     "--target", "-t", envvar=None, help="Which target to load for the given profile"
 )

core/dbt/config/project.py

@@ -667,6 +667,8 @@ def hashed_name(self):
 
     def get_selector(self, name: str) -> Union[SelectionSpec, bool]:
         if name not in self.selectors:
+            if name == 'arg_selector' and self.arg_selector:
+                return self.arg_selector
             raise DbtRuntimeError(
                 f"Could not find selector named {name}, expected one of {list(self.selectors)}"
             )

core/dbt/config/selectors.py

@@ -141,33 +141,28 @@ def validate_selector_default(selector_file: SelectorFile) -> None:
 # good to combine the two flows into one at some point.
 class SelectorDict:
     @classmethod
-    def parse_dict_definition(cls, definition, selector_dict={}):
+    def parse_dict_definition(cls, definition):
         key = list(definition)[0]
         value = definition[key]
         if isinstance(value, list):
             new_values = []
             for sel_def in value:
-                new_value = cls.parse_from_definition(sel_def, selector_dict=selector_dict)
+                new_value = cls.parse_from_definition(sel_def)
                 new_values.append(new_value)
             value = new_values
         if key == "exclude":
             definition = {key: value}
         elif len(definition) == 1:
             definition = {"method": key, "value": value}
-        elif key == "method" and value == "selector":
-            sel_def = definition.get("value")
-            if sel_def not in selector_dict:
-                raise DbtSelectorsError(f"Existing selector definition for {sel_def} not found.")
-            return selector_dict[definition["value"]]["definition"]
         return definition
 
     @classmethod
-    def parse_a_definition(cls, def_type, definition, selector_dict={}):
+    def parse_a_definition(cls, def_type, definition):
         # this definition must be a list
         new_dict = {def_type: []}
         for sel_def in definition[def_type]:
             if isinstance(sel_def, dict):
-                sel_def = cls.parse_from_definition(sel_def, selector_dict=selector_dict)
+                sel_def = cls.parse_from_definition(sel_def)
                 new_dict[def_type].append(sel_def)
             elif isinstance(sel_def, str):
                 sel_def = SelectionCriteria.dict_from_single_spec(sel_def)
@@ -177,17 +172,17 @@ def parse_a_definition(cls, def_type, definition, selector_dict={}):
         return new_dict
 
     @classmethod
-    def parse_from_definition(cls, definition, selector_dict={}):
+    def parse_from_definition(cls, definition):
         if isinstance(definition, str):
             definition = SelectionCriteria.dict_from_single_spec(definition)
         elif "union" in definition:
-            definition = cls.parse_a_definition("union", definition, selector_dict=selector_dict)
+            definition = cls.parse_a_definition("union", definition)
         elif "intersection" in definition:
             definition = cls.parse_a_definition(
-                "intersection", definition, selector_dict=selector_dict
+                "intersection", definition
             )
         elif isinstance(definition, dict):
-            definition = cls.parse_dict_definition(definition, selector_dict=selector_dict)
+            definition = cls.parse_dict_definition(definition)
         return definition
 
     # This is the normal entrypoint of this code. Give it the
@@ -198,8 +193,7 @@ def parse_from_selectors_list(cls, selectors):
         for selector in selectors:
             sel_name = selector["name"]
             selector_dict[sel_name] = selector
-            definition = cls.parse_from_definition(
-                selector["definition"], selector_dict=deepcopy(selector_dict)
+            selector_dict[sel_name]["definition"] = cls.parse_from_definition(
+                selector["definition"]
             )
-            selector_dict[sel_name]["definition"] = definition
         return selector_dict

core/dbt/constants.py

@@ -1,9 +1,17 @@
+import os
+
 SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
 DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"
 METADATA_ENV_PREFIX = "DBT_ENV_CUSTOM_ENV_"
 
-MAXIMUM_SEED_SIZE = 1 * 1024 * 1024
-MAXIMUM_SEED_SIZE_NAME = "1MB"
+def get_max_seed_size():
+    mx = os.getenv("DBT_MAXIMUM_SEED_SIZE", "1")
+    return int(mx)
+
+
+DEFAULT_MAXIMUM_SEED_SIZE = 1 * 1024 * 1024
+MAXIMUM_SEED_SIZE = get_max_seed_size() * DEFAULT_MAXIMUM_SEED_SIZE
+MAXIMUM_SEED_SIZE_NAME = str(get_max_seed_size()) + "MiB"
 
 PIN_PACKAGE_URL = (
     "https://docs.getdbt.com/docs/package-management#section-specifying-package-versions"

core/dbt/graph/cli.py

@@ -32,28 +32,63 @@ def parse_union(
     # turn ['a b', 'c'] -> ['a', 'b', 'c']
     raw_specs = itertools.chain.from_iterable(r.split(" ") for r in components)
     union_components: List[SelectionSpec] = []
+    exclude_components: List[SelectionSpec] = []
+    in_exclude = False
 
     # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd'))
     for raw_spec in raw_specs:
+        if in_exclude == False and raw_spec == 'EXCEPT':
+            in_exclude = True
+            if len(union_components) == 0:
+                for include in DEFAULT_INCLUDES:
+                    union_components.append(
+                        SelectionCriteria.from_single_spec(include, indirect_selection=False)
+                    )
+            continue
+
         intersection_components: List[SelectionSpec] = [
             SelectionCriteria.from_single_spec(part, indirect_selection=indirect_selection)
             for part in raw_spec.split(INTERSECTION_DELIMITER)
         ]
-        union_components.append(
-            SelectionIntersection(
-                components=intersection_components,
-                expect_exists=expect_exists,
-                raw=raw_spec,
-                indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION),
+        if in_exclude:
+            exclude_components.append(
+                SelectionIntersection(
+                    components=intersection_components,
+                    expect_exists=expect_exists,
+                    raw=raw_spec,
+                    indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION),
+                )
             )
-        )
-    return SelectionUnion(
+        else:
+            union_components.append(
+                SelectionIntersection(
+                    components=intersection_components,
+                    expect_exists=expect_exists,
+                    raw=raw_spec,
+                    indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION),
+                )
+            )
+
+    res = SelectionUnion(
         components=union_components,
         expect_exists=False,
         raw=components,
         indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION),
     )
-
+    if len(exclude_components) == 0:
+        return res
+    else:
+        return SelectionDifference(
+            components=[
+                res,
+                SelectionUnion(
+                    components=exclude_components,
+                    expect_exists=False,
+                    raw=components,
+                    indirect_selection=False
+                )
+            ]
+        )
 
 def parse_union_from_default(
     raw: Optional[List[str]],
@@ -118,9 +153,9 @@ def _get_list_dicts(dct: Dict[str, Any], key: str) -> List[RawDefinition]:
     return result
 
 
-def _parse_exclusions(definition, result={}) -> Optional[SelectionSpec]:
+def _parse_exclusions(definition) -> Optional[SelectionSpec]:
     exclusions = _get_list_dicts(definition, "exclude")
-    parsed_exclusions = [parse_from_definition(excl, result=result) for excl in exclusions]
+    parsed_exclusions = [parse_from_definition(excl) for excl in exclusions]
     if len(parsed_exclusions) == 1:
         return parsed_exclusions[0]
     elif len(parsed_exclusions) > 1:
@@ -130,7 +165,7 @@ def _parse_exclusions(definition, result={}) -> Optional[SelectionSpec]:
 
 
 def _parse_include_exclude_subdefs(
-    definitions: List[RawDefinition], result={}
+    definitions: List[RawDefinition]
 ) -> Tuple[List[SelectionSpec], Optional[SelectionSpec]]:
     include_parts: List[SelectionSpec] = []
     diff_arg: Optional[SelectionSpec] = None
@@ -144,16 +179,16 @@ def _parse_include_exclude_subdefs(
                     f"You cannot provide multiple exclude arguments to the "
                     f"same selector set operator:\n{yaml_sel_cfg}"
                 )
-            diff_arg = _parse_exclusions(definition, result=result)
+            diff_arg = _parse_exclusions(definition)
         else:
-            include_parts.append(parse_from_definition(definition, result=result))
+            include_parts.append(parse_from_definition(definition))
 
     return (include_parts, diff_arg)
 
 
-def parse_union_definition(definition: Dict[str, Any], result={}) -> SelectionSpec:
+def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
     union_def_parts = _get_list_dicts(definition, "union")
-    include, exclude = _parse_include_exclude_subdefs(union_def_parts, result=result)
+    include, exclude = _parse_include_exclude_subdefs(union_def_parts)
 
     union = SelectionUnion(components=include)
 
@@ -164,9 +199,9 @@ def parse_union_definition(definition: Dict[str, Any], result={}) -> SelectionSp
         return SelectionDifference(components=[union, exclude], raw=definition)
 
 
-def parse_intersection_definition(definition: Dict[str, Any], result={}) -> SelectionSpec:
+def parse_intersection_definition(definition: Dict[str, Any]) -> SelectionSpec:
     intersection_def_parts = _get_list_dicts(definition, "intersection")
-    include, exclude = _parse_include_exclude_subdefs(intersection_def_parts, result=result)
+    include, exclude = _parse_include_exclude_subdefs(intersection_def_parts)
     intersection = SelectionIntersection(components=include)
 
     if exclude is None:
@@ -176,7 +211,7 @@ def parse_intersection_definition(definition: Dict[str, Any], result={}) -> Sele
         return SelectionDifference(components=[intersection, exclude], raw=definition)
 
 
-def parse_dict_definition(definition: Dict[str, Any], result={}) -> SelectionSpec:
+def parse_dict_definition(definition: Dict[str, Any]) -> SelectionSpec:
     diff_arg: Optional[SelectionSpec] = None
     if len(definition) == 1:
         key = list(definition)[0]
@@ -189,15 +224,10 @@ def parse_dict_definition(definition: Dict[str, Any], result={}) -> SelectionSpe
             "method": key,
             "value": value,
         }
-    elif definition.get("method") == "selector":
-        sel_def = definition.get("value")
-        if sel_def not in result:
-            raise DbtValidationError(f"Existing selector definition for {sel_def} not found.")
-        return result[definition["value"]]["definition"]
     elif "method" in definition and "value" in definition:
         dct = definition
         if "exclude" in definition:
-            diff_arg = _parse_exclusions(definition, result=result)
+            diff_arg = _parse_exclusions(definition)
             dct = {k: v for k, v in dct.items() if k != "exclude"}
     else:
         raise DbtValidationError(
@@ -215,8 +245,7 @@ def parse_dict_definition(definition: Dict[str, Any], result={}) -> SelectionSpe
 
 def parse_from_definition(
     definition: RawDefinition,
-    rootlevel=False,
-    result: Dict[str, Dict[str, Union[SelectionSpec, bool]]] = {},
+    rootlevel=False
 ) -> SelectionSpec:
 
     if (
@@ -231,13 +260,13 @@ def parse_from_definition(
             f"in a root level selector definition; found {keys}."
         )
     if isinstance(definition, str):
-        return SelectionCriteria.from_single_spec(definition)
+        return parse_union_from_default(raw=[definition], default=[])
     elif "union" in definition:
-        return parse_union_definition(definition, result=result)
+        return parse_union_definition(definition)
     elif "intersection" in definition:
-        return parse_intersection_definition(definition, result=result)
+        return parse_intersection_definition(definition)
     elif isinstance(definition, dict):
-        return parse_dict_definition(definition, result=result)
+        return parse_dict_definition(definition)
     else:
         raise DbtValidationError(
             f"Expected to find union, intersection, str or dict, instead "
@@ -254,7 +283,7 @@ def parse_from_selectors_definition(
         result[selector.name] = {
             "default": selector.default,
             "definition": parse_from_definition(
-                selector.definition, rootlevel=True, result=deepcopy(result)
+                selector.definition, rootlevel=True
             ),
         }
     return result