diff --git a/.changes/unreleased/Dependencies-20240109-120530.yaml b/.changes/unreleased/Dependencies-20240109-120530.yaml new file mode 100644 index 00000000000..cde20b74760 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240109-120530.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: Remove logbook dependency +time: 2024-01-09T12:05:30.176656-06:00 +custom: + Author: emmyoop + PR: "9353" diff --git a/.changes/unreleased/Under the Hood-20240110-105734.yaml b/.changes/unreleased/Under the Hood-20240110-105734.yaml new file mode 100644 index 00000000000..5c8b26e550e --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240110-105734.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add dbt-common as a dependency and remove dbt/common +time: 2024-01-10T10:57:34.054908-05:00 +custom: + Author: michelleark emmyoop + Issue: "9357" diff --git a/core/dbt/README.md b/core/dbt/README.md index 79123a95f47..6b545ceb888 100644 --- a/core/dbt/README.md +++ b/core/dbt/README.md @@ -22,8 +22,6 @@ ### links.py -### logger.py - ### main.py ### node_types.py diff --git a/core/dbt/adapters/base/column.py b/core/dbt/adapters/base/column.py index 7d08780c4f4..72e5576d408 100644 --- a/core/dbt/adapters/base/column.py +++ b/core/dbt/adapters/base/column.py @@ -2,7 +2,7 @@ import re from typing import Dict, ClassVar, Any, Optional -from dbt.common.exceptions import DbtRuntimeError +from dbt_common.exceptions import DbtRuntimeError @dataclass diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index f347876f62e..cdb3f7467e1 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -25,7 +25,7 @@ import agate import dbt.adapters.exceptions -import dbt.common.exceptions.base +import dbt_common.exceptions.base from dbt.adapters.contracts.connection import ( Connection, Identifier, @@ -38,7 +38,7 @@ MacroQueryStringSetter, ) from dbt.adapters.events.logging import AdapterLogger -from dbt.common.events.functions import fire_event +from dbt_common.events.functions import fire_event from dbt.adapters.events.types import ( NewConnection, ConnectionReused, @@ -49,8 +49,8 @@ Rollback, RollbackFailed, ) -from dbt.common.events.contextvars import get_node_info -from dbt.common.utils import cast_to_str +from dbt_common.events.contextvars import get_node_info +from dbt_common.utils import cast_to_str SleepTime = Union[int, float] # As taken by time.sleep. AdapterHandle = Any # Adapter connection handle objects can be any class. @@ -99,7 +99,7 @@ def get_thread_connection(self) -> Connection: def set_thread_connection(self, conn: Connection) -> None: key = self.get_thread_identifier() if key in self.thread_connections: - raise dbt.common.exceptions.DbtInternalError( + raise dbt_common.exceptions.DbtInternalError( "In set_thread_connection, existing connection exists for {}" ) self.thread_connections[key] = conn @@ -139,7 +139,7 @@ def exception_handler(self, sql: str) -> ContextManager: :return: A context manager that handles exceptions raised by the underlying database. """ - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`exception_handler` is not implemented for this adapter!" ) @@ -275,7 +275,7 @@ def retry_connection( @abc.abstractmethod def cancel_open(self) -> Optional[List[str]]: """Cancel all open connections on the adapter. (passable)""" - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`cancel_open` is not implemented for this adapter!" ) @@ -290,7 +290,7 @@ def open(cls, connection: Connection) -> Connection: This should be thread-safe, or hold the lock if necessary. The given connection should not be in either in_use or available. """ - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`open` is not implemented for this adapter!" ) @@ -324,14 +324,14 @@ def cleanup_all(self) -> None: @abc.abstractmethod def begin(self) -> None: """Begin a transaction. (passable)""" - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`begin` is not implemented for this adapter!" ) @abc.abstractmethod def commit(self) -> None: """Commit a transaction. (passable)""" - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`commit` is not implemented for this adapter!" ) @@ -369,7 +369,7 @@ def _close_handle(cls, connection: Connection) -> None: def _rollback(cls, connection: Connection) -> None: """Roll back the given connection.""" if connection.transaction_open is False: - raise dbt.common.exceptions.DbtInternalError( + raise dbt_common.exceptions.DbtInternalError( f"Tried to rollback transaction on connection " f'"{connection.name}", but it does not have one open!' ) @@ -420,7 +420,7 @@ def execute( :return: A tuple of the query status and results (empty if fetch=False). :rtype: Tuple[AdapterResponse, agate.Table] """ - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`execute` is not implemented for this adapter!" ) @@ -432,7 +432,7 @@ def add_select_query(self, sql: str) -> Tuple[Connection, Any]: See https://github.com/dbt-labs/dbt-core/issues/8396 for more information. """ - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`add_select_query` is not implemented for this adapter!" ) @@ -440,6 +440,6 @@ def add_select_query(self, sql: str) -> Tuple[Connection, Any]: def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: """Get the string representation of the data type from the type_code.""" # https://peps.python.org/pep-0249/#type-objects - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`data_type_code_to_name` is not implemented for this adapter!" ) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 6aa6c587316..6849fca7c70 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -23,7 +23,7 @@ from multiprocessing.context import SpawnContext from dbt.adapters.capability import Capability, CapabilityDict -from dbt.common.contracts.constraints import ( +from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, @@ -44,7 +44,7 @@ QuoteConfigTypeError, ) -from dbt.common.exceptions import ( +from dbt_common.exceptions import ( NotImplementedError, DbtInternalError, DbtRuntimeError, @@ -58,15 +58,15 @@ AdapterConfig, MacroContextGeneratorCallable, ) -from dbt.common.clients.agate_helper import ( +from dbt_common.clients.agate_helper import ( empty_table, get_column_value_uncased, merge_tables, table_from_rows, Integer, ) -from dbt.common.clients.jinja import CallableMacroGenerator -from dbt.common.events.functions import fire_event, warn_or_error +from dbt_common.clients.jinja import CallableMacroGenerator +from dbt_common.events.functions import fire_event, warn_or_error from dbt.adapters.events.types import ( CacheMiss, ListRelations, @@ -76,7 +76,7 @@ ConstraintNotSupported, ConstraintNotEnforced, ) -from dbt.common.utils import filter_null_values, executor, cast_to_str, AttrDict +from dbt_common.utils import filter_null_values, executor, cast_to_str, AttrDict from dbt.adapters.contracts.relation import RelationConfig from dbt.adapters.base.connections import ( diff --git a/core/dbt/adapters/base/meta.py b/core/dbt/adapters/base/meta.py index 12f318d0c18..bf887a407ab 100644 --- a/core/dbt/adapters/base/meta.py +++ b/core/dbt/adapters/base/meta.py @@ -1,7 +1,7 @@ import abc from functools import wraps from typing import Callable, Optional, Any, FrozenSet, Dict, Set -from dbt.common.events.functions import warn_or_error +from dbt_common.events.functions import warn_or_error from dbt.adapters.events.types import AdapterDeprecationWarning Decorator = Callable[[Any], Callable] diff --git a/core/dbt/adapters/base/query_headers.py b/core/dbt/adapters/base/query_headers.py index b5f64d6214c..8ab8088aa3c 100644 --- a/core/dbt/adapters/base/query_headers.py +++ b/core/dbt/adapters/base/query_headers.py @@ -3,7 +3,7 @@ from dbt.adapters.clients.jinja import QueryStringGenerator from dbt.adapters.contracts.connection import AdapterRequiredConfig, QueryComment -from dbt.common.exceptions import DbtRuntimeError +from dbt_common.exceptions import DbtRuntimeError class QueryHeaderContextWrapper: diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index d68b2ce3c25..060dcb0a3ef 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -12,10 +12,10 @@ Path, ) from dbt.adapters.exceptions import MultipleDatabasesNotAllowedError, ApproximateMatchError -from dbt.common.utils import filter_null_values, deep_merge +from dbt_common.utils import filter_null_values, deep_merge from dbt.adapters.utils import classproperty -import dbt.common.exceptions +import dbt_common.exceptions Self = TypeVar("Self", bound="BaseRelation") @@ -97,7 +97,7 @@ def matches( if not search: # nothing was passed in - raise dbt.common.exceptions.DbtRuntimeError( + raise dbt_common.exceptions.DbtRuntimeError( "Tried to match relation, but no search path was passed!" ) @@ -360,7 +360,7 @@ class InformationSchema(BaseRelation): def __post_init__(self): if not isinstance(self.information_schema_view, (type(None), str)): - raise dbt.common.exceptions.CompilationError( + raise dbt_common.exceptions.CompilationError( "Got an invalid name: {}".format(self.information_schema_view) ) diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 01a8dc70d21..bc730aa158e 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -7,6 +7,7 @@ _make_ref_key_dict, _ReferenceKey, ) + from dbt.adapters.exceptions.cache import ( NewNameAlreadyInCacheError, ReferencedLinkNotCachedError, @@ -14,9 +15,9 @@ TruncatedModelNameCausedCollisionError, NoneRelationFoundError, ) -from dbt.common.events.functions import fire_event, fire_event_if +from dbt_common.events.functions import fire_event, fire_event_if from dbt.adapters.events.types import CacheAction, CacheDumpGraph -from dbt.common.utils.formatting import lowercase +from dbt_common.utils.formatting import lowercase def dot_separated(key: _ReferenceKey) -> str: diff --git a/core/dbt/adapters/clients/jinja.py b/core/dbt/adapters/clients/jinja.py index ace89c0d1d4..c2b6edbbfa7 100644 --- a/core/dbt/adapters/clients/jinja.py +++ b/core/dbt/adapters/clients/jinja.py @@ -1,5 +1,5 @@ from typing import Dict, Any -from dbt.common.clients.jinja import BaseMacroGenerator, get_environment +from dbt_common.clients.jinja import BaseMacroGenerator, get_environment class QueryStringGenerator(BaseMacroGenerator): diff --git a/core/dbt/adapters/contracts/connection.py b/core/dbt/adapters/contracts/connection.py index 0d6d36aa0cb..14e9f07e71d 100644 --- a/core/dbt/adapters/contracts/connection.py +++ b/core/dbt/adapters/contracts/connection.py @@ -16,21 +16,21 @@ from mashumaro.jsonschema.annotations import Pattern from dbt.adapters.utils import translate_aliases -from dbt.common.exceptions import DbtInternalError -from dbt.common.dataclass_schema import ( +from dbt_common.exceptions import DbtInternalError +from dbt_common.dataclass_schema import ( dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidatedStringMixin, ) -from dbt.common.contracts.util import Replaceable -from dbt.common.utils import md5 +from dbt_common.contracts.util import Replaceable +from dbt_common.utils import md5 -from dbt.common.events.functions import fire_event +from dbt_common.events.functions import fire_event from dbt.adapters.events.types import NewConnectionOpening # TODO: this is a very bad dependency - shared global state -from dbt.common.events.contextvars import get_node_info +from dbt_common.events.contextvars import get_node_info class Identifier(ValidatedStringMixin): diff --git a/core/dbt/adapters/contracts/macros.py b/core/dbt/adapters/contracts/macros.py index 151c9c44dde..6ffe58c1be2 100644 --- a/core/dbt/adapters/contracts/macros.py +++ b/core/dbt/adapters/contracts/macros.py @@ -1,7 +1,7 @@ from typing import Optional from typing_extensions import Protocol -from dbt.common.clients.jinja import MacroProtocol +from dbt_common.clients.jinja import MacroProtocol class MacroResolverProtocol(Protocol): diff --git a/core/dbt/adapters/contracts/relation.py b/core/dbt/adapters/contracts/relation.py index aea294e922c..8fcb8b43edc 100644 --- a/core/dbt/adapters/contracts/relation.py +++ b/core/dbt/adapters/contracts/relation.py @@ -6,11 +6,11 @@ ) from typing_extensions import Protocol -from dbt.common.dataclass_schema import dbtClassMixin, StrEnum +from dbt_common.dataclass_schema import dbtClassMixin, StrEnum -from dbt.common.contracts.util import Replaceable -from dbt.common.exceptions import CompilationError, DataclassNotDictError -from dbt.common.utils import deep_merge +from dbt_common.contracts.util import Replaceable +from dbt_common.exceptions import CompilationError, DataclassNotDictError +from dbt_common.utils import deep_merge class RelationType(StrEnum): diff --git a/core/dbt/adapters/events/base_types.py b/core/dbt/adapters/events/base_types.py index 3717fb44071..23de6ab2b73 100644 --- a/core/dbt/adapters/events/base_types.py +++ b/core/dbt/adapters/events/base_types.py @@ -1,5 +1,5 @@ # Aliasing common Level classes in order to make custom, but not overly-verbose versions that have PROTO_TYPES_MODULE set to the adapter-specific generated types_pb2 module -from dbt.common.events.base_types import ( +from dbt_common.events.base_types import ( BaseEvent, DynamicLevel as CommonDyanicLevel, TestLevel as CommonTestLevel, diff --git a/core/dbt/adapters/events/logging.py b/core/dbt/adapters/events/logging.py index f85b3358520..93f9d15fce1 100644 --- a/core/dbt/adapters/events/logging.py +++ b/core/dbt/adapters/events/logging.py @@ -7,10 +7,10 @@ AdapterEventWarning, AdapterEventError, ) -from dbt.common.events import get_event_manager -from dbt.common.events.contextvars import get_node_info -from dbt.common.events.event_handler import set_package_logging -from dbt.common.events.functions import fire_event +from dbt_common.events import get_event_manager +from dbt_common.events.contextvars import get_node_info +from dbt_common.events.event_handler import set_package_logging +from dbt_common.events.functions import fire_event @dataclass diff --git a/core/dbt/adapters/events/types.py b/core/dbt/adapters/events/types.py index d3aa0a87214..99fe1c1bf36 100644 --- a/core/dbt/adapters/events/types.py +++ b/core/dbt/adapters/events/types.py @@ -1,5 +1,5 @@ from dbt.adapters.events.base_types import WarnLevel, InfoLevel, ErrorLevel, DebugLevel -from dbt.common.ui import line_wrap_message, warning_tag +from dbt_common.ui import line_wrap_message, warning_tag def format_adapter_message(name, base_msg, args) -> str: diff --git a/core/dbt/adapters/exceptions/alias.py b/core/dbt/adapters/exceptions/alias.py index 68a677088d2..60426a4cb5d 100644 --- a/core/dbt/adapters/exceptions/alias.py +++ b/core/dbt/adapters/exceptions/alias.py @@ -1,6 +1,6 @@ from typing import Mapping, Any -from dbt.common.exceptions import DbtValidationError +from dbt_common.exceptions import DbtValidationError class AliasError(DbtValidationError): diff --git a/core/dbt/adapters/exceptions/cache.py b/core/dbt/adapters/exceptions/cache.py index d557be07d6c..9b51f26f9fb 100644 --- a/core/dbt/adapters/exceptions/cache.py +++ b/core/dbt/adapters/exceptions/cache.py @@ -1,7 +1,7 @@ import re from typing import Dict -from dbt.common.exceptions import DbtInternalError +from dbt_common.exceptions import DbtInternalError class CacheInconsistencyError(DbtInternalError): diff --git a/core/dbt/adapters/exceptions/compilation.py b/core/dbt/adapters/exceptions/compilation.py index c87e74b05e7..a66610d54d5 100644 --- a/core/dbt/adapters/exceptions/compilation.py +++ b/core/dbt/adapters/exceptions/compilation.py @@ -1,7 +1,7 @@ from typing import List, Mapping, Any -from dbt.common.exceptions import CompilationError, DbtDatabaseError -from dbt.common.ui import line_wrap_message +from dbt_common.exceptions import CompilationError, DbtDatabaseError +from dbt_common.ui import line_wrap_message class MissingConfigError(CompilationError): diff --git a/core/dbt/adapters/exceptions/connection.py b/core/dbt/adapters/exceptions/connection.py index aac55166407..870794fbe8d 100644 --- a/core/dbt/adapters/exceptions/connection.py +++ b/core/dbt/adapters/exceptions/connection.py @@ -1,6 +1,6 @@ from typing import List -from dbt.common.exceptions import DbtRuntimeError, DbtDatabaseError +from dbt_common.exceptions import DbtRuntimeError, DbtDatabaseError class InvalidConnectionError(DbtRuntimeError): diff --git a/core/dbt/adapters/exceptions/database.py b/core/dbt/adapters/exceptions/database.py index ff177289a03..066016636d6 100644 --- a/core/dbt/adapters/exceptions/database.py +++ b/core/dbt/adapters/exceptions/database.py @@ -1,6 +1,6 @@ from typing import Any -from dbt.common.exceptions import NotImplementedError, CompilationError +from dbt_common.exceptions import NotImplementedError, CompilationError class UnexpectedDbReferenceError(NotImplementedError): diff --git a/core/dbt/adapters/factory.py b/core/dbt/adapters/factory.py index 17023d8bf64..8124c5047d8 100644 --- a/core/dbt/adapters/factory.py +++ b/core/dbt/adapters/factory.py @@ -10,12 +10,12 @@ from dbt.adapters.base.plugin import AdapterPlugin from dbt.adapters.protocol import AdapterConfig, AdapterProtocol, RelationProtocol from dbt.adapters.contracts.connection import AdapterRequiredConfig, Credentials -from dbt.common.events.functions import fire_event +from dbt_common.events.functions import fire_event from dbt.adapters.events.types import AdapterImportError, PluginLoadError, AdapterRegistered -from dbt.common.exceptions import DbtInternalError, DbtRuntimeError +from dbt_common.exceptions import DbtInternalError, DbtRuntimeError from dbt.adapters.include.global_project import PACKAGE_PATH as GLOBAL_PROJECT_PATH from dbt.adapters.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME -from dbt.common.semver import VersionSpecifier +from dbt_common.semver import VersionSpecifier Adapter = AdapterProtocol diff --git a/core/dbt/adapters/protocol.py b/core/dbt/adapters/protocol.py index 2b0399acfc0..f2de7de50ac 100644 --- a/core/dbt/adapters/protocol.py +++ b/core/dbt/adapters/protocol.py @@ -18,8 +18,8 @@ from dbt.adapters.contracts.connection import Connection, AdapterRequiredConfig, AdapterResponse from dbt.adapters.contracts.macros import MacroResolverProtocol from dbt.adapters.contracts.relation import Policy, HasQuoting, RelationConfig -from dbt.common.contracts.config.base import BaseConfig -from dbt.common.clients.jinja import MacroProtocol +from dbt_common.contracts.config.base import BaseConfig +from dbt_common.clients.jinja import MacroProtocol @dataclass diff --git a/core/dbt/adapters/relation_configs/config_base.py b/core/dbt/adapters/relation_configs/config_base.py index 5bfaa8de233..57b791939bf 100644 --- a/core/dbt/adapters/relation_configs/config_base.py +++ b/core/dbt/adapters/relation_configs/config_base.py @@ -2,7 +2,7 @@ from typing import Union, Dict import agate -from dbt.common.utils import filter_null_values +from dbt_common.utils import filter_null_values """ diff --git a/core/dbt/adapters/relation_configs/config_change.py b/core/dbt/adapters/relation_configs/config_change.py index 94f7dd7e9ad..10a6781a24b 100644 --- a/core/dbt/adapters/relation_configs/config_change.py +++ b/core/dbt/adapters/relation_configs/config_change.py @@ -3,7 +3,7 @@ from typing import Hashable from dbt.adapters.relation_configs.config_base import RelationConfigBase -from dbt.common.dataclass_schema import StrEnum +from dbt_common.dataclass_schema import StrEnum class RelationConfigChangeAction(StrEnum): diff --git a/core/dbt/adapters/relation_configs/config_validation.py b/core/dbt/adapters/relation_configs/config_validation.py index ef7b18bb7bb..9442a60a091 100644 --- a/core/dbt/adapters/relation_configs/config_validation.py +++ b/core/dbt/adapters/relation_configs/config_validation.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Set, Optional -from dbt.common.exceptions import DbtRuntimeError +from dbt_common.exceptions import DbtRuntimeError @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/core/dbt/adapters/sql/connections.py b/core/dbt/adapters/sql/connections.py index 86de2dfbddc..324a84ff020 100644 --- a/core/dbt/adapters/sql/connections.py +++ b/core/dbt/adapters/sql/connections.py @@ -5,13 +5,13 @@ import agate from dbt.adapters.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus -import dbt.common.clients.agate_helper -import dbt.common.exceptions +import dbt_common.clients.agate_helper +import dbt_common.exceptions from dbt.adapters.base import BaseConnectionManager from dbt.adapters.contracts.connection import Connection, ConnectionState, AdapterResponse -from dbt.common.events.functions import fire_event -from dbt.common.events.contextvars import get_node_info -from dbt.common.utils import cast_to_str +from dbt_common.events.functions import fire_event +from dbt_common.events.contextvars import get_node_info +from dbt_common.utils import cast_to_str class SQLConnectionManager(BaseConnectionManager): @@ -27,7 +27,7 @@ class SQLConnectionManager(BaseConnectionManager): @abc.abstractmethod def cancel(self, connection: Connection): """Cancel the given connection.""" - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`cancel` is not implemented for this adapter!" ) @@ -95,7 +95,7 @@ def add_query( @abc.abstractmethod def get_response(cls, cursor: Any) -> AdapterResponse: """Get the status of the cursor.""" - raise dbt.common.exceptions.base.NotImplementedError( + raise dbt_common.exceptions.base.NotImplementedError( "`get_response` is not implemented for this adapter!" ) @@ -131,7 +131,7 @@ def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> agate.Tabl rows = cursor.fetchall() data = cls.process_results(column_names, rows) - return dbt.common.clients.agate_helper.table_from_data_flat(data, column_names) + return dbt_common.clients.agate_helper.table_from_data_flat(data, column_names) def execute( self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None @@ -142,7 +142,7 @@ def execute( if fetch: table = self.get_result_from_cursor(cursor, limit) else: - table = dbt.common.clients.agate_helper.empty_table() + table = dbt_common.clients.agate_helper.empty_table() return response, table def add_begin_query(self): @@ -158,7 +158,7 @@ def add_select_query(self, sql: str) -> Tuple[Connection, Any]: def begin(self): connection = self.get_thread_connection() if connection.transaction_open is True: - raise dbt.common.exceptions.DbtInternalError( + raise dbt_common.exceptions.DbtInternalError( 'Tried to begin a new transaction on connection "{}", but ' "it already had one open!".format(connection.name) ) @@ -171,7 +171,7 @@ def begin(self): def commit(self): connection = self.get_thread_connection() if connection.transaction_open is False: - raise dbt.common.exceptions.DbtInternalError( + raise dbt_common.exceptions.DbtInternalError( 'Tried to commit transaction on connection "{}", but ' "it does not have one open!".format(connection.name) ) diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index e8a712774de..e8cf8b45677 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -7,7 +7,7 @@ from dbt.adapters.base import BaseAdapter, available from dbt.adapters.cache import _make_ref_key_dict from dbt.adapters.sql import SQLConnectionManager -from dbt.common.events.functions import fire_event +from dbt_common.events.functions import fire_event from dbt.adapters.base.relation import BaseRelation diff --git a/core/dbt/artifacts/base.py b/core/dbt/artifacts/base.py index 69806915c43..ad94aa64e68 100644 --- a/core/dbt/artifacts/base.py +++ b/core/dbt/artifacts/base.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import ClassVar, Type, TypeVar, Dict, Any, Optional -from dbt.common.clients.system import write_json, read_json +from dbt_common.clients.system import write_json, read_json from dbt.exceptions import ( DbtInternalError, DbtRuntimeError, @@ -10,9 +10,9 @@ ) from dbt.version import __version__ -from dbt.common.events.functions import get_metadata_vars -from dbt.common.invocation import get_invocation_id -from dbt.common.dataclass_schema import dbtClassMixin +from dbt_common.events.functions import get_metadata_vars +from dbt_common.invocation import get_invocation_id +from dbt_common.dataclass_schema import dbtClassMixin from mashumaro.jsonschema import build_json_schema from mashumaro.jsonschema.dialects import DRAFT_2020_12 diff --git a/core/dbt/artifacts/catalog.py b/core/dbt/artifacts/catalog.py index 768992ddd1b..b5307576e14 100644 --- a/core/dbt/artifacts/catalog.py +++ b/core/dbt/artifacts/catalog.py @@ -2,9 +2,9 @@ from dataclasses import dataclass, field from datetime import datetime -from dbt.common.dataclass_schema import dbtClassMixin -from dbt.common.utils.formatting import lowercase -from dbt.common.contracts.util import Replaceable +from dbt_common.dataclass_schema import dbtClassMixin +from dbt_common.utils.formatting import lowercase +from dbt_common.contracts.util import Replaceable from dbt.artifacts.base import ArtifactMixin, BaseArtifactMetadata, schema_version Primitive = Union[bool, str, float, None] diff --git a/core/dbt/artifacts/freshness.py b/core/dbt/artifacts/freshness.py index ccef153c9ac..b77749bd8ac 100644 --- a/core/dbt/artifacts/freshness.py +++ b/core/dbt/artifacts/freshness.py @@ -4,8 +4,8 @@ from dbt.artifacts.results import ExecutionResult, FreshnessStatus, NodeResult, TimingInfo from dbt.artifacts.base import ArtifactMixin, VersionedSchema, schema_version, BaseArtifactMetadata -from dbt.common.dataclass_schema import dbtClassMixin, StrEnum -from dbt.common.exceptions import DbtInternalError +from dbt_common.dataclass_schema import dbtClassMixin, StrEnum +from dbt_common.exceptions import DbtInternalError from dbt.contracts.graph.unparsed import FreshnessThreshold from dbt.contracts.graph.nodes import SourceDefinition diff --git a/core/dbt/artifacts/results.py b/core/dbt/artifacts/results.py index 8bd5bf1fe54..f1cad47fac3 100644 --- a/core/dbt/artifacts/results.py +++ b/core/dbt/artifacts/results.py @@ -1,11 +1,10 @@ from dbt.contracts.graph.nodes import ResultNode -from dbt.common.events.functions import fire_event +from dbt_common.events.functions import fire_event from dbt.events.types import TimingInfoCollected -from dbt.common.events.contextvars import get_node_info -from dbt.common.events.helpers import datetime_to_json_string -from dbt.logger import TimingProcessor -from dbt.common.utils import cast_to_str, cast_to_int -from dbt.common.dataclass_schema import dbtClassMixin, StrEnum +from dbt_common.events.contextvars import get_node_info +from dbt_common.events.helpers import datetime_to_json_string +from dbt_common.utils import cast_to_str, cast_to_int +from dbt_common.dataclass_schema import dbtClassMixin, StrEnum from dataclasses import dataclass from datetime import datetime @@ -45,13 +44,11 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self.timing_info.end() self.callback(self.timing_info) - # Note: when legacy logger is removed, we can remove the following line - with TimingProcessor(self.timing_info): - fire_event( - TimingInfoCollected( - timing_info=self.timing_info.to_msg_dict(), node_info=get_node_info() - ) + fire_event( + TimingInfoCollected( + timing_info=self.timing_info.to_msg_dict(), node_info=get_node_info() ) + ) class RunningStatus(StrEnum): diff --git a/core/dbt/artifacts/run.py b/core/dbt/artifacts/run.py index c3e55248230..d6479bb2009 100644 --- a/core/dbt/artifacts/run.py +++ b/core/dbt/artifacts/run.py @@ -19,7 +19,7 @@ ResultNode, ExecutionResult, ) -from dbt.common.clients.system import write_json +from dbt_common.clients.system import write_json @dataclass diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index ffc73323df8..90ee795afe2 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -13,12 +13,12 @@ from dbt.cli.types import Command as CliCommand from dbt.config.project import read_project_flags from dbt.contracts.project import ProjectFlags -from dbt.common import ui -from dbt.common.events import functions -from dbt.common.exceptions import DbtInternalError -from dbt.common.clients import jinja +from dbt_common import ui +from dbt_common.events import functions +from dbt_common.exceptions import DbtInternalError +from dbt_common.clients import jinja from dbt.deprecations import renamed_env_var -from dbt.common.helper_types import WarnErrorOptions +from dbt_common.helper_types import WarnErrorOptions from dbt.events import ALL_EVENT_NAMES if os.name != "nt": diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 3525de95f96..714d7fd0f58 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -19,7 +19,7 @@ from dbt.contracts.graph.manifest import Manifest from dbt.artifacts.catalog import CatalogArtifact from dbt.artifacts.run import RunExecutionResult -from dbt.common.events.base_types import EventMsg +from dbt_common.events.base_types import EventMsg from dbt.task.build import BuildTask from dbt.task.clean import CleanTask from dbt.task.clone import CloneTask @@ -122,7 +122,6 @@ def global_flags(func): @p.cache_selected_only @p.debug @p.deprecated_print - @p.enable_legacy_logger @p.fail_fast @p.log_cache_events @p.log_file_max_bytes diff --git a/core/dbt/cli/option_types.py b/core/dbt/cli/option_types.py index f673ac279dc..7dc725f6b52 100644 --- a/core/dbt/cli/option_types.py +++ b/core/dbt/cli/option_types.py @@ -3,9 +3,9 @@ from dbt.config.utils import parse_cli_yaml_string from dbt.events import ALL_EVENT_NAMES from dbt.exceptions import ValidationError, OptionNotYamlDictError -from dbt.common.exceptions import DbtValidationError +from dbt_common.exceptions import DbtValidationError -from dbt.common.helper_types import WarnErrorOptions +from dbt_common.helper_types import WarnErrorOptions class YAML(ParamType): diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 3e03376f890..a15453e66a9 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -90,12 +90,6 @@ is_flag=True, ) -enable_legacy_logger = click.option( - "--enable-legacy-logger/--no-enable-legacy-logger", - envvar="DBT_ENABLE_LEGACY_LOGGER", - hidden=True, -) - exclude = click.option( "--exclude", envvar=None, diff --git a/core/dbt/cli/requires.py b/core/dbt/cli/requires.py index c00daddba9c..4667e583fa7 100644 --- a/core/dbt/cli/requires.py +++ b/core/dbt/cli/requires.py @@ -1,5 +1,5 @@ import dbt.tracking -from dbt.common.invocation import reset_invocation_id +from dbt_common.invocation import reset_invocation_id from dbt.version import installed as installed_version from dbt.adapters.factory import adapter_management from dbt.flags import set_flags, get_flag_dict @@ -11,8 +11,8 @@ from dbt.config import RuntimeConfig from dbt.config.runtime import load_project, load_profile, UnsetProfile -from dbt.common.events.base_types import EventLevel -from dbt.common.events.functions import ( +from dbt_common.events.base_types import EventLevel +from dbt_common.events.functions import ( fire_event, LOG_VERSION, ) @@ -22,14 +22,14 @@ MainReportArgs, MainTrackingUserState, ) -from dbt.common.events.helpers import get_json_string_utcnow +from dbt_common.events.helpers import get_json_string_utcnow from dbt.events.types import CommandCompleted, MainEncounteredError, MainStackTrace, ResourceReport -from dbt.common.exceptions import DbtBaseException as DbtException +from dbt_common.exceptions import DbtBaseException as DbtException from dbt.exceptions import DbtProjectError, FailFastError from dbt.parser.manifest import parse_manifest from dbt.profiler import profiler from dbt.tracking import active_user, initialize_from_flags, track_run -from dbt.common.utils import cast_dict_to_dict_of_strings +from dbt_common.utils import cast_dict_to_dict_of_strings from dbt.plugins import set_up_plugin_manager from click import Context diff --git a/core/dbt/cli/types.py b/core/dbt/cli/types.py index f43314c873f..1da078df902 100644 --- a/core/dbt/cli/types.py +++ b/core/dbt/cli/types.py @@ -1,7 +1,7 @@ from enum import Enum from typing import List -from dbt.common.exceptions import DbtInternalError +from dbt_common.exceptions import DbtInternalError class Command(Enum): diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index 2c033ae0c41..4da1c323327 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -1,8 +1,8 @@ import re import os.path -from dbt.common.clients.system import run_cmd, rmdir -from dbt.common.events.functions import fire_event +from dbt_common.clients.system import run_cmd, rmdir +from dbt_common.events.functions import fire_event from dbt.events.types import ( GitSparseCheckoutSubdirectory, GitProgressCheckoutRevision, diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index 1c6e4ae4570..37253e0ea92 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -10,13 +10,13 @@ import jinja2.parser import jinja2.sandbox -from dbt.common.clients.jinja import ( +from dbt_common.clients.jinja import ( render_template, get_template, CallableMacroGenerator, MacroProtocol, ) -from dbt.common.utils import deep_map_render +from dbt_common.utils import deep_map_render from dbt.contracts.graph.nodes import GenericTestNode from dbt.exceptions import ( diff --git a/core/dbt/clients/jinja_static.py b/core/dbt/clients/jinja_static.py index 2112503d9f9..a26710f01de 100644 --- a/core/dbt/clients/jinja_static.py +++ b/core/dbt/clients/jinja_static.py @@ -1,7 +1,7 @@ import jinja2 -from dbt.common.clients.jinja import get_environment +from dbt_common.clients.jinja import get_environment from dbt.exceptions import MacroNamespaceNotStringError -from dbt.common.exceptions.macros import MacroNameNotStringError +from dbt_common.exceptions.macros import MacroNameNotStringError def statically_extract_macro_calls(string, ctx, db_wrapper=None): diff --git a/core/dbt/clients/registry.py b/core/dbt/clients/registry.py index b22375fd8f4..3a160c7f791 100644 --- a/core/dbt/clients/registry.py +++ b/core/dbt/clients/registry.py @@ -1,7 +1,7 @@ import functools from typing import Any, Dict, List import requests -from dbt.common.events.functions import fire_event +from dbt_common.events.functions import fire_event from dbt.events.types import ( RegistryProgressGETRequest, RegistryProgressGETResponse, @@ -13,9 +13,9 @@ RegistryResponseExtraNestedKeys, ) from dbt.utils import memoized -from dbt.common.utils.connection import connection_exception_retry +from dbt_common.utils.connection import connection_exception_retry from dbt import deprecations -from dbt.common import semver +from dbt_common import semver import os if os.getenv("DBT_PACKAGE_HUB_URL"): diff --git a/core/dbt/clients/yaml_helper.py b/core/dbt/clients/yaml_helper.py index bfdf0ff189b..95f65e397ae 100644 --- a/core/dbt/clients/yaml_helper.py +++ b/core/dbt/clients/yaml_helper.py @@ -1,5 +1,5 @@ -import dbt.common.exceptions.base -import dbt.exceptions +import dbt_common.exceptions.base +import dbt_common.exceptions from typing import Any, Dict, Optional import yaml @@ -61,4 +61,4 @@ def load_yaml_text(contents, path=None): else: error = str(e) - raise dbt.common.exceptions.base.DbtValidationError(error) + raise dbt_common.exceptions.base.DbtValidationError(error) diff --git a/core/dbt/common/__init__.py b/core/dbt/common/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/core/dbt/common/clients/__init__.py b/core/dbt/common/clients/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/core/dbt/common/clients/_jinja_blocks.py b/core/dbt/common/clients/_jinja_blocks.py deleted file mode 100644 index 9eb85df13f2..00000000000 --- a/core/dbt/common/clients/_jinja_blocks.py +++ /dev/null @@ -1,360 +0,0 @@ -import re -from collections import namedtuple - -from dbt.common.exceptions import ( - BlockDefinitionNotAtTopError, - DbtInternalError, - MissingCloseTagError, - MissingControlFlowStartTagError, - NestedTagsError, - UnexpectedControlFlowEndTagError, - UnexpectedMacroEOFError, -) - - -def regex(pat): - return re.compile(pat, re.DOTALL | re.MULTILINE) - - -class BlockData: - """raw plaintext data from the top level of the file.""" - - def __init__(self, contents): - self.block_type_name = "__dbt__data" - self.contents = contents - self.full_block = contents - - -class BlockTag: - def __init__(self, block_type_name, block_name, contents=None, full_block=None, **kw): - self.block_type_name = block_type_name - self.block_name = block_name - self.contents = contents - self.full_block = full_block - - def __str__(self): - return "BlockTag({!r}, {!r})".format(self.block_type_name, self.block_name) - - def __repr__(self): - return str(self) - - @property - def end_block_type_name(self): - return "end{}".format(self.block_type_name) - - def end_pat(self): - # we don't want to use string formatting here because jinja uses most - # of the string formatting operators in its syntax... - pattern = "".join( - ( - r"(?P((?:\s*\{\%\-|\{\%)\s*", - self.end_block_type_name, - r"\s*(?:\-\%\}\s*|\%\})))", - ) - ) - return regex(pattern) - - -Tag = namedtuple("Tag", "block_type_name block_name start end") - - -_NAME_PATTERN = r"[A-Za-z_][A-Za-z_0-9]*" - -COMMENT_START_PATTERN = regex(r"(?:(?P(\s*\{\#)))") -COMMENT_END_PATTERN = regex(r"(.*?)(\s*\#\})") -RAW_START_PATTERN = regex(r"(?:\s*\{\%\-|\{\%)\s*(?P(raw))\s*(?:\-\%\}\s*|\%\})") -EXPR_START_PATTERN = regex(r"(?P(\{\{\s*))") -EXPR_END_PATTERN = regex(r"(?P(\s*\}\}))") - -BLOCK_START_PATTERN = regex( - "".join( - ( - r"(?:\s*\{\%\-|\{\%)\s*", - r"(?P({}))".format(_NAME_PATTERN), - # some blocks have a 'block name'. - r"(?:\s+(?P({})))?".format(_NAME_PATTERN), - ) - ) -) - - -RAW_BLOCK_PATTERN = regex( - "".join( - ( - r"(?:\s*\{\%\-|\{\%)\s*raw\s*(?:\-\%\}\s*|\%\})", - r"(?:.*?)", - r"(?:\s*\{\%\-|\{\%)\s*endraw\s*(?:\-\%\}\s*|\%\})", - ) - ) -) - -TAG_CLOSE_PATTERN = regex(r"(?:(?P(\-\%\}\s*|\%\})))") - -# stolen from jinja's lexer. Note that we've consumed all prefix whitespace by -# the time we want to use this. -STRING_PATTERN = regex(r"(?P('([^'\\]*(?:\\.[^'\\]*)*)'|" r'"([^"\\]*(?:\\.[^"\\]*)*)"))') - -QUOTE_START_PATTERN = regex(r"""(?P(['"]))""") - - -class TagIterator: - def __init__(self, data): - self.data = data - self.blocks = [] - self._parenthesis_stack = [] - self.pos = 0 - - def linepos(self, end=None) -> str: - """Given an absolute position in the input data, return a pair of - line number + relative position to the start of the line. - """ - end_val: int = self.pos if end is None else end - data = self.data[:end_val] - # if not found, rfind returns -1, and -1+1=0, which is perfect! - last_line_start = data.rfind("\n") + 1 - # it's easy to forget this, but line numbers are 1-indexed - line_number = data.count("\n") + 1 - return f"{line_number}:{end_val - last_line_start}" - - def advance(self, new_position): - self.pos = new_position - - def rewind(self, amount=1): - self.pos -= amount - - def _search(self, pattern): - return pattern.search(self.data, self.pos) - - def _match(self, pattern): - return pattern.match(self.data, self.pos) - - def _first_match(self, *patterns, **kwargs): - matches = [] - for pattern in patterns: - # default to 'search', but sometimes we want to 'match'. - if kwargs.get("method", "search") == "search": - match = self._search(pattern) - else: - match = self._match(pattern) - if match: - matches.append(match) - if not matches: - return None - # if there are multiple matches, pick the least greedy match - # TODO: do I need to account for m.start(), or is this ok? - return min(matches, key=lambda m: m.end()) - - def _expect_match(self, expected_name, *patterns, **kwargs): - match = self._first_match(*patterns, **kwargs) - if match is None: - raise UnexpectedMacroEOFError(expected_name, self.data[self.pos :]) - return match - - def handle_expr(self, match): - """Handle an expression. At this point we're at a string like: - {{ 1 + 2 }} - ^ right here - - And the match contains "{{ " - - We expect to find a `}}`, but we might find one in a string before - that. Imagine the case of `{{ 2 * "}}" }}`... - - You're not allowed to have blocks or comments inside an expr so it is - pretty straightforward, I hope: only strings can get in the way. - """ - self.advance(match.end()) - while True: - match = self._expect_match("}}", EXPR_END_PATTERN, QUOTE_START_PATTERN) - if match.groupdict().get("expr_end") is not None: - break - else: - # it's a quote. we haven't advanced for this match yet, so - # just slurp up the whole string, no need to rewind. - match = self._expect_match("string", STRING_PATTERN) - self.advance(match.end()) - - self.advance(match.end()) - - def handle_comment(self, match): - self.advance(match.end()) - match = self._expect_match("#}", COMMENT_END_PATTERN) - self.advance(match.end()) - - def _expect_block_close(self): - """Search for the tag close marker. - To the right of the type name, there are a few possiblities: - - a name (handled by the regex's 'block_name') - - any number of: `=`, `(`, `)`, strings, etc (arguments) - - nothing - - followed eventually by a %} - - So the only characters we actually have to worry about in this context - are quote and `%}` - nothing else can hide the %} and be valid jinja. - """ - while True: - end_match = self._expect_match( - 'tag close ("%}")', QUOTE_START_PATTERN, TAG_CLOSE_PATTERN - ) - self.advance(end_match.end()) - if end_match.groupdict().get("tag_close") is not None: - return - # must be a string. Rewind to its start and advance past it. - self.rewind() - string_match = self._expect_match("string", STRING_PATTERN) - self.advance(string_match.end()) - - def handle_raw(self): - # raw blocks are super special, they are a single complete regex - match = self._expect_match("{% raw %}...{% endraw %}", RAW_BLOCK_PATTERN) - self.advance(match.end()) - return match.end() - - def handle_tag(self, match): - """The tag could be one of a few things: - - {% mytag %} - {% mytag x = y %} - {% mytag x = "y" %} - {% mytag x.y() %} - {% mytag foo("a", "b", c="d") %} - - But the key here is that it's always going to be `{% mytag`! - """ - groups = match.groupdict() - # always a value - block_type_name = groups["block_type_name"] - # might be None - block_name = groups.get("block_name") - start_pos = self.pos - if block_type_name == "raw": - match = self._expect_match("{% raw %}...{% endraw %}", RAW_BLOCK_PATTERN) - self.advance(match.end()) - else: - self.advance(match.end()) - self._expect_block_close() - return Tag( - block_type_name=block_type_name, block_name=block_name, start=start_pos, end=self.pos - ) - - def find_tags(self): - while True: - match = self._first_match( - BLOCK_START_PATTERN, COMMENT_START_PATTERN, EXPR_START_PATTERN - ) - if match is None: - break - - self.advance(match.start()) - # start = self.pos - - groups = match.groupdict() - comment_start = groups.get("comment_start") - expr_start = groups.get("expr_start") - block_type_name = groups.get("block_type_name") - - if comment_start is not None: - self.handle_comment(match) - elif expr_start is not None: - self.handle_expr(match) - elif block_type_name is not None: - yield self.handle_tag(match) - else: - raise DbtInternalError( - "Invalid regex match in next_block, expected block start, " - "expr start, or comment start" - ) - - def __iter__(self): - return self.find_tags() - - -_CONTROL_FLOW_TAGS = { - "if": "endif", - "for": "endfor", -} - -_CONTROL_FLOW_END_TAGS = {v: k for k, v in _CONTROL_FLOW_TAGS.items()} - - -class BlockIterator: - def __init__(self, data): - self.tag_parser = TagIterator(data) - self.current = None - self.stack = [] - self.last_position = 0 - - @property - def current_end(self): - if self.current is None: - return 0 - else: - return self.current.end - - @property - def data(self): - return self.tag_parser.data - - def is_current_end(self, tag): - return ( - tag.block_type_name.startswith("end") - and self.current is not None - and tag.block_type_name[3:] == self.current.block_type_name - ) - - def find_blocks(self, allowed_blocks=None, collect_raw_data=True): - """Find all top-level blocks in the data.""" - if allowed_blocks is None: - allowed_blocks = {"snapshot", "macro", "materialization", "docs"} - - for tag in self.tag_parser.find_tags(): - if tag.block_type_name in _CONTROL_FLOW_TAGS: - self.stack.append(tag.block_type_name) - elif tag.block_type_name in _CONTROL_FLOW_END_TAGS: - found = None - if self.stack: - found = self.stack.pop() - else: - expected = _CONTROL_FLOW_END_TAGS[tag.block_type_name] - raise UnexpectedControlFlowEndTagError(tag, expected, self.tag_parser) - expected = _CONTROL_FLOW_TAGS[found] - if expected != tag.block_type_name: - raise MissingControlFlowStartTagError(tag, expected, self.tag_parser) - - if tag.block_type_name in allowed_blocks: - if self.stack: - raise BlockDefinitionNotAtTopError(self.tag_parser, tag.start) - if self.current is not None: - raise NestedTagsError(outer=self.current, inner=tag) - if collect_raw_data: - raw_data = self.data[self.last_position : tag.start] - self.last_position = tag.start - if raw_data: - yield BlockData(raw_data) - self.current = tag - - elif self.is_current_end(tag): - self.last_position = tag.end - assert self.current is not None - yield BlockTag( - block_type_name=self.current.block_type_name, - block_name=self.current.block_name, - contents=self.data[self.current.end : tag.start], - full_block=self.data[self.current.start : tag.end], - ) - self.current = None - - if self.current: - linecount = self.data[: self.current.end].count("\n") + 1 - raise MissingCloseTagError(self.current.block_type_name, linecount) - - if collect_raw_data: - raw_data = self.data[self.last_position :] - if raw_data: - yield BlockData(raw_data) - - def lex_for_blocks(self, allowed_blocks=None, collect_raw_data=True): - return list( - self.find_blocks(allowed_blocks=allowed_blocks, collect_raw_data=collect_raw_data) - ) diff --git a/core/dbt/common/clients/agate_helper.py b/core/dbt/common/clients/agate_helper.py deleted file mode 100644 index d7ac0916cdb..00000000000 --- a/core/dbt/common/clients/agate_helper.py +++ /dev/null @@ -1,251 +0,0 @@ -from codecs import BOM_UTF8 - -import agate -import datetime -import isodate -import json -from typing import Iterable, List, Dict, Union, Optional, Any - -from dbt.common.exceptions import DbtRuntimeError -from dbt.common.utils import ForgivingJSONEncoder - -BOM = BOM_UTF8.decode("utf-8") # '\ufeff' - - -class Integer(agate.data_types.DataType): - def cast(self, d): - # by default agate will cast none as a Number - # but we need to cast it as an Integer to preserve - # the type when merging and unioning tables - if type(d) == int or d is None: - return d - else: - raise agate.exceptions.CastError('Can not parse value "%s" as Integer.' % d) - - def jsonify(self, d): - return d - - -class Number(agate.data_types.Number): - # undo the change in https://github.com/wireservice/agate/pull/733 - # i.e. do not cast True and False to numeric 1 and 0 - def cast(self, d): - if type(d) == bool: - raise agate.exceptions.CastError("Do not cast True to 1 or False to 0.") - else: - return super().cast(d) - - -class ISODateTime(agate.data_types.DateTime): - def cast(self, d): - # this is agate.data_types.DateTime.cast with the "clever" bits removed - # so we only handle ISO8601 stuff - if isinstance(d, datetime.datetime) or d is None: - return d - elif isinstance(d, datetime.date): - return datetime.datetime.combine(d, datetime.time(0, 0, 0)) - elif isinstance(d, str): - d = d.strip() - if d.lower() in self.null_values: - return None - try: - return isodate.parse_datetime(d) - except: # noqa - pass - - raise agate.exceptions.CastError('Can not parse value "%s" as datetime.' % d) - - -def build_type_tester( - text_columns: Iterable[str], string_null_values: Optional[Iterable[str]] = ("null", "") -) -> agate.TypeTester: - - types = [ - Integer(null_values=("null", "")), - Number(null_values=("null", "")), - agate.data_types.Date(null_values=("null", ""), date_format="%Y-%m-%d"), - agate.data_types.DateTime(null_values=("null", ""), datetime_format="%Y-%m-%d %H:%M:%S"), - ISODateTime(null_values=("null", "")), - agate.data_types.Boolean( - true_values=("true",), false_values=("false",), null_values=("null", "") - ), - agate.data_types.Text(null_values=string_null_values), - ] - force = {k: agate.data_types.Text(null_values=string_null_values) for k in text_columns} - return agate.TypeTester(force=force, types=types) - - -DEFAULT_TYPE_TESTER = build_type_tester(()) - - -def table_from_rows( - rows: List[Any], - column_names: Iterable[str], - text_only_columns: Optional[Iterable[str]] = None, -) -> agate.Table: - if text_only_columns is None: - column_types = DEFAULT_TYPE_TESTER - else: - # If text_only_columns are present, prevent coercing empty string or - # literal 'null' strings to a None representation. - column_types = build_type_tester(text_only_columns, string_null_values=()) - - return agate.Table(rows, column_names, column_types=column_types) - - -def table_from_data(data, column_names: Iterable[str]) -> agate.Table: - "Convert a list of dictionaries into an Agate table" - - # The agate table is generated from a list of dicts, so the column order - # from `data` is not preserved. We can use `select` to reorder the columns - # - # If there is no data, create an empty table with the specified columns - - if len(data) == 0: - return agate.Table([], column_names=column_names) - else: - table = agate.Table.from_object(data, column_types=DEFAULT_TYPE_TESTER) - return table.select(column_names) - - -def table_from_data_flat(data, column_names: Iterable[str]) -> agate.Table: - """ - Convert a list of dictionaries into an Agate table. This method does not - coerce string values into more specific types (eg. '005' will not be - coerced to '5'). Additionally, this method does not coerce values to - None (eg. '' or 'null' will retain their string literal representations). - """ - - rows = [] - text_only_columns = set() - for _row in data: - row = [] - for col_name in column_names: - value = _row[col_name] - if isinstance(value, (dict, list, tuple)): - # Represent container types as json strings - value = json.dumps(value, cls=ForgivingJSONEncoder) - text_only_columns.add(col_name) - elif isinstance(value, str): - text_only_columns.add(col_name) - row.append(value) - - rows.append(row) - - return table_from_rows( - rows=rows, column_names=column_names, text_only_columns=text_only_columns - ) - - -def empty_table(): - "Returns an empty Agate table. To be used in place of None" - - return agate.Table(rows=[]) - - -def as_matrix(table): - "Return an agate table as a matrix of data sans columns" - - return [r.values() for r in table.rows.values()] - - -def from_csv(abspath, text_columns, delimiter=","): - type_tester = build_type_tester(text_columns=text_columns) - with open(abspath, encoding="utf-8") as fp: - if fp.read(1) != BOM: - fp.seek(0) - return agate.Table.from_csv(fp, column_types=type_tester, delimiter=delimiter) - - -class _NullMarker: - pass - - -NullableAgateType = Union[agate.data_types.DataType, _NullMarker] - - -class ColumnTypeBuilder(Dict[str, NullableAgateType]): - def __init__(self) -> None: - super().__init__() - - def __setitem__(self, key, value): - if key not in self: - super().__setitem__(key, value) - return - - existing_type = self[key] - if isinstance(existing_type, _NullMarker): - # overwrite - super().__setitem__(key, value) - elif isinstance(value, _NullMarker): - # use the existing value - return - # when one table column is Number while another is Integer, force the column to Number on merge - elif isinstance(value, Integer) and isinstance(existing_type, agate.data_types.Number): - # use the existing value - return - elif isinstance(existing_type, Integer) and isinstance(value, agate.data_types.Number): - # overwrite - super().__setitem__(key, value) - elif not isinstance(value, type(existing_type)): - # actual type mismatch! - raise DbtRuntimeError( - f"Tables contain columns with the same names ({key}), " - f"but different types ({value} vs {existing_type})" - ) - - def finalize(self) -> Dict[str, agate.data_types.DataType]: - result: Dict[str, agate.data_types.DataType] = {} - for key, value in self.items(): - if isinstance(value, _NullMarker): - # agate would make it a Number but we'll make it Integer so that if this column - # gets merged with another Integer column, it won't get forced to a Number - result[key] = Integer() - else: - result[key] = value - return result - - -def _merged_column_types(tables: List[agate.Table]) -> Dict[str, agate.data_types.DataType]: - # this is a lot like agate.Table.merge, but with handling for all-null - # rows being "any type". - new_columns: ColumnTypeBuilder = ColumnTypeBuilder() - for table in tables: - for i in range(len(table.columns)): - column_name: str = table.column_names[i] - column_type: NullableAgateType = table.column_types[i] - # avoid over-sensitive type inference - if all(x is None for x in table.columns[column_name]): - column_type = _NullMarker() - new_columns[column_name] = column_type - - return new_columns.finalize() - - -def merge_tables(tables: List[agate.Table]) -> agate.Table: - """This is similar to agate.Table.merge, but it handles rows of all 'null' - values more gracefully during merges. - """ - new_columns = _merged_column_types(tables) - column_names = tuple(new_columns.keys()) - column_types = tuple(new_columns.values()) - - rows: List[agate.Row] = [] - for table in tables: - if table.column_names == column_names and table.column_types == column_types: - rows.extend(table.rows) - else: - for row in table.rows: - data = [row.get(name, None) for name in column_names] - rows.append(agate.Row(data, column_names)) - # _is_fork to tell agate that we already made things into `Row`s. - return agate.Table(rows, column_names, column_types, _is_fork=True) - - -def get_column_value_uncased(column_name: str, row: agate.Row) -> Any: - """Get the value of a column in this row, ignoring the casing of the column name.""" - for key, value in row.items(): - if key.casefold() == column_name.casefold(): - return value - - raise KeyError diff --git a/core/dbt/common/clients/jinja.py b/core/dbt/common/clients/jinja.py deleted file mode 100644 index e01ad57093c..00000000000 --- a/core/dbt/common/clients/jinja.py +++ /dev/null @@ -1,505 +0,0 @@ -import codecs -import linecache -import os -import tempfile -from ast import literal_eval -from contextlib import contextmanager -from itertools import chain, islice -from typing import List, Union, Set, Optional, Dict, Any, Iterator, Type, Callable -from typing_extensions import Protocol - -import jinja2 -import jinja2.ext -import jinja2.nativetypes # type: ignore -import jinja2.nodes -import jinja2.parser -import jinja2.sandbox - -from dbt.common.utils import ( - get_dbt_macro_name, - get_docs_macro_name, - get_materialization_macro_name, - get_test_macro_name, -) -from dbt.common.clients._jinja_blocks import BlockIterator, BlockData, BlockTag - -from dbt.common.exceptions import ( - CompilationError, - DbtInternalError, - CaughtMacroErrorWithNodeError, - MaterializationArgError, - JinjaRenderingError, - UndefinedCompilationError, -) -from dbt.common.exceptions.macros import MacroReturn, UndefinedMacroError, CaughtMacroError - - -SUPPORTED_LANG_ARG = jinja2.nodes.Name("supported_languages", "param") - -# Global which can be set by dependents of dbt-common (e.g. core via flag parsing) -MACRO_DEBUGGING = False - - -def _linecache_inject(source, write): - if write: - # this is the only reliable way to accomplish this. Obviously, it's - # really darn noisy and will fill your temporary directory - tmp_file = tempfile.NamedTemporaryFile( - prefix="dbt-macro-compiled-", - suffix=".py", - delete=False, - mode="w+", - encoding="utf-8", - ) - tmp_file.write(source) - filename = tmp_file.name - else: - # `codecs.encode` actually takes a `bytes` as the first argument if - # the second argument is 'hex' - mypy does not know this. - rnd = codecs.encode(os.urandom(12), "hex") # type: ignore - filename = rnd.decode("ascii") - - # put ourselves in the cache - cache_entry = (len(source), None, [line + "\n" for line in source.splitlines()], filename) - # linecache does in fact have an attribute `cache`, thanks - linecache.cache[filename] = cache_entry # type: ignore - return filename - - -class MacroFuzzParser(jinja2.parser.Parser): - def parse_macro(self): - node = jinja2.nodes.Macro(lineno=next(self.stream).lineno) - - # modified to fuzz macros defined in the same file. this way - # dbt can understand the stack of macros being called. - # - @cmcarthur - node.name = get_dbt_macro_name(self.parse_assign_target(name_only=True).name) - - self.parse_signature(node) - node.body = self.parse_statements(("name:endmacro",), drop_needle=True) - return node - - -class MacroFuzzEnvironment(jinja2.sandbox.SandboxedEnvironment): - def _parse(self, source, name, filename): - return MacroFuzzParser(self, source, name, filename).parse() - - def _compile(self, source, filename): - """Override jinja's compilation to stash the rendered source inside - the python linecache for debugging when the appropriate environment - variable is set. - - If the value is 'write', also write the files to disk. - WARNING: This can write a ton of data if you aren't careful. - """ - if filename == "