From fe91793245aa38d121e0c89ebe554360d259e3df Mon Sep 17 00:00:00 2001 From: z3z1ma Date: Mon, 30 Dec 2024 00:31:01 -0700 Subject: [PATCH] chore: remove vendored mod --- src/dbt_osmosis/core/osmosis.py | 6 +- src/dbt_osmosis/vendored/__init__.py | 0 .../vendored/dbt_core_interface/VENDORED.md | 3 - .../vendored/dbt_core_interface/__init__.py | 3 - .../vendored/dbt_core_interface/project.py | 6510 ----------------- 5 files changed, 2 insertions(+), 6520 deletions(-) delete mode 100644 src/dbt_osmosis/vendored/__init__.py delete mode 100644 src/dbt_osmosis/vendored/dbt_core_interface/VENDORED.md delete mode 100644 src/dbt_osmosis/vendored/dbt_core_interface/__init__.py delete mode 100644 src/dbt_osmosis/vendored/dbt_core_interface/project.py diff --git a/src/dbt_osmosis/core/osmosis.py b/src/dbt_osmosis/core/osmosis.py index 9e4cc4e2..66af45de 100644 --- a/src/dbt_osmosis/core/osmosis.py +++ b/src/dbt_osmosis/core/osmosis.py @@ -483,12 +483,10 @@ def f(node: ResultNode) -> bool: yield uid, dbt_node -def normalize_column_name(column: str, credentials_type: str, to_lower: bool = False) -> str: +def normalize_column_name(column: str, credentials_type: str) -> str: """Apply case normalization to a column name based on the credentials type.""" if credentials_type == "snowflake" and column.startswith('"') and column.endswith('"'): return column - if to_lower: - return column.lower() if credentials_type == "snowflake": return column.upper() return column @@ -503,7 +501,7 @@ def _maybe_use_precise_dtype(col: t.Any, settings: YamlRefactorSettings) -> str: return col.dtype -def _get_catalog_key_for_node(node: ResultNode) -> CatalogKey: +def get_catalog_key_for_node(node: ResultNode) -> CatalogKey: """Make an appropriate catalog key for a dbt node.""" if node.resource_type == NodeType.Source: return CatalogKey(node.database, node.schema, node.identifier or node.name) diff --git a/src/dbt_osmosis/vendored/__init__.py b/src/dbt_osmosis/vendored/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/dbt_osmosis/vendored/dbt_core_interface/VENDORED.md b/src/dbt_osmosis/vendored/dbt_core_interface/VENDORED.md deleted file mode 100644 index b8d799d6..00000000 --- a/src/dbt_osmosis/vendored/dbt_core_interface/VENDORED.md +++ /dev/null @@ -1,3 +0,0 @@ -# Source - -`dbt-core-interface` is vendored from https://github.com/z3z1ma/dbt-core-interface diff --git a/src/dbt_osmosis/vendored/dbt_core_interface/__init__.py b/src/dbt_osmosis/vendored/dbt_core_interface/__init__.py deleted file mode 100644 index 7986e7ce..00000000 --- a/src/dbt_osmosis/vendored/dbt_core_interface/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Dbt Core Interface.""" - -from .project import * # noqa: F401, F403 diff --git a/src/dbt_osmosis/vendored/dbt_core_interface/project.py b/src/dbt_osmosis/vendored/dbt_core_interface/project.py deleted file mode 100644 index 9e8e674b..00000000 --- a/src/dbt_osmosis/vendored/dbt_core_interface/project.py +++ /dev/null @@ -1,6510 +0,0 @@ -#!/usr/bin/env python -"""The interface for interacting with dbt-core. - -We package the interface as a single python module that can be imported -and used in other python projects. You do not need to include dbt-core-interface -as a dependency in your project if you do not want to. You can simply copy -the dbt_core_interface folder into your project and import it from there. -""" - -# region dbt-core-interface imports & monkey patches - -if 1: # this stops ruff from complaining about the import order - import dbt.adapters.factory - - # See ... for more info on this monkey patch - dbt.adapters.factory.get_adapter = lambda config: config.adapter # type: ignore - -import _thread as thread -import base64 -import calendar -import cgi -import configparser -import decimal -import email.utils -import functools -import hashlib -import hmac -import http.client as httplib -import itertools -import json -import logging -import mimetypes -import os -import pickle -import re -import sys -import tempfile -import threading -import time -import uuid -import warnings -import weakref -from collections import OrderedDict, UserDict -from collections.abc import MutableMapping as DictMixin -from contextlib import contextmanager, redirect_stdout -from copy import copy -from dataclasses import asdict, dataclass, field -from datetime import date as datedate -from datetime import datetime, timedelta -from enum import Enum -from functools import lru_cache, wraps -from http.cookies import CookieError, Morsel, SimpleCookie -from inspect import getfullargspec -from io import BytesIO -from json import dumps as json_dumps -from json import loads as json_lds -from pathlib import Path -from tempfile import NamedTemporaryFile -from traceback import format_exc, print_exc -from types import FunctionType -from types import ModuleType as new_module # noqa -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generator, - List, - Optional, - Tuple, - Type, - TypeVar, - Union, - cast, -) -from unicodedata import normalize -from urllib.parse import SplitResult as UrlSplitResult -from urllib.parse import quote as urlquote -from urllib.parse import unquote as urlunquote -from urllib.parse import urlencode, urljoin - -import dbt.version -import yaml - -# We maintain the smallest possible surface area of dbt imports -from dbt.adapters.factory import get_adapter_class_by_name - -try: - # dbt >= 1.8 - from dbt_common.clients.system import get_env, make_directory - from dbt_common.context import set_invocation_context -except ImportError: - # dbt < 1.8 - from dbt.clients.system import make_directory - -from dbt.config.runtime import RuntimeConfig -from dbt.contracts.graph.nodes import ColumnInfo, ManifestNode -from dbt.flags import set_from_args -from dbt.node_types import NodeType -from dbt.parser.manifest import PARTIAL_PARSE_FILE_NAME, ManifestLoader, process_node -from dbt.parser.sql import SqlBlockParser, SqlMacroParser -from dbt.task.sql import SqlCompileRunner -from dbt.tracking import disable_tracking - -if TYPE_CHECKING: - # These imports are only used for type checking - from agate import Table # type: ignore # No stubs for agate - from dbt.adapters.base import BaseAdapter, BaseRelation # type: ignore - - try: - # dbt >= 1.8 - from dbt.adapters.contracts.connection import AdapterResponse - from dbt_common.semver import VersionSpecifier - - except ImportError: - # dbt < 1.8 - from dbt.contracts.connection import AdapterResponse - from dbt.semver import VersionSpecifier - from dbt.contracts.results import ExecutionResult, RunExecutionResult - from dbt.task.runnable import ManifestTask - -# dbt-core-interface is designed for non-standard use. There is no -# reason to track usage of this package. -disable_tracking() - -urlunquote = functools.partial(urlunquote, encoding="latin1") - -RAW_CODE = "raw_code" -COMPILED_CODE = "compiled_code" - - -def default_project_dir() -> Path: - if "DBT_PROJECT_DIR" in os.environ: - return Path(os.environ["DBT_PROJECT_DIR"]).resolve() - paths = list(Path.cwd().parents) - paths.insert(0, Path.cwd()) - return next((x for x in paths if (x / "dbt_project.yml").exists()), Path.cwd()) - - -def default_profiles_dir() -> Path: - if "DBT_PROFILES_DIR" in os.environ: - return Path(os.environ["DBT_PROFILES_DIR"]).resolve() - return Path.cwd() if (Path.cwd() / "profiles.yml").exists() else Path.home() / ".dbt" - - -DEFAULT_PROFILES_DIR = str(default_profiles_dir()) -DEFAULT_PROJECT_DIR = str(default_project_dir()) - - -def write_manifest_for_partial_parse(self: ManifestLoader): - """Monkey patch for dbt manifest loader.""" - path = os.path.join( - self.root_project.project_root, - self.root_project.target_path, - PARTIAL_PARSE_FILE_NAME, - ) - try: - if self.manifest.metadata.dbt_version != dbt.version.__version__: - self.manifest.metadata.dbt_version = dbt.version.__version__ - manifest_msgpack = self.manifest.to_msgpack() - make_directory(os.path.dirname(path)) - with open(path, "wb") as fp: - fp.write(manifest_msgpack) - except Exception: - raise - - -__all__ = [ - "DbtProject", - "DbtProjectContainer", - "DbtAdapterExecutionResult", - "DbtAdapterCompilationResult", - "DbtManifestProxy", - "DbtConfiguration", - "DEFAULT_PROFILES_DIR", - "DEFAULT_PROJECT_DIR", - "ServerRunResult", - "ServerCompileResult", - "ServerResetResult", - "ServerRegisterResult", - "ServerUnregisterResult", - "ServerErrorCode", - "ServerError", - "ServerErrorContainer", - "ServerPlugin", - "run_server", - "default_project_dir", - "default_profiles_dir", - "ColumnInfo", - "ManifestNode", -] - -T = TypeVar("T") -JINJA_CONTROL_SEQUENCES = ["{{", "}}", "{%", "%}", "{#", "#}"] - -LOGGER = logging.getLogger(__name__) - -__version__ = dbt.version.__version__ - -# endregion - -# region dbt-core-interface core - - -class DbtCommand(str, Enum): - """The dbt commands we support.""" - - RUN = "run" - BUILD = "build" - TEST = "test" - SEED = "seed" - RUN_OPERATION = "run-operation" - LIST = "list" - SNAPSHOT = "snapshot" - - -@dataclass -class DbtConfiguration: - """The configuration for dbt-core.""" - - project_dir: str = DEFAULT_PROJECT_DIR - profiles_dir: str = DEFAULT_PROFILES_DIR - profile: Optional[str] = None - target: Optional[str] = None - threads: int = 1 - single_threaded: bool = True - _vars: str = "{}" - # Mutes unwanted dbt output - quiet: bool = True - # We need single threaded, simple, jinja parsing -- no rust/pickling - use_experimental_parser: bool = False - static_parser: bool = False - partial_parse: bool = False - # A required attribute for dbt, not used by our interface - dependencies: List[str] = field(default_factory=list) - which: str = None - DEBUG: bool = False - REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES: bool = False - - def __post_init__(self) -> None: - """Post init hook to set single_threaded and remove target if not provided.""" - if self.target is None: - del self.target - self.single_threaded = self.threads == 1 - - @property - def vars(self) -> str: - """Access the vars attribute as a string.""" - return self._vars - - @vars.setter - def vars(self, v: Union[str, Dict[str, Any]]) -> None: - """Set the vars attribute as a string or dict. - - If dict then it will be converted to a string which is what dbt expects. - """ - if isinstance(v, str): - v = yaml.safe_load(v) - self._vars = v - - -class DbtManifestProxy(UserDict): # type: ignore - """Proxy for manifest dictionary object. - - If we need mutation then we should create a copy of the dict or interface with the dbt-core manifest object instead. - """ - - def _readonly(self, *args: Any, **kwargs: Any) -> None: - raise RuntimeError("Cannot modify DbtManifestProxy") - - __setitem__ = _readonly - __delitem__ = _readonly - pop = _readonly # type: ignore - popitem = _readonly # type: ignore - clear = _readonly - update = _readonly # type: ignore - setdefault = _readonly # type: ignore - - -@dataclass -class DbtAdapterExecutionResult: - """Interface for execution results. - - This keeps us 1 layer removed from dbt interfaces which may change. - """ - - adapter_response: "AdapterResponse" - table: "Table" - raw_code: str - compiled_code: str - - -@dataclass -class DbtAdapterCompilationResult: - """Interface for compilation results. - - This keeps us 1 layer removed from dbt interfaces which may change. - """ - - raw_code: str - compiled_code: str - node: "ManifestNode" - injected_code: Optional[str] = None - - -class DbtTaskConfiguration: - """A container for task configuration with sane defaults. - - Users should enforce an interface for their tasks via a factory method that returns an instance of this class. - """ - - def __init__(self, profile: str, target: str, **kwargs: Any) -> None: - """Initialize the task configuration.""" - self.profile: str = profile - self.target: str = target - self.kwargs: Dict[str, Any] = kwargs or {} - self.threads: int = kwargs.get("threads", 1) - self.single_threaded: bool = kwargs.get("single_threaded", self.threads == 1) - self.state_id: Optional[str] = kwargs.get("state_id") - self.version_check: bool = kwargs.get("version_check", False) - self.resource_types: Optional[List[str]] = kwargs.get("resource_types") - self.models: Union[None, str, List[str]] = kwargs.get("models") - self.select: Union[None, str, List[str]] = kwargs.get("select") - self.exclude: Union[None, str, List[str]] = kwargs.get("exclude") - self.selector_name: Optional[str] = kwargs.get("selector_name") - self.state: Optional[str] = kwargs.get("state") - self.defer: bool = kwargs.get("defer", False) - self.fail_fast: bool = kwargs.get("fail_fast", False) - self.full_refresh: bool = kwargs.get("full_refresh", False) - self.store_failures: bool = kwargs.get("store_failures", False) - self.indirect_selection: bool = kwargs.get("indirect_selection", False) - self.data: bool = kwargs.get("data", False) - self.schema: bool = kwargs.get("schema", False) - self.show: bool = kwargs.get("show", False) - self.output: str = kwargs.get("output", "name") - self.output_keys: Union[None, str, List[str]] = kwargs.get("output_keys") - self.macro: Optional[str] = kwargs.get("macro") - self.args: str = kwargs.get("args", "{}") - self.quiet: bool = kwargs.get("quiet", True) - self.defer_state: Path = kwargs.get("defer_state", None) - self.exclude_resource_types: List[str] = kwargs.get("exclude_resource_types", None) - self.selector: str = kwargs.get("selector", None) - self.write_json: bool = kwargs.get("write_json", False) - self.include_saved_query: bool = kwargs.get("include_saved_query", False) - - @classmethod - def from_runtime_config(cls, config: RuntimeConfig, **kwargs: Any) -> "DbtTaskConfiguration": - """Create a task configuration container from a DbtProject's runtime config. - - This is a good example of where static typing is not necessary. Developers can just - pass in whatever they want and it will be passed through to the task configuration container. - Users of the library are free to pass in any mapping derived from their own implementation for - their own custom task. - """ - threads = kwargs.pop("threads", config.threads) - kwargs.pop("single_threaded", None) # This is a derived property - return cls( - config.profile_name, - config.target_name, - threads=threads, - single_threaded=threads == 1, - **kwargs, - ) - - -class DbtProject: - """Container for a dbt project. - - The dbt attribute is the primary interface for dbt-core. The adapter attribute is the primary interface for the dbt adapter. - """ - - ADAPTER_TTL = 3600 - - def __init__( - self, - target: Optional[str] = None, - profiles_dir: str = DEFAULT_PROFILES_DIR, - project_dir: str = DEFAULT_PROJECT_DIR, - threads: int = 1, - vars: Optional[str] = None, - profile: Optional[str] = None, - ) -> None: - """Initialize the DbtProject.""" - self.base_config = DbtConfiguration( - threads=threads, - target=target, - profiles_dir=profiles_dir or DEFAULT_PROFILES_DIR, - project_dir=project_dir or DEFAULT_PROJECT_DIR, - profile=profile, - ) - if hasattr(sys.modules[__name__], "set_invocation_context"): - set_invocation_context(get_env()) - if vars is None: - vars = "{}" - self.base_config.vars = vars - - # Mutexes - self.adapter_mutex = threading.Lock() - self.parsing_mutex = threading.Lock() - self.manifest_mutation_mutex = threading.Lock() - - # First time initialization - self.parse_project(init=True) - - # Utilities - self._sql_parser: Optional[SqlBlockParser] = None - self._macro_parser: Optional[SqlMacroParser] = None - - @classmethod - def from_config(cls, config: DbtConfiguration) -> "DbtProject": - """Instatiate the DbtProject directly from a DbtConfiguration instance.""" - return cls( - target=config.target, - profiles_dir=config.profiles_dir, - project_dir=config.project_dir, - threads=config.threads, - ) - - def get_adapter_cls(self) -> Type["BaseAdapter"]: - """Get the adapter class associated with the dbt profile.""" - return get_adapter_class_by_name(self.config.credentials.type) - - def initialize_adapter(self) -> None: - """Initialize a dbt adapter.""" - if hasattr(self, "_adapter"): - # Clean up any existing connections, err on the side of runtime - # resiliency, don't let this fail. Maybe there is a world where - # it really matters, but I don't think so. Someone can make the case. - try: - self._adapter.connections.cleanup_all() - except Exception as e: - LOGGER.debug(f"Failed to cleanup adapter connections: {e}") - # The adapter.setter verifies connection, resets TTL, and updates adapter ref on config - # this is thread safe by virtue of the adapter_mutex on the adapter.setter - try: - self.adapter = self.get_adapter_cls()(self.config) - except TypeError: - from dbt.mp_context import get_mp_context - - self.adapter = self.get_adapter_cls()(self.config, get_mp_context()) - - try: - from dbt.context.providers import generate_runtime_macro_context - - self.adapter.set_macro_context_generator(generate_runtime_macro_context) - except Exception: - pass - - @property - def adapter(self) -> "BaseAdapter": - """dbt-core adapter with TTL and automatic reinstantiation. - - This supports long running processes that may have their connection to the database terminated by - the database server. It is transparent to the user. - """ - if time.time() - self._adapter_created_at > self.ADAPTER_TTL: - self.initialize_adapter() - return self._adapter - - @adapter.setter - def adapter(self, adapter: "BaseAdapter") -> None: - """Verify connection and reset TTL on adapter set, update adapter prop ref on config.""" - # Ensure safe concurrent access to the adapter - # Currently we choose to drop attempted mutations while an existing mutation is in progress - # This is a tradeoff between safety and performance, we could also choose to block - if self.adapter_mutex.acquire(blocking=False): - try: - self._adapter = adapter - self._adapter.connections.set_connection_name() - self._adapter_created_at = time.time() - self.config.adapter = self.adapter # type: ignore - finally: - self.adapter_mutex.release() - - def parse_project(self, init: bool = False) -> None: - """Parse project on disk. - - Uses the config from `DbtConfiguration` in args attribute, verifies connection to adapters database, - mutates config, adapter, and dbt attributes. Thread-safe. From an efficiency perspective, this is a - relatively expensive operation, so we want to avoid doing it more than necessary. - """ - # Threads will wait here if another thread is parsing the project - # however, it probably makes sense to not parse the project once the waiter - # has acquired the lock, TODO: Lets implement a debounce-like buffer here - with self.parsing_mutex: - if init: - set_from_args(self.base_config, self.base_config) - # We can think of `RuntimeConfig` as a dbt-core "context" object - # where a `Project` meets a `Profile` and is a superset of them both - self.config = RuntimeConfig.from_args(self.base_config) - self.initialize_adapter() - - _project_parser = ManifestLoader( - self.config, - self.config.load_dependencies(), - self.adapter.connections.set_query_header, - ) - - self.manifest = _project_parser.load() - self.manifest.build_flat_graph() - _project_parser.save_macros_to_adapter(self.adapter) - - self._sql_parser = None - self._macro_parser = None - - def safe_parse_project(self, reinit: bool = False) -> None: - """Safe version of parse_project that will not mutate the config if parsing fails.""" - if reinit: - self.clear_internal_caches() - _config_pointer = copy(self.config) - try: - self.parse_project(init=reinit) - except Exception as parse_error: - self.config = _config_pointer - raise parse_error - self.write_manifest_artifact() - - def _verify_connection(self, adapter: "BaseAdapter") -> "BaseAdapter": - """Verification for adapter + profile. - - Used as a passthrough, this also seeds the master connection. - """ - try: - adapter.connections.set_connection_name() - adapter.debug_query() - except Exception as query_exc: - raise RuntimeError("Could not connect to Database") from query_exc - else: - return adapter - - def adapter_probe(self) -> bool: - """Check adapter connection, useful for long running procsesses.""" - if not hasattr(self, "adapter") or self.adapter is None: - return False - try: - with self.adapter.connection_named("osmosis-heartbeat"): - self.adapter.debug_query() - except Exception: - # TODO: Should we preemptively reinitialize the adapter here? or leave it to userland to handle? - return False - return True - - def fn_threaded_conn(self, fn: Callable[..., T], *args: Any, **kwargs: Any) -> Callable[..., T]: - """For jobs which are intended to be submitted to a thread pool.""" - - @wraps(fn) - def _with_conn() -> T: - self.adapter.connections.set_connection_name() - return fn(*args, **kwargs) - - return _with_conn - - def generate_runtime_model_context(self, node: "ManifestNode") -> Dict[str, Any]: - """Wrap dbt context provider.""" - # Purposefully deferred due to its many dependencies - from dbt.context.providers import generate_runtime_model_context - - return generate_runtime_model_context(node, self.config, self.manifest) - - @property - def project_name(self) -> str: - """dbt project name.""" - return self.config.project_name - - @property - def project_root(self) -> str: - """dbt project root.""" - return self.config.project_root - - @property - def manifest_dict(self) -> DbtManifestProxy: - """dbt manifest dict.""" - return DbtManifestProxy(self.manifest.flat_graph) - - def write_manifest_artifact(self) -> None: - """Write a manifest.json to disk. - - Because our project is in memory, this is useful for integrating with other tools that - expect a manifest.json to be present in the target directory. - """ - artifact_path = os.path.join( - self.config.project_root, self.config.target_path, "manifest.json" - ) - self.manifest.write(artifact_path) - - def clear_internal_caches(self) -> None: - """Clear least recently used caches and reinstantiable container objects.""" - self.compile_code.cache_clear() - self.unsafe_compile_code.cache_clear() - - def get_ref_node(self, target_model_name: str) -> "ManifestNode": - """Get a `ManifestNode` from a dbt project model name. - - This is the same as one would in a {{ ref(...) }} macro call. - """ - return cast( - "ManifestNode", - self.manifest.resolve_ref( - target_model_name=target_model_name, - target_model_package=None, - current_project=self.config.project_name, - node_package=self.config.project_name, - ), - ) - - def get_source_node(self, target_source_name: str, target_table_name: str) -> "ManifestNode": - """Get a `ManifestNode` from a dbt project source name and table name. - - This is the same as one would in a {{ source(...) }} macro call. - """ - return cast( - "ManifestNode", - self.manifest.resolve_source( - target_source_name=target_source_name, - target_table_name=target_table_name, - current_project=self.config.project_name, - node_package=self.config.project_name, - ), - ) - - def get_node_by_path(self, path: str) -> Optional["ManifestNode"]: - """Find an existing node given relative file path. - - TODO: We can include Path obj support and make this more robust. - """ - for node in self.manifest.nodes.values(): - if node.original_file_path == path: - return node - return None - - @contextmanager - def generate_server_node( - self, sql: str, node_name: str = "anonymous_node" - ) -> Generator["ManifestNode", None, None]: - """Get a transient node for SQL execution against adapter. - - This is a context manager that will clear the node after execution and leverages a mutex during manifest mutation. - """ - with self.manifest_mutation_mutex: - self._clear_node(node_name) - sql_node = self.sql_parser.parse_remote(sql, node_name) - process_node(self.config, self.manifest, sql_node) - yield sql_node - self._clear_node(node_name) - - def unsafe_generate_server_node( - self, sql: str, node_name: str = "anonymous_node" - ) -> "ManifestNode": - """Get a transient node for SQL execution against adapter. - - This is faster than `generate_server_node` but does not clear the node after execution. - That is left to the caller. It is also not thread safe in and of itself and requires the caller to - manage jitter or mutexes. - """ - self._clear_node(node_name) - sql_node = self.sql_parser.parse_remote(sql, node_name) - process_node(self.config, self.manifest, sql_node) - return sql_node - - def inject_macro(self, macro_contents: str) -> None: - """Inject a macro into the project. - - This is useful for testing macros in isolation. It offers unique ways to integrate with dbt. - """ - macro_overrides = {} - for node in self.macro_parser.parse_remote(macro_contents): - macro_overrides[node.unique_id] = node - self.manifest.macros.update(macro_overrides) - - def get_macro_function(self, macro_name: str) -> Callable[..., Any]: - """Get macro as a function which behaves like a Python function.""" - - def _macro_fn(**kwargs: Any) -> Any: - return self.adapter.execute_macro(macro_name, self.manifest, kwargs=kwargs) - - return _macro_fn - - def execute_macro(self, macro: str, **kwargs: Any) -> Any: - """Wrap adapter execute_macro. Execute a macro like a python function.""" - return self.get_macro_function(macro)(**kwargs) - - def adapter_execute( - self, sql: str, auto_begin: bool = False, fetch: bool = False - ) -> Tuple["AdapterResponse", "Table"]: - """Wrap adapter.execute. Execute SQL against database. - - This is more on-the-rails than `execute_code` which intelligently handles jinja compilation provides a proxy result. - """ - return cast( - Tuple["AdapterResponse", "Table"], - self.adapter.execute(sql, auto_begin, fetch), - ) - - def execute_code(self, raw_code: str) -> DbtAdapterExecutionResult: - """Execute dbt SQL statement against database. - - This is a proxy for `adapter_execute` and the the recommended method for executing SQL against the database. - """ - # If no jinja chars then these are synonymous - compiled_code = str(raw_code) - if has_jinja(raw_code): - # Jinja found, compile it - compiled_code = self.compile_code(raw_code).compiled_code - return DbtAdapterExecutionResult( - *self.adapter_execute(compiled_code, fetch=True), - raw_code, - compiled_code, - ) - - def execute_from_node(self, node: "ManifestNode") -> DbtAdapterExecutionResult: - """Execute dbt SQL statement against database from a "ManifestNode".""" - raw_code: str = getattr(node, RAW_CODE) - compiled_code: Optional[str] = getattr(node, COMPILED_CODE, None) - if compiled_code: - # Node is compiled, execute the SQL - return self.execute_code(compiled_code) - # Node not compiled - if has_jinja(raw_code): - # Node has jinja in its SQL, compile it - compiled_code = self.compile_from_node(node).compiled_code - # Execute the SQL - return self.execute_code(compiled_code or raw_code) - - @lru_cache(maxsize=100) # noqa: B019 - def compile_code(self, raw_code: str) -> DbtAdapterCompilationResult: - """Create a node with `generate_server_node` method. Compile generated node. - - Has a retry built in because even uuidv4 cannot gaurantee uniqueness at the speed - in which we can call this function concurrently. A retry significantly increases the stability. - """ - temp_node_id = str(uuid.uuid4()) - with self.generate_server_node(raw_code, temp_node_id) as node: - return self.compile_from_node(node) - - @lru_cache(maxsize=100) # noqa: B019 - def unsafe_compile_code(self, raw_code: str, retry: int = 3) -> DbtAdapterCompilationResult: - """Create a node with `unsafe_generate_server_node` method. Compiles the generated node. - - Has a retry built in because even uuid4 cannot gaurantee uniqueness at the speed - in which we can call this function concurrently. A retry significantly increases the - stability. This is certainly the fastest way to compile SQL but it is yet to be benchmarked. - """ - temp_node_id = str(uuid.uuid4()) - try: - node = self.compile_from_node(self.unsafe_generate_server_node(raw_code, temp_node_id)) - except Exception as compilation_error: - if retry > 0: - return self.compile_code(raw_code, retry - 1) - raise compilation_error - else: - return node - finally: - self._clear_node(temp_node_id) - - def compile_from_node(self, node: "ManifestNode") -> DbtAdapterCompilationResult: - """Compiles existing node. ALL compilation passes through this code path. - - Raw SQL is marshalled by the caller into a mock node before being passed into this method. - Existing nodes can be passed in here directly. - """ - compiled_node = SqlCompileRunner( - self.config, self.adapter, node=node, node_index=1, num_nodes=1 - ).compile(self.manifest) - return DbtAdapterCompilationResult( - getattr(compiled_node, RAW_CODE), - getattr(compiled_node, COMPILED_CODE), - compiled_node, - ) - - def _clear_node(self, name: str = "anonymous_node") -> None: - """Clear remote node from dbt project.""" - self.manifest.nodes.pop(f"{NodeType.SqlOperation}.{self.project_name}.{name}", None) - - def get_relation(self, database: str, schema: str, name: str) -> Optional["BaseRelation"]: - """Wrap for `adapter.get_relation`.""" - return self.adapter.get_relation(database, schema, name) - - def relation_exists(self, database: str, schema: str, name: str) -> bool: - """Interface for checking if a relation exists in the database.""" - return self.adapter.get_relation(database, schema, name) is not None - - def node_exists(self, node: "ManifestNode") -> bool: - """Interface for checking if a node exists in the database.""" - return self.adapter.get_relation(self.create_relation_from_node(node)) is not None - - def create_relation(self, database: str, schema: str, name: str) -> "BaseRelation": - """Wrap `adapter.Relation.create`.""" - return self.adapter.Relation.create(database, schema, name) - - def create_relation_from_node(self, node: "ManifestNode") -> "BaseRelation": - """Wrap `adapter.Relation.create_from`.""" - return self.adapter.Relation.create_from(self.config, node) - - def get_or_create_relation( - self, database: str, schema: str, name: str - ) -> Tuple["BaseRelation", bool]: - """Get relation or create if not exists. - - Returns tuple of relation and boolean result of whether it existed ie: (relation, did_exist). - """ - ref = self.get_relation(database, schema, name) - return ( - (ref, True) - if ref is not None - else (self.create_relation(database, schema, name), False) - ) - - def create_schema(self, node: "ManifestNode") -> None: - """Create a schema in the database leveraging dbt-core's builtin macro.""" - self.execute_macro( - "create_schema", - kwargs={"relation": self.create_relation_from_node(node)}, - ) - - def get_columns_in_node(self, node: "ManifestNode") -> List[Any]: - """Wrap `adapter.get_columns_in_relation`.""" - return (self.adapter.get_columns_in_relation(self.create_relation_from_node(node)),) - - def get_columns(self, node: "ManifestNode") -> List[str]: - """Get a list of columns from a compiled node.""" - columns = [] - try: - columns.extend([c.name for c in self.get_columns_in_node(node)]) - except Exception: - # TODO: does this fallback make sense? - original_sql = str(getattr(node, RAW_CODE)) - setattr(node, RAW_CODE, f"SELECT * FROM ({original_sql}) WHERE 1=0") - result = self.execute_from_node(node) - setattr(node, RAW_CODE, original_sql), delattr(node, COMPILED_CODE) - node.compiled = False - columns.extend(result.table.column_names) - return columns - - def materialize( - self, node: "ManifestNode", temporary: bool = True - ) -> Tuple["AdapterResponse", None]: - """Materialize a table in the database. - - TODO: This is not fully baked. The API is stable but the implementation is not. - """ - return self.adapter_execute( - # Returns CTAS string so send to adapter.execute - self.execute_macro( - "create_table_as", - kwargs={ - "sql": getattr(node, COMPILED_CODE), - "relation": self.create_relation_from_node(node), - "temporary": temporary, - }, - ), - auto_begin=True, - ) - - @property - def sql_parser(self) -> SqlBlockParser: - """A dbt-core SQL parser capable of parsing and adding nodes to the manifest via `parse_remote` which will also return the added node to the caller. - - Note that post-parsing this still typically requires calls to `_process_nodes_for_ref` - and `_process_sources_for_ref` from the `dbt.parser.manifest` module in order to compile. - We have higher level methods that handle this for you. - """ - if self._sql_parser is None: - self._sql_parser = SqlBlockParser(self.config, self.manifest, self.config) - return self._sql_parser - - @property - def macro_parser(self) -> SqlMacroParser: - """A dbt-core macro parser. Parse macros with `parse_remote` and add them to the manifest. - - We have a higher level method `inject_macro` that handles this for you. - """ - if self._macro_parser is None: - self._macro_parser = SqlMacroParser(self.config, self.manifest) - return self._macro_parser - - def get_task_config(self, **kwargs) -> DbtTaskConfiguration: - """Get a dbt-core task configuration.""" - threads = kwargs.pop("threads", self.config.threads) - return DbtTaskConfiguration.from_runtime_config( - config=self.config, threads=threads, **kwargs - ) - - def get_task_cls(self, typ: DbtCommand) -> Type["ManifestTask"]: - """Get a dbt-core task class by type. - - This could be overridden to add custom tasks such as linting, etc. - so long as they are subclasses of `GraphRunnableTask`. - """ - # These are purposefully deferred imports - from dbt.task.build import BuildTask - from dbt.task.list import ListTask - from dbt.task.run import RunTask - from dbt.task.run_operation import RunOperationTask - from dbt.task.seed import SeedTask - from dbt.task.snapshot import SnapshotTask - from dbt.task.test import TestTask - - return { - DbtCommand.RUN: RunTask, - DbtCommand.BUILD: BuildTask, - DbtCommand.TEST: TestTask, - DbtCommand.SEED: SeedTask, - DbtCommand.LIST: ListTask, - DbtCommand.SNAPSHOT: SnapshotTask, - DbtCommand.RUN_OPERATION: RunOperationTask, - }[typ] - - def get_task(self, typ: DbtCommand, args: DbtTaskConfiguration) -> "ManifestTask": - """Get a dbt-core task by type.""" - try: - # DBT 1.8 requires manifest as 2-nd positional argument - task = self.get_task_cls(typ)(args, self.config, self.manifest) - except Exception as e: - task = self.get_task_cls(typ)(args, self.config) - # Render this a no-op on this class instance so that the tasks `run` - # method plumbing will defer to our existing in memory manifest. - task.load_manifest = lambda *args, **kwargs: None # type: ignore - task.manifest = self.manifest - return task - - def list( - self, - select: Optional[Union[str, List[str]]] = None, - exclude: Optional[Union[str, List[str]]] = None, - **kwargs: Dict[str, Any], - ) -> "ExecutionResult": - """List resources in the dbt project.""" - select, exclude = marshall_selection_args(select, exclude) - with redirect_stdout(None): - return self.get_task( # type: ignore - DbtCommand.LIST, - self.get_task_config(select=select, exclude=exclude, **kwargs), - ).run() - - def run( - self, - select: Optional[Union[str, List[str]]] = None, - exclude: Optional[Union[str, List[str]]] = None, - **kwargs: Dict[str, Any], - ) -> "RunExecutionResult": - """Run models in the dbt project.""" - select, exclude = marshall_selection_args(select, exclude) - with redirect_stdout(None): - return cast( - "RunExecutionResult", - self.get_task( - DbtCommand.RUN, - self.get_task_config(select=select, exclude=exclude, **kwargs), - ).run(), - ) - - def test( - self, - select: Optional[Union[str, List[str]]] = None, - exclude: Optional[Union[str, List[str]]] = None, - **kwargs: Dict[str, Any], - ) -> "ExecutionResult": - """Test models in the dbt project.""" - select, exclude = marshall_selection_args(select, exclude) - with redirect_stdout(None): - return self.get_task( # type: ignore - DbtCommand.TEST, - self.get_task_config(select=select, exclude=exclude, **kwargs), - ).run() - - def build( - self, - select: Optional[Union[str, List[str]]] = None, - exclude: Optional[Union[str, List[str]]] = None, - **kwargs: Dict[str, Any], - ) -> "ExecutionResult": - """Build resources in the dbt project.""" - select, exclude = marshall_selection_args(select, exclude) - with redirect_stdout(None): - return self.get_task( - DbtCommand.BUILD, - self.get_task_config(select=select, exclude=exclude, **kwargs), - ).run() - - -def marshall_selection_args( - select: Optional[Union[str, List[str]]] = None, - exclude: Optional[Union[str, List[str], None]] = None, -) -> Tuple[Union[str, List[str]], Union[str, List[str]]]: - """Marshall selection arguments to a list of strings.""" - if select is None: - select = [] - if exclude is None: - exclude = [] - if isinstance(select, (tuple, set, frozenset)): - select = list(select) - if isinstance(exclude, (tuple, set, frozenset)): - exclude = list(exclude) - # Permit standalone strings such as "my_model+ @some_other_model" - # as well as lists of strings such as ["my_model+", "@some_other_model"] - if not isinstance(select, list): - select = [select] - if not isinstance(exclude, list): - exclude = [exclude] - return select, exclude - - -class DbtProjectContainer: - """Manages multiple DbtProjects. - - A DbtProject corresponds to a single project. This interface is used - dbt projects in a single process. It enables basic multitenant servers. - """ - - def __init__(self) -> None: - """Initialize the container.""" - self._projects: Dict[str, DbtProject] = OrderedDict() - self._default_project: Optional[str] = None - - def get_project(self, project_name: str) -> Optional[DbtProject]: - """Primary interface to get a project and execute code.""" - return self._projects.get(project_name) - - def get_project_by_root_dir(self, root_dir: str) -> Optional[DbtProject]: - """Get a project by its root directory.""" - root_dir = os.path.abspath(os.path.normpath(root_dir)) - for project in self._projects.values(): - if os.path.abspath(project.project_root) == root_dir: - return project - return None - - def get_default_project(self) -> Optional[DbtProject]: - """Get the default project which is the earliest project inserted into the container.""" - default_project = self._default_project - if not default_project: - return None - return self._projects.get(default_project) - - def add_project( - self, - target: Optional[str] = None, - profiles_dir: str = DEFAULT_PROFILES_DIR, - project_dir: Optional[str] = None, - threads: int = 1, - vars: str = "{}", - name_override: str = "", - ) -> DbtProject: - """Add a DbtProject with arguments.""" - project = DbtProject(target, profiles_dir, project_dir, threads, vars) - project_name = name_override or project.config.project_name - if self._default_project is None: - self._default_project = project_name - self._projects[project_name] = project - return project - - def add_parsed_project(self, project: DbtProject) -> DbtProject: - """Add an already instantiated DbtProject.""" - self._projects.setdefault(project.config.project_name, project) - if self._default_project is None: - self._default_project = project.config.project_name - return project - - def add_project_from_args(self, config: DbtConfiguration) -> DbtProject: - """Add a DbtProject from a DbtConfiguration.""" - project = DbtProject.from_config(config) - self._projects.setdefault(project.config.project_name, project) - return project - - def drop_project(self, project_name: str) -> None: - """Drop a DbtProject.""" - project = self.get_project(project_name) - if project is None: - return - # Encourage garbage collection - project.clear_internal_caches() - project.adapter.connections.cleanup_all() - self._projects.pop(project_name) - if self._default_project == project_name: - if len(self) > 0: - self._default_project = list(self._projects.keys())[0] - else: - self._default_project = None - - def drop_all_projects(self) -> None: - """Drop all DbtProject's in the container.""" - self._default_project = None - for project in self._projects: - self.drop_project(project) - - def reparse_all_projects(self) -> None: - """Reparse all projects.""" - for project in self: - project.safe_parse_project() - - def registered_projects(self) -> List[str]: - """Grab all registered project names.""" - return list(self._projects.keys()) - - def __len__(self) -> int: - """Allow len(DbtProjectContainer).""" - return len(self._projects) - - def __getitem__(self, project: str) -> DbtProject: - """Allow DbtProjectContainer['jaffle_shop'].""" - maybe_project = self.get_project(project) - if maybe_project is None: - raise KeyError(project) - return maybe_project - - def __setitem__(self, name: str, project: DbtProject) -> None: - """Allow DbtProjectContainer['jaffle_shop'] = DbtProject.""" - if self._default_project is None: - self._default_project = name - self._projects[name] = project - - def __delitem__(self, project: str) -> None: - """Allow del DbtProjectContainer['jaffle_shop'].""" - self.drop_project(project) - - def __iter__(self) -> Generator[DbtProject, None, None]: - """Allow project for project in DbtProjectContainer.""" - for project in self._projects: - maybe_project = self.get_project(project) - if maybe_project is None: - continue - yield maybe_project - - def __contains__(self, project: str) -> bool: - """Allow 'jaffle_shop' in DbtProjectContainer.""" - return project in self._projects - - def __repr__(self) -> str: - """Canonical string representation of DbtProjectContainer instance.""" - return "\n".join( - f"Project: {project.project_name}, Dir: {project.project_root}" for project in self - ) - - -def has_jinja(query: str) -> bool: - """Check if a query contains any Jinja control sequences.""" - return any(seq in query for seq in JINJA_CONTROL_SEQUENCES) - - -def semvar_to_tuple(semvar: "VersionSpecifier") -> Tuple[int, int, int]: - """Convert a semvar to a tuple of ints.""" - return (int(semvar.major or 0), int(semvar.minor or 0), int(semvar.patch or 0)) - - -# endregion - -# region bottle.py - - -def _cli_parse(args): # pragma: no coverage - from argparse import ArgumentParser - - parser = ArgumentParser(prog=args[0], usage="%(prog)s [options] package.module:app") - opt = parser.add_argument - opt("--version", action="store_true", help="show version number.") - opt("-b", "--bind", metavar="ADDRESS", help="bind socket to ADDRESS.") - opt("-s", "--server", default="wsgiref", help="use SERVER as backend.") - opt("-p", "--plugin", action="append", help="install additional plugin/s.") - opt("-c", "--conf", action="append", metavar="FILE", help="load config values from FILE.") - opt("-C", "--param", action="append", metavar="NAME=VALUE", help="override config values.") - opt("--debug", action="store_true", help="start server in debug mode.") - opt("--reload", action="store_true", help="auto-reload on file changes.") - opt("app", help="WSGI app entry point.", nargs="?") - - cli_args = parser.parse_args(args[1:]) - - return cli_args, parser - - -py = sys.version_info -py3k = py.major > 2 - - -def getargspec(func): - spec = getfullargspec(func) - kwargs = makelist(spec[0]) + makelist(spec.kwonlyargs) - return kwargs, spec[1], spec[2], spec[3] - - -basestring = str -unicode = str -json_loads = lambda s: json_lds(touni(s)) -callable = lambda x: hasattr(x, "__call__") -imap = map - - -def _raise(*a): - raise a[0](a[1]).with_traceback(a[2]) - - -# Some helpers for string/byte handling -def tob(s, enc="utf8"): - if isinstance(s, unicode): - return s.encode(enc) - return b"" if s is None else bytes(s) - - -def touni(s, enc="utf8", err="strict"): - if isinstance(s, bytes): - return s.decode(enc, err) - return unicode("" if s is None else s) - - -tonat = touni if py3k else tob - - -def _stderr(*args): - try: - print(*args, file=sys.stderr) - except (OSError, AttributeError): - pass # Some environments do not allow printing (mod_wsgi) - - -# A bug in functools causes it to break if the wrapper is an instance method -def update_wrapper(wrapper, wrapped, *a, **ka): - try: - functools.update_wrapper(wrapper, wrapped, *a, **ka) - except AttributeError: - pass - - -# These helpers are used at module level and need to be defined first. -# And yes, I know PEP-8, but sometimes a lower-case classname makes more sense. - - -def depr(major, minor, cause, fix): - text = ( - "Warning: Use of deprecated feature or API. (Deprecated in Bottle-%d.%d)\n" - "Cause: %s\n" - "Fix: %s\n" % (major, minor, cause, fix) - ) - if DEBUG == "strict": - raise DeprecationWarning(text) - warnings.warn(text, DeprecationWarning, stacklevel=3) - return DeprecationWarning(text) - - -def makelist(data): # This is just too handy - if isinstance(data, (tuple, list, set, dict)): - return list(data) - elif data: - return [data] - else: - return [] - - -class DictProperty: - """Property that maps to a key in a local dict-like attribute.""" - - def __init__(self, attr, key=None, read_only=False): - self.attr, self.key, self.read_only = attr, key, read_only - - def __call__(self, func): - functools.update_wrapper(self, func, updated=[]) - self.getter, self.key = func, self.key or func.__name__ - return self - - def __get__(self, obj, cls): - if obj is None: - return self - key, storage = self.key, getattr(obj, self.attr) - if key not in storage: - storage[key] = self.getter(obj) - return storage[key] - - def __set__(self, obj, value): - if self.read_only: - raise AttributeError("Read-Only property.") - getattr(obj, self.attr)[self.key] = value - - def __delete__(self, obj): - if self.read_only: - raise AttributeError("Read-Only property.") - del getattr(obj, self.attr)[self.key] - - -class cached_property: - """A property that is only computed once per instance and then replaces - itself with an ordinary attribute. Deleting the attribute resets the - property.""" - - def __init__(self, func): - update_wrapper(self, func) - self.func = func - - def __get__(self, obj, cls): - if obj is None: - return self - value = obj.__dict__[self.func.__name__] = self.func(obj) - return value - - -class lazy_attribute: - """A property that caches itself to the class object.""" - - def __init__(self, func): - functools.update_wrapper(self, func, updated=[]) - self.getter = func - - def __get__(self, obj, cls): - value = self.getter(cls) - setattr(cls, self.__name__, value) - return value - - -class BottleException(Exception): - """A base class for exceptions used by bottle.""" - - pass - - -class RouteError(BottleException): - """This is a base class for all routing related exceptions""" - - -class RouteReset(BottleException): - """If raised by a plugin or request handler, the route is reset and all - plugins are re-applied.""" - - -class RouterUnknownModeError(RouteError): - pass - - -class RouteSyntaxError(RouteError): - """The route parser found something not supported by this router.""" - - -class RouteBuildError(RouteError): - """The route could not be built.""" - - -def _re_flatten(p): - """Turn all capturing groups in a regular expression pattern into - non-capturing groups.""" - if "(" not in p: - return p - return re.sub( - r"(\\*)(\(\?P<[^>]+>|\((?!\?))", - lambda m: m.group(0) if len(m.group(1)) % 2 else m.group(1) + "(?:", - p, - ) - - -class Router: - """A Router is an ordered collection of route->target pairs. It is used to - efficiently match WSGI requests against a number of routes and return - the first target that satisfies the request. The target may be anything, - usually a string, ID or callable object. A route consists of a path-rule - and a HTTP method. - - The path-rule is either a static path (e.g. `/contact`) or a dynamic - path that contains wildcards (e.g. `/wiki/`). The wildcard syntax - and details on the matching order are described in docs:`routing`. - """ - - default_pattern = "[^/]+" - default_filter = "re" - - #: The current CPython regexp implementation does not allow more - #: than 99 matching groups per regular expression. - _MAX_GROUPS_PER_PATTERN = 99 - - def __init__(self, strict=False): - self.rules = [] # All rules in order - self._groups = {} # index of regexes to find them in dyna_routes - self.builder = {} # Data structure for the url builder - self.static = {} # Search structure for static routes - self.dyna_routes = {} - self.dyna_regexes = {} # Search structure for dynamic routes - #: If true, static routes are no longer checked first. - self.strict_order = strict - self.filters = { - "re": lambda conf: (_re_flatten(conf or self.default_pattern), None, None), - "int": lambda conf: (r"-?\d+", int, lambda x: str(int(x))), - "float": lambda conf: (r"-?[\d.]+", float, lambda x: str(float(x))), - "path": lambda conf: (r".+?", None, None), - } - - def add_filter(self, name, func): - """Add a filter. The provided function is called with the configuration - string as parameter and must return a (regexp, to_python, to_url) tuple. - The first element is a string, the last two are callables or None.""" - self.filters[name] = func - - rule_syntax = re.compile( - "(\\\\*)" - "(?:(?::([a-zA-Z_][a-zA-Z_0-9]*)?()(?:#(.*?)#)?)" - "|(?:<([a-zA-Z_][a-zA-Z_0-9]*)?(?::([a-zA-Z_]*)" - "(?::((?:\\\\.|[^\\\\>])+)?)?)?>))" - ) - - def _itertokens(self, rule): - offset, prefix = 0, "" - for match in self.rule_syntax.finditer(rule): - prefix += rule[offset : match.start()] - g = match.groups() - if g[2] is not None: - depr(0, 13, "Use of old route syntax.", "Use instead of :name in routes.") - if len(g[0]) % 2: # Escaped wildcard - prefix += match.group(0)[len(g[0]) :] - offset = match.end() - continue - if prefix: - yield prefix, None, None - name, filtr, conf = g[4:7] if g[2] is None else g[1:4] - yield name, filtr or "default", conf or None - offset, prefix = match.end(), "" - if offset <= len(rule) or prefix: - yield prefix + rule[offset:], None, None - - def add(self, rule, method, target, name=None): - """Add a new rule or replace the target for an existing rule.""" - anons = 0 # Number of anonymous wildcards found - keys = [] # Names of keys - pattern = "" # Regular expression pattern with named groups - filters = [] # Lists of wildcard input filters - builder = [] # Data structure for the URL builder - is_static = True - - for key, mode, conf in self._itertokens(rule): - if mode: - is_static = False - if mode == "default": - mode = self.default_filter - mask, in_filter, out_filter = self.filters[mode](conf) - if not key: - pattern += "(?:%s)" % mask - key = "anon%d" % anons - anons += 1 - else: - pattern += f"(?P<{key}>{mask})" - keys.append(key) - if in_filter: - filters.append((key, in_filter)) - builder.append((key, out_filter or str)) - elif key: - pattern += re.escape(key) - builder.append((None, key)) - - self.builder[rule] = builder - if name: - self.builder[name] = builder - - if is_static and not self.strict_order: - self.static.setdefault(method, {}) - self.static[method][self.build(rule)] = (target, None) - return - - try: - re_pattern = re.compile("^(%s)$" % pattern) - re_match = re_pattern.match - except re.error as e: - raise RouteSyntaxError(f"Could not add Route: {rule} ({e})") - - if filters: - - def getargs(path): - url_args = re_match(path).groupdict() - for name, wildcard_filter in filters: - try: - url_args[name] = wildcard_filter(url_args[name]) - except ValueError: - raise HTTPError(400, "Path has wrong format.") - return url_args - - elif re_pattern.groupindex: - - def getargs(path): - return re_match(path).groupdict() - - else: - getargs = None - - flatpat = _re_flatten(pattern) - whole_rule = (rule, flatpat, target, getargs) - - if (flatpat, method) in self._groups: - if DEBUG: - msg = "Route <%s %s> overwrites a previously defined route" - warnings.warn(msg % (method, rule), RuntimeWarning) - self.dyna_routes[method][self._groups[flatpat, method]] = whole_rule - else: - self.dyna_routes.setdefault(method, []).append(whole_rule) - self._groups[flatpat, method] = len(self.dyna_routes[method]) - 1 - - self._compile(method) - - def _compile(self, method): - all_rules = self.dyna_routes[method] - comborules = self.dyna_regexes[method] = [] - maxgroups = self._MAX_GROUPS_PER_PATTERN - for x in range(0, len(all_rules), maxgroups): - some = all_rules[x : x + maxgroups] - combined = (flatpat for (_, flatpat, _, _) in some) - combined = "|".join("(^%s$)" % flatpat for flatpat in combined) - combined = re.compile(combined).match - rules = [(target, getargs) for (_, _, target, getargs) in some] - comborules.append((combined, rules)) - - def build(self, _name, *anons, **query): - """Build an URL by filling the wildcards in a rule.""" - builder = self.builder.get(_name) - if not builder: - raise RouteBuildError("No route with that name.", _name) - try: - for i, value in enumerate(anons): - query["anon%d" % i] = value - url = "".join([f(query.pop(n)) if n else f for (n, f) in builder]) - return url if not query else url + "?" + urlencode(query) - except KeyError as E: - raise RouteBuildError("Missing URL argument: %r" % E.args[0]) - - def match(self, environ): - """Return a (target, url_args) tuple or raise HTTPError(400/404/405).""" - verb = environ["REQUEST_METHOD"].upper() - path = environ["PATH_INFO"] or "/" - - methods = ("PROXY", "HEAD", "GET", "ANY") if verb == "HEAD" else ("PROXY", verb, "ANY") - - for method in methods: - if method in self.static and path in self.static[method]: - target, getargs = self.static[method][path] - return target, getargs(path) if getargs else {} - elif method in self.dyna_regexes: - for combined, rules in self.dyna_regexes[method]: - match = combined(path) - if match: - target, getargs = rules[match.lastindex - 1] - return target, getargs(path) if getargs else {} - - # No matching route found. Collect alternative methods for 405 response - allowed = set() - nocheck = set(methods) - for method in set(self.static) - nocheck: - if path in self.static[method]: - allowed.add(method) - for method in set(self.dyna_regexes) - allowed - nocheck: - for combined, rules in self.dyna_regexes[method]: - match = combined(path) - if match: - allowed.add(method) - if allowed: - allow_header = ",".join(sorted(allowed)) - raise HTTPError(405, "Method not allowed.", Allow=allow_header) - - # No matching route and no alternative method found. We give up - raise HTTPError(404, "Not found: " + repr(path)) - - -class Route: - """This class wraps a route callback along with route specific metadata and - configuration and applies Plugins on demand. It is also responsible for - turning an URL path rule into a regular expression usable by the Router. - """ - - def __init__( - self, app, rule, method, callback, name=None, plugins=None, skiplist=None, **config - ): - #: The application this route is installed to. - self.app = app - #: The path-rule string (e.g. ``/wiki/``). - self.rule = rule - #: The HTTP method as a string (e.g. ``GET``). - self.method = method - #: The original callback with no plugins applied. Useful for introspection. - self.callback = callback - #: The name of the route (if specified) or ``None``. - self.name = name or None - #: A list of route-specific plugins (see :meth:`Bottle.route`). - self.plugins = plugins or [] - #: A list of plugins to not apply to this route (see :meth:`Bottle.route`). - self.skiplist = skiplist or [] - #: Additional keyword arguments passed to the :meth:`Bottle.route` - #: decorator are stored in this dictionary. Used for route-specific - #: plugin configuration and meta-data. - self.config = app.config._make_overlay() - self.config.load_dict(config) - - @cached_property - def call(self): - """The route callback with all plugins applied. This property is - created on demand and then cached to speed up subsequent requests.""" - return self._make_callback() - - def reset(self): - """Forget any cached values. The next time :attr:`call` is accessed, - all plugins are re-applied.""" - self.__dict__.pop("call", None) - - def prepare(self): - """Do all on-demand work immediately (useful for debugging).""" - self.call - - def all_plugins(self): - """Yield all Plugins affecting this route.""" - unique = set() - for p in reversed(self.app.plugins + self.plugins): - if True in self.skiplist: - break - name = getattr(p, "name", False) - if name and (name in self.skiplist or name in unique): - continue - if p in self.skiplist or type(p) in self.skiplist: - continue - if name: - unique.add(name) - yield p - - def _make_callback(self): - callback = self.callback - for plugin in self.all_plugins(): - try: - if hasattr(plugin, "apply"): - callback = plugin.apply(callback, self) - else: - callback = plugin(callback) - except RouteReset: # Try again with changed configuration. - return self._make_callback() - if callback is not self.callback: - update_wrapper(callback, self.callback) - return callback - - def get_undecorated_callback(self): - """Return the callback. If the callback is a decorated function, try to - recover the original function.""" - func = self.callback - func = getattr(func, "__func__" if py3k else "im_func", func) - closure_attr = "__closure__" if py3k else "func_closure" - while hasattr(func, closure_attr) and getattr(func, closure_attr): - attributes = getattr(func, closure_attr) - func = attributes[0].cell_contents - - # in case of decorators with multiple arguments - if not isinstance(func, FunctionType): - # pick first FunctionType instance from multiple arguments - func = filter( - lambda x: isinstance(x, FunctionType), - map(lambda x: x.cell_contents, attributes), - ) - func = list(func)[0] # py3 support - return func - - def get_callback_args(self): - """Return a list of argument names the callback (most likely) accepts - as keyword arguments. If the callback is a decorated function, try - to recover the original function before inspection.""" - return getargspec(self.get_undecorated_callback())[0] - - def get_config(self, key, default=None): - """Lookup a config field and return its value, first checking the - route.config, then route.app.config.""" - depr( - 0, - 13, - "Route.get_config() is deprecated.", - "The Route.config property already includes values from the" - " application config for missing keys. Access it directly.", - ) - return self.config.get(key, default) - - def __repr__(self): - cb = self.get_undecorated_callback() - return f"<{self.method} {self.rule} -> {cb.__module__}:{cb.__name__}>" - - -class Bottle: - """Each Bottle object represents a single, distinct web application and - consists of routes, callbacks, plugins, resources and configuration. - Instances are callable WSGI applications. - - :param catchall: If true (default), handle all exceptions. Turn off to - let debugging middleware handle exceptions. - """ - - @lazy_attribute - def _global_config(cls): - cfg = ConfigDict() - cfg.meta_set("catchall", "validate", bool) - return cfg - - def __init__(self, **kwargs): - #: A :class:`ConfigDict` for app specific configuration. - self.config = self._global_config._make_overlay() - self.config._add_change_listener(functools.partial(self.trigger_hook, "config")) - - self.config.update({"catchall": True}) - - if kwargs.get("catchall") is False: - depr( - 0, - 13, - "Bottle(catchall) keyword argument.", - "The 'catchall' setting is now part of the app " - "configuration. Fix: `app.config['catchall'] = False`", - ) - self.config["catchall"] = False - if kwargs.get("autojson") is False: - depr( - 0, - 13, - "Bottle(autojson) keyword argument.", - "The 'autojson' setting is now part of the app " - "configuration. Fix: `app.config['json.enable'] = False`", - ) - self.config["json.disable"] = True - - self._mounts = [] - - #: A :class:`ResourceManager` for application files - self.resources = ResourceManager() - - self.routes = [] # List of installed :class:`Route` instances. - self.router = Router() # Maps requests to :class:`Route` instances. - self.error_handler = {} - - # Core plugins - self.plugins = [] # List of installed plugins. - self.install(JSONPlugin()) - self.install(TemplatePlugin()) - - #: If true, most exceptions are caught and returned as :exc:`HTTPError` - catchall = DictProperty("config", "catchall") - - __hook_names = "before_request", "after_request", "app_reset", "config" - __hook_reversed = {"after_request"} - - @cached_property - def _hooks(self): - return {name: [] for name in self.__hook_names} - - def add_hook(self, name, func): - """Attach a callback to a hook. Three hooks are currently implemented: - - before_request - Executed once before each request. The request context is - available, but no routing has happened yet. - after_request - Executed once after each request regardless of its outcome. - app_reset - Called whenever :meth:`Bottle.reset` is called. - """ - if name in self.__hook_reversed: - self._hooks[name].insert(0, func) - else: - self._hooks[name].append(func) - - def remove_hook(self, name, func): - """Remove a callback from a hook.""" - if name in self._hooks and func in self._hooks[name]: - self._hooks[name].remove(func) - return True - - def trigger_hook(self, __name, *args, **kwargs): - """Trigger a hook and return a list of results.""" - return [hook(*args, **kwargs) for hook in self._hooks[__name][:]] - - def hook(self, name): - """Return a decorator that attaches a callback to a hook. See - :meth:`add_hook` for details.""" - - def decorator(func): - self.add_hook(name, func) - return func - - return decorator - - def _mount_wsgi(self, prefix, app, **options): - segments = [p for p in prefix.split("/") if p] - if not segments: - raise ValueError('WSGI applications cannot be mounted to "/".') - path_depth = len(segments) - - def mountpoint_wrapper(): - try: - request.path_shift(path_depth) - rs = HTTPResponse([]) - - def start_response(status, headerlist, exc_info=None): - if exc_info: - _raise(*exc_info) - if py3k: - # Errors here mean that the mounted WSGI app did not - # follow PEP-3333 (which requires latin1) or used a - # pre-encoding other than utf8 :/ - status = status.encode("latin1").decode("utf8") - headerlist = [ - (k, v.encode("latin1").decode("utf8")) for (k, v) in headerlist - ] - rs.status = status - for name, value in headerlist: - rs.add_header(name, value) - return rs.body.append - - body = app(request.environ, start_response) - rs.body = itertools.chain(rs.body, body) if rs.body else body - return rs - finally: - request.path_shift(-path_depth) - - options.setdefault("skip", True) - options.setdefault("method", "PROXY") - options.setdefault("mountpoint", {"prefix": prefix, "target": app}) - options["callback"] = mountpoint_wrapper - - self.route("/%s/<:re:.*>" % "/".join(segments), **options) - if not prefix.endswith("/"): - self.route("/" + "/".join(segments), **options) - - def _mount_app(self, prefix, app, **options): - if app in self._mounts or "_mount.app" in app.config: - depr( - 0, - 13, - "Application mounted multiple times. Falling back to WSGI mount.", - "Clone application before mounting to a different location.", - ) - return self._mount_wsgi(prefix, app, **options) - - if options: - depr( - 0, - 13, - "Unsupported mount options. Falling back to WSGI mount.", - "Do not specify any route options when mounting bottle application.", - ) - return self._mount_wsgi(prefix, app, **options) - - if not prefix.endswith("/"): - depr( - 0, - 13, - "Prefix must end in '/'. Falling back to WSGI mount.", - "Consider adding an explicit redirect from '/prefix' to '/prefix/' in the" - " parent application.", - ) - return self._mount_wsgi(prefix, app, **options) - - self._mounts.append(app) - app.config["_mount.prefix"] = prefix - app.config["_mount.app"] = self - for route in app.routes: - route.rule = prefix + route.rule.lstrip("/") - self.add_route(route) - - def mount(self, prefix, app, **options): - """Mount an application (:class:`Bottle` or plain WSGI) to a specific - URL prefix. Example:: - - parent_app.mount('/prefix/', child_app) - - :param prefix: path prefix or `mount-point`. - :param app: an instance of :class:`Bottle` or a WSGI application. - - Plugins from the parent application are not applied to the routes - of the mounted child application. If you need plugins in the child - application, install them separately. - - While it is possible to use path wildcards within the prefix path - (:class:`Bottle` childs only), it is highly discouraged. - - The prefix path must end with a slash. If you want to access the - root of the child application via `/prefix` in addition to - `/prefix/`, consider adding a route with a 307 redirect to the - parent application. - """ - - if not prefix.startswith("/"): - raise ValueError("Prefix must start with '/'") - - if isinstance(app, Bottle): - return self._mount_app(prefix, app, **options) - else: - return self._mount_wsgi(prefix, app, **options) - - def merge(self, routes): - """Merge the routes of another :class:`Bottle` application or a list of - :class:`Route` objects into this application. The routes keep their - 'owner', meaning that the :data:`Route.app` attribute is not - changed.""" - if isinstance(routes, Bottle): - routes = routes.routes - for route in routes: - self.add_route(route) - - def install(self, plugin): - """Add a plugin to the list of plugins and prepare it for being - applied to all routes of this application. A plugin may be a simple - decorator or an object that implements the :class:`Plugin` API. - """ - if hasattr(plugin, "setup"): - plugin.setup(self) - if not callable(plugin) and not hasattr(plugin, "apply"): - raise TypeError("Plugins must be callable or implement .apply()") - self.plugins.append(plugin) - self.reset() - return plugin - - def uninstall(self, plugin): - """Uninstall plugins. Pass an instance to remove a specific plugin, a type - object to remove all plugins that match that type, a string to remove - all plugins with a matching ``name`` attribute or ``True`` to remove all - plugins. Return the list of removed plugins.""" - removed, remove = [], plugin - for i, plugin in list(enumerate(self.plugins))[::-1]: - if ( - remove is True - or remove is plugin - or remove is type(plugin) - or getattr(plugin, "name", True) == remove - ): - removed.append(plugin) - del self.plugins[i] - if hasattr(plugin, "close"): - plugin.close() - if removed: - self.reset() - return removed - - def reset(self, route=None): - """Reset all routes (force plugins to be re-applied) and clear all - caches. If an ID or route object is given, only that specific route - is affected.""" - if route is None: - routes = self.routes - elif isinstance(route, Route): - routes = [route] - else: - routes = [self.routes[route]] - for route in routes: - route.reset() - if DEBUG: - for route in routes: - route.prepare() - self.trigger_hook("app_reset") - - def close(self): - """Close the application and all installed plugins.""" - for plugin in self.plugins: - if hasattr(plugin, "close"): - plugin.close() - - def run(self, **kwargs): - """Calls :func:`run` with the same parameters.""" - run(self, **kwargs) - - def match(self, environ): - """Search for a matching route and return a (:class:`Route`, urlargs) - tuple. The second value is a dictionary with parameters extracted - from the URL. Raise :exc:`HTTPError` (404/405) on a non-match.""" - return self.router.match(environ) - - def get_url(self, routename, **kargs): - """Return a string that matches a named route""" - scriptname = request.environ.get("SCRIPT_NAME", "").strip("/") + "/" - location = self.router.build(routename, **kargs).lstrip("/") - return urljoin(urljoin("/", scriptname), location) - - def add_route(self, route): - """Add a route object, but do not change the :data:`Route.app` - attribute.""" - self.routes.append(route) - self.router.add(route.rule, route.method, route, name=route.name) - if DEBUG: - route.prepare() - - def route( - self, path=None, method="GET", callback=None, name=None, apply=None, skip=None, **config - ): - """A decorator to bind a function to a request URL. Example:: - - @app.route('/hello/') - def hello(name): - return 'Hello %s' % name - - The ```` part is a wildcard. See :class:`Router` for syntax - details. - - :param path: Request path or a list of paths to listen to. If no - path is specified, it is automatically generated from the - signature of the function. - :param method: HTTP method (`GET`, `POST`, `PUT`, ...) or a list of - methods to listen to. (default: `GET`) - :param callback: An optional shortcut to avoid the decorator - syntax. ``route(..., callback=func)`` equals ``route(...)(func)`` - :param name: The name for this route. (default: None) - :param apply: A decorator or plugin or a list of plugins. These are - applied to the route callback in addition to installed plugins. - :param skip: A list of plugins, plugin classes or names. Matching - plugins are not installed to this route. ``True`` skips all. - - Any additional keyword arguments are stored as route-specific - configuration and passed to plugins (see :meth:`Plugin.apply`). - """ - if callable(path): - path, callback = None, path - plugins = makelist(apply) - skiplist = makelist(skip) - - def decorator(callback): - if isinstance(callback, basestring): - callback = load(callback) - for rule in makelist(path) or yieldroutes(callback): - for verb in makelist(method): - verb = verb.upper() - route = Route( - self, - rule, - verb, - callback, - name=name, - plugins=plugins, - skiplist=skiplist, - **config, - ) - self.add_route(route) - return callback - - return decorator(callback) if callback else decorator - - def get(self, path=None, method="GET", **options): - """Equals :meth:`route`.""" - return self.route(path, method, **options) - - def post(self, path=None, method="POST", **options): - """Equals :meth:`route` with a ``POST`` method parameter.""" - return self.route(path, method, **options) - - def put(self, path=None, method="PUT", **options): - """Equals :meth:`route` with a ``PUT`` method parameter.""" - return self.route(path, method, **options) - - def delete(self, path=None, method="DELETE", **options): - """Equals :meth:`route` with a ``DELETE`` method parameter.""" - return self.route(path, method, **options) - - def patch(self, path=None, method="PATCH", **options): - """Equals :meth:`route` with a ``PATCH`` method parameter.""" - return self.route(path, method, **options) - - def error(self, code=500, callback=None): - """Register an output handler for a HTTP error code. Can - be used as a decorator or called directly :: - - def error_handler_500(error): - return 'error_handler_500' - - app.error(code=500, callback=error_handler_500) - - @app.error(404) - def error_handler_404(error): - return 'error_handler_404' - - """ - - def decorator(callback): - if isinstance(callback, basestring): - callback = load(callback) - self.error_handler[int(code)] = callback - return callback - - return decorator(callback) if callback else decorator - - def default_error_handler(self, res): - return tob( - template( - ERROR_PAGE_TEMPLATE, e=res, template_settings=dict(name="__ERROR_PAGE_TEMPLATE") - ) - ) - - def _handle(self, environ): - path = environ["bottle.raw_path"] = environ["PATH_INFO"] - if py3k: - environ["PATH_INFO"] = path.encode("latin1").decode("utf8", "ignore") - - environ["bottle.app"] = self - request.bind(environ) - response.bind() - - try: - while True: # Remove in 0.14 together with RouteReset - out = None - try: - self.trigger_hook("before_request") - route, args = self.router.match(environ) - environ["route.handle"] = route - environ["bottle.route"] = route - environ["route.url_args"] = args - out = route.call(**args) - break - except HTTPResponse as E: - out = E - break - except RouteReset: - depr( - 0, - 13, - "RouteReset exception deprecated", - "Call route.call() after route.reset() and return the result.", - ) - route.reset() - continue - finally: - if isinstance(out, HTTPResponse): - out.apply(response) - try: - self.trigger_hook("after_request") - except HTTPResponse as E: - out = E - out.apply(response) - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception as E: - if not self.catchall: - raise - stacktrace = format_exc() - environ["wsgi.errors"].write(stacktrace) - environ["wsgi.errors"].flush() - environ["bottle.exc_info"] = sys.exc_info() - out = HTTPError(500, "Internal Server Error", E, stacktrace) - out.apply(response) - - return out - - def _cast(self, out, peek=None): - """Try to convert the parameter into something WSGI compatible and set - correct HTTP headers when possible. - Support: False, str, unicode, dict, HTTPResponse, HTTPError, file-like, - iterable of strings and iterable of unicodes - """ - - # Empty output is done here - if not out: - if "Content-Length" not in response: - response["Content-Length"] = 0 - return [] - # Join lists of byte or unicode strings. Mixed lists are NOT supported - if isinstance(out, (tuple, list)) and isinstance(out[0], (bytes, unicode)): - out = out[0][0:0].join(out) # b'abc'[0:0] -> b'' - # Encode unicode strings - if isinstance(out, unicode): - out = out.encode(response.charset) - # Byte Strings are just returned - if isinstance(out, bytes): - if "Content-Length" not in response: - response["Content-Length"] = len(out) - return [out] - # HTTPError or HTTPException (recursive, because they may wrap anything) - # TODO: Handle these explicitly in handle() or make them iterable. - if isinstance(out, HTTPError): - out.apply(response) - out = self.error_handler.get(out.status_code, self.default_error_handler)(out) - return self._cast(out) - if isinstance(out, HTTPResponse): - out.apply(response) - return self._cast(out.body) - - # File-like objects. - if hasattr(out, "read"): - if "wsgi.file_wrapper" in request.environ: - return request.environ["wsgi.file_wrapper"](out) - elif hasattr(out, "close") or not hasattr(out, "__iter__"): - return WSGIFileWrapper(out) - - # Handle Iterables. We peek into them to detect their inner type. - try: - iout = iter(out) - first = next(iout) - while not first: - first = next(iout) - except StopIteration: - return self._cast("") - except HTTPResponse as E: - first = E - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception as error: - if not self.catchall: - raise - first = HTTPError(500, "Unhandled exception", error, format_exc()) - - # These are the inner types allowed in iterator or generator objects. - if isinstance(first, HTTPResponse): - return self._cast(first) - elif isinstance(first, bytes): - new_iter = itertools.chain([first], iout) - elif isinstance(first, unicode): - encoder = lambda x: x.encode(response.charset) - new_iter = imap(encoder, itertools.chain([first], iout)) - else: - msg = "Unsupported response type: %s" % type(first) - return self._cast(HTTPError(500, msg)) - if hasattr(out, "close"): - new_iter = _closeiter(new_iter, out.close) - return new_iter - - def wsgi(self, environ, start_response): - """The bottle WSGI-interface.""" - try: - out = self._cast(self._handle(environ)) - # rfc2616 section 4.3 - if response._status_code in (100, 101, 204, 304) or environ["REQUEST_METHOD"] == "HEAD": - if hasattr(out, "close"): - out.close() - out = [] - exc_info = environ.get("bottle.exc_info") - if exc_info is not None: - del environ["bottle.exc_info"] - start_response(response._wsgi_status_line(), response.headerlist, exc_info) - return out - except (KeyboardInterrupt, SystemExit, MemoryError): - raise - except Exception as E: - if not self.catchall: - raise - err = "

Critical error while processing request: %s

" % html_escape( - environ.get("PATH_INFO", "/") - ) - if DEBUG: - err += ( - "

Error:

\n
\n%s\n
\n

Traceback:

\n
\n%s\n
\n" - % (html_escape(repr(E)), html_escape(format_exc())) - ) - environ["wsgi.errors"].write(err) - environ["wsgi.errors"].flush() - headers = [("Content-Type", "text/html; charset=UTF-8")] - start_response("500 INTERNAL SERVER ERROR", headers, sys.exc_info()) - return [tob(err)] - - def __call__(self, environ, start_response): - """Each instance of :class:'Bottle' is a WSGI application.""" - return self.wsgi(environ, start_response) - - def __enter__(self): - """Use this application as default for all module-level shortcuts.""" - default_app.push(self) - return self - - def __exit__(self, exc_type, exc_value, traceback): - default_app.pop() - - def __setattr__(self, name, value): - if name in self.__dict__: - raise AttributeError("Attribute %s already defined. Plugin conflict?" % name) - self.__dict__[name] = value - - -class BaseRequest: - """A wrapper for WSGI environment dictionaries that adds a lot of - convenient access methods and properties. Most of them are read-only. - - Adding new attributes to a request actually adds them to the environ - dictionary (as 'bottle.request.ext.'). This is the recommended - way to store and access request-specific data. - """ - - __slots__ = ("environ",) - - #: Maximum size of memory buffer for :attr:`body` in bytes. - MEMFILE_MAX = 102400 - - def __init__(self, environ=None): - """Wrap a WSGI environ dictionary.""" - #: The wrapped WSGI environ dictionary. This is the only real attribute. - #: All other attributes actually are read-only properties. - self.environ = {} if environ is None else environ - self.environ["bottle.request"] = self - - @DictProperty("environ", "bottle.app", read_only=True) - def app(self): - """Bottle application handling this request.""" - raise RuntimeError("This request is not connected to an application.") - - @DictProperty("environ", "bottle.route", read_only=True) - def route(self): - """The bottle :class:`Route` object that matches this request.""" - raise RuntimeError("This request is not connected to a route.") - - @DictProperty("environ", "route.url_args", read_only=True) - def url_args(self): - """The arguments extracted from the URL.""" - raise RuntimeError("This request is not connected to a route.") - - @property - def path(self): - """The value of ``PATH_INFO`` with exactly one prefixed slash (to fix - broken clients and avoid the "empty path" edge case).""" - return "/" + self.environ.get("PATH_INFO", "").lstrip("/") - - @property - def method(self): - """The ``REQUEST_METHOD`` value as an uppercase string.""" - return self.environ.get("REQUEST_METHOD", "GET").upper() - - @DictProperty("environ", "bottle.request.headers", read_only=True) - def headers(self): - """A :class:`WSGIHeaderDict` that provides case-insensitive access to - HTTP request headers.""" - return WSGIHeaderDict(self.environ) - - def get_header(self, name, default=None): - """Return the value of a request header, or a given default value.""" - return self.headers.get(name, default) - - @DictProperty("environ", "bottle.request.cookies", read_only=True) - def cookies(self): - """Cookies parsed into a :class:`FormsDict`. Signed cookies are NOT - decoded. Use :meth:`get_cookie` if you expect signed cookies.""" - cookies = SimpleCookie(self.environ.get("HTTP_COOKIE", "")).values() - return FormsDict((c.key, c.value) for c in cookies) - - def get_cookie(self, key, default=None, secret=None, digestmod=hashlib.sha256): - """Return the content of a cookie. To read a `Signed Cookie`, the - `secret` must match the one used to create the cookie (see - :meth:`BaseResponse.set_cookie`). If anything goes wrong (missing - cookie or wrong signature), return a default value.""" - value = self.cookies.get(key) - if secret: - # See BaseResponse.set_cookie for details on signed cookies. - if value and value.startswith("!") and "?" in value: - sig, msg = map(tob, value[1:].split("?", 1)) - hash = hmac.new(tob(secret), msg, digestmod=digestmod).digest() - if _lscmp(sig, base64.b64encode(hash)): - dst = pickle.loads(base64.b64decode(msg)) - if dst and dst[0] == key: - return dst[1] - return default - return value or default - - @DictProperty("environ", "bottle.request.query", read_only=True) - def query(self): - """The :attr:`query_string` parsed into a :class:`FormsDict`. These - values are sometimes called "URL arguments" or "GET parameters", but - not to be confused with "URL wildcards" as they are provided by the - :class:`Router`.""" - get = self.environ["bottle.get"] = FormsDict() - pairs = _parse_qsl(self.environ.get("QUERY_STRING", "")) - for key, value in pairs: - get[key] = value - return get - - @DictProperty("environ", "bottle.request.forms", read_only=True) - def forms(self): - """Form values parsed from an `url-encoded` or `multipart/form-data` - encoded POST or PUT request body. The result is returned as a - :class:`FormsDict`. All keys and values are strings. File uploads - are stored separately in :attr:`files`.""" - forms = FormsDict() - forms.recode_unicode = self.POST.recode_unicode - for name, item in self.POST.allitems(): - if not isinstance(item, FileUpload): - forms[name] = item - return forms - - @DictProperty("environ", "bottle.request.params", read_only=True) - def params(self): - """A :class:`FormsDict` with the combined values of :attr:`query` and - :attr:`forms`. File uploads are stored in :attr:`files`.""" - params = FormsDict() - for key, value in self.query.allitems(): - params[key] = value - for key, value in self.forms.allitems(): - params[key] = value - return params - - @DictProperty("environ", "bottle.request.files", read_only=True) - def files(self): - """File uploads parsed from `multipart/form-data` encoded POST or PUT - request body. The values are instances of :class:`FileUpload`. - - """ - files = FormsDict() - files.recode_unicode = self.POST.recode_unicode - for name, item in self.POST.allitems(): - if isinstance(item, FileUpload): - files[name] = item - return files - - @DictProperty("environ", "bottle.request.json", read_only=True) - def json(self): - """If the ``Content-Type`` header is ``application/json`` or - ``application/json-rpc``, this property holds the parsed content - of the request body. Only requests smaller than :attr:`MEMFILE_MAX` - are processed to avoid memory exhaustion. - Invalid JSON raises a 400 error response. - """ - ctype = self.environ.get("CONTENT_TYPE", "").lower().split(";")[0] - if ctype in ("application/json", "application/json-rpc"): - b = self._get_body_string(self.MEMFILE_MAX) - if not b: - return None - try: - return json_loads(b) - except (ValueError, TypeError): - raise HTTPError(400, "Invalid JSON") - return None - - def _iter_body(self, read, bufsize): - maxread = max(0, self.content_length) - while maxread: - part = read(min(maxread, bufsize)) - if not part: - break - yield part - maxread -= len(part) - - @staticmethod - def _iter_chunked(read, bufsize): - err = HTTPError(400, "Error while parsing chunked transfer body.") - rn, sem, bs = tob("\r\n"), tob(";"), tob("") - while True: - header = read(1) - while header[-2:] != rn: - c = read(1) - header += c - if not c: - raise err - if len(header) > bufsize: - raise err - size, _, _ = header.partition(sem) - try: - maxread = int(tonat(size.strip()), 16) - except ValueError: - raise err - if maxread == 0: - break - buff = bs - while maxread > 0: - if not buff: - buff = read(min(maxread, bufsize)) - part, buff = buff[:maxread], buff[maxread:] - if not part: - raise err - yield part - maxread -= len(part) - if read(2) != rn: - raise err - - @DictProperty("environ", "bottle.request.body", read_only=True) - def _body(self): - try: - read_func = self.environ["wsgi.input"].read - except KeyError: - self.environ["wsgi.input"] = BytesIO() - return self.environ["wsgi.input"] - body_iter = self._iter_chunked if self.chunked else self._iter_body - body, body_size, is_temp_file = BytesIO(), 0, False - for part in body_iter(read_func, self.MEMFILE_MAX): - body.write(part) - body_size += len(part) - if not is_temp_file and body_size > self.MEMFILE_MAX: - body, tmp = NamedTemporaryFile(mode="w+b"), body - body.write(tmp.getvalue()) - del tmp - is_temp_file = True - self.environ["wsgi.input"] = body - body.seek(0) - return body - - def _get_body_string(self, maxread): - """Read body into a string. Raise HTTPError(413) on requests that are - too large.""" - if self.content_length > maxread: - raise HTTPError(413, "Request entity too large") - data = self.body.read(maxread + 1) - if len(data) > maxread: - raise HTTPError(413, "Request entity too large") - return data - - @property - def body(self): - """The HTTP request body as a seek-able file-like object. Depending on - :attr:`MEMFILE_MAX`, this is either a temporary file or a - :class:`io.BytesIO` instance. Accessing this property for the first - time reads and replaces the ``wsgi.input`` environ variable. - Subsequent accesses just do a `seek(0)` on the file object.""" - self._body.seek(0) - return self._body - - @property - def chunked(self): - """True if Chunked transfer encoding was.""" - return "chunked" in self.environ.get("HTTP_TRANSFER_ENCODING", "").lower() - - #: An alias for :attr:`query`. - GET = query - - @DictProperty("environ", "bottle.request.post", read_only=True) - def POST(self): - """The values of :attr:`forms` and :attr:`files` combined into a single - :class:`FormsDict`. Values are either strings (form values) or - instances of :class:`cgi.FieldStorage` (file uploads). - """ - post = FormsDict() - # We default to application/x-www-form-urlencoded for everything that - # is not multipart and take the fast path (also: 3.1 workaround) - if not self.content_type.startswith("multipart/"): - body = tonat(self._get_body_string(self.MEMFILE_MAX), "latin1") - for key, value in _parse_qsl(body): - post[key] = value - return post - - safe_env = {"QUERY_STRING": ""} # Build a safe environment for cgi - for key in ("REQUEST_METHOD", "CONTENT_TYPE", "CONTENT_LENGTH"): - if key in self.environ: - safe_env[key] = self.environ[key] - args = dict(fp=self.body, environ=safe_env, keep_blank_values=True) - - if py3k: - args["encoding"] = "utf8" - post.recode_unicode = False - data = cgi.FieldStorage(**args) - self["_cgi.FieldStorage"] = data # http://bugs.python.org/issue18394 - data = data.list or [] - for item in data: - if item.filename is None: - post[item.name] = item.value - else: - post[item.name] = FileUpload(item.file, item.name, item.filename, item.headers) - return post - - @property - def url(self): - """The full request URI including hostname and scheme. If your app - lives behind a reverse proxy or load balancer and you get confusing - results, make sure that the ``X-Forwarded-Host`` header is set - correctly.""" - return self.urlparts.geturl() - - @DictProperty("environ", "bottle.request.urlparts", read_only=True) - def urlparts(self): - """The :attr:`url` string as an :class:`urlparse.SplitResult` tuple. - The tuple contains (scheme, host, path, query_string and fragment), - but the fragment is always empty because it is not visible to the - server.""" - env = self.environ - http = env.get("HTTP_X_FORWARDED_PROTO") or env.get("wsgi.url_scheme", "http") - host = env.get("HTTP_X_FORWARDED_HOST") or env.get("HTTP_HOST") - if not host: - # HTTP 1.1 requires a Host-header. This is for HTTP/1.0 clients. - host = env.get("SERVER_NAME", "127.0.0.1") - port = env.get("SERVER_PORT") - if port and port != ("80" if http == "http" else "443"): - host += ":" + port - path = urlquote(self.fullpath) - return UrlSplitResult(http, host, path, env.get("QUERY_STRING"), "") - - @property - def fullpath(self): - """Request path including :attr:`script_name` (if present).""" - return urljoin(self.script_name, self.path.lstrip("/")) - - @property - def query_string(self): - """The raw :attr:`query` part of the URL (everything in between ``?`` - and ``#``) as a string.""" - return self.environ.get("QUERY_STRING", "") - - @property - def script_name(self): - """The initial portion of the URL's `path` that was removed by a higher - level (server or routing middleware) before the application was - called. This script path is returned with leading and tailing - slashes.""" - script_name = self.environ.get("SCRIPT_NAME", "").strip("/") - return "/" + script_name + "/" if script_name else "/" - - def path_shift(self, shift=1): - """Shift path segments from :attr:`path` to :attr:`script_name` and - vice versa. - - :param shift: The number of path segments to shift. May be negative - to change the shift direction. (default: 1) - """ - script, path = path_shift(self.environ.get("SCRIPT_NAME", "/"), self.path, shift) - self["SCRIPT_NAME"], self["PATH_INFO"] = script, path - - @property - def content_length(self): - """The request body length as an integer. The client is responsible to - set this header. Otherwise, the real length of the body is unknown - and -1 is returned. In this case, :attr:`body` will be empty.""" - return int(self.environ.get("CONTENT_LENGTH") or -1) - - @property - def content_type(self): - """The Content-Type header as a lowercase-string (default: empty).""" - return self.environ.get("CONTENT_TYPE", "").lower() - - @property - def is_xhr(self): - """True if the request was triggered by a XMLHttpRequest. This only - works with JavaScript libraries that support the `X-Requested-With` - header (most of the popular libraries do).""" - requested_with = self.environ.get("HTTP_X_REQUESTED_WITH", "") - return requested_with.lower() == "xmlhttprequest" - - @property - def is_ajax(self): - """Alias for :attr:`is_xhr`. "Ajax" is not the right term.""" - return self.is_xhr - - @property - def auth(self): - """HTTP authentication data as a (user, password) tuple. This - implementation currently supports basic (not digest) authentication - only. If the authentication happened at a higher level (e.g. in the - front web-server or a middleware), the password field is None, but - the user field is looked up from the ``REMOTE_USER`` environ - variable. On any errors, None is returned.""" - basic = parse_auth(self.environ.get("HTTP_AUTHORIZATION", "")) - if basic: - return basic - ruser = self.environ.get("REMOTE_USER") - if ruser: - return (ruser, None) - return None - - @property - def remote_route(self): - """A list of all IPs that were involved in this request, starting with - the client IP and followed by zero or more proxies. This does only - work if all proxies support the ```X-Forwarded-For`` header. Note - that this information can be forged by malicious clients.""" - proxy = self.environ.get("HTTP_X_FORWARDED_FOR") - if proxy: - return [ip.strip() for ip in proxy.split(",")] - remote = self.environ.get("REMOTE_ADDR") - return [remote] if remote else [] - - @property - def remote_addr(self): - """The client IP as a string. Note that this information can be forged - by malicious clients.""" - route = self.remote_route - return route[0] if route else None - - def copy(self): - """Return a new :class:`Request` with a shallow :attr:`environ` copy.""" - return Request(self.environ.copy()) - - def get(self, value, default=None): - return self.environ.get(value, default) - - def __getitem__(self, key): - return self.environ[key] - - def __delitem__(self, key): - self[key] = "" - del self.environ[key] - - def __iter__(self): - return iter(self.environ) - - def __len__(self): - return len(self.environ) - - def keys(self): - return self.environ.keys() - - def __setitem__(self, key, value): - """Change an environ value and clear all caches that depend on it.""" - - if self.environ.get("bottle.request.readonly"): - raise KeyError("The environ dictionary is read-only.") - - self.environ[key] = value - todelete = () - - if key == "wsgi.input": - todelete = ("body", "forms", "files", "params", "post", "json") - elif key == "QUERY_STRING": - todelete = ("query", "params") - elif key.startswith("HTTP_"): - todelete = ("headers", "cookies") - - for key in todelete: - self.environ.pop("bottle.request." + key, None) - - def __repr__(self): - return f"<{self.__class__.__name__}: {self.method} {self.url}>" - - def __getattr__(self, name): - """Search in self.environ for additional user defined attributes.""" - try: - var = self.environ["bottle.request.ext.%s" % name] - return var.__get__(self) if hasattr(var, "__get__") else var - except KeyError: - raise AttributeError("Attribute %r not defined." % name) - - def __setattr__(self, name, value): - if name == "environ": - return object.__setattr__(self, name, value) - key = "bottle.request.ext.%s" % name - if hasattr(self, name): - raise AttributeError("Attribute already defined: %s" % name) - self.environ[key] = value - - def __delattr__(self, name): - try: - del self.environ["bottle.request.ext.%s" % name] - except KeyError: - raise AttributeError("Attribute not defined: %s" % name) - - -def _hkey(key): - if "\n" in key or "\r" in key or "\0" in key: - raise ValueError("Header names must not contain control characters: %r" % key) - return key.title().replace("_", "-") - - -def _hval(value): - value = tonat(value) - if "\n" in value or "\r" in value or "\0" in value: - raise ValueError("Header value must not contain control characters: %r" % value) - return value - - -class HeaderProperty: - def __init__(self, name, reader=None, writer=None, default=""): - self.name, self.default = name, default - self.reader, self.writer = reader, writer - self.__doc__ = "Current value of the %r header." % name.title() - - def __get__(self, obj, _): - if obj is None: - return self - value = obj.get_header(self.name, self.default) - return self.reader(value) if self.reader else value - - def __set__(self, obj, value): - obj[self.name] = self.writer(value) if self.writer else value - - def __delete__(self, obj): - del obj[self.name] - - -class BaseResponse: - """Storage class for a response body as well as headers and cookies. - - This class does support dict-like case-insensitive item-access to - headers, but is NOT a dict. Most notably, iterating over a response - yields parts of the body and not the headers. - - :param body: The response body as one of the supported types. - :param status: Either an HTTP status code (e.g. 200) or a status line - including the reason phrase (e.g. '200 OK'). - :param headers: A dictionary or a list of name-value pairs. - - Additional keyword arguments are added to the list of headers. - Underscores in the header name are replaced with dashes. - """ - - default_status = 200 - default_content_type = "text/html; charset=UTF-8" - - # Header denylist for specific response codes - # (rfc2616 section 10.2.3 and 10.3.5) - bad_headers = { - 204: frozenset(("Content-Type", "Content-Length")), - 304: frozenset( - ( - "Allow", - "Content-Encoding", - "Content-Language", - "Content-Length", - "Content-Range", - "Content-Type", - "Content-Md5", - "Last-Modified", - ) - ), - } - - def __init__(self, body="", status=None, headers=None, **more_headers): - self._cookies = None - self._headers = {} - self.body = body - self.status = status or self.default_status - if headers: - if isinstance(headers, dict): - headers = headers.items() - for name, value in headers: - self.add_header(name, value) - if more_headers: - for name, value in more_headers.items(): - self.add_header(name, value) - - def copy(self, cls=None): - """Returns a copy of self.""" - cls = cls or BaseResponse - assert issubclass(cls, BaseResponse) - copy = cls() - copy.status = self.status - copy._headers = {k: v[:] for (k, v) in self._headers.items()} - if self._cookies: - cookies = copy._cookies = SimpleCookie() - for k, v in self._cookies.items(): - cookies[k] = v.value - cookies[k].update(v) # also copy cookie attributes - return copy - - def __iter__(self): - return iter(self.body) - - def close(self): - if hasattr(self.body, "close"): - self.body.close() - - @property - def status_line(self): - """The HTTP status line as a string (e.g. ``404 Not Found``).""" - return self._status_line - - @property - def status_code(self): - """The HTTP status code as an integer (e.g. 404).""" - return self._status_code - - def _set_status(self, status): - if isinstance(status, int): - code, status = status, _HTTP_STATUS_LINES.get(status) - elif " " in status: - if "\n" in status or "\r" in status or "\0" in status: - raise ValueError("Status line must not include control chars.") - status = status.strip() - code = int(status.split()[0]) - else: - raise ValueError("String status line without a reason phrase.") - if not 100 <= code <= 999: - raise ValueError("Status code out of range.") - self._status_code = code - self._status_line = str(status or ("%d Unknown" % code)) - - def _get_status(self): - return self._status_line - - status = property( - _get_status, - _set_status, - None, - """ A writeable property to change the HTTP response status. It accepts - either a numeric code (100-999) or a string with a custom reason - phrase (e.g. "404 Brain not found"). Both :data:`status_line` and - :data:`status_code` are updated accordingly. The return value is - always a status string. """, - ) - del _get_status, _set_status - - @property - def headers(self): - """An instance of :class:`HeaderDict`, a case-insensitive dict-like - view on the response headers.""" - hdict = HeaderDict() - hdict.dict = self._headers - return hdict - - def __contains__(self, name): - return _hkey(name) in self._headers - - def __delitem__(self, name): - del self._headers[_hkey(name)] - - def __getitem__(self, name): - return self._headers[_hkey(name)][-1] - - def __setitem__(self, name, value): - self._headers[_hkey(name)] = [_hval(value)] - - def get_header(self, name, default=None): - """Return the value of a previously defined header. If there is no - header with that name, return a default value.""" - return self._headers.get(_hkey(name), [default])[-1] - - def set_header(self, name, value): - """Create a new response header, replacing any previously defined - headers with the same name.""" - self._headers[_hkey(name)] = [_hval(value)] - - def add_header(self, name, value): - """Add an additional response header, not removing duplicates.""" - self._headers.setdefault(_hkey(name), []).append(_hval(value)) - - def iter_headers(self): - """Yield (header, value) tuples, skipping headers that are not - allowed with the current response status code.""" - return self.headerlist - - def _wsgi_status_line(self): - """WSGI conform status line (latin1-encodeable)""" - if py3k: - return self._status_line.encode("utf8").decode("latin1") - return self._status_line - - @property - def headerlist(self): - """WSGI conform list of (header, value) tuples.""" - out = [] - headers = list(self._headers.items()) - if "Content-Type" not in self._headers: - headers.append(("Content-Type", [self.default_content_type])) - if self._status_code in self.bad_headers: - bad_headers = self.bad_headers[self._status_code] - headers = [h for h in headers if h[0] not in bad_headers] - out += [(name, val) for (name, vals) in headers for val in vals] - if self._cookies: - for c in self._cookies.values(): - out.append(("Set-Cookie", _hval(c.OutputString()))) - if py3k: - out = [(k, v.encode("utf8").decode("latin1")) for (k, v) in out] - return out - - content_type = HeaderProperty("Content-Type") - content_length = HeaderProperty("Content-Length", reader=int, default=-1) - expires = HeaderProperty( - "Expires", - reader=lambda x: datetime.utcfromtimestamp(parse_date(x)), - writer=lambda x: http_date(x), - ) - - @property - def charset(self, default="UTF-8"): - """Return the charset specified in the content-type header (default: utf8).""" - if "charset=" in self.content_type: - return self.content_type.split("charset=")[-1].split(";")[0].strip() - return default - - def set_cookie(self, name, value, secret=None, digestmod=hashlib.sha256, **options): - """Create a new cookie or replace an old one. If the `secret` parameter is - set, create a `Signed Cookie` (described below). - - :param name: the name of the cookie. - :param value: the value of the cookie. - :param secret: a signature key required for signed cookies. - - Additionally, this method accepts all RFC 2109 attributes that are - supported by :class:`cookie.Morsel`, including: - - :param maxage: maximum age in seconds. (default: None) - :param expires: a datetime object or UNIX timestamp. (default: None) - :param domain: the domain that is allowed to read the cookie. - (default: current domain) - :param path: limits the cookie to a given path (default: current path) - :param secure: limit the cookie to HTTPS connections (default: off). - :param httponly: prevents client-side javascript to read this cookie - (default: off, requires Python 2.6 or newer). - :param samesite: Control or disable third-party use for this cookie. - Possible values: `lax`, `strict` or `none` (default). - - If neither `expires` nor `maxage` is set (default), the cookie will - expire at the end of the browser session (as soon as the browser - window is closed). - - Signed cookies may store any pickle-able object and are - cryptographically signed to prevent manipulation. Keep in mind that - cookies are limited to 4kb in most browsers. - - Warning: Pickle is a potentially dangerous format. If an attacker - gains access to the secret key, he could forge cookies that execute - code on server side if unpickled. Using pickle is discouraged and - support for it will be removed in later versions of bottle. - - Warning: Signed cookies are not encrypted (the client can still see - the content) and not copy-protected (the client can restore an old - cookie). The main intention is to make pickling and unpickling - save, not to store secret information at client side. - """ - if not self._cookies: - self._cookies = SimpleCookie() - - # Monkey-patch Cookie lib to support 'SameSite' parameter - # https://tools.ietf.org/html/draft-west-first-party-cookies-07#section-4.1 - if py < (3, 8, 0): - Morsel._reserved.setdefault("samesite", "SameSite") - - if secret: - if not isinstance(value, basestring): - depr( - 0, - 13, - "Pickling of arbitrary objects into cookies is deprecated.", - "Only store strings in cookies. JSON strings are fine, too.", - ) - encoded = base64.b64encode(pickle.dumps([name, value], -1)) - sig = base64.b64encode(hmac.new(tob(secret), encoded, digestmod=digestmod).digest()) - value = touni(tob("!") + sig + tob("?") + encoded) - elif not isinstance(value, basestring): - raise TypeError("Secret key required for non-string cookies.") - - # Cookie size plus options must not exceed 4kb. - if len(name) + len(value) > 3800: - raise ValueError("Content does not fit into a cookie.") - - self._cookies[name] = value - - for key, value in options.items(): - if key in ("max_age", "maxage"): # 'maxage' variant added in 0.13 - key = "max-age" - if isinstance(value, timedelta): - value = value.seconds + value.days * 24 * 3600 - if key == "expires": - value = http_date(value) - if key in ("same_site", "samesite"): # 'samesite' variant added in 0.13 - key, value = "samesite", (value or "none").lower() - if value not in ("lax", "strict", "none"): - raise CookieError("Invalid value for SameSite") - if key in ("secure", "httponly") and not value: - continue - self._cookies[name][key] = value - - def delete_cookie(self, key, **kwargs): - """Delete a cookie. Be sure to use the same `domain` and `path` - settings as used to create the cookie.""" - kwargs["max_age"] = -1 - kwargs["expires"] = 0 - self.set_cookie(key, "", **kwargs) - - def __repr__(self): - out = "" - for name, value in self.headerlist: - out += f"{name.title()}: {value.strip()}\n" - return out - - -def _local_property(): - ls = threading.local() - - def fget(_): - try: - return ls.var - except AttributeError: - raise RuntimeError("Request context not initialized.") - - def fset(_, value): - ls.var = value - - def fdel(_): - del ls.var - - return property(fget, fset, fdel, "Thread-local property") - - -class LocalRequest(BaseRequest): - """A thread-local subclass of :class:`BaseRequest` with a different - set of attributes for each thread. There is usually only one global - instance of this class (:data:`request`). If accessed during a - request/response cycle, this instance always refers to the *current* - request (even on a multithreaded server).""" - - bind = BaseRequest.__init__ - environ = _local_property() - - -class LocalResponse(BaseResponse): - """A thread-local subclass of :class:`BaseResponse` with a different - set of attributes for each thread. There is usually only one global - instance of this class (:data:`response`). Its attributes are used - to build the HTTP response at the end of the request/response cycle. - """ - - bind = BaseResponse.__init__ - _status_line = _local_property() - _status_code = _local_property() - _cookies = _local_property() - _headers = _local_property() - body = _local_property() - - -Request = BaseRequest -Response = BaseResponse - - -class HTTPResponse(Response, BottleException): - def __init__(self, body="", status=None, headers=None, **more_headers): - super().__init__(body, status, headers, **more_headers) - - def apply(self, other): - other._status_code = self._status_code - other._status_line = self._status_line - other._headers = self._headers - other._cookies = self._cookies - other.body = self.body - - -class HTTPError(HTTPResponse): - default_status = 500 - - def __init__(self, status=None, body=None, exception=None, traceback=None, **more_headers): - self.exception = exception - self.traceback = traceback - super().__init__(body, status, **more_headers) - - -class PluginError(BottleException): - pass - - -class JSONPlugin: - name = "json" - api = 2 - - def __init__(self, json_dumps=json_dumps): - self.json_dumps = json_dumps - - def setup(self, app): - app.config._define( - "json.enable", - default=True, - validate=bool, - help="Enable or disable automatic dict->json filter.", - ) - app.config._define( - "json.ascii", - default=False, - validate=bool, - help="Use only 7-bit ASCII characters in output.", - ) - app.config._define( - "json.indent", - default=True, - validate=bool, - help="Add whitespace to make json more readable.", - ) - app.config._define( - "json.dump_func", - default=None, - help=( - "If defined, use this function to transform" - " dict into json. The other options no longer" - " apply." - ), - ) - - def apply(self, callback, route): - dumps = self.json_dumps - if not self.json_dumps: - return callback - - @functools.wraps(callback) - def wrapper(*a, **ka): - try: - rv = callback(*a, **ka) - except HTTPResponse as resp: - rv = resp - - if isinstance(rv, dict): - # Attempt to serialize, raises exception on failure - json_response = dumps(rv) - # Set content type only if serialization successful - response.content_type = "application/json" - return json_response - elif isinstance(rv, HTTPResponse) and isinstance(rv.body, dict): - rv.body = dumps(rv.body) - rv.content_type = "application/json" - return rv - - return wrapper - - -class TemplatePlugin: - """This plugin applies the :func:`view` decorator to all routes with a - `template` config parameter. If the parameter is a tuple, the second - element must be a dict with additional options (e.g. `template_engine`) - or default variables for the template.""" - - name = "template" - api = 2 - - def setup(self, app): - app.tpl = self - - def apply(self, callback, route): - conf = route.config.get("template") - if isinstance(conf, (tuple, list)) and len(conf) == 2: - return view(conf[0], **conf[1])(callback) - elif isinstance(conf, str): - return view(conf)(callback) - else: - return callback - - -#: Not a plugin, but part of the plugin API. TODO: Find a better place. -class _ImportRedirect: - def __init__(self, name, impmask): - """Create a virtual package that redirects imports (see PEP 302).""" - self.name = name - self.impmask = impmask - self.module = sys.modules.setdefault(name, new_module(name)) - self.module.__dict__.update( - {"__file__": __file__, "__path__": [], "__all__": [], "__loader__": self} - ) - sys.meta_path.append(self) - - def find_spec(self, fullname, path, target=None): - if "." not in fullname: - return - if fullname.rsplit(".", 1)[0] != self.name: - return - from importlib.util import spec_from_loader - - return spec_from_loader(fullname, self) - - def find_module(self, fullname, path=None): - if "." not in fullname: - return - if fullname.rsplit(".", 1)[0] != self.name: - return - return self - - def load_module(self, fullname): - if fullname in sys.modules: - return sys.modules[fullname] - modname = fullname.rsplit(".", 1)[1] - realname = self.impmask % modname - __import__(realname) - module = sys.modules[fullname] = sys.modules[realname] - setattr(self.module, modname, module) - module.__loader__ = self - return module - - -class MultiDict(DictMixin): - """This dict stores multiple values per key, but behaves exactly like a - normal dict in that it returns only the newest value for any given key. - There are special methods available to access the full list of values. - """ - - def __init__(self, *a, **k): - self.dict = {k: [v] for (k, v) in dict(*a, **k).items()} - - def __len__(self): - return len(self.dict) - - def __iter__(self): - return iter(self.dict) - - def __contains__(self, key): - return key in self.dict - - def __delitem__(self, key): - del self.dict[key] - - def __getitem__(self, key): - return self.dict[key][-1] - - def __setitem__(self, key, value): - self.append(key, value) - - def keys(self): - return self.dict.keys() - - if py3k: - - def values(self): - return (v[-1] for v in self.dict.values()) - - def items(self): - return ((k, v[-1]) for k, v in self.dict.items()) - - def allitems(self): - return ((k, v) for k, vl in self.dict.items() for v in vl) - - iterkeys = keys - itervalues = values - iteritems = items - iterallitems = allitems - - else: - - def values(self): - return [v[-1] for v in self.dict.values()] - - def items(self): - return [(k, v[-1]) for k, v in self.dict.items()] - - def iterkeys(self): - return self.dict.iterkeys() - - def itervalues(self): - return (v[-1] for v in self.dict.itervalues()) - - def iteritems(self): - return ((k, v[-1]) for k, v in self.dict.iteritems()) - - def iterallitems(self): - return ((k, v) for k, vl in self.dict.iteritems() for v in vl) - - def allitems(self): - return [(k, v) for k, vl in self.dict.iteritems() for v in vl] - - def get(self, key, default=None, index=-1, type=None): - """Return the most recent value for a key. - - :param default: The default value to be returned if the key is not - present or the type conversion fails. - :param index: An index for the list of available values. - :param type: If defined, this callable is used to cast the value - into a specific type. Exception are suppressed and result in - the default value to be returned. - """ - try: - val = self.dict[key][index] - return type(val) if type else val - except Exception: - pass - return default - - def append(self, key, value): - """Add a new value to the list of values for this key.""" - self.dict.setdefault(key, []).append(value) - - def replace(self, key, value): - """Replace the list of values with a single value.""" - self.dict[key] = [value] - - def getall(self, key): - """Return a (possibly empty) list of values for a key.""" - return self.dict.get(key) or [] - - #: Aliases for WTForms to mimic other multi-dict APIs (Django) - getone = get - getlist = getall - - -class FormsDict(MultiDict): - """This :class:`MultiDict` subclass is used to store request form data. - Additionally to the normal dict-like item access methods (which return - unmodified data as native strings), this container also supports - attribute-like access to its values. Attributes are automatically de- - or recoded to match :attr:`input_encoding` (default: 'utf8'). Missing - attributes default to an empty string.""" - - #: Encoding used for attribute values. - input_encoding = "utf8" - #: If true (default), unicode strings are first encoded with `latin1` - #: and then decoded to match :attr:`input_encoding`. - recode_unicode = True - - def _fix(self, s, encoding=None): - if isinstance(s, unicode) and self.recode_unicode: # Python 3 WSGI - return s.encode("latin1").decode(encoding or self.input_encoding) - elif isinstance(s, bytes): # Python 2 WSGI - return s.decode(encoding or self.input_encoding) - else: - return s - - def decode(self, encoding=None): - """Returns a copy with all keys and values de- or recoded to match - :attr:`input_encoding`. Some libraries (e.g. WTForms) want a - unicode dictionary.""" - copy = FormsDict() - enc = copy.input_encoding = encoding or self.input_encoding - copy.recode_unicode = False - for key, value in self.allitems(): - copy.append(self._fix(key, enc), self._fix(value, enc)) - return copy - - def getunicode(self, name, default=None, encoding=None): - """Return the value as a unicode string, or the default.""" - try: - return self._fix(self[name], encoding) - except (UnicodeError, KeyError): - return default - - def __getattr__(self, name, default=unicode()): - # Without this guard, pickle generates a cryptic TypeError: - if name.startswith("__") and name.endswith("__"): - return super().__getattr__(name) - return self.getunicode(name, default=default) - - -class HeaderDict(MultiDict): - """A case-insensitive version of :class:`MultiDict` that defaults to - replace the old value instead of appending it.""" - - def __init__(self, *a, **ka): - self.dict = {} - if a or ka: - self.update(*a, **ka) - - def __contains__(self, key): - return _hkey(key) in self.dict - - def __delitem__(self, key): - del self.dict[_hkey(key)] - - def __getitem__(self, key): - return self.dict[_hkey(key)][-1] - - def __setitem__(self, key, value): - self.dict[_hkey(key)] = [_hval(value)] - - def append(self, key, value): - self.dict.setdefault(_hkey(key), []).append(_hval(value)) - - def replace(self, key, value): - self.dict[_hkey(key)] = [_hval(value)] - - def getall(self, key): - return self.dict.get(_hkey(key)) or [] - - def get(self, key, default=None, index=-1): - return MultiDict.get(self, _hkey(key), default, index) - - def filter(self, names): - for name in (_hkey(n) for n in names): - if name in self.dict: - del self.dict[name] - - -class WSGIHeaderDict(DictMixin): - """This dict-like class wraps a WSGI environ dict and provides convenient - access to HTTP_* fields. Keys and values are native strings - (2.x bytes or 3.x unicode) and keys are case-insensitive. If the WSGI - environment contains non-native string values, these are de- or encoded - using a lossless 'latin1' character set. - - The API will remain stable even on changes to the relevant PEPs. - Currently PEP 333, 444 and 3333 are supported. (PEP 444 is the only one - that uses non-native strings.) - """ - - #: List of keys that do not have a ``HTTP_`` prefix. - cgikeys = ("CONTENT_TYPE", "CONTENT_LENGTH") - - def __init__(self, environ): - self.environ = environ - - def _ekey(self, key): - """Translate header field name to CGI/WSGI environ key.""" - key = key.replace("-", "_").upper() - if key in self.cgikeys: - return key - return "HTTP_" + key - - def raw(self, key, default=None): - """Return the header value as is (may be bytes or unicode).""" - return self.environ.get(self._ekey(key), default) - - def __getitem__(self, key): - val = self.environ[self._ekey(key)] - if py3k: - if isinstance(val, unicode): - val = val.encode("latin1").decode("utf8") - else: - val = val.decode("utf8") - return val - - def __setitem__(self, key, value): - raise TypeError("%s is read-only." % self.__class__) - - def __delitem__(self, key): - raise TypeError("%s is read-only." % self.__class__) - - def __iter__(self): - for key in self.environ: - if key[:5] == "HTTP_": - yield _hkey(key[5:]) - elif key in self.cgikeys: - yield _hkey(key) - - def keys(self): - return [x for x in self] - - def __len__(self): - return len(self.keys()) - - def __contains__(self, key): - return self._ekey(key) in self.environ - - -_UNSET = object() - - -class ConfigDict(dict): - """A dict-like configuration storage with additional support for - namespaces, validators, meta-data, overlays and more. - - This dict-like class is heavily optimized for read access. All read-only - methods as well as item access should be as fast as the built-in dict. - """ - - __slots__ = ( - "_meta", - "_change_listener", - "_overlays", - "_virtual_keys", - "_source", - "__weakref__", - ) - - def __init__(self): - self._meta = {} - self._change_listener = [] - #: Weak references of overlays that need to be kept in sync. - self._overlays = [] - #: Config that is the source for this overlay. - self._source = None - #: Keys of values copied from the source (values we do not own) - self._virtual_keys = set() - - def load_module(self, path, squash=True): - """Load values from a Python module. - - Example modue ``config.py``:: - - DEBUG = True - SQLITE = { - "db": ":memory:" - } - - - >>> c = ConfigDict() - >>> c.load_module('config') - {DEBUG: True, 'SQLITE.DB': 'memory'} - >>> c.load_module("config", False) - {'DEBUG': True, 'SQLITE': {'DB': 'memory'}} - - :param squash: If true (default), dictionary values are assumed to - represent namespaces (see :meth:`load_dict`). - """ - config_obj = load(path) - obj = {key: getattr(config_obj, key) for key in dir(config_obj) if key.isupper()} - - if squash: - self.load_dict(obj) - else: - self.update(obj) - return self - - def load_config(self, filename, **options): - """Load values from an ``*.ini`` style config file. - - A configuration file consists of sections, each led by a - ``[section]`` header, followed by key/value entries separated by - either ``=`` or ``:``. Section names and keys are case-insensitive. - Leading and trailing whitespace is removed from keys and values. - Values can be omitted, in which case the key/value delimiter may - also be left out. Values can also span multiple lines, as long as - they are indented deeper than the first line of the value. Commands - are prefixed by ``#`` or ``;`` and may only appear on their own on - an otherwise empty line. - - Both section and key names may contain dots (``.``) as namespace - separators. The actual configuration parameter name is constructed - by joining section name and key name together and converting to - lower case. - - The special sections ``bottle`` and ``ROOT`` refer to the root - namespace and the ``DEFAULT`` section defines default values for all - other sections. - - With Python 3, extended string interpolation is enabled. - - :param filename: The path of a config file, or a list of paths. - :param options: All keyword parameters are passed to the underlying - :class:`python:configparser.ConfigParser` constructor call. - - """ - options.setdefault("allow_no_value", True) - if py3k: - options.setdefault("interpolation", configparser.ExtendedInterpolation()) - conf = configparser.ConfigParser(**options) - conf.read(filename) - for section in conf.sections(): - for key in conf.options(section): - value = conf.get(section, key) - if section not in ("bottle", "ROOT"): - key = section + "." + key - self[key.lower()] = value - return self - - def load_dict(self, source, namespace=""): - """Load values from a dictionary structure. Nesting can be used to - represent namespaces. - - >>> c = ConfigDict() - >>> c.load_dict({'some': {'namespace': {'key': 'value'} } }) - {'some.namespace.key': 'value'} - """ - for key, value in source.items(): - if isinstance(key, basestring): - nskey = (namespace + "." + key).strip(".") - if isinstance(value, dict): - self.load_dict(value, namespace=nskey) - else: - self[nskey] = value - else: - raise TypeError("Key has type %r (not a string)" % type(key)) - return self - - def update(self, *a, **ka): - """If the first parameter is a string, all keys are prefixed with this - namespace. Apart from that it works just as the usual dict.update(). - - >>> c = ConfigDict() - >>> c.update('some.namespace', key='value') - """ - prefix = "" - if a and isinstance(a[0], basestring): - prefix = a[0].strip(".") + "." - a = a[1:] - for key, value in dict(*a, **ka).items(): - self[prefix + key] = value - - def setdefault(self, key, value): - if key not in self: - self[key] = value - return self[key] - - def __setitem__(self, key, value): - if not isinstance(key, basestring): - raise TypeError("Key has type %r (not a string)" % type(key)) - - self._virtual_keys.discard(key) - - value = self.meta_get(key, "filter", lambda x: x)(value) - if key in self and self[key] is value: - return - - self._on_change(key, value) - dict.__setitem__(self, key, value) - - for overlay in self._iter_overlays(): - overlay._set_virtual(key, value) - - def __delitem__(self, key): - if key not in self: - raise KeyError(key) - if key in self._virtual_keys: - raise KeyError("Virtual keys cannot be deleted: %s" % key) - - if self._source and key in self._source: - # Not virtual, but present in source -> Restore virtual value - dict.__delitem__(self, key) - self._set_virtual(key, self._source[key]) - else: # not virtual, not present in source. This is OUR value - self._on_change(key, None) - dict.__delitem__(self, key) - for overlay in self._iter_overlays(): - overlay._delete_virtual(key) - - def _set_virtual(self, key, value): - """Recursively set or update virtual keys. Do nothing if non-virtual - value is present.""" - if key in self and key not in self._virtual_keys: - return # Do nothing for non-virtual keys. - - self._virtual_keys.add(key) - if key in self and self[key] is not value: - self._on_change(key, value) - dict.__setitem__(self, key, value) - for overlay in self._iter_overlays(): - overlay._set_virtual(key, value) - - def _delete_virtual(self, key): - """Recursively delete virtual entry. Do nothing if key is not virtual.""" - if key not in self._virtual_keys: - return # Do nothing for non-virtual keys. - - if key in self: - self._on_change(key, None) - dict.__delitem__(self, key) - self._virtual_keys.discard(key) - for overlay in self._iter_overlays(): - overlay._delete_virtual(key) - - def _on_change(self, key, value): - for cb in self._change_listener: - if cb(self, key, value): - return True - - def _add_change_listener(self, func): - self._change_listener.append(func) - return func - - def meta_get(self, key, metafield, default=None): - """Return the value of a meta field for a key.""" - return self._meta.get(key, {}).get(metafield, default) - - def meta_set(self, key, metafield, value): - """Set the meta field for a key to a new value.""" - self._meta.setdefault(key, {})[metafield] = value - - def meta_list(self, key): - """Return an iterable of meta field names defined for a key.""" - return self._meta.get(key, {}).keys() - - def _define(self, key, default=_UNSET, help=_UNSET, validate=_UNSET): - """(Unstable) Shortcut for plugins to define own config parameters.""" - if default is not _UNSET: - self.setdefault(key, default) - if help is not _UNSET: - self.meta_set(key, "help", help) - if validate is not _UNSET: - self.meta_set(key, "validate", validate) - - def _iter_overlays(self): - for ref in self._overlays: - overlay = ref() - if overlay is not None: - yield overlay - - def _make_overlay(self): - """(Unstable) Create a new overlay that acts like a chained map: Values - missing in the overlay are copied from the source map. Both maps - share the same meta entries. - - Entries that were copied from the source are called 'virtual'. You - can not delete virtual keys, but overwrite them, which turns them - into non-virtual entries. Setting keys on an overlay never affects - its source, but may affect any number of child overlays. - - Other than collections.ChainMap or most other implementations, this - approach does not resolve missing keys on demand, but instead - actively copies all values from the source to the overlay and keeps - track of virtual and non-virtual keys internally. This removes any - lookup-overhead. Read-access is as fast as a build-in dict for both - virtual and non-virtual keys. - - Changes are propagated recursively and depth-first. A failing - on-change handler in an overlay stops the propagation of virtual - values and may result in an partly updated tree. Take extra care - here and make sure that on-change handlers never fail. - - Used by Route.config - """ - # Cleanup dead references - self._overlays[:] = [ref for ref in self._overlays if ref() is not None] - - overlay = ConfigDict() - overlay._meta = self._meta - overlay._source = self - self._overlays.append(weakref.ref(overlay)) - for key in self: - overlay._set_virtual(key, self[key]) - return overlay - - -class AppStack(list): - """A stack-like list. Calling it returns the head of the stack.""" - - def __call__(self): - """Return the current default application.""" - return self.default - - def push(self, value=None): - """Add a new :class:`Bottle` instance to the stack""" - if not isinstance(value, Bottle): - value = Bottle() - self.append(value) - return value - - new_app = push - - @property - def default(self): - try: - return self[-1] - except IndexError: - return self.push() - - -class WSGIFileWrapper: - def __init__(self, fp, buffer_size=1024 * 64): - self.fp, self.buffer_size = fp, buffer_size - for attr in "fileno", "close", "read", "readlines", "tell", "seek": - if hasattr(fp, attr): - setattr(self, attr, getattr(fp, attr)) - - def __iter__(self): - buff, read = self.buffer_size, self.read - part = read(buff) - while part: - yield part - part = read(buff) - - -class _closeiter: - """This only exists to be able to attach a .close method to iterators that - do not support attribute assignment (most of itertools).""" - - def __init__(self, iterator, close=None): - self.iterator = iterator - self.close_callbacks = makelist(close) - - def __iter__(self): - return iter(self.iterator) - - def close(self): - for func in self.close_callbacks: - func() - - -class ResourceManager: - """This class manages a list of search paths and helps to find and open - application-bound resources (files). - - :param base: default value for :meth:`add_path` calls. - :param opener: callable used to open resources. - :param cachemode: controls which lookups are cached. One of 'all', - 'found' or 'none'. - """ - - def __init__(self, base="./", opener=open, cachemode="all"): - self.opener = opener - self.base = base - self.cachemode = cachemode - - #: A list of search paths. See :meth:`add_path` for details. - self.path = [] - #: A cache for resolved paths. ``res.cache.clear()`` clears the cache. - self.cache = {} - - def add_path(self, path, base=None, index=None, create=False): - """Add a new path to the list of search paths. Return False if the - path does not exist. - - :param path: The new search path. Relative paths are turned into - an absolute and normalized form. If the path looks like a file - (not ending in `/`), the filename is stripped off. - :param base: Path used to absolutize relative search paths. - Defaults to :attr:`base` which defaults to ``os.getcwd()``. - :param index: Position within the list of search paths. Defaults - to last index (appends to the list). - - The `base` parameter makes it easy to reference files installed - along with a python module or package:: - - res.add_path('./resources/', __file__) - """ - base = os.path.abspath(os.path.dirname(base or self.base)) - path = os.path.abspath(os.path.join(base, os.path.dirname(path))) - path += os.sep - if path in self.path: - self.path.remove(path) - if create and not os.path.isdir(path): - os.makedirs(path) - if index is None: - self.path.append(path) - else: - self.path.insert(index, path) - self.cache.clear() - return os.path.exists(path) - - def __iter__(self): - """Iterate over all existing files in all registered paths.""" - search = self.path[:] - while search: - path = search.pop() - if not os.path.isdir(path): - continue - for name in os.listdir(path): - full = os.path.join(path, name) - if os.path.isdir(full): - search.append(full) - else: - yield full - - def lookup(self, name): - """Search for a resource and return an absolute file path, or `None`. - - The :attr:`path` list is searched in order. The first match is - returned. Symlinks are followed. The result is cached to speed up - future lookups.""" - if name not in self.cache or DEBUG: - for path in self.path: - fpath = os.path.join(path, name) - if os.path.isfile(fpath): - if self.cachemode in ("all", "found"): - self.cache[name] = fpath - return fpath - if self.cachemode == "all": - self.cache[name] = None - return self.cache[name] - - def open(self, name, mode="r", *args, **kwargs): - """Find a resource and return a file object, or raise IOError.""" - fname = self.lookup(name) - if not fname: - raise OSError("Resource %r not found." % name) - return self.opener(fname, mode=mode, *args, **kwargs) - - -class FileUpload: - def __init__(self, fileobj, name, filename, headers=None): - """Wrapper for file uploads.""" - #: Open file(-like) object (BytesIO buffer or temporary file) - self.file = fileobj - #: Name of the upload form field - self.name = name - #: Raw filename as sent by the client (may contain unsafe characters) - self.raw_filename = filename - #: A :class:`HeaderDict` with additional headers (e.g. content-type) - self.headers = HeaderDict(headers) if headers else HeaderDict() - - content_type = HeaderProperty("Content-Type") - content_length = HeaderProperty("Content-Length", reader=int, default=-1) - - def get_header(self, name, default=None): - """Return the value of a header within the multipart part.""" - return self.headers.get(name, default) - - @cached_property - def filename(self): - """Name of the file on the client file system, but normalized to ensure - file system compatibility. An empty filename is returned as 'empty'. - - Only ASCII letters, digits, dashes, underscores and dots are - allowed in the final filename. Accents are removed, if possible. - Whitespace is replaced by a single dash. Leading or tailing dots - or dashes are removed. The filename is limited to 255 characters. - """ - fname = self.raw_filename - if not isinstance(fname, unicode): - fname = fname.decode("utf8", "ignore") - fname = normalize("NFKD", fname) - fname = fname.encode("ASCII", "ignore").decode("ASCII") - fname = os.path.basename(fname.replace("\\", os.path.sep)) - fname = re.sub(r"[^a-zA-Z0-9-_.\s]", "", fname).strip() - fname = re.sub(r"[-\s]+", "-", fname).strip(".-") - return fname[:255] or "empty" - - def _copy_file(self, fp, chunk_size=2**16): - read, write, offset = self.file.read, fp.write, self.file.tell() - while 1: - buf = read(chunk_size) - if not buf: - break - write(buf) - self.file.seek(offset) - - def save(self, destination, overwrite=False, chunk_size=2**16): - """Save file to disk or copy its content to an open file(-like) object. - If *destination* is a directory, :attr:`filename` is added to the - path. Existing files are not overwritten by default (IOError). - - :param destination: File path, directory or file(-like) object. - :param overwrite: If True, replace existing files. (default: False) - :param chunk_size: Bytes to read at a time. (default: 64kb) - """ - if isinstance(destination, basestring): # Except file-likes here - if os.path.isdir(destination): - destination = os.path.join(destination, self.filename) - if not overwrite and os.path.exists(destination): - raise OSError("File exists.") - with open(destination, "wb") as fp: - self._copy_file(fp, chunk_size) - else: - self._copy_file(destination, chunk_size) - - -############################################################################### -# Application Helper ########################################################### -############################################################################### - - -def abort(code=500, text="Unknown Error."): - """Aborts execution and causes a HTTP error.""" - raise HTTPError(code, text) - - -def redirect(url, code=None): - """Aborts execution and causes a 303 or 302 redirect, depending on - the HTTP protocol version.""" - if not code: - code = 303 if request.get("SERVER_PROTOCOL") == "HTTP/1.1" else 302 - res = response.copy(cls=HTTPResponse) - res.status = code - res.body = "" - res.set_header("Location", urljoin(request.url, url)) - raise res - - -def _rangeiter(fp, offset, limit, bufsize=1024 * 1024): - """Yield chunks from a range in a file.""" - fp.seek(offset) - while limit > 0: - part = fp.read(min(limit, bufsize)) - if not part: - break - limit -= len(part) - yield part - - -def static_file( - filename, root, mimetype=True, download=False, charset="UTF-8", etag=None, headers=None -): - """Open a file in a safe way and return an instance of :exc:`HTTPResponse` - that can be sent back to the client. - - :param filename: Name or path of the file to send, relative to ``root``. - :param root: Root path for file lookups. Should be an absolute directory - path. - :param mimetype: Provide the content-type header (default: guess from - file extension) - :param download: If True, ask the browser to open a `Save as...` dialog - instead of opening the file with the associated program. You can - specify a custom filename as a string. If not specified, the - original filename is used (default: False). - :param charset: The charset for files with a ``text/*`` mime-type. - (default: UTF-8) - :param etag: Provide a pre-computed ETag header. If set to ``False``, - ETag handling is disabled. (default: auto-generate ETag header) - :param headers: Additional headers dict to add to the response. - - While checking user input is always a good idea, this function provides - additional protection against malicious ``filename`` parameters from - breaking out of the ``root`` directory and leaking sensitive information - to an attacker. - - Read-protected files or files outside of the ``root`` directory are - answered with ``403 Access Denied``. Missing files result in a - ``404 Not Found`` response. Conditional requests (``If-Modified-Since``, - ``If-None-Match``) are answered with ``304 Not Modified`` whenever - possible. ``HEAD`` and ``Range`` requests (used by download managers to - check or continue partial downloads) are also handled automatically. - - """ - - root = os.path.join(os.path.abspath(root), "") - filename = os.path.abspath(os.path.join(root, filename.strip("/\\"))) - headers = headers.copy() if headers else {} - - if not filename.startswith(root): - return HTTPError(403, "Access denied.") - if not os.path.exists(filename) or not os.path.isfile(filename): - return HTTPError(404, "File does not exist.") - if not os.access(filename, os.R_OK): - return HTTPError(403, "You do not have permission to access this file.") - - if mimetype is True: - if download and download is not True: - mimetype, encoding = mimetypes.guess_type(download) - else: - mimetype, encoding = mimetypes.guess_type(filename) - if encoding: - headers["Content-Encoding"] = encoding - - if mimetype: - if ( - (mimetype[:5] == "text/" or mimetype == "application/javascript") - and charset - and "charset" not in mimetype - ): - mimetype += "; charset=%s" % charset - headers["Content-Type"] = mimetype - - if download: - download = os.path.basename(filename if download is True else download) - headers["Content-Disposition"] = 'attachment; filename="%s"' % download - - stats = os.stat(filename) - headers["Content-Length"] = clen = stats.st_size - headers["Last-Modified"] = email.utils.formatdate(stats.st_mtime, usegmt=True) - headers["Date"] = email.utils.formatdate(time.time(), usegmt=True) - - getenv = request.environ.get - - if etag is None: - etag = "%d:%d:%d:%d:%s" % (stats.st_dev, stats.st_ino, stats.st_mtime, clen, filename) - etag = hashlib.sha1(tob(etag)).hexdigest() - - if etag: - headers["ETag"] = etag - check = getenv("HTTP_IF_NONE_MATCH") - if check and check == etag: - return HTTPResponse(status=304, **headers) - - ims = getenv("HTTP_IF_MODIFIED_SINCE") - if ims: - ims = parse_date(ims.split(";")[0].strip()) - if ims is not None and ims >= int(stats.st_mtime): - return HTTPResponse(status=304, **headers) - - body = "" if request.method == "HEAD" else open(filename, "rb") - - headers["Accept-Ranges"] = "bytes" - range_header = getenv("HTTP_RANGE") - if range_header: - ranges = list(parse_range_header(range_header, clen)) - if not ranges: - return HTTPError(416, "Requested Range Not Satisfiable") - offset, end = ranges[0] - rlen = end - offset - headers["Content-Range"] = "bytes %d-%d/%d" % (offset, end - 1, clen) - headers["Content-Length"] = str(rlen) - if body: - body = _closeiter(_rangeiter(body, offset, rlen), body.close) - return HTTPResponse(body, status=206, **headers) - return HTTPResponse(body, **headers) - - -def debug(mode=True): - """Change the debug level. - There is only one debug level supported at the moment.""" - global DEBUG - if mode: - warnings.simplefilter("default") - DEBUG = bool(mode) - - -def http_date(value): - if isinstance(value, basestring): - return value - if isinstance(value, datetime): - # aware datetime.datetime is converted to UTC time - # naive datetime.datetime is treated as UTC time - value = value.utctimetuple() - elif isinstance(value, datedate): - # datetime.date is naive, and is treated as UTC time - value = value.timetuple() - if not isinstance(value, (int, float)): - # convert struct_time in UTC to UNIX timestamp - value = calendar.timegm(value) - return email.utils.formatdate(value, usegmt=True) - - -def parse_date(ims): - """Parse rfc1123, rfc850 and asctime timestamps and return UTC epoch.""" - try: - ts = email.utils.parsedate_tz(ims) - return calendar.timegm(ts[:8] + (0,)) - (ts[9] or 0) - except (TypeError, ValueError, IndexError, OverflowError): - return None - - -def parse_auth(header): - """Parse rfc2617 HTTP authentication header string (basic) and return (user,pass) tuple or None""" - try: - method, data = header.split(None, 1) - if method.lower() == "basic": - user, pwd = touni(base64.b64decode(tob(data))).split(":", 1) - return user, pwd - except (KeyError, ValueError): - return None - - -def parse_range_header(header, maxlen=0): - """Yield (start, end) ranges parsed from a HTTP Range header. Skip - unsatisfiable ranges. The end index is non-inclusive.""" - if not header or header[:6] != "bytes=": - return - ranges = [r.split("-", 1) for r in header[6:].split(",") if "-" in r] - for start, end in ranges: - try: - if not start: # bytes=-100 -> last 100 bytes - start, end = max(0, maxlen - int(end)), maxlen - elif not end: # bytes=100- -> all but the first 99 bytes - start, end = int(start), maxlen - else: # bytes=100-200 -> bytes 100-200 (inclusive) - start, end = int(start), min(int(end) + 1, maxlen) - if 0 <= start < end <= maxlen: - yield start, end - except ValueError: - pass - - -#: Header tokenizer used by _parse_http_header() -_hsplit = re.compile('(?:(?:"((?:[^"\\\\]|\\\\.)*)")|([^;,=]+))([;,=]?)').findall - - -def _parse_http_header(h): - """Parses a typical multi-valued and parametrised HTTP header (e.g. Accept headers) and returns a list of values - and parameters. For non-standard or broken input, this implementation may return partial results. - :param h: A header string (e.g. ``text/html,text/plain;q=0.9,*/*;q=0.8``) - :return: List of (value, params) tuples. The second element is a (possibly empty) dict. - """ - values = [] - if '"' not in h: # INFO: Fast path without regexp (~2x faster) - for value in h.split(","): - parts = value.split(";") - values.append((parts[0].strip(), {})) - for attr in parts[1:]: - name, value = attr.split("=", 1) - values[-1][1][name.strip()] = value.strip() - else: - lop, key, attrs = ",", None, {} - for quoted, plain, tok in _hsplit(h): - value = plain.strip() if plain else quoted.replace('\\"', '"') - if lop == ",": - attrs = {} - values.append((value, attrs)) - elif lop == ";": - if tok == "=": - key = value - else: - attrs[value] = "" - elif lop == "=" and key: - attrs[key] = value - key = None - lop = tok - return values - - -def _parse_qsl(qs): - r = [] - for pair in qs.split("&"): - if not pair: - continue - nv = pair.split("=", 1) - if len(nv) != 2: - nv.append("") - key = urlunquote(nv[0].replace("+", " ")) - value = urlunquote(nv[1].replace("+", " ")) - r.append((key, value)) - return r - - -def _lscmp(a, b): - """Compares two strings in a cryptographically safe way: - Runtime is not affected by length of common prefix.""" - return not sum(0 if x == y else 1 for x, y in zip(a, b)) and len(a) == len(b) - - -def cookie_encode(data, key, digestmod=None): - """Encode and sign a pickle-able object. Return a (byte) string""" - depr(0, 13, "cookie_encode() will be removed soon.", "Do not use this API directly.") - digestmod = digestmod or hashlib.sha256 - msg = base64.b64encode(pickle.dumps(data, -1)) - sig = base64.b64encode(hmac.new(tob(key), msg, digestmod=digestmod).digest()) - return tob("!") + sig + tob("?") + msg - - -def cookie_decode(data, key, digestmod=None): - """Verify and decode an encoded string. Return an object or None.""" - depr(0, 13, "cookie_decode() will be removed soon.", "Do not use this API directly.") - data = tob(data) - if cookie_is_encoded(data): - sig, msg = data.split(tob("?"), 1) - digestmod = digestmod or hashlib.sha256 - hashed = hmac.new(tob(key), msg, digestmod=digestmod).digest() - if _lscmp(sig[1:], base64.b64encode(hashed)): - return pickle.loads(base64.b64decode(msg)) - return None - - -def cookie_is_encoded(data): - """Return True if the argument looks like a encoded cookie.""" - depr(0, 13, "cookie_is_encoded() will be removed soon.", "Do not use this API directly.") - return bool(data.startswith(tob("!")) and tob("?") in data) - - -def html_escape(string): - """Escape HTML special characters ``&<>`` and quotes ``'"``.""" - return ( - string.replace("&", "&") - .replace("<", "<") - .replace(">", ">") - .replace('"', """) - .replace("'", "'") - ) - - -def html_quote(string): - """Escape and quote a string to be used as an HTTP attribute.""" - return '"%s"' % html_escape(string).replace("\n", " ").replace("\r", " ").replace( - "\t", " " - ) - - -def yieldroutes(func): - """Return a generator for routes that match the signature (name, args) - of the func parameter. This may yield more than one route if the function - takes optional keyword arguments. The output is best described by example:: - - a() -> '/a' - b(x, y) -> '/b//' - c(x, y=5) -> '/c/' and '/c//' - d(x=5, y=6) -> '/d' and '/d/' and '/d//' - """ - path = "/" + func.__name__.replace("__", "/").lstrip("/") - spec = getargspec(func) - argc = len(spec[0]) - len(spec[3] or []) - path += ("/<%s>" * argc) % tuple(spec[0][:argc]) - yield path - for arg in spec[0][argc:]: - path += "/<%s>" % arg - yield path - - -def path_shift(script_name, path_info, shift=1): - """Shift path fragments from PATH_INFO to SCRIPT_NAME and vice versa. - - :return: The modified paths. - :param script_name: The SCRIPT_NAME path. - :param script_name: The PATH_INFO path. - :param shift: The number of path fragments to shift. May be negative to - change the shift direction. (default: 1) - """ - if shift == 0: - return script_name, path_info - pathlist = path_info.strip("/").split("/") - scriptlist = script_name.strip("/").split("/") - if pathlist and pathlist[0] == "": - pathlist = [] - if scriptlist and scriptlist[0] == "": - scriptlist = [] - if 0 < shift <= len(pathlist): - moved = pathlist[:shift] - scriptlist = scriptlist + moved - pathlist = pathlist[shift:] - elif 0 > shift >= -len(scriptlist): - moved = scriptlist[shift:] - pathlist = moved + pathlist - scriptlist = scriptlist[:shift] - else: - empty = "SCRIPT_NAME" if shift < 0 else "PATH_INFO" - raise AssertionError("Cannot shift. Nothing left from %s" % empty) - new_script_name = "/" + "/".join(scriptlist) - new_path_info = "/" + "/".join(pathlist) - if path_info.endswith("/") and pathlist: - new_path_info += "/" - return new_script_name, new_path_info - - -def auth_basic(check, realm="private", text="Access denied"): - """Callback decorator to require HTTP auth (basic). - TODO: Add route(check_auth=...) parameter.""" - - def decorator(func): - @functools.wraps(func) - def wrapper(*a, **ka): - user, password = request.auth or (None, None) - if user is None or not check(user, password): - err = HTTPError(401, text) - err.add_header("WWW-Authenticate", 'Basic realm="%s"' % realm) - return err - return func(*a, **ka) - - return wrapper - - return decorator - - -def make_default_app_wrapper(name): - """Return a callable that relays calls to the current default app.""" - - @functools.wraps(getattr(Bottle, name)) - def wrapper(*a, **ka): - return getattr(app(), name)(*a, **ka) - - return wrapper - - -route = make_default_app_wrapper("route") -get = make_default_app_wrapper("get") -post = make_default_app_wrapper("post") -put = make_default_app_wrapper("put") -delete = make_default_app_wrapper("delete") -patch = make_default_app_wrapper("patch") -error = make_default_app_wrapper("error") -mount = make_default_app_wrapper("mount") -hook = make_default_app_wrapper("hook") -install = make_default_app_wrapper("install") -uninstall = make_default_app_wrapper("uninstall") -url = make_default_app_wrapper("get_url") - - -class ServerAdapter: - quiet = False - - def __init__(self, host="127.0.0.1", port=8080, **options): - self.options = options - self.host = host - self.port = int(port) - - def run(self, handler): # pragma: no cover - pass - - def __repr__(self): - args = ", ".join(f"{k}={repr(v)}" for k, v in self.options.items()) - return f"{self.__class__.__name__}({args})" - - -class CGIServer(ServerAdapter): - quiet = True - - def run(self, handler): # pragma: no cover - from wsgiref.handlers import CGIHandler - - def fixed_environ(environ, start_response): - environ.setdefault("PATH_INFO", "") - return handler(environ, start_response) - - CGIHandler().run(fixed_environ) - - -class FlupFCGIServer(ServerAdapter): - def run(self, handler): # pragma: no cover - import flup.server.fcgi - - self.options.setdefault("bindAddress", (self.host, self.port)) - flup.server.fcgi.WSGIServer(handler, **self.options).run() - - -class WSGIRefServer(ServerAdapter): - def run(self, app): # pragma: no cover - import socket - from wsgiref.simple_server import WSGIRequestHandler, WSGIServer, make_server - - class FixedHandler(WSGIRequestHandler): - def address_string(self): # Prevent reverse DNS lookups please. - return self.client_address[0] - - def log_request(*args, **kw): - if not self.quiet: - return WSGIRequestHandler.log_request(*args, **kw) - - handler_cls = self.options.get("handler_class", FixedHandler) - server_cls = self.options.get("server_class", WSGIServer) - - if ":" in self.host: # Fix wsgiref for IPv6 addresses. - if getattr(server_cls, "address_family") == socket.AF_INET: - - class server_cls(server_cls): - address_family = socket.AF_INET6 - - self.srv = make_server(self.host, self.port, app, server_cls, handler_cls) - self.port = self.srv.server_port # update port actual port (0 means random) - try: - self.srv.serve_forever() - except KeyboardInterrupt: - self.srv.server_close() # Prevent ResourceWarning: unclosed socket - raise - - -class CherryPyServer(ServerAdapter): - def run(self, handler): # pragma: no cover - depr( - 0, - 13, - "The wsgi server part of cherrypy was split into a new project called 'cheroot'.", - "Use the 'cheroot' server adapter instead of cherrypy.", - ) - from cherrypy import wsgiserver # This will fail for CherryPy >= 9 - - self.options["bind_addr"] = (self.host, self.port) - self.options["wsgi_app"] = handler - - certfile = self.options.get("certfile") - if certfile: - del self.options["certfile"] - keyfile = self.options.get("keyfile") - if keyfile: - del self.options["keyfile"] - - server = wsgiserver.CherryPyWSGIServer(**self.options) - if certfile: - server.ssl_certificate = certfile - if keyfile: - server.ssl_private_key = keyfile - - try: - server.start() - finally: - server.stop() - - -class CherootServer(ServerAdapter): - def run(self, handler): # pragma: no cover - from cheroot import wsgi - from cheroot.ssl import builtin - - self.options["bind_addr"] = (self.host, self.port) - self.options["wsgi_app"] = handler - certfile = self.options.pop("certfile", None) - keyfile = self.options.pop("keyfile", None) - chainfile = self.options.pop("chainfile", None) - server = wsgi.Server(**self.options) - if certfile and keyfile: - server.ssl_adapter = builtin.BuiltinSSLAdapter(certfile, keyfile, chainfile) - try: - server.start() - finally: - server.stop() - - -class WaitressServer(ServerAdapter): - def run(self, handler): - from waitress import serve - - serve(handler, host=self.host, port=self.port, _quiet=self.quiet, **self.options) - - -class PasteServer(ServerAdapter): - def run(self, handler): # pragma: no cover - from paste import httpserver - from paste.translogger import TransLogger - - handler = TransLogger(handler, setup_console_handler=(not self.quiet)) - httpserver.serve(handler, host=self.host, port=str(self.port), **self.options) - - -class MeinheldServer(ServerAdapter): - def run(self, handler): - from meinheld import server - - server.listen((self.host, self.port)) - server.run(handler) - - -class FapwsServer(ServerAdapter): - """Extremely fast webserver using libev. See https://github.com/william-os4y/fapws3""" - - def run(self, handler): # pragma: no cover - depr(0, 13, "fapws3 is not maintained and support will be dropped.") - import fapws._evwsgi as evwsgi - from fapws import base, config - - port = self.port - if float(config.SERVER_IDENT[-2:]) > 0.4: - # fapws3 silently changed its API in 0.5 - port = str(port) - evwsgi.start(self.host, port) - # fapws3 never releases the GIL. Complain upstream. I tried. No luck. - if "BOTTLE_CHILD" in os.environ and not self.quiet: - _stderr("WARNING: Auto-reloading does not work with Fapws3.") - _stderr(" (Fapws3 breaks python thread support)") - evwsgi.set_base_module(base) - - def app(environ, start_response): - environ["wsgi.multiprocess"] = False - return handler(environ, start_response) - - evwsgi.wsgi_cb(("", app)) - evwsgi.run() - - -class TornadoServer(ServerAdapter): - """The super hyped asynchronous server by facebook. Untested.""" - - def run(self, handler): # pragma: no cover - import tornado.httpserver - import tornado.ioloop - import tornado.wsgi - - container = tornado.wsgi.WSGIContainer(handler) - server = tornado.httpserver.HTTPServer(container) - server.listen(port=self.port, address=self.host) - tornado.ioloop.IOLoop.instance().start() - - -class AppEngineServer(ServerAdapter): - """Adapter for Google App Engine.""" - - quiet = True - - def run(self, handler): - depr( - 0, - 13, - "AppEngineServer no longer required", - "Configure your application directly in your app.yaml", - ) - from google.appengine.ext.webapp import util - - # A main() function in the handler script enables 'App Caching'. - # Lets makes sure it is there. This _really_ improves performance. - module = sys.modules.get("__main__") - if module and not hasattr(module, "main"): - module.main = lambda: util.run_wsgi_app(handler) - util.run_wsgi_app(handler) - - -class TwistedServer(ServerAdapter): - """Untested.""" - - def run(self, handler): - from twisted.internet import reactor - from twisted.python.threadpool import ThreadPool - from twisted.web import server, wsgi - - thread_pool = ThreadPool() - thread_pool.start() - reactor.addSystemEventTrigger("after", "shutdown", thread_pool.stop) - factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler)) - reactor.listenTCP(self.port, factory, interface=self.host) - if not reactor.running: - reactor.run() - - -class DieselServer(ServerAdapter): - """Untested.""" - - def run(self, handler): - depr(0, 13, "Diesel is not tested or supported and will be removed.") - from diesel.protocols.wsgi import WSGIApplication - - app = WSGIApplication(handler, port=self.port) - app.run() - - -class GeventServer(ServerAdapter): - """Untested. Options: - - * See gevent.wsgi.WSGIServer() documentation for more options. - """ - - def run(self, handler): - from gevent import local, pywsgi - - if not isinstance(threading.local(), local.local): - msg = "Bottle requires gevent.monkey.patch_all() (before import)" - raise RuntimeError(msg) - if self.quiet: - self.options["log"] = None - address = (self.host, self.port) - server = pywsgi.WSGIServer(address, handler, **self.options) - if "BOTTLE_CHILD" in os.environ: - import signal - - signal.signal(signal.SIGINT, lambda s, f: server.stop()) - server.serve_forever() - - -class GunicornServer(ServerAdapter): - """Untested. See http://gunicorn.org/configure.html for options.""" - - def run(self, handler): - from gunicorn.app.base import BaseApplication - - if self.host.startswith("unix:"): - config = {"bind": self.host} - else: - config = {"bind": "%s:%d" % (self.host, self.port)} - - config.update(self.options) - - class GunicornApplication(BaseApplication): - def load_config(self): - for key, value in config.items(): - self.cfg.set(key, value) - - def load(self): - return handler - - GunicornApplication().run() - - -class EventletServer(ServerAdapter): - """Untested. Options: - - * `backlog` adjust the eventlet backlog parameter which is the maximum - number of queued connections. Should be at least 1; the maximum - value is system-dependent. - * `family`: (default is 2) socket family, optional. See socket - documentation for available families. - """ - - def run(self, handler): - from eventlet import listen, patcher, wsgi - - if not patcher.is_monkey_patched(os): - msg = "Bottle requires eventlet.monkey_patch() (before import)" - raise RuntimeError(msg) - socket_args = {} - for arg in ("backlog", "family"): - try: - socket_args[arg] = self.options.pop(arg) - except KeyError: - pass - address = (self.host, self.port) - try: - wsgi.server(listen(address, **socket_args), handler, log_output=(not self.quiet)) - except TypeError: - # Fallback, if we have old version of eventlet - wsgi.server(listen(address), handler) - - -class BjoernServer(ServerAdapter): - """Fast server written in C: https://github.com/jonashaag/bjoern""" - - def run(self, handler): - from bjoern import run - - run(handler, self.host, self.port, reuse_port=True) - - -class AsyncioServerAdapter(ServerAdapter): - """Extend ServerAdapter for adding custom event loop""" - - def get_event_loop(self): - pass - - -class AiohttpServer(AsyncioServerAdapter): - """Asynchronous HTTP client/server framework for asyncio - https://pypi.python.org/pypi/aiohttp/ - https://pypi.org/project/aiohttp-wsgi/ - """ - - def get_event_loop(self): - import asyncio - - return asyncio.new_event_loop() - - def run(self, handler): - import asyncio - - from aiohttp_wsgi.wsgi import serve - - self.loop = self.get_event_loop() - asyncio.set_event_loop(self.loop) - - if "BOTTLE_CHILD" in os.environ: - import signal - - signal.signal(signal.SIGINT, lambda s, f: self.loop.stop()) - - serve(handler, host=self.host, port=self.port) - - -class AiohttpUVLoopServer(AiohttpServer): - """uvloop - https://github.com/MagicStack/uvloop - """ - - def get_event_loop(self): - import uvloop - - return uvloop.new_event_loop() - - -class AutoServer(ServerAdapter): - """Untested.""" - - adapters = [ - WaitressServer, - PasteServer, - TwistedServer, - CherryPyServer, - CherootServer, - WSGIRefServer, - ] - - def run(self, handler): - for sa in self.adapters: - try: - return sa(self.host, self.port, **self.options).run(handler) - except ImportError: - pass - - -server_names = { - "cgi": CGIServer, - "flup": FlupFCGIServer, - "wsgiref": WSGIRefServer, - "waitress": WaitressServer, - "cherrypy": CherryPyServer, - "cheroot": CherootServer, - "paste": PasteServer, - "fapws3": FapwsServer, - "tornado": TornadoServer, - "gae": AppEngineServer, - "twisted": TwistedServer, - "diesel": DieselServer, - "meinheld": MeinheldServer, - "gunicorn": GunicornServer, - "eventlet": EventletServer, - "gevent": GeventServer, - "bjoern": BjoernServer, - "aiohttp": AiohttpServer, - "uvloop": AiohttpUVLoopServer, - "auto": AutoServer, -} - - -def load(target, **namespace): - """Import a module or fetch an object from a module. - - * ``package.module`` returns `module` as a module object. - * ``pack.mod:name`` returns the module variable `name` from `pack.mod`. - * ``pack.mod:func()`` calls `pack.mod.func()` and returns the result. - - The last form accepts not only function calls, but any type of - expression. Keyword arguments passed to this function are available as - local variables. Example: ``import_string('re:compile(x)', x='[a-z]')`` - """ - module, target = target.split(":", 1) if ":" in target else (target, None) - if module not in sys.modules: - __import__(module) - if not target: - return sys.modules[module] - if target.isalnum(): - return getattr(sys.modules[module], target) - package_name = module.split(".")[0] - namespace[package_name] = sys.modules[package_name] - return eval(f"{module}.{target}", namespace) - - -def load_app(target): - """Load a bottle application from a module and make sure that the import - does not affect the current default application, but returns a separate - application object. See :func:`load` for the target parameter.""" - global NORUN - NORUN, nr_old = True, NORUN - tmp = default_app.push() # Create a new "default application" - try: - rv = load(target) # Import the target module - return rv if callable(rv) else tmp - finally: - default_app.remove(tmp) # Remove the temporary added default application - NORUN = nr_old - - -_debug = debug - - -def run( - app=None, - server="wsgiref", - host="127.0.0.1", - port=8080, - interval=1, - reloader=False, - quiet=False, - plugins=None, - debug=None, - config=None, - **kargs, -): - """Start a server instance. This method blocks until the server terminates. - - :param app: WSGI application or target string supported by - :func:`load_app`. (default: :func:`default_app`) - :param server: Server adapter to use. See :data:`server_names` keys - for valid names or pass a :class:`ServerAdapter` subclass. - (default: `wsgiref`) - :param host: Server address to bind to. Pass ``0.0.0.0`` to listens on - all interfaces including the external one. (default: 127.0.0.1) - :param port: Server port to bind to. Values below 1024 require root - privileges. (default: 8080) - :param reloader: Start auto-reloading server? (default: False) - :param interval: Auto-reloader interval in seconds (default: 1) - :param quiet: Suppress output to stdout and stderr? (default: False) - :param options: Options passed to the server adapter. - """ - if NORUN: - return - if reloader and not os.environ.get("BOTTLE_CHILD"): - import subprocess - - fd, lockfile = tempfile.mkstemp(prefix="bottle.", suffix=".lock") - environ = os.environ.copy() - environ["BOTTLE_CHILD"] = "true" - environ["BOTTLE_LOCKFILE"] = lockfile - args = [sys.executable] + sys.argv - # If a package was loaded with `python -m`, then `sys.argv` needs to be - # restored to the original value, or imports might break. See #1336 - if getattr(sys.modules.get("__main__"), "__package__", None): - args[1:1] = ["-m", sys.modules["__main__"].__package__] - - try: - os.close(fd) # We never write to this file - while os.path.exists(lockfile): - p = subprocess.Popen(args, env=environ) - while p.poll() is None: - os.utime(lockfile, None) # Tell child we are still alive - time.sleep(interval) - if p.returncode == 3: # Child wants to be restarted - continue - sys.exit(p.returncode) - except KeyboardInterrupt: - pass - finally: - if os.path.exists(lockfile): - os.unlink(lockfile) - return - - try: - if debug is not None: - _debug(debug) - app = app or default_app() - if isinstance(app, basestring): - app = load_app(app) - if not callable(app): - raise ValueError("Application is not callable: %r" % app) - - for plugin in plugins or []: - if isinstance(plugin, basestring): - plugin = load(plugin) - app.install(plugin) - - if config: - app.config.update(config) - - if server in server_names: - server = server_names.get(server) - if isinstance(server, basestring): - server = load(server) - if isinstance(server, type): - server = server(host=host, port=port, **kargs) - if not isinstance(server, ServerAdapter): - raise ValueError("Unknown or unsupported server: %r" % server) - - server.quiet = server.quiet or quiet - if not server.quiet: - _stderr( - "Bottle (dbt v{}) server starting up (using {})...".format( - __version__, repr(server) - ) - ) - if server.host.startswith("unix:"): - _stderr("Listening on %s" % server.host) - else: - _stderr("Listening on http://%s:%d/" % (server.host, server.port)) - _stderr("Hit Ctrl-C to quit.\n") - - if reloader: - lockfile = os.environ.get("BOTTLE_LOCKFILE") - bgcheck = FileCheckerThread(lockfile, interval) - with bgcheck: - server.run(app) - if bgcheck.status == "reload": - sys.exit(3) - else: - server.run(app) - except KeyboardInterrupt: - pass - except (SystemExit, MemoryError): - raise - except: - if not reloader: - raise - if not getattr(server, "quiet", quiet): - print_exc() - time.sleep(interval) - sys.exit(3) - - -class FileCheckerThread(threading.Thread): - """Interrupt main-thread as soon as a changed module file is detected, - the lockfile gets deleted or gets too old.""" - - def __init__(self, lockfile, interval): - threading.Thread.__init__(self) - self.daemon = True - self.lockfile, self.interval = lockfile, interval - #: Is one of 'reload', 'error' or 'exit' - self.status = None - - def run(self): - exists = os.path.exists - mtime = lambda p: os.stat(p).st_mtime - files = dict() - - for module in list(sys.modules.values()): - path = getattr(module, "__file__", "") or "" - if path[-4:] in (".pyo", ".pyc"): - path = path[:-1] - if path and exists(path): - files[path] = mtime(path) - - while not self.status: - if not exists(self.lockfile) or mtime(self.lockfile) < time.time() - self.interval - 5: - self.status = "error" - thread.interrupt_main() - for path, lmtime in list(files.items()): - if not exists(path) or mtime(path) > lmtime: - self.status = "reload" - thread.interrupt_main() - break - time.sleep(self.interval) - - def __enter__(self): - self.start() - - def __exit__(self, exc_type, *_): - if not self.status: - self.status = "exit" # silent exit - self.join() - return exc_type is not None and issubclass(exc_type, KeyboardInterrupt) - - -class TemplateError(BottleException): - pass - - -class BaseTemplate: - """Base class and minimal API for template adapters""" - - extensions = ["tpl", "html", "thtml", "stpl"] - settings = {} # used in prepare() - defaults = {} # used in render() - - def __init__(self, source=None, name=None, lookup=None, encoding="utf8", **settings): - """Create a new template. - If the source parameter (str or buffer) is missing, the name argument - is used to guess a template filename. Subclasses can assume that - self.source and/or self.filename are set. Both are strings. - The lookup, encoding and settings parameters are stored as instance - variables. - The lookup parameter stores a list containing directory paths. - The encoding parameter should be used to decode byte strings or files. - The settings parameter contains a dict for engine-specific settings. - """ - self.name = name - self.source = source.read() if hasattr(source, "read") else source - self.filename = source.filename if hasattr(source, "filename") else None - self.lookup = [os.path.abspath(x) for x in lookup] if lookup else [] - self.encoding = encoding - self.settings = self.settings.copy() # Copy from class variable - self.settings.update(settings) # Apply - if not self.source and self.name: - self.filename = self.search(self.name, self.lookup) - if not self.filename: - raise TemplateError("Template %s not found." % repr(name)) - if not self.source and not self.filename: - raise TemplateError("No template specified.") - self.prepare(**self.settings) - - @classmethod - def search(cls, name, lookup=None): - """Search name in all directories specified in lookup. - First without, then with common extensions. Return first hit.""" - if not lookup: - raise depr(0, 12, "Empty template lookup path.", "Configure a template lookup path.") - - if os.path.isabs(name): - raise depr( - 0, - 12, - "Use of absolute path for template name.", - "Refer to templates with names or paths relative to the lookup path.", - ) - - for spath in lookup: - spath = os.path.abspath(spath) + os.sep - fname = os.path.abspath(os.path.join(spath, name)) - if not fname.startswith(spath): - continue - if os.path.isfile(fname): - return fname - for ext in cls.extensions: - if os.path.isfile(f"{fname}.{ext}"): - return f"{fname}.{ext}" - - @classmethod - def global_config(cls, key, *args): - """This reads or sets the global settings stored in class.settings.""" - if args: - cls.settings = cls.settings.copy() # Make settings local to class - cls.settings[key] = args[0] - else: - return cls.settings[key] - - def prepare(self, **options): - """Run preparations (parsing, caching, ...). - It should be possible to call this again to refresh a template or to - update settings. - """ - raise NotImplementedError - - def render(self, *args, **kwargs): - """Render the template with the specified local variables and return - a single byte or unicode string. If it is a byte string, the encoding - must match self.encoding. This method must be thread-safe! - Local variables may be provided in dictionaries (args) - or directly, as keywords (kwargs). - """ - raise NotImplementedError - - -class MakoTemplate(BaseTemplate): - def prepare(self, **options): - from mako.lookup import TemplateLookup - from mako.template import Template - - options.update({"input_encoding": self.encoding}) - options.setdefault("format_exceptions", bool(DEBUG)) - lookup = TemplateLookup(directories=self.lookup, **options) - if self.source: - self.tpl = Template(self.source, lookup=lookup, **options) - else: - self.tpl = Template(uri=self.name, filename=self.filename, lookup=lookup, **options) - - def render(self, *args, **kwargs): - for dictarg in args: - kwargs.update(dictarg) - _defaults = self.defaults.copy() - _defaults.update(kwargs) - return self.tpl.render(**_defaults) - - -class CheetahTemplate(BaseTemplate): - def prepare(self, **options): - from Cheetah.Template import Template - - self.context = threading.local() - self.context.vars = {} - options["searchList"] = [self.context.vars] - if self.source: - self.tpl = Template(source=self.source, **options) - else: - self.tpl = Template(file=self.filename, **options) - - def render(self, *args, **kwargs): - for dictarg in args: - kwargs.update(dictarg) - self.context.vars.update(self.defaults) - self.context.vars.update(kwargs) - out = str(self.tpl) - self.context.vars.clear() - return out - - -class Jinja2Template(BaseTemplate): - def prepare(self, filters=None, tests=None, globals={}, **kwargs): - from jinja2 import Environment, FunctionLoader - - self.env = Environment(loader=FunctionLoader(self.loader), **kwargs) - if filters: - self.env.filters.update(filters) - if tests: - self.env.tests.update(tests) - if globals: - self.env.globals.update(globals) - if self.source: - self.tpl = self.env.from_string(self.source) - else: - self.tpl = self.env.get_template(self.name) - - def render(self, *args, **kwargs): - for dictarg in args: - kwargs.update(dictarg) - _defaults = self.defaults.copy() - _defaults.update(kwargs) - return self.tpl.render(**_defaults) - - def loader(self, name): - if name == self.filename: - fname = name - else: - fname = self.search(name, self.lookup) - if not fname: - return - with open(fname, "rb") as f: - return (f.read().decode(self.encoding), fname, lambda: False) - - -class SimpleTemplate(BaseTemplate): - def prepare(self, escape_func=html_escape, noescape=False, syntax=None, **ka): - self.cache = {} - enc = self.encoding - self._str = lambda x: touni(x, enc) - self._escape = lambda x: escape_func(touni(x, enc)) - self.syntax = syntax - if noescape: - self._str, self._escape = self._escape, self._str - - @cached_property - def co(self): - return compile(self.code, self.filename or "", "exec") - - @cached_property - def code(self): - source = self.source - if not source: - with open(self.filename, "rb") as f: - source = f.read() - try: - source, encoding = touni(source), "utf8" - except UnicodeError: - raise depr(0, 11, "Unsupported template encodings.", "Use utf-8 for templates.") - parser = StplParser(source, encoding=encoding, syntax=self.syntax) - code = parser.translate() - self.encoding = parser.encoding - return code - - def _rebase(self, _env, _name=None, **kwargs): - _env["_rebase"] = (_name, kwargs) - - def _include(self, _env, _name=None, **kwargs): - env = _env.copy() - env.update(kwargs) - if _name not in self.cache: - self.cache[_name] = self.__class__(name=_name, lookup=self.lookup, syntax=self.syntax) - return self.cache[_name].execute(env["_stdout"], env) - - def execute(self, _stdout, kwargs): - env = self.defaults.copy() - env.update(kwargs) - env.update( - { - "_stdout": _stdout, - "_printlist": _stdout.extend, - "include": functools.partial(self._include, env), - "rebase": functools.partial(self._rebase, env), - "_rebase": None, - "_str": self._str, - "_escape": self._escape, - "get": env.get, - "setdefault": env.setdefault, - "defined": env.__contains__, - } - ) - exec(self.co, env) - if env.get("_rebase"): - subtpl, rargs = env.pop("_rebase") - rargs["base"] = "".join(_stdout) # copy stdout - del _stdout[:] # clear stdout - return self._include(env, subtpl, **rargs) - return env - - def render(self, *args, **kwargs): - """Render the template using keyword arguments as local variables.""" - env = {} - stdout = [] - for dictarg in args: - env.update(dictarg) - env.update(kwargs) - self.execute(stdout, env) - return "".join(stdout) - - -class StplSyntaxError(TemplateError): - pass - - -class StplParser: - """Parser for stpl templates.""" - - _re_cache = {} #: Cache for compiled re patterns - - # This huge pile of voodoo magic splits python code into 8 different tokens. - # We use the verbose (?x) regex mode to make this more manageable - - _re_tok = r"""( - [urbURB]* - (?: ''(?!') - |""(?!") - |'{6} - |"{6} - |'(?:[^\\']|\\.)+?' - |"(?:[^\\"]|\\.)+?" - |'{3}(?:[^\\]|\\.|\n)+?'{3} - |"{3}(?:[^\\]|\\.|\n)+?"{3} - ) - )""" - - _re_inl = _re_tok.replace(r"|\n", "") # We re-use this string pattern later - - _re_tok += r""" - # 2: Comments (until end of line, but not the newline itself) - |(\#.*) - - # 3: Open and close (4) grouping tokens - |([\[\{\(]) - |([\]\}\)]) - - # 5,6: Keywords that start or continue a python block (only start of line) - |^([\ \t]*(?:if|for|while|with|try|def|class)\b) - |^([\ \t]*(?:elif|else|except|finally)\b) - - # 7: Our special 'end' keyword (but only if it stands alone) - |((?:^|;)[\ \t]*end[\ \t]*(?=(?:%(block_close)s[\ \t]*)?\r?$|;|\#)) - - # 8: A customizable end-of-code-block template token (only end of line) - |(%(block_close)s[\ \t]*(?=\r?$)) - - # 9: And finally, a single newline. The 10th token is 'everything else' - |(\r?\n) - """ - - # Match the start tokens of code areas in a template - _re_split = r"""(?m)^[ \t]*(\\?)((%(line_start)s)|(%(block_start)s))""" - # Match inline statements (may contain python strings) - _re_inl = r"""%%(inline_start)s((?:%s|[^'"\n])*?)%%(inline_end)s""" % _re_inl - - # add the flag in front of the regexp to avoid Deprecation warning (see Issue #949) - # verbose and dot-matches-newline mode - _re_tok = "(?mx)" + _re_tok - _re_inl = "(?mx)" + _re_inl - - default_syntax = "<% %> % {{ }}" - - def __init__(self, source, syntax=None, encoding="utf8"): - self.source, self.encoding = touni(source, encoding), encoding - self.set_syntax(syntax or self.default_syntax) - self.code_buffer, self.text_buffer = [], [] - self.lineno, self.offset = 1, 0 - self.indent, self.indent_mod = 0, 0 - self.paren_depth = 0 - - def get_syntax(self): - """Tokens as a space separated string (default: <% %> % {{ }})""" - return self._syntax - - def set_syntax(self, syntax): - self._syntax = syntax - self._tokens = syntax.split() - if syntax not in self._re_cache: - names = "block_start block_close line_start inline_start inline_end" - etokens = map(re.escape, self._tokens) - pattern_vars = dict(zip(names.split(), etokens)) - patterns = (self._re_split, self._re_tok, self._re_inl) - patterns = [re.compile(p % pattern_vars) for p in patterns] - self._re_cache[syntax] = patterns - self.re_split, self.re_tok, self.re_inl = self._re_cache[syntax] - - syntax = property(get_syntax, set_syntax) - - def translate(self): - if self.offset: - raise RuntimeError("Parser is a one time instance.") - while True: - m = self.re_split.search(self.source, pos=self.offset) - if m: - text = self.source[self.offset : m.start()] - self.text_buffer.append(text) - self.offset = m.end() - if m.group(1): # Escape syntax - line, sep, _ = self.source[self.offset :].partition("\n") - self.text_buffer.append( - self.source[m.start() : m.start(1)] + m.group(2) + line + sep - ) - self.offset += len(line + sep) - continue - self.flush_text() - self.offset += self.read_code( - self.source[self.offset :], multiline=bool(m.group(4)) - ) - else: - break - self.text_buffer.append(self.source[self.offset :]) - self.flush_text() - return "".join(self.code_buffer) - - def read_code(self, pysource, multiline): - code_line, comment = "", "" - offset = 0 - while True: - m = self.re_tok.search(pysource, pos=offset) - if not m: - code_line += pysource[offset:] - offset = len(pysource) - self.write_code(code_line.strip(), comment) - break - code_line += pysource[offset : m.start()] - offset = m.end() - _str, _com, _po, _pc, _blk1, _blk2, _end, _cend, _nl = m.groups() - if self.paren_depth > 0 and (_blk1 or _blk2): # a if b else c - code_line += _blk1 or _blk2 - continue - if _str: # Python string - code_line += _str - elif _com: # Python comment (up to EOL) - comment = _com - if multiline and _com.strip().endswith(self._tokens[1]): - multiline = False # Allow end-of-block in comments - elif _po: # open parenthesis - self.paren_depth += 1 - code_line += _po - elif _pc: # close parenthesis - if self.paren_depth > 0: - # we could check for matching parentheses here, but it's - # easier to leave that to python - just check counts - self.paren_depth -= 1 - code_line += _pc - elif _blk1: # Start-block keyword (if/for/while/def/try/...) - code_line = _blk1 - self.indent += 1 - self.indent_mod -= 1 - elif _blk2: # Continue-block keyword (else/elif/except/...) - code_line = _blk2 - self.indent_mod -= 1 - elif _cend: # The end-code-block template token (usually '%>') - if multiline: - multiline = False - else: - code_line += _cend - elif _end: - self.indent -= 1 - self.indent_mod += 1 - else: # \n - self.write_code(code_line.strip(), comment) - self.lineno += 1 - code_line, comment, self.indent_mod = "", "", 0 - if not multiline: - break - - return offset - - def flush_text(self): - text = "".join(self.text_buffer) - del self.text_buffer[:] - if not text: - return - parts, pos, nl = [], 0, "\\\n" + " " * self.indent - for m in self.re_inl.finditer(text): - prefix, pos = text[pos : m.start()], m.end() - if prefix: - parts.append(nl.join(map(repr, prefix.splitlines(True)))) - if prefix.endswith("\n"): - parts[-1] += nl - parts.append(self.process_inline(m.group(1).strip())) - if pos < len(text): - prefix = text[pos:] - lines = prefix.splitlines(True) - if lines[-1].endswith("\\\\\n"): - lines[-1] = lines[-1][:-3] - elif lines[-1].endswith("\\\\\r\n"): - lines[-1] = lines[-1][:-4] - parts.append(nl.join(map(repr, lines))) - code = "_printlist((%s,))" % ", ".join(parts) - self.lineno += code.count("\n") + 1 - self.write_code(code) - - @staticmethod - def process_inline(chunk): - if chunk[0] == "!": - return "_str(%s)" % chunk[1:] - return "_escape(%s)" % chunk - - def write_code(self, line, comment=""): - code = " " * (self.indent + self.indent_mod) - code += line.lstrip() + comment + "\n" - self.code_buffer.append(code) - - -def template(*args, **kwargs): - """ - Get a rendered template as a string iterator. - You can use a name, a filename or a template string as first parameter. - Template rendering arguments can be passed as dictionaries - or directly (as keyword arguments). - """ - tpl = args[0] if args else None - for dictarg in args[1:]: - kwargs.update(dictarg) - adapter = kwargs.pop("template_adapter", SimpleTemplate) - lookup = kwargs.pop("template_lookup", TEMPLATE_PATH) - tplid = (id(lookup), tpl) - if tplid not in TEMPLATES or DEBUG: - settings = kwargs.pop("template_settings", {}) - if isinstance(tpl, adapter): - TEMPLATES[tplid] = tpl - if settings: - TEMPLATES[tplid].prepare(**settings) - elif "\n" in tpl or "{" in tpl or "%" in tpl or "$" in tpl: - TEMPLATES[tplid] = adapter(source=tpl, lookup=lookup, **settings) - else: - TEMPLATES[tplid] = adapter(name=tpl, lookup=lookup, **settings) - if not TEMPLATES[tplid]: - abort(500, "Template (%s) not found" % tpl) - return TEMPLATES[tplid].render(kwargs) - - -mako_template = functools.partial(template, template_adapter=MakoTemplate) -cheetah_template = functools.partial(template, template_adapter=CheetahTemplate) -jinja2_template = functools.partial(template, template_adapter=Jinja2Template) - - -def view(tpl_name, **defaults): - """Decorator: renders a template for a handler. - The handler can control its behavior like that: - - - return a dict of template vars to fill out the template - - return something other than a dict and the view decorator will not - process the template, but return the handler result as is. - This includes returning a HTTPResponse(dict) to get, - for instance, JSON with autojson or other castfilters. - """ - - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - result = func(*args, **kwargs) - if isinstance(result, (dict, DictMixin)): - tplvars = defaults.copy() - tplvars.update(result) - return template(tpl_name, **tplvars) - elif result is None: - return template(tpl_name, **defaults) - return result - - return wrapper - - return decorator - - -mako_view = functools.partial(view, template_adapter=MakoTemplate) -cheetah_view = functools.partial(view, template_adapter=CheetahTemplate) -jinja2_view = functools.partial(view, template_adapter=Jinja2Template) - -TEMPLATE_PATH = ["./", "./views/"] -TEMPLATES = {} -DEBUG = False -NORUN = False # If set, run() does nothing. Used by load_app() - -#: A dict to map HTTP status codes (e.g. 404) to phrases (e.g. 'Not Found') -HTTP_CODES = httplib.responses.copy() -HTTP_CODES[418] = "I'm a teapot" # RFC 2324 -HTTP_CODES[428] = "Precondition Required" -HTTP_CODES[429] = "Too Many Requests" -HTTP_CODES[431] = "Request Header Fields Too Large" -HTTP_CODES[451] = "Unavailable For Legal Reasons" # RFC 7725 -HTTP_CODES[511] = "Network Authentication Required" -_HTTP_STATUS_LINES = {k: "%d %s" % (k, v) for (k, v) in HTTP_CODES.items()} - -#: The default template used for error pages. Override with @error() -ERROR_PAGE_TEMPLATE = ( - """ -%%try: - %%from %s import DEBUG, request - - - - Error: {{e.status}} - - - -

Error: {{e.status}}

-

Sorry, the requested URL {{repr(request.url)}} - caused an error:

- 
{{e.body}}
- %%if DEBUG and e.exception: -

Exception:

- %%try: - %%exc = repr(e.exception) - %%except: - %%exc = '' %% type(e.exception).__name__ - %%end - 
{{exc}}
- %%end - %%if DEBUG and e.traceback: -

Traceback:

- 
{{e.traceback}}
- %%end - - -%%except ImportError: - ImportError: Could not generate the error page. Please add bottle to - the import path. -%%end -""" - % __name__ -) - -#: A thread-safe instance of :class:`LocalRequest`. If accessed from within a -#: request callback, this instance always refers to the *current* request -#: (even on a multi-threaded server). -request = LocalRequest() - -#: A thread-safe instance of :class:`LocalResponse`. It is used to change the -#: HTTP response for the *current* request. -response = LocalResponse() - -#: A thread-safe namespace. Not used by Bottle. -local = threading.local() - -# Initialize app stack (create first empty Bottle app now deferred until needed) -# BC: 0.6.4 and needed for run() -apps = app = default_app = AppStack() - -#: A virtual package that redirects import statements. -#: Example: ``import bottle.ext.sqlite`` actually imports `bottle_sqlite`. -ext = _ImportRedirect( - "bottle.ext" if __name__ == "__main__" else __name__ + ".ext", "bottle_%s" -).module - - -def _main(argv): # pragma: no coverage - args, parser = _cli_parse(argv) - - def _cli_error(cli_msg): - parser.print_help() - _stderr("\nError: %s\n" % cli_msg) - sys.exit(1) - - if args.version: - print("Bottle %s" % __version__) - sys.exit(0) - if not args.app: - _cli_error("No application entry point specified.") - - sys.path.insert(0, ".") - sys.modules.setdefault("bottle", sys.modules["__main__"]) - - host, port = (args.bind or "localhost"), 8080 - if ":" in host and host.rfind("]") < host.rfind(":"): - host, port = host.rsplit(":", 1) - host = host.strip("[]") - - config = ConfigDict() - - for cfile in args.conf or []: - try: - if cfile.endswith(".json"): - with open(cfile, "rb") as fp: - config.load_dict(json_loads(fp.read())) - else: - config.load_config(cfile) - except configparser.Error as parse_error: - _cli_error(parse_error) - except OSError: - _cli_error("Unable to read config file %r" % cfile) - except (UnicodeError, TypeError, ValueError) as error: - _cli_error(f"Unable to parse config file {cfile!r}: {error}") - - for cval in args.param or []: - if "=" in cval: - config.update((cval.split("=", 1),)) - else: - config[cval] = True - - run( - args.app, - host=host, - port=int(port), - server=args.server, - reloader=args.reload, - plugins=args.plugin, - debug=args.debug, - config=config, - ) - - -# endregion - -# region: dbt-core-interface server - -SERVER_MUTEX = threading.Lock() - - -@dataclass -class ServerRunResult: - """The result of running a query.""" - - column_names: List[str] - rows: List[List[Any]] - raw_code: str - executed_code: str - - -@dataclass -class ServerCompileResult: - """The result of compiling a project.""" - - result: str - - -@dataclass -class ServerResetResult: - """The result of resetting the Server database.""" - - result: str - - -@dataclass -class ServerRegisterResult: - """The result of registering a project.""" - - added: str - projects: List[str] - - -@dataclass -class ServerUnregisterResult: - """The result of unregistering a project.""" - - removed: str - projects: List[str] - - -class ServerErrorCode(Enum): - """The error codes that can be returned by the Server API.""" - - FailedToReachServer = -1 - CompileSqlFailure = 1 - ExecuteSqlFailure = 2 - ProjectParseFailure = 3 - ProjectNotRegistered = 4 - ProjectHeaderNotSupplied = 5 - - -@dataclass -class ServerError: - """An error that can be serialized to JSON.""" - - code: ServerErrorCode - message: str - data: Dict[str, Any] - - -@dataclass -class ServerErrorContainer: - """A container for an ServerError that can be serialized to JSON.""" - - error: ServerError - - -def server_serializer(o): - """Encode JSON. Handles server-specific types.""" - if isinstance(o, decimal.Decimal): - return float(o) - if isinstance(o, ServerErrorCode): - return o.value - return str(o) - - -def remove_comments(string) -> str: # noqa: C901 - """Remove comments from a string.""" - pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|//[^\r\n]*$)" - # first group captures quoted strings (double or single) - # second group captures comments (//single-line or /* multi-line */) - regex = re.compile(pattern, re.MULTILINE | re.DOTALL) - - def _replacer(match): - # if the 2nd group (capturing comments) is not None, - # it means we have captured a non-quoted (real) comment string. - if match.group(2) is not None: - return "" # so we will return empty to remove the comment - else: # otherwise, we will return the 1st group - return match.group(1) # captured quoted-string - - multiline_comments_removed = regex.sub(_replacer, string) + "\n" - output = "" - for line in multiline_comments_removed.splitlines(keepends=True): - if line.strip().startswith("--"): - continue - s_quote_c = 0 - d_quote_c = 0 - cmt_dash = 0 - split_ix = -1 - for i, c in enumerate(line): - if cmt_dash >= 2: - # found 2 sequential dashes, split here - split_ix = i - cmt_dash - break - if c == '"': - # inc quote count - d_quote_c += 1 - elif c == "'": - # inc quote count - s_quote_c += 1 - elif c == "-" and d_quote_c % 2 == 0 and s_quote_c % 2 == 0: - # dash and not in a quote, inc dash count - cmt_dash += 1 - continue - # reset dash count each iteration - cmt_dash = 0 - if split_ix > 0: - output += line[:split_ix] + "\n" - else: - output += line - return "".join(output) - - -class DbtInterfaceServerPlugin: - """Used to inject the dbt-core-interface runner into the request context.""" - - name = "dbt-interface-server" - api = 2 - - def __init__(self, runner: Optional[DbtProject] = None): - """Initialize the plugin with the runner to inject into the request context.""" - self.runners = DbtProjectContainer() - if runner: - self.runners.add_parsed_project(runner) - - def apply(self, callback, route): - """Apply the plugin to the route callback.""" - - def wrapper(*args, **kwargs): - start = time.time() - body = callback(*args, **kwargs, runners=self.runners) - end = time.time() - response.headers["X-dbt-Exec-Time"] = str(end - start) - return body - - return wrapper - - -@route("/run", method="POST") -def run_sql(runners: DbtProjectContainer) -> Union[ServerRunResult, ServerErrorContainer, str]: - """Run SQL against a dbt project.""" - # Project Support - project_runner = ( - runners.get_project(request.get_header("X-dbt-Project")) or runners.get_default_project() - ) - if not project_runner: - response.status = 400 - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectNotRegistered, - message=( - "Project is not registered. Make a POST request to the /register endpoint" - " first to register a runner" - ), - data={"registered_projects": runners.registered_projects()}, - ) - ) - ) - - # Query Construction - query = remove_comments(request.body.read().decode("utf-8")) - limit = request.query.get("limit", 200) - query_with_limit = ( - # we need to support `TOP` too - f"select * from ({query}) as __server_query limit {limit}" - ) - - try: - result = project_runner.execute_code(query_with_limit) - except Exception as execution_err: - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ExecuteSqlFailure, - message=str(execution_err), - data=execution_err.__dict__, - ) - ) - ) - - # Re-extract compiled query and return data structure - compiled_query = re.search( - r"select \* from \(([\w\W]+)\) as __server_query", result.compiled_code - ).groups()[0] - return asdict( - ServerRunResult( - rows=[list(row) for row in result.table.rows], - column_names=result.table.column_names, - executed_code=compiled_query.strip(), - raw_code=query, - ) - ) - - -@route("/compile", method="POST") -def compile_sql( - runners: DbtProjectContainer, -) -> Union[ServerCompileResult, ServerErrorContainer, str]: - """Compiles a SQL query.""" - # Project Support - project_runner = ( - runners.get_project(request.get_header("X-dbt-Project")) or runners.get_default_project() - ) - if not project_runner: - response.status = 400 - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectNotRegistered, - message=( - "Project is not registered. Make a POST request to the /register endpoint" - " first to register a runner" - ), - data={"registered_projects": runners.registered_projects()}, - ) - ) - ) - - # Query Compilation - query: str = request.body.read().decode("utf-8").strip() - if has_jinja(query): - try: - compiled_query = project_runner.compile_code(query).compiled_code - except Exception as compile_err: - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.CompileSqlFailure, - message=str(compile_err), - data=compile_err.__dict__, - ) - ) - ) - else: - compiled_query = query - - return asdict(ServerCompileResult(result=compiled_query)) - - -@route(["/parse", "/reset"]) -def reset(runners: DbtProjectContainer) -> Union[ServerResetResult, ServerErrorContainer, str]: - """Reset the runner and clear the cache.""" - # Project Support - project_runner = ( - runners.get_project(request.get_header("X-dbt-Project")) or runners.get_default_project() - ) - if not project_runner: - response.status = 400 - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectNotRegistered, - message=( - "Project is not registered. Make a POST request to the /register endpoint" - " first to register a runner" - ), - data={"registered_projects": runners.registered_projects()}, - ) - ) - ) - - # Determines if we should clear caches and reset config before re-seeding runner - reset = str(request.query.get("reset", "false")).lower() == "true" - - # Get targets - old_target = getattr(project_runner.base_config, "target", project_runner.config.target_name) - new_target = request.query.get("target", old_target) - - if not reset and old_target == new_target: - # Async (target same) - if SERVER_MUTEX.acquire(blocking=False): - LOGGER.debug("Mutex locked") - parse_job = threading.Thread( - target=_reset, args=(project_runner, reset, old_target, new_target) - ) - parse_job.start() - return asdict(ServerResetResult(result="Initializing project parsing")) - else: - LOGGER.debug("Mutex is locked, reparse in progress") - return asdict(ServerResetResult(result="Currently reparsing project")) - else: - # Sync (target changed or reset is true) - if SERVER_MUTEX.acquire(blocking=old_target != new_target): - LOGGER.debug("Mutex locked") - return asdict(_reset(project_runner, reset, old_target, new_target)) - else: - LOGGER.debug("Mutex is locked, reparse in progress") - return asdict(ServerResetResult(result="Currently reparsing project")) - - -def _reset( - runner: DbtProject, reset: bool, old_target: str, new_target: str -) -> Union[ServerResetResult, ServerErrorContainer]: - """Reset the project runner. - - Can be called asynchronously or synchronously. - """ - target_did_change = old_target != new_target - try: - runner.base_config.target = new_target - LOGGER.debug("Starting reparse") - runner.safe_parse_project(reinit=reset or target_did_change) - except Exception as reparse_err: - LOGGER.debug("Reparse error") - runner.base_config.target = old_target - rv = ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectParseFailure, - message=str(reparse_err), - data=reparse_err.__dict__, - ) - ) - else: - LOGGER.debug("Reparse success") - rv = ServerResetResult( - result=( - f"Profile target changed from {old_target} to {new_target}!" - if target_did_change - else f"Reparsed project with profile {old_target}!" - ) - ) - finally: - LOGGER.debug("Unlocking mutex") - SERVER_MUTEX.release() - return rv - - -@route("/register", method="POST") -def register(runners: DbtProjectContainer) -> Union[ServerResetResult, ServerErrorContainer, str]: - """Register a new project runner.""" - # Project Support - project = request.get_header("X-dbt-Project") - if not project: - response.status = 400 - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectHeaderNotSupplied, - message=( - "Project header `X-dbt-Project` was not supplied but is required for this" - " endpoint" - ), - data=dict(request.headers), - ) - ) - ) - if project in runners: - # Idempotent - return asdict(ServerRegisterResult(added=project, projects=runners.registered_projects())) - - # Inputs - project_dir = request.json["project_dir"] - profiles_dir = request.json["profiles_dir"] - target = request.json.get("target") - - try: - new_runner = DbtProject( - project_dir=project_dir, - profiles_dir=profiles_dir, - target=target, - ) - except Exception as init_err: - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectParseFailure, - message=str(init_err), - data=init_err.__dict__, - ) - ) - ) - - runners[project] = new_runner - runners.add_parsed_project - return asdict(ServerRegisterResult(added=project, projects=runners.registered_projects())) - - -@route("/unregister", method="POST") -def unregister(runners: DbtProjectContainer) -> Union[ServerResetResult, ServerErrorContainer, str]: - """Unregister a project runner from the server.""" - # Project Support - project = request.get_header("X-dbt-Project") - if not project: - response.status = 400 - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectHeaderNotSupplied, - message=( - "Project header `X-dbt-Project` was not supplied but is required for this" - " endpoint" - ), - data=dict(request.headers), - ) - ) - ) - if project not in runners: - response.status = 400 - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectNotRegistered, - message=( - "Project is not registered. Make a POST request to the /register endpoint" - " first to register a runner" - ), - data={"registered_projects": runners.registered_projects()}, - ) - ) - ) - runners.drop_project(project) - return asdict(ServerUnregisterResult(removed=project, projects=runners.registered_projects())) - - -@route("/v1/info", methods="GET") -def trino_info(runners: DbtProjectContainer): - """Trino info endpoint.""" - return { - "coordinator": {}, - "workers": [], - "memory": {}, - "jvm": {}, - "system": {}, - } - - -# TODO: These are here while we map agate types to trino types. -# public final class ClientStandardTypes -# { -# public static final String BIGINT = "bigint"; -# public static final String INTEGER = "integer"; -# public static final String SMALLINT = "smallint"; -# public static final String TINYINT = "tinyint"; -# public static final String BOOLEAN = "boolean"; -# public static final String DATE = "date"; -# public static final String DECIMAL = "decimal"; -# public static final String REAL = "real"; -# public static final String DOUBLE = "double"; -# public static final String HYPER_LOG_LOG = "HyperLogLog"; -# public static final String QDIGEST = "qdigest"; -# public static final String P4_HYPER_LOG_LOG = "P4HyperLogLog"; -# public static final String INTERVAL_DAY_TO_SECOND = "interval day to second"; -# public static final String INTERVAL_YEAR_TO_MONTH = "interval year to month"; -# public static final String TIMESTAMP = "timestamp"; -# public static final String TIMESTAMP_WITH_TIME_ZONE = "timestamp with time zone"; -# public static final String TIME = "time"; -# public static final String TIME_WITH_TIME_ZONE = "time with time zone"; -# public static final String VARBINARY = "varbinary"; -# public static final String VARCHAR = "varchar"; -# public static final String CHAR = "char"; -# public static final String ROW = "row"; -# public static final String ARRAY = "array"; -# public static final String MAP = "map"; -# public static final String JSON = "json"; -# public static final String IPADDRESS = "ipaddress"; -# public static final String UUID = "uuid"; -# public static final String GEOMETRY = "Geometry"; -# public static final String SPHERICAL_GEOGRAPHY = "SphericalGeography"; -# public static final String BING_TILE = "BingTile"; -# private ClientStandardTypes() {} -# } - -# TODO: This is just a note to acknowledge that we cannot use the default trino catalog -# yet some integrations may expect this type of query to succeed. -# select schema_name AS label, -# schema_name AS schema, -# 'connection.schema' as type, -# 'group-by-ref-type' as iconId, -# '' as database -# from .information_schema.schemata - - -@route("/v1/statement", method="POST") -def trino_statement(runners: DbtProjectContainer): - """Trino statement endpoint. - - This endpoint is used to execute queries and return the results. - It is very minimal right now. The only purpose is to proxy SELECT queries - to dbt from a JDBC and return the results to the JDBC. - """ - from agate import data_types - - # User Support - _user = request.headers.get("X-Presto-User", "default") - # Project Support - project_runner = ( - runners.get_project(request.get_header("X-dbt-Project")) or runners.get_default_project() - ) - if not project_runner: - return { - "errorName": "ProjectNotRegistered", - "errorType": "USER_ERROR", - "errorLocation": {"lineNumber": 1, "columnNumber": 1}, - "error": "Project is not registered. Make a POST request to the /register endpoint", - } - query = request.body.read().decode("utf-8") - res = project_runner.execute_code(query) - columns = [] - for column in res.table.columns: - if isinstance(column.data_type, data_types.Text): - columns += [ - { - "name": column.name, - "type": "varchar", - "typeSignature": { - "rawType": "varchar", - "arguments": [{"kind": "LONG_LITERAL", "value": 255}], - }, - } - ] - elif isinstance(column.data_type, data_types.Number): - columns += [ - { - "name": column.name, - "type": "bigint", - "typeSignature": {"rawType": "bigint", "arguments": []}, - } - ] - elif isinstance(column.data_type, data_types.Boolean): - columns += [ - { - "name": column.name, - "type": "boolean", - "typeSignature": {"rawType": "boolean", "arguments": []}, - } - ] - elif isinstance(column.data_type, data_types.Date): - columns += [ - { - "name": column.name, - "type": "date", - "typeSignature": {"rawType": "date", "arguments": []}, - } - ] - elif isinstance(column.data_type, data_types.DateTime): - columns += [ - { - "name": column.name, - "type": "timestamp", - "typeSignature": {"rawType": "timestamp", "arguments": []}, - } - ] - elif isinstance(column.data_type, data_types.TimeDelta): - columns += [ - { - "name": column.name, - "type": "interval day to second", - "typeSignature": { - "rawType": "interval day to second", - "arguments": [], - }, - } - ] - else: - columns += [ - { - "name": column.name, - "type": "varchar", - "typeSignature": { - "rawType": "varchar", - "arguments": [{"kind": "LONG_LITERAL", "value": 255}], - }, - } - ] - return { - "id": "someId", - # TODO: this should not be static - "infoUri": "http://localhost:8581/v1/info", - "columns": columns, - "data": [list(row) for row in res.table.rows], - "stats": { - "state": "FINISHED", - "nodes": 1, - }, - } - - -@route(["/health", "/api/health"], methods="GET") -def health_check(runners: DbtProjectContainer) -> dict: - """Health check endpoint.""" - # Project Support - project_runner = ( - runners.get_project(request.get_header("X-dbt-Project")) or runners.get_default_project() - ) - if not project_runner: - response.status = 400 - return asdict( - ServerErrorContainer( - error=ServerError( - code=ServerErrorCode.ProjectNotRegistered, - message=( - "Project is not registered. Make a POST request to the /register endpoint" - " first to register a runner" - ), - data={"registered_projects": runners.registered_projects()}, - ) - ) - ) - return { - "result": { - "status": "ready", - "project_name": project_runner.config.project_name, - "target_name": project_runner.config.target_name, - "profile_name": project_runner.config.project_name, - "logs": project_runner.config.log_path, - "timestamp": str(datetime.utcnow()), - "error": None, - }, - "id": str(uuid.uuid4()), - "dbt-interface-server": __name__, - } - - -ServerPlugin = DbtInterfaceServerPlugin() -install(ServerPlugin) -install(JSONPlugin(json_dumps=lambda body: json.dumps(body, default=server_serializer))) - - -def run_server(runner: Optional[DbtProject] = None, host="localhost", port=8581): - """Run the dbt core interface server. - - See supported servers below. By default, the server will run with the - `WSGIRefServer` which is a pure Python server. If you want to use a different server, - you will need to install the dependencies for that server. - - (CGIServer, FlupFCGIServer, WSGIRefServer, WaitressServer, - CherryPyServer, CherootServer, PasteServer, FapwsServer, - TornadoServer, AppEngineServer, TwistedServer, DieselServer, - MeinheldServer, GunicornServer, EventletServer, GeventServer, - BjoernServer, AiohttpServer, AiohttpUVLoopServer, AutoServer) - """ - if runner: - ServerPlugin.runners.add_parsed_project(runner) - run(host=host, port=port) - - -# endregion - -# region: dbt-core-interface data diffs - -try: - # TODO: we can drop this and use subprocesses to run git commands - from git import Repo -except ImportError: - pass -else: - from pathlib import Path - - def build_diff_queries(model: str, runner: DbtProject) -> Tuple[str, str]: - """Leverage git to build two temporary tables for diffing the results of a query throughout a change.""" - # Resolve git node - node = runner.get_ref_node(model) - dbt_path = Path(node.root_path) - repo = Repo(dbt_path, search_parent_directories=True) - t = next(Path(repo.working_dir).rglob(node.original_file_path)).relative_to( - repo.working_dir - ) - sha = repo.head.object.hexsha - target = repo.head.object.tree[str(t)] - - # Create original node - git_node_name = "z_" + sha[-7:] - original_node = runner.get_server_node( - target.data_stream.read().decode("utf-8"), git_node_name - ) - - # Alias changed node - changed_node = node - - # Compile models - original_node = runner.compile_node(original_node) - changed_node = runner.compile_node(changed_node) - - return original_node.compiled_sql, changed_node.compiled_sql - - def build_diff_tables(model: str, runner: DbtProject) -> Tuple["BaseRelation", "BaseRelation"]: - """Leverage git to build two temporary tables for diffing the results of a query throughout a change.""" - # Resolve git node - node = runner.get_ref_node(model) - dbt_path = Path(node.root_path) - repo = Repo(dbt_path, search_parent_directories=True) - t = next(Path(repo.working_dir).rglob(node.original_file_path)).relative_to( - repo.working_dir - ) - sha = repo.head.object.hexsha - target = repo.head.object.tree[str(t)] - - # Create original node - git_node_name = "z_" + sha[-7:] - original_node = runner.get_server_node( - target.data_stream.read().decode("utf-8"), git_node_name - ) - - # Alias changed node - changed_node = node - - # Compile models - original_node = runner.compile_node(original_node).node - changed_node = runner.compile_node(changed_node).node - - # Lookup and resolve original ref based on git sha - git_node_parts = original_node.database, "dbt_diff", git_node_name - ref_a, did_exist = runner.get_or_create_relation(*git_node_parts) - if not did_exist: - LOGGER.info("Creating new relation for %s", ref_a) - with runner.adapter.connection_named("dbt-osmosis"): - runner.execute_macro( - "create_schema", - kwargs={"relation": ref_a}, - ) - runner.execute_macro( - "create_table_as", - kwargs={ - "sql": original_node.compiled_sql, - "relation": ref_a, - "temporary": True, - }, - run_compiled_sql=True, - ) - - # Resolve modified fake ref based on hash of it compiled SQL - temp_node_name = ( - "z_" - + hashlib.md5( - changed_node.compiled_sql.encode("utf-8"), - usedforsecurity=False, - ).hexdigest()[-7:] - ) - git_node_parts = original_node.database, "dbt_diff", temp_node_name - ref_b, did_exist = runner.get_or_create_relation(*git_node_parts) - if not did_exist: - ref_b = runner.adapter.Relation.create(*git_node_parts) - LOGGER.info("Creating new relation for %s", ref_b) - with runner.adapter.connection_named("dbt-osmosis"): - runner.execute_macro( - "create_schema", - kwargs={"relation": ref_b}, - ) - runner.execute_macro( - "create_table_as", - kwargs={ - "sql": original_node.compiled_sql, - "relation": ref_b, - "temporary": True, - }, - run_compiled_sql=True, - ) - - return ref_a, ref_b - - def diff_tables( - ref_a: "BaseRelation", - ref_b: "BaseRelation", - pk: str, - runner: DbtProject, - aggregate: bool = True, - ) -> "Table": - """Given two relations, compare the results and return a table of the differences.""" - LOGGER.info("Running diff") - _, table = runner.adapter_execute( - runner.execute_macro( - ( - "_dbt_osmosis_compare_relations_agg" - if aggregate - else "_dbt_osmosis_compare_relations" - ), - kwargs={ - "a_relation": ref_a, - "b_relation": ref_b, - "primary_key": pk, - }, - ), - auto_begin=True, - fetch=True, - ) - return table - - def diff_queries( - sql_a: str, sql_b: str, pk: str, runner: DbtProject, aggregate: bool = True - ) -> "Table": - """Given two queries, compare the results and return a table of the differences.""" - LOGGER.info("Running diff") - _, table = runner.adapter_execute( - runner.execute_macro( - "_dbt_osmosis_compare_queries_agg" if aggregate else "_dbt_osmosis_compare_queries", - kwargs={ - "a_query": sql_a, - "b_query": sql_b, - "primary_key": pk, - }, - ), - auto_begin=True, - fetch=True, - ) - return table - - def diff_and_print_to_console( - model: str, - pk: str, - runner: DbtProject, - make_temp_tables: bool = False, - agg: bool = True, - output: str = "table", - ) -> None: - """Compare two tables and print the results to the console.""" - import agate - - if make_temp_tables: - table = diff_tables(*build_diff_tables(model, runner), pk, runner, agg) - else: - table = diff_queries(*build_diff_queries(model, runner), pk, runner, agg) - print("") - output = output.lower() - if output == "table": - table.print_table() - elif output in ("chart", "bar"): - if not agg: - LOGGER.warn( - "Cannot render output format chart with --no-agg option, defaulting to table" - ) - table.print_table() - else: - _table = table.compute( - [ - ( - "in_original, in_changed", - agate.Formula(agate.Text(), lambda r: "%(in_a)s, %(in_b)s" % r), - ) - ] - ) - _table.print_bars( - label_column_name="in_original, in_changed", value_column_name="count" - ) - elif output == "csv": - table.to_csv("dbt-osmosis-diff.csv") - else: - LOGGER.warn("No such output format %s, defaulting to table", output) - table.print_table() - - -# endregion - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser( - description="Run the dbt interface server. Defaults to the WSGIRefServer" - ) - parser.add_argument( - "--host", - default="localhost", - help="The host to run the server on. Defaults to localhost", - ) - parser.add_argument( - "--port", - default=8581, - help="The port to run the server on. Defaults to 8581", - ) - parser.add_argument( - "--project", - default=None, - help="The path to the dbt project to run. Defaults to None", - ) - args = parser.parse_args() - if args.project: - run_server(DbtProject.from_path(args.project), host=args.host, port=args.port) - else: - run_server(host=args.host, port=args.port)