Merge branch 'master' into DOC-302-new-etl-tutorial

.buildkite/dagster-buildkite/dagster_buildkite/package_spec.py

@@ -125,6 +125,7 @@ class PackageSpec:
     queue: Optional[BuildkiteQueue] = None
     run_pytest: bool = True
     always_run_if: Optional[Callable[[], bool]] = None
+    skip_if: Optional[Callable[[], Optional[str]]] = None
 
     def __post_init__(self):
         if not self.name:
@@ -161,9 +162,13 @@ def build_steps(self) -> List[BuildkiteTopLevelStep]:
                     if v not in unsupported_python_versions
                 ]
 
-                pytest_python_versions = sorted(
-                    list(set(default_python_versions) - set(unsupported_python_versions))
-                )
+                pytest_python_versions = [
+                    AvailablePythonVersion(v)
+                    for v in sorted(
+                        set(e.value for e in default_python_versions)
+                        - set(e.value for e in unsupported_python_versions)
+                    )
+                ]
                 # Use highest supported python version if no defaults_match
                 if len(pytest_python_versions) == 0:
                     pytest_python_versions = [supported_python_versions[-1]]
@@ -243,22 +248,39 @@ def requirements(self):
 
     @property
     def skip_reason(self) -> Optional[str]:
-        # Memoize so we don't log twice
-        if self._should_skip is False:
-            return None
-
-        if self.always_run_if and self.always_run_if():
-            self._should_skip = False
-            return None
-
-        if self._skip_reason:
+        """Provides a message if this package's steps should be skipped on this run, and no message if the package's steps should be run.
+        We actually use this to determine whether or not to run the package.
+
+        Because we use an archaic version of python to build our images, we can't use `cached_property`, and so we reinvent the wheel here with
+        self._should_skip and self._skip_reason. When we determine definitively that a package should or shouldn't be skipped, we cache the result on self._should_skip
+        as a boolean (it starts out as None), and cache the skip reason (or lack thereof) on self._skip_reason.
+        """
+        # If self._should_skip is not None, then the result is cached on self._skip_reason and we can return it.
+        if self._should_skip is not None:
+            if self._should_skip is True:
+                assert (
+                    self._skip_reason is not None
+                ), "Expected skip reason to be set if self._should_skip is True."
             return self._skip_reason
 
+        # If the result is not cached, check for NO_SKIP signifier first, so that it always
+        # takes precedent.
         if message_contains("NO_SKIP"):
             logging.info(f"Building {self.name} because NO_SKIP set")
             self._should_skip = False
             return None
+        if self.always_run_if and self.always_run_if():
+            self._should_skip = False
+            self._skip_reason = None
+            return None
+        if self.skip_if and self.skip_if():
+            self._skip_reason = self.skip_if()
+            self._should_skip = True
+            return self._skip_reason
 
+        # Take account of feature_branch changes _after_ skip_if so that skip_if
+        # takes precedent. This way, integration tests can run on branch but won't be
+        # forced to run on every master commit.
         if not is_feature_branch(os.getenv("BUILDKITE_BRANCH", "")):
             logging.info(f"Building {self.name} we're not on a feature branch")
             self._should_skip = False

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/dagster_oss_main.py

@@ -52,6 +52,7 @@ def build_dagster_oss_main_steps() -> List[BuildkiteStep]:
                     ),
                     "DAGSTER_CHECKOUT_DEPTH": _get_setting("DAGSTER_CHECKOUT_DEPTH") or "100",
                     "OSS_COMPAT_SLIM": "1" if oss_compat_slim else "",
+                    "DAGSTER_FROM_OSS": "1" if pipeline_name == "internal" else "0",
                 },
             ),
         )

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/dagster_oss_nightly_pipeline.py

@@ -42,6 +42,28 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                 ],
                 pytest_extra_cmds=k8s_extra_cmds,
             ),
+            PackageSpec(
+                "examples/experimental/dagster-dlift/kitchen-sink",
+                name="dbt-cloud-live-tests",
+                env_vars=[
+                    "KS_DBT_CLOUD_ACCOUNT_ID",
+                    "KS_DBT_CLOUD_PROJECT_ID",
+                    "KS_DBT_CLOUD_TOKEN",
+                    "KS_DBT_CLOUD_ACCESS_URL",
+                    "KS_DBT_CLOUD_DISCOVERY_API_URL",
+                ],
+            ),
+            PackageSpec(
+                "examples/experimental/dagster-airlift/examples/dbt-example",
+                name="airlift-demo-live-tests",
+                env_vars=[
+                    "KS_DBT_CLOUD_ACCOUNT_ID",
+                    "KS_DBT_CLOUD_PROJECT_ID",
+                    "KS_DBT_CLOUD_TOKEN",
+                    "KS_DBT_CLOUD_ACCESS_URL",
+                    "KS_DBT_CLOUD_DISCOVERY_API_URL",
+                ],
+            ),
         ]
     )
 

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/prerelease_package.py

@@ -1,5 +1,3 @@
-import re
-from pathlib import Path
 from typing import List
 
 from dagster_buildkite.python_version import AvailablePythonVersion
@@ -17,14 +15,6 @@ def build_prerelease_package_steps() -> List[BuildkiteStep]:
         + _get_uncustomized_pkg_roots("examples/experimental", [])
     )
 
-    # Get only packages that have a fixed version in setup.py
-    filtered_packages = []
-    for package in packages:
-        setup_file = Path(package) / "setup.py"
-        contents = setup_file.read_text()
-        if re.findall(r"version=\"[\d\.]+\"", contents):
-            filtered_packages.append(package)
-
     input_step: BlockStep = {
         "block": ":question: Choose package",
         "prompt": None,
@@ -39,7 +29,7 @@ def build_prerelease_package_steps() -> List[BuildkiteStep]:
                         else package,
                         "value": package,
                     }
-                    for package in filtered_packages
+                    for package in packages
                 ],
                 "hint": None,
                 "default": None,

.buildkite/dagster-buildkite/dagster_buildkite/python_version.py

@@ -5,9 +5,8 @@
 from dagster_buildkite.utils import is_release_branch, safe_getenv
 
 
-class AvailablePythonVersion(str, Enum):
+class AvailablePythonVersion(Enum):
     # Ordering is important here, because some steps will take the highest/lowest available version.
-    V3_8 = "3.8"
     V3_9 = "3.9"
     V3_10 = "3.10"
     V3_11 = "3.11"
@@ -21,6 +20,12 @@ def get_all(cls) -> List["AvailablePythonVersion"]:
     def get_default(cls) -> "AvailablePythonVersion":
         return cls["V3_11"]
 
+    # Useful for providing to `PackageSpec.unsupported_python_versions` when you only want to test
+    # the default version.
+    @classmethod
+    def get_all_except_default(cls) -> List["AvailablePythonVersion"]:
+        return [v for v in cls.get_all() if v != cls.get_default()]
+
     @classmethod
     def get_pytest_defaults(cls) -> List["AvailablePythonVersion"]:
         branch_name = safe_getenv("BUILDKITE_BRANCH")
@@ -55,6 +60,6 @@ def from_major_minor(cls, major: int, minor: int) -> "AvailablePythonVersion":
 
     @classmethod
     def to_tox_factor(cls, version: "AvailablePythonVersion") -> str:
-        ver_parts = version.split(".")
+        ver_parts = version.value.split(".")
         major, minor = ver_parts[0], ver_parts[1]
         return f"py{major}{minor}"

.buildkite/dagster-buildkite/dagster_buildkite/step_builder.py

@@ -109,7 +109,7 @@ def on_test_image(
             raise Exception(f"Unsupported python version for test image: {ver}.")
 
         return self.on_python_image(
-            image=f"buildkite-test:py{ver}-{BUILDKITE_TEST_IMAGE_VERSION}",
+            image=f"buildkite-test:py{ver.value}-{BUILDKITE_TEST_IMAGE_VERSION}",
             env=env,
         )
 

.buildkite/dagster-buildkite/dagster_buildkite/steps/helm.py

@@ -20,7 +20,6 @@ def build_helm_steps() -> List[BuildkiteStep]:
         os.path.join("helm", "dagster", "schema"),
         unsupported_python_versions=[
             # run helm schema tests only once, on the latest python version
-            AvailablePythonVersion.V3_8,
             AvailablePythonVersion.V3_9,
             AvailablePythonVersion.V3_10,
             AvailablePythonVersion.V3_11,

.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py

@@ -6,7 +6,11 @@
     GCP_CREDS_LOCAL_FILE,
     LATEST_DAGSTER_RELEASE,
 )
-from dagster_buildkite.package_spec import PackageSpec, UnsupportedVersionsFunction
+from dagster_buildkite.package_spec import (
+    PackageSpec,
+    PytestExtraCommandsFunction,
+    UnsupportedVersionsFunction,
+)
 from dagster_buildkite.python_version import AvailablePythonVersion
 from dagster_buildkite.step_builder import BuildkiteQueue
 from dagster_buildkite.steps.test_project import test_project_depends_fn
@@ -60,12 +64,12 @@ def build_backcompat_suite_steps() -> List[BuildkiteTopLevelStep]:
     )
 
 
-def backcompat_extra_cmds(_, factor: str) -> List[str]:
+def backcompat_extra_cmds(_, factor: Optional[str]) -> List[str]:
     tox_factor_map = {
         "user-code-latest-release": LATEST_DAGSTER_RELEASE,
         "user-code-earliest-release": EARLIEST_TESTED_RELEASE,
     }
-
+    assert factor
     webserver_version = DAGSTER_CURRENT_BRANCH
     webserver_library_version = _get_library_version(webserver_version)
     user_code_version = tox_factor_map[factor]
@@ -163,7 +167,7 @@ def build_auto_materialize_perf_suite_steps():
 
 def daemon_pytest_extra_cmds(version: AvailablePythonVersion, _):
     return [
-        "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version,
+        "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value,
         'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"',
         "pushd integration_tests/test_suites/daemon-test-suite/monitoring_daemon_tests/",
         "docker-compose up -d --remove-orphans",
@@ -182,7 +186,7 @@ def daemon_pytest_extra_cmds(version: AvailablePythonVersion, _):
 # ########################
 
 
-def build_k8s_suite_steps():
+def build_k8s_suite_steps() -> List[BuildkiteTopLevelStep]:
     pytest_tox_factors = ["-default", "-subchart"]
     directory = os.path.join("integration_tests", "test_suites", "k8s-test-suite")
     return build_integration_suite_steps(
@@ -201,7 +205,7 @@ def build_k8s_suite_steps():
 def build_integration_suite_steps(
     directory: str,
     pytest_tox_factors: Optional[List[str]],
-    pytest_extra_cmds: Optional[Callable] = None,
+    pytest_extra_cmds: Optional[PytestExtraCommandsFunction] = None,
     queue=None,
     always_run_if: Optional[Callable[[], bool]] = None,
     unsupported_python_versions: Optional[
@@ -229,19 +233,19 @@ def build_integration_suite_steps(
     ).build_steps()
 
 
-def k8s_integration_suite_pytest_extra_cmds(version: str, _) -> List[str]:
+def k8s_integration_suite_pytest_extra_cmds(version: AvailablePythonVersion, _) -> List[str]:
     return [
-        "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version,
+        "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value,
         'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"',
         "aws ecr get-login --no-include-email --region us-west-2 | sh",
     ]
 
 
-def celery_k8s_integration_suite_pytest_extra_cmds(version: str, _) -> List[str]:
+def celery_k8s_integration_suite_pytest_extra_cmds(version: AvailablePythonVersion, _) -> List[str]:
     cmds = [
         'export AIRFLOW_HOME="/airflow"',
         "mkdir -p $${AIRFLOW_HOME}",
-        "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version,
+        "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value,
         'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"',
         "aws ecr get-login --no-include-email --region us-west-2 | sh",
     ]