diff --git a/Cargo.lock b/Cargo.lock index e46e4b99..79529f4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3606,6 +3606,8 @@ dependencies = [ "clap", "clap_complete", "clap_mangen", + "once_cell", + "regex", "serde_json", "snafu", "stackable-cockpitd", diff --git a/Cargo.toml b/Cargo.toml index 12625f31..d7cc0f73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,10 +29,12 @@ k8s-openapi = { version = "0.19", default-features = false, features = ["v1_27"] kube = { version = "0.85", default-features = false, features = ["client", "rustls-tls"] } lazy_static = "1.4" nu-ansi-term = "0.49" +once_cell = "1.18" phf = "0.11" phf_codegen = "0.11" rand = "0.8" -reqwest = { version = "0.11.16", default-features = false, features = ["rustls-tls"] } +regex = "1.9" +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] } semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/docs/antora.yml b/docs/antora.yml index d522680b..45968107 100644 --- a/docs/antora.yml +++ b/docs/antora.yml @@ -1,3 +1,7 @@ --- -name: home +name: management +title: SDP Management version: "nightly" +nav: + - modules/cockpit/nav.adoc + - modules/stackablectl/nav.adoc diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc new file mode 100644 index 00000000..6dd74e8c --- /dev/null +++ b/docs/modules/ROOT/pages/index.adoc @@ -0,0 +1,4 @@ += Management Tools for the Stackable Data Platform + +* xref:cockpit:index.adoc[] +* xref:stackablectl:index.adoc[] diff --git a/docs/modules/cockpit/nav.adoc b/docs/modules/cockpit/nav.adoc new file mode 100644 index 00000000..fd44482e --- /dev/null +++ b/docs/modules/cockpit/nav.adoc @@ -0,0 +1,2 @@ +* xref:index.adoc[Cockpit] +** xref:installation.adoc[Installation] diff --git a/docs/modules/cockpit/pages/index.adoc b/docs/modules/cockpit/pages/index.adoc index d5e697a0..06accaa4 100644 --- a/docs/modules/cockpit/pages/index.adoc +++ b/docs/modules/cockpit/pages/index.adoc @@ -2,5 +2,5 @@ This is a visual dashboard to monitor and control Stackable Data Platform clusters. -NOTE: The Stackable Cockpit is currently an early preview, and is not yet a +IMPORTANT: The Stackable Cockpit is currently an early preview, and is not yet a fully supported component of the Stackable Data Platform. diff --git a/docs/modules/cockpit/pages/installation.adoc b/docs/modules/cockpit/pages/installation.adoc index 2d9b2a08..a5e83981 100644 --- a/docs/modules/cockpit/pages/installation.adoc +++ b/docs/modules/cockpit/pages/installation.adoc @@ -11,7 +11,8 @@ You will need: * Helm * htpasswd (from Apache HTTPD) -Resource sizing depends on cluster type(s), usage and scope, but as a starting point we recommend a minimum of the following resources for this service: +Resource sizing depends on cluster type(s), usage and scope, but as a starting point we recommend a minimum of the +following resources for this service: * 0.2 cores (e.g. i5 or similar) * 256MB RAM diff --git a/docs/modules/cockpit/partials/nav.adoc b/docs/modules/cockpit/partials/nav.adoc deleted file mode 100644 index 8dc3d46f..00000000 --- a/docs/modules/cockpit/partials/nav.adoc +++ /dev/null @@ -1 +0,0 @@ -* xref:cockpit:installation.adoc[] diff --git a/docs/modules/stackablectl/images/layers.png b/docs/modules/stackablectl/images/layers.png new file mode 100644 index 00000000..69e5a5a5 Binary files /dev/null and b/docs/modules/stackablectl/images/layers.png differ diff --git a/docs/modules/stackablectl/nav.adoc b/docs/modules/stackablectl/nav.adoc new file mode 100644 index 00000000..db667849 --- /dev/null +++ b/docs/modules/stackablectl/nav.adoc @@ -0,0 +1,16 @@ +* xref:index.adoc[stackablectl] +** xref:installation.adoc[Installation] +** xref:quickstart.adoc[Quickstart] +** xref:commands/index.adoc[Commands] +*** xref:commands/cache.adoc[cache] +*** xref:commands/completions.adoc[completions] +*** xref:commands/demo.adoc[demo] +*** xref:commands/operator.adoc[operator] +*** xref:commands/release.adoc[release] +*** xref:commands/stack.adoc[stack] +*** xref:commands/stacklets.adoc[stacklets] +** xref:customization/index.adoc[] +*** xref:customization/add-demo.adoc[] +*** xref:customization/add-stack.adoc[] +*** xref:customization/add-release.adoc[] +// *** xref:customization/working-with-feature-branches.adoc[] diff --git a/docs/modules/stackablectl/pages/commands/cache.adoc b/docs/modules/stackablectl/pages/commands/cache.adoc new file mode 100644 index 00000000..773e8ef1 --- /dev/null +++ b/docs/modules/stackablectl/pages/commands/cache.adoc @@ -0,0 +1,85 @@ += stackablectl cache + +// Autogenerated by cargo xtask gen-docs. DO NOT CHANGE MANUALLY! +[source,console] +---- +$ stackablectl cache +Interact with locally cached files + +Usage: cache [OPTIONS] + +Commands: + list List cached files + clean Clean cached files + help Print this message or the help of the given subcommand(s) + +Options: + -l, --log-level + Log level this application uses + + --no-cache + Do not cache the remote (default) demo, stack and release files + + Cached files are saved at '$XDG_CACHE_HOME/stackablectl', which is usually + '$HOME/.cache/stackablectl' when not explicitly set. + + --offline + Do not request any remote files via the network + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +File options: + -d, --demo-file + Provide one or more additional (custom) demo file(s) + + Demos are loaded in the following order: Remote (default) demo file, custom + demo files provided via the 'STACKABLE_DEMO_FILES' environment variable, and + lastly demo files provided via the '-d/--demo-file' argument(s). If there are + demos with the same name, the last demo definition will be used. + + Use "stackablectl [OPTIONS] -d path/to/demos1.yaml -d path/to/demos2.yaml" + to provide multiple additional demo files. + + -s, --stack-file + Provide one or more additional (custom) stack file(s) + + Stacks are loaded in the following order: Remote (default) stack file, custom + stack files provided via the 'STACKABLE_STACK_FILES' environment variable, and + lastly demo files provided via the '-s/--stack-file' argument(s). If there are + stacks with the same name, the last stack definition will be used. + + Use "stackablectl [OPTIONS] -s path/to/stacks1.yaml -s path/to/stacks2.yaml" + to provide multiple additional stack files. + + -r, --release-file + Provide one or more additional (custom) release file(s) + + Releases are loaded in the following order: Remote (default) release file, + custom release files provided via the 'STACKABLE_RELEASE_FILES' environment + variable, and lastly release files provided via the '-r/--release-file' + argument(s). If there are releases with the same name, the last release + definition will be used. + + Use "stackablectl [OPTIONS] -r path/to/releases1.yaml -r path/to/releases2.yaml" + to provide multiple additional release files. + +Helm repository options: + --helm-repo-stable + Provide a custom Helm stable repository URL + + [default: https://repo.stackable.tech/repository/helm-stable/] + + --helm-repo-test + Provide a custom Helm test repository URL + + [default: https://repo.stackable.tech/repository/helm-test/] + + --helm-repo-dev + Provide a custom Helm dev repository URL + + [default: https://repo.stackable.tech/repository/helm-dev/] +---- diff --git a/docs/modules/stackablectl/pages/commands/completions.adoc b/docs/modules/stackablectl/pages/commands/completions.adoc new file mode 100644 index 00000000..07922e3c --- /dev/null +++ b/docs/modules/stackablectl/pages/commands/completions.adoc @@ -0,0 +1,86 @@ += stackablectl completions + +// Autogenerated by cargo xtask gen-docs. DO NOT CHANGE MANUALLY! +[source,console] +---- +$ stackablectl completions +Generate shell completions for this tool + +Usage: completions [OPTIONS] + +Commands: + bash Generate shell completions for Bash + fish Generate shell completions for Fish + zsh Generate shell completions for ZSH + help Print this message or the help of the given subcommand(s) + +Options: + -l, --log-level + Log level this application uses + + --no-cache + Do not cache the remote (default) demo, stack and release files + + Cached files are saved at '$XDG_CACHE_HOME/stackablectl', which is usually + '$HOME/.cache/stackablectl' when not explicitly set. + + --offline + Do not request any remote files via the network + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +File options: + -d, --demo-file + Provide one or more additional (custom) demo file(s) + + Demos are loaded in the following order: Remote (default) demo file, custom + demo files provided via the 'STACKABLE_DEMO_FILES' environment variable, and + lastly demo files provided via the '-d/--demo-file' argument(s). If there are + demos with the same name, the last demo definition will be used. + + Use "stackablectl [OPTIONS] -d path/to/demos1.yaml -d path/to/demos2.yaml" + to provide multiple additional demo files. + + -s, --stack-file + Provide one or more additional (custom) stack file(s) + + Stacks are loaded in the following order: Remote (default) stack file, custom + stack files provided via the 'STACKABLE_STACK_FILES' environment variable, and + lastly demo files provided via the '-s/--stack-file' argument(s). If there are + stacks with the same name, the last stack definition will be used. + + Use "stackablectl [OPTIONS] -s path/to/stacks1.yaml -s path/to/stacks2.yaml" + to provide multiple additional stack files. + + -r, --release-file + Provide one or more additional (custom) release file(s) + + Releases are loaded in the following order: Remote (default) release file, + custom release files provided via the 'STACKABLE_RELEASE_FILES' environment + variable, and lastly release files provided via the '-r/--release-file' + argument(s). If there are releases with the same name, the last release + definition will be used. + + Use "stackablectl [OPTIONS] -r path/to/releases1.yaml -r path/to/releases2.yaml" + to provide multiple additional release files. + +Helm repository options: + --helm-repo-stable + Provide a custom Helm stable repository URL + + [default: https://repo.stackable.tech/repository/helm-stable/] + + --helm-repo-test + Provide a custom Helm test repository URL + + [default: https://repo.stackable.tech/repository/helm-test/] + + --helm-repo-dev + Provide a custom Helm dev repository URL + + [default: https://repo.stackable.tech/repository/helm-dev/] +---- diff --git a/docs/modules/stackablectl/pages/commands/demo.adoc b/docs/modules/stackablectl/pages/commands/demo.adoc new file mode 100644 index 00000000..e5c77b8a --- /dev/null +++ b/docs/modules/stackablectl/pages/commands/demo.adoc @@ -0,0 +1,256 @@ += stackablectl demo +:page-aliases: stackablectl::commands/demo.adoc + +A demo is an end-to-end demonstration of the usage of the Stackable data platform. It is tied to a specific stack of the +Stackable data platform, which will provide the required products for the demo. + +== General Usage + +// Autogenerated by cargo xtask gen-docs. DO NOT CHANGE MANUALLY! +[source,console] +---- +$ stackablectl demo +Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform + +Usage: demo [OPTIONS] + +Commands: + list List available demos + describe Print out detailed demo information + install Install a specific demo + help Print this message or the help of the given subcommand(s) + +Options: + -l, --log-level + Log level this application uses + + --no-cache + Do not cache the remote (default) demo, stack and release files + + Cached files are saved at '$XDG_CACHE_HOME/stackablectl', which is usually + '$HOME/.cache/stackablectl' when not explicitly set. + + --offline + Do not request any remote files via the network + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +File options: + -d, --demo-file + Provide one or more additional (custom) demo file(s) + + Demos are loaded in the following order: Remote (default) demo file, custom + demo files provided via the 'STACKABLE_DEMO_FILES' environment variable, and + lastly demo files provided via the '-d/--demo-file' argument(s). If there are + demos with the same name, the last demo definition will be used. + + Use "stackablectl [OPTIONS] -d path/to/demos1.yaml -d path/to/demos2.yaml" + to provide multiple additional demo files. + + -s, --stack-file + Provide one or more additional (custom) stack file(s) + + Stacks are loaded in the following order: Remote (default) stack file, custom + stack files provided via the 'STACKABLE_STACK_FILES' environment variable, and + lastly demo files provided via the '-s/--stack-file' argument(s). If there are + stacks with the same name, the last stack definition will be used. + + Use "stackablectl [OPTIONS] -s path/to/stacks1.yaml -s path/to/stacks2.yaml" + to provide multiple additional stack files. + + -r, --release-file + Provide one or more additional (custom) release file(s) + + Releases are loaded in the following order: Remote (default) release file, + custom release files provided via the 'STACKABLE_RELEASE_FILES' environment + variable, and lastly release files provided via the '-r/--release-file' + argument(s). If there are releases with the same name, the last release + definition will be used. + + Use "stackablectl [OPTIONS] -r path/to/releases1.yaml -r path/to/releases2.yaml" + to provide multiple additional release files. + +Helm repository options: + --helm-repo-stable + Provide a custom Helm stable repository URL + + [default: https://repo.stackable.tech/repository/helm-stable/] + + --helm-repo-test + Provide a custom Helm test repository URL + + [default: https://repo.stackable.tech/repository/helm-test/] + + --helm-repo-dev + Provide a custom Helm dev repository URL + + [default: https://repo.stackable.tech/repository/helm-dev/] +---- + +== Browse Available Demos + +To list the available demos, run the following command: + +[source,console] +---- +$ stackablectl demo list +┌────┬───────────────────────┬─────────┬─────────────────────────────────────────────────────────┐ +│ # ┆ NAME ┆ STACK ┆ DESCRIPTION │ +╞════╪═══════════════════════╪═════════╪═════════════════════════════════════════════════════════╡ +│ 1 ┆ airflow-scheduled-job ┆ airflow ┆ Activate a simple Airflow DAG to run continuously at a │ +│ ┆ ┆ ┆ set interval │ +├╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 2 ┆ ... ┆ ... ┆ ... │ +├╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 11 ┆ local-demo-test-env ┆ test ┆ Test demo. Provided with the STACKABLE_DEMO_FILES env │ +│ ┆ ┆ ┆ var in the .env file │ +└────┴───────────────────────┴─────────┴─────────────────────────────────────────────────────────┘ +---- + +Detailed information of a demo can be queried using the `describe` command: + +[source,console] +---- +$ stackablectl demo describe trino-taxi-data + DEMO trino-taxi-data + DESCRIPTION Demo loading 2.5 years of New York taxi data into S3 bucket, creating a Trino table and a Superset dashboard + DOCUMENTATION https://docs.stackable.tech/stackablectl/stable/demos/trino-taxi-data.html + STACK trino-superset-s3 + LABELS trino, superset, minio, s3, ny-taxi-data +---- + +== Installing a Demo + +=== Using an Existing Kubernetes Cluster + +If you want to access a Kubernetes cluster, make sure your https://kubernetes.io/docs/tasks/tools/#kubectl[`kubectl`] +Kubernetes client is configured to interact with the Kubernetes cluster. After that, run the following command: + + +[source,console] +---- +$ stackablectl demo install trino-taxi-data + +---- + +=== Using a Local Kubernetes Cluster + +If you don't have a Kubernetes cluster available, `stackablectl` can spin up a https://kind.sigs.k8s.io/[kind] or +https://minikube.sigs.k8s.io/docs/[minikube] Kubernetes cluster for you. Based on the type of local cluster you want to +use, ensure you have either `kind` or `minikube` installed on your system. `stackablectl` will perform a check to verify +that these tools are available in your `PATH` and check if Docker is running. + +==== Local Kind Kubernetes Cluster + +[source,console] +---- +$ stackablectl demo install trino-taxi-data -c kind +Creating cluster "stackable-data-platform" ... + ✓ Ensuring node image (kindest/node:v1.26.3) 🖼 + ✓ Preparing nodes 📦 📦 + ✓ Writing configuration 📜 + ✓ Starting control-plane 🕹️ + ✓ Installing CNI 🔌 + ✓ Installing StorageClass 💾 + ✓ Joining worker nodes 🚜 +Set kubectl context to "kind-stackable-data-platform" +You can now use your cluster with: + +kubectl cluster-info --context kind-stackable-data-platform + +Have a question, bug, or feature request? Let us know! https://kind.sigs.k8s.io/#community 🙂 +The release commons-operator was successfully installed. +The release hive-operator was successfully installed. +The release opa-operator was successfully installed. +The release secret-operator was successfully installed. +The release superset-operator was successfully installed. +The release trino-operator was successfully installed. +Installed demo trino-taxi-data + +Use "stackablectl operator installed" to display the installed operators +Use "stackablectl stacklet list" to display the installed stacklets +---- + +==== Local Minikube Kubernetes Cluster + +[source,console] +---- +$ stackablectl demo install trino-taxi-data -c minikube +😄 [stackable-data-platform] minikube v1.30.1 on Ubuntu 22.04.2 +✨ Using the docker driver based on user configuration +🎉 minikube 1.31.2 is available! Download it: https://github.com/kubernetes/minikube/releases/tag/v1.31.2 +💡 To disable this notice, run: 'minikube config set WantUpdateNotification false' + +📌 Using Docker driver with root privileges +👍 Starting control plane node stackable-data-platform in cluster stackable-data-platform +🚜 Pulling base image ... +🔥 Creating docker container (CPUs=2, Memory=8000MB) ... +🐳 Preparing Kubernetes v1.26.3 on Docker 23.0.2 ... + ▪ Generating certificates and keys ... + ▪ Booting up control plane ... + ▪ Configuring RBAC rules ... +🔗 Configuring CNI (Container Networking Interface) ... + ▪ Using image gcr.io/k8s-minikube/storage-provisioner:v5 +🌟 Enabled addons: storage-provisioner, default-storageclass +🔎 Verifying Kubernetes components... + +👍 Starting worker node stackable-data-platform-m02 in cluster stackable-data-platform +🚜 Pulling base image ... +🔥 Creating docker container (CPUs=2, Memory=8000MB) ... +🌐 Found network options: + ▪ NO_PROXY=192.168.58.2 +🐳 Preparing Kubernetes v1.26.3 on Docker 23.0.2 ... + ▪ env NO_PROXY=192.168.58.2 +🔎 Verifying Kubernetes components... +🏄 Done! kubectl is now configured to use "stackable-data-platform" cluster and "default" namespace by default +The release commons-operator was successfully installed. +The release hive-operator was successfully installed. +The release opa-operator was successfully installed. +The release secret-operator was successfully installed. +The release superset-operator was successfully installed. +The release trino-operator was successfully installed. +Installed demo trino-taxi-data + +Use "stackablectl operator installed" to display the installed operators +Use "stackablectl stacklet list" to display the installed stacklets +---- + +''' + +The demos create Kubernetes jobs that will populate test data and interact with the installed products to process the +data. Until the products are ready, it is expected that the pods of these Jobs will fail with an error. They will get +retried with an exponentially growing back-off time. After the products are ready, they should turn green, and +everything should settle down. + +=== Listing Deployed Stacklets + +After installing your demo you can use the xref:commands/stacklets.adoc[`stackablectl stacklets`] command to list the +installed stacklets as follows: + +[source,console] +---- +$ stackablectl stacklets list +┌──────────┬───────────────┬───────────┬─────────────────────────────────────────────┬────────────────────────────────────────────┐ +│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │ +╞══════════╪═══════════════╪═══════════╪═════════════════════════════════════════════╪════════════════════════════════════════════╡ +│ hive ┆ hive ┆ default ┆ ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ opa ┆ opa ┆ default ┆ ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ superset ┆ superset ┆ default ┆ external-superset http://192.168.58.2:30788 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ trino ┆ trino ┆ default ┆ ┆ Unavailable: See [1], Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ minio ┆ minio-console ┆ default ┆ ┆ │ +└──────────┴───────────────┴───────────┴─────────────────────────────────────────────┴────────────────────────────────────────────┘ + +[1]: StatefulSet ["trino-coordinator-default", "trino-worker-default"] missing ready replicas. +---- + +== Uninstalling a Demo + +Currently, there is no support for uninstalling a demo again. However, this functionality will come soon. diff --git a/docs/modules/stackablectl/pages/commands/index.adoc b/docs/modules/stackablectl/pages/commands/index.adoc new file mode 100644 index 00000000..1d9d38df --- /dev/null +++ b/docs/modules/stackablectl/pages/commands/index.adoc @@ -0,0 +1,90 @@ +== General Usage + +// Autogenerated by cargo xtask gen-docs. DO NOT CHANGE MANUALLY! +[source,console] +---- +$ stackablectl +Command line tool to interact with the Stackable Data Platform + +Usage: stackablectl [OPTIONS] + +Commands: + operator Interact with single operator instead of the full platform + release Interact with all operators of the platform which are released together + stack Interact with stacks, which are ready-to-use product combinations + stacklets Interact with deployed stacklets, which are bundles of resources and containers required to run the product + demo Interact with demos, which are end-to-end usage demonstrations of the Stackable data platform + completions Generate shell completions for this tool + cache Interact with locally cached files + help Print this message or the help of the given subcommand(s) + +Options: + -l, --log-level + Log level this application uses + + --no-cache + Do not cache the remote (default) demo, stack and release files + + Cached files are saved at '$XDG_CACHE_HOME/stackablectl', which is usually + '$HOME/.cache/stackablectl' when not explicitly set. + + --offline + Do not request any remote files via the network + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +File options: + -d, --demo-file + Provide one or more additional (custom) demo file(s) + + Demos are loaded in the following order: Remote (default) demo file, custom + demo files provided via the 'STACKABLE_DEMO_FILES' environment variable, and + lastly demo files provided via the '-d/--demo-file' argument(s). If there are + demos with the same name, the last demo definition will be used. + + Use "stackablectl [OPTIONS] -d path/to/demos1.yaml -d path/to/demos2.yaml" + to provide multiple additional demo files. + + -s, --stack-file + Provide one or more additional (custom) stack file(s) + + Stacks are loaded in the following order: Remote (default) stack file, custom + stack files provided via the 'STACKABLE_STACK_FILES' environment variable, and + lastly demo files provided via the '-s/--stack-file' argument(s). If there are + stacks with the same name, the last stack definition will be used. + + Use "stackablectl [OPTIONS] -s path/to/stacks1.yaml -s path/to/stacks2.yaml" + to provide multiple additional stack files. + + -r, --release-file + Provide one or more additional (custom) release file(s) + + Releases are loaded in the following order: Remote (default) release file, + custom release files provided via the 'STACKABLE_RELEASE_FILES' environment + variable, and lastly release files provided via the '-r/--release-file' + argument(s). If there are releases with the same name, the last release + definition will be used. + + Use "stackablectl [OPTIONS] -r path/to/releases1.yaml -r path/to/releases2.yaml" + to provide multiple additional release files. + +Helm repository options: + --helm-repo-stable + Provide a custom Helm stable repository URL + + [default: https://repo.stackable.tech/repository/helm-stable/] + + --helm-repo-test + Provide a custom Helm test repository URL + + [default: https://repo.stackable.tech/repository/helm-test/] + + --helm-repo-dev + Provide a custom Helm dev repository URL + + [default: https://repo.stackable.tech/repository/helm-dev/] +---- diff --git a/docs/modules/stackablectl/pages/commands/operator.adoc b/docs/modules/stackablectl/pages/commands/operator.adoc new file mode 100644 index 00000000..a20c84f3 --- /dev/null +++ b/docs/modules/stackablectl/pages/commands/operator.adoc @@ -0,0 +1,203 @@ += stackablectl operator +:page-aliases: stackablectl::commands/operator.adoc + +The `stackable operator` command allows the listing, installation and removal of Stackable operators. Operators manage +the individual data products of the Stackable Data Platform. + +This command manages individual operators. It is intended for people with experience working on the Stackable Data +Platform. If you want an easy way to get started or don't know which products and versions to install, using the +xref:commands/release.adoc[`stackablectl release`] command is recommended. This command will install a bundle of +operators from an official Stackable release. + +== General Usage + +// Autogenerated by cargo xtask gen-docs. DO NOT CHANGE MANUALLY! +[source,console] +---- +$ stackablectl operator +Interact with single operator instead of the full platform + +Usage: operator [OPTIONS] + +Commands: + list List available operators + describe Print out detailed operator information + install Install one or more operators + uninstall Uninstall one or more operators + installed List installed operators + help Print this message or the help of the given subcommand(s) + +Options: + -l, --log-level + Log level this application uses + + --no-cache + Do not cache the remote (default) demo, stack and release files + + Cached files are saved at '$XDG_CACHE_HOME/stackablectl', which is usually + '$HOME/.cache/stackablectl' when not explicitly set. + + --offline + Do not request any remote files via the network + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +File options: + -d, --demo-file + Provide one or more additional (custom) demo file(s) + + Demos are loaded in the following order: Remote (default) demo file, custom + demo files provided via the 'STACKABLE_DEMO_FILES' environment variable, and + lastly demo files provided via the '-d/--demo-file' argument(s). If there are + demos with the same name, the last demo definition will be used. + + Use "stackablectl [OPTIONS] -d path/to/demos1.yaml -d path/to/demos2.yaml" + to provide multiple additional demo files. + + -s, --stack-file + Provide one or more additional (custom) stack file(s) + + Stacks are loaded in the following order: Remote (default) stack file, custom + stack files provided via the 'STACKABLE_STACK_FILES' environment variable, and + lastly demo files provided via the '-s/--stack-file' argument(s). If there are + stacks with the same name, the last stack definition will be used. + + Use "stackablectl [OPTIONS] -s path/to/stacks1.yaml -s path/to/stacks2.yaml" + to provide multiple additional stack files. + + -r, --release-file + Provide one or more additional (custom) release file(s) + + Releases are loaded in the following order: Remote (default) release file, + custom release files provided via the 'STACKABLE_RELEASE_FILES' environment + variable, and lastly release files provided via the '-r/--release-file' + argument(s). If there are releases with the same name, the last release + definition will be used. + + Use "stackablectl [OPTIONS] -r path/to/releases1.yaml -r path/to/releases2.yaml" + to provide multiple additional release files. + +Helm repository options: + --helm-repo-stable + Provide a custom Helm stable repository URL + + [default: https://repo.stackable.tech/repository/helm-stable/] + + --helm-repo-test + Provide a custom Helm test repository URL + + [default: https://repo.stackable.tech/repository/helm-test/] + + --helm-repo-dev + Provide a custom Helm dev repository URL + + [default: https://repo.stackable.tech/repository/helm-dev/] +---- + +== Listing Available Operators + +To list the operators that are part of the Stackable Data Platform and their stable versions, run the following +command: + +[source,console] +---- +$ stackablectl operator list +┌────┬───────────┬────────────────────────────────────────────────────────────────────────────────────────┐ +│ # ┆ OPERATOR ┆ STABLE VERSIONS │ +╞════╪═══════════╪════════════════════════════════════════════════════════════════════════════════════════╡ +│ 1 ┆ airflow ┆ 0.1.0, 0.2.0, 0.3.0, 0.4.0, 0.5.0, 0.6.0, 23.1.0, 23.4.0, 23.4.1, 23.7.0 │ +├╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 2 ┆ ... ┆ ... │ +├╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 15 ┆ zookeeper ┆ 0.10.0, 0.11.0, 0.12.0, 0.6.0, 0.7.0, 0.8.0, 0.9.0, 23.1.0, 23.4.0, 23.4.1, 23.7.0 │ +└────┴───────────┴────────────────────────────────────────────────────────────────────────────────────────┘ +---- + +This command only includes the stable versions of every operator for clarity. If you're interested in a particular +version of an operator, you can use the `describe` command to get more details for a specific operator as follows: + +[source,console] +---- +$ stackablectl operator describe airflow + OPERATOR airflow + STABLE VERSIONS 0.1.0, 0.2.0, 0.3.0, 0.4.0, 0.5.0, 0.6.0, 23.1.0, 23.4.0, 23.4.1, 23.7.0 + TEST VERSIONS 0.0.0-pr303, 0.0.0-pr304, 0.0.0-pr305, 0.0.0-pr307, 0.0.0-pr308, 0.0.0-pr310, 0.0.0-pr311, 0.0.0-pr312, 0.0.0-pr314, 0.0.0-pr315, + 0.0.0-pr316, 0.0.0-pr317 + DEV VERSIONS 0.0.0-dev +---- + +== Installing Operators + +If you want to access a Kubernetes cluster, make sure your https://kubernetes.io/docs/tasks/tools/#kubectl[`kubectl`] +Kubernetes client is configured to interact with the Kubernetes cluster. After that, run the following command, which +will install the operators in their latest nightly version - built from the main branch of the operators. + +[source,console] +---- +$ stackablectl operator install airflow commons secret +Installing 3 operators +Installed airflow operator +Installed commons operator +Installed secret operator +Installed 3 operators +---- + +If you don't have a Kubernetes cluster available, `stackablectl` can spin up a https://kind.sigs.k8s.io/[kind] or +https://minikube.sigs.k8s.io/docs/[minikube] Kubernetes cluster for you. Based on the type of local cluster you want to +use, ensure you have either `kind` or `minikube` installed on your system. See +xref:commands/demo.adoc#_using_a_local_kubernetes_cluster[here] for more information. + +With this command, we installed the operator for Apache Airflow and two operators needed internally by the Stackable +Data Platform (commons and secret). As we didn't specify a specific version to install, the operators were installed in +the latest nightly version - built from the main branch of the operators. If you want to install a specific version, you +can add the version to each operator to install as follows: + +[source,console] +---- +$ stackablectl operator install airflow=23.7 commons=23.7 secret=23.7 +Installing 3 operators +Installed airflow=23.7 operator +Installed commons=23.7 operator +Installed secret=23.7 operator +Installed 3 operators +---- + +As you can see, the three operators were installed in the requested version. + +Remember: If you want to install a recommended and tested set of operator versions, look at the +xref:commands/release.adoc[`stackablectl release`] command. + +== Listing Installed Operators + +After installing some operators, you can list which operators are installed in your Kubernetes cluster: + +[source,console] +---- +$ stackablectl operator installed +┌──────────────────┬─────────┬─────────────────────┬──────────┬──────────────────────────────────────────┐ +│ OPERATOR ┆ VERSION ┆ NAMESPACE ┆ STATUS ┆ LAST UPDATED │ +╞══════════════════╪═════════╪═════════════════════╪══════════╪══════════════════════════════════════════╡ +│ airflow-operator ┆ 23.7.0 ┆ stackable-operators ┆ deployed ┆ 2023-08-23 17:33:01.509777626 +0200 CEST │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ commons-operator ┆ 23.7.0 ┆ stackable-operators ┆ deployed ┆ 2023-08-23 17:33:04.012698515 +0200 CEST │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ secret-operator ┆ 23.7.0 ┆ stackable-operators ┆ deployed ┆ 2023-08-23 17:33:06.328410802 +0200 CEST │ +└──────────────────┴─────────┴─────────────────────┴──────────┴──────────────────────────────────────────┘ +---- + +== Uninstalling Operators + +You can use the `stackablectl operator uninstall` command to uninstall the operators again. + +[source,console] +---- +$ stackablectl operator uninstall airflow commons secret +The release airflow-operator was successfully uninstalled. +The release commons-operator was successfully uninstalled. +The release secret-operator was successfully uninstalled. +Uninstalled 3 operators +---- diff --git a/docs/modules/stackablectl/pages/commands/release.adoc b/docs/modules/stackablectl/pages/commands/release.adoc new file mode 100644 index 00000000..bffde9a2 --- /dev/null +++ b/docs/modules/stackablectl/pages/commands/release.adoc @@ -0,0 +1,214 @@ += stackablectl release +:page-aliases: stackablectl::commands/release.adoc + +A release is a bundle of operators of a specific stable version. The stable versions of the operators are tested and +proven to work hand in hand. If you want to install a single individual operator, look at the +xref:commands/operator.adoc[] command. + +// Autogenerated by cargo xtask gen-docs. DO NOT CHANGE MANUALLY! +[source,console] +---- +$ stackablectl release +Interact with all operators of the platform which are released together + +Usage: release [OPTIONS] + +Commands: + list List available releases + describe Print out detailed release information + install Install a specific release + uninstall Uninstall a release + help Print this message or the help of the given subcommand(s) + +Options: + -l, --log-level + Log level this application uses + + --no-cache + Do not cache the remote (default) demo, stack and release files + + Cached files are saved at '$XDG_CACHE_HOME/stackablectl', which is usually + '$HOME/.cache/stackablectl' when not explicitly set. + + --offline + Do not request any remote files via the network + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +File options: + -d, --demo-file + Provide one or more additional (custom) demo file(s) + + Demos are loaded in the following order: Remote (default) demo file, custom + demo files provided via the 'STACKABLE_DEMO_FILES' environment variable, and + lastly demo files provided via the '-d/--demo-file' argument(s). If there are + demos with the same name, the last demo definition will be used. + + Use "stackablectl [OPTIONS] -d path/to/demos1.yaml -d path/to/demos2.yaml" + to provide multiple additional demo files. + + -s, --stack-file + Provide one or more additional (custom) stack file(s) + + Stacks are loaded in the following order: Remote (default) stack file, custom + stack files provided via the 'STACKABLE_STACK_FILES' environment variable, and + lastly demo files provided via the '-s/--stack-file' argument(s). If there are + stacks with the same name, the last stack definition will be used. + + Use "stackablectl [OPTIONS] -s path/to/stacks1.yaml -s path/to/stacks2.yaml" + to provide multiple additional stack files. + + -r, --release-file + Provide one or more additional (custom) release file(s) + + Releases are loaded in the following order: Remote (default) release file, + custom release files provided via the 'STACKABLE_RELEASE_FILES' environment + variable, and lastly release files provided via the '-r/--release-file' + argument(s). If there are releases with the same name, the last release + definition will be used. + + Use "stackablectl [OPTIONS] -r path/to/releases1.yaml -r path/to/releases2.yaml" + to provide multiple additional release files. + +Helm repository options: + --helm-repo-stable + Provide a custom Helm stable repository URL + + [default: https://repo.stackable.tech/repository/helm-stable/] + + --helm-repo-test + Provide a custom Helm test repository URL + + [default: https://repo.stackable.tech/repository/helm-test/] + + --helm-repo-dev + Provide a custom Helm dev repository URL + + [default: https://repo.stackable.tech/repository/helm-dev/] +---- + +== Browsing Available Releases + +To list the available Stackable releases run the following command: + +[source,console] +---- +$ stackablectl release list +┌───┬─────────┬──────────────┬─────────────────────────────────────────────────────────────────────────────┐ +│ # ┆ RELEASE ┆ RELEASE DATE ┆ DESCRIPTION │ +╞═══╪═════════╪══════════════╪═════════════════════════════════════════════════════════════════════════════╡ +│ 1 ┆ 23.7 ┆ 2023-07-26 ┆ Sixth release focusing on resources and pod overrides │ +├╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 2 ┆ 23.4 ┆ 2023-05-17 ┆ Fifth release focusing on affinities and product status │ +├╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 3 ┆ 23.1 ┆ 2023-01-27 ┆ Fourth release focusing on image selection and logging │ +├╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 4 ┆ 22.11 ┆ 2022-11-14 ┆ Third release focusing on resource management │ +├╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 5 ┆ 22.09 ┆ 2022-09-09 ┆ Second release focusing on security and OpenShift support │ +├╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 6 ┆ 22.06 ┆ 2022-06-30 ┆ First official release of the Stackable Data Platform │ +├╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 7 ┆ latest ┆ 2023-07-26 ┆ Always pointing to the latest stable version of the Stackable Data Platform │ +├╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 8 ┆ dev ┆ 2023-01-27 ┆ Development versions from main branch. Not stable! │ +└───┴─────────┴──────────────┴─────────────────────────────────────────────────────────────────────────────┘ +---- + +Detailed information of a release can be queried with the `stackablectl release describe` command: + +[source,console] +---- +$ stackablectl release describe 23.7 + RELEASE 23.7 + RELEASE DATE 2023-07-26 + DESCRIPTION Sixth release focusing on resources and pod overrides + INCLUDED PRODUCTS PRODUCT OPERATOR VERSION + airflow 23.7.0 + commons 23.7.0 + druid 23.7.0 + hbase 23.7.0 + hdfs 23.7.0 + hive 23.7.0 + kafka 23.7.0 + listener 23.7.0 + nifi 23.7.0 + opa 23.7.0 + secret 23.7.0 + spark-k8s 23.7.0 + superset 23.7.0 + trino 23.7.0 + zookeeper 23.7.0 +---- + +In the output you can see which product operators are included in the specific release. + +== Installing Releases + +If you want to access a Kubernetes cluster, make sure your https://kubernetes.io/docs/tasks/tools/#kubectl[`kubectl`] +Kubernetes client is configured to interact with the Kubernetes cluster. After that, run the following command: + +[source,console] +---- +$ stackablectl release install 23.7 +Installed product airflow=23.7.0 +Installed product commons=23.7.0 +Installed product druid=23.7.0 +Installed product hbase=23.7.0 +Installed product hdfs=23.7.0 +Installed product hive=23.7.0 +Installed product kafka=23.7.0 +Installed product listener=23.7.0 +Installed product nifi=23.7.0 +Installed product opa=23.7.0 +Installed product secret=23.7.0 +Installed product spark-k8s=23.7.0 +Installed product superset=23.7.0 +Installed product trino=23.7.0 +Installed product zookeeper=23.7.0 +Installed release 23.7 +---- + +If you don't have a Kubernetes cluster available, `stackablectl` can spin up a https://kind.sigs.k8s.io/[kind] or +https://minikube.sigs.k8s.io/docs/[minikube] Kubernetes cluster for you. Based on the type of local cluster you want to +use, ensure you have either `kind` or `minikube` installed on your system. See +xref:commands/demo.adoc#_using_a_local_kubernetes_cluster[here] for more information. + +[source,console] +---- +$ stackablectl release install 23.7 -c kind +Creating cluster "stackable-data-platform" ... + ✓ Ensuring node image (kindest/node:v1.26.3) 🖼 + ✓ Preparing nodes 📦 📦 + ✓ Writing configuration 📜 + ✓ Starting control-plane 🕹️ + ✓ Installing CNI 🔌 + ✓ Installing StorageClass 💾 + ✓ Joining worker nodes 🚜 +Set kubectl context to "kind-stackable-data-platform" +You can now use your cluster with: + +kubectl cluster-info --context kind-stackable-data-platform + +Have a nice day! 👋 +Installed product airflow=23.7.0 +Installed product commons=23.7.0 +Installed product druid=23.7.0 +Installed product hbase=23.7.0 +Installed product hdfs=23.7.0 +Installed product hive=23.7.0 +Installed product kafka=23.7.0 +Installed product listener=23.7.0 +Installed product nifi=23.7.0 +Installed product opa=23.7.0 +Installed product secret=23.7.0 +Installed product spark-k8s=23.7.0 +Installed product superset=23.7.0 +Installed product trino=23.7.0 +Installed product zookeeper=23.7.0 +Installed release 23.7 +---- diff --git a/docs/modules/stackablectl/pages/commands/stack.adoc b/docs/modules/stackablectl/pages/commands/stack.adoc new file mode 100644 index 00000000..6e6ee220 --- /dev/null +++ b/docs/modules/stackablectl/pages/commands/stack.adoc @@ -0,0 +1,87 @@ += stackablectl stack +:page-aliases: stackablectl::commands/stack.adoc + +// Autogenerated by cargo xtask gen-docs. DO NOT CHANGE MANUALLY! +[source,console] +---- +$ stackablectl stack +Interact with stacks, which are ready-to-use product combinations + +Usage: stack [OPTIONS] + +Commands: + list List available stacks + describe Describe a specific stack + install Install a specific stack + help Print this message or the help of the given subcommand(s) + +Options: + -l, --log-level + Log level this application uses + + --no-cache + Do not cache the remote (default) demo, stack and release files + + Cached files are saved at '$XDG_CACHE_HOME/stackablectl', which is usually + '$HOME/.cache/stackablectl' when not explicitly set. + + --offline + Do not request any remote files via the network + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +File options: + -d, --demo-file + Provide one or more additional (custom) demo file(s) + + Demos are loaded in the following order: Remote (default) demo file, custom + demo files provided via the 'STACKABLE_DEMO_FILES' environment variable, and + lastly demo files provided via the '-d/--demo-file' argument(s). If there are + demos with the same name, the last demo definition will be used. + + Use "stackablectl [OPTIONS] -d path/to/demos1.yaml -d path/to/demos2.yaml" + to provide multiple additional demo files. + + -s, --stack-file + Provide one or more additional (custom) stack file(s) + + Stacks are loaded in the following order: Remote (default) stack file, custom + stack files provided via the 'STACKABLE_STACK_FILES' environment variable, and + lastly demo files provided via the '-s/--stack-file' argument(s). If there are + stacks with the same name, the last stack definition will be used. + + Use "stackablectl [OPTIONS] -s path/to/stacks1.yaml -s path/to/stacks2.yaml" + to provide multiple additional stack files. + + -r, --release-file + Provide one or more additional (custom) release file(s) + + Releases are loaded in the following order: Remote (default) release file, + custom release files provided via the 'STACKABLE_RELEASE_FILES' environment + variable, and lastly release files provided via the '-r/--release-file' + argument(s). If there are releases with the same name, the last release + definition will be used. + + Use "stackablectl [OPTIONS] -r path/to/releases1.yaml -r path/to/releases2.yaml" + to provide multiple additional release files. + +Helm repository options: + --helm-repo-stable + Provide a custom Helm stable repository URL + + [default: https://repo.stackable.tech/repository/helm-stable/] + + --helm-repo-test + Provide a custom Helm test repository URL + + [default: https://repo.stackable.tech/repository/helm-test/] + + --helm-repo-dev + Provide a custom Helm dev repository URL + + [default: https://repo.stackable.tech/repository/helm-dev/] +---- diff --git a/docs/modules/stackablectl/pages/commands/stacklets.adoc b/docs/modules/stackablectl/pages/commands/stacklets.adoc new file mode 100644 index 00000000..e00a2eab --- /dev/null +++ b/docs/modules/stackablectl/pages/commands/stacklets.adoc @@ -0,0 +1,90 @@ += stackablectl stacklets +:page-aliases: stackablectl::commands/services.adoc + +// Autogenerated by cargo xtask gen-docs. DO NOT CHANGE MANUALLY! +[source,console] +---- +$ stackablectl stacklets +Interact with deployed stacklets, which are bundles of resources and containers +required to run the product. + +Each stacklet consists of init containers, app containers, sidecar containers +and additional Kubernetes resources like StatefulSets, ConfigMaps, Services and +CRDs. + +Usage: stacklets [OPTIONS] + +Commands: + list List deployed services + help Print this message or the help of the given subcommand(s) + +Options: + -l, --log-level + Log level this application uses + + --no-cache + Do not cache the remote (default) demo, stack and release files + + Cached files are saved at '$XDG_CACHE_HOME/stackablectl', which is usually + '$HOME/.cache/stackablectl' when not explicitly set. + + --offline + Do not request any remote files via the network + + -h, --help + Print help (see a summary with '-h') + + -V, --version + Print version + +File options: + -d, --demo-file + Provide one or more additional (custom) demo file(s) + + Demos are loaded in the following order: Remote (default) demo file, custom + demo files provided via the 'STACKABLE_DEMO_FILES' environment variable, and + lastly demo files provided via the '-d/--demo-file' argument(s). If there are + demos with the same name, the last demo definition will be used. + + Use "stackablectl [OPTIONS] -d path/to/demos1.yaml -d path/to/demos2.yaml" + to provide multiple additional demo files. + + -s, --stack-file + Provide one or more additional (custom) stack file(s) + + Stacks are loaded in the following order: Remote (default) stack file, custom + stack files provided via the 'STACKABLE_STACK_FILES' environment variable, and + lastly demo files provided via the '-s/--stack-file' argument(s). If there are + stacks with the same name, the last stack definition will be used. + + Use "stackablectl [OPTIONS] -s path/to/stacks1.yaml -s path/to/stacks2.yaml" + to provide multiple additional stack files. + + -r, --release-file + Provide one or more additional (custom) release file(s) + + Releases are loaded in the following order: Remote (default) release file, + custom release files provided via the 'STACKABLE_RELEASE_FILES' environment + variable, and lastly release files provided via the '-r/--release-file' + argument(s). If there are releases with the same name, the last release + definition will be used. + + Use "stackablectl [OPTIONS] -r path/to/releases1.yaml -r path/to/releases2.yaml" + to provide multiple additional release files. + +Helm repository options: + --helm-repo-stable + Provide a custom Helm stable repository URL + + [default: https://repo.stackable.tech/repository/helm-stable/] + + --helm-repo-test + Provide a custom Helm test repository URL + + [default: https://repo.stackable.tech/repository/helm-test/] + + --helm-repo-dev + Provide a custom Helm dev repository URL + + [default: https://repo.stackable.tech/repository/helm-dev/] +---- diff --git a/docs/modules/stackablectl/pages/customization/add-demo.adoc b/docs/modules/stackablectl/pages/customization/add-demo.adoc new file mode 100644 index 00000000..8c0b3f61 --- /dev/null +++ b/docs/modules/stackablectl/pages/customization/add-demo.adoc @@ -0,0 +1,55 @@ += Add a Demo +:page-aliases: stackablectl::customization/add_demo.adoc + +== Motivation + +When you have developed a new data pipeline or product, you often want to show it to other colleagues or potential +clients. You can create a custom demo to achieve this - enabling easy reproduction and sharing with others. + +Please remember that a demo requires a stack to run on. Look at the chapter xref:customization/add-stack.adoc[] on +creating your stack. + +== 1. Create a demos.yaml + +For a custom demo, you must create a `mycorp-demos.yaml` containing demos according to the format defined by +https://github.com/stackabletech/stackablectl/blob/main/demos/demos-v1.yaml[the Stackable provided demos]. + +As of writing, a `demos.yaml` file could look as follows: + +[source,yaml] +---- +demos: + mycorp-warehouse-realtime-analysis: + description: Using our internal warehouse stack we show how you can analyze real-time data and build interactive Dashboards + documentation: https://my.corp/some-blogpost-anouncing-demo.html + stackableStack: mycorp-warehouse + labels: + - mycorp + - warehouse + - real-time + - dashboards + manifests: + - plainYaml: https://my.corp/demos/mycorp-warehouse-realtime-analysis/create-testdata-ingestion-job.yaml + - plainYaml: https://my.corp/demos/mycorp-warehouse-realtime-analysis/create-trino-tables.yaml + - plainYaml: https://my.corp/demos/mycorp-warehouse-realtime-analysis/setup-superset.yaml +---- + +== 2. Using the Custom `demos.yaml` File + +After creating the `mycorp-demos.yaml` file, it can be added to the available demos in `stackablectl` via the CLI +argument `--demo-file mycorp-demos.yaml`. + +The argument to `--demo-file` can be a path to a file on the local filesystem or a URL. For example, the demo file can +be put into a central Git repository and referenced by all teams or clients. Multiple "`--demo-file` flags can be +specified to include multiple demo files. + +Additionally, the custom file can be provided using an environment variable. The variable can be defined by `export` in +the shell or a `.env` file. + +[source,ini] +---- +STACKABLECTL_DEMO_FILES=demos1.yml,demos2.yml +---- + +Every additional demo will be added to the already existing demos in `stackablectl`, so all the available demo files +will be merged. diff --git a/docs/modules/stackablectl/pages/customization/add-release.adoc b/docs/modules/stackablectl/pages/customization/add-release.adoc new file mode 100644 index 00000000..5e85ecf0 --- /dev/null +++ b/docs/modules/stackablectl/pages/customization/add-release.adoc @@ -0,0 +1,63 @@ += Add a Release +:page-aliases: stackablectl::customization/add_release.adoc + +== Motivation + +If advanced users of the Stackable Platform want to define their internal release within their company, they can easily +add their release, which has the following benefits: + +* Identical operator versions across the whole company produce more uniform environments, making debugging and helping + other teams easier. +* Suppose the company is only interested in a subset of the available operators. In that case, you can only add your + relevant operators to your release and install some other operators. + +== 1. Create a releases.yaml + +For a custom release you must create a `mycorp-releases.yaml` containing releases according to the format defined by +https://github.com/stackabletech/release/blob/main/releases.yaml[the Stackable provided releases]. You can pick any +number of operators in arbitrary versions. + +As of writing, a `releases.yaml` file could look as follows: + +[source,yaml] +---- +releases: + mycorp-release1: + releaseDate: 2022-11-10 + description: Internal release of the SDP + products: + commons: + operatorVersion: 0.4.0 + hive: + operatorVersion: 0.8.0 + opa: + operatorVersion: 0.11.0 + secret: + operatorVersion: 0.6.0 + spark-k8s: + operatorVersion: 0.6.0 + superset: + operatorVersion: 0.7.0 + trino: + operatorVersion: 0.8.0 +---- + +== 2. Using the Custom `releases.yaml` File + +After creating the `mycorp-releases.yaml` file, it can be added to the available releases in `stackablectl` via the CLI +argument `--release-file mycorp-demos.yaml`. + +The argument to `--release-file` can be a path to a file on the local filesystem or a URL. For example, the release file +can be put into a central Git repository and referenced by all teams or clients. Multiple "`--release-file` flags can be +specified to include multiple release files. + +Additionally, the custom file can be provided using an environment variable. The variable can be defined by `export` in +the shell or a `.env` file. + +[source,ini] +---- +STACKABLECTL_RELEASE_FILES=releases1.yml,releases2.yml +---- + +Every additional release will be added to the already existing releases in `stackablectl`, so all the available release +files will be merged. diff --git a/docs/modules/stackablectl/pages/customization/add-stack.adoc b/docs/modules/stackablectl/pages/customization/add-stack.adoc new file mode 100644 index 00000000..7b7d0309 --- /dev/null +++ b/docs/modules/stackablectl/pages/customization/add-stack.adoc @@ -0,0 +1,67 @@ += Add a Stack +:page-aliases: stackablectl::customization/add_stack.adoc + +== Motivation + +If your company or clients have multiple similar setups or reference architectures, making them readily available to all +employees or clients could make sense. All product versions are pinned in the custom-defined stack, so you can easily +spin up a stack containing the identical versions as your production setup. You can use your defined stack to give it to +colleagues or potential customers to show the overall architecture of the Data Platform you will build. + +Please keep in mind that a stack requires a release to run on. In most cases, the stackable provided release should work +fine, but you can also look at the chapter xref:customization/add-release.adoc[] on creating your release. + +== 1. Create a stacks.yaml + +For a custom stack you need to create a `mycorp-stacks.yaml` containing stacks according to the format defined by +https://github.com/stackabletech/stackablectl/blob/main/stacks/stacks-v1.yaml[the Stackable provided stacks]. + +As of writing a `stacks.yaml` file could look as follows: + +[source,yaml] +---- +stacks: + mycorp-warehouse: + description: Internal stack we use to build our warehouses + stackableRelease: 22.09 # or use your custom release mycorp-release1 + labels: + - mycorp + - warehouse + manifests: + # We have Superset in out Stack, which needs a postgressql instance + # So let's install that first + - helmChart: &template-postgresql-superset + releaseName: postgresql-superset + name: postgresql + repo: + name: bitnami + url: https://charts.bitnami.com/bitnami/ + version: 11.0.0 + options: + auth: + username: superset + password: superset + database: superset + - plainYaml: https://my.corp/stacks/mycorp-warehouse/trino.yaml + - plainYaml: https://my.corp/stacks/mycorp-warehouse/superset.yaml +---- + +== 2. Using the Custom `stacks.yaml` File + +After creating the `mycorp-stacks.yaml` file, it can be added to the available stacks in `stackablectl` via the CLI +argument `--stack-file mycorp-demos.yaml`. + +The argument to `--stack-file` can be a path to a file on the local filesystem or a URL. For example, the demo file can +be put into a central Git repository and referenced by all teams or clients. Multiple "`--stack-file` flags can be +specified to include multiple stack files. + +Additionally, the custom file can be provided using an environment variable. The variable can be defined by `export` in +the shell or a `.env` file. + +[source,ini] +---- +STACKABLECTL_STACK_FILES=stacks1.yml,stacks2.yml +---- + +Every additional stack will be added to the already existing stacks in `stackablectl`, so all the available stack files +will be merged. diff --git a/docs/modules/stackablectl/pages/customization/index.adoc b/docs/modules/stackablectl/pages/customization/index.adoc new file mode 100644 index 00000000..43edb209 --- /dev/null +++ b/docs/modules/stackablectl/pages/customization/index.adoc @@ -0,0 +1,8 @@ += Customization +:page-aliases: stackablectl::customization/index.adoc + +If you're working for a large company, chances are that there are multiple teams using the Stackable Data Platform. +A single team can also operate multiple Stackable Data Platforms. `stackablectl` is build in a way customers or +developers can define their own releases, stacks and demos! This way it is possible to cover the following use-cases. + +Any additional demos/stacks/releases you specify, will be added to the already existing ones provided by Stackable. diff --git a/docs/modules/stackablectl/pages/index.adoc b/docs/modules/stackablectl/pages/index.adoc new file mode 100644 index 00000000..e2e2f6ba --- /dev/null +++ b/docs/modules/stackablectl/pages/index.adoc @@ -0,0 +1,54 @@ += stackablectl +:page-aliases: stackablectl::index.adoc + +The `stackablectl` command line tool interacts with the Stackable data platform. It can install individual +operators as well as platform releases. It also ships with a set of pre-built xref:commands/demo.adoc[`demos`] that +utilize different data products of the Platform, e.g. an end-to-end data pipeline. + +The installation of `stackablectl` is described in xref:installation.adoc[installation guide]. To get started, please +follow the xref:quickstart.adoc[quickstart guide]. + +In general, use `stackablectl --help` to find out more about how to use the tool or specific options. Every subcommand +supports the help flag. For example, `stackablectl release install --help` will show the usage test for installing a +release. You can also use an abbreviation instead of typing out the complete commands. E.g. `stackablectl operator list` +can also be written as `stackablectl op ls` + +A Kubernetes cluster is required to use the Stackable Data Platform, as all products and operators run on Kubernetes. If +you don't have a Kubernetes cluster, `stackablectl` can spin up a https://kind.sigs.k8s.io/[kind] Kubernetes Cluster for +you. + +The deployed services are separated into three different layers, as illustrated below: + +image::layers.png[Layers of the deployed services] + +== Operators + +This layer consists of Stackable operators managing the individual data products. They can be installed one by one with +the xref:commands/operator.adoc[`operator`] command or from a release with the xref:commands/release.adoc[`release`] +command, which is recommended. A release is a well-playing bundle of operators that Stackable has extensively tested. + +== Stacks + +A stack is a collection of ready-to-use Stackable data products and required third-party services like Postgresql or +MinIO. + +Stacks are installed with the xref:commands/stack.adoc[`stack`] command. A stack needs a release (of Stackable +operators) to run on. That's why a stack depends on a release, which gets automatically installed when a stack is +installed. + +== Demos + +A demo is an end-to-end demonstration of the usage of the Stackable data platform. It contains: + +. Installing a Stackable release +. Spinning up a stack +. Performing the actual demo +.. Prepare some test data +.. Process test data +.. Visualize results (optional) + +Demos are installed with the xref:commands/demo.adoc[`demo`] command. A demo needs a stack to run on. That's why a demo +depends on a stack, which gets automatically installed when a demo is installed. The stack, in turn, will install the +needed Stackable release. + +You can browse the available demos on the xref:demos:index.adoc[demo] page. diff --git a/docs/modules/stackablectl/pages/installation.adoc b/docs/modules/stackablectl/pages/installation.adoc new file mode 100644 index 00000000..e5ec9231 --- /dev/null +++ b/docs/modules/stackablectl/pages/installation.adoc @@ -0,0 +1,125 @@ += Installation +:page-aliases: stackablectl::installation.adoc + +:latest-release: https://github.com/stackabletech/stackable-cockpit/releases/tag/stackablectl-1.0.0-rc1 +:fish-comp-loations: https://fishshell.com/docs/current/completions.html#where-to-put-completions + +== Using Pre-Compiled Binaries + +Stackable provides pre-compiled binaries of stackablectl, which should work on most environments such as Windows, macOS, +and Linux distributions like Ubuntu and Arch. You can also build the binary from source. More information about the +manual building steps can be found in xref:#building-from-source[this] section. + +=== Linux + +Download the `stackablectl-x86_64-unknown-linux-gnu` binary file from the link:{latest-release}[latest release], then +rename the file to `stackablectl`. You can also use the following command: + +[source,console] +---- +wget -O stackablectl https://github.com/stackabletech/stackable-cockpit/releases/download/stackablectl-1.0.0-rc1/stackablectl-x86_64-unknown-linux-gnu +# or +curl -L -o stackablectl https://github.com/stackabletech/stackable-cockpit/releases/download/stackablectl-1.0.0-rc1/stackablectl-x86_64-unknown-linux-gnu +---- + +Mark the binary as executable: + +[source,console] +---- +chmod +x stackablectl +---- + +Then, make sure it is present in your `$PATH`, like `/usr/local/bin`. + +=== macOS + +Download the `stackablectl-x86_64-apple-darwin` binary file for Intel-based devices or the +`stackablectl-aarch64-apple-darwin` binary file for ARM-based devices from the link:{latest-release}[latest release], +then rename the file to `stackablectl`. You can also use the following command: + +[source,console] +---- +wget -O stackablectl https://github.com/stackabletech/stackable-cockpit/releases/download/stackablectl-1.0.0-rc1/stackablectl-x86_64-apple-darwin +# or +wget -O stackablectl https://github.com/stackabletech/stackable-cockpit/releases/download/stackablectl-1.0.0-rc1/stackablectl-aarch64-apple-darwin +---- + +Mark the binary as executable: + +[source,console] +---- +chmod +x stackablectl +---- + +If macOS denies the execution of `stackablectl` go to Settings -> Security & Privacy -> General. Here you will see a pop +up asking if you want to allow access for `stackablectl`. You must allow access. + +=== Windows + +Currently, there are no pre-built binaries available for Windows. Please refer to xref:#building-from-source[this] +section to learn how to build the binary from source. + +[#building-from-source] +== Building from Source + +To build `stackablectl` from source you need to have the following tools installed: + +* *The Rust toolchain:* Needed for compiling the source code of `stackablectl` itself. Use https://rustup.rs/[rustup] to + easily install all required tools. +* *The Go toolchain:* Needed for compiling a wrapper around the Go library `go-helm-client`. + +Continue by cloning the repository located at https://github.com/stackabletech/stackable-cockpit. Then, compile the +binary using the following command: + +[source,console] +---- +cargo build -p stackablectl --release +---- + +After a successful build, the binary will be placed in `target/release/stackablectl`. Copy it to your systems path to +access it from anywhere if you like: + +[source,console] +---- +cp target/release/stackablectl /usr/local/bin +---- + +== Shell Completions + +We provide completions for `stackablectl` for major shells out there. Currently, ZSH, Fish and Bash are supported. The +repository provides pre-generated completion files. These can be downloaded and copied to the appropriate location on +your system. + +=== ZSH + +Download the completions file and place it in `/usr/local/share/zsh/site-functions/` to load it automatically. + +[source,console] +---- +wget https://raw.githubusercontent.com/stackabletech/stackable-cockpit/main/extra/completions/_stackablectl +mv _stackablectl /usr/local/share/zsh/site-functions/ +---- + +=== Fish + +Download the completions file and place it in any of the supported location listed {fish-comp-loations}[here]. + +[source,console] +---- +wget https://raw.githubusercontent.com/stackabletech/stackable-cockpit/main/extra/completions/stackablectl.fish +---- + +=== Bash + +Download the completions file and place it in `/etc/bash_completion.d/` to load it automatically. + +[source,console] +---- +wget https://raw.githubusercontent.com/stackabletech/stackable-cockpit/main/extra/completions/stackablectl.bash +mv stackablectl.bash /etc/bash_completion.d/ +---- + +''' + +You can generate the completions on your own by using the `stackablectl completions` command. See +xref:commands/completions.adoc[here] for more information. diff --git a/docs/modules/stackablectl/pages/quickstart.adoc b/docs/modules/stackablectl/pages/quickstart.adoc new file mode 100644 index 00000000..825b6a14 --- /dev/null +++ b/docs/modules/stackablectl/pages/quickstart.adoc @@ -0,0 +1,104 @@ += Quickstart +:page-aliases: stackablectl::quickstart.adoc + +In this Quickstart guide, you will install a xref:commands/demo.adoc[demo], which is an end-to-end demonstration of the +usage of the Stackable data platform. Please follow the xref:installation.adoc[Installation documentation] to install +the `stackablectl` tool. + +== Browse Available Demos + +Stackable provides a set of ready-to-use demos. They will automatically appear as `stackablectl` fetches the available +list of demos via the internet. To list the available demos, run the following command: + +[source,console] +---- +$ stackablectl demo list +┌────┬───────────────────────┬─────────┬─────────────────────────────────────────────────────────┐ +│ # ┆ NAME ┆ STACK ┆ DESCRIPTION │ +╞════╪═══════════════════════╪═════════╪═════════════════════════════════════════════════════════╡ +│ 1 ┆ airflow-scheduled-job ┆ airflow ┆ Activate a simple Airflow DAG to run continuously at a │ +│ ┆ ┆ ┆ set interval │ +├╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 2 ┆ ... ┆ ... ┆ ... │ +├╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ 11 ┆ local-demo-test-env ┆ test ┆ Test demo. Provided with the STACKABLE_DEMO_FILES env │ +│ ┆ ┆ ┆ var in the .env file │ +└────┴───────────────────────┴─────────┴─────────────────────────────────────────────────────────┘ +---- + +[NOTE] +==== +When you are on a Windows system you have to replace the `stackablectl` command with `stackablectl.exe`, e.g. +`stackablectl.exe demo list`. This applies to all commands below. +==== + +For this guide, we will use the xref:demos:trino-taxi-data.adoc[] demo. The installation of other available demos +should work the same way. You need to use the name of the chosen demo instead of `trino-taxi-data` in the following +commands. + +== Install Demo + +The installation depends on whether you already have a Kubernetes cluster to run the Stackable Data Platform. + +=== Using Existing Kubernetes Cluster + +If you want to access a Kubernetes cluster, make sure your https://kubernetes.io/docs/tasks/tools/#kubectl[`kubectl`] +Kubernetes client is configured to interact with the Kubernetes cluster. After that, run the following command: + +[source,console] +---- +$ stackablectl demo install trino-taxi-data +The release commons-operator was successfully installed. +The release hive-operator was successfully installed. +The release opa-operator was successfully installed. +The release secret-operator was successfully installed. +The release superset-operator was successfully installed. +The release trino-operator was successfully installed. +Installed demo trino-taxi-data + +Use "stackablectl operator installed" to display the installed operators +Use "stackablectl stacklet list" to display the installed stacklets +---- + +=== Using Local Kind Cluster + +If you don't have a Kubernetes cluster available, `stackablectl` can spin up a https://kind.sigs.k8s.io/[kind] +Kubernetes cluster for you. Make sure you have `kind` installed and run the following command: + +[source,console] +---- +$ stackablectl demo install trino-taxi-data -c kind +Creating cluster "stackable-data-platform" ... + ✓ Ensuring node image (kindest/node:v1.26.3) 🖼 + ✓ Preparing nodes 📦 📦 + ✓ Writing configuration 📜 + ✓ Starting control-plane 🕹️ + ✓ Installing CNI 🔌 + ✓ Installing StorageClass 💾 + ✓ Joining worker nodes 🚜 +Set kubectl context to "kind-stackable-data-platform" +You can now use your cluster with: + +kubectl cluster-info --context kind-stackable-data-platform + +Have a question, bug, or feature request? Let us know! https://kind.sigs.k8s.io/#community 🙂 +The release commons-operator was successfully installed. +The release hive-operator was successfully installed. +The release opa-operator was successfully installed. +The release secret-operator was successfully installed. +The release superset-operator was successfully installed. +The release trino-operator was successfully installed. +Installed demo trino-taxi-data + +Use "stackablectl operator installed" to display the installed operators +Use "stackablectl stacklet list" to display the installed stacklets +---- + +The demos create Kubernetes jobs that will populate test data and talk to the installed products to process the data. +Until the products are ready, the pods of these jobs are expected to fail with an error. They will get retried with an +exponentially growing backoff time. After the products are ready, they should turn green, and everything should settle +down. + +== Proceed with the Demo + +Please read the documentation on the demo xref:demos:trino-taxi-data.adoc[] on how to proceed with the demo. diff --git a/docs/modules/stackablectl/partials/instance-hint.adoc b/docs/modules/stackablectl/partials/instance-hint.adoc new file mode 100644 index 00000000..3860c9d4 --- /dev/null +++ b/docs/modules/stackablectl/partials/instance-hint.adoc @@ -0,0 +1,6 @@ +[NOTE] +==== +When a product instance has not finished starting yet, the service will have no endpoint. Depending on your internet +connectivity, creating all the product instances might take considerable time. A warning might be shown if the product +is not ready yet. +==== diff --git a/rust/stackable-cockpit/src/helm.rs b/rust/stackable-cockpit/src/helm.rs index 2d0cc4a6..d6ed4688 100644 --- a/rust/stackable-cockpit/src/helm.rs +++ b/rust/stackable-cockpit/src/helm.rs @@ -180,7 +180,7 @@ impl Display for HelmUninstallReleaseStatus { HelmUninstallReleaseStatus::Uninstalled(release_name) => { write!( f, - "The releas {} was successfully uninstalled.", + "The release {} was successfully uninstalled.", release_name ) } diff --git a/rust/stackable-cockpit/src/platform/operator/spec.rs b/rust/stackable-cockpit/src/platform/operator/spec.rs index 52d88976..fcd90062 100644 --- a/rust/stackable-cockpit/src/platform/operator/spec.rs +++ b/rust/stackable-cockpit/src/platform/operator/spec.rs @@ -1,7 +1,7 @@ use std::{fmt::Display, str::FromStr}; use snafu::Snafu; -use tracing::{debug, info, instrument}; +use tracing::{info, instrument}; use crate::{ constants::{HELM_REPO_NAME_DEV, HELM_REPO_NAME_STABLE, HELM_REPO_NAME_TEST}, @@ -175,7 +175,7 @@ impl OperatorSpec { let version = self.version.as_deref(); // Install using Helm - match helm::install_release_from_repo( + helm::install_release_from_repo( &self.name, &helm_name, helm::ChartVersion { @@ -186,13 +186,9 @@ impl OperatorSpec { None, namespace, true, - ) { - Ok(status) => { - debug!("{status}"); - Ok(()) - } - Err(err) => Err(err), - } + )?; + + Ok(()) } /// Uninstalls the operator using Helm. diff --git a/rust/stackable-cockpit/src/platform/stacklet/mod.rs b/rust/stackable-cockpit/src/platform/stacklet/mod.rs index b6404a1b..c8e1ca10 100644 --- a/rust/stackable-cockpit/src/platform/stacklet/mod.rs +++ b/rust/stackable-cockpit/src/platform/stacklet/mod.rs @@ -3,7 +3,7 @@ use kube::{core::GroupVersionKind, ResourceExt}; use serde::Serialize; use snafu::{ResultExt, Snafu}; use stackable_operator::status::condition::ClusterCondition; -use tracing::warn; +use tracing::info; #[cfg(feature = "openapi")] use utoipa::ToSchema; @@ -89,7 +89,7 @@ async fn list_stackable_stacklets( { Some(obj) => obj, None => { - warn!( + info!( "Failed to list services because the gvk {product_gvk:?} can not be resolved" ); continue; diff --git a/rust/stackablectl/src/cmds/operator.rs b/rust/stackablectl/src/cmds/operator.rs index 1c3f1fe7..e9a62a29 100644 --- a/rust/stackablectl/src/cmds/operator.rs +++ b/rust/stackablectl/src/cmds/operator.rs @@ -282,10 +282,8 @@ async fn install_cmd(args: &OperatorInstallArgs, common_args: &Cli) -> Result println!("Installed {} operator", operator.name), + Ok(_) => println!("Installed {} operator", operator), Err(err) => { return Err(CmdError::HelmError { source: err }); } diff --git a/rust/xtask/Cargo.toml b/rust/xtask/Cargo.toml index 3c0a95a2..35581ffc 100644 --- a/rust/xtask/Cargo.toml +++ b/rust/xtask/Cargo.toml @@ -5,10 +5,12 @@ edition = "2021" publish = false [dependencies] -stackablectl = { path = "../stackablectl" } -stackable-cockpitd = { path = "../stackable-cockpitd" } -clap_complete = { workspace = true } -snafu = { workspace = true } -clap.workspace = true +clap_complete = { workspace = true } clap_mangen = "0.2.10" +clap.workspace = true +once_cell.workspace = true +regex.workspace = true serde_json.workspace = true +snafu.workspace = true +stackable-cockpitd = { path = "../stackable-cockpitd" } +stackablectl = { path = "../stackablectl" } diff --git a/rust/xtask/src/completions.rs b/rust/xtask/src/completions.rs new file mode 100644 index 00000000..9ed85464 --- /dev/null +++ b/rust/xtask/src/completions.rs @@ -0,0 +1,33 @@ +use std::fs; + +use clap::CommandFactory; +use clap_complete::{generate as generate_comps, Shell}; +use snafu::{ResultExt, Snafu}; +use stackablectl::cli::Cli; + +#[derive(Debug, Snafu)] +pub enum GenCompError { + #[snafu(display("io error"))] + Io { source: std::io::Error }, +} + +pub fn generate() -> Result<(), GenCompError> { + let mut cmd = Cli::command(); + let name = cmd.get_name().to_string(); + + fs::create_dir_all("extra/completions").context(IoSnafu)?; + + // Bash completions + let mut f = fs::File::create("extra/completions/stackablectl.bash").context(IoSnafu)?; + generate_comps(Shell::Bash, &mut cmd, name.clone(), &mut f); + + // Fish completions + let mut f = fs::File::create("extra/completions/stackablectl.fish").context(IoSnafu)?; + generate_comps(Shell::Fish, &mut cmd, name.clone(), &mut f); + + // ZSH completions + let mut f = fs::File::create("extra/completions/_stackablectl").context(IoSnafu)?; + generate_comps(Shell::Zsh, &mut cmd, name, &mut f); + + Ok(()) +} diff --git a/rust/xtask/src/docs.rs b/rust/xtask/src/docs.rs new file mode 100644 index 00000000..21437b23 --- /dev/null +++ b/rust/xtask/src/docs.rs @@ -0,0 +1,85 @@ +use std::{fs, path::Path}; + +use clap::CommandFactory; +use once_cell::sync::Lazy; +use regex::Regex; +use snafu::{ResultExt, Snafu}; +use stackablectl::cli::Cli; + +const COMMANDS: &[&str] = &[ + "completions", + "stacklets", + "operator", + "release", + "stack", + "cache", + "demo", + ".", +]; + +static CODE_LISTING_PATTERN: Lazy = Lazy::new(|| { + Regex::new(r"(?m)// Autogenerated by cargo xtask gen-docs\. DO NOT CHANGE MANUALLY!\n\[source,console\]\n----\n\$ stackablectl.*\n").unwrap() +}); +const DOCS_BASE_PATH: &str = "docs/modules/stackablectl/pages/commands"; +const USAGE_END_STRING: &str = "\n----"; + +#[derive(Debug, Snafu)] +pub enum GenDocsError { + #[snafu(display("No such subcommand: {name}"))] + NoSuchSubcommand { name: String }, + + #[snafu(display("io error"))] + Io { source: std::io::Error }, +} + +pub fn generate() -> Result<(), GenDocsError> { + let mut cli = Cli::command(); + + for command_page_name in COMMANDS { + let usage_text = if command_page_name == &"." { + cli.render_long_help().to_string() + } else { + match cli.find_subcommand_mut(command_page_name) { + Some(cmd) => cmd.render_long_help().to_string(), + None => { + return Err(NoSuchSubcommandSnafu { + name: command_page_name.to_string(), + } + .build()) + } + } + }; + + // Needed to remove trailing whitespaces in empty lines + let usage_text: Vec<_> = usage_text.lines().map(|l| l.trim_end()).collect(); + let usage_text = usage_text.join("\n"); + + let page_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .unwrap() + .join(DOCS_BASE_PATH) + .join(format!( + "{}.adoc", + if command_page_name == &"." { + "index" + } else { + command_page_name + } + )); + + let mut doc_page = fs::read_to_string(&page_path).context(IoSnafu)?; + + let m = CODE_LISTING_PATTERN.find(&doc_page).unwrap(); + let usage_start = m.end(); + let usage_end = doc_page[usage_start..].find(USAGE_END_STRING).unwrap(); + + doc_page.replace_range( + usage_start..usage_start + usage_end, + usage_text.to_string().trim(), + ); + fs::write(page_path, doc_page).context(IoSnafu)?; + } + + Ok(()) +} diff --git a/rust/xtask/src/main.rs b/rust/xtask/src/main.rs index ca3901fd..fc05773c 100644 --- a/rust/xtask/src/main.rs +++ b/rust/xtask/src/main.rs @@ -1,18 +1,16 @@ -use clap::{CommandFactory, Parser}; -use clap_complete::{generate, Shell}; -use clap_mangen::Man; -use snafu::{ensure, ResultExt, Snafu}; -use stackable_cockpitd::api_doc::openapi; -use stackablectl::cli::Cli; +use clap::Parser; +use snafu::Snafu; -use std::{ - fs, - io::Write, - path::Path, - process::{Command, Stdio}, +use crate::{ + completions::GenCompError, docs::GenDocsError, man::GenManError, openapi::GenOpenapiError, + readme::GenReadmeError, }; -const USAGE_STRING: &str = "Command line tool to interact with the Stackable Data Platform\n\nUsage: stackablectl [OPTIONS] \n"; +mod completions; +mod docs; +mod man; +mod openapi; +mod readme; #[derive(clap::Parser)] #[allow(clippy::enum_variant_names)] @@ -21,95 +19,41 @@ enum XtaskCommand { GenComp, GenOpenapi, GenCtlReadme, + GenDocs, } #[derive(Debug, Snafu)] enum TaskError { - #[snafu(display("io error"))] - Io { source: std::io::Error }, - - #[snafu(display("error serializing openapi"))] - SerializeOpenApi { source: serde_json::Error }, - - #[snafu(display("error running importing openapi schema importer"))] - ImportOpenapiSchemaRun { source: std::io::Error }, - - #[snafu(display("openapi schema importer failed with error code {error_code:?}"))] - ImportOpenapiSchema { error_code: Option }, - - #[snafu(display("error writing openapi schema into importer"))] - WriteOpenapiSchema { source: std::io::Error }, + #[snafu(display("failed to generate man pages"), context(false))] + Man { source: GenManError }, + + #[snafu(display("failed to generate shell completions"), context(false))] + Comp { source: GenCompError }, + + #[snafu( + display("failed to generate OpenAPI TypeScript schema based on the OpenAPI JSON spec"), + context(false) + )] + Openapi { source: GenOpenapiError }, + + #[snafu( + display("failed to generate stackablectl usage README file"), + context(false) + )] + Readme { source: GenReadmeError }, + + #[snafu(display("failed to generate stackablectl doc pages"), context(false))] + Docs { source: GenDocsError }, } #[snafu::report] fn main() -> Result<(), TaskError> { match XtaskCommand::parse() { - XtaskCommand::GenMan => { - let cmd = Cli::command(); - - fs::create_dir_all("extra/man").context(IoSnafu)?; - let mut f = fs::File::create("extra/man/stackablectl.1").context(IoSnafu)?; - - let man = Man::new(cmd); - man.render(&mut f).context(IoSnafu)? - } - XtaskCommand::GenComp => { - let mut cmd = Cli::command(); - let name = cmd.get_name().to_string(); - - fs::create_dir_all("extra/completions").context(IoSnafu)?; - - // Bash completions - let mut f = fs::File::create("extra/completions/stackablectl.bash").context(IoSnafu)?; - generate(Shell::Bash, &mut cmd, name.clone(), &mut f); - - // Fish completions - let mut f = fs::File::create("extra/completions/stackablectl.fish").context(IoSnafu)?; - generate(Shell::Fish, &mut cmd, name.clone(), &mut f); - - // ZSH completions - let mut f = fs::File::create("extra/completions/_stackablectl").context(IoSnafu)?; - generate(Shell::Zsh, &mut cmd, name, &mut f); - } - XtaskCommand::GenOpenapi => { - let openapi_json = openapi().to_json().context(SerializeOpenApiSnafu)?; - let mut codegen = Command::new("yarn") - .args(["--cwd", "web", "run", "openapi-codegen"]) - .stdin(Stdio::piped()) - .spawn() - .context(ImportOpenapiSchemaRunSnafu)?; - codegen - .stdin - .take() - .expect("child stdin must be available") - .write_all(openapi_json.as_bytes()) - .context(WriteOpenapiSchemaSnafu)?; - let status = codegen.wait().context(ImportOpenapiSchemaRunSnafu)?; - ensure!( - status.success(), - ImportOpenapiSchemaSnafu { - error_code: status.code() - } - ); - } - XtaskCommand::GenCtlReadme => { - let mut cmd = Cli::command(); - let usage_text = cmd.render_long_help().to_string(); - let usage_text: Vec<_> = usage_text.lines().map(|l| l.trim_end()).collect(); - let usage_text = usage_text.join("\n"); - - let readme_path = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("stackablectl/README.md"); - - let mut readme = fs::read_to_string(&readme_path).context(IoSnafu)?; - let usage_start = readme.find(USAGE_STRING).unwrap(); - let usage_end = readme[usage_start..].find("\n```").unwrap(); - - readme.replace_range(usage_start..usage_start + usage_end, &usage_text); - fs::write(readme_path, readme).context(IoSnafu)? - } + XtaskCommand::GenMan => man::generate()?, + XtaskCommand::GenComp => completions::generate()?, + XtaskCommand::GenOpenapi => openapi::generate()?, + XtaskCommand::GenCtlReadme => readme::generate()?, + XtaskCommand::GenDocs => docs::generate()?, } Ok(()) diff --git a/rust/xtask/src/man.rs b/rust/xtask/src/man.rs new file mode 100644 index 00000000..b162551e --- /dev/null +++ b/rust/xtask/src/man.rs @@ -0,0 +1,22 @@ +use std::fs; + +use clap::CommandFactory; +use clap_mangen::Man; +use snafu::{ResultExt, Snafu}; +use stackablectl::cli::Cli; + +#[derive(Debug, Snafu)] +pub enum GenManError { + #[snafu(display("io error"))] + Io { source: std::io::Error }, +} + +pub fn generate() -> Result<(), GenManError> { + let cmd = Cli::command(); + + fs::create_dir_all("extra/man").context(IoSnafu)?; + let mut f = fs::File::create("extra/man/stackablectl.1").context(IoSnafu)?; + + let man = Man::new(cmd); + man.render(&mut f).context(IoSnafu) +} diff --git a/rust/xtask/src/openapi.rs b/rust/xtask/src/openapi.rs new file mode 100644 index 00000000..fd93af9e --- /dev/null +++ b/rust/xtask/src/openapi.rs @@ -0,0 +1,46 @@ +use std::{ + io::Write, + process::{Command, Stdio}, +}; + +use snafu::{ensure, ResultExt, Snafu}; +use stackable_cockpitd::api_doc::openapi; + +#[derive(Debug, Snafu)] +pub enum GenOpenapiError { + #[snafu(display("error serializing openapi"))] + SerializeOpenApi { source: serde_json::Error }, + + #[snafu(display("error running importing openapi schema importer"))] + ImportOpenapiSchemaRun { source: std::io::Error }, + + #[snafu(display("openapi schema importer failed with error code {error_code:?}"))] + ImportOpenapiSchema { error_code: Option }, + + #[snafu(display("error writing openapi schema into importer"))] + WriteOpenapiSchema { source: std::io::Error }, +} + +pub fn generate() -> Result<(), GenOpenapiError> { + let openapi_json = openapi().to_json().context(SerializeOpenApiSnafu)?; + let mut codegen = Command::new("yarn") + .args(["--cwd", "web", "run", "openapi-codegen"]) + .stdin(Stdio::piped()) + .spawn() + .context(ImportOpenapiSchemaRunSnafu)?; + codegen + .stdin + .take() + .expect("child stdin must be available") + .write_all(openapi_json.as_bytes()) + .context(WriteOpenapiSchemaSnafu)?; + let status = codegen.wait().context(ImportOpenapiSchemaRunSnafu)?; + ensure!( + status.success(), + ImportOpenapiSchemaSnafu { + error_code: status.code() + } + ); + + Ok(()) +} diff --git a/rust/xtask/src/readme.rs b/rust/xtask/src/readme.rs new file mode 100644 index 00000000..d23f54e8 --- /dev/null +++ b/rust/xtask/src/readme.rs @@ -0,0 +1,33 @@ +use std::{fs, path::Path}; + +use clap::CommandFactory; +use snafu::{ResultExt, Snafu}; +use stackablectl::cli::Cli; + +const USAGE_START_STRING: &str = "Command line tool to interact with the Stackable Data Platform\n\nUsage: stackablectl [OPTIONS] \n"; +const USAGE_END_STRING: &str = "\n```"; + +#[derive(Debug, Snafu)] +pub enum GenReadmeError { + #[snafu(display("io error"))] + Io { source: std::io::Error }, +} + +pub fn generate() -> Result<(), GenReadmeError> { + let mut cmd = Cli::command(); + let usage_text = cmd.render_long_help().to_string(); + let usage_text: Vec<_> = usage_text.lines().map(|l| l.trim_end()).collect(); + let usage_text = usage_text.join("\n"); + + let readme_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("stackablectl/README.md"); + + let mut readme = fs::read_to_string(&readme_path).context(IoSnafu)?; + let usage_start = readme.find(USAGE_START_STRING).unwrap(); + let usage_end = readme[usage_start..].find(USAGE_END_STRING).unwrap(); + + readme.replace_range(usage_start..usage_start + usage_end, &usage_text); + fs::write(readme_path, readme).context(IoSnafu) +}