Skip to content

Commit

Permalink
[reconfigurator] clickhouse-admin endpoints to generate ClickHouse co…
Browse files Browse the repository at this point in the history
…nfig files (#6476)

## Overview

This commit introduces two new endpoints to the `clickhouse-admin` API.
One to generate a ClickHouse server XML configuration file and the other
to generate a ClickHouse keeper XML configuration file.

## Purpose

"Reconfigurator" will need to call these endpoints to generate the
necessary files when bringing up new `clickhouse_server` and
`clickhouse_keeper` zones. They will also be necessary to update
existing zones with the keeper and server information from new zones.

## API structure

Moving forward the API endpoints will follow the following structure:

- `/keeper/` : For endpoints that will perform actions on keeper nodes
solely.
- `/server/` : For endpoints that will perform actions on server nodes
solely.

## Usage

To generate a server XML config file:

```console
$ curl -X put http://example/server/config \ 
-H "Content-Type: application/json" \
-d '{"generation": {GENERATION_NUM}, settings: {"id": {ID}, "keepers": [{"ipv6|ipv4|domain": "{ADDRESS}"}], "remote_servers": [{"ipv6|ipv4|domain": "{ADDRESS}"}], "config_dir": "{CONFIG_PATH}", "datastore_path": "{DATA_PATH}", "listen_addr": "{LISTEN_ADDRESS}" }}'
```

To generate a keeper XML config file:

```console
$ curl -X put http://example/keeper/config \ 
-H "Content-Type: application/json" \
-d '{"generation": {GENERATION_NUM}, settings: {"node_id": {ID}, "raft_servers": [{"id": {NODE_ID}, "host": {"ipv6|ipv4|domain": "{ADDRESS}"}}], "config_dir": "{CONFIG_PATH}", "datastore_path": "{DATA_PATH}", "listen_addr": "{LISTEN_ADDRESS}"}}'
```

## Caveats

For the time being the generation number isn't used for anything. In a
follow up PR we will use it to keep track of the configuration version
to make sure we're not generating an incorrect XML configuration file.

## Testing

Dropshot server:

```console
$ cargo run --bin=clickhouse-admin -- run -c ./smf/clickhouse-admin/config.toml -a [::1]:8888
   Compiling omicron-clickhouse-admin v0.1.0 (/Users/karcar/src/omicron/clickhouse-admin)
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 2.52s
     Running `target/debug/clickhouse-admin run -c ./smf/clickhouse-admin/config.toml -a '[::1]:8888'`
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:32:00.106783Z","hostname":"ixchel","pid":11318,"local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"}
{"msg":"accepted connection","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:32:17.626437Z","hostname":"ixchel","pid":11318,"local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[::1]:51896"}
Path { inner: GenerationNum { generation: Generation(123) } }
{"msg":"request completed","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:32:17.628323Z","hostname":"ixchel","pid":11318,"uri":"/node/server/generate-config/123","method":"post","req_id":"0b623f6b-441c-4418-9189-99504fbea5bf","remote_addr":"[::1]:51896","local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":1044,"response_code":"201"}
{"msg":"accepted connection","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:34:07.829199Z","hostname":"ixchel","pid":11318,"local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[::1]:52117"}
Path { inner: GenerationNum { generation: Generation(123) } }
{"msg":"request completed","v":0,"name":"clickhouse-admin","level":30,"time":"2024-09-02T07:34:07.830355Z","hostname":"ixchel","pid":11318,"uri":"/node/keeper/generate-config/123","method":"post","req_id":"1c729e0c-04bf-4e75-af0a-b23eef40f20f","remote_addr":"[::1]:52117","local_addr":"[::1]:8888","component":"dropshot","file":"/Users/karcar/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":985,"response_code":"201"}

```

Generate server XML file:

```console
$ curl -X put http://[::1]:8888/server/config -H "Content-Type: application/json" -d '{"generation": 3, "settings": {"id": 45, "keepers": [{"ipv6": "ff::01"}, {"ipv4": "127.0.0.1"}, {"domain_name": "hi.there"}], "remote_servers": [{"ipv6": "ff::08"}, {"ipv4": "127.0.0.2"}], "config_dir": "./", "datastore_path": "./", "listen_addr": "::1" }}' | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   858  100   602  100   256   210k  91789 --:--:-- --:--:-- --:--:--  418k
{
  "logger": {
    "level": "trace",
    "log": "./log/clickhouse.log",
    "errorlog": "./log/clickhouse.err.log",
    "size": 100,
    "count": 1
  },
  "macros": {
    "shard": 1,
    "replica": 45,
    "cluster": "oximeter_cluster"
  },
  "listen_host": "::1",
  "http_port": 8123,
  "tcp_port": 9000,
  "interserver_http_port": 9009,
  "remote_servers": {
    "cluster": "oximeter_cluster",
    "secret": "some-unique-value",
    "replicas": [
      {
        "host": {
          "ipv6": "ff::8"
        },
        "port": 9000
      },
      {
        "host": {
          "ipv4": "127.0.0.2"
        },
        "port": 9000
      }
    ]
  },
  "keepers": {
    "nodes": [
      {
        "host": {
          "ipv6": "ff::1"
        },
        "port": 9181
      },
      {
        "host": {
          "ipv4": "127.0.0.1"
        },
        "port": 9181
      },
      {
        "host": {
          "domain_name": "hi.there"
        },
        "port": 9181
      }
    ]
  },
  "data_path": "./data"
}
```

Generate keeper XML file:

```console
$ curl -X put http://[::1]:8888/keeper/config -H "Content-Type: application/json" -d '{"generation": 3, "settings": {"id": 1, "raft_servers": [{"id": 1, "host": {"ipv6": "ff::01"}}, {"id": 2, "host":{"ipv4": "127.0.0.1"}}, {"id": 3, "host":{"domain_name": "hi.there"}}], "config_dir": "./", "datastore_path": "./", "listen_addr": "::1" }}' | jq
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   820  100   568  100   252   301k   133k --:--:-- --:--:-- --:--:--  800k
{
  "logger": {
    "level": "trace",
    "log": "./log/clickhouse-keeper.log",
    "errorlog": "./log/clickhouse-keeper.err.log",
    "size": 100,
    "count": 1
  },
  "listen_host": "::1",
  "tcp_port": 9181,
  "server_id": 1,
  "log_storage_path": "./coordination/log",
  "snapshot_storage_path": "./coordination/snapshots",
  "coordination_settings": {
    "operation_timeout_ms": 10000,
    "session_timeout_ms": 30000,
    "raft_logs_level": "trace"
  },
  "raft_config": {
    "servers": [
      {
        "id": 1,
        "hostname": {
          "ipv6": "ff::1"
        },
        "port": 9234
      },
      {
        "id": 2,
        "hostname": {
          "ipv4": "127.0.0.1"
        },
        "port": 9234
      },
      {
        "id": 3,
        "hostname": {
          "domain_name": "hi.there"
        },
        "port": 9234
      }
    ]
  }
}
```


Related: #5999 , #3824
  • Loading branch information
karencfv authored Sep 5, 2024
1 parent e7c6e97 commit 61cac94
Show file tree
Hide file tree
Showing 13 changed files with 881 additions and 159 deletions.
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions clickhouse-admin/api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ license = "MPL-2.0"
workspace = true

[dependencies]
clickhouse-admin-types.workspace = true
dropshot.workspace = true
omicron-common.workspace = true
omicron-uuid-kinds.workspace = true
Expand Down
51 changes: 38 additions & 13 deletions clickhouse-admin/api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,52 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use dropshot::{HttpError, HttpResponseOk, RequestContext};
use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use clickhouse_admin_types::{KeeperSettings, ServerSettings};
use dropshot::{HttpError, HttpResponseCreated, RequestContext, TypedBody};
use omicron_common::api::external::Generation;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use std::net::SocketAddrV6;
use serde::Deserialize;

#[derive(Debug, Deserialize, JsonSchema)]
pub struct ServerConfigurableSettings {
/// A unique identifier for the configuration generation.
pub generation: Generation,
/// Configurable settings for a ClickHouse replica server node.
pub settings: ServerSettings,
}

#[derive(Debug, Deserialize, JsonSchema)]
pub struct KeeperConfigurableSettings {
/// A unique identifier for the configuration generation.
pub generation: Generation,
/// Configurable settings for a ClickHouse keeper node.
pub settings: KeeperSettings,
}

#[dropshot::api_description]
pub trait ClickhouseAdminApi {
type Context;

/// Retrieve the address the ClickHouse server or keeper node is listening on
/// Generate a ClickHouse configuration file for a server node on a specified
/// directory.
#[endpoint {
method = GET,
path = "/node/address",
method = PUT,
path = "/server/config",
}]
async fn clickhouse_address(
async fn generate_server_config(
rqctx: RequestContext<Self::Context>,
) -> Result<HttpResponseOk<ClickhouseAddress>, HttpError>;
}
body: TypedBody<ServerConfigurableSettings>,
) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError>;

#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub struct ClickhouseAddress {
pub clickhouse_address: SocketAddrV6,
/// Generate a ClickHouse configuration file for a keeper node on a specified
/// directory.
#[endpoint {
method = PUT,
path = "/keeper/config",
}]
async fn generate_keeper_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<KeeperConfigurableSettings>,
) -> Result<HttpResponseCreated<KeeperConfig>, HttpError>;
}
12 changes: 3 additions & 9 deletions clickhouse-admin/src/bin/clickhouse-admin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,8 @@ use std::net::{SocketAddr, SocketAddrV6};
enum Args {
/// Start the ClickHouse admin server
Run {
// TODO: This address is solely for testing now. We should remove it
// once we have more endpoints up and running.
/// Socket address for a running clickhouse server or keeper instance
#[clap(long, short = 'a', action)]
clickhouse_address: SocketAddrV6,

/// Address on which this server should run
#[clap(long, short = 'H', action)]
#[clap(long, short = 'a', action)]
http_address: SocketAddrV6,

/// Path to the server configuration file
Expand All @@ -47,12 +41,12 @@ async fn main_impl() -> Result<(), CmdError> {
let args = Args::parse();

match args {
Args::Run { clickhouse_address, http_address, config } => {
Args::Run { http_address, config } => {
let mut config = Config::from_file(&config)
.map_err(|err| CmdError::Failure(anyhow!(err)))?;
config.dropshot.bind_address = SocketAddr::V6(http_address);

let clickward = Clickward::new(clickhouse_address);
let clickward = Clickward::new();

let server =
omicron_clickhouse_admin::start_server(clickward, config)
Expand Down
39 changes: 26 additions & 13 deletions clickhouse-admin/src/clickward.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,17 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use clickhouse_admin_api::ClickhouseAddress;
use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use clickhouse_admin_types::{KeeperSettings, ServerSettings};
use dropshot::HttpError;
use slog_error_chain::{InlineErrorChain, SlogInlineError};
use std::io;
use std::net::SocketAddrV6;

#[derive(Debug, thiserror::Error, SlogInlineError)]
pub enum ClickwardError {
#[error("clickward failure")]
#[error("clickward XML generation failure")]
Failure {
#[source]
err: io::Error,
err: anyhow::Error,
},
}

Expand All @@ -34,18 +33,32 @@ impl From<ClickwardError> for HttpError {
}

#[derive(Debug)]
pub struct Clickward {
clickhouse_address: SocketAddrV6,
}
pub struct Clickward {}

impl Clickward {
pub fn new(clickhouse_address: SocketAddrV6) -> Self {
Self { clickhouse_address }
pub fn new() -> Self {
Self {}
}

pub fn generate_server_config(
&self,
settings: ServerSettings,
) -> Result<ReplicaConfig, ClickwardError> {
let replica_config = settings
.generate_xml_file()
.map_err(|e| ClickwardError::Failure { err: e })?;

Ok(replica_config)
}

pub fn clickhouse_address(
pub fn generate_keeper_config(
&self,
) -> Result<ClickhouseAddress, ClickwardError> {
Ok(ClickhouseAddress { clickhouse_address: self.clickhouse_address })
settings: KeeperSettings,
) -> Result<KeeperConfig, ClickwardError> {
let keeper_config = settings
.generate_xml_file()
.map_err(|e| ClickwardError::Failure { err: e })?;

Ok(keeper_config)
}
}
30 changes: 23 additions & 7 deletions clickhouse-admin/src/http_entrypoints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@

use crate::context::ServerContext;
use clickhouse_admin_api::*;
use dropshot::HttpError;
use dropshot::HttpResponseOk;
use dropshot::RequestContext;
use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use dropshot::{HttpError, HttpResponseCreated, RequestContext, TypedBody};
use std::sync::Arc;

type ClickhouseApiDescription = dropshot::ApiDescription<Arc<ServerContext>>;
Expand All @@ -21,11 +20,28 @@ enum ClickhouseAdminImpl {}
impl ClickhouseAdminApi for ClickhouseAdminImpl {
type Context = Arc<ServerContext>;

async fn clickhouse_address(
async fn generate_server_config(
rqctx: RequestContext<Self::Context>,
) -> Result<HttpResponseOk<ClickhouseAddress>, HttpError> {
body: TypedBody<ServerConfigurableSettings>,
) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError> {
let ctx = rqctx.context();
let output = ctx.clickward().clickhouse_address()?;
Ok(HttpResponseOk(output))
let replica_server = body.into_inner();
// TODO(https://github.com/oxidecomputer/omicron/issues/5999): Do something
// with the generation number `replica_server.generation`
let output =
ctx.clickward().generate_server_config(replica_server.settings)?;
Ok(HttpResponseCreated(output))
}

async fn generate_keeper_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<KeeperConfigurableSettings>,
) -> Result<HttpResponseCreated<KeeperConfig>, HttpError> {
let ctx = rqctx.context();
let keeper = body.into_inner();
// TODO(https://github.com/oxidecomputer/omicron/issues/5999): Do something
// with the generation number `keeper.generation`
let output = ctx.clickward().generate_keeper_config(keeper.settings)?;
Ok(HttpResponseCreated(output))
}
}
Loading

0 comments on commit 61cac94

Please sign in to comment.