Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make changes for prometheus metrics #1418

Merged
merged 5 commits into from
Mar 15, 2024
Merged

Make changes for prometheus metrics #1418

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
26f45f5
Make changes for prometheus metrics
G-D-Petrov Mar 13, 2024
2b1db8f
Address PR comments
G-D-Petrov Mar 13, 2024
3788c27
Rename to MetricsConfig
G-D-Petrov Mar 13, 2024
1332727
Change log message to debug
G-D-Petrov Mar 13, 2024
336d6b0
Make the Prometheus cfg public
G-D-Petrov Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 24 additions & 28 deletions cpp/arcticdb/entity/metrics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,45 +27,39 @@ namespace arcticdb {
std::shared_ptr<PrometheusInstance> PrometheusInstance::instance_;
std::once_flag PrometheusInstance::init_flag_;

std::shared_ptr<PrometheusConfigInstance> PrometheusConfigInstance::instance(){
std::call_once(PrometheusConfigInstance::init_flag_, &PrometheusConfigInstance::init);
return PrometheusConfigInstance::instance_;
PrometheusInstance::PrometheusInstance() : configured_(false) {
arcticdb::log::version().debug("PrometheusInstance created");
}

std::shared_ptr<PrometheusConfigInstance> PrometheusConfigInstance::instance_;
std::once_flag PrometheusConfigInstance::init_flag_;

PrometheusInstance::PrometheusInstance() {

auto cfg = PrometheusConfigInstance::instance()->config;

if (cfg.prometheus_model() == PrometheusConfigInstance::Proto::PUSH) {
// PUSH MODE
if (cfg.instance().empty() || cfg.host().empty() || cfg.port().empty() || cfg.job_name().empty()) {
util::raise_rte( "Invalid Push PrometheusConfig {}", arcticdb::util::format(cfg));
}
void PrometheusInstance::configure(const MetricsConfig& config, const bool reconfigure) {
if (configured_ && !reconfigure) {
arcticdb::log::version().warn("Prometheus already configured");
return;
}

cfg_ = config;

if (cfg_.model_ == MetricsConfig::Model::PUSH) {
// IMP: This is the GROUPING_KEY - every push overwrites the previous grouping key
auto labels = prometheus::Gateway::GetInstanceLabel(getHostName());
mongo_instance_ = cfg.instance();
mongo_instance_ = cfg_.instance;
labels.try_emplace(MONGO_INSTANCE_LABEL, mongo_instance_);
labels.try_emplace(PROMETHEUS_ENV_LABEL, cfg.prometheus_env());
gateway_= std::make_shared<prometheus::Gateway>(cfg.host(), cfg.port(), cfg.job_name(), labels);
labels.try_emplace(PROMETHEUS_ENV_LABEL, cfg_.prometheus_env);
gateway_= std::make_shared<prometheus::Gateway>(cfg_.host, cfg_.port, cfg_.job_name, labels);
registry_ = std::make_shared<prometheus::Registry>();
gateway_->RegisterCollectable(registry_);

arcticdb::log::version().info("Prometheus Push created with settings {}", arcticdb::util::format(cfg));
arcticdb::log::version().info("Prometheus Push created with settings {}", cfg_);

} else if (cfg.prometheus_model() == PrometheusConfigInstance::Proto::WEB) {

// WEB SERVER MODE
if (cfg.port().empty()) {
util::raise_rte( "PrometheusConfig web mode port not set {}", arcticdb::util::format(cfg));
}
} else if (cfg_.model_ == MetricsConfig::Model::PULL) {

// create an http server ie "http://hostname:"+port()+"/metrics"
std::string hostname = getHostName();
std::string endpoint = hostname + ":" + cfg.port();
std::string endpoint = cfg_.host + ":" + cfg_.port;

if (exposer_.use_count() > 0) {
exposer_->RemoveCollectable(registry_, "/metrics");
exposer_.reset();
}

// default to 2 threads
exposer_ = std::make_shared<prometheus::Exposer>(endpoint, 2);
Expand All @@ -79,8 +73,10 @@ namespace arcticdb {
arcticdb::log::version().info("Prometheus endpoint created on {}/metrics", endpoint);
}
else {
arcticdb::log::version().info("Prometheus not configured {}", arcticdb::util::format(cfg));
arcticdb::log::version().info("Prometheus not configured {}", cfg_);
}

configured_ = true;
}

// new mechanism, labels at runtime
Expand Down
68 changes: 55 additions & 13 deletions cpp/arcticdb/entity/metrics.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <map>
#include <unordered_map>
#include <memory>
#include <fmt/format.h>

namespace arcticdb {

Expand All @@ -34,6 +35,42 @@ const std::string PROMETHEUS_ENV_LABEL = "env";
const int SUMMARY_MAX_AGE = 30;
const int SUMMARY_AGE_BUCKETS = 5;

class MetricsConfig {
public:
enum class Model {
NO_INIT,
PUSH,
PULL
};
MetricsConfig() : model_(Model::NO_INIT) {}

MetricsConfig(const std::string& host,
const std::string& port,
const std::string& job_name,
const std::string& instance,
const std::string& prometheus_env,
const Model model)
: host(host)
, port(port)
, job_name(job_name)
, instance(instance)
, prometheus_env(prometheus_env)
, model_(model) {
util::check(!host.empty(), "MetricsConfig: host is empty");
util::check(!port.empty(), "MetricsConfig: port is empty");
util::check(!job_name.empty(), "MetricsConfig: job_name is empty");
util::check(!instance.empty(), "MetricsConfig: instance is empty");
util::check(!prometheus_env.empty(), "MetricsConfig: instance is empty");
util::check(!prometheus_env.empty(), "MetricsConfig: prometheus_env is empty");
}

std::string host;
std::string port;
std::string job_name;
std::string instance;
std::string prometheus_env;
Model model_;
};

class PrometheusInstance {
public:
Expand Down Expand Up @@ -67,6 +104,10 @@ class PrometheusInstance {

int push();

void configure(const MetricsConfig& config, const bool reconfigure = false);

MetricsConfig cfg_;

private:

struct HistogramInfo {
Expand All @@ -88,20 +129,8 @@ class PrometheusInstance {
// push gateway
std::string mongo_instance_;
std::shared_ptr<prometheus::Gateway> gateway_;
};
bool configured_;

class PrometheusConfigInstance {
public:
static std::shared_ptr<PrometheusConfigInstance> instance();

using Proto = arcticdb::proto::utils::PrometheusConfig;
Proto config;
static std::shared_ptr<PrometheusConfigInstance> instance_;
static std::once_flag init_flag_;

static void init(){
instance_ = std::make_shared<PrometheusConfigInstance>();
}
};

inline void log_prometheus_gauge(const std::string& metric_name, const std::string& metric_desc, size_t val) {
Expand All @@ -121,3 +150,16 @@ inline void log_prometheus_counter(const std::string& metric_name, const std::st
}

} // Namespace arcticdb

template<>
struct fmt::formatter<arcticdb::MetricsConfig> {

template<typename ParseContext>
constexpr auto parse(ParseContext &ctx) { return ctx.begin(); }

template<typename FormatContext>
auto format(const arcticdb::MetricsConfig k, FormatContext &ctx) const {
return fmt::format_to(ctx.out(), "MetricsConfig: host={}, port={}, job_name={}, instance={}, prometheus_env={}, model={}",
k.host, k.port, k.job_name, k.instance, k.prometheus_env, static_cast<int>(k.model_));
}
};
16 changes: 11 additions & 5 deletions cpp/arcticdb/python/python_module.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,13 +273,19 @@ void register_instrumentation(py::module && m){
}

void register_metrics(py::module && m){

auto prometheus = m.def_submodule("prometheus");
py::class_<arcticdb::PrometheusInstance, std::shared_ptr<arcticdb::PrometheusInstance>>(prometheus, "Instance");
prometheus.def("configure", [](const py::object & py_config){
arcticdb::proto::utils::PrometheusConfig config;
arcticdb::python_util::pb_from_python(py_config, config);
arcticdb::PrometheusConfigInstance::instance()->config.CopyFrom(config);
});

py::class_<arcticdb::MetricsConfig, std::shared_ptr<arcticdb::MetricsConfig>>(prometheus, "MetricsConfig")
.def(py::init<const std::string&, const std::string&, const std::string&, const std::string&, const std::string&, const arcticdb::MetricsConfig::Model>());

py::enum_<arcticdb::MetricsConfig::Model>(prometheus, "MetricsConfigModel")
.value("NO_INIT", arcticdb::MetricsConfig::Model::NO_INIT)
.value("PUSH", arcticdb::MetricsConfig::Model::PUSH)
.value("PULL", arcticdb::MetricsConfig::Model::PULL)
.export_values()
;
}

/// Register handling of non-trivial types. For more information @see arcticdb::TypeHandlerRegistry and
Expand Down
4 changes: 0 additions & 4 deletions cpp/arcticdb/version/local_versioned_engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1730,10 +1730,6 @@ void LocalVersionedEngine::configure(const storage::LibraryDescriptor::VariantSt
if(cfg.write_options().has_sync_passive()) {
version_map->set_log_changes(cfg.write_options().sync_passive().enabled());
}
if (cfg.has_prometheus_config()) {
PrometheusConfigInstance::instance()->config.CopyFrom(cfg.prometheus_config());
ARCTICDB_DEBUG(log::version(), "prometheus configured");
}
},
[](const auto& conf){
util::raise_rte(
Expand Down
Loading