diff --git a/Dockerfile b/Dockerfile index 80d2ff0..77e47ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ -FROM maven:3.5.3-jdk-8-slim +FROM apache/nifi:1.8.0 -COPY ./nifi-prometheus-nar /app/nifi-prometheus-nar -COPY ./nifi-prometheus-reporting-task /app/nifi-prometheus-reporting-task -COPY pom.xml /app -WORKDIR /app +ADD https://github.com/mkjoerg/nifi-prometheus-reporter/releases/download/1.8.0/nifi-prometheus-nar-1.8.0.nar ${NIFI_BASE_DIR}/nifi-current/lib + +USER root + +# Setup NiFi user and create necessary directories +RUN chown -R nifi:nifi ${NIFI_BASE_DIR}/nifi-current/lib + +USER nifi -RUN ls -l -RUN mvn clean install diff --git a/Readme.md b/Readme.md index 14edbf2..192ae7e 100644 --- a/Readme.md +++ b/Readme.md @@ -17,11 +17,18 @@ This will bootstrap: * A Prometheus server that runs under: http://localhost:9090 * A Pushgateway that runs under: http://localhost:9091 * A Grafana instance that runs under: http://localhost:3000 +* A Nifi instance, containing the reporting task under: http://localhost:8080/nifi A sample dashboard can be found here: [Sample Dashboard](https://grafana.com/dashboards/3294) After setting up a simple flow and the ReportingTask, the flow can be started and the results should be visible in the Grafana dashboard. +## Docs + +See the docs for more details: + +1. [Configuration](docs/Configuration.md) + ### Prerequisites To test or use the PrometheusReportingTask the following systems should be @@ -32,9 +39,21 @@ setup and running. The tools can be setup with Docker or manually. -### Installing +### Install to running Nifi instance +First download the [current release](https://github.com/mkjoerg/nifi-prometheus-reporter/releases) and then +copy the nar file into your Nifi lib folder. (Most times under __/opt/nifi//lib__) + +After this, just restart Nifi. + +### Limitations +The Reporting Task can't send custom metrics from processors to the Pushgateway. If you want +something like this, you have to setup your own processor, that can read FlowFiles, generate custom metrics +and send them to a Pushgateway. Because this is such a custom thing, it can't be done with this Reporting Task +and it is also not the scope of this project. -The project can be build with maven as the standard fasion of building +### Build it yourself + +The project can be build with maven as the standard fashion of building nifi-processor-bundles. Following snippet shows the entire setup with pre-installed Nifi: ```sh # Clone project @@ -45,9 +64,6 @@ cd nifi-prometheus-reporter # Build project mvn clean install ``` - -## Deployment - The previously built .nar archive has to be copied into the nifi/lib directory and can be used after a restart of nifi. ```sh @@ -59,7 +75,6 @@ NIFI_HOME/bin/nifi.sh start # Or restart if already running NIFI_HOME/bin/nifi.sh restart -``` ## Authors diff --git a/docker-compose.yml b/docker-compose.yml index b1ca76b..95d68cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,11 @@ services: ports: - "9091:9091" + nifi: + build: . + ports: + - "8080:8080" + grafana: image: grafana/grafana:latest ports: diff --git a/docs/Configuration.md b/docs/Configuration.md new file mode 100644 index 0000000..749f590 --- /dev/null +++ b/docs/Configuration.md @@ -0,0 +1,21 @@ +# Configuration + +Open your Nifi under http://localhost:8080/nifi and go to the menu in the upper right: +![Controller Settings](./img/00-controller-settings.png) + +Go to Controller Settings and add a new reporting task: +![Add Reporting Task](./img/01-add-reporting-task.png) + +After the task is added, it has to be configured: +![Configure Reporting Task](./img/02-configure-reporting-task.png) + +The following settings can be edited: (See the help texts for more details) +![Settings Reporting Task](./img/03-settings-reporting-task.png) + +After the reporting task is configured, run it: +![Run Reporting Task](./img/04-run-reporting-task.png) + +If everything went well, the prometheus pushgateway (localhost:9091) should now provide your metrics: +![Check Pushgateway](./img/05-pushgateway-view.png) + +After this the metrics can be scraped by prometheus and then visualized in Grafana. \ No newline at end of file diff --git a/docs/img/00-controller-settings.png b/docs/img/00-controller-settings.png new file mode 100644 index 0000000..bea73b9 Binary files /dev/null and b/docs/img/00-controller-settings.png differ diff --git a/docs/img/01-add-reporting-task.png b/docs/img/01-add-reporting-task.png new file mode 100644 index 0000000..1b0e521 Binary files /dev/null and b/docs/img/01-add-reporting-task.png differ diff --git a/docs/img/02-configure-reporting-task.png b/docs/img/02-configure-reporting-task.png new file mode 100644 index 0000000..ef6bc2f Binary files /dev/null and b/docs/img/02-configure-reporting-task.png differ diff --git a/docs/img/03-settings-reporting-task.png b/docs/img/03-settings-reporting-task.png new file mode 100644 index 0000000..a36e5f7 Binary files /dev/null and b/docs/img/03-settings-reporting-task.png differ diff --git a/docs/img/04-run-reporting-task.png b/docs/img/04-run-reporting-task.png new file mode 100644 index 0000000..1314bce Binary files /dev/null and b/docs/img/04-run-reporting-task.png differ diff --git a/docs/img/05-pushgateway-view.png b/docs/img/05-pushgateway-view.png new file mode 100644 index 0000000..62a3ce8 Binary files /dev/null and b/docs/img/05-pushgateway-view.png differ diff --git a/nifi-prometheus-reporting-task/src/main/java/org/apache/nifi/reporting/prometheus/api/PrometheusMetricsFactory.java b/nifi-prometheus-reporting-task/src/main/java/org/apache/nifi/reporting/prometheus/api/PrometheusMetricsFactory.java index 022bdff..2507ea6 100644 --- a/nifi-prometheus-reporting-task/src/main/java/org/apache/nifi/reporting/prometheus/api/PrometheusMetricsFactory.java +++ b/nifi-prometheus-reporting-task/src/main/java/org/apache/nifi/reporting/prometheus/api/PrometheusMetricsFactory.java @@ -13,7 +13,7 @@ public class PrometheusMetricsFactory { - private static final CollectorRegistry NIFI_REGISTRY = new CollectorRegistry(); + private static final CollectorRegistry NIFI_METRICS_REGISTRY = new CollectorRegistry(); private static final CollectorRegistry JVM_REGISTRY = new CollectorRegistry(); @@ -21,31 +21,31 @@ public class PrometheusMetricsFactory { .name("process_group_amount_flowfiles_total") .help("Total number of FlowFiles in ProcessGroup") .labelNames("status", "application", "process_group") - .register(NIFI_REGISTRY); + .register(NIFI_METRICS_REGISTRY); private static final Gauge AMOUNT_BYTES_TOTAL = Gauge.build() .name("process_group_amount_bytes_total") .help("Total number of Bytes in ProcessGroup") .labelNames("status", "application", "process_group") - .register(NIFI_REGISTRY); + .register(NIFI_METRICS_REGISTRY); private static final Gauge AMOUNT_THREADS_TOTAL = Gauge.build() .name("process_group_amount_threads_total") .help("Total amount of threads in ProcessGroup") .labelNames("status", "application", "process_group") - .register(NIFI_REGISTRY); + .register(NIFI_METRICS_REGISTRY); private static final Gauge SIZE_CONTENT_TOTAL = Gauge.build() .name("process_group_size_content_total") .help("Total size of content in ProcessGroup") .labelNames("status", "application", "process_group") - .register(NIFI_REGISTRY); + .register(NIFI_METRICS_REGISTRY); private static final Gauge AMOUNT_ITEMS = Gauge.build() .name("process_group_amount_items") .help("Total amount of items in ProcessGroup") .labelNames("status", "application", "process_group") - .register(NIFI_REGISTRY); + .register(NIFI_METRICS_REGISTRY); private static final Gauge JVM_HEAP = Gauge.build() .name("jvm_heap_stats") @@ -99,7 +99,7 @@ public static CollectorRegistry createNifiMetrics(ProcessGroupStatus status, Str AMOUNT_THREADS_TOTAL.labels("nano", applicationId, processGroupName).set(status.getActiveThreadCount()); - return NIFI_REGISTRY; + return NIFI_METRICS_REGISTRY; } public static CollectorRegistry createJvmMetrics(VirtualMachineMetrics jvmMetrics) {