From 9dc31833f48b546c2fba13e8c26aad779028650a Mon Sep 17 00:00:00 2001 From: diwanshu Date: Tue, 8 Aug 2023 13:14:42 -0400 Subject: [PATCH 1/2] Add metrics for tracking blacklist rules usage --- carbonj.service/Dockerfile | 5 +- .../carbonj/service/engine/MetricList.java | 63 ++++++++++++++----- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/carbonj.service/Dockerfile b/carbonj.service/Dockerfile index 93a8d711..fb79de18 100644 --- a/carbonj.service/Dockerfile +++ b/carbonj.service/Dockerfile @@ -26,7 +26,7 @@ RUN yum update -y && \ RUN yum install -y gcc-c++ gcc make libtool automake autoconf make python3-devel RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ - curl -o /etc/yum.repos.d/zulu.repo http://repos.azulsystems.com/rhel/zulu.repo && \ + curl -L -o /etc/yum.repos.d/zulu.repo http://repos.azulsystems.com/rhel/zulu.repo && \ yum update -y && \ yum install -y zulu-11 \ python3 \ @@ -35,7 +35,8 @@ RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \ yum clean all # Install aiohttp required by the script -RUN pip3 install aiohttp +RUN pip3 install --upgrade pip && \ + pip3 install aiohttp # Ensure crontab starts RUN /sbin/chkconfig crond on && \ diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java index a7998638..a8dced6f 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java @@ -6,6 +6,7 @@ */ package com.demandware.carbonj.service.engine; +import org.apache.commons.lang3.tuple.Pair; import com.codahale.metrics.Counter; import com.codahale.metrics.MetricRegistry; import com.demandware.carbonj.service.db.util.StatsAware; @@ -36,20 +37,23 @@ public class MetricList implements StatsAware volatile private List configLines = new ArrayList<>( ); - volatile private List patterns = new ArrayList<>( ); + volatile private List> patternsAndCounters = new ArrayList<>( ); - volatile private boolean empty = patterns.isEmpty(); + volatile private boolean empty = patternsAndCounters.isEmpty(); private final String confSrc; private final ConfigServerUtil configServerUtil; + private final MetricRegistry metricRegistry; + public MetricList(MetricRegistry metricRegistry, String name, File confFile, String confSrc, - ConfigServerUtil configServerUtil ) + ConfigServerUtil configServerUtil ) { this.name = Preconditions.checkNotNull(name); this.confFile = Preconditions.checkNotNull( confFile ); log.info( String.format("Creating metric list [%s] with config file [%s]", name, confFile) ); + this.metricRegistry = metricRegistry; this.droppedMetrics = metricRegistry.counter( MetricRegistry.name( name, "drop" ) ); this.confSrc = confSrc; this.configServerUtil = configServerUtil; @@ -59,25 +63,39 @@ public MetricList(MetricRegistry metricRegistry, String name, File confFile, St public boolean match(String name) { - if ( patterns.isEmpty() ) + if ( patternsAndCounters.isEmpty() ) { return false; } - List currentPatterns = patterns; // copy so we don't keep hitting the volatile barrier - for ( Pattern p : currentPatterns ) + List> currentPatternsAndCounters = patternsAndCounters; // copy so we don't keep hitting the volatile barrier + for ( int i = 0; i < currentPatternsAndCounters.size(); i++ ) { - if( ".*".equals( p.pattern() ) ) + Pair p = currentPatternsAndCounters.get(i); + + if( ".*".equals( p.getLeft().pattern() ) ) { + patternsAndCounters.get(i).getRight().inc(); droppedMetrics.inc(); return true; } - if ( p.matcher( name ).find() ) + long startTime = 0; + if (log.isDebugEnabled()) { + startTime = System.nanoTime(); // Record start time + } + if ( p.getLeft().matcher( name ).find() ) { + patternsAndCounters.get(i).getRight().inc(); droppedMetrics.inc(); + if (log.isDebugEnabled()) { + long endTime = System.nanoTime(); // Record end time + long duration = endTime - startTime; // Calculate duration in nanoseconds + log.debug("Pattern match runtime for {}: {} nanoseconds", p.getLeft().pattern(), duration); + } return true; } + } return false; } @@ -122,9 +140,8 @@ public void reload() log.info(String.format("Metric list [%s] configuration file has changed. File: [%s]", name, confFile)); List oldLines = this.configLines; - List newPatterns = parseConfig( lines ); - - this.patterns = newPatterns; + List> newPatternsAndCounters = parseConfig( lines ); + this.patternsAndCounters = newPatternsAndCounters; this.configLines = lines; log.info(String.format("Metric list [%s] updated.", name)); if( log.isDebugEnabled() ) @@ -138,12 +155,24 @@ public void reload() } } - private List parseConfig(List lines) + private List> parseConfig(List lines) { - return lines.stream() - .map( String::trim ) - .filter( line -> line.length() > 0 && !line.startsWith( "#" ) ) - .map( Pattern::compile ).collect( Collectors.toList() ); + // Create an empty list to hold pairs of Pattern and Counter + List> patternCounterPairs = lines.stream() + .map(String::trim) + .filter(line -> line.length() > 0 && !line.startsWith("#")) + .map(line -> { + Pattern pattern = Pattern.compile(line); + Counter counter = metricRegistry.counter( MetricRegistry.name( name, "blacklist" ) ); + return Pair.of(pattern, counter); // Create and return the Pair + }) + .collect(Collectors.toList()); + + // Reset the counters here if needed + for (Pair pair : patternCounterPairs) { + pair.getValue().dec(pair.getValue().getCount()); // Reset the counter to zero + } + return patternCounterPairs; } @Override @@ -154,6 +183,6 @@ public void dumpStats() public boolean isEmpty() { - return patterns.isEmpty(); + return patternsAndCounters.isEmpty(); } } From 69b220f2b2bf4f68ba50faed27096e355b30446c Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 20 Sep 2023 10:31:34 -0400 Subject: [PATCH 2/2] Merge fix --- .../demandware/carbonj/service/engine/MetricList.java | 10 +--------- gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java index f68cfa93..45a3080a 100644 --- a/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java +++ b/carbonj.service/src/main/java/com/demandware/carbonj/service/engine/MetricList.java @@ -40,8 +40,6 @@ public class MetricList implements StatsAware volatile private List> patternsAndCounters = new ArrayList<>( ); - volatile private boolean empty = patternsAndCounters.isEmpty(); - private final String confSrc; private final ConfigServerUtil configServerUtil; @@ -156,13 +154,7 @@ public void reload() log.info(String.format("Metric list [%s] configuration file has changed. File: [%s]", name, confFile)); List oldLines = this.configLines; -<<<<<<< HEAD - List> newPatternsAndCounters = parseConfig( lines ); - this.patternsAndCounters = newPatternsAndCounters; -======= - - this.patterns = parseConfig( lines ); ->>>>>>> origin/master + this.patternsAndCounters = parseConfig( lines ); this.configLines = lines; StringsCache.invalidateCache(); log.info(String.format("Metric list [%s] updated.", name)); diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 070cb702..98debb84 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists