opencast · berthob98 · Jan 13, 2021
diff --git a/docs/opencast-influxdb-adapter.properties b/docs/opencast-influxdb-adapter.properties
@@ -2,11 +2,15 @@ influxdb.uri=http://localhost:8086
 influxdb.user=root
 influxdb.password=root
 influxdb.db-name=opencast
+# Can be "debug", "info" and "error"
 influxdb.log-level=info
 # Can be off by default, will use default RP
 # influxdb.retention-policy=infinite
 log-file=/var/log/httpd/access_log
-# Can be "debug", "info" and "error"
+# Can be off by default, will use default
+# adapter.httpd.logline-regex=^(?<ip>(?:[0-9]{1,3}\\.){3}[0-9]{1,3}) - (-|[^ ]+) \\[(?<date>[^]]+)\\] "(?<request>[^"]*)" (?<httpret>[0-9]+) (?<unknown1>(?:[0-9]+|-)) "(?<referrer>[^"]*)" "(?<agent>[^"]+)"
+# adapter.httpd.requestline-regex=^(?<method>[^ ]+) /(static/)?(?<organizationid>[^/]+)/(?<publicationchannel>[^/]+)/(?<episodeid>[^/]+)/(?<assetid>[^/]+)/[^/ ]+ .+$
+# adapter.httpd.datetime-format=dd/MMM/yyyy:HH:mm:ss Z
 adapter.view-interval-iso-duration=PT2H
 adapter.log-configuration-file=logback-sample.xml
 adapter.invalid-user-agents=Ruby,slurp,bot,spider,curl

diff --git a/src/main/java/org/opencastproject/influxdbadapter/ConfigFile.java b/src/main/java/org/opencastproject/influxdbadapter/ConfigFile.java
@@ -29,6 +29,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
+import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.util.Collections;
 import java.util.Properties;
@@ -58,6 +59,9 @@ public final class ConfigFile {
   private static final String OPENCAST_SERIES_ARE_OPTIONAL = "opencast.series-are-optional";
   private static final String LOG_FILE = "log-file";
   private static final String ADAPTER_LOG_CONFIGURATION_FILE = "adapter.log-configuration-file";
+  private static final String HTTPD_LOG_LINE_PATTERN = "adapter.httpd.logline-regex";
+  private static final String HTTPD_REQUESTLINE_PATTERN = "adapter.httpd.requestline-regex";
+  private static final String HTTPD_DATETIME_PATTERN = "adapter.httpd.datetime-format";
   private static final String ADAPTER_VIEW_INTERVAL = "adapter.view-interval-iso-duration";
   private static final String ADAPTER_INVALID_USER_AGENTS = "adapter.invalid-user-agents";
   private static final String ADAPTER_VALID_FILE_EXTENSIONS = "adapter.valid-file-extensions";
@@ -71,6 +75,9 @@ public final class ConfigFile {
   private final Set<String> invalidUserAgents;
   private final Set<String> validFileExtensions;
   private final Set<String> invalidPublicationChannels;
+  private final Pattern httpLogLinePattern;
+  private final Pattern httpRequestLinePattern;
+  private final String httpDateTimePattern;
 
   private ConfigFile(
           final InfluxDBConfig influxDBConfig,
@@ -80,7 +87,11 @@ private ConfigFile(
           final Path logConfigurationFile,
           final Set<String> invalidUserAgents,
           final Set<String> validFileExtensions,
-          final Set<String> invalidPublicationChannels) {
+          final Set<String> invalidPublicationChannels,
+          final Pattern httpLogLinePattern,
+          final Pattern httpRequestLinePattern,
+          final String httpDateTimePattern
+          ) {
     this.influxDBConfig = influxDBConfig;
     this.opencastConfig = opencastConfig;
     this.logFile = logFile;
@@ -89,6 +100,10 @@ private ConfigFile(
     this.invalidUserAgents = invalidUserAgents;
     this.validFileExtensions = validFileExtensions;
     this.invalidPublicationChannels = invalidPublicationChannels;
+    this.httpLogLinePattern = httpLogLinePattern;
+    this.httpRequestLinePattern = httpRequestLinePattern;
+    this.httpDateTimePattern = httpDateTimePattern;
+
   }
 
   private static Set<String> propertySet(final String propertyName, final Properties properties) {
@@ -120,6 +135,46 @@ public static ConfigFile readFile(final Path p) {
       System.exit(ExitStatuses.CONFIG_FILE_PARSE_ERROR);
     }
     final String logConfigurationFile = parsed.getProperty(ADAPTER_LOG_CONFIGURATION_FILE);
+
+    String httpLogLineRegex = parsed.getProperty(HTTPD_LOG_LINE_PATTERN);
+
+    if (httpLogLineRegex == null) {
+      httpLogLineRegex = "^(?<ip>(?:[0-9]{1,3}\\.){3}[0-9]{1,3}) - (-|[^ ]+) \\[(?<date>[^]]+)] \"(?<request>[^\"]*)\" (?<httpret>[0-9]+) (?<unknown1>(?:[0-9]+|-)) \"(?<referrer>[^\"]*)\" \"(?<agent>[^\"]+)\"";
+    }
+    Pattern httpLogLinePattern = null;
+
+    String httpDateTimeFormat = parsed.getProperty(HTTPD_DATETIME_PATTERN);
+
+    if (httpDateTimeFormat == null) {
+      httpDateTimeFormat = "dd/MMM/yyyy:HH:mm:ss Z";
+    }
+    String httpDateTimePattern = null;
+    try {
+        LOGGER.info("Using \"{}\" as pattern", httpLogLineRegex);
+        httpLogLinePattern = Pattern.compile(httpLogLineRegex);
+        LOGGER.info("Using \"{}\" as format", httpDateTimeFormat);
+        httpDateTimePattern = httpDateTimeFormat;
+        LogLine.setLogLineConfiguration(new LogLineConfiguration(httpLogLinePattern, DateTimeFormatter.ofPattern(httpDateTimePattern)));
+    } catch (Exception e) {
+        LOGGER.error("Error parsing {} \"{}\" {} \"{}\": {}",  HTTPD_LOG_LINE_PATTERN, httpLogLineRegex, HTTPD_DATETIME_PATTERN, httpDateTimeFormat, e.getMessage());
+        System.exit(ExitStatuses.CONFIG_FILE_PARSE_ERROR);
+    }
+
+    String httpRequestLineRegex = parsed.getProperty(HTTPD_REQUESTLINE_PATTERN);
+
+    if (httpRequestLineRegex == null) {
+      httpRequestLineRegex = "^(?<method>[^ ]+) /(static/)?(?<organizationid>[^/]+)/(?<publicationchannel>[^/]+)/(?<episodeid>[^/]+)/(?<assetid>[^/]+)/[^/ ]+ .+$";
+    }
+    Pattern httpRequestLinePattern = null;
+    try {
+        LOGGER.info("Using \"{}\" as pattern", httpRequestLineRegex);
+        httpRequestLinePattern = Pattern.compile(httpRequestLineRegex);
+        RequestLine.setRequestLineConfiguration(new RequestLineConfiguration(httpRequestLinePattern));
+    } catch (Exception e) {
+        LOGGER.error("Error parsing {} \"{}\": {}",  HTTPD_REQUESTLINE_PATTERN, httpRequestLinePattern, e.getMessage());
+        System.exit(ExitStatuses.CONFIG_FILE_PARSE_ERROR);
+    }
+
     Duration viewDuration = null;
     try {
       viewDuration = Duration.parse(parsed.getProperty(ADAPTER_VIEW_INTERVAL, "PT2H"));
@@ -178,7 +233,11 @@ else if (opencastSeriesAreOptionalStr.equals("false"))
                           logConfigurationFile != null ? Paths.get(logConfigurationFile) : null,
                           propertySet(ADAPTER_INVALID_USER_AGENTS, parsed),
                           propertySet(ADAPTER_VALID_FILE_EXTENSIONS, parsed),
-                          propertySet(ADAPTER_INVALID_PUBLICATION_CHANNELS, parsed));
+                          propertySet(ADAPTER_INVALID_PUBLICATION_CHANNELS, parsed),
+                          httpLogLinePattern,
+                          httpRequestLinePattern,
+                          httpDateTimePattern
+                          );
   }
 
   public InfluxDBConfig getInfluxDBConfig() {
@@ -212,4 +271,16 @@ public Set<String> getValidFileExtensions() {
   public Set<String> getInvalidPublicationChannels() {
     return this.invalidPublicationChannels;
   }
+
+  public Pattern gethttpLogLinePattern() {
+    return httpLogLinePattern;
+  }
+
+  public Pattern getHttpRequestLinePattern() {
+    return httpRequestLinePattern;
+  }
+
+  public String getHttpDateTimePattern() {
+    return httpDateTimePattern;
+  }
 }
diff --git a/src/main/java/org/opencastproject/influxdbadapter/ConfigurationException.java b/src/main/java/org/opencastproject/influxdbadapter/ConfigurationException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to The Apereo Foundation under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ *
+ * The Apereo Foundation licenses this file to you under the Educational
+ * Community License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License
+ * at:
+ *
+ *   http://opensource.org/licenses/ecl2.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package org.opencastproject.influxdbadapter;
+
+@SuppressWarnings("serial")
+public class ConfigurationException extends RuntimeException {
+
+  public ConfigurationException() {
+    super();
+  }
+
+  public ConfigurationException(String message, Throwable cause, boolean enableSuppression,
+          boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+
+  public ConfigurationException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public ConfigurationException(String message) {
+    super(message);
+  }
+
+  public ConfigurationException(Throwable cause) {
+    super(cause);
+  }
+
+}
diff --git a/src/main/java/org/opencastproject/influxdbadapter/LogLine.java b/src/main/java/org/opencastproject/influxdbadapter/LogLine.java
@@ -24,12 +24,9 @@
 import org.slf4j.LoggerFactory;
 
 import java.time.OffsetDateTime;
-import java.time.format.DateTimeFormatter;
 import java.util.Collection;
-import java.util.Locale;
 import java.util.Optional;
 import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import io.reactivex.Flowable;
 
@@ -46,12 +43,8 @@
 public final class LogLine {
   private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(LogLine.class);
 
-  private static final Pattern APACHE_LOG_LINE_PATTERN = Pattern.compile(
-          "^(?<ip>(?:[0-9]{1,3}\\.){3}[0-9]{1,3}) - - \\[(?<date>[^]]+)] \"(?<request>[^\"]*)\" (?<httpret>[0-9]+) (?<unknown1>(?:[0-9]+|-)) \"(?<referrer>[^\"]*)\" \"(?<agent>[^\"]+)\"");
+  private static LogLineConfiguration logLineConfiguration = null;
 
-  // Example: 10/Feb/2019:03:38:22 +0100
-  private static final DateTimeFormatter LOG_TIME_FORMATTER = DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z")
-                                                                               .withLocale(Locale.ENGLISH);
 
   private final CharSequence origin;
   private final String ip;
@@ -93,18 +86,21 @@ public final class LogLine {
    * @return An empty <code>Flowable</code> if the line was not successfully parsed, else a singleton <code>Flowable</code>
    */
   public static Flowable<LogLine> fromLine(final CharSequence line) {
+    if (logLineConfiguration == null) {
+      throw new ConfigurationException("Configuration not set. Abort.");
+    }
     if (line.length() == 0) {
       return Flowable.empty();
     }
-    final Matcher m = APACHE_LOG_LINE_PATTERN.matcher(line);
+    final Matcher m = logLineConfiguration.getPattern().matcher(line);
     if (!m.matches()) {
       LOGGER.debug("SKIP, wrong line pattern: {}", line);
       return Flowable.empty();
     }
     final String requestStr = m.group("request");
     return Flowable.just(new LogLine(line,
                                      m.group("ip"),
-                                     OffsetDateTime.parse(m.group("date"), LOG_TIME_FORMATTER),
+                                     OffsetDateTime.parse(m.group("date"), logLineConfiguration.getDateTimeFormatter()),
                                      requestStr,
                                      Integer.parseInt(m.group("httpret")),
                                      m.group("unknown1"),
@@ -150,6 +146,7 @@ public Flowable<RawImpression> toRawImpression(
         LOGGER.debug("SKIP, invalid agent \"{}\": {}", this.agent, this.origin);
         return Flowable.empty();
       }
+      LOGGER.debug("TAKE: {}", this.origin);
       return Flowable.just(new RawImpression(
               this,
               rl.getEpisodeId(),
@@ -167,4 +164,17 @@ private boolean invalidAgent(final Collection<String> invalidAgents) {
   public CharSequence getOrigin() {
     return this.origin;
   }
+
+  public static void setLogLineConfiguration(LogLineConfiguration logLineConfiguration) {
+    LogLine.logLineConfiguration = logLineConfiguration;
+  }
+
+  public String getReferrer() {
+    return referrer;
+  }
+
+  public String getIp() {
+    return ip;
+  }
+
 }
diff --git a/src/main/java/org/opencastproject/influxdbadapter/LogLineConfiguration.java b/src/main/java/org/opencastproject/influxdbadapter/LogLineConfiguration.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to The Apereo Foundation under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ *
+ * The Apereo Foundation licenses this file to you under the Educational
+ * Community License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License
+ * at:
+ *
+ *   http://opensource.org/licenses/ecl2.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package org.opencastproject.influxdbadapter;
+
+import java.time.format.DateTimeFormatter;
+import java.util.regex.Pattern;
+
+public class LogLineConfiguration {
+
+  public LogLineConfiguration(Pattern pattern, DateTimeFormatter dateTimeFormatter) {
+    super();
+    this.pattern = pattern;
+    this.dateTimeFormatter = dateTimeFormatter;
+  }
+
+  private Pattern pattern;
+  private DateTimeFormatter dateTimeFormatter;
+
+  public Pattern getPattern() {
+    return pattern;
+  }
+
+  public void setPattern(Pattern pattern) {
+    this.pattern = pattern;
+  }
+
+  public DateTimeFormatter getDateTimeFormatter() {
+    return dateTimeFormatter;
+  }
+
+  public void setDateTimeFormatter(DateTimeFormatter dateTimeFormatter) {
+    this.dateTimeFormatter = dateTimeFormatter;
+  }
+
+}
diff --git a/src/main/java/org/opencastproject/influxdbadapter/Main.java b/src/main/java/org/opencastproject/influxdbadapter/Main.java
@@ -48,6 +48,9 @@
 public final class Main {
   private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(Main.class);
 
+  private static ConfigFile configFile;
+
+
   private Main() {
   }
 
@@ -90,7 +93,7 @@ private static void processLines(
   public static void main(final String[] args) {
     // Preliminaries: command line parsing, config file parsing
     final CommandLine commandLine = CommandLine.parse(args);
-    final ConfigFile configFile = ConfigFile.readFile(commandLine.getConfigFile());
+    configFile = ConfigFile.readFile(commandLine.getConfigFile());
     configureLog(configFile);
     LOGGER.info("Logging configured");
     // Connect and configure InfluxDB
@@ -202,4 +205,12 @@ private static void processError(final Throwable e) {
     System.exit(ExitStatuses.UNKNOWN);
   }
 
+  public static ConfigFile getConfigFile() {
+    return configFile;
+  }
+
+  public static void setConfigFile(ConfigFile configFile) {
+    Main.configFile = configFile;
+  }
+
 }