fluent.conf.rt

<source>
  @type tail
  @id access_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-service.log.pos"
  tag jfrog.rt.access.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id artifactory_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-service.log.pos"
  tag jfrog.rt.artifactory.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id frontend_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-service.log.pos"
  tag jfrog.rt.frontend.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id metadata_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-service.log.pos"
  tag jfrog.rt.metadata.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id router_service_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos"
  tag jfrog.rt.router.service
  <parse>
    @type none
  </parse>
</source>
<filter jfrog.rt.**.service>
  @type record_transformer
  enable_ruby true
  <record>
    message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')}
  </record>
</filter>
<filter jfrog.rt.**.service>
  @type parser
  key_name message
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
      time_key timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
  emit_invalid_record_to_error false
</filter>
<source>
  @type tail
  @id router_traefik_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos"
  tag jfrog.rt.router.traefik
  <parse>
      @type multiline
      format_firstline /\d{4}-\d{1,2}-\d{1,2}/
      format1 /^(?<timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.*)$/
      time_key timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<source>
  @type tail
  @id access_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-request.log.pos"
  tag jfrog.rt.access.request
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<source>
  @type tail
  @id artifactory_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-request.log.pos"
  tag jfrog.rt.artifactory.request
  <parse>
    @type none
  </parse>
</source>
<filter jfrog.rt.artifactory.request>
  @type parser
  key_name message
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
  emit_invalid_record_to_error false
</filter>
<source>
  @type tail
  @id frontend_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/frontend-request.log.pos"
  tag jfrog.rt.frontend.request
  <parse>
    @type none
  </parse>
</source>
<filter jfrog.rt.frontend.request>
  @type parser
  key_name message
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
  emit_invalid_record_to_error false
</filter>
<source>
  @type tail
  @id metadata_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/metadata-request.log.pos"
  tag jfrog.rt.metadata.request
  <parse>
    @type none
  </parse>
</source>
<filter jfrog.rt.metadata.request>
  @type parser
  key_name message
  <parse>
    @type regexp
    expression ^(?<timestamp>[^ ]*)\|(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
  emit_invalid_record_to_error false
</filter>
<filter jfrog.rt.artifactory.request>
  @type record_transformer
  enable_ruby true
  <record>
    repo ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2") ? (record["request_url"].split('/')[3]) : ("")}
    image ${record["request_url"].include?("/api/docker") && !record["request_url"].include?("/api/docker/null") && !record["request_url"].include?("/api/docker/v2")  ? (record["request_url"].split('/')[5]) : ("")}
  </record>
</filter>
<source>
  @type tail
  @id router_request_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
  tag jfrog.rt.router.request
  <parse>
    @type json
    time_key time
    time_format %Y-%m-%dT%H:%M:%S%:z
  </parse>
</source>
<source>
  @type tail
  @id artifactory_access_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/artifactory-access.log.pos"
  tag jfrog.rt.artifactory.access
  <parse>
    @type regexp
    expression /^(?<timestamp>[^ ]*) \[(?<trace_id>[^\]]*)\] \[(?<action_response>[^\]]*)\] (?<repo_path>.*) for client : (?<username>.+)/(?<ip>.+).$/
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<source>
  @type tail
  @id access_security_audit_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/access-security-audit.log.pos"
  tag jfrog.rt.access.audit
  <parse>
    @type regexp
    expression /^(?<timestamp>[^ ]*)\|(?<token_id>[^ ]*)\|(?<user_ip>[^ ]*)\|(?<user>[^ ]*)\|(?<logged_principal>[^ ]*)\|(?<entity_name>[^ ]*)\|(?<event_type>[^ ]*)\|(?<event>[^ ]*)\|(?<data_changed>.*)/
    time_key timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
<filter jfrog.**>
  @type record_transformer
  <record>
    hostname "#{Socket.gethostname}"
    log_source ${tag}
  </record>
</filter>
<filter jfrog.rt.artifactory.access>
  @type record_transformer
  enable_ruby true
  <record>
    impacted_artifacts ${if record['repo_path'].length > 1; "default/" + record["repo_path"].split(':')[0] + "/" + record["repo_path"].split(':')[1].rstrip ; end;}
  </record>
</filter>

<source>
  @type jfrog_metrics
  @id metrics_http_jfrt
  tag jfrog.metrics.artifactory
  interval 5s
  metric_prefix 'jfrog.artifactory'
  jpd_url JPD_URL
  username ADMIN_USERNAME
  apikey API_KEY
  token JFROG_ADMIN_TOKEN
  common_jpd COMMON_JPD
</source>

## CALL HOME
<source>
  @type exec
  tag jfrog.callhome
  command "curl --request GET 'JPD_URL/artifactory/api/system/version' -H 'Authorization: Bearer JFROG_ADMIN_TOKEN'"
  run_interval 1d
  <parse>
    @type json
  </parse>
</source>
<filter jfrog.callhome>
  @type record_transformer
  renew_record true
  keep_keys 'productId,features'
  enable_ruby true
  <record>
    productId 'jfrogLogAnalyticsSplunk/0.13.0'
    features ${return([{"featureId" => "ArtifactoryVersion/" + record["version"]}])}
  </record>
</filter>
<match jfrog.callhome>
  @type http
  endpoint JPD_URL/artifactory/api/system/usage
  open_timeout 5
  content_type application/json
  headers {"Authorization":"Bearer JFROG_ADMIN_TOKEN"}
  <format>
    @type json
  </format>
  <buffer>
    flush_interval 5s
  </buffer>
</match>

####################
# SPLUNK OUTPUT
####################

<match jfrog.metrics.**>
  @type splunk_hec
  protocol COM_PROTOCOL
  data_type metric
  hec_host HEC_HOST
  hec_port HEC_PORT
  hec_token METRICS_HEC_TOKEN
  flush_interval 5s
  source ${tag}
  index jfrog_splunk_metrics
  metric_name_key metric_name
  metric_value_key value
  insecure_ssl INSECURE_SSL
  <fields>
  </fields>
</match>

<match jfrog.**>
  @type splunk_hec
  protocol COM_PROTOCOL
  hec_host HEC_HOST
  hec_port HEC_PORT
  hec_token HEC_TOKEN
  index jfrog_splunk
  format json
  sourcetype_key log_source
  use_fluentd_time false
  # buffered output parameter
  flush_interval 10s
  insecure_ssl INSECURE_SSL
  # ssl parameter
  #use_ssl true
  #ca_file /path/to/ca.pem
</match>