fluent.forwarder.conf

#############################################################################################
# Fluentd Configuration File                                                                #
#                                                                                           #
# In v1 configuration, type and id are @ prefix parameters.                                 #
# @type and @id are recommended. type and id are still available for backward compatibility #
#############################################################################################

<system>
  # equal to -v option
  log_level debug
</system>


################################
#            Source            #
################################
## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
  @type forward
  @id forward_input

  port 24224
</source>

# Listen DRb for debug
<source>
  @type debug_agent
  @id debug_agent_input

  bind 127.0.0.1
  port 24230
</source>

# HTTP input
# http://localhost:8888/<tag>?json=<json>
#<source>
#  @type http
#  @id http_input

#  port 8888
#</source>

# Listen HTTP for monitoring
# http://localhost:24220/api/plugins
# http://localhost:24220/api/plugins?type=TYPE
# http://localhost:24220/api/plugins?tag=MYTAG
<source>
  @type monitor_agent
  @id monitor_agent_input

  port 24220
</source>

<source>
  @type tail

  path /opt/wso2esb-4.9.0-wkr-2/repository/logs/wso2carbon.log
  pos_file /cellcard/fluent/wso2.esb.service.pos
  tag wso2.esb.service.test
  format /^([TID:]+ [^ ]+ [^ ]+ \[(?<time>[^\]]+)\]) ([^***]+[^=]+[^ ]+(?<transaction_id>[^,]*)[^=]+[^ ]+(?<service_name>[^,]*)[^=]+[^ ]+(?<data>[^,]*))/
  time_format %Y-%m-%d %H:%M:%S
  keep_time_key true
</source>

<source>
  @type tail

  path /opt/wso2esb-4.9.0-wkr-2/repository/logs/wso2carbon.log
  pos_file /cellcard/fluent/wso2.esb.ne.surepay.pos
  tag wso2.esb.ne.surepay
  format /^([TID:]+ [^ ]+ [^ ]+ \[(?<time>[^\]]+)\]) ([^***]+[^=]+[^ ]+(?<service_name>[^,]*)[^=]+[^ ]+(?<transaction_id>[^,]*)[^<?]+(?<payload>[^{]*))/
  time_format %Y-%m-%d %H:%M:%S
  keep_time_key true
</source>

<source>
  @type tail

  path /opt/wso2esb-4.9.0-wkr-2/repository/logs/wso2carbon.log
  pos_file /cellcard/fluent/wso2.esb.surepay.trigger.pos
  tag wso2.esb.surepay.trigger
  format /^([TID:]+ [^ ]+ [^ ]+ \[(?<time>[^\]]+)\]) ([^*]+[^=]+[^ ]+(?<client_ip>[^,]*)[^=]+[^ ]+(?<service_name>[^,]*)[^=]+[^ ]+(?<req_id>[^,]*)[^=]+[^ ]+(?<content_message>[^,]*)[^=]+[^ ]+)/
  time_format %Y-%m-%d %H:%M:%S
  keep_time_key true
</source>


###########################
#        Filter           #
###########################
<filter fluent.**>
  @type grep

  <exclude>
    key     message
    pattern ^pattern
  </exclude>
</filter>

<filter wso2.esb.service.**>
  @type grep

  <regexp>
    key     service_name
    pattern ^\sNew
  </regexp>
</filter>

<filter wso2.esb.service.**>
  @type record_transformer
  enable_ruby

  <record>
    data ${record["data"].strip.split(";").each_slice(2).to_h.to_json}
  </record>
</filter>

<filter wso2.esb.service.**>
  @type parser

  format json
  key_name data
</filter>

<filter wso2.esb.ne.surepay>
  @type grep

  <regexp>
    key     service_name
    pattern ^\sNE_SurePayEp
  </regexp>
</filter>

<filter wso2.esb.ne.surepay>
  @type record_transformer
  remove_keys payload
  enable_ruby

  <record>
    service_name ${record["service_name"].strip!}
    transaction_id ${record["transaction_id"].strip!}
    payload1 ${record["payload"][0,(record["payload"].size/2.0).round]}
    payload2 ${record["payload"][(record["payload"].size/2.0).round..-1]}
  </record>
</filter>

<filter wso2.esb.surepay.trigger>
  @type grep

  <regexp>
    key     service_name
    pattern ^\sSurePayPassiveTrigger
  </regexp>
</filter>

<filter wso2.esb.surepay.trigger>
  @type record_transformer
  enable_ruby

  <record>
    client_ip ${record["client_ip"].strip!}
    service_name ${record["service_name"].strip!}
    req_id ${record["req_id"].strip!}
    content_massage ${record["content_massage"].strip!}
  </record>
</filter>


###########################
#        Output           #
###########################
## Fluentd internal log
## match tag=fluent.**
<match fluent.**>
  @type file
  @id fluentd_internal_log

  path  /cellcard/fluent/log/fluent
  time_slice_format %Y%m%d
  time_slice_wait   10m
</match>

## Debug
## match tag=debug.** and dump to console
<match debug.**>
  @type stdout
  @id stdout_output
</match>

## ESB Service Log
## match tag=wso2.esb.**. Forward to Fluentd Collector (, stdout for debug) and write to file
<match wso2.esb.**>
  @type copy

  <store>
    @type forward
    @id forward_output
    buffer_path /cellcard/fluent/buffer/fluentd.forward
    buffer_type file
    flush_interval 5
    disable_retry_limit false
    retry_limit 15
    retry_wait 2
    send_timeout 60
    heartbeat_type tcp
    heartbeat_interval 20

    <server>
      host 172.16.100.243
      port 24224
    </server>
    ## If have sencondary fluentd server for fail-over, enable <secondary> block
    # <secondary>
    #   <server>
    #     host 192.168.0.12
    #   </server>
    # </secondary>
  </store>

  <store>
    @type file
    @id file_output

    path  /cellcard/fluent/log/wso2
    time_slice_format %Y%m%d%H
    time_slice_wait   10m
    time_format       %Y-%m-%d %H:%M:%S%z
  </store>

  # <store>
  #   @type stdout
  # </store>
</match>