From 77eddf83e5b53f85fc1991d5789941b96222c45e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 16 May 2017 12:10:43 +0900 Subject: [PATCH 1/7] Use Fluentd v0.14 API --- .../plugin/out_azureeventhubs_buffered.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/fluent/plugin/out_azureeventhubs_buffered.rb b/lib/fluent/plugin/out_azureeventhubs_buffered.rb index 488de16..2389312 100644 --- a/lib/fluent/plugin/out_azureeventhubs_buffered.rb +++ b/lib/fluent/plugin/out_azureeventhubs_buffered.rb @@ -1,22 +1,32 @@ -#module Fluent +module Fluent::Plugin - class AzureEventHubsOutputBuffered < Fluent::BufferedOutput + class AzureEventHubsOutputBuffered < Output Fluent::Plugin.register_output('azureeventhubs_buffered', self) + helpers :compat_parameters + + DEFAULT_BUFFER_TYPE = "memory" + config_param :connection_string, :string config_param :hub_name, :string config_param :include_tag, :bool, :default => false config_param :include_time, :bool, :default => false config_param :tag_time_name, :string, :default => 'time' config_param :expiry_interval, :integer, :default => 3600 # 60min - config_param :type, :string, :default => 'https' # https / amqps (Not Implemented) + config_param :type, :string, :default => 'https' # https / amqps (Not Implemented) config_param :proxy_addr, :string, :default => '' config_param :proxy_port, :integer,:default => 3128 config_param :open_timeout, :integer,:default => 60 config_param :read_timeout, :integer,:default => 60 config_param :message_properties, :hash, :default => nil + config_section :buffer do + config_set_default :@type, DEFAULT_BUFFER_TYPE + config_set_default :chunk_keys, ['tag'] + end + def configure(conf) + compat_parameters_convert(conf, :buffer) super case @type when 'amqps' @@ -44,5 +54,4 @@ def write(chunk) } end end -#end - +end From c25b74b3e378cbc1f6638efcedee2043c5445c2b Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 16 May 2017 12:11:17 +0900 Subject: [PATCH 2/7] Depends on Fluentd v0.14.15 or later --- fluent-plugin-azureeventhubs.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/fluent-plugin-azureeventhubs.gemspec b/fluent-plugin-azureeventhubs.gemspec index 732003d..016ad53 100644 --- a/fluent-plugin-azureeventhubs.gemspec +++ b/fluent-plugin-azureeventhubs.gemspec @@ -19,4 +19,5 @@ Gem::Specification.new do |spec| spec.add_development_dependency "bundler", "~> 1.7" spec.add_development_dependency "rake", "~> 10.0" + spec.add_dependency "fluentd", [">= 0.14.15", "< 2"] end From 3670deb73faf02f3ac7ac4e1308f8ace85bcfd38 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 16 May 2017 12:17:05 +0900 Subject: [PATCH 3/7] Introduce inject plugin helper --- lib/fluent/plugin/out_azureeventhubs_buffered.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/out_azureeventhubs_buffered.rb b/lib/fluent/plugin/out_azureeventhubs_buffered.rb index 2389312..3bd787e 100644 --- a/lib/fluent/plugin/out_azureeventhubs_buffered.rb +++ b/lib/fluent/plugin/out_azureeventhubs_buffered.rb @@ -3,7 +3,7 @@ module Fluent::Plugin class AzureEventHubsOutputBuffered < Output Fluent::Plugin.register_output('azureeventhubs_buffered', self) - helpers :compat_parameters + helpers :compat_parameters, :inject DEFAULT_BUFFER_TYPE = "memory" @@ -26,7 +26,7 @@ class AzureEventHubsOutputBuffered < Output end def configure(conf) - compat_parameters_convert(conf, :buffer) + compat_parameters_convert(conf, :buffer, :inject) super case @type when 'amqps' @@ -38,6 +38,7 @@ def configure(conf) end def format(tag, time, record) + record = inject_values_to_record(tag, time, record) [tag, time, record].to_msgpack end From 49e4ee8d24a27e9e3024103155424737a6e49efe Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 16 May 2017 12:46:08 +0900 Subject: [PATCH 4/7] Mark chunk as msgpack --- lib/fluent/plugin/out_azureeventhubs_buffered.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/fluent/plugin/out_azureeventhubs_buffered.rb b/lib/fluent/plugin/out_azureeventhubs_buffered.rb index 3bd787e..511114e 100644 --- a/lib/fluent/plugin/out_azureeventhubs_buffered.rb +++ b/lib/fluent/plugin/out_azureeventhubs_buffered.rb @@ -42,6 +42,10 @@ def format(tag, time, record) [tag, time, record].to_msgpack end + def formatted_to_msgpack_binary? + true + end + def write(chunk) chunk.msgpack_each { |tag, time, record| p record.to_s From 7611de5ead0b92d70dc24a56297f36519c1cba0c Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Tue, 16 May 2017 12:48:34 +0900 Subject: [PATCH 5/7] Add existence check for tag in chunk_keys --- lib/fluent/plugin/out_azureeventhubs_buffered.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/fluent/plugin/out_azureeventhubs_buffered.rb b/lib/fluent/plugin/out_azureeventhubs_buffered.rb index 511114e..cb52369 100644 --- a/lib/fluent/plugin/out_azureeventhubs_buffered.rb +++ b/lib/fluent/plugin/out_azureeventhubs_buffered.rb @@ -35,6 +35,7 @@ def configure(conf) require_relative 'azureeventhubs/http' @sender = AzureEventHubsHttpSender.new(@connection_string, @hub_name, @expiry_interval,@proxy_addr,@proxy_port,@open_timeout,@read_timeout) end + raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag end def format(tag, time, record) From 1f85cfa9fa693ddd4dcc08849f45a72e862972c8 Mon Sep 17 00:00:00 2001 From: Toddy Mladenov Date: Wed, 6 Sep 2017 16:25:16 -0700 Subject: [PATCH 6/7] Updated version and atuhor information --- fluent-plugin-azureeventhubs.gemspec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fluent-plugin-azureeventhubs.gemspec b/fluent-plugin-azureeventhubs.gemspec index 016ad53..f381e8a 100644 --- a/fluent-plugin-azureeventhubs.gemspec +++ b/fluent-plugin-azureeventhubs.gemspec @@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = "fluent-plugin-azureeventhubs" - spec.version = "0.0.5" - spec.authors = ["Hidemasa Togashi", "Justin Seely"] - spec.email = ["togachiro@gmail.com"] + spec.version = "0.0.6" + spec.authors = ["Hidemasa Togashi", "Toddy Mladenov", "Justin Seely"] + spec.email = ["togachiro@gmail.com", "toddysm@gmail.com"] spec.summary = "Fluentd output plugin for Azure Event Hubs" spec.description = "Fluentd output plugin for Azure Event Hubs" spec.homepage = "https://github.com/htgc/fluent-plugin-azureeventhubs" From d13129f0300aed057ecf0c7155893605c4eb4160 Mon Sep 17 00:00:00 2001 From: Toddy Mladenov Date: Wed, 6 Sep 2017 16:25:33 -0700 Subject: [PATCH 7/7] Added .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ae3fdc2..70d4d2f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ *.o *.a mkmf.log +*.gem