Nr 91173 fluent bit 3 windows (#1909)

* NR-91173 feat: fluent bit 3.x under FF * NR-91173 feat: fluent-bit 3.x on Windows * eat: Upgrade to FB on windows 3.0.6 * feat: fluent_bit19 remove fflag check on linux and add new fflag for windows --------- Co-authored-by: Ruben Ruiz de Gauna <[email protected]>
newrelic · Aug 12, 2024 · 723f535 · 723f535
1 parent 681300c
commit 723f535
Show file tree

Hide file tree

Showing 19 changed files with 247 additions and 24 deletions.
diff --git a/build/embed/fluent-bit.version b/build/embed/fluent-bit.version
@@ -6,4 +6,6 @@
 # OS, newrelic_plugin_version, fluent-bit
 
 linux,1.19.1
-windows,1.19.1,1.9.3
+windows,1.19.1,3.0.6
+#To be removed on removal of the ff fluent_bit_19
+windows-legacy,1.19.1,1.9.3
diff --git a/build/package/windows/newrelic-infra-386-installer/newrelic-infra/Product.wxs b/build/package/windows/newrelic-infra-386-installer/newrelic-infra/Product.wxs
@@ -37,13 +37,15 @@
             <ComponentRef Id="CMP_V1_CUSTOM_PLUGINS" />
             <ComponentRef Id="CMP_V1_NR_PLUGINS" />
             <ComponentRef Id="CMP_LOGGING_TOOL" />
+            <ComponentRef Id="CMP_LOGGING_TOOL_FB2" />
             <ComponentRef Id="CMP_LOGGING_TOOL_CFG" />
             <ComponentRef Id="NewRelicInfraDataComponent" />
             <ComponentGroupRef Id="ProductComponents" />
             <ComponentGroupRef Id="BundledPluginDefinitionComponents" />
             <ComponentGroupRef Id="BundledPluginConfigComponents" />
             <ComponentGroupRef Id="BundledPluginComponents" />
             <ComponentGroupRef Id="LoggingToolComponents" />
+            <ComponentGroupRef Id="LoggingToolFB2Components" />
             <ComponentGroupRef Id="LoggingToolCfgComponents" />
         </Feature>
     </Product>
@@ -70,7 +72,8 @@
                         <Directory Id="LoggingToolCfg" Name="logging.d" />
                         <Directory Id="V1CustomPluginsFolder" Name="custom-integrations" />
                         <Directory Id="V1NRPluginsFolder" Name="newrelic-integrations">
-                            <Directory Id="LoggingTool" Name="logging" />
+                            <Directory Id="LoggingTool" Name="logging-legacy" />
+                            <Directory Id="LoggingToolFB2" Name="logging" />
                         </Directory>
                     </Directory>
                 </Directory>
@@ -99,6 +102,11 @@
                 <CreateFolder />
             </Component>
         </DirectoryRef>
+        <DirectoryRef Id="LoggingToolFB2">
+            <Component Id="CMP_LOGGING_TOOL_FB2" Guid="D66DA13A-2F34-409B-9FF2-4CA00B0BB8B7" KeyPath="yes">
+                <CreateFolder />
+            </Component>
+        </DirectoryRef>
         <DirectoryRef Id="V1NRPluginsFolder">
             <Component Id="CMP_V1_NR_PLUGINS" Guid="070708D0-87B5-4543-83C3-5A037A15BD5B" KeyPath="yes">
                 <CreateFolder />
@@ -243,6 +251,7 @@
         </ComponentGroup>
     </Fragment>
 
+  <!-- <To be removed on removal of the ff fluent_bit_19> -->
   <Fragment>
     <ComponentGroup Id="LoggingToolComponents" Directory="LoggingTool">
         <Component Id="CMP_LOGGING_TOOL_BIN" Guid="015B10A1-7843-4961-B221-CBB449A6646C" Win64="no">
@@ -267,6 +276,32 @@
         </Component>
     </ComponentGroup>
   </Fragment>
+  <!-- </To be removed on removal of the ff fluent_bit_19> -->
+
+  <Fragment>
+    <ComponentGroup Id="LoggingToolFB2Components" Directory="LoggingToolFB2">
+        <Component Id="CMP_LOGGING_FB2_TOOL_BIN" Guid="A6D1104A-F822-4D5C-AD0E-7B90B4CADA81" Win64="no">
+          <File Id="FILE_LOGGING_FB2_TOOL_BIN"
+              Source="$(var.EmbedBinariesPath)logging\nrfb2\fluent-bit.exe"
+                KeyPath="yes" />
+        </Component>
+        <Component Id="CMP_LOGGING_FB2_TOOL_DLL" Guid="7F9069E1-BBFB-4A3F-966E-8E3C092E23C6" Win64="no">
+          <File Id="FILE_LOGGING_FB2_TOOL_DLL"
+              Source="$(var.EmbedBinariesPath)logging\nrfb2\fluent-bit.dll"
+                KeyPath="yes" />
+        </Component>
+        <Component Id="CMP_NR_LOGGING_FB2_TOOL_DLL" Guid="FA321297-4693-4402-BE78-105B0234C75A" Win64="no">
+          <File Id="FILE_NR_LOGGING_FB2_TOOL_DLL"
+              Source="$(var.EmbedBinariesPath)logging\nrfb2\out_newrelic.dll"
+                KeyPath="yes" />
+        </Component>
+        <Component Id="CMP_NR_LOGGING_FB2_TOOL_PARSERS" Guid="C99EA5B8-447F-4653-821B-BF30F994F4CE" Win64="no">
+          <File Id="FILE_NR_LOGGING_FB2_TOOL_PARSERS"
+              Source="$(var.ExternalFilesPath)examples\logging\parsers.conf"
+                KeyPath="yes" />
+        </Component>
+    </ComponentGroup>
+  </Fragment>
 
   <Fragment>
     <ComponentGroup Id="LoggingToolCfgComponents" Directory="LoggingToolCfg">

diff --git a/build/package/windows/newrelic-infra-amd64-installer/newrelic-infra/Product.wxs b/build/package/windows/newrelic-infra-amd64-installer/newrelic-infra/Product.wxs
@@ -37,12 +37,14 @@
             <ComponentRef Id="CMP_V1_CUSTOM_PLUGINS" />
             <ComponentRef Id="CMP_V1_NR_PLUGINS" />
             <ComponentRef Id="CMP_LOGGING_TOOL" />
+            <ComponentRef Id="CMP_LOGGING_TOOL_FB2" />
             <ComponentRef Id="CMP_LOGGING_TOOL_CFG" />
             <ComponentRef Id="NewRelicInfraDataComponent" />
             <ComponentGroupRef Id="ProductComponents" />
             <ComponentGroupRef Id="BundledPluginConfigComponents" />
             <ComponentGroupRef Id="BundledPluginComponents" />
             <ComponentGroupRef Id="LoggingToolComponents" />
+            <ComponentGroupRef Id="LoggingToolFB2Components" />
             <ComponentGroupRef Id="LoggingToolCfgComponents" />
         </Feature>
     </Product>
@@ -71,7 +73,8 @@
                         <Directory Id="LoggingToolCfg" Name="logging.d" />
                         <Directory Id="V1CustomPluginsFolder" Name="custom-integrations" />
                         <Directory Id="V1NRPluginsFolder" Name="newrelic-integrations">
-                            <Directory Id="LoggingTool" Name="logging" />
+                            <Directory Id="LoggingTool" Name="logging-legacy" />
+                            <Directory Id="LoggingToolFB2" Name="logging" />
                         </Directory>
                     </Directory>
                 </Directory>
@@ -99,6 +102,11 @@
           <Component Id="CMP_LOGGING_TOOL" Guid="97CBE086-FE32-44FE-B035-9B7D66A422AD" KeyPath="yes">
               <CreateFolder />
           </Component>
+      </DirectoryRef>
+        <DirectoryRef Id="LoggingToolFB2">
+            <Component Id="CMP_LOGGING_TOOL_FB2" Guid="AA775C42-74B2-4260-A477-3CC57B5D5842" KeyPath="yes">
+              <CreateFolder />
+          </Component>
       </DirectoryRef>
       <DirectoryRef Id="V1NRPluginsFolder">
           <Component Id="CMP_V1_NR_PLUGINS" Guid="48198DB9-B622-49E6-B7B4-AEED7B7378DB" KeyPath="yes">
@@ -226,6 +234,7 @@
         </InstallExecuteSequence>
     </Fragment>
 
+  <!-- <To be removed on removal of the ff fluent_bit_19> -->
   <Fragment>
     <ComponentGroup Id="LoggingToolComponents" Directory="LoggingTool">
         <Component Id="CMP_LOGGING_TOOL_BIN" Guid="015B10A1-7843-4961-B221-CBB449A6646C" Win64="yes">
@@ -250,6 +259,32 @@
         </Component>
     </ComponentGroup>
   </Fragment>
+  <!-- </To be removed on removal of the ff fluent_bit_19> -->
+
+  <Fragment>
+      <ComponentGroup Id="LoggingToolFB2Components" Directory="LoggingToolFB2">
+          <Component Id="CMP_LOGGING_FB2_TOOL_BIN" Guid="297401DE-6C77-4523-AE2D-E2B4982A34E2" Win64="yes">
+            <File Id="FILE_LOGGING_FB2_TOOL_BIN"
+                Source="$(var.EmbedBinariesPath)logging\nrfb2\fluent-bit.exe"
+                  KeyPath="yes" />
+          </Component>
+          <Component Id="CMP_LOGGING_FB2_TOOL_DLL" Guid="B0652EBD-1F42-46CA-B1F6-2BAB76117A04" Win64="yes">
+            <File Id="FILE_LOGGING_FB2_TOOL_DLL"
+                Source="$(var.EmbedBinariesPath)logging\nrfb2\fluent-bit.dll"
+                  KeyPath="yes" />
+          </Component>
+          <Component Id="CMP_NR_LOGGING_FB2_TOOL_DLL" Guid="76F95029-80F0-4CB2-96B2-D414608E92F7" Win64="yes">
+            <File Id="FILE_NR_LOGGING_FB2_TOOL_DLL"
+                Source="$(var.EmbedBinariesPath)logging\nrfb2\out_newrelic.dll"
+                  KeyPath="yes" />
+          </Component>
+          <Component Id="CMP_NR_LOGGING_FB2_TOOL_PARSERS" Guid="52DC6B7E-EF16-4A49-8A2D-F963030B0916" Win64="yes">
+            <File Id="FILE_NR_LOGGING_FB2_TOOL_PARSERS"
+                Source="$(var.ExternalFilesPath)examples\logging\parsers.conf"
+                  KeyPath="yes" />
+          </Component>
+      </ComponentGroup>
+    </Fragment>
 
   <Fragment>
     <ComponentGroup Id="LoggingToolCfgComponents" Directory="LoggingToolCfg">

diff --git a/build/windows/scripts/embed_ohis.ps1 b/build/windows/scripts/embed_ohis.ps1
@@ -73,17 +73,33 @@ Function EmbedPrometheus {
 Function EmbedFluentBit {
     Write-Output "--- Embedding fluent-bit"
 
+    # <To be removed on removal of the ff fluent_bit_19>
+    # td-agent-bit (1.9)
+    $pluginLegacyVersion = GetFluentBitLegacyPluginVersion
+    $nrfbLegacyVersion = GetFluentBitLegacyVersion
+
+    [string]$pluginLegacyUrl = "https://github.com/newrelic/newrelic-fluent-bit-output/releases/download/v$pluginLegacyVersion/out_newrelic-windows-$arch-$pluginLegacyVersion.dll"
+    DownloadFile -dest:"$downloadPath\logging\nrfb" -outFile:"out_newrelic.dll" -url:"$pluginLegacyUrl"
+
+    [string]$nrfbUrl = "https://github.com/newrelic-experimental/fluent-bit-package/releases/download/$nrfbLegacyVersion/fb-windows-$arch.zip"
+    DownloadAndExtractZip -dest:"$downloadPath\logging\nrfb" -url:"$nrfbUrl"
+    # </To be removed on removal of the ff fluent_bit_19>
+
+    ## fluent-bit (3.x)
     $pluginVersion = GetFluentBitPluginVersion
     $nrfbVersion = GetFluentBitVersion
 
     [string]$pluginUrl = "https://github.com/newrelic/newrelic-fluent-bit-output/releases/download/v$pluginVersion/out_newrelic-windows-$arch-$pluginVersion.dll"
-    DownloadFile -dest:"$downloadPath\logging\nrfb" -outFile:"out_newrelic.dll" -url:"$pluginUrl"
+    DownloadFile -dest:"$downloadPath\logging\nrfb2" -outFile:"out_newrelic.dll" -url:"$pluginUrl"
 
-    [string]$nrfbUrl = "https://github.com/newrelic-experimental/fluent-bit-package/releases/download/$nrfbVersion/fb-windows-$arch.zip"
-    DownloadAndExtractZip -dest:"$downloadPath\logging\nrfb" -url:"$nrfbUrl"
+    [string]$nrfbUrl = "https://logging-fb-windows-packages.s3.us-east-2.amazonaws.com/fb-windows-$nrfbVersion-$arch.zip"
+    DownloadAndExtractZip -dest:"$downloadPath\logging\nrfb2" -url:"$nrfbUrl"
 
     if (-Not $skipSigning) {
+        # <To be removed on removal of the ff fluent_bit_19>
         SignExecutable -executable "$downloadPath\logging\nrfb\fluent-bit.exe"
+        # </To be removed on removal of the ff fluent_bit_19>
+        SignExecutable -executable "$downloadPath\logging\nrfb2\fluent-bit.exe"
     }
 }
 

diff --git a/build/windows/scripts/functions.ps1 b/build/windows/scripts/functions.ps1
@@ -30,10 +30,33 @@ Function GetIntegrationVersion {
     return $version
 }
 
+# <To be removed on removal of the ff fluent_bit_19>
+Function GetFluentBitLegacyPluginVersion {
+    $dir = "$scriptPath\..\..\embed"
+
+    $pluginVersion = $(Get-Content "$dir/fluent-bit.version" | %{if($_ -match "^windows-legacy,") { $_.Split(',')[1]; }})
+    if ([string]::IsNullOrWhitespace($pluginVersion)) {
+        throw "failed to read nr fluent-bit plugin version"
+    }
+
+    return $pluginVersion
+}
+
+Function GetFluentBitLegacyVersion {
+    $dir = "$scriptPath\..\..\embed"
+    $fbVersion = $(Get-Content "$dir/fluent-bit.version" | %{if($_ -match "^windows-legacy,") { $_.Split(',')[2]; }})
+    if ([string]::IsNullOrWhitespace($fbVersion)) {
+        throw "failed to read nr fluent-bit 1.x version"
+    }
+
+    return $fbVersion
+}
+# </To be removed on removal of the ff fluent_bit_19>
+
 Function GetFluentBitPluginVersion {
     $dir = "$scriptPath\..\..\embed"
 
-    $pluginVersion = $(Get-Content "$dir/fluent-bit.version" | %{if($_ -match "^windows") { $_.Split(',')[1]; }})
+    $pluginVersion = $(Get-Content "$dir/fluent-bit.version" | %{if($_ -match "^windows,") { $_.Split(',')[1]; }})
     if ([string]::IsNullOrWhitespace($pluginVersion)) {
         throw "failed to read nr fluent-bit plugin version"
     }
@@ -42,9 +65,9 @@ Function GetFluentBitPluginVersion {
 }
 Function GetFluentBitVersion {
     $dir = "$scriptPath\..\..\embed"
-    $fbVersion = $(Get-Content "$dir/fluent-bit.version" | %{if($_ -match "^windows") { $_.Split(',')[2]; }})
+    $fbVersion = $(Get-Content "$dir/fluent-bit.version" | %{if($_ -match "^windows,") { $_.Split(',')[2]; }})
     if ([string]::IsNullOrWhitespace($fbVersion)) {
-        throw "failed to read nr fluent-bit version"
+        throw "failed to read nr fluent-bit 3.x version"
     }
 
     return $fbVersion

diff --git a/cmd/newrelic-infra/newrelic-infra.go b/cmd/newrelic-infra/newrelic-infra.go
@@ -455,6 +455,7 @@ func initializeAgentAndRun(c *config.Config, logFwCfg config.LogForward) error {
 		os.Exit(1)
 	}
 
+	fbVerbose := c.Log.Level == config.LogLevelTrace && c.Log.HasIncludeFilter(config.TracesFieldName, config.SupervisorTrace)
 	confTempFolder := filepath.Join(c.AgentTempDir, v4.FbConfTempFolderNameDefault)
 	fbIntCfg := v4.NewFBSupervisorConfig(
 		ffManager,
@@ -464,7 +465,7 @@ func initializeAgentAndRun(c *config.Config, logFwCfg config.LogForward) error {
 		c.FluentBitExePath,
 		c.FluentBitNRLibPath,
 		c.FluentBitParsersPath,
-		logFwCfg.FluentBitVerbose,
+		fbVerbose,
 		confTempFolder,
 	)
 

diff --git a/internal/agent/cmdchannel/fflag/ffhandler.go b/internal/agent/cmdchannel/fflag/ffhandler.go
@@ -26,7 +26,7 @@ const (
 	FlagProtocolV4           = "protocol_v4_enabled"
 	FlagFullProcess          = "full_process_sampling"
 	FlagDmRegisterDeprecated = "dm_register_deprecated"
-	FlagFluentBit19          = "fluent_bit_19"
+	FlagFluentBit19          = "fluent_bit_19_win"
 	// Config
 	CfgYmlRegisterEnabled              = "register_enabled"
 	CfgYmlParallelizeInventory         = "inventory_queue_len"

diff --git a/internal/agent/cmdchannel/fflag/ffhandler_test.go b/internal/agent/cmdchannel/fflag/ffhandler_test.go
@@ -310,7 +310,7 @@ func TestSrv_InitialFetch_EnablesFb19(t *testing.T) {
 				"name": "set_feature_flag",
 				"arguments": {
 					"category": "Infra_Agent",
-					"flag": "fluent_bit_19",
+					"flag": "fluent_bit_19_win",
 					"enabled": true
 				}
 			}

diff --git a/pkg/integrations/v4/supervisor_fb_conf_linux.go b/pkg/integrations/v4/supervisor_fb_conf_linux.go
@@ -19,17 +19,17 @@ const (
 	defaultFluentBitExecutable2 = "fluent-bit"
 )
 
-func (c *fBSupervisorConfig) defaultLoggingBinDir(ffExists bool, ffEnabled bool) string {
-	if (ffExists && ffEnabled) || onlyTdAgentInstalled() {
+func (c *fBSupervisorConfig) defaultLoggingBinDir(_ bool, _ bool) string {
+	if onlyTdAgentInstalled() {
 		return defaultLoggingBinDir1
 	}
 
 	return defaultLoggingBinDir2
 }
 
-func (c *fBSupervisorConfig) defaultFluentBitExePath(ffExists bool, ffEnabled bool, loggingBinDir string) string {
+func (c *fBSupervisorConfig) defaultFluentBitExePath(_ bool, _ bool, loggingBinDir string) string {
 	defaultFluentBitExe := defaultFluentBitExecutable2
-	if (ffExists && ffEnabled) || onlyTdAgentInstalled() {
+	if onlyTdAgentInstalled() {
 		defaultFluentBitExe = defaultFluentBitExecutable1
 	}
 

diff --git a/pkg/integrations/v4/supervisor_fb_conf_linux_test.go b/pkg/integrations/v4/supervisor_fb_conf_linux_test.go
@@ -40,7 +40,7 @@ func Test_defaultLoggingBinDir(t *testing.T) {
 			name:           "enabled ff",
 			ffExists:       true,
 			ffEnabled:      true,
-			expectedBinDir: "/opt/td-agent-bit/bin",
+			expectedBinDir: "/opt/fluent-bit/bin",
 		},
 	}
 
@@ -86,7 +86,7 @@ func Test_defaultFluentBitExePath(t *testing.T) {
 			name:            "enabled ff",
 			ffExists:        true,
 			ffEnabled:       true,
-			expectedExePath: "/opt/td-agent-bit/bin/td-agent-bit",
+			expectedExePath: "/opt/fluent-bit/bin/fluent-bit",
 		},
 	}
 

diff --git a/pkg/integrations/v4/supervisor_fb_conf_windows.go b/pkg/integrations/v4/supervisor_fb_conf_windows.go
@@ -8,7 +8,7 @@ import "path/filepath"
 
 const (
 	// defaults for td-agent-bit (<=1.9).
-	defaultLoggingBinDir1 = "logging"
+	defaultLoggingBinDir1 = "logging-legacy"
 	// defaults for fluent-bit (>=2.0).
 	defaultLoggingBinDir2 = "logging"
 	// both versions have the same name.

diff --git a/pkg/integrations/v4/supervisor_fb_conf_windows_test.go b/pkg/integrations/v4/supervisor_fb_conf_windows_test.go
@@ -42,7 +42,7 @@ func Test_defaultLoggingBinDir(t *testing.T) {
 			name:                  "enabled ff",
 			ffExists:              true,
 			ffEnabled:             true,
-			expectedLoggingBinDir: "C:\\some\\agent\\dir\\integrations_dir\\logging",
+			expectedLoggingBinDir: "C:\\some\\agent\\dir\\integrations_dir\\logging-legacy",
 		},
 	}
 
@@ -90,7 +90,7 @@ func Test_defaultFluentBitExePath(t *testing.T) {
 			name:            "enabled ff",
 			ffExists:        true,
 			ffEnabled:       true,
-			expectedExePath: "C:\\some\\agent\\dir\\integrations_dir\\logging\\fluent-bit.exe",
+			expectedExePath: "C:\\some\\agent\\dir\\integrations_dir\\logging-legacy\\fluent-bit.exe",
 		},
 	}
 

diff --git a/pkg/integrations/v4/supervisor_fb_test.go b/pkg/integrations/v4/supervisor_fb_test.go
@@ -311,7 +311,7 @@ func TestNewSupervisorConfig(t *testing.T) {
 			integrationsDir:     "integrationsDir",
 			ffEnabled:           true,
 			ffExists:            true,
-			expectedPathLinux:   filepath.Join("loggingBinDir", "td-agent-bit"),
+			expectedPathLinux:   filepath.Join("loggingBinDir", "fluent-bit"),
 			expectedPathWindows: filepath.Join("loggingBinDir", "fluent-bit.exe"),
 		},
 		{
@@ -327,8 +327,8 @@ func TestNewSupervisorConfig(t *testing.T) {
 			ffExists:            true,
 			integrationsDir:     "integrationsDir",
 			agentDir:            "some_agent_dir",
-			expectedPathLinux:   filepath.Join("/opt/td-agent-bit/bin", "td-agent-bit"),
-			expectedPathWindows: filepath.Join("some_agent_dir", "integrationsDir", "logging", "fluent-bit.exe"),
+			expectedPathLinux:   filepath.Join("/opt/fluent-bit/bin", "fluent-bit"),
+			expectedPathWindows: filepath.Join("some_agent_dir", "integrationsDir", "logging-legacy", "fluent-bit.exe"),
 		},
 	}