Add nodeName template var and update doc (#6088)

Signed-off-by: Ketian Xu <[email protected]>
Signed-off-by: Eduardo Apolinario <[email protected]>
Co-authored-by: Eduardo Apolinario <[email protected]>

docs/user_guide/productionizing/configuring_logging_links_in_the_ui.md

@@ -40,7 +40,9 @@ The parameters can be used to generate a unique URL to the logs using a template
    * - ``{{ .logName }}``
      - A deployment specific name where to expect the logs to be
    * - ``{{ .hostname }}``
-     - The hostname where the pod is running and logs reside
+     - The value used to override the hostname the pod uses internally within its own network namespace (i.e., the pod's ``.spec.hostname``)
+   * - ``{{ .nodeName }}``
+     - The hostname of the node where the pod is running and logs reside (i.e., the pod's ``.spec.nodeName``)
    * - ``{{ .podRFC3339StartTime }}``
      - The pod creation time (in RFC3339 format, e.g. "2021-01-01T02:07:14Z", also conforming to ISO 8601)
    * - ``{{ .podRFC3339FinishTime }}``

flyteplugins/go/tasks/logs/logging_utils.go

@@ -64,6 +64,7 @@ func GetLogsForContainerInPod(ctx context.Context, logPlugin tasklog.Plugin, tas
 			ExtraTemplateVars:    extraLogTemplateVars,
 			TaskTemplate:         taskTemplate,
 			HostName:             pod.Spec.Hostname,
+			NodeName:             pod.Spec.NodeName,
 		},
 	)
 

flyteplugins/go/tasks/logs/logging_utils_test.go

@@ -302,7 +302,7 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) {
 				MessageFormat: core.TaskLog_JSON,
 				Name:          "Stackdriver Logs my-Suffix",
 			},
-		}, "")
+		}, "", "")
 	})
 
 	t.Run("StackDriver", func(t *testing.T) {
@@ -315,11 +315,11 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) {
 				MessageFormat: core.TaskLog_JSON,
 				Name:          "Stackdriver Logs my-Suffix",
 			},
-		}, "")
+		}, "", "")
 	})
 }
 
-func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string) {
+func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string, nodeName string) {
 	logPlugin, err := InitializeLogPlugins(config)
 	assert.NoError(tb, err)
 
@@ -335,6 +335,7 @@ func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.Ta
 				},
 			},
 			Hostname: hostname,
+			NodeName: nodeName,
 		},
 		Status: v1.PodStatus{
 			ContainerStatuses: []v1.ContainerStatus{
@@ -382,7 +383,7 @@ func TestGetLogsForContainerInPod_Templates(t *testing.T) {
 			MessageFormat: core.TaskLog_JSON,
 			Name:          "Internal my-Suffix",
 		},
-	}, "")
+	}, "", "")
 }
 
 func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) {
@@ -402,7 +403,27 @@ func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) {
 			MessageFormat: core.TaskLog_JSON,
 			Name:          "StackDriver my-Suffix",
 		},
-	}, "my-hostname")
+	}, "my-hostname", "")
+}
+
+func TestGetLogsForContainerInPodTemplates_NodeName(t *testing.T) {
+	assertTestSucceeded(t, &LogConfig{
+		Templates: []tasklog.TemplateLogPlugin{
+			{
+				DisplayName: "MyLog",
+				TemplateURIs: []string{
+					"{{ .nodeName }}/{{ .namespace }}/{{ .podName }}/{{ .containerName }}/{{ .containerId }}",
+				},
+				MessageFormat: core.TaskLog_JSON,
+			},
+		},
+	}, nil, []*core.TaskLog{
+		{
+			Uri:           "ip-1-2-3-4/my-namespace/my-pod/ContainerName/ContainerID",
+			MessageFormat: core.TaskLog_JSON,
+			Name:          "MyLog my-Suffix",
+		},
+	}, "my-hostname", "ip-1-2-3-4")
 }
 
 func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) {
@@ -562,7 +583,7 @@ func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "")
+			assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "", "")
 		})
 	}
 }

flyteplugins/go/tasks/pluginmachinery/tasklog/plugin.go

@@ -28,6 +28,7 @@ type TemplateVar struct {
 // log links.
 type Input struct {
 	HostName             string
+	NodeName             string
 	PodName              string
 	Namespace            string
 	ContainerName        string

flyteplugins/go/tasks/pluginmachinery/tasklog/template.go

@@ -27,6 +27,7 @@ type templateRegexes struct {
 	ContainerName        *regexp.Regexp
 	ContainerID          *regexp.Regexp
 	Hostname             *regexp.Regexp
+	NodeName             *regexp.Regexp
 	PodRFC3339StartTime  *regexp.Regexp
 	PodRFC3339FinishTime *regexp.Regexp
 	PodUnixStartTime     *regexp.Regexp
@@ -52,6 +53,7 @@ func initDefaultRegexes() templateRegexes {
 		MustCreateRegex("containerName"),
 		MustCreateRegex("containerID"),
 		MustCreateRegex("hostname"),
+		MustCreateRegex("nodeName"),
 		MustCreateRegex("podRFC3339StartTime"),
 		MustCreateRegex("podRFC3339FinishTime"),
 		MustCreateRegex("podUnixStartTime"),
@@ -105,6 +107,7 @@ func (input Input) templateVars() []TemplateVar {
 		TemplateVar{defaultRegexes.ContainerName, input.ContainerName},
 		TemplateVar{defaultRegexes.ContainerID, containerID},
 		TemplateVar{defaultRegexes.Hostname, input.HostName},
+		TemplateVar{defaultRegexes.NodeName, input.NodeName},
 	)
 	if input.TaskExecutionID != nil {
 		taskExecutionIdentifier := input.TaskExecutionID.GetID()

flyteplugins/go/tasks/pluginmachinery/tasklog/template_test.go

@@ -63,6 +63,7 @@ func Test_Input_templateVars(t *testing.T) {
 	}
 	podBase := Input{
 		HostName:             "my-host",
+		NodeName:             "my-node-name",
 		PodName:              "my-pod",
 		PodUID:               "my-pod-uid",
 		Namespace:            "my-namespace",
@@ -103,6 +104,7 @@ func Test_Input_templateVars(t *testing.T) {
 				{defaultRegexes.ContainerName, "my-container"},
 				{defaultRegexes.ContainerID, "containerID"},
 				{defaultRegexes.Hostname, "my-host"},
+				{defaultRegexes.NodeName, "my-node-name"},
 				{defaultRegexes.PodRFC3339StartTime, "1970-01-01T01:02:03+01:00"},
 				{defaultRegexes.PodRFC3339FinishTime, "1970-01-01T04:25:45+01:00"},
 				{defaultRegexes.PodUnixStartTime, "123"},
@@ -139,6 +141,7 @@ func Test_Input_templateVars(t *testing.T) {
 				{defaultRegexes.ContainerName, ""},
 				{defaultRegexes.ContainerID, ""},
 				{defaultRegexes.Hostname, ""},
+				{defaultRegexes.NodeName, ""},
 				{defaultRegexes.NodeID, "n0-0-n0"},
 				{defaultRegexes.GeneratedName, "generated-name"},
 				{defaultRegexes.TaskRetryAttempt, "1"},
@@ -327,12 +330,13 @@ func TestTemplateLogPlugin(t *testing.T) {
 		{
 			"ddog",
 			TemplateLogPlugin{
-				TemplateURIs:  []TemplateURI{"https://app.datadoghq.com/logs?event&from_ts={{ .podUnixStartTime }}&live=true&query=pod_name%3A{{ .podName }}&to_ts={{ .podUnixFinishTime }}"},
+				TemplateURIs:  []TemplateURI{"https://app.datadoghq.com/logs?event&from_ts={{ .podUnixStartTime }}&live=true&query=pod_name%3A{{ .podName }}&to_ts={{ .podUnixFinishTime }}&host={{ .nodeName }}"},
 				MessageFormat: core.TaskLog_JSON,
 			},
 			args{
 				input: Input{
 					HostName:             "my-host",
+					NodeName:             "ip-1-2-3-4",
 					PodName:              "my-pod",
 					Namespace:            "my-namespace",
 					ContainerName:        "my-container",
@@ -347,7 +351,7 @@ func TestTemplateLogPlugin(t *testing.T) {
 			Output{
 				TaskLogs: []*core.TaskLog{
 					{
-						Uri:           "https://app.datadoghq.com/logs?event&from_ts=123&live=true&query=pod_name%3Amy-pod&to_ts=12345",
+						Uri:           "https://app.datadoghq.com/logs?event&from_ts=123&live=true&query=pod_name%3Amy-pod&to_ts=12345&host=ip-1-2-3-4",
 						MessageFormat: core.TaskLog_JSON,
 						Name:          "main_logs",
 					},