Skip to content

Commit

Permalink
feat: logs filter
Browse files Browse the repository at this point in the history
Signed-off-by: Lin Yang <[email protected]>
  • Loading branch information
reaver-flomesh committed Sep 17, 2024
1 parent 0b7e794 commit 2fa7a79
Show file tree
Hide file tree
Showing 32 changed files with 1,234 additions and 68 deletions.
Binary file modified charts/fsm/components/scripts.tar.gz
Binary file not shown.
42 changes: 42 additions & 0 deletions charts/fsm/components/scripts/gateways/filters/http/AccessLog.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
export default function (config) {
var log = console.log
var $ctx

return pipeline($=>$
.onStart(c => { $ctx = c })
.pipeNext()
.handleMessageEnd(() => {
var inbound = $ctx.parent.inbound
var headers = $ctx.head.headers
var response = $ctx.response
var target = $ctx.target
log({
protocol: $ctx.head.protocol || '',
upstream_service_time: response.headTime,
upstream_local_address: target || '',
duration: response.tailTime - $ctx.headTime,
upstream_transport_failure_reason: '',
route_name: '',
downstream_local_address: inbound.localAddress,
user_agent: headers['user-agent'] || '',
response_code: response.head?.status,
response_flags: '',
start_time: $ctx.headTime,
method: $ctx.head.method || '',
request_id: $ctx.id,
upstream_host: target,
x_forward_for: headers['x-forwarded-for'] || '',
client_ip: inbound.remoteAddress,
requested_server_name: '',
bytes_received: $ctx.tail.headSize + $ctx.tail.bodySize,
bytes_sent: response.tail ? response.tail.headSize + response.tail.bodySize : 0,
upstream_cluster: $ctx.backendResource?.metadata?.name,
downstream_remote_address: inbound.remoteAddress,
authority: $ctx.head.authority || '',
path: $ctx.path,
response_code_details: response.head?.statusText,
trace_id: headers['x-b3-traceid'] || '',
})
})
)
}
6 changes: 3 additions & 3 deletions charts/fsm/templates/fsm-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,13 @@ rules:

# GatewayAPI Extension
- apiGroups: [ "extension.gateway.flomesh.io" ]
resources: [ "filters", "filterdefinitions", "listenerfilters", "circuitbreakers", "faultinjections", "ratelimits" ]
resources: [ "filters", "filterdefinitions", "listenerfilters", "circuitbreakers", "faultinjections", "ratelimits", "httplogs" ]
verbs: [ "get", "list", "watch", "create", "update", "patch", "delete" ]
- apiGroups: [ "extension.gateway.flomesh.io" ]
resources: [ "filters/finalizers", "filterdefinitions/finalizers", "listenerfilters/finalizers", "circuitbreakers/finalizers", "faultinjections/finalizers", "ratelimits/finalizers" ]
resources: [ "filters/finalizers", "filterdefinitions/finalizers", "listenerfilters/finalizers", "circuitbreakers/finalizers", "faultinjections/finalizers", "ratelimits/finalizers", "httplogs/finalizers" ]
verbs: [ "update" ]
- apiGroups: [ "extension.gateway.flomesh.io" ]
resources: [ "filters/status", "filterdefinitions/status", "listenerfilters/status", "circuitbreakers/status", "faultinjections/status", "ratelimits/status" ]
resources: [ "filters/status", "filterdefinitions/status", "listenerfilters/status", "circuitbreakers/status", "faultinjections/status", "ratelimits/status", "httplogs/status" ]
verbs: [ "get", "patch", "update" ]

# PolicyAttachment
Expand Down
200 changes: 200 additions & 0 deletions cmd/fsm-bootstrap/crds/extension.gateway.flomesh.io_httplogs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.15.0
labels:
app.kubernetes.io/name: flomesh.io
name: httplogs.extension.gateway.flomesh.io
spec:
group: extension.gateway.flomesh.io
names:
categories:
- gateway-api
kind: HTTPLog
listKind: HTTPLogList
plural: httplogs
singular: httplog
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
schema:
openAPIV3Schema:
description: HTTPLog is the Schema for the HTTPLog API
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: HTTPLogSpec defines the desired state of HTTPLog
properties:
batch:
default:
interval: 1s
postfix: ""
prefix: ""
separator: |2+
size: 1000
description: Batch is the batch configuration of the logs
properties:
interval:
default: 1s
description: Interval is the interval to send a batch, default
is 1s
type: string
postfix:
default: ""
description: Postfix is the postfix of the batch, default is ""
type: string
prefix:
default: ""
description: Prefix is the prefix of the batch, default is ""
type: string
separator:
default: |2+
description: Separator is the separator of the logs in the batch,
default is "\n"
type: string
size:
default: 1000
description: Size is the maximum number of logs in a batch, default
is 1000
format: int32
minimum: 1
type: integer
type: object
bufferLimit:
default: 1048576
description: BufferLimit is the maximum size of the buffer in bytes,
default is 1048576(1MB)
format: int64
minimum: 1
type: integer
headers:
additionalProperties:
type: string
description: Headers is the HTTP headers of the log request
type: object
method:
default: POST
description: Method is the HTTP method of the HTTPLog service, default
is POST
enum:
- GET
- POST
- PUT
- DELETE
- PATCH
- HEAD
- OPTIONS
type: string
target:
description: Target is the URL of the HTTPLog service
type: string
required:
- target
type: object
status:
description: HTTPLogStatus defines the observed state of HTTPLog
properties:
conditions:
description: Conditions describe the current conditions of the HTTPLog.
items:
description: "Condition contains details for one aspect of the current
state of this API Resource.\n---\nThis struct is intended for
direct use as an array at the field path .status.conditions. For
example,\n\n\n\ttype FooStatus struct{\n\t // Represents the
observations of a foo's current state.\n\t // Known .status.conditions.type
are: \"Available\", \"Progressing\", and \"Degraded\"\n\t //
+patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t
\ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\"
patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t
\ // other fields\n\t}"
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: |-
type of condition in CamelCase or in foo.example.com/CamelCase.
---
Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be
useful (see .node.status.conditions), the ability to deconflict is important.
The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
maxItems: 8
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
type: object
type: object
served: true
storage: true
subresources:
status: {}
11 changes: 11 additions & 0 deletions pkg/announcements/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -644,6 +644,17 @@ const (

// ListenerFilterUpdated is the type of announcement emitted when we observe an update to listenerfilters.extension.gateway.flomesh.io
ListenerFilterUpdated Kind = "listenerfilter-updated"

// --

// HTTPLogAdded is the type of announcement emitted when we observe an addition of httplogs.extension.gateway.flomesh.io
HTTPLogAdded Kind = "httplog-added"

// HTTPLogDeleted the type of announcement emitted when we observe a deletion of httplogs.extension.gateway.flomesh.io
HTTPLogDeleted Kind = "httplog-deleted"

// HTTPLogUpdated is the type of announcement emitted when we observe an update to httplogs.extension.gateway.flomesh.io
HTTPLogUpdated Kind = "httplog-updated"
)

// Announcement is a struct for messages between various components of FSM signaling a need for a change in Sidecar proxy configuration
Expand Down
100 changes: 100 additions & 0 deletions pkg/apis/extension/v1alpha1/httplog.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package v1alpha1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// HTTPLogSpec defines the desired state of HTTPLog
type HTTPLogSpec struct {
// +kubebuilder:validation:Required
// Target is the URL of the HTTPLog service
Target string `json:"target"`

// +optional
// +kubebuilder:default="POST"
// +kubebuilder:validation:Enum=GET;POST;PUT;DELETE;PATCH;HEAD;OPTIONS
// Method is the HTTP method of the HTTPLog service, default is POST
Method *string `json:"method,omitempty"`

// +optional
// Headers is the HTTP headers of the log request
Headers map[string]string `json:"headers,omitempty"`

// +optional
// +kubebuilder:default=1048576
// +kubebuilder:validation:Minimum=1
// BufferLimit is the maximum size of the buffer in bytes, default is 1048576(1MB)
BufferLimit *int64 `json:"bufferLimit,omitempty"`

// +optional
// +kubebuilder:default={size: 1000, interval: "1s", prefix: "", postfix: "", separator: "\n"}
// Batch is the batch configuration of the logs
Batch *HTTPLogBatch `json:"batch,omitempty"`
}

type HTTPLogBatch struct {
// +optional
// +kubebuilder:default=1000
// +kubebuilder:validation:Minimum=1
// Size is the maximum number of logs in a batch, default is 1000
Size *int32 `json:"size,omitempty"`

// +optional
// +kubebuilder:default="1s"
// Interval is the interval to send a batch, default is 1s
Interval *metav1.Duration `json:"interval,omitempty"`

// +optional
// +kubebuilder:default=""
// Prefix is the prefix of the batch, default is ""
Prefix *string `json:"prefix,omitempty"`

// +optional
// +kubebuilder:default=""
// Postfix is the postfix of the batch, default is ""
Postfix *string `json:"postfix,omitempty"`

// +optional
// +kubebuilder:default="\n"
// Separator is the separator of the logs in the batch, default is "\n"
Separator *string `json:"separator,omitempty"`
}

// HTTPLogStatus defines the observed state of HTTPLog
type HTTPLogStatus struct {
// Conditions describe the current conditions of the HTTPLog.
//
// +optional
// +listType=map
// +listMapKey=type
// +kubebuilder:validation:MaxItems=8
Conditions []metav1.Condition `json:"conditions,omitempty"`
}

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +kubebuilder:storageversion
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:scope=Namespaced,categories=gateway-api
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
// +kubebuilder:metadata:labels={app.kubernetes.io/name=flomesh.io}

// HTTPLog is the Schema for the HTTPLog API
type HTTPLog struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Spec HTTPLogSpec `json:"spec,omitempty"`
Status HTTPLogStatus `json:"status,omitempty"`
}

// +kubebuilder:object:root=true
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// HTTPLogList contains a list of HTTPLog
type HTTPLogList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []HTTPLog `json:"items"`
}
Loading

0 comments on commit 2fa7a79

Please sign in to comment.