From 817aea918756d362eeee9c9db816b67ce90079f1 Mon Sep 17 00:00:00 2001 From: Juliano Martinez Date: Mon, 23 Sep 2024 10:08:22 +0200 Subject: [PATCH 1/4] update readme and update development setup --- .gitignore | 4 + README.md | 155 +++++++++++++++++- configs/{docker => development}/Dockerfile | 0 configs/{docker => development}/Makefile | 0 .../config/config.yaml | 0 .../docker-compose.yaml | 0 .../{docker => development}/scripts/writer.sh | 0 7 files changed, 157 insertions(+), 2 deletions(-) rename configs/{docker => development}/Dockerfile (100%) rename configs/{docker => development}/Makefile (100%) rename configs/{docker => development}/config/config.yaml (100%) rename configs/{docker => development}/docker-compose.yaml (100%) rename configs/{docker => development}/scripts/writer.sh (100%) diff --git a/.gitignore b/.gitignore index 517439d..a84b379 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,7 @@ go.work.sum # idea .idea + +# ignore the log directory + +configs/development/logs/ diff --git a/README.md b/README.md index 39b6675..4fc3a39 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,153 @@ -# vault-audit-filter - A lightweight tool designed to receive audit events from the vault and apply filtering using a rule-based engine +# Vault Audit Filter + +`vault-audit-filter` is a Go-based tool designed to filter and log HashiCorp Vault audit logs based on configurable rules. It provides fine-grained control over how Vault audit events are processed and categorized, allowing you to capture critical events while reducing noise from routine operations. + +## Features + +- **Configurable Rule-Based Filtering**: Define rules to match specific audit events, such as read, write, delete, or specific paths in Vault. +- **Multiple Rule Groups**: Organize rules into groups and log them to separate files. +- **Dynamic Logging**: Log audit events to specified files with log rotation and size limits. +- **Supports Multiple Operations**: Filters common Vault operations, including KV operations, metadata updates, and deletion events. +- **Performance-Oriented**: Built with `gnet` to handle high concurrency. + +## Table of Contents + +- [Getting Started](#getting-started) +- [Installation](#installation) +- [Configuration](#configuration) +- [Usage](#usage) +- [Testing](#testing) +- [Contributing](#contributing) +- [License](#license) + +## Getting Started + +These instructions will help you set up and run `vault-audit-filter` on your local machine. + +### Prerequisites + +- **Go**: Ensure you have Go 1.21 or later installed. You can download it here: +- **Vault**: You should have HashiCorp Vault installed and configured. Instructions can be found here: + +### Installation + +Clone the repository: + + git clone https://github.com/ncode/vault-audit-filter.git + cd vault-audit-filter + +### Build the Project + +To build the binary: + + go build -o vault-audit-filter . + +### Running the Application + +Once you have built the project, you can run the `vault-audit-filter` executable: + + ./vault-audit-filter --config config.yaml + +## Configuration + +`vault-audit-filter` uses a YAML-based configuration file that allows you to define rule groups, specify logging files, and configure Vault settings. + +### Sample Configuration (`config.yaml`) + + vault: + address: "http://127.0.0.1:8200" + token: "your-vault-token" + audit_path: "/vault-audit-filter" + audit_address: "127.0.0.1:1269" + audit_description: "Vault Audit Filter Device" + + rule_groups: + - name: "normal_operations" + rules: + - 'Request.Operation in ["read", "update"] && Request.Path startsWith "secret/data/" && Auth.PolicyResults.Allowed == true' + log_file: + file_path: "/var/log/vault_normal_operations.log" + max_size: 100 # Max size in MB + max_backups: 5 # Max number of backup files + max_age: 30 # Max age in days + compress: true # Compress rotated files + + - name: "critical_events" + rules: + - 'Request.Operation == "delete" && Auth.PolicyResults.Allowed == true' + - 'Request.Path startsWith "secret/metadata/" && Auth.PolicyResults.Allowed == true' + log_file: + file_path: "/var/log/vault_critical_events.log" + max_size: 100 + max_backups: 5 + max_age: 30 + compress: true + +### Configuration Parameters + +- **Vault Settings**: + - `vault.address`: The address of your Vault instance. + - `vault.token`: Vault token for authentication. + - `vault.audit_path`: The path for Vault's audit device. + - `vault.audit_address`: The address for receiving audit logs. + - `vault.audit_description`: Description for the Vault audit device. + +- **Rule Groups**: + - `rule_groups.name`: The name of the rule group. + - `rule_groups.rules`: A list of expressions using `expr` to define rules for audit log filtering. + - `log_file.file_path`: The file path where matching logs will be written. + - `log_file.max_size`: The maximum size of the log file in MB before rotation. + - `log_file.max_backups`: The number of backup logs to keep. + - `log_file.max_age`: The maximum number of days to retain logs. + - `log_file.compress`: Whether to compress the old log files. + +### Rule Syntax + +Rules are written using the `expr` language, a simple and safe expression language for Go. Rules can be based on the following properties of audit logs: + +- `Request.Operation`: The type of operation (`read`, `update`, `delete`, etc.). +- `Request.Path`: The Vault path being accessed. +- `Auth.PolicyResults.Allowed`: Whether the operation was allowed. + +**Example Rule**: + + 'Request.Operation == "update" && Request.Path startsWith "secret/data/" && Auth.PolicyResults.Allowed == true' + +## Usage + +To run `vault-audit-filter` with your configuration file, use: + +```bash +$ ./vault-audit-filter --config config.yaml +``` + +### Command-Line Options + +- `--config`: Specify the path to the configuration file (default is `config.yaml`). +- `--log-level`: Set the logging level (`debug`, `info`, `warn`, `error`). + +### Environment Variables + +You can also define environment variables to override configuration file values. For example: + +```bash +$ export VAULT_ADDRESS="http://127.0.0.1:8200" +$ export VAULT_TOKEN="your-vault-token" +``` + +### Development + +For development purposes, you can use the provided Makefile located at `configs/development/Makefile` to build and run the project using Docker and Docker Compose. This is how I test my changes and have a playground of sorts. + +## Contributing + +We welcome contributions from the community! +Before submitting a pull request, ensure that: + +- The code compiles without errors. +- All tests pass. +- Your changes are well-documented. + +## License + +This project is licensed under the Apache License, Version 2.0. See the `LICENSE` file for details. diff --git a/configs/docker/Dockerfile b/configs/development/Dockerfile similarity index 100% rename from configs/docker/Dockerfile rename to configs/development/Dockerfile diff --git a/configs/docker/Makefile b/configs/development/Makefile similarity index 100% rename from configs/docker/Makefile rename to configs/development/Makefile diff --git a/configs/docker/config/config.yaml b/configs/development/config/config.yaml similarity index 100% rename from configs/docker/config/config.yaml rename to configs/development/config/config.yaml diff --git a/configs/docker/docker-compose.yaml b/configs/development/docker-compose.yaml similarity index 100% rename from configs/docker/docker-compose.yaml rename to configs/development/docker-compose.yaml diff --git a/configs/docker/scripts/writer.sh b/configs/development/scripts/writer.sh similarity index 100% rename from configs/docker/scripts/writer.sh rename to configs/development/scripts/writer.sh From 6bc0dbd1e76db85af34d705ac6fcca9ba8845bba Mon Sep 17 00:00:00 2001 From: Juliano Martinez Date: Mon, 23 Sep 2024 10:15:00 +0200 Subject: [PATCH 2/4] adds build and test workflows --- .github/workflows/ci.yml | 22 +++++++++++ .github/workflows/codeql.yml | 74 ++++++++++++++++++++++++++++++++++++ .github/workflows/go.yml | 28 ++++++++++++++ README.md | 4 ++ 4 files changed, 128 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..56101f3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: Test and coverage + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/setup-go@v3 + with: + go-version: '1.22' + - name: Run coverage + run: go test -coverpkg=./... ./... -race -coverprofile=coverage.out -covermode=atomic + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..6e53432 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,74 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '35 15 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..6092f4a --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,28 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: Go + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.23' + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... diff --git a/README.md b/README.md index 4fc3a39..768fad9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +[![Go Report Card](https://goreportcard.com/badge/github.com/ncode/vault-audit-filter)](https://goreportcard.com/report/github.com/ncode/vault-audit-filter) +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![codecov](https://codecov.io/gh/ncode/vault-audit-filter/graph/badge.svg?token=PTW9OYF19R)](https://codecov.io/gh/ncode/vault-audit-filter) + # Vault Audit Filter `vault-audit-filter` is a Go-based tool designed to filter and log HashiCorp Vault audit logs based on configurable rules. It provides fine-grained control over how Vault audit events are processed and categorized, allowing you to capture critical events while reducing noise from routine operations. From 265f41088a6eb56d7b48e8744e9a23afef073657 Mon Sep 17 00:00:00 2001 From: Juliano Martinez Date: Mon, 23 Sep 2024 10:18:02 +0200 Subject: [PATCH 3/4] update version of actions --- .github/workflows/ci.yml | 8 ++++---- .github/workflows/codeql.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56101f3..b090520 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,16 +6,16 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: - go-version: '1.22' + go-version: '1.23' - name: Run coverage run: go test -coverpkg=./... ./... -race -coverprofile=coverage.out -covermode=atomic - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: verbose: true env: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6e53432..af84bd2 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,11 +38,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -56,7 +56,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -69,6 +69,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" From c92b32477931ac7546959b7cc0b51918c9bbf20b Mon Sep 17 00:00:00 2001 From: Juliano Martinez Date: Mon, 23 Sep 2024 10:22:16 +0200 Subject: [PATCH 4/4] cleanup copied tests from courier --- pkg/vault/vault.go | 18 ------ pkg/vault/vault_test.go | 119 ---------------------------------------- 2 files changed, 137 deletions(-) diff --git a/pkg/vault/vault.go b/pkg/vault/vault.go index 85a76ec..34d7eea 100644 --- a/pkg/vault/vault.go +++ b/pkg/vault/vault.go @@ -42,11 +42,6 @@ type JWTAuth struct { JWT string } -type K8sAuth struct { - Role string - JWT string -} - func (t TokenAuth) Authenticate(client *vault.Client) error { client.SetToken(t.Token) return nil @@ -110,19 +105,6 @@ func (j JWTAuth) ConfigureTLS(*vault.Config) error { return nil } -func (k K8sAuth) Authenticate(client *vault.Client) error { - data := map[string]interface{}{ - "role": k.Role, - "jwt": k.JWT, - } - secret, err := client.Logical().Write("auth/kubernetes/login", data) - if err != nil { - return fmt.Errorf("failed to authenticate with Kubernetes: %w", err) - } - client.SetToken(secret.Auth.ClientToken) - return nil -} - func NewVaultClient(address string, authMethod AuthMethod) (*VaultClient, error) { config := vault.DefaultConfig() config.Address = address diff --git a/pkg/vault/vault_test.go b/pkg/vault/vault_test.go index b1b7ad8..72fca17 100644 --- a/pkg/vault/vault_test.go +++ b/pkg/vault/vault_test.go @@ -99,26 +99,6 @@ func TestNewVaultClient(t *testing.T) { }, wantErr: false, }, - { - name: "K8sAuth_Success", - authMethod: K8sAuth{ - Role: "test-role", - JWT: "test-jwt", - }, - setupMock: func(s *httptest.Server) { - s.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/v1/auth/kubernetes/login", r.URL.Path) - assert.Equal(t, http.MethodPut, r.Method) - var payload map[string]interface{} - json.NewDecoder(r.Body).Decode(&payload) - assert.Equal(t, "test-role", payload["role"]) - assert.Equal(t, "test-jwt", payload["jwt"]) - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"auth": {"client_token": "test-client-token"}}`)) - }) - }, - wantErr: false, - }, { name: "TokenAuth_Failure", authMethod: TokenAuth{Token: "invalid-token"}, @@ -176,20 +156,6 @@ func TestNewVaultClient(t *testing.T) { }, wantErr: true, }, - { - name: "K8sAuth_Failure", - authMethod: K8sAuth{ - Role: "invalid-role", - JWT: "invalid-jwt", - }, - setupMock: func(s *httptest.Server) { - s.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte(`{"errors": ["invalid Kubernetes credentials"]}`)) - }) - }, - wantErr: true, - }, } for _, tt := range tests { @@ -240,85 +206,6 @@ func TestVaultClient_Operations(t *testing.T) { expectedErr bool checkResult func(t *testing.T, result interface{}) }{ - { - name: "ReadSecret_Success", - operation: "Read", - path: "secret/data/test", - setupMock: func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/v1/secret/data/test", r.URL.Path) - assert.Equal(t, http.MethodGet, r.Method) - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"data": {"data": {"foo": "bar"}}}`)) - }, - expectedErr: false, - checkResult: func(t *testing.T, result interface{}) { - data, ok := result.(map[string]interface{}) - assert.True(t, ok) - assert.Equal(t, "bar", data["data"].(map[string]interface{})["foo"]) - }, - }, - { - name: "ReadSecret_NotFound", - operation: "Read", - path: "secret/data/nonexistent", - setupMock: func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/v1/secret/data/nonexistent", r.URL.Path) - assert.Equal(t, http.MethodGet, r.Method) - w.WriteHeader(http.StatusNotFound) - }, - expectedErr: true, - }, - { - name: "WriteSecret_Success", - operation: "Write", - path: "secret/data/test", - input: map[string]interface{}{"foo": "bar"}, - setupMock: func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/v1/secret/data/test", r.URL.Path) - assert.Equal(t, http.MethodPut, r.Method) - var payload map[string]interface{} - json.NewDecoder(r.Body).Decode(&payload) - assert.Equal(t, map[string]interface{}{"foo": "bar"}, payload) - w.WriteHeader(http.StatusNoContent) - }, - expectedErr: false, - }, - { - name: "WriteSecret_Failure", - operation: "Write", - path: "secret/data/test", - input: map[string]interface{}{"foo": "bar"}, - setupMock: func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/v1/secret/data/test", r.URL.Path) - assert.Equal(t, http.MethodPut, r.Method) - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(`{"errors": ["permission denied"]}`)) - }, - expectedErr: true, - }, - { - name: "DeleteSecret_Success", - operation: "Delete", - path: "secret/data/test", - setupMock: func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/v1/secret/data/test", r.URL.Path) - assert.Equal(t, http.MethodDelete, r.Method) - w.WriteHeader(http.StatusNoContent) - }, - expectedErr: false, - }, - { - name: "DeleteSecret_Failure", - operation: "Delete", - path: "secret/data/test", - setupMock: func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "/v1/secret/data/test", r.URL.Path) - assert.Equal(t, http.MethodDelete, r.Method) - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(`{"errors": ["permission denied"]}`)) - }, - expectedErr: true, - }, { name: "EnableAuditDevice_Success", operation: "EnableAudit", @@ -385,12 +272,6 @@ func TestVaultClient_Operations(t *testing.T) { var result interface{} switch tt.operation { - case "Read": - result, err = vaultClient.ReadSecret(tt.path) - case "Write": - err = vaultClient.WriteSecret(tt.path, tt.input) - case "Delete": - err = vaultClient.DeleteSecret(tt.path) case "EnableAudit": err = vaultClient.EnableAuditDevice(tt.path, tt.input["type"].(string), tt.input["description"].(string), tt.input["options"].(map[string]string)) }