SNOW-1571136 Add network tcp-check command (#472)

* SNOW-1571136 Add `network tcp-check` command * SNOW-1571136 Add set up dev env to readme * SNOW-1571136 Add golangci to pre-commit
Snowflake-Labs · Aug 27, 2024 · 4d1444b · 4d1444b
1 parent 85e40d2
commit 4d1444b
Show file tree

Hide file tree

Showing 38 changed files with 2,992 additions and 64 deletions.
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
@@ -20,7 +20,7 @@ jobs:
         go-version-file: 'go.mod'
     - uses: actions/setup-python@v3
     - uses: pre-commit/[email protected]
-  test:
+  unit_test:
     name: Unit tests
     runs-on: ubuntu-20.04
     steps:
@@ -34,6 +34,20 @@ jobs:
           sudo apt-get install ansible
           sudo apt-get install gdb
           sudo apt-get install python3
-      - name: integration tests
+      - name: integration bash tests
         run: ./testing/integrate.sh
         shell: bash
+      - name: integration tests
+        run: go test -tags=integration ./...
+        shell: bash
+  integration_test:
+    name: Unit tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+      - name: integration tests
+        run: INTEGRATION_TEST=yes go test -run "^TestIntegration.*$" ./...
+        shell: bash
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -14,3 +14,9 @@ repos:
     - id: go-fmt
     - id: go-build
     - id: go-mod-tidy
+- repo: https://github.com/golangci/golangci-lint
+  rev: v1.59.1
+  hooks:
+      - id: golangci-lint
+        args: ['--fast']
+        stages: [commit]
diff --git a/README.md b/README.md
@@ -127,6 +127,17 @@ do this for you, as well as re-generating the service proto files.
 $ go generate tools.go
 ```
 
+### Dev Environment setup
+#### Required tools
+- [pre-commit 3.8.0+](https://pre-commit.com/index.html)
+- [golangci-lint 1.59.1+](https://golangci-lint.run/welcome/install/#local-installation)
+
+Configuration:
+- Set up git pre-commit hooks
+```bash
+pre-commit install
+```
+
 ### Creating your own certificates
 
 As an alternative to copying auth/mtls/testdata, you can create your own example mTLS certs. See the
@@ -148,6 +159,32 @@ $ GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$HOME/.sansshell/root.pem grpc_cli \
 
 NOTE: This connects to the proxy. Change to 50042 if you want to connect to the sansshell-server.
 
+### Testing
+To run unit tests, run the following command:
+```bash
+go test ./...
+```
+
+To run integration tests, run the following command:
+```bash
+# Run go integration tests
+INTEGRATION_TEST=yes go test -run "^TestIntegration.*$" ./...
+
+# Run bash integration tests
+./test/integration.sh
+```
+
+#### Integration testing
+To implement integration tests, you need to:
+- Create a new test file name satisfy pattern `<file-name>_integration_test.go`
+- Name test functions satisfy pattern `TestIntegration<FunctionName>`
+- Add check to skip tests when unit test is running:
+```go
+if os.Getenv("INTEGRATION_TEST") == "" {
+    t.Skip("skipping integration test")
+}
+```
+
 ## A tour of the codebase
 
 SansShell is composed of 5 primary concepts:
@@ -170,6 +207,8 @@ implementations of the SansShell Server to easily import services they wish to
 use, and have zero overhead or risk from services they do not import at compile
 time.
 
+[Here](/docs/services-architecture.md) you could read more about services architecture.
+
 #### List of available Services
 
 1. Ansible: Run a local ansible playbook and return output
@@ -180,6 +219,8 @@ time.
 1. Package operations: Install, Upgrade, List, Repolist
 1. Process operations: List, Get stacks (native or Java), Get dumps (core or Java heap)
 1. MPA operations: Multi party authorization for commands
+1. [Network](./services/network):
+   1. [TCP-Check](./services/network/README.md#sanssh-network-tcp-check) - Check if a TCP port is open on a remote host
 1. Service operations: List, Status, Start/stop/restart
 
 TODO: Document service/.../client expectations.

diff --git a/cmd/proxy-server/default-policy.rego b/cmd/proxy-server/default-policy.rego
@@ -62,6 +62,13 @@ allow {
 	startswith(input.method, "/Mpa.Mpa/")
 }
 
+#######################################################
+# Network service related policies
+#
+allow {
+    input.method = "/Network.Network/TCPCheck"
+}
+
 # More complex example: allow stat of any file in /etc/ for
 # hosts in the 10.0.0.0/8 subnet, for callers in the 'admin'
 # group.

diff --git a/cmd/proxy-server/main.go b/cmd/proxy-server/main.go
@@ -51,6 +51,7 @@ import (
 	_ "github.com/Snowflake-Labs/sansshell/services/httpoverrpc"
 	_ "github.com/Snowflake-Labs/sansshell/services/localfile"
 	_ "github.com/Snowflake-Labs/sansshell/services/mpa"
+	_ "github.com/Snowflake-Labs/sansshell/services/network"
 	_ "github.com/Snowflake-Labs/sansshell/services/packages"
 	_ "github.com/Snowflake-Labs/sansshell/services/process"
 	_ "github.com/Snowflake-Labs/sansshell/services/sansshell"

diff --git a/cmd/sanssh/client/client.go b/cmd/sanssh/client/client.go
@@ -20,7 +20,6 @@
 package client
 
 import (
-	"bytes"
 	"context"
 	"flag"
 	"fmt"
@@ -39,6 +38,7 @@ import (
 	cmdUtil "github.com/Snowflake-Labs/sansshell/cmd/util"
 	"github.com/Snowflake-Labs/sansshell/services/mpa/mpahooks"
 	"github.com/Snowflake-Labs/sansshell/services/util"
+	writerUtils "github.com/Snowflake-Labs/sansshell/services/util/writer"
 )
 
 func init() {
@@ -83,52 +83,6 @@ const (
 	defaultOutput = "-"
 )
 
-type prefixWriter struct {
-	prefix []byte
-	start  bool
-	dest   io.Writer
-}
-
-func (p *prefixWriter) Write(b []byte) (n int, err error) {
-	// Exit early if we're not writing any bytes
-	if len(b) == 0 {
-		return 0, nil
-	}
-
-	// Keep track of the size of the incoming buf as we'll print more
-	// but clients want to know we wrote what they asked for.
-	tot := len(b)
-
-	// If we just started emit a prefix
-	if p.start {
-		n, err = p.dest.Write(p.prefix)
-		if err != nil {
-			return n, err
-		}
-		p.start = false
-	}
-
-	// Find any newlines and augment them with the prefix appended onto them.
-
-	// If the last byte is a newline we don't want to add a prefix yet as this may be the end of output.
-	// Instead we'll remark start so the next one prints and remove the newline for now.
-	if n := bytes.LastIndex(b, []byte{'\n'}); n == len(b)-1 {
-		p.start = true
-		b = b[:len(b)-1]
-	}
-	b = bytes.ReplaceAll(b, []byte{'\n'}, append(append([]byte{}, byte('\n')), p.prefix...))
-	// If start got set above we need to add back the newline we dropped so output looks correct.
-	// Thankfully b is now a new slice as Write() isn't supposed to directly modify the incoming one.
-	if p.start {
-		b = append(b, '\n')
-	}
-	n, err = p.dest.Write(b)
-	if err != nil {
-		return n, err
-	}
-	return tot, nil
-}
-
 // UnaryClientTimeoutInterceptor returns a grpc.UnaryClientInterceptor that adds a deadline to every request
 func UnaryClientTimeoutInterceptor(timeout time.Duration) grpc.UnaryClientInterceptor {
 	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
@@ -342,11 +296,11 @@ func Run(ctx context.Context, rs RunState) {
 	makeWriter := func(prefix bool, i int, dest io.Writer) io.Writer {
 		if prefix {
 			targetName := cmdUtil.StripTimeout(rs.Targets[i])
-			dest = &prefixWriter{
-				start:  true,
-				dest:   dest,
-				prefix: []byte(fmt.Sprintf("%d-%s: ", i, targetName)),
-			}
+			dest = writerUtils.GetPrefixedWriter(
+				[]byte(fmt.Sprintf("%d-%s: ", i, targetName)),
+				true,
+				dest,
+			)
 		}
 		return dest
 	}

diff --git a/cmd/sanssh/main.go b/cmd/sanssh/main.go
@@ -48,6 +48,7 @@ import (
 	_ "github.com/Snowflake-Labs/sansshell/services/httpoverrpc/client"
 	_ "github.com/Snowflake-Labs/sansshell/services/localfile/client"
 	_ "github.com/Snowflake-Labs/sansshell/services/mpa/client"
+	_ "github.com/Snowflake-Labs/sansshell/services/network/client"
 	_ "github.com/Snowflake-Labs/sansshell/services/packages/client"
 	_ "github.com/Snowflake-Labs/sansshell/services/power/client"
 	_ "github.com/Snowflake-Labs/sansshell/services/process/client"

diff --git a/cmd/sansshell-server/default-policy.rego b/cmd/sansshell-server/default-policy.rego
@@ -106,3 +106,7 @@ allow {
 	not input.metadata["proxied-sansshell-identity"]
 	input.method = ["/Mpa.Mpa/Store", "/Mpa.Mpa/Approve"][_]
 }
+
+allow {
+    input.method = "/Network.Network/TCPCheck"
+}
diff --git a/cmd/sansshell-server/main.go b/cmd/sansshell-server/main.go
@@ -65,6 +65,7 @@ import (
 	_ "github.com/Snowflake-Labs/sansshell/services/healthcheck/server"
 	_ "github.com/Snowflake-Labs/sansshell/services/localfile/server"
 	mpa "github.com/Snowflake-Labs/sansshell/services/mpa/server"
+	_ "github.com/Snowflake-Labs/sansshell/services/network/server"
 	_ "github.com/Snowflake-Labs/sansshell/services/power/server"
 
 	// Packages needs a real import to bind flags.

diff --git a/docs/services-architecture.md b/docs/services-architecture.md
@@ -0,0 +1,23 @@
+## Services architecture
+SansShell is built on a principle of "Don't pay for what you don't use". This
+is advantageous in both minimizing the resources of SansShell server (binary
+size, memory footprint, etc) as well as reducing the security risk of running
+it. To accomplish that, all of the SansShell services are independent modules,
+which can be optionally included at build time.
+
+It is divided into 2 parts:
+- `client`, part of cli app running on local
+- `server`, part of server app running on remote machine
+
+Each part follows hexagonal architecture. It divides into following layers:
+- `application`, contains the application logic
+- `infrastructure`, contains the implementation of the ports and adapters
+    - `input`, contains user interface/api adapters implementation, such as GRPC controllers, CLI command handlers and etc
+    - `output`, contains adapters to external systems implementation, such as HTTP/GRPC client, repositories and etc
+
+Other:
+- `./<service-name>.go` contains the service related commands
+- `./<service-name>.proto` protobuf definition of client-server communication
+- `./<service-name>.pb.go` GoLang definition of `./<service-name>.proto` file. **Auto-generated**. Do not edit manually
+- `./<service-name>_grpc.pb.go` Pure definition of GRPC client and server. **Auto-generated**. Do not edit manually
+- `./<service-name>_grpcproxy.pb.go` Sansshell extension of pure GRPC client and server, go [here](../proxy/README.md) to know more. **Auto-generated**. Do not edit manually
diff --git a/go.mod b/go.mod
@@ -25,6 +25,7 @@ require (
 	gocloud.dev v0.32.0
 	golang.org/x/sync v0.8.0
 	golang.org/x/sys v0.24.0
+	golang.org/x/term v0.22.0
 	google.golang.org/grpc v1.65.0
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1
 	google.golang.org/protobuf v1.34.2
@@ -105,7 +106,6 @@ require (
 	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/term v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect

diff --git a/services/network/README.md b/services/network/README.md
@@ -0,0 +1,21 @@
+# Network
+This directory contains the network services. This defines network related commands like `ping`, `traceroute`, `netstat`, etc.
+
+What is not part of network service? Commands related to protocols build on top of network layer like `http`, `ftp`, `ssh`, etc.
+
+## Usage
+
+### sanssh network tcp-check
+Check if a TCP port is open on a remote host.
+
+```bash
+sanssh <sanssh-args> network tcp-check <host>:<port> [--timeout <timeout>]
+```
+Where:
+- `<sanssh-args>` common sanssh arguments
+- `<host>` is the host to check
+- `<port>` is the port to check
+- `<timeout>` timeout in seconds to wait for the connection to be established. Default is 3 seconds.
+
+# Useful docs
+- [service-architecture](../../docs/services-architecture.md)