From 2988d4e601db5a897a7957836833649122381b2a Mon Sep 17 00:00:00 2001 From: niqdev Date: Sat, 23 Sep 2023 10:25:50 +0100 Subject: [PATCH] fix task logs --- README.md | 38 +++++------- docs/task-htb-example.txt | 106 +++++++++++++++++++++++++++++++++ internal/command/task/task.go | 6 +- pkg/client/docker/client.go | 19 +++--- pkg/client/docker/options.go | 2 +- pkg/task/docker/docker.go | 41 +++++-------- pkg/task/docker/docker_test.go | 12 ++++ pkg/task/docker/events.go | 6 +- 8 files changed, 169 insertions(+), 61 deletions(-) create mode 100644 docs/task-htb-example.txt create mode 100644 pkg/task/docker/docker_test.go diff --git a/README.md b/README.md index 41cbf184..d39a4dd5 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,8 @@ hckctl task nuclei --network-vpn htb --input target=10.10.10.3 # TODO ffuf ``` +See [output](./docs/task-htb-example.txt) example + ### Flow (cloud preview) Launch multiple tasks in parallel, collect and combine the results @@ -132,27 +134,11 @@ hckctl config --reset ## Roadmap -> TODO create issues and add links - -### Machine - -> create and access AWS EC2, Azure Virtual Machines, DigitalOcean Droplet, QEMU etc. - -### Man - -> combine tldr and cheat - -### Plugin - -> add custom commands - -### TUI - -> similar to lazydocker and k9s - -### Prompt - -> chatgpt prompt style +* Machine: instead of just containers, create and access VMs e.g. DigitalOcean Droplet, AWS EC2, Azure Virtual Machines, QEMU etc. +* Man: combine tldr and cheat +* Plugin: add your custom commands +* TUI: similar to lazydocker and k9s +* Prompt: chatgpt prompt style ## Setup @@ -183,6 +169,12 @@ just > TODO example of how to point to a specific pr/revision in a forked repo diff --git a/docs/task-htb-example.txt b/docs/task-htb-example.txt new file mode 100644 index 00000000..af023468 --- /dev/null +++ b/docs/task-htb-example.txt @@ -0,0 +1,106 @@ +############################## nmap ############################## + +hckctl task nmap --network-vpn htb --command full --input address=10.10.10.3 +Starting Nmap 7.94 ( https://nmap.org ) at 2023-09-23 09:11 UTC +Nmap scan report for 10.10.10.3 +Host is up (0.11s latency). +Not shown: 996 filtered tcp ports (no-response) +PORT STATE SERVICE VERSION +21/tcp open ftp vsftpd 2.3.4 +|_ftp-anon: Anonymous FTP login allowed (FTP code 230) +| ftp-syst: +| STAT: +| FTP server status: +| Connected to 10.10.14.2 +| Logged in as ftp +| TYPE: ASCII +| No session bandwidth limit +| Session timeout in seconds is 300 +| Control connection is plain text +| Data connections will be plain text +| vsFTPd 2.3.4 - secure, fast, stable +|_End of status +22/tcp open ssh OpenSSH 4.7p1 Debian 8ubuntu1 (protocol 2.0) +| ssh-hostkey: +| 1024 60:0f:cf:e1:c0:5f:6a:74:d6:90:24:fa:c4:d5:6c:cd (DSA) +|_ 2048 56:56:24:0f:21:1d:de:a7:2b:ae:61:b1:24:3d:e8:f3 (RSA) +139/tcp open netbios-ssn Samba smbd 3.X - 4.X (workgroup: WORKGROUP) +445/tcp open netbios-ssn Samba smbd 3.0.20-Debian (workgroup: WORKGROUP) +Service Info: OSs: Unix, Linux; CPE: cpe:/o:linux:linux_kernel + +Host script results: +|_clock-skew: mean: 2h00m55s, deviation: 2h49m45s, median: 52s +| smb-os-discovery: +| OS: Unix (Samba 3.0.20-Debian) +| Computer name: lame +| NetBIOS computer name: +| Domain name: hackthebox.gr +| FQDN: lame.hackthebox.gr +|_ System time: 2023-09-23T05:12:20-04:00 +| smb-security-mode: +| account_used: guest +| authentication_level: user +| challenge_response: supported +|_ message_signing: disabled (dangerous, but default) +|_smb2-time: Protocol negotiation failed (SMB2) + +Service detection performed. Please report any incorrect results at https://nmap.org/submit/ . +Nmap done: 1 IP address (1 host up) scanned in 62.99 seconds + +############################## rustscan ############################## + +hckctl task rustscan --network-vpn htb --inline -- -a 10.10.10.3 --ulimit 5000 +.----. .-. .-. .----..---. .----. .---. .--. .-. .-. +| {} }| { } |{ {__ {_ _}{ {__ / ___} / {} \ | `| | +| .-. \| {_} |.-._} } | | .-._} }\ }/ /\ \| |\ | +`-' `-'`-----'`----' `-' `----' `---' `-' `-'`-' `-' +The Modern Day Port Scanner. +________________________________________ +: http://discord.skerritt.blog : +: https://github.com/RustScan/RustScan : + -------------------------------------- +Nmap? More like slowmap.🐢 + +[~] The config file is expected to be at "/home/rustscan/.rustscan.toml" +[~] Automatically increasing ulimit value to 5000. +Open 10.10.10.3:21 +Open 10.10.10.3:22 +Open 10.10.10.3:139 +Open 10.10.10.3:445 +[~] Starting Script(s) +[~] Starting Nmap 7.80 ( https://nmap.org ) at 2023-09-23 09:12 UTC +Initiating Ping Scan at 09:12 +Scanning 10.10.10.3 [2 ports] +Completed Ping Scan at 09:12, 3.00s elapsed (1 total hosts) +Nmap scan report for 10.10.10.3 [host down, received no-response] +Read data files from: /usr/bin/../share/nmap +Note: Host seems down. If it is really up, but blocking our ping probes, try -Pn +Nmap done: 1 IP address (0 hosts up) scanned in 3.05 seconds + +############################## nuclei ############################## + +hckctl task nuclei --network-vpn htb --input target=10.10.10.3 + + __ _ + ____ __ _______/ /__ (_) + / __ \/ / / / ___/ / _ \/ / + / / / / /_/ / /__/ / __/ / +/_/ /_/\__,_/\___/_/\___/_/ v2.9.15 + + projectdiscovery.io + +[INF] nuclei-templates are not installed, installing... +[INF] Successfully installed nuclei-templates at /root/nuclei-templates +[INF] Current nuclei version: v2.9.15 (latest) +[INF] Current nuclei-templates version: v9.6.4 (latest) +[INF] New templates added in latest release: 121 +[INF] Templates loaded for current scan: 6892 +[INF] Targets loaded for current scan: 1 +[INF] Running httpx on input host +[INF] Found 0 URL from httpx +[INF] Templates clustered: 1194 (Reduced 1133 Requests) +[vsftpd-backdoor] [tcp] [critical] 10.10.10.3:21 +[ftp-anonymous-login] [tcp] [medium] 10.10.10.3:21 +[openssh-detect] [tcp] [info] 10.10.10.3:22 [SSH-2.0-OpenSSH_4.7p1 Debian-8ubuntu1] +[samba-detect] [tcp] [info] 10.10.10.3:139 +ubuntu@dell in ~/Projects/hckops/hckctl on (main ▲) $ diff --git a/internal/command/task/task.go b/internal/command/task/task.go index da6548bc..614fcb4b 100644 --- a/internal/command/task/task.go +++ b/internal/command/task/task.go @@ -164,7 +164,11 @@ func (opts *taskCmdOptions) runTask(sourceLoader template.SourceLoader[taskModel StreamOpts: commonModel.NewStdStreamOpts(false), } - return taskClient.Run(runOpts) + if err := taskClient.Run(runOpts); err != nil { + log.Warn().Err(err).Msg("error run task") + return errors.New("error run task") + } + return nil } func newDefaultTaskClient(provider taskModel.TaskProvider, configRef *config.ConfigRef, loader *commonCmd.Loader) (task.TaskClient, error) { diff --git a/pkg/client/docker/client.go b/pkg/client/docker/client.go index 13691229..27f0df05 100644 --- a/pkg/client/docker/client.go +++ b/pkg/client/docker/client.go @@ -388,28 +388,25 @@ func (client *DockerClient) NetworkUpsert(networkName string) (string, error) { } func (client *DockerClient) CopyFileToContainer(containerId string, localPath string, containerPath string) error { - // TODO https://github.com/moby/moby/issues/38035 - // https://github.com/moby/moby/issues/26652 - // https://github.com/docker/cli/blob/b1d27091e50595fecd8a2a4429557b70681395b2/cli/command/container/cp.go#L182-L282 - // TODO https://github.com/docker/cli/blob/master/cli/command/container/cp.go#L182 + // see https://github.com/docker/cli/blob/b1d27091e50595fecd8a2a4429557b70681395b2/cli/command/container/cp.go#L182-L282 // Get an absolute source path. srcPath, err := resolveLocalPath(localPath) if err != nil { - return err + return errors.Wrap(err, "error copy file to container: resolve local path") } // Prepare destination copy info by stat-ing the container path. dstInfo := archive.CopyInfo{Path: containerPath} dstStat, err := client.docker.ContainerStatPath(client.ctx, containerId, containerPath) if err != nil { - // TODO error is ignore - //return err + // Ignore any error and assume that the parent directory of the destination + // path exists, in which case the copy may still succeed. } // Validate the destination path. if err := validateOutputPathFileMode(dstStat.Mode); err != nil { - return errors.Wrapf(err, `destination "%s:%s" must be a directory or a regular file`, containerId, containerPath) + return errors.Wrapf(err, `error copy file to container: destination "%s:%s" must be a directory or a regular file`, containerId, containerPath) } // ??? @@ -418,18 +415,18 @@ func (client *DockerClient) CopyFileToContainer(containerId string, localPath st // Prepare source copy info. srcInfo, err := archive.CopyInfoSourcePath(srcPath, false) if err != nil { - return err + return errors.Wrap(err, "error copy file to container: source copy info") } srcArchive, err := archive.TarResource(srcInfo) if err != nil { - return err + return errors.Wrap(err, "error copy file to container: tar archive") } defer srcArchive.Close() dstDir, preparedArchive, err := archive.PrepareArchiveCopy(srcArchive, srcInfo, dstInfo) if err != nil { - return err + return errors.Wrap(err, "error copy file to container: prepare archive") } defer preparedArchive.Close() diff --git a/pkg/client/docker/options.go b/pkg/client/docker/options.go index aac02864..5eb1da5f 100644 --- a/pkg/client/docker/options.go +++ b/pkg/client/docker/options.go @@ -42,7 +42,7 @@ type ContainerCreateOpts struct { WaitStatus bool OnContainerCreateCallback func(containerId string) error OnContainerWaitCallback func(containerId string) error - OnContainerStatusCallback func(string) + OnContainerStatusCallback func(status string) OnContainerStartCallback func() } diff --git a/pkg/task/docker/docker.go b/pkg/task/docker/docker.go index f956df22..f262c658 100644 --- a/pkg/task/docker/docker.go +++ b/pkg/task/docker/docker.go @@ -2,9 +2,10 @@ package docker import ( "fmt" - "github.com/pkg/errors" "strings" + "github.com/pkg/errors" + "github.com/hckops/hckctl/pkg/client/docker" commonModel "github.com/hckops/hckctl/pkg/common/model" taskModel "github.com/hckops/hckctl/pkg/task/model" @@ -44,7 +45,7 @@ func (task *DockerTaskClient) runTask(opts *taskModel.RunOptions) error { return err } else { networkMode = docker.ContainerNetworkMode(sidecarContainerId) - // remove on exit + // remove sidecar on exit defer task.client.ContainerRemove(sidecarContainerId) } } else { @@ -94,21 +95,16 @@ func (task *DockerTaskClient) runTask(opts *taskModel.RunOptions) error { WaitStatus: true, // block OnContainerCreateCallback: func(string) error { return nil }, OnContainerWaitCallback: func(containerId string) error { - task.eventBus.Publish(newContainerStartDockerLoaderEvent()) - - // TODO error event - // tail logs before start waiting - if err := task.client.ContainerLogsStd(containerId); err != nil { - return err - } - return nil + // stop loader + task.eventBus.Publish(newContainerWaitDockerLoaderEvent()) + + // tail logs before blocking + return task.client.ContainerLogsStd(containerId) }, OnContainerStatusCallback: func(status string) { task.eventBus.Publish(newContainerCreateStatusDockerEvent(status)) }, - OnContainerStartCallback: func() { - task.eventBus.Publish(newContainerStartDockerLoaderEvent()) - }, + OnContainerStartCallback: func() {}, } // taskId containerId, err := task.client.ContainerCreate(containerOpts) @@ -117,10 +113,8 @@ func (task *DockerTaskClient) runTask(opts *taskModel.RunOptions) error { } task.eventBus.Publish(newContainerCreateDockerEvent(opts.Template.Name, containerName, containerId)) - if err := task.client.ContainerRemove(containerId); err != nil { - return err - } - return nil + // remove temporary container + return task.client.ContainerRemove(containerId) } func (task *DockerTaskClient) pullImage(imageName string, pullEvent *dockerTaskEvent) error { @@ -159,12 +153,12 @@ func (task *DockerTaskClient) pullImage(imageName string, pullEvent *dockerTaskE } func buildVpnSidecarName(taskName string) string { - return fmt.Sprintf("sidecar-vpn-%s", strings.Split(taskName, "-")[2]) + tokens := strings.Split(taskName, "-") + return fmt.Sprintf("sidecar-vpn-%s", tokens[len(tokens)-1]) } func (task *DockerTaskClient) startVpnSidecar(containerName string, vpnInfo *commonModel.VpnNetworkInfo) (string, error) { - // TODO context timeout imageName := "hckops/alpine-openvpn:latest" // base directory must exist vpnConfigPath := "/usr/share/client.ovpn" @@ -179,11 +173,8 @@ func (task *DockerTaskClient) startVpnSidecar(containerName string, vpnInfo *com HostConfig: docker.BuildVpnHostConfig(), WaitStatus: false, OnContainerCreateCallback: func(containerId string) error { - // TODO error event - if err := task.client.CopyFileToContainer(containerId, vpnInfo.LocalPath, vpnConfigPath); err != nil { - return err - } - return nil + // upload openvpn config file + return task.client.CopyFileToContainer(containerId, vpnInfo.LocalPath, vpnConfigPath) }, OnContainerStatusCallback: func(status string) { task.eventBus.Publish(newContainerCreateStatusDockerEvent(status)) @@ -195,7 +186,7 @@ func (task *DockerTaskClient) startVpnSidecar(containerName string, vpnInfo *com if err != nil { return "", err } - //task.eventBus.Publish(newContainerCreateDockerEvent(opts.Template.Name, containerName, containerId)) + task.eventBus.Publish(newContainerCreateDockerEvent("sidecar-vpn", containerName, containerId)) return containerId, nil } diff --git a/pkg/task/docker/docker_test.go b/pkg/task/docker/docker_test.go new file mode 100644 index 00000000..e7a1adfc --- /dev/null +++ b/pkg/task/docker/docker_test.go @@ -0,0 +1,12 @@ +package docker + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildVpnSidecarName(t *testing.T) { + expected := "sidecar-vpn-12345" + assert.Equal(t, expected, buildVpnSidecarName("aaa-bbb-ccc-ddd-12345")) +} diff --git a/pkg/task/docker/events.go b/pkg/task/docker/events.go index 2afb8eeb..2f64ed5b 100644 --- a/pkg/task/docker/events.go +++ b/pkg/task/docker/events.go @@ -68,10 +68,14 @@ func newContainerCreateDockerLoaderEvent() *dockerTaskEvent { return &dockerTaskEvent{kind: event.LoaderUpdate, value: "running"} } -func newContainerStartDockerLoaderEvent() *dockerTaskEvent { +func newContainerWaitDockerLoaderEvent() *dockerTaskEvent { return &dockerTaskEvent{kind: event.LoaderStop, value: "waiting"} } func newVpnConnectDockerLoaderEvent(vpnName string) *dockerTaskEvent { return &dockerTaskEvent{kind: event.LoaderUpdate, value: fmt.Sprintf("connecting to %s", vpnName)} } + +func newContainerErrorDockerEvent(containerId string, err error) *dockerTaskEvent { + return &dockerTaskEvent{kind: event.LogError, value: fmt.Sprintf("container exec error: containerId=%s error=%v", containerId, err)} +}