diff --git a/README.md b/README.md index ab119f1f..98993b72 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ OUTPUT: -o, -output string file to write output to -sr, -store-response store http requests/responses -srd, -store-response-dir string store http requests/responses to custom directory + -sfd, -store-field-dir string store per-host field to custom directory -or, -omit-raw omit raw requests/responses from jsonl output -ob, -omit-body omit response body from jsonl output -j, -jsonl write output in jsonl format @@ -683,7 +684,7 @@ katana -u https://tesla.com -f email,phone *`-store-field`* --- -To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url. +To compliment `field` option which is useful to filter output at run time, there is `-sf, -store-fields` option which works exactly like field option except instead of filtering, it stores all the information on the disk under `katana_field` directory sorted by target url. Use `-sfd` or `-store-field-dir` to store data in a different location. ``` katana -u https://tesla.com -sf key,fqdn,qurl -silent diff --git a/cmd/katana/main.go b/cmd/katana/main.go index a7336296..1be5933d 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -172,6 +172,8 @@ pipelines offering both headless and non-headless crawling.`) flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"), + flagSet.BoolVarP(&options.NoClobber, "no-clobber", "ncb", false, "do not overwrite output file"), + flagSet.StringVarP(&options.StoreFieldDir, "store-field-dir", "sfd", "", "store per-host field to custom directory"), flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"), flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"), flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"), diff --git a/go.mod b/go.mod index 3da6d1b6..950ecba5 100644 --- a/go.mod +++ b/go.mod @@ -11,22 +11,22 @@ require ( github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/dsl v0.0.57 - github.com/projectdiscovery/fastdialer v0.0.72 - github.com/projectdiscovery/goflags v0.1.52 + github.com/projectdiscovery/dsl v0.1.1 + github.com/projectdiscovery/fastdialer v0.1.5 + github.com/projectdiscovery/goflags v0.1.57 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.42 + github.com/projectdiscovery/hmap v0.0.47 github.com/projectdiscovery/mapcidr v1.1.34 - github.com/projectdiscovery/ratelimit v0.0.41 - github.com/projectdiscovery/retryablehttp-go v1.0.60 - github.com/projectdiscovery/useragent v0.0.52 - github.com/projectdiscovery/utils v0.0.94 - github.com/projectdiscovery/wappalyzergo v0.1.1 + github.com/projectdiscovery/ratelimit v0.0.45 + github.com/projectdiscovery/retryablehttp-go v1.0.65 + github.com/projectdiscovery/useragent v0.0.57 + github.com/projectdiscovery/utils v0.1.5 + github.com/projectdiscovery/wappalyzergo v0.1.8 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.25.0 + golang.org/x/net v0.26.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -39,6 +39,7 @@ require ( github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/brotli v1.0.6 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -46,10 +47,12 @@ require ( github.com/dlclark/regexp2 v1.8.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/gaissmai/bart v0.9.5 // indirect github.com/gaukas/godicttls v0.0.4 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hdm/jarm-go v0.0.7 // indirect @@ -94,7 +97,9 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/term v0.20.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect ) @@ -120,8 +125,8 @@ require ( github.com/nwaples/rardecode v1.1.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/projectdiscovery/networkpolicy v0.0.8 - github.com/projectdiscovery/retryabledns v1.0.59 // indirect + github.com/projectdiscovery/networkpolicy v0.0.9 + github.com/projectdiscovery/retryabledns v1.0.64 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -129,7 +134,6 @@ require ( github.com/ulikunitz/xz v0.5.11 // indirect github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect - github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect @@ -137,12 +141,12 @@ require ( github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/crypto v0.23.0 // indirect + golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index bd6cb2dd..2b22615f 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= -github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= @@ -66,6 +66,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= +github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -92,8 +94,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= @@ -103,6 +106,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= @@ -206,36 +211,36 @@ github.com/projectdiscovery/asnmap v1.1.0 h1:ynvbLB5cNpyQ2+k9IP0Rpla+0JmCJpd3mw6 github.com/projectdiscovery/asnmap v1.1.0/go.mod h1:QNjBnGLxUBEZAgaYk/Av5cjKKWFY3i/FOfoIWCUApoY= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI= -github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k= -github.com/projectdiscovery/fastdialer v0.0.72 h1:CbKNFqvJotGmn9uBeHoR2vJQRg8QMuQs9NIOc8HW02E= -github.com/projectdiscovery/fastdialer v0.0.72/go.mod h1:sfeBKELnLnkpwEYcK5Qf8DRXLcdmR34u4TxtFwxNNQ0= -github.com/projectdiscovery/goflags v0.1.52 h1:rVYZOtq7iA8e6ceyVZbp6OcuMhcwh5weiXSuDoXsivU= -github.com/projectdiscovery/goflags v0.1.52/go.mod h1:tcBQ0EVGP4Wafza7gx57ZktkGxyfdLn+eQWUUQrV84c= +github.com/projectdiscovery/dsl v0.1.1 h1:67aqIMMui7dbAXOS3W75DJ3hinGxex48ZBTESJk822M= +github.com/projectdiscovery/dsl v0.1.1/go.mod h1:Jqeu8Q4pqPbmpRwumgF59jOjilw9qYCdt3BYQTs3GtY= +github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= +github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= +github.com/projectdiscovery/goflags v0.1.57 h1:mIL9rGo3Cykdv2AU0LlxoWD18bQ1Iyg4qRnLbPztigY= +github.com/projectdiscovery/goflags v0.1.57/go.mod h1:/5JI3q/mVHqMMS1MYj0CPkTAwDoAKn9R5syf7Mhw9vk= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.42 h1:+P8CC7gAeTG0phe0d1FB7i3Vl15v1K+dJApwX4rvMAM= -github.com/projectdiscovery/hmap v0.0.42/go.mod h1:lbGBuL/bLoYWdlgphZmHXjZCYzteVDf4WfKsR/aH57c= +github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk= +github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= -github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= -github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.41 h1:hlGYBspQL1ikUFnRFd4NDe1gZQaEoTm2HFxguA8a+6E= -github.com/projectdiscovery/ratelimit v0.0.41/go.mod h1:3764AOAYQZL3mQyJv3GTrIwPye7s3XZGxQHu3Z71YOc= -github.com/projectdiscovery/retryabledns v1.0.59 h1:8pMN+VibEBp29RIUior9LXUbx0RsBTjPC0008t2hfGU= -github.com/projectdiscovery/retryabledns v1.0.59/go.mod h1:CwyQLDt9oqNIO/2ArALhAnUHJjZYdvJRSfGERRNPtoQ= -github.com/projectdiscovery/retryablehttp-go v1.0.60 h1:sXbx6Rdh22SZ3AFhY3P7LC+p8GPLlANMgPHlkBXJlv8= -github.com/projectdiscovery/retryablehttp-go v1.0.60/go.mod h1:rgRdV7LSrrTTlvN7yKsYxtvWm39VZB6pgD2t1p1ma64= -github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU= -github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ= -github.com/projectdiscovery/utils v0.0.94 h1:2zzFEjMkq/Ei/o3NIA2SWTkhfGHMkBy0T3aIzq0vizo= -github.com/projectdiscovery/utils v0.0.94/go.mod h1:wxPi+kCsLm5JCLMkZJyGwS+4Mn4PaPHHf0ayE8JphOw= -github.com/projectdiscovery/wappalyzergo v0.1.1 h1:HDxuqawatylDiOlfJf4IsabS0wA/Iyvqm7Dn18TVGjU= -github.com/projectdiscovery/wappalyzergo v0.1.1/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM= +github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= +github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= +github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= +github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= +github.com/projectdiscovery/retryabledns v1.0.64 h1:bhaKarpUnPjTXN9A0ApA/IA4SWfugdCke1yT0YxIj0k= +github.com/projectdiscovery/retryabledns v1.0.64/go.mod h1:n1N7LMgxMNQT/bUoE5/OQZBTxVApkfeAvDtYQ1OnuAg= +github.com/projectdiscovery/retryablehttp-go v1.0.65 h1:GCZI9CUbYkldy9Iup4C7w6aVTxZuAHpSZTVy92qHPo4= +github.com/projectdiscovery/retryablehttp-go v1.0.65/go.mod h1:qF0UuglJiYJcezEoMIyYVYhHGwaOBy8bh3gRGz7CBs8= +github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= +github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= +github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0= +github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4= +github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= +github.com/projectdiscovery/wappalyzergo v0.1.8/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= @@ -376,15 +381,15 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -403,8 +408,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -414,8 +419,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -444,16 +449,16 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -463,15 +468,17 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/runner/options.go b/internal/runner/options.go index 5bb847fe..61805e16 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,6 +25,13 @@ func validateOptions(options *types.Options) error { return errorutil.New("no inputs specified for crawler") } + // Disabling automatic form fill (-aff) for headless navigation due to incorrect implementation. + // Form filling should be handled via headless actions within the page context + if options.Headless && options.AutomaticFormFill { + options.AutomaticFormFill = false + gologger.Info().Msgf("Automatic form fill (-aff) has been disabled for headless navigation.") + } + if (options.HeadlessOptionalArguments != nil || options.HeadlessNoSandbox || options.SystemChromePath != "") && !options.Headless { return errorutil.New("headless mode (-hl) is required if -ho, -nos or -scp are set") } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 7aacf1f8..9fcd09fe 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -55,7 +55,7 @@ func New(options *types.Options) (*Runner, error) { } options.URLs = mapsutil.GetKeys(runnerState.InFlightUrls.GetAll()) } - + options.ConfigureOutput() showBanner() if options.Version { diff --git a/pkg/engine/common/base.go b/pkg/engine/common/base.go index 7eec2a49..55576a0c 100644 --- a/pkg/engine/common/base.go +++ b/pkg/engine/common/base.go @@ -144,7 +144,7 @@ func (s *Shared) NewCrawlSessionWithURL(URL string) (*CrawlSession, error) { cancel() return nil, err } - queue.Push(&navigation.Request{Method: http.MethodGet, URL: URL, Depth: 0}, 0) + queue.Push(&navigation.Request{Method: http.MethodGet, URL: URL, Depth: 0, SkipValidation: true}, 0) if s.KnownFiles != nil { navigationRequests, err := s.KnownFiles.Request(URL) @@ -204,7 +204,12 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error { continue } - if ok, err := s.Options.ValidateScope(req.URL, crawlSession.Hostname); err != nil || !ok { + inScope, scopeErr := s.Options.ValidateScope(req.URL, crawlSession.Hostname) + if scopeErr != nil { + gologger.Debug().Msgf("Error validating scope for `%v`: %v. skipping", req.URL, scopeErr) + continue + } + if !req.SkipValidation && !inScope { gologger.Debug().Msgf("`%v` not in scope. skipping", req.URL) continue } @@ -223,7 +228,9 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error { resp, err := doRequest(crawlSession, req) - s.Output(req, resp, err) + if inScope { + s.Output(req, resp, err) + } if err != nil { gologger.Warning().Msgf("Could not request seed URL %s: %s\n", req.URL, err) diff --git a/pkg/navigation/request.go b/pkg/navigation/request.go index f427d0d0..dcd9c96e 100644 --- a/pkg/navigation/request.go +++ b/pkg/navigation/request.go @@ -10,17 +10,18 @@ type Depth struct{} // Request is a navigation request for the crawler type Request struct { - Method string `json:"method,omitempty"` - URL string `json:"endpoint,omitempty"` - Body string `json:"body,omitempty"` - Depth int `json:"-"` - Headers map[string]string `json:"headers,omitempty"` - Tag string `json:"tag,omitempty"` - Attribute string `json:"attribute,omitempty"` - RootHostname string `json:"-"` - Source string `json:"source,omitempty"` - CustomFields map[string][]string `json:"-"` - Raw string `json:"raw,omitempty"` + Method string `json:"method,omitempty"` + URL string `json:"endpoint,omitempty"` + Body string `json:"body,omitempty"` + Depth int `json:"-"` + SkipValidation bool `json:"-"` + Headers map[string]string `json:"headers,omitempty"` + Tag string `json:"tag,omitempty"` + Attribute string `json:"attribute,omitempty"` + RootHostname string `json:"-"` + Source string `json:"source,omitempty"` + CustomFields map[string][]string `json:"-"` + Raw string `json:"raw,omitempty"` } // RequestURL returns the request URL for the navigation diff --git a/pkg/navigation/response.go b/pkg/navigation/response.go index cb6f0ef2..666b4fc7 100644 --- a/pkg/navigation/response.go +++ b/pkg/navigation/response.go @@ -20,7 +20,7 @@ type Form struct { func (h *Headers) MarshalJSON() ([]byte, error) { hCopy := make(Headers) for k, v := range *h { - k := strings.ReplaceAll(strings.ToLower(k), "-", "_") + k := strings.ToLower(k) hCopy[k] = v } return jsoniter.Marshal(hCopy) diff --git a/pkg/output/fields.go b/pkg/output/fields.go index ab8666b7..cf074a83 100644 --- a/pkg/output/fields.go +++ b/pkg/output/fields.go @@ -5,7 +5,6 @@ import ( "net/url" "os" "path" - "path/filepath" "strings" "github.com/projectdiscovery/gologger" @@ -85,7 +84,7 @@ func storeFields(output *Result, storeFields []string) { } func appendToFileField(parsed *url.URL, field, data string) { - file, err := os.OpenFile(filepath.Join(storeFieldsDirectory, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + file, err := os.OpenFile(path.Join(storeFieldDir, fmt.Sprintf("%s_%s_%s.txt", parsed.Scheme, parsed.Hostname(), field)), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { return } diff --git a/pkg/output/options.go b/pkg/output/options.go index 03e1bb37..99faa32a 100644 --- a/pkg/output/options.go +++ b/pkg/output/options.go @@ -12,12 +12,14 @@ type Options struct { JSON bool Verbose bool StoreResponse bool + NoClobber bool OmitRaw bool OmitBody bool OutputFile string Fields string StoreFields string StoreResponseDir string + StoreFieldDir string FieldConfig string ErrorLogFile string MatchRegex []*regexp.Regexp diff --git a/pkg/output/output.go b/pkg/output/output.go index e879ea43..0e0e1983 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "sync" @@ -16,15 +17,16 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/katana/pkg/utils/extensions" errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" ) const ( - storeFieldsDirectory = "katana_field" - indexFile = "index.txt" - DefaultResponseDir = "katana_response" + indexFile = "index.txt" + DefaultResponseDir = "katana_response" ) var ( + storeFieldDir = "katana_field" decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) ) @@ -48,6 +50,7 @@ type StandardWriter struct { outputMutex *sync.Mutex storeResponse bool storeResponseDir string + noClobber bool omitRaw bool omitBody bool errorFile *fileWriter @@ -68,6 +71,7 @@ func New(options Options) (Writer, error) { outputMutex: &sync.Mutex{}, storeResponse: options.StoreResponse, storeResponseDir: options.StoreResponseDir, + noClobber: options.NoClobber, omitRaw: options.OmitRaw, omitBody: options.OmitBody, matchRegex: options.MatchRegex, @@ -76,6 +80,10 @@ func New(options Options) (Writer, error) { outputMatchCondition: options.OutputMatchCondition, outputFilterCondition: options.OutputFilterCondition, } + + if options.StoreFieldDir != "" { + storeFieldDir = options.StoreFieldDir + } // if fieldConfig empty get the default file if options.FieldConfig == "" { var err error @@ -99,7 +107,7 @@ func New(options Options) (Writer, error) { } } if options.StoreFields != "" { - _ = os.MkdirAll(storeFieldsDirectory, os.ModePerm) + _ = os.MkdirAll(storeFieldDir, os.ModePerm) if err := validateFieldNames(options.StoreFields); err != nil { return nil, errorutil.NewWithTag("output", "could not validate store fields").Wrap(err) } @@ -117,8 +125,13 @@ func New(options Options) (Writer, error) { if options.StoreResponseDir != DefaultResponseDir && options.StoreResponseDir != "" { writer.storeResponseDir = options.StoreResponseDir } - _ = os.RemoveAll(writer.storeResponseDir) - _ = os.MkdirAll(writer.storeResponseDir, os.ModePerm) + if options.NoClobber { + writer.storeResponseDir = createDirNameNoClobber(writer.storeResponseDir) + _ = os.MkdirAll(writer.storeResponseDir, os.ModePerm) + } else { + removeDirsWithSuffix(writer.storeResponseDir) + _ = os.MkdirAll(writer.storeResponseDir, os.ModePerm) + } // todo: the index file seems never used? _, err := newFileOutputWriter(filepath.Join(writer.storeResponseDir, indexFile)) if err != nil { @@ -252,6 +265,53 @@ func (w *StandardWriter) Close() error { return nil } +func createDirNameNoClobber(dir string) string { + if !fileutil.FolderExists(dir) { + return dir + } + + parentDir, dirName := filepath.Dir(dir), filepath.Base(dir) + entries, err := os.ReadDir(parentDir) + if err != nil { + return dirName + } + + highestNum := 0 + regex := regexp.MustCompile(fmt.Sprintf("^%s(\\d+)$", regexp.QuoteMeta(dirName))) + for _, entry := range entries { + if entry.IsDir() { + name := entry.Name() + matches := regex.FindStringSubmatch(name) + if matches != nil { + if num, err := strconv.Atoi(matches[1]); err == nil && num > highestNum { + highestNum = num + } + } + } + } + + newDirName := fmt.Sprintf("%s%d", dirName, highestNum+1) + newFullPath := filepath.Join(parentDir, newDirName) + return newFullPath +} + +func removeDirsWithSuffix(dir string) { + parentDir, dirName := filepath.Dir(dir), filepath.Base(dir) + entries, _ := os.ReadDir(parentDir) + + pattern := fmt.Sprintf("^%s(\\d*)$", regexp.QuoteMeta(dirName)) + regex := regexp.MustCompile(pattern) + for _, entry := range entries { + if entry.IsDir() { + name := entry.Name() + if regex.MatchString(name) { + fullPath := filepath.Join(parentDir, name) + _ = os.RemoveAll(fullPath) + } + } + } +} + // matchOutput checks if the event matches the output regex func (w *StandardWriter) matchOutput(event *Result) bool { if w.matchRegex == nil && w.outputMatchCondition == "" { diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index 8186780d..86cb6592 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -6,15 +6,12 @@ import ( "time" "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/katana/pkg/output" "github.com/projectdiscovery/katana/pkg/utils/extensions" "github.com/projectdiscovery/katana/pkg/utils/filters" "github.com/projectdiscovery/katana/pkg/utils/scope" "github.com/projectdiscovery/ratelimit" errorutil "github.com/projectdiscovery/utils/errors" - logutil "github.com/projectdiscovery/utils/log" urlutil "github.com/projectdiscovery/utils/url" wappalyzer "github.com/projectdiscovery/wappalyzergo" ) @@ -42,7 +39,7 @@ type CrawlerOptions struct { // NewCrawlerOptions creates a new crawler options structure // from user specified options. func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { - configureOutput(options) + options.ConfigureOutput() extensionsValidator := extensions.NewValidator(options.ExtensionsMatch, options.ExtensionFilter) dialerOpts := fastdialer.DefaultOptions @@ -72,6 +69,8 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { Fields: options.Fields, StoreFields: options.StoreFields, StoreResponseDir: options.StoreResponseDir, + NoClobber: options.NoClobber, + StoreFieldDir: options.StoreFieldDir, OmitRaw: options.OmitRaw, OmitBody: options.OmitBody, FieldConfig: options.FieldConfig, @@ -151,18 +150,3 @@ func (c *CrawlerOptions) ValidateScope(absURL, rootHostname string) (bool, error } return true, nil } - -// configureOutput configures the output logging levels to be displayed on the screen -func configureOutput(options *Options) { - if options.Silent { - gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) - } else if options.Verbose { - gologger.DefaultLogger.SetMaxLevel(levels.LevelWarning) - } else if options.Debug { - gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) - } else { - gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) - } - - logutil.DisableDefaultLogger() -} diff --git a/pkg/types/options.go b/pkg/types/options.go index 1963830b..43e6e8f3 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -6,8 +6,11 @@ import ( "time" "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/katana/pkg/output" fileutil "github.com/projectdiscovery/utils/file" + logutil "github.com/projectdiscovery/utils/log" ) // OnResultCallback (output.Result) @@ -114,6 +117,10 @@ type Options struct { StoreResponse bool // StoreResponseDir specifies if katana should use a custom directory to store http requests/responses StoreResponseDir string + // NoClobber specifies if katana should overwrite existing output files + NoClobber bool + // StoreFieldDir specifies if katana should use a custom directory to store fields + StoreFieldDir string // OmitRaw omits raw requests/responses from the output OmitRaw bool // OmitBody omits the response body from the output @@ -188,3 +195,18 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string { func (options *Options) ShouldResume() bool { return options.Resume != "" && fileutil.FileExists(options.Resume) } + +// ConfigureOutput configures the output logging levels to be displayed on the screen +func (options *Options) ConfigureOutput() { + if options.Silent { + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + } else if options.Verbose { + gologger.DefaultLogger.SetMaxLevel(levels.LevelWarning) + } else if options.Debug { + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) + } else { + gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) + } + + logutil.DisableDefaultLogger() +} diff --git a/pkg/utils/jsluice.go b/pkg/utils/jsluice.go index 38e5c7b8..3314956d 100644 --- a/pkg/utils/jsluice.go +++ b/pkg/utils/jsluice.go @@ -10,7 +10,7 @@ import ( var ( // CommonJSLibraryFileRegex is a regex to match common js library files. - CommonJSLibraryFileRegex = `(?i)(?:amplify|quantserve|slideshow|jquery|modernizr|polyfill|vendor|modules|gtm|underscore?|tween|retina|selectivizr|cufon|angular|swf|sha1|freestyle|bootstrap|d3|backbone|videojs|google[-_]analytics|material|redux|knockout|datepicker|datetimepicker|ember|react|ng|fusion|analytics|libs?|vendors?|node[-_]modules|lodash|moment|chart|highcharts|raphael|prototype|mootools|dojo|ext|yui|web[-_]?components|polymer|vue|svelte|next|nuxt|gatsby|express|koa|hapi|socket[-_.]?io|axios|superagent|request|bluebird|rxjs|ramda|immutable|flux|redux[-_]saga|mobx|relay|apollo|graphql|three|phaser|pixi|babylon|cannon|hammer|howler|gsap|velocity|mo[-_.]?js|popper|shepherd|prism|highlight|markdown[-_]?it|codemirror|ace[-_]?editor|tinymce|ckeditor|quill|simplemde|monaco[-_]?editor|pdf[-_.]?js|jspdf|fabric|paper|konva|p5|processing|matter[-_.]?js|box2d|planck)(?:[-._][\w\d]*)*\.js$` + CommonJSLibraryFileRegex = `(?i)(?:amplify|quantserve|slideshow|jquery|modernizr|polyfill|vendor|modules|gtm|underscore?|tween|retina|selectivizr|cufon|angular|swf|sha1|freestyle|bootstrap|d3|backbone|videojs|google[-_]analytics|material|redux|knockout|datepicker|datetimepicker|ember|react|ng|fusion|analytics|libs?|vendors?|node[-_]modules|lodash|moment|chart|highcharts|raphael|prototype|mootools|dojo|ext|yui|web[-_]?components|polymer|vue|svelte|next|nuxt|gatsby|express|koa|hapi|socket[-_.]?io|axios|superagent|request|bluebird|rxjs|ramda|immutable|flux|redux[-_]saga|mobx|relay|apollo|graphql|three|phaser|pixi|babylon|cannon|hammer|howler|gsap|velocity|mo[-_.]?js|popper|shepherd|prism|highlight|markdown[-_]?it|codemirror|ace[-_]?editor|tinymce|ckeditor|quill|simplemde|monaco[-_]?editor|pdf[-_.]?js|jspdf|fabric|paper|konva|p5|processing|matter[-_.]?js|box2d|planck|chart[-_.]?js|plotly|echarts|d3[-_.]?force|sigma|c3|nvd3|amcharts|vis[-_.]?js|dagre[-_.]?d3|cytoscape|leaflet|openlayers|ol3|mapbox|cesium|turf|moment[-_.]?timezone|luxon|dayjs|date[-_.]?fns|date[-_.]?io|flatpickr|pikaday|fullcalendar|draggable|interact|sortable|dragula|dropzone|filepond|uppy|fine[-_.]?uploader|plyr|mediaelement|flowplayer|jwplayer|video[-_.]?js|mediaelement[-_.]?js|dash[-_.]?js|hls[-_.]?js|videojs|wavesurfer|soundmanager|amplitude|pizzicato|tone|adroll|doubleclick|facebook-pixel|ga-audiences|googlesyndication|adsbygoogle|gpt|amazon-adsystem|criteo|taboola|outbrain|bidswitch|bidswitch.net|spotxchange|yahoo|media.net|contextweb|openx|pubmatic|rubiconproject|indexexchange|appnexus|liveintent|triplelift|verizonmedia|synacor|sonobi|yieldmo|gumgum|smartadserver|mopub|pubnative|inmobi|chartboost|tapjoy|admob|unityads|vungle|flurry|matomy|altitude|dataxu|thetradedesk|exponential|zypmedia|quantcast|mediamath|bidswitch|mgid|revcontent|powerlinks|rhythmone|airpush|smaato|adcolony|mopub|leadbolt|mobfox|nativo|revjet|smartyads|avocarrot|epom|imobile|supersonicads|loopme|applovin|pandora|mytarget|bidvertiser|chitika|popads|propellerads|buysellads|adhit|hilltopads|plugrush|popcash|popunder|revenuehits|trafficjunky|trafficfactory|zero-|smartoasis)(?:[-._][\w\d]*)*\.js$` commonJSLibraryFileRegexCompiled = regexp.MustCompile(CommonJSLibraryFileRegex) ) diff --git a/pkg/utils/regex.go b/pkg/utils/regex.go index 61ead03f..4c2db7c2 100644 --- a/pkg/utils/regex.go +++ b/pkg/utils/regex.go @@ -7,10 +7,10 @@ import ( var ( BodyA0 = `(?:` BodyB0 = `(` - BodyC0 = `(?:[\.]{1,2}/[A-Za-z0-9\-_/\\?&@\.?=%]+)` - BodyC1 = `|(https?://[A-Za-z0-9_\-\.]+([\.]{0,2})?\/[A-Za-z0-9\-_/\\?&@\.?=%]+)` - BodyC2 = `|(/[A-Za-z0-9\-_/\\?&@\.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(p|on)?|pdf|php5?|py|rss))` - BodyC3 = `|([A-Za-z0-9\-_?&@\.%]+/[A-Za-z0-9/\\\-_?&@\.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(p|on)?|pdf|php5?|py|rss))` + BodyC0 = `(?:\.\./[A-Za-z0-9\-_/\\?&@.=%]+)` + BodyC1 = `|(https?://[A-Za-z0-9_\-.]+(?:\.\./)?/[A-Za-z0-9\-_/\\?&@.=%]+)` + BodyC2 = `|(/[A-Za-z0-9\-_/\\?&@.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(?:p|on)?|pdf|php5?|py|rss))` + BodyC3 = `|([A-Za-z0-9\-_?&@.%]+/[A-Za-z0-9/\\\-_?&@.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(?:p|on)?|pdf|php5?|py|rss))` BodyB1 = `)` BodyA1 = `)` // pageBodyRegex extracts endpoints from page body @@ -18,10 +18,10 @@ var ( JsA0 = `(?:"|'|\s)` JsB0 = `(` - JsC0 = `((https?://[A-Za-z0-9_\-\.]+(:\d{1,5})?)+([\.]{1,2})?/[A-Za-z0-9/\-_\.\\%]+([\?|#][^"']+)?)` - JsC1 = `|((\.{1,2}/)?[a-zA-Z0-9\-_/\\%]+\.(aspx?|js(on|p)?|html|php5?|html|action|do)([\?|#][^"']+)?)` - JsC2 = `|((\.{0,2}/)[a-zA-Z0-9\-_/\\%]+(/|\\)[a-zA-Z0-9\-_]{3,}([\?|#][^"|']+)?)` - JsC3 = `|((\.{0,2})[a-zA-Z0-9\-_/\\%]{3,}/)` + JsC0 = `((https?://[A-Za-z0-9_\-.]+(?:\:\d{1,5})?)+(?:\.\./)?/[A-Za-z0-9/\-_\\.%]+(?:[\?|#][^"']+)?)` + JsC1 = `|((?:\.\./)?[a-zA-Z0-9\-_/\\%]+\.(aspx?|js(?:on|p)?|html|php5?|action|do)(?:[\?|#][^"']+)?)` + JsC2 = `|((?:\.\./)[a-zA-Z0-9\-_/\\%]+(?:/|\\)[a-zA-Z0-9\-_]{3,}(?:[\?|#][^"']+)?)` + JsC3 = `|((?:\.\./)[a-zA-Z0-9\-_/\\%]{3,}/)` JsB1 = `)` JsA1 = `(?:"|'|\s)` // relativeEndpointsRegex is the regex to find endpoints in js files. diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 80c2d90c..3b355a45 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -65,7 +65,7 @@ func ParseRefreshTag(value string) string { // WebUserAgent returns the chrome-web user agent func WebUserAgent() string { - return "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" + return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36" } func FlattenHeaders(headers map[string][]string) map[string]string {