Merge pull request #899 from projectdiscovery/remove_passive_crawling

Remove passive crawling

README.md

@@ -30,7 +30,6 @@
 
  - Fast And fully configurable web crawling
  - **Standard** and **Headless** mode
- - **Active** and **Passive** mode
  - **JavaScript** parsing / crawling
  - Customizable **automatic form filling**
  - **Scope control** - Preconfigured field / Regex 
@@ -157,10 +156,6 @@ HEADLESS:
    -cwu, -chrome-ws-url string       use chrome browser instance launched elsewhere with the debugger listening at this URL
    -xhr, -xhr-extraction             extract xhr request url,method in jsonl output
 
-PASSIVE:
-   -ps, -passive                   enable passive sources to discover target endpoints
-   -pss, -passive-source string[]  passive source to use for url discovery (waybackarchive,commoncrawl,alienvault)
-
 SCOPE:
    -cs, -crawl-scope string[]       in scope url regex to be followed by crawler
    -cos, -crawl-out-scope string[]  out of scope url regex to be excluded by crawler

cmd/katana/main.go

@@ -134,10 +134,6 @@ pipelines offering both headless and non-headless crawling.`)
 		flagSet.StringVarP(&options.ChromeWSUrl, "chrome-ws-url", "cwu", "", "use chrome browser instance launched elsewhere with the debugger listening at this URL"),
 		flagSet.BoolVarP(&options.XhrExtraction, "xhr-extraction", "xhr", false, "extract xhr request url,method in jsonl output"),
 	)
-	flagSet.CreateGroup("passive", "Passive",
-		flagSet.BoolVarP(&options.Passive, "passive", "ps", false, "enable passive sources to discover target endpoints"),
-		flagSet.StringSliceVarP(&options.PassiveSource, "passive-source", "pss", nil, "passive source to use for url discovery (waybackarchive,commoncrawl,alienvault)", goflags.NormalizedStringSliceOptions),
-	)
 
 	flagSet.CreateGroup("scope", "Scope",
 		flagSet.StringSliceVarP(&options.Scope, "crawl-scope", "cs", nil, "in scope url regex to be followed by crawler", goflags.FileCommaSeparatedStringSliceOptions),

internal/runner/options.go

@@ -32,10 +32,6 @@ func validateOptions(options *types.Options) error {
 		gologger.Info().Msgf("Automatic form fill (-aff) has been disabled for headless navigation.")
 	}
 
-	if options.Headless && options.Passive {
-		return errorutil.New("headless mode (-headless) and passive mode (-passive) cannot be used together")
-	}
-
 	if (options.HeadlessOptionalArguments != nil || options.HeadlessNoSandbox || options.SystemChromePath != "") && !options.Headless {
 		return errorutil.New("headless mode (-hl) is required if -ho, -nos or -scp are set")
 	}

internal/runner/runner.go

@@ -9,7 +9,6 @@ import (
 	"github.com/projectdiscovery/katana/pkg/engine"
 	"github.com/projectdiscovery/katana/pkg/engine/hybrid"
 	"github.com/projectdiscovery/katana/pkg/engine/parser"
-	"github.com/projectdiscovery/katana/pkg/engine/passive"
 	"github.com/projectdiscovery/katana/pkg/engine/standard"
 	"github.com/projectdiscovery/katana/pkg/types"
 	"github.com/projectdiscovery/mapcidr"
@@ -98,8 +97,6 @@ func New(options *types.Options) (*Runner, error) {
 	switch {
 	case options.Headless:
 		crawler, err = hybrid.New(crawlerOptions)
-	case options.Passive:
-		crawler, err = passive.New(crawlerOptions)
 	default:
 		crawler, err = standard.New(crawlerOptions)
 	}

pkg/engine/common/base.go

@@ -72,7 +72,7 @@ func (s *Shared) Enqueue(queue *queue.Queue, navigationRequests ...*navigation.R
 			// if the user requested anyway out of scope items
 			// they are sent to output without visiting
 			if s.Options.Options.DisplayOutScope {
-				s.Output(nr, nil, nil, ErrOutOfScope)
+				s.Output(nr, nil, ErrOutOfScope)
 			}
 			continue
 		}
@@ -95,18 +95,17 @@ func (s *Shared) ValidateScope(URL string, root string) bool {
 	return err == nil && scopeValidated
 }
 
-func (s *Shared) Output(navigationRequest *navigation.Request, navigationResponse *navigation.Response, passiveReference *navigation.PassiveReference, err error) {
+func (s *Shared) Output(navigationRequest *navigation.Request, navigationResponse *navigation.Response, err error) {
 	var errData string
 	if err != nil {
 		errData = err.Error()
 	}
 	// Write the found result to output
 	result := &output.Result{
-		Timestamp:        time.Now(),
-		Request:          navigationRequest,
-		Response:         navigationResponse,
-		PassiveReference: passiveReference,
-		Error:            errData,
+		Timestamp: time.Now(),
+		Request:   navigationRequest,
+		Response:  navigationResponse,
+		Error:     errData,
 	}
 
 	outputErr := s.Options.OutputWriter.Write(result)
@@ -230,7 +229,7 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error {
 			resp, err := doRequest(crawlSession, req)
 
 			if inScope {
-				s.Output(req, resp, nil, err)
+				s.Output(req, resp, err)
 			}
 
 			if err != nil {