From 9ba078242121204059730fed6b8878273561cb8b Mon Sep 17 00:00:00 2001
From: Daniel Igarashi <79033724+gigarashi@users.noreply.github.com>
Date: Sat, 20 Feb 2021 09:13:39 +0800
Subject: [PATCH] Trim white space prefix in line of robots.txt

---
 pkg/collector/collector.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go
index 7d586f6..6a6f77b 100644
--- a/pkg/collector/collector.go
+++ b/pkg/collector/collector.go
@@ -346,7 +346,7 @@ func (c *Collector) parseRobots(url string, reqsMade *syncList) {
 
         for _, line := range lines {
                 if re.MatchString(line) {
-                        urlstring := re.ReplaceAllString(line, "")
+                        urlstring := strings.TrimLeft(re.ReplaceAllString(line, ""), " ")
                         if c.conf.IncludeRobots || c.conf.IncludeAll {
                                 _ = c.recordIfInScope(c.au.BrightMagenta("[robots]"), url, url+urlstring, reqsMade)
                         }