Merge pull request #85 from gigarashi/master

Trim white space prefix in line of robots.txt
hakluke · Feb 22, 2021 · bd3d30a · bd3d30a
2 parents e39a514 + 9ba0782
commit bd3d30a
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go
@@ -346,7 +346,7 @@ func (c *Collector) parseRobots(url string, reqsMade *syncList) {
 
         for _, line := range lines {
                 if re.MatchString(line) {
-                        urlstring := re.ReplaceAllString(line, "")
+                        urlstring := strings.TrimLeft(re.ReplaceAllString(line, ""), " ")
                         if c.conf.IncludeRobots || c.conf.IncludeAll {
                                 _ = c.recordIfInScope(c.au.BrightMagenta("[robots]"), url, url+urlstring, reqsMade)
                         }