Don't rescrape the same page

pppontusw · May 9, 2024 · c3955a1 · c3955a1
1 parent 402a8fc
commit c3955a1
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/pkg/scraper/scrapers.go b/pkg/scraper/scrapers.go
@@ -78,6 +78,10 @@ func (bs *BaseScraper) Scrape(maxWorkers int) ([]models.Product, error) {
 					break // Exit loop if there's no next URL
 				}
 
+				if nextURL == currentURL {
+					break // Exit loop if the next URL is the same as the current URL
+				}
+
 				currentURL = nextURL
 			}
 		}(url)