Merge pull request #8 from pppontusw/no-refetch-same-page

Don't rescrape the same page
pppontusw · May 9, 2024 · 417dd38 · 417dd38
2 parents 402a8fc + c3955a1
commit 417dd38
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/pkg/scraper/scrapers.go b/pkg/scraper/scrapers.go
@@ -78,6 +78,10 @@ func (bs *BaseScraper) Scrape(maxWorkers int) ([]models.Product, error) {
 					break // Exit loop if there's no next URL
 				}
 
+				if nextURL == currentURL {
+					break // Exit loop if the next URL is the same as the current URL
+				}
+
 				currentURL = nextURL
 			}
 		}(url)