From 9f5eb7e3392e6bc2ad979ad32d3dd27ef9c00b20 Mon Sep 17 00:00:00 2001 From: wbond Date: Fri, 13 Aug 2021 21:14:52 -0400 Subject: [PATCH] Reduce the batch size when crawling --- app/tasks/crawl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/tasks/crawl.py b/app/tasks/crawl.py index fda43aa..0029668 100644 --- a/app/tasks/crawl.py +++ b/app/tasks/crawl.py @@ -34,9 +34,8 @@ valid_sources = package.sources.sources_for(explicit_package) valid_dependency_sources = [] else: - # Limit each run to 400 packages - valid_sources = package.sources.outdated_sources(60, 400) - valid_dependency_sources = dependency.outdated_sources(60, 400) + valid_sources = package.sources.outdated_sources(60, 200) + valid_dependency_sources = dependency.outdated_sources(60, 200) invalid_sources = package.sources.invalid_sources(valid_sources) invalid_dependency_sources = dependency.invalid_sources(valid_dependency_sources)