diff --git a/src/scrapyscript/__init__.py b/src/scrapyscript/__init__.py index 64001f0..de9fc11 100644 --- a/src/scrapyscript/__init__.py +++ b/src/scrapyscript/__init__.py @@ -40,7 +40,7 @@ class Processor(Process): Blocks until all have finished. """ - def __init__(self, settings=None): + def __init__(self, settings=None, install_root_handler=True): """ Parms: settings (scrapy.settings.Settings) - settings to apply. Defaults @@ -51,6 +51,7 @@ def __init__(self, settings=None): self.results = Queue(**kwargs) self.items = [] self.settings = settings or Settings() + self.install_root_handler = install_root_handler dispatcher.connect(self._item_scraped, signals.item_scraped) def _item_scraped(self, item): @@ -62,7 +63,10 @@ def _crawl(self, requests): requests (Request) - One or more Jobs. All will be loaded into a single invocation of the reactor. """ - self.crawler = CrawlerProcess(self.settings) + self.crawler = CrawlerProcess( + settings=self.settings, + install_root_handler=self.install_root_handler, + ) # crawl can be called multiple times to queue several requests for req in requests: diff --git a/tests/test_celery.py b/tests/test_celery.py index 6c9f3f1..efd178a 100644 --- a/tests/test_celery.py +++ b/tests/test_celery.py @@ -16,9 +16,11 @@ def celery_job(url): @app.task -def celery_job_with_custom_settings(url, settings): +def celery_job_with_custom_settings(url, settings, install_root_handler): job = Job(ItemSpider, url=url) - return Processor(settings=settings).run(job) + return Processor(settings=settings, install_root_handler=install_root_handler).run( + job + ) class TestScrapyScriptCelery: @@ -32,7 +34,7 @@ def test_celery_job_with_settings(self): settings["BOT_NAME"] = "alpha" task = celery_job_with_custom_settings.s( - "https://www.python.org", settings + "https://www.python.org", settings, False ).apply() print(task.result[0]) assert task.result[0]["bot"] == "alpha"