2026-10-30 03:10:23 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): Getting https://www.trekbikes.com/us/en_US/store/149438/ took longer than 15.0 seconds.
2026-10-30 03:10:41 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1092, in _runCallbacks
current.result = callback( # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 537, in _cb_timeout
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
scrapy.exceptions.DownloadTimeoutError: Getting https://www.trekbikes.com/us/en_US/store/149438/ took longer than 15.0 seconds.
2026-10-30 03:10:46 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): Getting https://www.trekbikes.com/us/en_US/store/149393/ took longer than 15.0 seconds.
2026-10-30 03:11:11 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1092, in _runCallbacks
current.result = callback( # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 537, in _cb_timeout
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
scrapy.exceptions.DownloadTimeoutError: Getting https://www.trekbikes.com/us/en_US/store/149393/ took longer than 15.0 seconds.
2026-10-30 03:11:20 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): []
2026-10-30 03:11:34 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): Getting https://www.trekbikes.com/us/en_US/store/5310131/ took longer than 15.0 seconds.
2026-10-30 03:11:38 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): []
2026-10-30 03:11:47 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 55, in wrap_twisted_exceptions
yield
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
twisted.web._newclient.ResponseNeverReceived: []
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 139, in download_request
with wrap_twisted_exceptions():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 65, in wrap_twisted_exceptions
raise DownloadFailedError(str(e)) from e
scrapy.exceptions.DownloadFailedError: []
2026-10-30 03:11:50 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1092, in _runCallbacks
current.result = callback( # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 537, in _cb_timeout
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
scrapy.exceptions.DownloadTimeoutError: Getting https://www.trekbikes.com/us/en_US/store/5310131/ took longer than 15.0 seconds.
2026-10-30 03:11:53 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 55, in wrap_twisted_exceptions
yield
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
twisted.web._newclient.ResponseNeverReceived: []
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 139, in download_request
with wrap_twisted_exceptions():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 65, in wrap_twisted_exceptions
raise DownloadFailedError(str(e)) from e
scrapy.exceptions.DownloadFailedError: []
2026-10-30 03:12:11 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): []
2026-10-30 03:12:35 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 55, in wrap_twisted_exceptions
yield
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
twisted.web._newclient.ResponseNeverReceived: []
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 139, in download_request
with wrap_twisted_exceptions():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 65, in wrap_twisted_exceptions
raise DownloadFailedError(str(e)) from e
scrapy.exceptions.DownloadFailedError: []
2026-10-30 03:12:40 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): Getting https://www.trekbikes.com/us/en_US/store/3808822/ took longer than 15.0 seconds.
2026-10-30 03:13:13 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1092, in _runCallbacks
current.result = callback( # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 537, in _cb_timeout
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
scrapy.exceptions.DownloadTimeoutError: Getting https://www.trekbikes.com/us/en_US/store/3808822/ took longer than 15.0 seconds.
2026-10-30 04:08:36 [trek_bikes] ERROR: Invalid value "http://https://www.trekbikes.com/jp/ja_JP/retail/yokkaichi/" for attribute "website". Hostname is not valid.
2026-10-30 05:00:08 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): Getting https://www.trekbikes.com/us/en_US/store/4100693/ took longer than 15.0 seconds.
2026-10-30 05:01:08 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): []
2026-10-30 05:01:34 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1092, in _runCallbacks
current.result = callback( # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 537, in _cb_timeout
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
scrapy.exceptions.DownloadTimeoutError: Getting https://www.trekbikes.com/us/en_US/store/4100693/ took longer than 15.0 seconds.
2026-10-30 05:02:26 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): User timeout caused connection failure.
2026-10-30 05:02:52 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 55, in wrap_twisted_exceptions
yield
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
twisted.web._newclient.ResponseNeverReceived: []
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 139, in download_request
with wrap_twisted_exceptions():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 65, in wrap_twisted_exceptions
raise DownloadFailedError(str(e)) from e
scrapy.exceptions.DownloadFailedError: []
2026-10-30 05:03:55 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 55, in wrap_twisted_exceptions
yield
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
twisted.internet.error.TimeoutError: User timeout caused connection failure.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 139, in download_request
with wrap_twisted_exceptions():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 67, in wrap_twisted_exceptions
raise DownloadTimeoutError(str(e)) from e
scrapy.exceptions.DownloadTimeoutError: User timeout caused connection failure.
2026-10-30 05:04:05 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): Getting https://www.trekbikes.com/us/en_US/store/3880062/ took longer than 15.0 seconds.
2026-10-30 05:05:10 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): User timeout caused connection failure.
2026-10-30 05:05:12 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): User timeout caused connection failure.
2026-10-30 05:06:16 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1092, in _runCallbacks
current.result = callback( # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 537, in _cb_timeout
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
scrapy.exceptions.DownloadTimeoutError: Getting https://www.trekbikes.com/us/en_US/store/3880062/ took longer than 15.0 seconds.
2026-10-30 05:06:22 [scrapy.downloadermiddlewares.retry] ERROR: Gave up retrying (failed 3 times): Getting https://www.trekbikes.com/us/en_US/store/6118091/ took longer than 15.0 seconds.
2026-10-30 05:06:39 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 55, in wrap_twisted_exceptions
yield
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
twisted.internet.error.TimeoutError: User timeout caused connection failure.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 139, in download_request
with wrap_twisted_exceptions():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 67, in wrap_twisted_exceptions
raise DownloadTimeoutError(str(e)) from e
scrapy.exceptions.DownloadTimeoutError: User timeout caused connection failure.
2026-10-30 05:06:44 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 55, in wrap_twisted_exceptions
yield
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
twisted.internet.error.TimeoutError: User timeout caused connection failure.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 139, in download_request
with wrap_twisted_exceptions():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/.venv/lib/python3.11/site-packages/scrapy/utils/_download_handlers.py", line 67, in wrap_twisted_exceptions
raise DownloadTimeoutError(str(e)) from e
scrapy.exceptions.DownloadTimeoutError: User timeout caused connection failure.
2026-10-30 05:06:59 [scrapy.core.scraper] ERROR: Error downloading
Traceback (most recent call last):
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/engine.py", line 514, in _download
result = yield self.downloader.fetch(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1853, in _inlineCallbacks
result = context.run(
^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/python/failure.py", line 467, in throwExceptionIntoGenerator
return g.throw(self.value.with_traceback(self.tb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 226, in fetch
result: Response | Request = yield (
^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1257, in adapt
extracted: _SelfResultT | Failure = result.result()
^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 87, in download_async
result = await self._process_exception(ex, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 156, in _process_exception
raise exception
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 80, in download_async
result: Response | Request = await self._process_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/middleware.py", line 114, in _process_request
return await download_func(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 281, in _enqueue_request
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 348, in _wait_for_download
response = await self._download(slot, request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/__init__.py", line 318, in _download
response: Response = await self.handlers.download_request_async(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/__init__.py", line 168, in download_request_async
return await handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy_zyte_api/handler.py", line 251, in download_request
return await fallback_handler.download_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 140, in download_request
return await maybe_deferred_to_future(agent.download_request(request))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/twisted/internet/defer.py", line 1092, in _runCallbacks
current.result = callback( # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/.venv/lib/python3.11/site-packages/scrapy/core/downloader/handlers/http11.py", line 537, in _cb_timeout
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
scrapy.exceptions.DownloadTimeoutError: Getting https://www.trekbikes.com/us/en_US/store/6118091/ took longer than 15.0 seconds.