From e7d007a46915bafc601dbb33a3546f531e6eb997 Mon Sep 17 00:00:00 2001 From: Dhananjay Shah Date: Fri, 22 Nov 2024 15:14:33 +0100 Subject: [PATCH] gui-crawl: Reduce memory required to end a test-run. Earlier, If URLs were not found for 100 iterations, then test-run was exited. This took ~8 mins on a local system. On a busy CI node, the time taken could be scaled by a fator of 3. Now, If URLs are not found for 25 iterations, then exit the test-run. Removed `time.sleep(rate_of_tasks)`. processing the tasks as a TaskGroup ensures that URLs are found faster than the tasks being created. CMK-20093 Change-Id: I0fd9e9b4023d29e7c1a1cf66853ba4ab086e4b1d --- tests/testlib/crawler.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/testlib/crawler.py b/tests/testlib/crawler.py index 9d3f14a1d28..2c8f1e2c687 100644 --- a/tests/testlib/crawler.py +++ b/tests/testlib/crawler.py @@ -211,13 +211,11 @@ async def crawl(self, max_tasks: int, max_url_batch_size: int = 100) -> None: """Crawl through URLs using simultaneously using tasks / coroutines. A group of tasks / coroutines is added every `rate_create_crawl_task` seconds. - Crawling stop when URLs are not found for last `memory_size_urls_exist` iterations - (`rate_create_crawl_task` x `memory_size_urls_exist` seconds). + Crawling stop when URLs are not found for last `memory_size_urls_exist` iterations. debug-mode: Crawling URLs stop when at least `--max-urls` number of URLs have been crawled. """ - rate_create_crawl_task = 0.1 # seconds - memory_size_urls_exist = 100 # iterations + memory_size_urls_exist = 25 # iterations search_limited_urls: bool = self._max_urls > 0 # special-case if search_limited_urls and self._max_urls < max_tasks: @@ -256,9 +254,6 @@ async def crawl(self, max_tasks: int, max_url_batch_size: int = 100) -> None: logger.info("No more URLs to crawl. Stopping ...") # ---- - # ensure rate of URL collection in `self._todos` > rate of new tasks added - time.sleep(rate_create_crawl_task) - async def setup_checkmk_context( self, browser: playwright.async_api.Browser ) -> playwright.async_api.BrowserContext: