From e9cc61c2008bd76b5287c180075b9f4cb7ecdfe7 Mon Sep 17 00:00:00 2001 From: Gabriel Ferreira Date: Tue, 14 Feb 2023 12:12:46 -0300 Subject: [PATCH] tests: update dead url test code in test-ns3.py --- utils/tests/test-ns3.py | 57 ++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/utils/tests/test-ns3.py b/utils/tests/test-ns3.py index f7b6be018..5f01f16eb 100755 --- a/utils/tests/test-ns3.py +++ b/utils/tests/test-ns3.py @@ -2771,6 +2771,8 @@ class NS3QualityControlTestCase(unittest.TestCase): # Skip this test if requests library is not available try: import requests + import urllib3 + urllib3.disable_warnings() except ImportError: requests = None # noqa self.skipTest("Requests library is not available") @@ -2850,37 +2852,39 @@ class NS3QualityControlTestCase(unittest.TestCase): validate_url(test_url) except ValidationError: dead_link_msg = "%s: URL %s, invalid URL" % (test_filepath, test_url) + except Exception as e: + self.assertEqual(False, True, msg=e.__str__()) + if dead_link_msg is not None: + return dead_link_msg + tries = 3 # Check if valid URLs are alive - if dead_link_msg is None: + while tries > 0: + # Not verifying the certificate (verify=False) is potentially dangerous + # HEAD checks are not as reliable as GET ones, + # in some cases they may return bogus error codes and reasons try: - tries = 3 - while tries > 0: - # Not verifying the certificate (verify=False) is potentially dangerous - # HEAD checks are not as reliable as GET ones, - # in some cases they may return bogus error codes and reasons - response = requests.get(test_url, verify=False, headers=headers) + response = requests.get(test_url, verify=False, headers=headers, timeout=50) - # In case of success and redirection - if response.status_code in [200, 301]: + # In case of success and redirection + if response.status_code in [200, 301]: + dead_link_msg = None + break + + # People use the wrong code, but the reason + # can still be correct + if response.status_code in [302, 308, 500, 503]: + if response.reason.lower() in ['found', + 'moved temporarily', + 'permanent redirect', + 'ok', + 'service temporarily unavailable' + ]: dead_link_msg = None break - - # People use the wrong code, but the reason - # can still be correct - if response.status_code in [302, 308, 500, 503]: - if response.reason.lower() in ['found', - 'moved temporarily', - 'permanent redirect', - 'ok', - 'service temporarily unavailable' - ]: - dead_link_msg = None - break - # In case it didn't pass in any of the previous tests, - # set dead_link_msg with the most recent error and try again - dead_link_msg = "%s: URL %s: returned code %d" % (test_filepath, test_url, response.status_code) - tries -= 1 + # In case it didn't pass in any of the previous tests, + # set dead_link_msg with the most recent error and try again + dead_link_msg = "%s: URL %s: returned code %d" % (test_filepath, test_url, response.status_code) except requests.exceptions.InvalidURL: dead_link_msg = "%s: URL %s: invalid URL" % (test_filepath, test_url) except requests.exceptions.SSLError: @@ -2893,11 +2897,12 @@ class NS3QualityControlTestCase(unittest.TestCase): except AttributeError: error_msg = e.args[0] dead_link_msg = "%s: URL %s: failed with exception: %s" % (test_filepath, test_url, error_msg) + tries -= 1 return dead_link_msg # Dispatch threads to test multiple URLs concurrently from concurrent.futures import ThreadPoolExecutor - with ThreadPoolExecutor(max_workers=20) as executor: + with ThreadPoolExecutor(max_workers=100) as executor: dead_links = list(executor.map(test_file_url, list(files_and_urls))) # Filter out None entries