Skip to content
Snippets Groups Projects

Speedup test_urls

Merged Emanuele Aina requested to merge wip/em/speedup-test-urls into master
1 file
+ 7
2
Compare changes
  • Side-by-side
  • Inline
+ 7
2
@@ -84,11 +84,16 @@ headers={
"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36",
}
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(max_retries=3)
session.mount('http://', adapter)
session.mount('https://', adapter)
def url_check(url):
start = time.perf_counter()
resp = None
try:
resp = requests.head(url, headers=headers, allow_redirects=True, timeout=60, verify=False)
resp = session.head(url, headers=headers, allow_redirects=True, timeout=60, verify=False)
status = resp.ok
resp.close()
except Exception as e:
@@ -100,7 +105,7 @@ def url_check(url):
# that's not got a 200 status code with GET as well.
if not status:
resp = None
resp = requests.get(url, headers=headers, allow_redirects=True, timeout=60, verify=False)
resp = session.get(url, headers=headers, allow_redirects=True, timeout=60, verify=False)
status = resp.ok
resp.close()
except Exception as e:
Loading