https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=832003502e6d2cb5dd1eeb52bf3bd1e7ea2e2635 commit 832003502e6d2cb5dd1eeb52bf3bd1e7ea2e2635 Author: Jon Turney <jon.turney@dronecode.org.uk> Date: Mon Apr 13 16:46:27 2020 +0100 Add a tool for demonstrating version sort https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=1e59f9df68ad944cf05a1e0ebd8c7e6631d3d866 commit 1e59f9df68ad944cf05a1e0ebd8c7e6631d3d866 Author: Jon Turney <jon.turney@dronecode.org.uk> Date: Mon Apr 13 15:51:36 2020 +0100 Enhance homepage fixing Handle: * Trivial terminal '/' redirects * 308 Permanent Redirect as well as 301 Moved Permanently * sites which block urllib's default user-agent * Slow to respond sites with a timeout Diff: --- calm/fixes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/calm/fixes.py b/calm/fixes.py index a89896a..33c1087 100644 --- a/calm/fixes.py +++ b/calm/fixes.py @@ -26,6 +26,7 @@ import logging import os import re import shutil +import socket import tarfile import urllib.request import urllib.error @@ -72,11 +73,12 @@ class NoRedirection(urllib.request.HTTPErrorProcessor): def follow_redirect(homepage): opener = urllib.request.build_opener(NoRedirection) opener.addheaders = [('User-Agent', 'calm')] + request = urllib.request.Request(homepage, method='HEAD') try: - response = opener.open(homepage) + response = opener.open(request, timeout=60) if response.code in [301, 308]: return response.headers['Location'] - except (ConnectionResetError, ValueError, urllib.error.URLError) as e: + except (ConnectionResetError, ValueError, socket.timeout, urllib.error.URLError) as e: logging.warning('error %s checking homepage:%s' % (e, homepage)) return homepage