Work around Python 2.7 urllib2 bug
If the remote is using authenticated HTTP, but does not have
$GIT_URL/clone.bundle files in each repository, an initial sync
would fail around 8 projects in due to the library not resetting
the number of failures after getting a 404.
Work around this by updating the retry counter ourselves.
The urllib2 library is also not thread-safe. Make it somewhat
safer by wrapping the critical section with a lock.
Change-Id: I886e2750ef4793cbe2150c3b5396eb9f10974f7f
Signed-off-by: Shawn O. Pearce <sop@google.com>
diff --git a/main.py b/main.py
index 8ffdfcc..22e6fa4 100755
--- a/main.py
+++ b/main.py
@@ -273,6 +273,15 @@
req.add_header('User-Agent', _UserAgent())
return req
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+ def http_error_auth_reqed(self, authreq, host, req, headers):
+ try:
+ return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+ self, authreq, host, req, headers)
+ except:
+ self.reset_retry_count()
+ raise
+
def init_http():
handlers = [_UserAgentHandler()]
@@ -287,7 +296,7 @@
pass
except IOError:
pass
- handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+ handlers.append(_BasicAuthHandler(mgr))
if 'http_proxy' in os.environ:
url = os.environ['http_proxy']
diff --git a/project.py b/project.py
index 5f8369d..4bc54de 100644
--- a/project.py
+++ b/project.py
@@ -24,6 +24,11 @@
import time
import urllib2
+try:
+ import threading as _threading
+except ImportError:
+ import dummy_threading as _threading
+
from color import Coloring
from git_command import GitCommand
from git_config import GitConfig, IsId, GetSchemeFromUrl
@@ -34,6 +39,8 @@
from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
+_urllib_lock = _threading.Lock()
+
def _lwrite(path, content):
lock = '%s.lock' % path
@@ -1458,40 +1465,44 @@
dest.seek(0, os.SEEK_END)
pos = dest.tell()
- req = urllib2.Request(srcUrl)
- if pos > 0:
- req.add_header('Range', 'bytes=%d-' % pos)
-
+ _urllib_lock.acquire()
try:
- r = urllib2.urlopen(req)
- except urllib2.HTTPError, e:
- def _content_type():
- try:
- return e.info()['content-type']
- except:
- return None
+ req = urllib2.Request(srcUrl)
+ if pos > 0:
+ req.add_header('Range', 'bytes=%d-' % pos)
- if e.code == 404:
- keep = False
- return False
- elif _content_type() == 'text/plain':
- try:
- msg = e.read()
- if len(msg) > 0 and msg[-1] == '\n':
- msg = msg[0:-1]
- msg = ' (%s)' % msg
- except:
- msg = ''
- else:
- try:
- from BaseHTTPServer import BaseHTTPRequestHandler
- res = BaseHTTPRequestHandler.responses[e.code]
- msg = ' (%s: %s)' % (res[0], res[1])
- except:
- msg = ''
- raise DownloadError('HTTP %s%s' % (e.code, msg))
- except urllib2.URLError, e:
- raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+ try:
+ r = urllib2.urlopen(req)
+ except urllib2.HTTPError, e:
+ def _content_type():
+ try:
+ return e.info()['content-type']
+ except:
+ return None
+
+ if e.code == 404:
+ keep = False
+ return False
+ elif _content_type() == 'text/plain':
+ try:
+ msg = e.read()
+ if len(msg) > 0 and msg[-1] == '\n':
+ msg = msg[0:-1]
+ msg = ' (%s)' % msg
+ except:
+ msg = ''
+ else:
+ try:
+ from BaseHTTPServer import BaseHTTPRequestHandler
+ res = BaseHTTPRequestHandler.responses[e.code]
+ msg = ' (%s: %s)' % (res[0], res[1])
+ except:
+ msg = ''
+ raise DownloadError('HTTP %s%s' % (e.code, msg))
+ except urllib2.URLError, e:
+ raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+ finally:
+ _urllib_lock.release()
p = None
try: