Work around Python 2.7 urllib2 bug
If the remote is using authenticated HTTP, but does not have
$GIT_URL/clone.bundle files in each repository, an initial sync
would fail around 8 projects in due to the library not resetting
the number of failures after getting a 404.
Work around this by updating the retry counter ourselves.
The urllib2 library is also not thread-safe. Make it somewhat
safer by wrapping the critical section with a lock.
Change-Id: I886e2750ef4793cbe2150c3b5396eb9f10974f7f
Signed-off-by: Shawn O. Pearce <sop@google.com>
diff --git a/project.py b/project.py
index 5f8369d..4bc54de 100644
--- a/project.py
+++ b/project.py
@@ -24,6 +24,11 @@
import time
import urllib2
+try:
+ import threading as _threading
+except ImportError:
+ import dummy_threading as _threading
+
from color import Coloring
from git_command import GitCommand
from git_config import GitConfig, IsId, GetSchemeFromUrl
@@ -34,6 +39,8 @@
from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
+_urllib_lock = _threading.Lock()
+
def _lwrite(path, content):
lock = '%s.lock' % path
@@ -1458,40 +1465,44 @@
dest.seek(0, os.SEEK_END)
pos = dest.tell()
- req = urllib2.Request(srcUrl)
- if pos > 0:
- req.add_header('Range', 'bytes=%d-' % pos)
-
+ _urllib_lock.acquire()
try:
- r = urllib2.urlopen(req)
- except urllib2.HTTPError, e:
- def _content_type():
- try:
- return e.info()['content-type']
- except:
- return None
+ req = urllib2.Request(srcUrl)
+ if pos > 0:
+ req.add_header('Range', 'bytes=%d-' % pos)
- if e.code == 404:
- keep = False
- return False
- elif _content_type() == 'text/plain':
- try:
- msg = e.read()
- if len(msg) > 0 and msg[-1] == '\n':
- msg = msg[0:-1]
- msg = ' (%s)' % msg
- except:
- msg = ''
- else:
- try:
- from BaseHTTPServer import BaseHTTPRequestHandler
- res = BaseHTTPRequestHandler.responses[e.code]
- msg = ' (%s: %s)' % (res[0], res[1])
- except:
- msg = ''
- raise DownloadError('HTTP %s%s' % (e.code, msg))
- except urllib2.URLError, e:
- raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+ try:
+ r = urllib2.urlopen(req)
+ except urllib2.HTTPError, e:
+ def _content_type():
+ try:
+ return e.info()['content-type']
+ except:
+ return None
+
+ if e.code == 404:
+ keep = False
+ return False
+ elif _content_type() == 'text/plain':
+ try:
+ msg = e.read()
+ if len(msg) > 0 and msg[-1] == '\n':
+ msg = msg[0:-1]
+ msg = ' (%s)' % msg
+ except:
+ msg = ''
+ else:
+ try:
+ from BaseHTTPServer import BaseHTTPRequestHandler
+ res = BaseHTTPRequestHandler.responses[e.code]
+ msg = ' (%s: %s)' % (res[0], res[1])
+ except:
+ msg = ''
+ raise DownloadError('HTTP %s%s' % (e.code, msg))
+ except urllib2.URLError, e:
+ raise DownloadError('%s: %s ' % (req.get_host(), str(e)))
+ finally:
+ _urllib_lock.release()
p = None
try: