Automatically use SSH control master support during sync
By creating a background ssh "control master" process which lives
for the duration of our sync cycle we can easily cut the time for
a no-op sync of 132 projects from 60s to 18s.
Bug: REPO-11
Signed-off-by: Shawn O. Pearce <sop@google.com>
diff --git a/git_config.py b/git_config.py
index 7e642a4..163b080 100644
--- a/git_config.py
+++ b/git_config.py
@@ -16,11 +16,14 @@
import cPickle
import os
import re
+import subprocess
import sys
+import time
+from signal import SIGTERM
from urllib2 import urlopen, HTTPError
from error import GitError, UploadError
from trace import Trace
-from git_command import GitCommand
+from git_command import GitCommand, _ssh_sock
R_HEADS = 'refs/heads/'
R_TAGS = 'refs/tags/'
@@ -331,6 +334,79 @@
return s
+_ssh_cache = {}
+_ssh_master = True
+
+def _open_ssh(host, port=None):
+ global _ssh_master
+
+ if port is None:
+ port = 22
+
+ key = '%s:%s' % (host, port)
+ if key in _ssh_cache:
+ return True
+
+ if not _ssh_master \
+ or 'GIT_SSH' in os.environ \
+ or sys.platform == 'win32':
+ # failed earlier, or cygwin ssh can't do this
+ #
+ return False
+
+ command = ['ssh',
+ '-o','ControlPath %s' % _ssh_sock(),
+ '-p',str(port),
+ '-M',
+ '-N',
+ host]
+ try:
+ Trace(': %s', ' '.join(command))
+ p = subprocess.Popen(command)
+ except Exception, e:
+ _ssh_master = False
+ print >>sys.stderr, \
+ '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
+ % (host,port, str(e))
+ return False
+
+ _ssh_cache[key] = p
+ time.sleep(1)
+ return True
+
+def close_ssh():
+ for key,p in _ssh_cache.iteritems():
+ os.kill(p.pid, SIGTERM)
+ p.wait()
+ _ssh_cache.clear()
+
+ d = _ssh_sock(create=False)
+ if d:
+ try:
+ os.rmdir(os.path.dirname(d))
+ except OSError:
+ pass
+
+URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
+URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/])/')
+
+def _preconnect(url):
+ m = URI_ALL.match(url)
+ if m:
+ scheme = m.group(1)
+ host = m.group(2)
+ if ':' in host:
+ host, port = host.split(':')
+ if scheme in ('ssh', 'git+ssh', 'ssh+git'):
+ return _open_ssh(host, port)
+ return False
+
+ m = URI_SCP.match(url)
+ if m:
+ host = m.group(1)
+ return _open_ssh(host)
+
+
class Remote(object):
"""Configuration options related to a remote.
"""
@@ -344,6 +420,9 @@
self._Get('fetch', all=True))
self._review_protocol = None
+ def PreConnectFetch(self):
+ return _preconnect(self.url)
+
@property
def ReviewProtocol(self):
if self._review_protocol is None: