Merge "Add extend-project tag to support adding groups to an existing project"
diff --git a/docs/manifest-format.txt b/docs/manifest-format.txt
index 65cd70b..d5c6a02 100644
--- a/docs/manifest-format.txt
+++ b/docs/manifest-format.txt
@@ -143,14 +143,14 @@
this value. If this value is not set, projects will use `revision`
by default instead.
-Attribute `sync_j`: Number of parallel jobs to use when synching.
+Attribute `sync-j`: Number of parallel jobs to use when synching.
-Attribute `sync_c`: Set to true to only sync the given Git
+Attribute `sync-c`: Set to true to only sync the given Git
branch (specified in the `revision` attribute) rather than the
-whole ref space. Project elements lacking a sync_c element of
+whole ref space. Project elements lacking a sync-c element of
their own will use this value.
-Attribute `sync_s`: Set to true to also sync sub-projects.
+Attribute `sync-s`: Set to true to also sync sub-projects.
Element manifest-server
@@ -238,11 +238,11 @@
If the project has a parent element, the `name` and `path` here
are the prefixed ones.
-Attribute `sync_c`: Set to true to only sync the given Git
+Attribute `sync-c`: Set to true to only sync the given Git
branch (specified in the `revision` attribute) rather than the
whole ref space.
-Attribute `sync_s`: Set to true to also sync sub-projects.
+Attribute `sync-s`: Set to true to also sync sub-projects.
Attribute `upstream`: Name of the Git branch in which a sha1
can be found. Used when syncing a revision locked manifest in
diff --git a/git_command.py b/git_command.py
index 354fc71..53b3e75 100644
--- a/git_command.py
+++ b/git_command.py
@@ -80,13 +80,13 @@
def version(self):
p = GitCommand(None, ['--version'], capture_stdout=True)
if p.Wait() == 0:
- return p.stdout
+ return p.stdout.decode('utf-8')
return None
def version_tuple(self):
global _git_version
if _git_version is None:
- ver_str = git.version().decode('utf-8')
+ ver_str = git.version()
_git_version = Wrapper().ParseGitVersion(ver_str)
if _git_version is None:
print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
diff --git a/git_config.py b/git_config.py
index 380bdd2..aa07d1b 100644
--- a/git_config.py
+++ b/git_config.py
@@ -216,9 +216,9 @@
"""Resolve any url.*.insteadof references.
"""
for new_url in self.GetSubSections('url'):
- old_url = self.GetString('url.%s.insteadof' % new_url)
- if old_url is not None and url.startswith(old_url):
- return new_url + url[len(old_url):]
+ for old_url in self.GetString('url.%s.insteadof' % new_url, True):
+ if old_url is not None and url.startswith(old_url):
+ return new_url + url[len(old_url):]
return url
@property
@@ -697,7 +697,7 @@
self._Set('merge', self.merge)
else:
- fd = open(self._config.file, 'ab')
+ fd = open(self._config.file, 'a')
try:
fd.write('[branch "%s"]\n' % self.name)
if self.remote:
diff --git a/hooks/commit-msg b/hooks/commit-msg
index 5ca2b11..d8f009b 100755
--- a/hooks/commit-msg
+++ b/hooks/commit-msg
@@ -1,5 +1,4 @@
#!/bin/sh
-# From Gerrit Code Review 2.6
#
# Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
#
@@ -27,7 +26,7 @@
#
add_ChangeId() {
clean_message=`sed -e '
- /^diff --git a\/.*/{
+ /^diff --git .*/{
s///
q
}
@@ -39,6 +38,11 @@
return
fi
+ if test "false" = "`git config --bool --get gerrit.createChangeId`"
+ then
+ return
+ fi
+
# Does Change-Id: already exist? if so, exit (no change).
if grep -i '^Change-Id:' "$MSG" >/dev/null
then
@@ -77,7 +81,7 @@
# Skip the line starting with the diff command and everything after it,
# up to the end of the file, assuming it is only patch data.
# If more than one line before the diff was empty, strip all but one.
- /^diff --git a/ {
+ /^diff --git / {
blankLines = 0
while (getline) { }
next
diff --git a/manifest_xml.py b/manifest_xml.py
index bd1ab69..890c954 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -86,16 +86,20 @@
# about here are:
# * no scheme in the base url, like <hostname:port>
# * persistent-https://
+ # * rpc://
# We handle this by replacing these with obscure protocols
# and then replacing them with the original when we are done.
# gopher -> <none>
# wais -> persistent-https
+ # nntp -> rpc
if manifestUrl.find(':') != manifestUrl.find('/') - 1:
manifestUrl = 'gopher://' + manifestUrl
manifestUrl = re.sub(r'^persistent-https://', 'wais://', manifestUrl)
+ manifestUrl = re.sub(r'^rpc://', 'nntp://', manifestUrl)
url = urllib.parse.urljoin(manifestUrl, url)
url = re.sub(r'^gopher://', '', url)
url = re.sub(r'^wais://', 'persistent-https://', url)
+ url = re.sub(r'^nntp://', 'rpc://', url)
return url
def ToRemoteSpec(self, projectName):
@@ -264,6 +268,8 @@
revision = self.remotes[remoteName].revision or d.revisionExpr
if not revision or revision != p.revisionExpr:
e.setAttribute('revision', p.revisionExpr)
+ if p.upstream and p.upstream != p.revisionExpr:
+ e.setAttribute('upstream', p.upstream)
for c in p.copyfiles:
ce = doc.createElement('copyfile')
diff --git a/project.py b/project.py
index e070351..95403cc 100644
--- a/project.py
+++ b/project.py
@@ -46,7 +46,7 @@
def _lwrite(path, content):
lock = '%s.lock' % path
- fd = open(lock, 'wb')
+ fd = open(lock, 'w')
try:
fd.write(content)
finally:
@@ -1706,6 +1706,7 @@
if command.Wait() != 0:
raise GitError('git archive %s: %s' % (self.name, command.stderr))
+
def _RemoteFetch(self, name=None,
current_branch_only=False,
initial=False,
@@ -1808,19 +1809,30 @@
else:
cmd.append('--tags')
+ spec = []
if not current_branch_only:
# Fetch whole repo
- cmd.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
+ spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
elif tag_name is not None:
- cmd.append('tag')
- cmd.append(tag_name)
+ spec.append('tag')
+ spec.append(tag_name)
else:
branch = self.revisionExpr
if is_sha1:
branch = self.upstream
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
- cmd.append(str((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch)))
+ spec.append(str((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch)))
+ cmd.extend(spec)
+
+ shallowfetch = self.config.GetString('repo.shallowfetch')
+ if shallowfetch and shallowfetch != ' '.join(spec):
+ GitCommand(self, ['fetch', '--unshallow', name] + shallowfetch.split(),
+ bare=True, ssh_proxy=ssh_proxy).Wait()
+ if depth:
+ self.config.SetString('repo.shallowfetch', ' '.join(spec))
+ else:
+ self.config.SetString('repo.shallowfetch', None)
ok = False
for _i in range(2):
@@ -2205,6 +2217,14 @@
if name in symlink_dirs and not os.path.lexists(src):
os.makedirs(src)
+ # If the source file doesn't exist, ensure the destination
+ # file doesn't either.
+ if name in symlink_files and not os.path.lexists(src):
+ try:
+ os.remove(dst)
+ except OSError:
+ pass
+
if name in to_symlink:
os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
elif copy_all and not os.path.islink(dst):
diff --git a/repo b/repo
index 3fd0166..6338483 100755
--- a/repo
+++ b/repo
@@ -738,7 +738,7 @@
try:
_Init(args)
except CloneFailure:
- shutil.rmtree(repodir, ignore_errors=True)
+ shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
sys.exit(1)
repo_main, rel_repo_dir = _FindRepo()
else:
diff --git a/subcmds/branches.py b/subcmds/branches.py
index f714c1e..2902684 100644
--- a/subcmds/branches.py
+++ b/subcmds/branches.py
@@ -47,6 +47,10 @@
return self.current > 0
@property
+ def IsSplitCurrent(self):
+ return self.current != 0 and self.current != len(self.projects)
+
+ @property
def IsPublished(self):
return self.published > 0
@@ -139,10 +143,14 @@
if in_cnt < project_cnt:
fmt = out.write
paths = []
- if in_cnt < project_cnt - in_cnt:
+ non_cur_paths = []
+ if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
in_type = 'in'
for b in i.projects:
- paths.append(b.project.relpath)
+ if not i.IsSplitCurrent or b.current:
+ paths.append(b.project.relpath)
+ else:
+ non_cur_paths.append(b.project.relpath)
else:
fmt = out.notinproject
in_type = 'not in'
@@ -154,13 +162,19 @@
paths.append(p.relpath)
s = ' %s %s' % (in_type, ', '.join(paths))
- if width + 7 + len(s) < 80:
+ if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
+ fmt = out.current if i.IsCurrent else fmt
fmt(s)
else:
fmt(' %s:' % in_type)
+ fmt = out.current if i.IsCurrent else out.write
for p in paths:
out.nl()
fmt(width*' ' + ' %s' % p)
+ fmt = out.write
+ for p in non_cur_paths:
+ out.nl()
+ fmt(width*' ' + ' %s' % p)
else:
out.write(' in all projects')
out.nl()
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 03ebcb2..7771ec1 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -14,7 +14,9 @@
# limitations under the License.
from __future__ import print_function
+import errno
import fcntl
+import multiprocessing
import re
import os
import select
@@ -31,6 +33,7 @@
'log',
]
+
class ForallColoring(Coloring):
def __init__(self, config):
Coloring.__init__(self, config, 'forall')
@@ -132,9 +135,31 @@
g.add_option('-v', '--verbose',
dest='verbose', action='store_true',
help='Show command error messages')
+ g.add_option('-j', '--jobs',
+ dest='jobs', action='store', type='int', default=1,
+ help='number of commands to execute simultaneously')
def WantPager(self, opt):
- return opt.project_header
+ return opt.project_header and opt.jobs == 1
+
+ def _SerializeProject(self, project):
+ """ Serialize a project._GitGetByExec instance.
+
+ project._GitGetByExec is not pickle-able. Instead of trying to pass it
+ around between processes, make a dict ourselves containing only the
+ attributes that we need.
+
+ """
+ return {
+ 'name': project.name,
+ 'relpath': project.relpath,
+ 'remote_name': project.remote.name,
+ 'lrev': project.GetRevisionId(),
+ 'rrev': project.revisionExpr,
+ 'annotations': dict((a.name, a.value) for a in project.annotations),
+ 'gitdir': project.gitdir,
+ 'worktree': project.worktree,
+ }
def Execute(self, opt, args):
if not opt.command:
@@ -173,11 +198,7 @@
# pylint: enable=W0631
mirror = self.manifest.IsMirror
- out = ForallColoring(self.manifest.manifestProject.config)
- out.redirect(sys.stdout)
-
rc = 0
- first = True
if not opt.regex:
projects = self.GetProjects(args)
@@ -186,113 +207,156 @@
os.environ['REPO_COUNT'] = str(len(projects))
- for (cnt, project) in enumerate(projects):
- env = os.environ.copy()
- def setenv(name, val):
- if val is None:
- val = ''
- env[name] = val.encode()
-
- setenv('REPO_PROJECT', project.name)
- setenv('REPO_PATH', project.relpath)
- setenv('REPO_REMOTE', project.remote.name)
- setenv('REPO_LREV', project.GetRevisionId())
- setenv('REPO_RREV', project.revisionExpr)
- setenv('REPO_I', str(cnt + 1))
- for a in project.annotations:
- setenv("REPO__%s" % (a.name), a.value)
-
- if mirror:
- setenv('GIT_DIR', project.gitdir)
- cwd = project.gitdir
- else:
- cwd = project.worktree
-
- if not os.path.exists(cwd):
- if (opt.project_header and opt.verbose) \
- or not opt.project_header:
- print('skipping %s/' % project.relpath, file=sys.stderr)
- continue
-
- if opt.project_header:
- stdin = subprocess.PIPE
- stdout = subprocess.PIPE
- stderr = subprocess.PIPE
- else:
- stdin = None
- stdout = None
- stderr = None
-
- p = subprocess.Popen(cmd,
- cwd = cwd,
- shell = shell,
- env = env,
- stdin = stdin,
- stdout = stdout,
- stderr = stderr)
-
- if opt.project_header:
- class sfd(object):
- def __init__(self, fd, dest):
- self.fd = fd
- self.dest = dest
- def fileno(self):
- return self.fd.fileno()
-
- empty = True
- errbuf = ''
-
- p.stdin.close()
- s_in = [sfd(p.stdout, sys.stdout),
- sfd(p.stderr, sys.stderr)]
-
- for s in s_in:
- flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
- fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
- while s_in:
- in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
- for s in in_ready:
- buf = s.fd.read(4096)
- if not buf:
- s.fd.close()
- s_in.remove(s)
- continue
-
- if not opt.verbose:
- if s.fd != p.stdout:
- errbuf += buf
- continue
-
- if empty:
- if first:
- first = False
- else:
- out.nl()
-
- if mirror:
- project_header_path = project.name
- else:
- project_header_path = project.relpath
- out.project('project %s/', project_header_path)
- out.nl()
- out.flush()
- if errbuf:
- sys.stderr.write(errbuf)
- sys.stderr.flush()
- errbuf = ''
- empty = False
-
- s.dest.write(buf)
- s.dest.flush()
-
- r = p.wait()
- if r != 0:
- if r != rc:
- rc = r
- if opt.abort_on_errors:
- print("error: %s: Aborting due to previous error" % project.relpath,
- file=sys.stderr)
- sys.exit(r)
+ pool = multiprocessing.Pool(opt.jobs)
+ try:
+ config = self.manifest.manifestProject.config
+ results_it = pool.imap(
+ DoWorkWrapper,
+ [[mirror, opt, cmd, shell, cnt, config, self._SerializeProject(p)]
+ for cnt, p in enumerate(projects)]
+ )
+ pool.close()
+ for r in results_it:
+ rc = rc or r
+ if r != 0 and opt.abort_on_errors:
+ raise Exception('Aborting due to previous error')
+ except (KeyboardInterrupt, WorkerKeyboardInterrupt):
+ # Catch KeyboardInterrupt raised inside and outside of workers
+ print('Interrupted - terminating the pool')
+ pool.terminate()
+ rc = rc or errno.EINTR
+ except Exception as e:
+ # Catch any other exceptions raised
+ print('Got an error, terminating the pool: %r' % e,
+ file=sys.stderr)
+ pool.terminate()
+ rc = rc or getattr(e, 'errno', 1)
+ finally:
+ pool.join()
if rc != 0:
sys.exit(rc)
+
+
+class WorkerKeyboardInterrupt(Exception):
+ """ Keyboard interrupt exception for worker processes. """
+ pass
+
+
+def DoWorkWrapper(args):
+ """ A wrapper around the DoWork() method.
+
+ Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
+ ``Exception``-based exception to stop it flooding the console with stacktraces
+ and making the parent hang indefinitely.
+
+ """
+ project = args.pop()
+ try:
+ return DoWork(project, *args)
+ except KeyboardInterrupt:
+ print('%s: Worker interrupted' % project['name'])
+ raise WorkerKeyboardInterrupt()
+
+
+def DoWork(project, mirror, opt, cmd, shell, cnt, config):
+ env = os.environ.copy()
+ def setenv(name, val):
+ if val is None:
+ val = ''
+ env[name] = val.encode()
+
+ setenv('REPO_PROJECT', project['name'])
+ setenv('REPO_PATH', project['relpath'])
+ setenv('REPO_REMOTE', project['remote_name'])
+ setenv('REPO_LREV', project['lrev'])
+ setenv('REPO_RREV', project['rrev'])
+ setenv('REPO_I', str(cnt + 1))
+ for name in project['annotations']:
+ setenv("REPO__%s" % (name), project['annotations'][name])
+
+ if mirror:
+ setenv('GIT_DIR', project['gitdir'])
+ cwd = project['gitdir']
+ else:
+ cwd = project['worktree']
+
+ if not os.path.exists(cwd):
+ if (opt.project_header and opt.verbose) \
+ or not opt.project_header:
+ print('skipping %s/' % project['relpath'], file=sys.stderr)
+ return
+
+ if opt.project_header:
+ stdin = subprocess.PIPE
+ stdout = subprocess.PIPE
+ stderr = subprocess.PIPE
+ else:
+ stdin = None
+ stdout = None
+ stderr = None
+
+ p = subprocess.Popen(cmd,
+ cwd=cwd,
+ shell=shell,
+ env=env,
+ stdin=stdin,
+ stdout=stdout,
+ stderr=stderr)
+
+ if opt.project_header:
+ out = ForallColoring(config)
+ out.redirect(sys.stdout)
+ class sfd(object):
+ def __init__(self, fd, dest):
+ self.fd = fd
+ self.dest = dest
+ def fileno(self):
+ return self.fd.fileno()
+
+ empty = True
+ errbuf = ''
+
+ p.stdin.close()
+ s_in = [sfd(p.stdout, sys.stdout),
+ sfd(p.stderr, sys.stderr)]
+
+ for s in s_in:
+ flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+ fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+ while s_in:
+ in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
+ for s in in_ready:
+ buf = s.fd.read(4096)
+ if not buf:
+ s.fd.close()
+ s_in.remove(s)
+ continue
+
+ if not opt.verbose:
+ if s.fd != p.stdout:
+ errbuf += buf
+ continue
+
+ if empty and out:
+ if not cnt == 0:
+ out.nl()
+
+ if mirror:
+ project_header_path = project['name']
+ else:
+ project_header_path = project['relpath']
+ out.project('project %s/', project_header_path)
+ out.nl()
+ out.flush()
+ if errbuf:
+ sys.stderr.write(errbuf)
+ sys.stderr.flush()
+ errbuf = ''
+ empty = False
+
+ s.dest.write(buf)
+ s.dest.flush()
+
+ r = p.wait()
+ return r