Merge "repo: Support multiple branches for the same project."
diff --git a/command.py b/command.py
index 287f4d3..207ef46 100644
--- a/command.py
+++ b/command.py
@@ -129,7 +129,7 @@
def GetProjects(self, args, missing_ok=False, submodules_ok=False):
"""A list of projects that match the arguments.
"""
- all_projects = self.manifest.projects
+ all_projects_list = self.manifest.projects
result = []
mp = self.manifest.manifestProject
@@ -140,7 +140,6 @@
groups = [x for x in re.split(r'[,\s]+', groups) if x]
if not args:
- all_projects_list = list(all_projects.values())
derived_projects = {}
for project in all_projects_list:
if submodules_ok or project.sync_s:
@@ -152,12 +151,12 @@
project.MatchesGroups(groups)):
result.append(project)
else:
- self._ResetPathToProjectMap(all_projects.values())
+ self._ResetPathToProjectMap(all_projects_list)
for arg in args:
- project = all_projects.get(arg)
+ projects = self.manifest.GetProjectsWithName(arg)
- if not project:
+ if not projects:
path = os.path.abspath(arg).replace('\\', '/')
project = self._GetProjectByPath(path)
@@ -172,14 +171,19 @@
if search_again:
project = self._GetProjectByPath(path) or project
- if not project:
- raise NoSuchProjectError(arg)
- if not missing_ok and not project.Exists:
- raise NoSuchProjectError(arg)
- if not project.MatchesGroups(groups):
- raise InvalidProjectGroupsError(arg)
+ if project:
+ projects = [project]
- result.append(project)
+ if not projects:
+ raise NoSuchProjectError(arg)
+
+ for project in projects:
+ if not missing_ok and not project.Exists:
+ raise NoSuchProjectError(arg)
+ if not project.MatchesGroups(groups):
+ raise InvalidProjectGroupsError(arg)
+
+ result.extend(projects)
def _getpath(x):
return x.relpath
diff --git a/manifest_xml.py b/manifest_xml.py
index 785976b..c5f3bcc 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -215,8 +215,9 @@
root.appendChild(doc.createTextNode(''))
def output_projects(parent, parent_node, projects):
- for p in projects:
- output_project(parent, parent_node, self.projects[p])
+ for project_name in projects:
+ for project in self._projects[project_name]:
+ output_project(parent, parent_node, project)
def output_project(parent, parent_node, p):
if not p.MatchesGroups(groups):
@@ -277,13 +278,11 @@
e.setAttribute('sync-s', 'true')
if p.subprojects:
- sort_projects = list(sorted([subp.name for subp in p.subprojects]))
- output_projects(p, e, sort_projects)
+ subprojects = set(subp.name for subp in p.subprojects)
+ output_projects(p, e, list(sorted(subprojects)))
- sort_projects = list(sorted([key for key, value in self.projects.items()
- if not value.parent]))
- sort_projects.sort()
- output_projects(None, root, sort_projects)
+ projects = set(p.name for p in self._paths.values() if not p.parent)
+ output_projects(None, root, list(sorted(projects)))
if self._repo_hooks_project:
root.appendChild(doc.createTextNode(''))
@@ -296,9 +295,14 @@
doc.writexml(fd, '', ' ', '\n', 'UTF-8')
@property
+ def paths(self):
+ self._Load()
+ return self._paths
+
+ @property
def projects(self):
self._Load()
- return self._projects
+ return self._paths.values()
@property
def remotes(self):
@@ -336,6 +340,7 @@
def _Unload(self):
self._loaded = False
self._projects = {}
+ self._paths = {}
self._remotes = {}
self._default = None
self._repo_hooks_project = None
@@ -467,11 +472,17 @@
self._manifest_server = url
def recursively_add_projects(project):
- if self._projects.get(project.name):
+ projects = self._projects.setdefault(project.name, [])
+ if project.relpath is None:
raise ManifestParseError(
- 'duplicate project %s in %s' %
+ 'missing path for %s in %s' %
(project.name, self.manifestFile))
- self._projects[project.name] = project
+ if project.relpath in self._paths:
+ raise ManifestParseError(
+ 'duplicate path %s in %s' %
+ (project.relpath, self.manifestFile))
+ self._paths[project.relpath] = project
+ projects.append(project)
for subproject in project.subprojects:
recursively_add_projects(subproject)
@@ -492,12 +503,18 @@
# Store a reference to the Project.
try:
- self._repo_hooks_project = self._projects[repo_hooks_project]
+ repo_hooks_projects = self._projects[repo_hooks_project]
except KeyError:
raise ManifestParseError(
'project %s not found for repo-hooks' %
(repo_hooks_project))
+ if len(repo_hooks_projects) != 1:
+ raise ManifestParseError(
+ 'internal error parsing repo-hooks in %s' %
+ (self.manifestFile))
+ self._repo_hooks_project = repo_hooks_projects[0]
+
# Store the enabled hooks in the Project object.
self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
if node.nodeName == 'remove-project':
@@ -544,11 +561,12 @@
name = name,
remote = remote.ToRemoteSpec(name),
gitdir = gitdir,
+ objdir = gitdir,
worktree = None,
relpath = None,
revisionExpr = m.revisionExpr,
revisionId = None)
- self._projects[project.name] = project
+ self._projects[project.name] = [project]
def _ParseRemote(self, node):
"""
@@ -708,9 +726,10 @@
groups = [x for x in re.split(r'[,\s]+', groups) if x]
if parent is None:
- relpath, worktree, gitdir = self.GetProjectPaths(name, path)
+ relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
else:
- relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path)
+ relpath, worktree, gitdir, objdir = \
+ self.GetSubprojectPaths(parent, name, path)
default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
groups.extend(set(default_groups).difference(groups))
@@ -723,6 +742,7 @@
name = name,
remote = remote.ToRemoteSpec(name),
gitdir = gitdir,
+ objdir = objdir,
worktree = worktree,
relpath = relpath,
revisionExpr = revisionExpr,
@@ -751,10 +771,15 @@
if self.IsMirror:
worktree = None
gitdir = os.path.join(self.topdir, '%s.git' % name)
+ objdir = gitdir
else:
worktree = os.path.join(self.topdir, path).replace('\\', '/')
gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
- return relpath, worktree, gitdir
+ objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+ return relpath, worktree, gitdir, objdir
+
+ def GetProjectsWithName(self, name):
+ return self._projects.get(name, [])
def GetSubprojectName(self, parent, submodule_path):
return os.path.join(parent.name, submodule_path)
@@ -765,14 +790,15 @@
def _UnjoinRelpath(self, parent_relpath, relpath):
return os.path.relpath(relpath, parent_relpath)
- def GetSubprojectPaths(self, parent, path):
+ def GetSubprojectPaths(self, parent, name, path):
relpath = self._JoinRelpath(parent.relpath, path)
gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
+ objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
if self.IsMirror:
worktree = None
else:
worktree = os.path.join(parent.worktree, path).replace('\\', '/')
- return relpath, worktree, gitdir
+ return relpath, worktree, gitdir, objdir
def _ParseCopyFile(self, project, node):
src = self._reqatt(node, 'src')
diff --git a/project.py b/project.py
index eb8bad3..46f3b8f 100644
--- a/project.py
+++ b/project.py
@@ -488,6 +488,7 @@
name,
remote,
gitdir,
+ objdir,
worktree,
relpath,
revisionExpr,
@@ -508,6 +509,7 @@
name: The `name` attribute of manifest.xml's project element.
remote: RemoteSpec object specifying its remote's properties.
gitdir: Absolute path of git directory.
+ objdir: Absolute path of directory to store git objects.
worktree: Absolute path of git working tree.
relpath: Relative path of git working tree to repo's top directory.
revisionExpr: The `revision` attribute of manifest.xml's project element.
@@ -526,6 +528,7 @@
self.name = name
self.remote = remote
self.gitdir = gitdir.replace('\\', '/')
+ self.objdir = objdir.replace('\\', '/')
if worktree:
self.worktree = worktree.replace('\\', '/')
else:
@@ -558,11 +561,12 @@
defaults = self.manifest.globalConfig)
if self.worktree:
- self.work_git = self._GitGetByExec(self, bare=False)
+ self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
else:
self.work_git = None
- self.bare_git = self._GitGetByExec(self, bare=True)
+ self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
self.bare_ref = GitRefs(gitdir)
+ self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
self.dest_branch = dest_branch
# This will be filled in if a project is later identified to be the
@@ -1117,6 +1121,7 @@
"""Perform only the local IO portion of the sync process.
Network access is not required.
"""
+ self._InitWorkTree()
all_refs = self.bare_ref.all
self.CleanPublishedCache(all_refs)
revid = self.GetRevisionId(all_refs)
@@ -1125,7 +1130,6 @@
self._FastForward(revid)
self._CopyFiles()
- self._InitWorkTree()
head = self.work_git.GetHead()
if head.startswith(R_HEADS):
branch = head[len(R_HEADS):]
@@ -1592,11 +1596,13 @@
return result
for rev, path, url in self._GetSubmodules():
name = self.manifest.GetSubprojectName(self, path)
- project = self.manifest.projects.get(name)
+ relpath, worktree, gitdir, objdir = \
+ self.manifest.GetSubprojectPaths(self, name, path)
+ project = self.manifest.paths.get(relpath)
if project:
result.extend(project.GetDerivedSubprojects())
continue
- relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path)
+
remote = RemoteSpec(self.remote.name,
url = url,
review = self.remote.review)
@@ -1604,6 +1610,7 @@
name = name,
remote = remote,
gitdir = gitdir,
+ objdir = objdir,
worktree = worktree,
relpath = relpath,
revisionExpr = self.revisionExpr,
@@ -1966,8 +1973,17 @@
def _InitGitDir(self, mirror_git=None):
if not os.path.exists(self.gitdir):
- os.makedirs(self.gitdir)
- self.bare_git.init()
+
+ # Initialize the bare repository, which contains all of the objects.
+ if not os.path.exists(self.objdir):
+ os.makedirs(self.objdir)
+ self.bare_objdir.init()
+
+ # If we have a separate directory to hold refs, initialize it as well.
+ if self.objdir != self.gitdir:
+ os.makedirs(self.gitdir)
+ self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
+ copy_all=True)
mp = self.manifest.manifestProject
ref_dir = mp.config.GetString('repo.reference') or ''
@@ -2083,33 +2099,61 @@
msg = 'manifest set to %s' % self.revisionExpr
self.bare_git.symbolic_ref('-m', msg, ref, dst)
+ def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
+ """Update |dotgit| to reference |gitdir|, using symlinks where possible.
+
+ Args:
+ gitdir: The bare git repository. Must already be initialized.
+ dotgit: The repository you would like to initialize.
+ share_refs: If true, |dotgit| will store its refs under |gitdir|.
+ Only one work tree can store refs under a given |gitdir|.
+ copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
+ This saves you the effort of initializing |dotgit| yourself.
+ """
+ # These objects can be shared between several working trees.
+ symlink_files = ['description', 'info']
+ symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
+ if share_refs:
+ # These objects can only be used by a single working tree.
+ symlink_files += ['config', 'packed-refs']
+ symlink_dirs += ['logs', 'refs']
+ to_symlink = symlink_files + symlink_dirs
+
+ to_copy = []
+ if copy_all:
+ to_copy = os.listdir(gitdir)
+
+ for name in set(to_copy).union(to_symlink):
+ try:
+ src = os.path.realpath(os.path.join(gitdir, name))
+ dst = os.path.realpath(os.path.join(dotgit, name))
+
+ if os.path.lexists(dst) and not os.path.islink(dst):
+ raise GitError('cannot overwrite a local work tree')
+
+ # If the source dir doesn't exist, create an empty dir.
+ if name in symlink_dirs and not os.path.lexists(src):
+ os.makedirs(src)
+
+ if name in to_symlink:
+ os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
+ elif copy_all and not os.path.islink(dst):
+ if os.path.isdir(src):
+ shutil.copytree(src, dst)
+ elif os.path.isfile(src):
+ shutil.copy(src, dst)
+ except OSError as e:
+ if e.errno == errno.EPERM:
+ raise GitError('filesystem must support symlinks')
+ else:
+ raise
+
def _InitWorkTree(self):
dotgit = os.path.join(self.worktree, '.git')
if not os.path.exists(dotgit):
os.makedirs(dotgit)
-
- for name in ['config',
- 'description',
- 'hooks',
- 'info',
- 'logs',
- 'objects',
- 'packed-refs',
- 'refs',
- 'rr-cache',
- 'svn']:
- try:
- src = os.path.join(self.gitdir, name)
- dst = os.path.join(dotgit, name)
- if os.path.islink(dst) or not os.path.exists(dst):
- os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
- else:
- raise GitError('cannot overwrite a local work tree')
- except OSError as e:
- if e.errno == errno.EPERM:
- raise GitError('filesystem must support symlinks')
- else:
- raise
+ self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
+ copy_all=False)
_lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
@@ -2119,14 +2163,10 @@
if GitCommand(self, cmd).Wait() != 0:
raise GitError("cannot initialize work tree")
- rr_cache = os.path.join(self.gitdir, 'rr-cache')
- if not os.path.exists(rr_cache):
- os.makedirs(rr_cache)
-
self._CopyFiles()
def _gitdir_path(self, path):
- return os.path.join(self.gitdir, path)
+ return os.path.realpath(os.path.join(self.gitdir, path))
def _revlist(self, *args, **kw):
a = []
@@ -2139,9 +2179,10 @@
return self.bare_ref.all
class _GitGetByExec(object):
- def __init__(self, project, bare):
+ def __init__(self, project, bare, gitdir):
self._project = project
self._bare = bare
+ self._gitdir = gitdir
def LsOthers(self):
p = GitCommand(self._project,
@@ -2150,6 +2191,7 @@
'--others',
'--exclude-standard'],
bare = False,
+ gitdir=self._gitdir,
capture_stdout = True,
capture_stderr = True)
if p.Wait() == 0:
@@ -2165,6 +2207,7 @@
cmd.extend(args)
p = GitCommand(self._project,
cmd,
+ gitdir=self._gitdir,
bare = False,
capture_stdout = True,
capture_stderr = True)
@@ -2274,6 +2317,7 @@
p = GitCommand(self._project,
cmdv,
bare = self._bare,
+ gitdir=self._gitdir,
capture_stdout = True,
capture_stderr = True)
r = []
@@ -2326,6 +2370,7 @@
p = GitCommand(self._project,
cmdv,
bare = self._bare,
+ gitdir=self._gitdir,
capture_stdout = True,
capture_stderr = True)
if p.Wait() != 0:
@@ -2459,6 +2504,7 @@
manifest = manifest,
name = name,
gitdir = gitdir,
+ objdir = gitdir,
worktree = worktree,
remote = RemoteSpec('origin'),
relpath = '.repo/%s' % name,
diff --git a/subcmds/rebase.py b/subcmds/rebase.py
index b9a7774..1bdc1f0 100644
--- a/subcmds/rebase.py
+++ b/subcmds/rebase.py
@@ -62,6 +62,9 @@
if opt.interactive and not one_project:
print('error: interactive rebase not supported with multiple projects',
file=sys.stderr)
+ if len(args) == 1:
+ print('note: project %s is mapped to more than one path' % (args[0],),
+ file=sys.stderr)
return -1
for project in all_projects:
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 0279ff6..5e7385d 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -219,9 +219,25 @@
dest='repo_upgraded', action='store_true',
help=SUPPRESS_HELP)
- def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
+ def _FetchProjectList(self, opt, projects, *args):
"""Main function of the fetch threads when jobs are > 1.
+ Delegates most of the work to _FetchHelper.
+
+ Args:
+ opt: Program options returned from optparse. See _Options().
+ projects: Projects to fetch.
+ *args: Remaining arguments to pass to _FetchHelper. See the
+ _FetchHelper docstring for details.
+ """
+ for project in projects:
+ success = self._FetchHelper(opt, project, *args)
+ if not success and not opt.force_broken:
+ break
+
+ def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
+ """Fetch git objects for a single project.
+
Args:
opt: Program options returned from optparse. See _Options().
project: Project object for the project to fetch.
@@ -235,6 +251,9 @@
can be started up.
err_event: We'll set this event in the case of an error (after printing
out info about the error).
+
+ Returns:
+ Whether the fetch was successful.
"""
# We'll set to true once we've locked the lock.
did_lock = False
@@ -281,6 +300,8 @@
lock.release()
sem.release()
+ return success
+
def _Fetch(self, projects, opt):
fetched = set()
pm = Progress('Fetching projects', len(projects))
@@ -304,20 +325,24 @@
else:
sys.exit(1)
else:
+ objdir_project_map = dict()
+ for project in projects:
+ objdir_project_map.setdefault(project.objdir, []).append(project)
+
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
err_event = _threading.Event()
- for project in projects:
+ for project_list in objdir_project_map.values():
# Check for any errors before starting any new threads.
# ...we'll let existing threads finish, though.
if err_event.isSet():
break
sem.acquire()
- t = _threading.Thread(target = self._FetchHelper,
+ t = _threading.Thread(target = self._FetchProjectList,
args = (opt,
- project,
+ project_list,
lock,
fetched,
pm,
@@ -345,6 +370,10 @@
return fetched
def _GCProjects(self, projects):
+ gitdirs = {}
+ for project in projects:
+ gitdirs[project.gitdir] = project.bare_git
+
has_dash_c = git_require((1, 7, 2))
if multiprocessing and has_dash_c:
cpu_count = multiprocessing.cpu_count()
@@ -353,8 +382,8 @@
jobs = min(self.jobs, cpu_count)
if jobs < 2:
- for project in projects:
- project.bare_git.gc('--auto')
+ for bare_git in gitdirs.values():
+ bare_git.gc('--auto')
return
config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
@@ -363,10 +392,10 @@
sem = _threading.Semaphore(jobs)
err_event = _threading.Event()
- def GC(project):
+ def GC(bare_git):
try:
try:
- project.bare_git.gc('--auto', config=config)
+ bare_git.gc('--auto', config=config)
except GitError:
err_event.set()
except:
@@ -375,11 +404,11 @@
finally:
sem.release()
- for project in projects:
+ for bare_git in gitdirs.values():
if err_event.isSet():
break
sem.acquire()
- t = _threading.Thread(target=GC, args=(project,))
+ t = _threading.Thread(target=GC, args=(bare_git,))
t.daemon = True
threads.add(t)
t.start()
@@ -419,12 +448,13 @@
if path not in new_project_paths:
# If the path has already been deleted, we don't need to do it
if os.path.exists(self.manifest.topdir + '/' + path):
+ gitdir = os.path.join(self.manifest.topdir, path, '.git')
project = Project(
manifest = self.manifest,
name = path,
remote = RemoteSpec('origin'),
- gitdir = os.path.join(self.manifest.topdir,
- path, '.git'),
+ gitdir = gitdir,
+ objdir = gitdir,
worktree = os.path.join(self.manifest.topdir, path),
relpath = path,
revisionExpr = 'HEAD',
diff --git a/subcmds/upload.py b/subcmds/upload.py
index 7f7585a..5621240 100644
--- a/subcmds/upload.py
+++ b/subcmds/upload.py
@@ -432,8 +432,10 @@
hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
self.manifest.topdir, abort_if_user_denies=True)
pending_proj_names = [project.name for (project, avail) in pending]
+ pending_worktrees = [project.worktree for (project, avail) in pending]
try:
- hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
+ hook.Run(opt.allow_all_hooks, project_list=pending_proj_names,
+ worktree_list=pending_worktrees)
except HookError as e:
print("ERROR: %s" % str(e), file=sys.stderr)
return