Introduce manifest format using git submodules
If a manifest top level directory contains '.gitmodules' we now
assume this is a git module format manifest and switch to using
that code, rather than the legacy XML based manifest.
At the same time, we move the bare repository for a project from
$TOP/.repo/projects/$REPO_PATH.git to be $REPO_NAME.git instead.
This makes it easier for us to later support a repo init from an
existing work tree, as we can more accurately predict the path of
the project's repository in the workspace. It also means that the
$TOP/.repo/projects/ directory is layed out like a mirror would be.
Signed-off-by: Shawn O. Pearce <sop@google.com>
diff --git a/manifest_submodule.py b/manifest_submodule.py
new file mode 100644
index 0000000..92f187a
--- /dev/null
+++ b/manifest_submodule.py
@@ -0,0 +1,474 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+import shutil
+
+from error import GitError
+from error import ManifestParseError
+from git_command import GitCommand
+from git_config import GitConfig
+from git_config import IsId
+from manifest import Manifest
+from progress import Progress
+from project import RemoteSpec
+from project import Project
+from project import MetaProject
+from project import R_HEADS
+from project import HEAD
+from project import _lwrite
+
+import manifest_xml
+
+GITLINK = '160000'
+
+def _rmdir(dir, top):
+ while dir != top:
+ try:
+ os.rmdir(dir)
+ except OSError:
+ break
+ dir = os.path.dirname(dir)
+
+def _rmref(gitdir, ref):
+ os.remove(os.path.join(gitdir, ref))
+ log = os.path.join(gitdir, 'logs', ref)
+ if os.path.exists(log):
+ os.remove(log)
+ _rmdir(os.path.dirname(log), gitdir)
+
+def _has_gitmodules(d):
+ return os.path.exists(os.path.join(d, '.gitmodules'))
+
+class SubmoduleManifest(Manifest):
+ """manifest from .gitmodules file"""
+
+ @classmethod
+ def Is(cls, repodir):
+ return _has_gitmodules(os.path.dirname(repodir)) \
+ or _has_gitmodules(os.path.join(repodir, 'manifest')) \
+ or _has_gitmodules(os.path.join(repodir, 'manifests'))
+
+ @classmethod
+ def IsBare(cls, p):
+ try:
+ p.bare_git.cat_file('-e', '%s:.gitmodules' % p.GetRevisionId())
+ except GitError:
+ return False
+ return True
+
+ def __init__(self, repodir):
+ Manifest.__init__(self, repodir)
+
+ gitdir = os.path.join(repodir, 'manifest.git')
+ config = GitConfig.ForRepository(gitdir = gitdir)
+
+ if config.GetBoolean('repo.mirror'):
+ worktree = os.path.join(repodir, 'manifest')
+ relpath = None
+ else:
+ worktree = self.topdir
+ relpath = '.'
+
+ self.manifestProject = MetaProject(self, '__manifest__',
+ gitdir = gitdir,
+ worktree = worktree,
+ relpath = relpath)
+ self._modules = GitConfig(os.path.join(worktree, '.gitmodules'),
+ pickleFile = os.path.join(
+ repodir, '.repopickle_gitmodules'
+ ))
+ self._review = GitConfig(os.path.join(worktree, '.review'),
+ pickleFile = os.path.join(
+ repodir, '.repopickle_review'
+ ))
+ self._Unload()
+
+ @property
+ def projects(self):
+ self._Load()
+ return self._projects
+
+ def InitBranch(self):
+ m = self.manifestProject
+ if m.CurrentBranch is None:
+ b = m.revisionExpr
+ if b.startswith(R_HEADS):
+ b = b[len(R_HEADS):]
+ return m.StartBranch(b)
+ return True
+
+ def SetMRefs(self, project):
+ if project.revisionId is None:
+ # Special project, e.g. the manifest or repo executable.
+ #
+ return
+
+ ref = 'refs/remotes/m'
+ cur = project.bare_ref.get(ref)
+ exp = project.revisionId
+ if cur != exp:
+ msg = 'manifest set to %s' % exp
+ project.bare_git.UpdateRef(ref, exp, message = msg, detach = True)
+
+ ref = 'refs/remotes/m-revision'
+ cur = project.bare_ref.symref(ref)
+ exp = project.revisionExpr
+ if exp is None:
+ if cur:
+ _rmref(project.gitdir, ref)
+ elif cur != exp:
+ remote = project.GetRemote(project.remote.name)
+ dst = remote.ToLocal(exp)
+ msg = 'manifest set to %s (%s)' % (exp, dst)
+ project.bare_git.symbolic_ref('-m', msg, ref, dst)
+
+ def Upgrade_Local(self, old):
+ if isinstance(old, manifest_xml.XmlManifest):
+ self.FromXml_Local_1(old, checkout=True)
+ self.FromXml_Local_2(old)
+ else:
+ raise ManifestParseError, 'cannot upgrade manifest'
+
+ def FromXml_Local_1(self, old, checkout):
+ os.rename(old.manifestProject.gitdir,
+ os.path.join(old.repodir, 'manifest.git'))
+
+ oldmp = old.manifestProject
+ oldBranch = oldmp.CurrentBranch
+ b = oldmp.GetBranch(oldBranch).merge
+ if not b:
+ raise ManifestParseError, 'cannot upgrade manifest'
+ if b.startswith(R_HEADS):
+ b = b[len(R_HEADS):]
+
+ newmp = self.manifestProject
+ self._CleanOldMRefs(newmp)
+ if oldBranch != b:
+ newmp.bare_git.branch('-m', oldBranch, b)
+ newmp.config.ClearCache()
+
+ old_remote = newmp.GetBranch(b).remote.name
+ act_remote = self._GuessRemoteName(old)
+ if old_remote != act_remote:
+ newmp.bare_git.remote('rename', old_remote, act_remote)
+ newmp.config.ClearCache()
+ newmp.remote.name = act_remote
+ print >>sys.stderr, "Assuming remote named '%s'" % act_remote
+
+ if checkout:
+ for p in old.projects.values():
+ for c in p.copyfiles:
+ if os.path.exists(c.abs_dest):
+ os.remove(c.abs_dest)
+ newmp._InitWorkTree()
+ else:
+ newmp._LinkWorkTree()
+
+ _lwrite(os.path.join(newmp.worktree,'.git',HEAD),
+ 'ref: refs/heads/%s\n' % b)
+
+ def _GuessRemoteName(self, old):
+ used = {}
+ for p in old.projects.values():
+ n = p.remote.name
+ used[n] = used.get(n, 0) + 1
+
+ remote_name = 'origin'
+ remote_used = 0
+ for n in used.keys():
+ if remote_used < used[n]:
+ remote_used = used[n]
+ remote_name = n
+ return remote_name
+
+ def FromXml_Local_2(self, old):
+ shutil.rmtree(old.manifestProject.worktree)
+ os.remove(old._manifestFile)
+
+ my_remote = self._Remote().name
+ new_base = os.path.join(self.repodir, 'projects')
+ old_base = os.path.join(self.repodir, 'projects.old')
+ os.rename(new_base, old_base)
+ os.makedirs(new_base)
+
+ info = []
+ pm = Progress('Converting projects', len(self.projects))
+ for p in self.projects.values():
+ pm.update()
+
+ old_p = old.projects.get(p.name)
+ old_gitdir = os.path.join(old_base, '%s.git' % p.relpath)
+ if not os.path.isdir(old_gitdir):
+ continue
+
+ parent = os.path.dirname(p.gitdir)
+ if not os.path.isdir(parent):
+ os.makedirs(parent)
+ os.rename(old_gitdir, p.gitdir)
+ _rmdir(os.path.dirname(old_gitdir), self.repodir)
+
+ if not os.path.isdir(p.worktree):
+ os.makedirs(p.worktree)
+
+ if os.path.isdir(os.path.join(p.worktree, '.git')):
+ p._LinkWorkTree(relink=True)
+
+ self._CleanOldMRefs(p)
+ if old_p and old_p.remote.name != my_remote:
+ info.append("%s/: renamed remote '%s' to '%s'" \
+ % (p.relpath, old_p.remote.name, my_remote))
+ p.bare_git.remote('rename', old_p.remote.name, my_remote)
+ p.config.ClearCache()
+
+ self.SetMRefs(p)
+ pm.end()
+ for i in info:
+ print >>sys.stderr, i
+
+ def _CleanOldMRefs(self, p):
+ all_refs = p._allrefs
+ for ref in all_refs.keys():
+ if ref.startswith(manifest_xml.R_M):
+ if p.bare_ref.symref(ref) != '':
+ _rmref(p.gitdir, ref)
+ else:
+ p.bare_git.DeleteRef(ref, all_refs[ref])
+
+ def FromXml_Definition(self, old):
+ """Convert another manifest representation to this one.
+ """
+ mp = self.manifestProject
+ gm = self._modules
+ gr = self._review
+
+ fd = open(os.path.join(mp.worktree, '.gitignore'), 'ab')
+ fd.write('/.repo\n')
+ fd.close()
+
+ sort_projects = list(old.projects.keys())
+ sort_projects.sort()
+
+ b = mp.GetBranch(mp.CurrentBranch).merge
+ if b.startswith(R_HEADS):
+ b = b[len(R_HEADS):]
+
+ info = []
+ pm = Progress('Converting manifest', len(sort_projects))
+ for p in sort_projects:
+ pm.update()
+ p = old.projects[p]
+
+ gm.SetString('submodule.%s.path' % p.name, p.relpath)
+ gm.SetString('submodule.%s.url' % p.name, p.remote.url)
+
+ if gr.GetString('review.url') is None:
+ gr.SetString('review.url', p.remote.review)
+ elif gr.GetString('review.url') != p.remote.review:
+ gr.SetString('review.%s.url' % p.name, p.remote.review)
+
+ r = p.revisionExpr
+ if r and not IsId(r):
+ if r.startswith(R_HEADS):
+ r = r[len(R_HEADS):]
+ if r == b:
+ r = '.'
+ gm.SetString('submodule.%s.revision' % p.name, r)
+
+ for c in p.copyfiles:
+ info.append('Moved %s out of %s' % (c.src, p.relpath))
+ c._Copy()
+ p.work_git.rm(c.src)
+ mp.work_git.add(c.dest)
+
+ self.SetRevisionId(p.relpath, p.GetRevisionId())
+ mp.work_git.add('.gitignore', '.gitmodules', '.review')
+ pm.end()
+ for i in info:
+ print >>sys.stderr, i
+
+ def _Unload(self):
+ self._loaded = False
+ self._projects = {}
+ self._revisionIds = None
+ self.branch = None
+
+ def _Load(self):
+ if not self._loaded:
+ f = os.path.join(self.repodir, manifest_xml.LOCAL_MANIFEST_NAME)
+ if os.path.exists(f):
+ print >>sys.stderr, 'warning: ignoring %s' % f
+
+ m = self.manifestProject
+ b = m.CurrentBranch
+ if not b:
+ raise ManifestParseError, 'manifest cannot be on detached HEAD'
+ b = m.GetBranch(b).merge
+ if b.startswith(R_HEADS):
+ b = b[len(R_HEADS):]
+ self.branch = b
+ m.remote.name = self._Remote().name
+
+ self._ParseModules()
+
+ if self.IsMirror:
+ self._AddMetaProjectMirror(self.repoProject)
+ self._AddMetaProjectMirror(self.manifestProject)
+
+ self._loaded = True
+
+ def _ParseModules(self):
+ byPath = dict()
+ for name in self._modules.GetSubSections('submodule'):
+ p = self._ParseProject(name)
+ if self._projects.get(p.name):
+ raise ManifestParseError, 'duplicate project "%s"' % p.name
+ if byPath.get(p.relpath):
+ raise ManifestParseError, 'duplicate path "%s"' % p.relpath
+ self._projects[p.name] = p
+ byPath[p.relpath] = p
+
+ for relpath in self._allRevisionIds.keys():
+ if relpath not in byPath:
+ raise ManifestParseError, \
+ 'project "%s" not in .gitmodules' \
+ % relpath
+
+ def _Remote(self):
+ m = self.manifestProject
+ b = m.GetBranch(m.CurrentBranch)
+ return b.remote
+
+ def _ResolveUrl(self, url):
+ if url.startswith('./') or url.startswith('../'):
+ base = self._Remote().url
+ try:
+ base = base[:base.rindex('/')+1]
+ except ValueError:
+ base = base[:base.rindex(':')+1]
+ if url.startswith('./'):
+ url = url[2:]
+ while '/' in base and url.startswith('../'):
+ base = base[:base.rindex('/')+1]
+ url = url[3:]
+ return base + url
+ return url
+
+ def _GetRevisionId(self, path):
+ return self._allRevisionIds.get(path)
+
+ @property
+ def _allRevisionIds(self):
+ if self._revisionIds is None:
+ a = dict()
+ p = GitCommand(self.manifestProject,
+ ['ls-files','-z','--stage'],
+ capture_stdout = True)
+ for line in p.process.stdout.read().split('\0')[:-1]:
+ l_info, l_path = line.split('\t', 2)
+ l_mode, l_id, l_stage = l_info.split(' ', 2)
+ if l_mode == GITLINK and l_stage == '0':
+ a[l_path] = l_id
+ p.Wait()
+ self._revisionIds = a
+ return self._revisionIds
+
+ def SetRevisionId(self, path, id):
+ self.manifestProject.work_git.update_index(
+ '--add','--cacheinfo', GITLINK, id, path)
+
+ def _ParseProject(self, name):
+ gm = self._modules
+ gr = self._review
+
+ path = gm.GetString('submodule.%s.path' % name)
+ if not path:
+ path = name
+
+ revId = self._GetRevisionId(path)
+ if not revId:
+ raise ManifestParseError(
+ 'submodule "%s" has no revision at "%s"' \
+ % (name, path))
+
+ url = gm.GetString('submodule.%s.url' % name)
+ if not url:
+ url = name
+ url = self._ResolveUrl(url)
+
+ review = gr.GetString('review.%s.url' % name)
+ if not review:
+ review = gr.GetString('review.url')
+ if not review:
+ review = self._Remote().review
+
+ remote = RemoteSpec(self._Remote().name, url, review)
+ revExpr = gm.GetString('submodule.%s.revision' % name)
+ if revExpr == '.':
+ revExpr = self.branch
+
+ if self.IsMirror:
+ relpath = None
+ worktree = None
+ gitdir = os.path.join(self.topdir, '%s.git' % name)
+ else:
+ worktree = os.path.join(self.topdir, path)
+ gitdir = os.path.join(self.repodir, 'projects/%s.git' % name)
+
+ return Project(manifest = self,
+ name = name,
+ remote = remote,
+ gitdir = gitdir,
+ worktree = worktree,
+ relpath = path,
+ revisionExpr = revExpr,
+ revisionId = revId)
+
+ def _AddMetaProjectMirror(self, m):
+ m_url = m.GetRemote(m.remote.name).url
+ if m_url.endswith('/.git'):
+ raise ManifestParseError, 'refusing to mirror %s' % m_url
+
+ name = self._GuessMetaName(m_url)
+ if name.endswith('.git'):
+ name = name[:-4]
+
+ if name not in self._projects:
+ m.PreSync()
+ gitdir = os.path.join(self.topdir, '%s.git' % name)
+ project = Project(manifest = self,
+ name = name,
+ remote = RemoteSpec(self._Remote().name, m_url),
+ gitdir = gitdir,
+ worktree = None,
+ relpath = None,
+ revisionExpr = m.revisionExpr,
+ revisionId = None)
+ self._projects[project.name] = project
+
+ def _GuessMetaName(self, m_url):
+ parts = m_url.split('/')
+ name = parts[-1]
+ parts = parts[0:-1]
+ s = len(parts) - 1
+ while s > 0:
+ l = '/'.join(parts[0:s]) + '/'
+ r = '/'.join(parts[s:]) + '/'
+ for p in self._projects.values():
+ if p.name.startswith(r) and p.remote.url.startswith(l):
+ return r + name
+ s -= 1
+ return m_url[m_url.rindex('/') + 1:]