Merge branch 'master' into maint
master's original purpose was to forge ahead on using git submodules,
but this route has been abandoned.
Change-Id: I164a9efc7821bcd1b941ad76649764722046081b
diff --git a/SUBMITTING_PATCHES b/SUBMITTING_PATCHES
index f68906c..63b0e57 100644
--- a/SUBMITTING_PATCHES
+++ b/SUBMITTING_PATCHES
@@ -5,7 +5,7 @@
- Make sure all code is under the Apache License, 2.0.
- Publish your changes for review:
- git push ssh://review.source.android.com:29418/tools/repo.git HEAD:refs/for/master
+ git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/maint
Long Version:
@@ -55,24 +55,23 @@
Instead, login to the Gerrit Code Review tool at:
- https://review.source.android.com/
+ https://gerrit-review.googlesource.com/
Ensure you have completed one of the necessary contributor
agreements, providing documentation to the project maintainers that
they have right to redistribute your work under the Apache License:
- https://review.source.android.com/#settings,agreements
+ https://gerrit-review.googlesource.com/#/settings/agreements
-Ensure you have registered one or more SSH public keys, so you can
-push your commits directly over SSH:
+Ensure you have obtained an HTTP password to authenticate:
- https://review.source.android.com/#settings,ssh-keys
+ https://gerrit-review.googlesource.com/new-password
-Push your patches over SSH to the review server, possibly through
+Push your patches over HTTPS to the review server, possibly through
a remembered remote to make this easier in the future:
- git config remote.review.url ssh://review.source.android.com:29418/tools/repo.git
- git config remote.review.push HEAD:refs/for/master
+ git config remote.review.url https://gerrit-review.googlesource.com/git-repo
+ git config remote.review.push HEAD:refs/for/maint
git push review
diff --git a/command.py b/command.py
index 4e0253f..4dbe2e7 100644
--- a/command.py
+++ b/command.py
@@ -15,17 +15,19 @@
import os
import optparse
+import platform
+import re
import sys
-import manifest_loader
-
from error import NoSuchProjectError
+from error import InvalidProjectGroupsError
class Command(object):
"""Base class for any command line action in repo.
"""
common = False
+ manifest = None
_optparse = None
def WantPager(self, opt):
@@ -57,31 +59,24 @@
"""Perform the action, after option parsing is complete.
"""
raise NotImplementedError
-
- @property
- def manifest(self):
- return self.GetManifest()
-
- def GetManifest(self, reparse=False, type=None):
- return manifest_loader.GetManifest(self.repodir,
- reparse=reparse,
- type=type)
def GetProjects(self, args, missing_ok=False):
"""A list of projects that match the arguments.
"""
all = self.manifest.projects
+ result = []
mp = self.manifest.manifestProject
- if mp.relpath == '.':
- all = dict(all)
- all[mp.name] = mp
- result = []
+ groups = mp.config.GetString('manifest.groups')
+ if not groups:
+ groups = 'default,platform-' + platform.system().lower()
+ groups = [x for x in re.split('[,\s]+', groups) if x]
if not args:
for project in all.values():
- if missing_ok or project.Exists:
+ if ((missing_ok or project.Exists) and
+ project.MatchesGroups(groups)):
result.append(project)
else:
by_path = None
@@ -97,9 +92,7 @@
for p in all.values():
by_path[p.worktree] = p
- try:
- project = by_path[path]
- except KeyError:
+ if os.path.exists(path):
oldpath = None
while path \
and path != oldpath \
@@ -110,11 +103,18 @@
except KeyError:
oldpath = path
path = os.path.dirname(path)
+ else:
+ try:
+ project = by_path[path]
+ except KeyError:
+ pass
if not project:
raise NoSuchProjectError(arg)
if not missing_ok and not project.Exists:
raise NoSuchProjectError(arg)
+ if not project.MatchesGroups(groups):
+ raise InvalidProjectGroupsError(arg)
result.append(project)
diff --git a/docs/manifest_xml.txt b/docs/manifest-format.txt
similarity index 72%
rename from docs/manifest_xml.txt
rename to docs/manifest-format.txt
index 37fbd5c..38868f1 100644
--- a/docs/manifest_xml.txt
+++ b/docs/manifest-format.txt
@@ -25,30 +25,48 @@
default?,
manifest-server?,
remove-project*,
- project*)>
+ project*,
+ repo-hooks?)>
<!ELEMENT notice (#PCDATA)>
<!ELEMENT remote (EMPTY)>
<!ATTLIST remote name ID #REQUIRED>
+ <!ATTLIST remote alias CDATA #IMPLIED>
<!ATTLIST remote fetch CDATA #REQUIRED>
<!ATTLIST remote review CDATA #IMPLIED>
<!ELEMENT default (EMPTY)>
<!ATTLIST default remote IDREF #IMPLIED>
<!ATTLIST default revision CDATA #IMPLIED>
-
+ <!ATTLIST default sync-j CDATA #IMPLIED>
+ <!ATTLIST default sync-c CDATA #IMPLIED>
+
<!ELEMENT manifest-server (EMPTY)>
<!ATTLIST url CDATA #REQUIRED>
- <!ELEMENT project (EMPTY)>
+ <!ELEMENT project (annotation?)>
<!ATTLIST project name CDATA #REQUIRED>
<!ATTLIST project path CDATA #IMPLIED>
<!ATTLIST project remote IDREF #IMPLIED>
<!ATTLIST project revision CDATA #IMPLIED>
+ <!ATTLIST project groups CDATA #IMPLIED>
+ <!ATTLIST project sync-c CDATA #IMPLIED>
+
+ <!ELEMENT annotation (EMPTY)>
+ <!ATTLIST annotation name CDATA #REQUIRED>
+ <!ATTLIST annotation value CDATA #REQUIRED>
+ <!ATTLIST annotation keep CDATA "true">
<!ELEMENT remove-project (EMPTY)>
<!ATTLIST remove-project name CDATA #REQUIRED>
+
+ <!ELEMENT repo-hooks (EMPTY)>
+ <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
+ <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
+
+ <!ELEMENT include (EMPTY)>
+ <!ATTLIST include name CDATA #REQUIRED>
]>
A description of the elements and their attributes follows.
@@ -72,6 +90,12 @@
.git/config, and is therefore automatically available to commands
like `git fetch`, `git remote`, `git pull` and `git push`.
+Attribute `alias`: The alias, if specified, is used to override
+`name` to be set as the remote name in each project's .git/config.
+Its value can be duplicated while attribute `name` has to be unique
+in the manifest file. This helps each project to be able to have
+same remote name which actually points to different remote url.
+
Attribute `fetch`: The Git URL prefix for all projects which use
this remote. Each project's name is appended to this prefix to
form the actual URL used to clone the project.
@@ -152,6 +176,25 @@
been extensively tested. If not supplied the revision given by
the default element is used.
+Attribute `groups`: List of groups to which this project belongs,
+whitespace or comma separated. All projects belong to the group
+"default", and each project automatically belongs to a group of
+it's name:`name` and path:`path`. E.g. for
+<project name="monkeys" path="barrel-of"/>, that project
+definition is implicitly in the following manifest groups:
+default, name:monkeys, and path:barrel-of.
+
+Element annotation
+------------------
+
+Zero or more annotation elements may be specified as children of a
+project element. Each element describes a name-value pair that will be
+exported into each project's environment during a 'forall' command,
+prefixed with REPO__. In addition, there is an optional attribute
+"keep" which accepts the case insensitive values "true" (default) or
+"false". This attribute determines whether or not the annotation will
+be kept when exported with the manifest subcommand.
+
Element remove-project
----------------------
@@ -163,6 +206,16 @@
the user can remove a project, and possibly replace it with their
own definition.
+Element include
+---------------
+
+This element provides the capability of including another manifest
+file into the originating manifest. Normal rules apply for the
+target manifest to include- it must be a usable manifest on it's own.
+
+Attribute `name`; the manifest to include, specified relative to
+the manifest repositories root.
+
Local Manifest
==============
diff --git a/docs/manifest_submodule.txt b/docs/manifest_submodule.txt
deleted file mode 100644
index 1718284..0000000
--- a/docs/manifest_submodule.txt
+++ /dev/null
@@ -1,136 +0,0 @@
-repo Manifest Format (submodule)
-================================
-
-A repo manifest describes the structure of a repo client; that is
-the directories that are visible and where they should be obtained
-from with git.
-
-The basic structure of a manifest is a bare Git repository holding
-a 'gitmodules' file in the top level directory, and one or more
-gitlink references pointing at commits from the referenced projects.
-This is the same structure as used by 'git submodule'.
-
-Manifests are inherently version controlled, since they are kept
-within a Git repository. Updates to manifests are automatically
-obtained by clients during `repo sync`.
-
-.gitmodules
-===========
-
-The '.gitmodules' file, located in the top-level directory of the
-client's working tree (or manifest repository), is a text file with
-a syntax matching the requirements of 'git config'.
-
-This file contains one subsection per project (also called a
-submodule by git), and the subsection value is a unique name to
-describe the project. Each submodule section must contain the
-following required keys:
-
- * path
- * url
-
-submodule.<name>.path
----------------------
-
-Defines the path, relative to the top-level directory of the client's
-working tree, where the project is expected to be checked out. The
-path name must not end with a '/'. All paths must be unique within
-the .gitmodules file.
-
-At the specified path within the manifest repository a gitlink
-tree entry (an entry with file mode 160000) must exist referencing
-a commit SHA-1 from the project. This tree entry specifies the
-exact version of the project that `repo sync` will synchronize the
-client's working tree to.
-
-submodule.<name>.url
---------------------
-
-Defines a URL from where the project repository can be cloned.
-By default `repo sync` will clone from this URL whenever a user
-needs to access this project.
-
-submodule.<name>.revision
--------------------------
-
-Name of the branch in the project repository that Gerrit Code Review
-should automatically refresh the project's gitlink entry from.
-
-If set, during submit of a change within the referenced project,
-Gerrit Code Review will automatically update the manifest
-repository's corresponding gitlink to the new commit SHA-1 of
-this branch.
-
-Valid values are a short branch name (e.g. 'master'), a full ref
-name (e.g. 'refs/heads/master'), or '.' to request using the same
-branch name as the manifest branch itself. Since '.' automatically
-uses the manifest branch, '.' is the recommended value.
-
-If this key is not set, Gerrit Code Review will NOT automatically
-update the gitlink. An unset key requires the manifest maintainer
-to manually update the gitlink when it is necessary to reference
-a different revision of the project.
-
-submodule.<name>.update
------------------------
-
-This key is not supported by repo. If set, it will be ignored.
-
-repo.notice
------------
-
-A message displayed when repo sync uses this manifest.
-
-
-.review
-=======
-
-The optional '.review' file, located in the top-level directory of
-the client's working tree (or manifest repository), is a text file
-with a syntax matching the requirements of 'git config'.
-
-This file describes how `repo upload` should interact with the
-project's preferred code review system.
-
-review.url
-----------
-
-URL of the default Gerrit Code Review server. If a project does
-not have a specific URL in the '.review' file, this default URL
-will be used instead.
-
-review.<name>.url
------------------
-
-Project specific URL of the Gerrit Code Review server, for the
-submodule whose project name is <name>.
-
-Example
-=======
-
- $ cat .gitmodules
- [submodule "app/Clock"]
- path = clock
- url = git://vcs.example.com/ClockWidget.git
- revision = .
- [submodule "app/Browser"]
- path = net/browser
- url = git://netgroup.example.com/network/web/Browser.git
- revision = .
-
- $ cat .review
- [review]
- url = vcs-gerrit.example.com
- [review "app/Browser"]
- url = netgroup.example.com
-
-In the above example, the app/Clock project will send its code
-reviews to the default server, vcs-gerrit.example.com, while
-app/Browser will send its code reviews to netgroup.example.com.
-
-See Also
-========
-
- * http://www.kernel.org/pub/software/scm/git/docs/gitmodules.html
- * http://www.kernel.org/pub/software/scm/git/docs/git-config.html
- * http://code.google.com/p/gerrit/
diff --git a/error.py b/error.py
index cb3b725..78c5c0e 100644
--- a/error.py
+++ b/error.py
@@ -57,6 +57,15 @@
def __str__(self):
return self.reason
+class DownloadError(Exception):
+ """Cannot download a repository.
+ """
+ def __init__(self, reason):
+ self.reason = reason
+
+ def __str__(self):
+ return self.reason
+
class NoSuchProjectError(Exception):
"""A specified project does not exist in the work tree.
"""
@@ -68,6 +77,18 @@
return 'in current directory'
return self.name
+
+class InvalidProjectGroupsError(Exception):
+ """A specified project is not suitable for the specified groups
+ """
+ def __init__(self, name=None):
+ self.name = name
+
+ def __str__(self):
+ if self.Name is None:
+ return 'in current directory'
+ return self.name
+
class RepoChangedException(Exception):
"""Thrown if 'repo sync' results in repo updating its internal
repo or manifest repositories. In this special case we must
@@ -75,3 +96,10 @@
"""
def __init__(self, extra_args=[]):
self.extra_args = extra_args
+
+class HookError(Exception):
+ """Thrown if a 'repo-hook' could not be run.
+
+ The common case is that the file wasn't present when we tried to run it.
+ """
+ pass
diff --git a/git_command.py b/git_command.py
index 513b9eb..5988cc2 100644
--- a/git_command.py
+++ b/git_command.py
@@ -72,6 +72,8 @@
pass
_ssh_clients = []
+_git_version = None
+
class _GitCall(object):
def version(self):
p = GitCommand(None, ['--version'], capture_stdout=True)
@@ -79,6 +81,21 @@
return p.stdout
return None
+ def version_tuple(self):
+ global _git_version
+
+ if _git_version is None:
+ ver_str = git.version()
+ if ver_str.startswith('git version '):
+ _git_version = tuple(
+ map(lambda x: int(x),
+ ver_str[len('git version '):].strip().split('-')[0].split('.')[0:3]
+ ))
+ else:
+ print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
+ sys.exit(1)
+ return _git_version
+
def __getattr__(self, name):
name = name.replace('_','-')
def fun(*cmdv):
@@ -88,23 +105,9 @@
return fun
git = _GitCall()
-_git_version = None
-
def git_require(min_version, fail=False):
- global _git_version
-
- if _git_version is None:
- ver_str = git.version()
- if ver_str.startswith('git version '):
- _git_version = tuple(
- map(lambda x: int(x),
- ver_str[len('git version '):].strip().split('.')[0:3]
- ))
- else:
- print >>sys.stderr, 'fatal: "%s" unsupported' % ver_str
- sys.exit(1)
-
- if min_version <= _git_version:
+ git_version = git.version_tuple()
+ if min_version <= git_version:
return True
if fail:
need = '.'.join(map(lambda x: str(x), min_version))
@@ -144,6 +147,12 @@
if ssh_proxy:
_setenv(env, 'REPO_SSH_SOCK', ssh_sock())
_setenv(env, 'GIT_SSH', _ssh_proxy())
+ if 'http_proxy' in env and 'darwin' == sys.platform:
+ s = "'http.proxy=%s'" % (env['http_proxy'],)
+ p = env.get('GIT_CONFIG_PARAMETERS')
+ if p is not None:
+ s = p + ' ' + s
+ _setenv(env, 'GIT_CONFIG_PARAMETERS', s)
if project:
if not cwd:
@@ -218,26 +227,10 @@
self.stdin = p.stdin
def Wait(self):
- p = self.process
-
- if p.stdin:
- p.stdin.close()
- self.stdin = None
-
- if p.stdout:
- self.stdout = p.stdout.read()
- p.stdout.close()
- else:
- p.stdout = None
-
- if p.stderr:
- self.stderr = p.stderr.read()
- p.stderr.close()
- else:
- p.stderr = None
-
try:
- rc = p.wait()
+ p = self.process
+ (self.stdout, self.stderr) = p.communicate()
+ rc = p.returncode
finally:
_remove_ssh_client(p)
return rc
diff --git a/git_config.py b/git_config.py
index ff815e3..eb532d0 100644
--- a/git_config.py
+++ b/git_config.py
@@ -26,7 +26,6 @@
import urllib2
from signal import SIGTERM
-from urllib2 import urlopen, HTTPError
from error import GitError, UploadError
from trace import Trace
@@ -80,14 +79,6 @@
else:
self._pickle = pickleFile
- def ClearCache(self):
- if os.path.exists(self._pickle):
- os.remove(self._pickle)
- self._cache_dict = None
- self._section_dict = None
- self._remotes = {}
- self._branches = {}
-
def Has(self, name, include_defaults = True):
"""Return true if this configuration file has the key.
"""
@@ -206,6 +197,15 @@
except KeyError:
return False
+ def UrlInsteadOf(self, url):
+ """Resolve any url.*.insteadof references.
+ """
+ for new_url in self.GetSubSections('url'):
+ old_url = self.GetString('url.%s.insteadof' % new_url)
+ if old_url is not None and url.startswith(old_url):
+ return new_url + url[len(old_url):]
+ return url
+
@property
def _sections(self):
d = self._section_dict
@@ -488,7 +488,13 @@
_master_keys_lock = None
URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
-URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
+URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
+
+def GetSchemeFromUrl(url):
+ m = URI_ALL.match(url)
+ if m:
+ return m.group(1)
+ return None
def _preconnect(url):
m = URI_ALL.match(url)
@@ -521,7 +527,7 @@
self.projectname = self._Get('projectname')
self.fetch = map(lambda x: RefSpec.FromString(x),
self._Get('fetch', all=True))
- self._review_protocol = None
+ self._review_url = None
def _InsteadOf(self):
globCfg = GitConfig.ForUser()
@@ -548,9 +554,8 @@
connectionUrl = self._InsteadOf()
return _preconnect(connectionUrl)
- @property
- def ReviewProtocol(self):
- if self._review_protocol is None:
+ def ReviewUrl(self, userEmail):
+ if self._review_url is None:
if self.review is None:
return None
@@ -559,57 +564,47 @@
u = 'http://%s' % u
if u.endswith('/Gerrit'):
u = u[:len(u) - len('/Gerrit')]
- if not u.endswith('/ssh_info'):
- if not u.endswith('/'):
- u += '/'
- u += 'ssh_info'
+ if u.endswith('/ssh_info'):
+ u = u[:len(u) - len('/ssh_info')]
+ if not u.endswith('/'):
+ u += '/'
+ http_url = u
if u in REVIEW_CACHE:
- info = REVIEW_CACHE[u]
- self._review_protocol = info[0]
- self._review_host = info[1]
- self._review_port = info[2]
+ self._review_url = REVIEW_CACHE[u]
+ elif 'REPO_HOST_PORT_INFO' in os.environ:
+ host, port = os.environ['REPO_HOST_PORT_INFO'].split()
+ self._review_url = self._SshReviewUrl(userEmail, host, port)
+ REVIEW_CACHE[u] = self._review_url
else:
try:
- info = urlopen(u).read()
- if info == 'NOT_AVAILABLE':
- raise UploadError('%s: SSH disabled' % self.review)
+ info_url = u + 'ssh_info'
+ info = urllib2.urlopen(info_url).read()
if '<' in info:
# Assume the server gave us some sort of HTML
# response back, like maybe a login page.
#
- raise UploadError('%s: Cannot parse response' % u)
+ raise UploadError('%s: Cannot parse response' % info_url)
- self._review_protocol = 'ssh'
- self._review_host = info.split(" ")[0]
- self._review_port = info.split(" ")[1]
- except urllib2.URLError, e:
- raise UploadError('%s: %s' % (self.review, e.reason[1]))
- except HTTPError, e:
- if e.code == 404:
- self._review_protocol = 'http-post'
- self._review_host = None
- self._review_port = None
+ if info == 'NOT_AVAILABLE':
+ # Assume HTTP if SSH is not enabled.
+ self._review_url = http_url + 'p/'
else:
- raise UploadError('Upload over ssh unavailable')
+ host, port = info.split()
+ self._review_url = self._SshReviewUrl(userEmail, host, port)
+ except urllib2.HTTPError, e:
+ raise UploadError('%s: %s' % (self.review, str(e)))
+ except urllib2.URLError, e:
+ raise UploadError('%s: %s' % (self.review, str(e)))
- REVIEW_CACHE[u] = (
- self._review_protocol,
- self._review_host,
- self._review_port)
- return self._review_protocol
+ REVIEW_CACHE[u] = self._review_url
+ return self._review_url + self.projectname
- def SshReviewUrl(self, userEmail):
- if self.ReviewProtocol != 'ssh':
- return None
+ def _SshReviewUrl(self, userEmail, host, port):
username = self._config.GetString('review.%s.username' % self.review)
if username is None:
- username = userEmail.split("@")[0]
- return 'ssh://%s@%s:%s/%s' % (
- username,
- self._review_host,
- self._review_port,
- self.projectname)
+ username = userEmail.split('@')[0]
+ return 'ssh://%s@%s:%s/' % (username, host, port)
def ToLocal(self, rev):
"""Convert a remote revision string to something we have locally.
diff --git a/git_refs.py b/git_refs.py
index b24a0b4..0e3cc82 100644
--- a/git_refs.py
+++ b/git_refs.py
@@ -21,6 +21,7 @@
R_HEADS = 'refs/heads/'
R_TAGS = 'refs/tags/'
R_PUB = 'refs/published/'
+R_M = 'refs/remotes/m/'
class GitRefs(object):
@@ -138,13 +139,15 @@
def _ReadLoose1(self, path, name):
try:
fd = open(path, 'rb')
- mtime = os.path.getmtime(path)
- except OSError:
+ except:
return
- except IOError:
- return
+
try:
- id = fd.readline()
+ try:
+ mtime = os.path.getmtime(path)
+ id = fd.readline()
+ except:
+ return
finally:
fd.close()
diff --git a/hooks/pre-auto-gc b/hooks/pre-auto-gc
index 110e319..360e5e1 100755
--- a/hooks/pre-auto-gc
+++ b/hooks/pre-auto-gc
@@ -38,6 +38,11 @@
grep -q "Currently drawing from 'AC Power'"
then
exit 0
+elif test -d /sys/bus/acpi/drivers/battery && test 0 = \
+ "$(find /sys/bus/acpi/drivers/battery/ -type l | wc -l)";
+then
+ # No battery exists.
+ exit 0
fi
echo "Auto packing deferred; not on AC"
diff --git a/main.py b/main.py
index 07b26ef..ea29851 100755
--- a/main.py
+++ b/main.py
@@ -22,19 +22,27 @@
del sys.argv[-1]
del magic
+import netrc
import optparse
import os
import re
import sys
+import time
+import urllib2
from trace import SetTrace
+from git_command import git, GitCommand
from git_config import init_ssh, close_ssh
from command import InteractiveCommand
from command import MirrorSafeCommand
from command import PagedCommand
+from subcmds.version import Version
+from editor import Editor
+from error import DownloadError
from error import ManifestInvalidRevisionError
from error import NoSuchProjectError
from error import RepoChangedException
+from manifest_xml import XmlManifest
from pager import RunPager
from subcmds import all as all_commands
@@ -51,6 +59,9 @@
global_options.add_option('--trace',
dest='trace', action='store_true',
help='trace git command execution')
+global_options.add_option('--time',
+ dest='time', action='store_true',
+ help='time repo command execution')
global_options.add_option('--version',
dest='show_version', action='store_true',
help='display this version of repo')
@@ -63,6 +74,7 @@
all_commands['branch'] = all_commands['branches']
def _Run(self, argv):
+ result = 0
name = None
glob = []
@@ -86,7 +98,7 @@
name = 'version'
else:
print >>sys.stderr, 'fatal: invalid usage of --version'
- sys.exit(1)
+ return 1
try:
cmd = self.commands[name]
@@ -94,15 +106,17 @@
print >>sys.stderr,\
"repo: '%s' is not a repo command. See 'repo help'."\
% name
- sys.exit(1)
+ return 1
cmd.repodir = self.repodir
+ cmd.manifest = XmlManifest(cmd.repodir)
+ Editor.globalConfig = cmd.manifest.globalConfig
if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
print >>sys.stderr, \
"fatal: '%s' requires a working directory"\
% name
- sys.exit(1)
+ return 1
copts, cargs = cmd.OptionParser.parse_args(argv)
@@ -118,16 +132,37 @@
RunPager(config)
try:
- cmd.Execute(copts, cargs)
+ start = time.time()
+ try:
+ result = cmd.Execute(copts, cargs)
+ finally:
+ elapsed = time.time() - start
+ hours, remainder = divmod(elapsed, 3600)
+ minutes, seconds = divmod(remainder, 60)
+ if gopts.time:
+ if hours == 0:
+ print >>sys.stderr, 'real\t%dm%.3fs' \
+ % (minutes, seconds)
+ else:
+ print >>sys.stderr, 'real\t%dh%dm%.3fs' \
+ % (hours, minutes, seconds)
+ except DownloadError, e:
+ print >>sys.stderr, 'error: %s' % str(e)
+ return 1
except ManifestInvalidRevisionError, e:
print >>sys.stderr, 'error: %s' % str(e)
- sys.exit(1)
+ return 1
except NoSuchProjectError, e:
if e.name:
print >>sys.stderr, 'error: project %s not found' % e.name
else:
print >>sys.stderr, 'error: no project in current directory'
- sys.exit(1)
+ return 1
+
+ return result
+
+def _MyRepoPath():
+ return os.path.dirname(__file__)
def _MyWrapperPath():
return os.path.join(os.path.dirname(__file__), 'repo')
@@ -195,7 +230,117 @@
continue
i += 1
+_user_agent = None
+
+def _UserAgent():
+ global _user_agent
+
+ if _user_agent is None:
+ py_version = sys.version_info
+
+ os_name = sys.platform
+ if os_name == 'linux2':
+ os_name = 'Linux'
+ elif os_name == 'win32':
+ os_name = 'Win32'
+ elif os_name == 'cygwin':
+ os_name = 'Cygwin'
+ elif os_name == 'darwin':
+ os_name = 'Darwin'
+
+ p = GitCommand(
+ None, ['describe', 'HEAD'],
+ cwd = _MyRepoPath(),
+ capture_stdout = True)
+ if p.Wait() == 0:
+ repo_version = p.stdout
+ if len(repo_version) > 0 and repo_version[-1] == '\n':
+ repo_version = repo_version[0:-1]
+ if len(repo_version) > 0 and repo_version[0] == 'v':
+ repo_version = repo_version[1:]
+ else:
+ repo_version = 'unknown'
+
+ _user_agent = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
+ repo_version,
+ os_name,
+ '.'.join(map(lambda d: str(d), git.version_tuple())),
+ py_version[0], py_version[1], py_version[2])
+ return _user_agent
+
+class _UserAgentHandler(urllib2.BaseHandler):
+ def http_request(self, req):
+ req.add_header('User-Agent', _UserAgent())
+ return req
+
+ def https_request(self, req):
+ req.add_header('User-Agent', _UserAgent())
+ return req
+
+class _BasicAuthHandler(urllib2.HTTPBasicAuthHandler):
+ def http_error_auth_reqed(self, authreq, host, req, headers):
+ try:
+ old_add_header = req.add_header
+ def _add_header(name, val):
+ val = val.replace('\n', '')
+ old_add_header(name, val)
+ req.add_header = _add_header
+ return urllib2.AbstractBasicAuthHandler.http_error_auth_reqed(
+ self, authreq, host, req, headers)
+ except:
+ reset = getattr(self, 'reset_retry_count', None)
+ if reset is not None:
+ reset()
+ elif getattr(self, 'retried', None):
+ self.retried = 0
+ raise
+
+class _DigestAuthHandler(urllib2.HTTPDigestAuthHandler):
+ def http_error_auth_reqed(self, auth_header, host, req, headers):
+ try:
+ old_add_header = req.add_header
+ def _add_header(name, val):
+ val = val.replace('\n', '')
+ old_add_header(name, val)
+ req.add_header = _add_header
+ return urllib2.AbstractDigestAuthHandler.http_error_auth_reqed(
+ self, auth_header, host, req, headers)
+ except:
+ reset = getattr(self, 'reset_retry_count', None)
+ if reset is not None:
+ reset()
+ elif getattr(self, 'retried', None):
+ self.retried = 0
+ raise
+
+def init_http():
+ handlers = [_UserAgentHandler()]
+
+ mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ try:
+ n = netrc.netrc()
+ for host in n.hosts:
+ p = n.hosts[host]
+ mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
+ mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
+ except netrc.NetrcParseError:
+ pass
+ except IOError:
+ pass
+ handlers.append(_BasicAuthHandler(mgr))
+ handlers.append(_DigestAuthHandler(mgr))
+
+ if 'http_proxy' in os.environ:
+ url = os.environ['http_proxy']
+ handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+ if 'REPO_CURL_VERBOSE' in os.environ:
+ handlers.append(urllib2.HTTPHandler(debuglevel=1))
+ handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+ urllib2.install_opener(urllib2.build_opener(*handlers))
+
def _Main(argv):
+ result = 0
+
opt = optparse.OptionParser(usage="repo wrapperinfo -- ...")
opt.add_option("--repo-dir", dest="repodir",
help="path to .repo/")
@@ -209,15 +354,19 @@
_CheckWrapperVersion(opt.wrapper_version, opt.wrapper_path)
_CheckRepoDir(opt.repodir)
+ Version.wrapper_version = opt.wrapper_version
+ Version.wrapper_path = opt.wrapper_path
+
repo = _Repo(opt.repodir)
try:
try:
init_ssh()
- repo._Run(argv)
+ init_http()
+ result = repo._Run(argv) or 0
finally:
close_ssh()
except KeyboardInterrupt:
- sys.exit(1)
+ result = 1
except RepoChangedException, rce:
# If repo changed, re-exec ourselves.
#
@@ -228,7 +377,9 @@
except OSError, e:
print >>sys.stderr, 'fatal: cannot restart repo after upgrade'
print >>sys.stderr, 'fatal: %s' % e
- sys.exit(128)
+ result = 128
+
+ sys.exit(result)
if __name__ == '__main__':
_Main(sys.argv[1:])
diff --git a/manifest.py b/manifest.py
deleted file mode 100644
index c03cb4a..0000000
--- a/manifest.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#
-# Copyright (C) 2009 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-from error import ManifestParseError
-from editor import Editor
-from git_config import GitConfig
-from project import MetaProject
-
-class Manifest(object):
- """any manifest format"""
-
- def __init__(self, repodir):
- self.repodir = os.path.abspath(repodir)
- self.topdir = os.path.dirname(self.repodir)
- self.globalConfig = GitConfig.ForUser()
- Editor.globalConfig = self.globalConfig
-
- self.repoProject = MetaProject(self, 'repo',
- gitdir = os.path.join(repodir, 'repo/.git'),
- worktree = os.path.join(repodir, 'repo'))
-
- @property
- def IsMirror(self):
- return self.manifestProject.config.GetBoolean('repo.mirror')
-
- @property
- def projects(self):
- return {}
-
- @property
- def notice(self):
- return None
-
- @property
- def manifest_server(self):
- return None
-
- def InitBranch(self):
- pass
-
- def SetMRefs(self, project):
- pass
-
- def Upgrade_Local(self, old):
- raise ManifestParseError, 'unsupported upgrade path'
diff --git a/manifest_loader.py b/manifest_loader.py
deleted file mode 100644
index 467cb42..0000000
--- a/manifest_loader.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Copyright (C) 2009 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from manifest_submodule import SubmoduleManifest
-from manifest_xml import XmlManifest
-
-def ParseManifest(repodir, type=None):
- if type:
- return type(repodir)
- if SubmoduleManifest.Is(repodir):
- return SubmoduleManifest(repodir)
- return XmlManifest(repodir)
-
-_manifest = None
-
-def GetManifest(repodir, reparse=False, type=None):
- global _manifest
- if _manifest is None \
- or reparse \
- or (type and _manifest.__class__ != type):
- _manifest = ParseManifest(repodir, type=type)
- return _manifest
diff --git a/manifest_submodule.py b/manifest_submodule.py
deleted file mode 100644
index cac271c..0000000
--- a/manifest_submodule.py
+++ /dev/null
@@ -1,481 +0,0 @@
-#
-# Copyright (C) 2009 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-import os
-import shutil
-
-from error import GitError
-from error import ManifestParseError
-from git_command import GitCommand
-from git_config import GitConfig
-from git_config import IsId
-from manifest import Manifest
-from progress import Progress
-from project import RemoteSpec
-from project import Project
-from project import MetaProject
-from project import R_HEADS
-from project import HEAD
-from project import _lwrite
-
-import manifest_xml
-
-GITLINK = '160000'
-
-def _rmdir(dir, top):
- while dir != top:
- try:
- os.rmdir(dir)
- except OSError:
- break
- dir = os.path.dirname(dir)
-
-def _rmref(gitdir, ref):
- os.remove(os.path.join(gitdir, ref))
- log = os.path.join(gitdir, 'logs', ref)
- if os.path.exists(log):
- os.remove(log)
- _rmdir(os.path.dirname(log), gitdir)
-
-def _has_gitmodules(d):
- return os.path.exists(os.path.join(d, '.gitmodules'))
-
-class SubmoduleManifest(Manifest):
- """manifest from .gitmodules file"""
-
- @classmethod
- def Is(cls, repodir):
- return _has_gitmodules(os.path.dirname(repodir)) \
- or _has_gitmodules(os.path.join(repodir, 'manifest')) \
- or _has_gitmodules(os.path.join(repodir, 'manifests'))
-
- @classmethod
- def IsBare(cls, p):
- try:
- p.bare_git.cat_file('-e', '%s:.gitmodules' % p.GetRevisionId())
- except GitError:
- return False
- return True
-
- def __init__(self, repodir):
- Manifest.__init__(self, repodir)
-
- gitdir = os.path.join(repodir, 'manifest.git')
- config = GitConfig.ForRepository(gitdir = gitdir)
-
- if config.GetBoolean('repo.mirror'):
- worktree = os.path.join(repodir, 'manifest')
- relpath = None
- else:
- worktree = self.topdir
- relpath = '.'
-
- self.manifestProject = MetaProject(self, '__manifest__',
- gitdir = gitdir,
- worktree = worktree,
- relpath = relpath)
- self._modules = GitConfig(os.path.join(worktree, '.gitmodules'),
- pickleFile = os.path.join(
- repodir, '.repopickle_gitmodules'
- ))
- self._review = GitConfig(os.path.join(worktree, '.review'),
- pickleFile = os.path.join(
- repodir, '.repopickle_review'
- ))
- self._Unload()
-
- @property
- def projects(self):
- self._Load()
- return self._projects
-
- @property
- def notice(self):
- return self._modules.GetString('repo.notice')
-
- def InitBranch(self):
- m = self.manifestProject
- if m.CurrentBranch is None:
- b = m.revisionExpr
- if b.startswith(R_HEADS):
- b = b[len(R_HEADS):]
- return m.StartBranch(b)
- return True
-
- def SetMRefs(self, project):
- if project.revisionId is None:
- # Special project, e.g. the manifest or repo executable.
- #
- return
-
- ref = 'refs/remotes/m'
- cur = project.bare_ref.get(ref)
- exp = project.revisionId
- if cur != exp:
- msg = 'manifest set to %s' % exp
- project.bare_git.UpdateRef(ref, exp, message = msg, detach = True)
-
- ref = 'refs/remotes/m-revision'
- cur = project.bare_ref.symref(ref)
- exp = project.revisionExpr
- if exp is None:
- if cur:
- _rmref(project.gitdir, ref)
- elif cur != exp:
- remote = project.GetRemote(project.remote.name)
- dst = remote.ToLocal(exp)
- msg = 'manifest set to %s (%s)' % (exp, dst)
- project.bare_git.symbolic_ref('-m', msg, ref, dst)
-
- def Upgrade_Local(self, old):
- if isinstance(old, manifest_xml.XmlManifest):
- self.FromXml_Local_1(old, checkout=True)
- self.FromXml_Local_2(old)
- else:
- raise ManifestParseError, 'cannot upgrade manifest'
-
- def FromXml_Local_1(self, old, checkout):
- os.rename(old.manifestProject.gitdir,
- os.path.join(old.repodir, 'manifest.git'))
-
- oldmp = old.manifestProject
- oldBranch = oldmp.CurrentBranch
- b = oldmp.GetBranch(oldBranch).merge
- if not b:
- raise ManifestParseError, 'cannot upgrade manifest'
- if b.startswith(R_HEADS):
- b = b[len(R_HEADS):]
-
- newmp = self.manifestProject
- self._CleanOldMRefs(newmp)
- if oldBranch != b:
- newmp.bare_git.branch('-m', oldBranch, b)
- newmp.config.ClearCache()
-
- old_remote = newmp.GetBranch(b).remote.name
- act_remote = self._GuessRemoteName(old)
- if old_remote != act_remote:
- newmp.bare_git.remote('rename', old_remote, act_remote)
- newmp.config.ClearCache()
- newmp.remote.name = act_remote
- print >>sys.stderr, "Assuming remote named '%s'" % act_remote
-
- if checkout:
- for p in old.projects.values():
- for c in p.copyfiles:
- if os.path.exists(c.abs_dest):
- os.remove(c.abs_dest)
- newmp._InitWorkTree()
- else:
- newmp._LinkWorkTree()
-
- _lwrite(os.path.join(newmp.worktree,'.git',HEAD),
- 'ref: refs/heads/%s\n' % b)
-
- def _GuessRemoteName(self, old):
- used = {}
- for p in old.projects.values():
- n = p.remote.name
- used[n] = used.get(n, 0) + 1
-
- remote_name = 'origin'
- remote_used = 0
- for n in used.keys():
- if remote_used < used[n]:
- remote_used = used[n]
- remote_name = n
- return remote_name
-
- def FromXml_Local_2(self, old):
- shutil.rmtree(old.manifestProject.worktree)
- os.remove(old._manifestFile)
-
- my_remote = self._Remote().name
- new_base = os.path.join(self.repodir, 'projects')
- old_base = os.path.join(self.repodir, 'projects.old')
- os.rename(new_base, old_base)
- os.makedirs(new_base)
-
- info = []
- pm = Progress('Converting projects', len(self.projects))
- for p in self.projects.values():
- pm.update()
-
- old_p = old.projects.get(p.name)
- old_gitdir = os.path.join(old_base, '%s.git' % p.relpath)
- if not os.path.isdir(old_gitdir):
- continue
-
- parent = os.path.dirname(p.gitdir)
- if not os.path.isdir(parent):
- os.makedirs(parent)
- os.rename(old_gitdir, p.gitdir)
- _rmdir(os.path.dirname(old_gitdir), self.repodir)
-
- if not os.path.isdir(p.worktree):
- os.makedirs(p.worktree)
-
- if os.path.isdir(os.path.join(p.worktree, '.git')):
- p._LinkWorkTree(relink=True)
-
- self._CleanOldMRefs(p)
- if old_p and old_p.remote.name != my_remote:
- info.append("%s/: renamed remote '%s' to '%s'" \
- % (p.relpath, old_p.remote.name, my_remote))
- p.bare_git.remote('rename', old_p.remote.name, my_remote)
- p.config.ClearCache()
-
- self.SetMRefs(p)
- pm.end()
- for i in info:
- print >>sys.stderr, i
-
- def _CleanOldMRefs(self, p):
- all_refs = p._allrefs
- for ref in all_refs.keys():
- if ref.startswith(manifest_xml.R_M):
- if p.bare_ref.symref(ref) != '':
- _rmref(p.gitdir, ref)
- else:
- p.bare_git.DeleteRef(ref, all_refs[ref])
-
- def FromXml_Definition(self, old):
- """Convert another manifest representation to this one.
- """
- mp = self.manifestProject
- gm = self._modules
- gr = self._review
-
- fd = open(os.path.join(mp.worktree, '.gitignore'), 'ab')
- fd.write('/.repo\n')
- fd.close()
-
- sort_projects = list(old.projects.keys())
- sort_projects.sort()
-
- b = mp.GetBranch(mp.CurrentBranch).merge
- if b.startswith(R_HEADS):
- b = b[len(R_HEADS):]
-
- if old.notice:
- gm.SetString('repo.notice', old.notice)
-
- info = []
- pm = Progress('Converting manifest', len(sort_projects))
- for p in sort_projects:
- pm.update()
- p = old.projects[p]
-
- gm.SetString('submodule.%s.path' % p.name, p.relpath)
- gm.SetString('submodule.%s.url' % p.name, p.remote.url)
-
- if gr.GetString('review.url') is None:
- gr.SetString('review.url', p.remote.review)
- elif gr.GetString('review.url') != p.remote.review:
- gr.SetString('review.%s.url' % p.name, p.remote.review)
-
- r = p.revisionExpr
- if r and not IsId(r):
- if r.startswith(R_HEADS):
- r = r[len(R_HEADS):]
- if r == b:
- r = '.'
- gm.SetString('submodule.%s.revision' % p.name, r)
-
- for c in p.copyfiles:
- info.append('Moved %s out of %s' % (c.src, p.relpath))
- c._Copy()
- p.work_git.rm(c.src)
- mp.work_git.add(c.dest)
-
- self.SetRevisionId(p.relpath, p.GetRevisionId())
- mp.work_git.add('.gitignore', '.gitmodules', '.review')
- pm.end()
- for i in info:
- print >>sys.stderr, i
-
- def _Unload(self):
- self._loaded = False
- self._projects = {}
- self._revisionIds = None
- self.branch = None
-
- def _Load(self):
- if not self._loaded:
- f = os.path.join(self.repodir, manifest_xml.LOCAL_MANIFEST_NAME)
- if os.path.exists(f):
- print >>sys.stderr, 'warning: ignoring %s' % f
-
- m = self.manifestProject
- b = m.CurrentBranch
- if not b:
- raise ManifestParseError, 'manifest cannot be on detached HEAD'
- b = m.GetBranch(b).merge
- if b.startswith(R_HEADS):
- b = b[len(R_HEADS):]
- self.branch = b
- m.remote.name = self._Remote().name
-
- self._ParseModules()
-
- if self.IsMirror:
- self._AddMetaProjectMirror(self.repoProject)
- self._AddMetaProjectMirror(self.manifestProject)
-
- self._loaded = True
-
- def _ParseModules(self):
- byPath = dict()
- for name in self._modules.GetSubSections('submodule'):
- p = self._ParseProject(name)
- if self._projects.get(p.name):
- raise ManifestParseError, 'duplicate project "%s"' % p.name
- if byPath.get(p.relpath):
- raise ManifestParseError, 'duplicate path "%s"' % p.relpath
- self._projects[p.name] = p
- byPath[p.relpath] = p
-
- for relpath in self._allRevisionIds.keys():
- if relpath not in byPath:
- raise ManifestParseError, \
- 'project "%s" not in .gitmodules' \
- % relpath
-
- def _Remote(self):
- m = self.manifestProject
- b = m.GetBranch(m.CurrentBranch)
- return b.remote
-
- def _ResolveUrl(self, url):
- if url.startswith('./') or url.startswith('../'):
- base = self._Remote().url
- try:
- base = base[:base.rindex('/')+1]
- except ValueError:
- base = base[:base.rindex(':')+1]
- if url.startswith('./'):
- url = url[2:]
- while '/' in base and url.startswith('../'):
- base = base[:base.rindex('/')+1]
- url = url[3:]
- return base + url
- return url
-
- def _GetRevisionId(self, path):
- return self._allRevisionIds.get(path)
-
- @property
- def _allRevisionIds(self):
- if self._revisionIds is None:
- a = dict()
- p = GitCommand(self.manifestProject,
- ['ls-files','-z','--stage'],
- capture_stdout = True)
- for line in p.process.stdout.read().split('\0')[:-1]:
- l_info, l_path = line.split('\t', 2)
- l_mode, l_id, l_stage = l_info.split(' ', 2)
- if l_mode == GITLINK and l_stage == '0':
- a[l_path] = l_id
- p.Wait()
- self._revisionIds = a
- return self._revisionIds
-
- def SetRevisionId(self, path, id):
- self.manifestProject.work_git.update_index(
- '--add','--cacheinfo', GITLINK, id, path)
-
- def _ParseProject(self, name):
- gm = self._modules
- gr = self._review
-
- path = gm.GetString('submodule.%s.path' % name)
- if not path:
- path = name
-
- revId = self._GetRevisionId(path)
- if not revId:
- raise ManifestParseError(
- 'submodule "%s" has no revision at "%s"' \
- % (name, path))
-
- url = gm.GetString('submodule.%s.url' % name)
- if not url:
- url = name
- url = self._ResolveUrl(url)
-
- review = gr.GetString('review.%s.url' % name)
- if not review:
- review = gr.GetString('review.url')
- if not review:
- review = self._Remote().review
-
- remote = RemoteSpec(self._Remote().name, url, review)
- revExpr = gm.GetString('submodule.%s.revision' % name)
- if revExpr == '.':
- revExpr = self.branch
-
- if self.IsMirror:
- relpath = None
- worktree = None
- gitdir = os.path.join(self.topdir, '%s.git' % name)
- else:
- worktree = os.path.join(self.topdir, path)
- gitdir = os.path.join(self.repodir, 'projects/%s.git' % name)
-
- return Project(manifest = self,
- name = name,
- remote = remote,
- gitdir = gitdir,
- worktree = worktree,
- relpath = path,
- revisionExpr = revExpr,
- revisionId = revId)
-
- def _AddMetaProjectMirror(self, m):
- m_url = m.GetRemote(m.remote.name).url
- if m_url.endswith('/.git'):
- raise ManifestParseError, 'refusing to mirror %s' % m_url
-
- name = self._GuessMetaName(m_url)
- if name.endswith('.git'):
- name = name[:-4]
-
- if name not in self._projects:
- m.PreSync()
- gitdir = os.path.join(self.topdir, '%s.git' % name)
- project = Project(manifest = self,
- name = name,
- remote = RemoteSpec(self._Remote().name, m_url),
- gitdir = gitdir,
- worktree = None,
- relpath = None,
- revisionExpr = m.revisionExpr,
- revisionId = None)
- self._projects[project.name] = project
-
- def _GuessMetaName(self, m_url):
- parts = m_url.split('/')
- name = parts[-1]
- parts = parts[0:-1]
- s = len(parts) - 1
- while s > 0:
- l = '/'.join(parts[0:s]) + '/'
- r = '/'.join(parts[s:]) + '/'
- for p in self._projects.values():
- if p.name.startswith(r) and p.remote.url.startswith(l):
- return r + name
- s -= 1
- return m_url[m_url.rindex('/') + 1:]
diff --git a/manifest_xml.py b/manifest_xml.py
index 1d02f9d..26cc14f 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -13,53 +13,75 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import itertools
import os
+import re
import sys
+import urlparse
import xml.dom.minidom
-from git_config import GitConfig
-from git_config import IsId
-from manifest import Manifest
-from project import RemoteSpec
-from project import Project
-from project import MetaProject
-from project import R_HEADS
-from project import HEAD
+from git_config import GitConfig, IsId
+from project import RemoteSpec, Project, MetaProject, R_HEADS, HEAD
from error import ManifestParseError
MANIFEST_FILE_NAME = 'manifest.xml'
LOCAL_MANIFEST_NAME = 'local_manifest.xml'
-R_M = 'refs/remotes/m/'
+
+urlparse.uses_relative.extend(['ssh', 'git'])
+urlparse.uses_netloc.extend(['ssh', 'git'])
class _Default(object):
"""Project defaults within the manifest."""
revisionExpr = None
remote = None
+ sync_j = 1
+ sync_c = False
class _XmlRemote(object):
def __init__(self,
name,
+ alias=None,
fetch=None,
+ manifestUrl=None,
review=None):
self.name = name
self.fetchUrl = fetch
+ self.manifestUrl = manifestUrl
+ self.remoteAlias = alias
self.reviewUrl = review
+ self.resolvedFetchUrl = self._resolveFetchUrl()
+
+ def _resolveFetchUrl(self):
+ url = self.fetchUrl.rstrip('/')
+ manifestUrl = self.manifestUrl.rstrip('/')
+ # urljoin will get confused if there is no scheme in the base url
+ # ie, if manifestUrl is of the form <hostname:port>
+ if manifestUrl.find(':') != manifestUrl.find('/') - 1:
+ manifestUrl = 'gopher://' + manifestUrl
+ url = urlparse.urljoin(manifestUrl, url)
+ return re.sub(r'^gopher://', '', url)
def ToRemoteSpec(self, projectName):
- url = self.fetchUrl
- while url.endswith('/'):
- url = url[:-1]
- url += '/%s.git' % projectName
- return RemoteSpec(self.name, url, self.reviewUrl)
+ url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
+ remoteName = self.name
+ if self.remoteAlias:
+ remoteName = self.remoteAlias
+ return RemoteSpec(remoteName, url, self.reviewUrl)
-class XmlManifest(Manifest):
+class XmlManifest(object):
"""manages the repo configuration file"""
def __init__(self, repodir):
- Manifest.__init__(self, repodir)
+ self.repodir = os.path.abspath(repodir)
+ self.topdir = os.path.dirname(self.repodir)
+ self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
+ self.globalConfig = GitConfig.ForUser()
- self._manifestFile = os.path.join(repodir, MANIFEST_FILE_NAME)
+ self.repoProject = MetaProject(self, 'repo',
+ gitdir = os.path.join(repodir, 'repo/.git'),
+ worktree = os.path.join(repodir, 'repo'))
+
self.manifestProject = MetaProject(self, 'manifests',
gitdir = os.path.join(repodir, 'manifests.git'),
worktree = os.path.join(repodir, 'manifests'))
@@ -73,13 +95,13 @@
if not os.path.isfile(path):
raise ManifestParseError('manifest %s not found' % name)
- old = self._manifestFile
+ old = self.manifestFile
try:
- self._manifestFile = path
+ self.manifestFile = path
self._Unload()
self._Load()
finally:
- self._manifestFile = old
+ self.manifestFile = old
def Link(self, name):
"""Update the repo metadata to use a different manifest.
@@ -87,9 +109,9 @@
self.Override(name)
try:
- if os.path.exists(self._manifestFile):
- os.remove(self._manifestFile)
- os.symlink('manifests/%s' % name, self._manifestFile)
+ if os.path.exists(self.manifestFile):
+ os.remove(self.manifestFile)
+ os.symlink('manifests/%s' % name, self.manifestFile)
except OSError, e:
raise ManifestParseError('cannot link manifest %s' % name)
@@ -104,6 +126,13 @@
def Save(self, fd, peg_rev=False):
"""Write the current manifest out to the given file descriptor.
"""
+ mp = self.manifestProject
+
+ groups = mp.config.GetString('manifest.groups')
+ if not groups:
+ groups = 'default'
+ groups = [x for x in re.split(r'[,\s]+', groups) if x]
+
doc = xml.dom.minidom.Document()
root = doc.createElement('manifest')
doc.appendChild(root)
@@ -134,6 +163,12 @@
if d.revisionExpr:
have_default = True
e.setAttribute('revision', d.revisionExpr)
+ if d.sync_j > 1:
+ have_default = True
+ e.setAttribute('sync-j', '%d' % d.sync_j)
+ if d.sync_c:
+ have_default = True
+ e.setAttribute('sync-c', 'true')
if have_default:
root.appendChild(e)
root.appendChild(doc.createTextNode(''))
@@ -149,6 +184,10 @@
for p in sort_projects:
p = self.projects[p]
+
+ if not p.MatchesGroups(groups):
+ continue
+
e = doc.createElement('project')
root.appendChild(e)
e.setAttribute('name', p.name)
@@ -172,6 +211,29 @@
ce.setAttribute('dest', c.dest)
e.appendChild(ce)
+ default_groups = ['default', 'name:%s' % p.name, 'path:%s' % p.relpath]
+ egroups = [g for g in p.groups if g not in default_groups]
+ if egroups:
+ e.setAttribute('groups', ','.join(egroups))
+
+ for a in p.annotations:
+ if a.keep == "true":
+ ae = doc.createElement('annotation')
+ ae.setAttribute('name', a.name)
+ ae.setAttribute('value', a.value)
+ e.appendChild(ae)
+
+ if p.sync_c:
+ e.setAttribute('sync-c', 'true')
+
+ if self._repo_hooks_project:
+ root.appendChild(doc.createTextNode(''))
+ e = doc.createElement('repo-hooks')
+ e.setAttribute('in-project', self._repo_hooks_project.name)
+ e.setAttribute('enabled-list',
+ ' '.join(self._repo_hooks_project.enabled_repo_hooks))
+ root.appendChild(e)
+
doc.writexml(fd, '', ' ', '\n', 'UTF-8')
@property
@@ -190,6 +252,11 @@
return self._default
@property
+ def repo_hooks_project(self):
+ self._Load()
+ return self._repo_hooks_project
+
+ @property
def notice(self):
self._Load()
return self._notice
@@ -199,21 +266,16 @@
self._Load()
return self._manifest_server
- def InitBranch(self):
- m = self.manifestProject
- if m.CurrentBranch is None:
- return m.StartBranch('default')
- return True
-
- def SetMRefs(self, project):
- if self.branch:
- project._InitAnyMRef(R_M + self.branch)
+ @property
+ def IsMirror(self):
+ return self.manifestProject.config.GetBoolean('repo.mirror')
def _Unload(self):
self._loaded = False
self._projects = {}
self._remotes = {}
self._default = None
+ self._repo_hooks_project = None
self._notice = None
self.branch = None
self._manifest_server = None
@@ -221,24 +283,20 @@
def _Load(self):
if not self._loaded:
m = self.manifestProject
- b = m.GetBranch(m.CurrentBranch)
- if b.remote and b.remote.name:
- m.remote.name = b.remote.name
- b = b.merge
+ b = m.GetBranch(m.CurrentBranch).merge
if b is not None and b.startswith(R_HEADS):
b = b[len(R_HEADS):]
self.branch = b
- self._ParseManifest(True)
+ nodes = []
+ nodes.append(self._ParseManifestXml(self.manifestFile,
+ self.manifestProject.worktree))
local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
if os.path.exists(local):
- try:
- real = self._manifestFile
- self._manifestFile = local
- self._ParseManifest(False)
- finally:
- self._manifestFile = real
+ nodes.append(self._ParseManifestXml(local, self.repodir))
+
+ self._ParseManifest(nodes)
if self.IsMirror:
self._AddMetaProjectMirror(self.repoProject)
@@ -246,73 +304,117 @@
self._loaded = True
- def _ParseManifest(self, is_root_file):
- root = xml.dom.minidom.parse(self._manifestFile)
+ def _ParseManifestXml(self, path, include_root):
+ root = xml.dom.minidom.parse(path)
if not root or not root.childNodes:
- raise ManifestParseError, \
- "no root node in %s" % \
- self._manifestFile
+ raise ManifestParseError("no root node in %s" % (path,))
config = root.childNodes[0]
if config.nodeName != 'manifest':
- raise ManifestParseError, \
- "no <manifest> in %s" % \
- self._manifestFile
+ raise ManifestParseError("no <manifest> in %s" % (path,))
+ nodes = []
for node in config.childNodes:
+ if node.nodeName == 'include':
+ name = self._reqatt(node, 'name')
+ fp = os.path.join(include_root, name)
+ if not os.path.isfile(fp):
+ raise ManifestParseError, \
+ "include %s doesn't exist or isn't a file" % \
+ (name,)
+ try:
+ nodes.extend(self._ParseManifestXml(fp, include_root))
+ # should isolate this to the exact exception, but that's
+ # tricky. actual parsing implementation may vary.
+ except (KeyboardInterrupt, RuntimeError, SystemExit):
+ raise
+ except Exception, e:
+ raise ManifestParseError(
+ "failed parsing included manifest %s: %s", (name, e))
+ else:
+ nodes.append(node)
+ return nodes
+
+ def _ParseManifest(self, node_list):
+ for node in itertools.chain(*node_list):
+ if node.nodeName == 'remote':
+ remote = self._ParseRemote(node)
+ if self._remotes.get(remote.name):
+ raise ManifestParseError(
+ 'duplicate remote %s in %s' %
+ (remote.name, self.manifestFile))
+ self._remotes[remote.name] = remote
+
+ for node in itertools.chain(*node_list):
+ if node.nodeName == 'default':
+ if self._default is not None:
+ raise ManifestParseError(
+ 'duplicate default in %s' %
+ (self.manifestFile))
+ self._default = self._ParseDefault(node)
+ if self._default is None:
+ self._default = _Default()
+
+ for node in itertools.chain(*node_list):
+ if node.nodeName == 'notice':
+ if self._notice is not None:
+ raise ManifestParseError(
+ 'duplicate notice in %s' %
+ (self.manifestFile))
+ self._notice = self._ParseNotice(node)
+
+ for node in itertools.chain(*node_list):
+ if node.nodeName == 'manifest-server':
+ url = self._reqatt(node, 'url')
+ if self._manifest_server is not None:
+ raise ManifestParseError(
+ 'duplicate manifest-server in %s' %
+ (self.manifestFile))
+ self._manifest_server = url
+
+ for node in itertools.chain(*node_list):
+ if node.nodeName == 'project':
+ project = self._ParseProject(node)
+ if self._projects.get(project.name):
+ raise ManifestParseError(
+ 'duplicate project %s in %s' %
+ (project.name, self.manifestFile))
+ self._projects[project.name] = project
+ if node.nodeName == 'repo-hooks':
+ # Get the name of the project and the (space-separated) list of enabled.
+ repo_hooks_project = self._reqatt(node, 'in-project')
+ enabled_repo_hooks = self._reqatt(node, 'enabled-list').split()
+
+ # Only one project can be the hooks project
+ if self._repo_hooks_project is not None:
+ raise ManifestParseError(
+ 'duplicate repo-hooks in %s' %
+ (self.manifestFile))
+
+ # Store a reference to the Project.
+ try:
+ self._repo_hooks_project = self._projects[repo_hooks_project]
+ except KeyError:
+ raise ManifestParseError(
+ 'project %s not found for repo-hooks' %
+ (repo_hooks_project))
+
+ # Store the enabled hooks in the Project object.
+ self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
if node.nodeName == 'remove-project':
name = self._reqatt(node, 'name')
try:
del self._projects[name]
except KeyError:
- raise ManifestParseError, \
- 'project %s not found' % \
- (name)
+ raise ManifestParseError(
+ 'project %s not found' %
+ (name))
- for node in config.childNodes:
- if node.nodeName == 'remote':
- remote = self._ParseRemote(node)
- if self._remotes.get(remote.name):
- raise ManifestParseError, \
- 'duplicate remote %s in %s' % \
- (remote.name, self._manifestFile)
- self._remotes[remote.name] = remote
+ # If the manifest removes the hooks project, treat it as if it deleted
+ # the repo-hooks element too.
+ if self._repo_hooks_project and (self._repo_hooks_project.name == name):
+ self._repo_hooks_project = None
- for node in config.childNodes:
- if node.nodeName == 'default':
- if self._default is not None:
- raise ManifestParseError, \
- 'duplicate default in %s' % \
- (self._manifestFile)
- self._default = self._ParseDefault(node)
- if self._default is None:
- self._default = _Default()
-
- for node in config.childNodes:
- if node.nodeName == 'notice':
- if self._notice is not None:
- raise ManifestParseError, \
- 'duplicate notice in %s' % \
- (self.manifestFile)
- self._notice = self._ParseNotice(node)
-
- for node in config.childNodes:
- if node.nodeName == 'manifest-server':
- url = self._reqatt(node, 'url')
- if self._manifest_server is not None:
- raise ManifestParseError, \
- 'duplicate manifest-server in %s' % \
- (self.manifestFile)
- self._manifest_server = url
-
- for node in config.childNodes:
- if node.nodeName == 'project':
- project = self._ParseProject(node)
- if self._projects.get(project.name):
- raise ManifestParseError, \
- 'duplicate project %s in %s' % \
- (project.name, self._manifestFile)
- self._projects[project.name] = project
def _AddMetaProjectMirror(self, m):
name = None
@@ -321,7 +423,7 @@
raise ManifestParseError, 'refusing to mirror %s' % m_url
if self._default and self._default.remote:
- url = self._default.remote.fetchUrl
+ url = self._default.remote.resolvedFetchUrl
if not url.endswith('/'):
url += '/'
if m_url.startswith(url):
@@ -330,7 +432,8 @@
if name is None:
s = m_url.rindex('/') + 1
- remote = _XmlRemote('origin', m_url[:s])
+ manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
+ remote = _XmlRemote('origin', fetch=m_url[:s], manifestUrl=manifestUrl)
name = m_url[s:]
if name.endswith('.git'):
@@ -354,11 +457,15 @@
reads a <remote> element from the manifest file
"""
name = self._reqatt(node, 'name')
+ alias = node.getAttribute('alias')
+ if alias == '':
+ alias = None
fetch = self._reqatt(node, 'fetch')
review = node.getAttribute('review')
if review == '':
review = None
- return _XmlRemote(name, fetch, review)
+ manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
+ return _XmlRemote(name, alias, fetch, manifestUrl, review)
def _ParseDefault(self, node):
"""
@@ -369,6 +476,18 @@
d.revisionExpr = node.getAttribute('revision')
if d.revisionExpr == '':
d.revisionExpr = None
+
+ sync_j = node.getAttribute('sync-j')
+ if sync_j == '' or sync_j is None:
+ d.sync_j = 1
+ else:
+ d.sync_j = int(sync_j)
+
+ sync_c = node.getAttribute('sync-c')
+ if not sync_c:
+ d.sync_c = False
+ else:
+ d.sync_c = sync_c.lower() in ("yes", "true", "1")
return d
def _ParseNotice(self, node):
@@ -422,7 +541,7 @@
if remote is None:
raise ManifestParseError, \
"no remote for project %s within %s" % \
- (name, self._manifestFile)
+ (name, self.manifestFile)
revisionExpr = node.getAttribute('revision')
if not revisionExpr:
@@ -430,7 +549,7 @@
if not revisionExpr:
raise ManifestParseError, \
"no revision for project %s within %s" % \
- (name, self._manifestFile)
+ (name, self.manifestFile)
path = node.getAttribute('path')
if not path:
@@ -438,7 +557,27 @@
if path.startswith('/'):
raise ManifestParseError, \
"project %s path cannot be absolute in %s" % \
- (name, self._manifestFile)
+ (name, self.manifestFile)
+
+ rebase = node.getAttribute('rebase')
+ if not rebase:
+ rebase = True
+ else:
+ rebase = rebase.lower() in ("yes", "true", "1")
+
+ sync_c = node.getAttribute('sync-c')
+ if not sync_c:
+ sync_c = False
+ else:
+ sync_c = sync_c.lower() in ("yes", "true", "1")
+
+ groups = ''
+ if node.hasAttribute('groups'):
+ groups = node.getAttribute('groups')
+ groups = [x for x in re.split('[,\s]+', groups) if x]
+
+ default_groups = ['default', 'name:%s' % name, 'path:%s' % path]
+ groups.extend(set(default_groups).difference(groups))
if self.IsMirror:
relpath = None
@@ -455,11 +594,16 @@
worktree = worktree,
relpath = path,
revisionExpr = revisionExpr,
- revisionId = None)
+ revisionId = None,
+ rebase = rebase,
+ groups = groups,
+ sync_c = sync_c)
for n in node.childNodes:
if n.nodeName == 'copyfile':
self._ParseCopyFile(project, n)
+ if n.nodeName == 'annotation':
+ self._ParseAnnotation(project, n)
return project
@@ -471,6 +615,17 @@
# dest is relative to the top of the tree
project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
+ def _ParseAnnotation(self, project, node):
+ name = self._reqatt(node, 'name')
+ value = self._reqatt(node, 'value')
+ try:
+ keep = self._reqatt(node, 'keep').lower()
+ except ManifestParseError:
+ keep = "true"
+ if keep != "true" and keep != "false":
+ raise ManifestParseError, "optional \"keep\" attribute must be \"true\" or \"false\""
+ project.AddAnnotation(name, value, keep)
+
def _get_remote(self, node):
name = node.getAttribute('remote')
if not name:
@@ -480,7 +635,7 @@
if not v:
raise ManifestParseError, \
"remote %s not defined in %s" % \
- (name, self._manifestFile)
+ (name, self.manifestFile)
return v
def _reqatt(self, node, attname):
@@ -491,5 +646,5 @@
if not v:
raise ManifestParseError, \
"no %s in <%s> within %s" % \
- (attname, node.nodeName, self._manifestFile)
+ (attname, node.nodeName, self.manifestFile)
return v
diff --git a/progress.py b/progress.py
index 2ace701..d948654 100644
--- a/progress.py
+++ b/progress.py
@@ -21,13 +21,14 @@
_NOT_TTY = not os.isatty(2)
class Progress(object):
- def __init__(self, title, total=0):
+ def __init__(self, title, total=0, units=''):
self._title = title
self._total = total
self._done = 0
self._lastp = -1
self._start = time()
self._show = False
+ self._units = units
def update(self, inc=1):
self._done += inc
@@ -51,11 +52,11 @@
if self._lastp != p:
self._lastp = p
- sys.stderr.write('\r%s: %3d%% (%d/%d) ' % (
+ sys.stderr.write('\r%s: %3d%% (%d%s/%d%s) ' % (
self._title,
p,
- self._done,
- self._total))
+ self._done, self._units,
+ self._total, self._units))
sys.stderr.flush()
def end(self):
@@ -69,9 +70,9 @@
sys.stderr.flush()
else:
p = (100 * self._done) / self._total
- sys.stderr.write('\r%s: %3d%% (%d/%d), done. \n' % (
+ sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done. \n' % (
self._title,
p,
- self._done,
- self._total))
+ self._done, self._units,
+ self._total, self._units))
sys.stderr.flush()
diff --git a/project.py b/project.py
index b404494..00ebb17 100644
--- a/project.py
+++ b/project.py
@@ -12,22 +12,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import traceback
import errno
import filecmp
import os
+import random
import re
import shutil
import stat
+import subprocess
import sys
-import urllib2
+import time
from color import Coloring
from git_command import GitCommand
-from git_config import GitConfig, IsId
-from error import GitError, ImportError, UploadError
+from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
+from error import DownloadError
+from error import GitError, HookError, ImportError, UploadError
from error import ManifestInvalidRevisionError
+from progress import Progress
+from trace import IsTrace, Trace
-from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB
+from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
def _lwrite(path, content):
lock = '%s.lock' % path
@@ -54,29 +60,25 @@
def sq(r):
return "'" + r.replace("'", "'\''") + "'"
-hook_list = None
-def repo_hooks():
- global hook_list
- if hook_list is None:
+_project_hook_list = None
+def _ProjectHooks():
+ """List the hooks present in the 'hooks' directory.
+
+ These hooks are project hooks and are copied to the '.git/hooks' directory
+ of all subprojects.
+
+ This function caches the list of hooks (based on the contents of the
+ 'repo/hooks' directory) on the first call.
+
+ Returns:
+ A list of absolute paths to all of the files in the hooks directory.
+ """
+ global _project_hook_list
+ if _project_hook_list is None:
d = os.path.abspath(os.path.dirname(__file__))
d = os.path.join(d , 'hooks')
- hook_list = map(lambda x: os.path.join(d, x), os.listdir(d))
- return hook_list
-
-def relpath(dst, src):
- src = os.path.dirname(src)
- top = os.path.commonprefix([dst, src])
- if top.endswith('/'):
- top = top[:-1]
- else:
- top = os.path.dirname(top)
-
- tmp = src
- rel = ''
- while top != tmp:
- rel += '../'
- tmp = os.path.dirname(tmp)
- return rel + dst[len(top) + 1:]
+ _project_hook_list = map(lambda x: os.path.join(d, x), os.listdir(d))
+ return _project_hook_list
class DownloadedChange(object):
@@ -148,10 +150,11 @@
R_HEADS + self.name,
'--')
- def UploadForReview(self, people, auto_topic=False):
+ def UploadForReview(self, people, auto_topic=False, draft=False):
self.project.UploadForReview(self.name,
people,
- auto_topic=auto_topic)
+ auto_topic=auto_topic,
+ draft=draft)
def GetPublishedRefs(self):
refs = {}
@@ -185,6 +188,11 @@
Coloring.__init__(self, config, 'diff')
self.project = self.printer('header', attr = 'bold')
+class _Annotation:
+ def __init__(self, name, value, keep):
+ self.name = name
+ self.value = value
+ self.keep = keep
class _CopyFile:
def __init__(self, src, dest, abssrc, absdest):
@@ -223,6 +231,249 @@
self.url = url
self.review = review
+class RepoHook(object):
+ """A RepoHook contains information about a script to run as a hook.
+
+ Hooks are used to run a python script before running an upload (for instance,
+ to run presubmit checks). Eventually, we may have hooks for other actions.
+
+ This shouldn't be confused with files in the 'repo/hooks' directory. Those
+ files are copied into each '.git/hooks' folder for each project. Repo-level
+ hooks are associated instead with repo actions.
+
+ Hooks are always python. When a hook is run, we will load the hook into the
+ interpreter and execute its main() function.
+ """
+ def __init__(self,
+ hook_type,
+ hooks_project,
+ topdir,
+ abort_if_user_denies=False):
+ """RepoHook constructor.
+
+ Params:
+ hook_type: A string representing the type of hook. This is also used
+ to figure out the name of the file containing the hook. For
+ example: 'pre-upload'.
+ hooks_project: The project containing the repo hooks. If you have a
+ manifest, this is manifest.repo_hooks_project. OK if this is None,
+ which will make the hook a no-op.
+ topdir: Repo's top directory (the one containing the .repo directory).
+ Scripts will run with CWD as this directory. If you have a manifest,
+ this is manifest.topdir
+ abort_if_user_denies: If True, we'll throw a HookError() if the user
+ doesn't allow us to run the hook.
+ """
+ self._hook_type = hook_type
+ self._hooks_project = hooks_project
+ self._topdir = topdir
+ self._abort_if_user_denies = abort_if_user_denies
+
+ # Store the full path to the script for convenience.
+ if self._hooks_project:
+ self._script_fullpath = os.path.join(self._hooks_project.worktree,
+ self._hook_type + '.py')
+ else:
+ self._script_fullpath = None
+
+ def _GetHash(self):
+ """Return a hash of the contents of the hooks directory.
+
+ We'll just use git to do this. This hash has the property that if anything
+ changes in the directory we will return a different has.
+
+ SECURITY CONSIDERATION:
+ This hash only represents the contents of files in the hook directory, not
+ any other files imported or called by hooks. Changes to imported files
+ can change the script behavior without affecting the hash.
+
+ Returns:
+ A string representing the hash. This will always be ASCII so that it can
+ be printed to the user easily.
+ """
+ assert self._hooks_project, "Must have hooks to calculate their hash."
+
+ # We will use the work_git object rather than just calling GetRevisionId().
+ # That gives us a hash of the latest checked in version of the files that
+ # the user will actually be executing. Specifically, GetRevisionId()
+ # doesn't appear to change even if a user checks out a different version
+ # of the hooks repo (via git checkout) nor if a user commits their own revs.
+ #
+ # NOTE: Local (non-committed) changes will not be factored into this hash.
+ # I think this is OK, since we're really only worried about warning the user
+ # about upstream changes.
+ return self._hooks_project.work_git.rev_parse('HEAD')
+
+ def _GetMustVerb(self):
+ """Return 'must' if the hook is required; 'should' if not."""
+ if self._abort_if_user_denies:
+ return 'must'
+ else:
+ return 'should'
+
+ def _CheckForHookApproval(self):
+ """Check to see whether this hook has been approved.
+
+ We'll look at the hash of all of the hooks. If this matches the hash that
+ the user last approved, we're done. If it doesn't, we'll ask the user
+ about approval.
+
+ Note that we ask permission for each individual hook even though we use
+ the hash of all hooks when detecting changes. We'd like the user to be
+ able to approve / deny each hook individually. We only use the hash of all
+ hooks because there is no other easy way to detect changes to local imports.
+
+ Returns:
+ True if this hook is approved to run; False otherwise.
+
+ Raises:
+ HookError: Raised if the user doesn't approve and abort_if_user_denies
+ was passed to the consturctor.
+ """
+ hooks_dir = self._hooks_project.worktree
+ hooks_config = self._hooks_project.config
+ git_approval_key = 'repo.hooks.%s.approvedhash' % self._hook_type
+
+ # Get the last hash that the user approved for this hook; may be None.
+ old_hash = hooks_config.GetString(git_approval_key)
+
+ # Get the current hash so we can tell if scripts changed since approval.
+ new_hash = self._GetHash()
+
+ if old_hash is not None:
+ # User previously approved hook and asked not to be prompted again.
+ if new_hash == old_hash:
+ # Approval matched. We're done.
+ return True
+ else:
+ # Give the user a reason why we're prompting, since they last told
+ # us to "never ask again".
+ prompt = 'WARNING: Scripts have changed since %s was allowed.\n\n' % (
+ self._hook_type)
+ else:
+ prompt = ''
+
+ # Prompt the user if we're not on a tty; on a tty we'll assume "no".
+ if sys.stdout.isatty():
+ prompt += ('Repo %s run the script:\n'
+ ' %s\n'
+ '\n'
+ 'Do you want to allow this script to run '
+ '(yes/yes-never-ask-again/NO)? ') % (
+ self._GetMustVerb(), self._script_fullpath)
+ response = raw_input(prompt).lower()
+ print
+
+ # User is doing a one-time approval.
+ if response in ('y', 'yes'):
+ return True
+ elif response == 'yes-never-ask-again':
+ hooks_config.SetString(git_approval_key, new_hash)
+ return True
+
+ # For anything else, we'll assume no approval.
+ if self._abort_if_user_denies:
+ raise HookError('You must allow the %s hook or use --no-verify.' %
+ self._hook_type)
+
+ return False
+
+ def _ExecuteHook(self, **kwargs):
+ """Actually execute the given hook.
+
+ This will run the hook's 'main' function in our python interpreter.
+
+ Args:
+ kwargs: Keyword arguments to pass to the hook. These are often specific
+ to the hook type. For instance, pre-upload hooks will contain
+ a project_list.
+ """
+ # Keep sys.path and CWD stashed away so that we can always restore them
+ # upon function exit.
+ orig_path = os.getcwd()
+ orig_syspath = sys.path
+
+ try:
+ # Always run hooks with CWD as topdir.
+ os.chdir(self._topdir)
+
+ # Put the hook dir as the first item of sys.path so hooks can do
+ # relative imports. We want to replace the repo dir as [0] so
+ # hooks can't import repo files.
+ sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
+
+ # Exec, storing global context in the context dict. We catch exceptions
+ # and convert to a HookError w/ just the failing traceback.
+ context = {}
+ try:
+ execfile(self._script_fullpath, context)
+ except Exception:
+ raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
+ traceback.format_exc(), self._hook_type))
+
+ # Running the script should have defined a main() function.
+ if 'main' not in context:
+ raise HookError('Missing main() in: "%s"' % self._script_fullpath)
+
+
+ # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
+ # We don't actually want hooks to define their main with this argument--
+ # it's there to remind them that their hook should always take **kwargs.
+ # For instance, a pre-upload hook should be defined like:
+ # def main(project_list, **kwargs):
+ #
+ # This allows us to later expand the API without breaking old hooks.
+ kwargs = kwargs.copy()
+ kwargs['hook_should_take_kwargs'] = True
+
+ # Call the main function in the hook. If the hook should cause the
+ # build to fail, it will raise an Exception. We'll catch that convert
+ # to a HookError w/ just the failing traceback.
+ try:
+ context['main'](**kwargs)
+ except Exception:
+ raise HookError('%s\nFailed to run main() for %s hook; see traceback '
+ 'above.' % (
+ traceback.format_exc(), self._hook_type))
+ finally:
+ # Restore sys.path and CWD.
+ sys.path = orig_syspath
+ os.chdir(orig_path)
+
+ def Run(self, user_allows_all_hooks, **kwargs):
+ """Run the hook.
+
+ If the hook doesn't exist (because there is no hooks project or because
+ this particular hook is not enabled), this is a no-op.
+
+ Args:
+ user_allows_all_hooks: If True, we will never prompt about running the
+ hook--we'll just assume it's OK to run it.
+ kwargs: Keyword arguments to pass to the hook. These are often specific
+ to the hook type. For instance, pre-upload hooks will contain
+ a project_list.
+
+ Raises:
+ HookError: If there was a problem finding the hook or the user declined
+ to run a required hook (from _CheckForHookApproval).
+ """
+ # No-op if there is no hooks project or if hook is disabled.
+ if ((not self._hooks_project) or
+ (self._hook_type not in self._hooks_project.enabled_repo_hooks)):
+ return
+
+ # Bail with a nice error if we can't find the hook.
+ if not os.path.isfile(self._script_fullpath):
+ raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
+
+ # Make sure the user is OK with running the hook.
+ if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
+ return
+
+ # Run the hook with the same version of python we're using.
+ self._ExecuteHook(**kwargs)
+
+
class Project(object):
def __init__(self,
manifest,
@@ -232,7 +483,10 @@
worktree,
relpath,
revisionExpr,
- revisionId):
+ revisionId,
+ rebase = True,
+ groups = None,
+ sync_c = False):
self.manifest = manifest
self.name = name
self.remote = remote
@@ -251,8 +505,13 @@
else:
self.revisionId = revisionId
+ self.rebase = rebase
+ self.groups = groups
+ self.sync_c = sync_c
+
self.snapshots = {}
self.copyfiles = []
+ self.annotations = []
self.config = GitConfig.ForRepository(
gitdir = self.gitdir,
defaults = self.manifest.globalConfig)
@@ -264,6 +523,10 @@
self.bare_git = self._GitGetByExec(self, bare=True)
self.bare_ref = GitRefs(gitdir)
+ # This will be filled in if a project is later identified to be the
+ # project containing repo hooks.
+ self.enabled_repo_hooks = []
+
@property
def Exists(self):
return os.path.isdir(self.gitdir)
@@ -367,6 +630,27 @@
return heads
+ def MatchesGroups(self, manifest_groups):
+ """Returns true if the manifest groups specified at init should cause
+ this project to be synced.
+ Prefixing a manifest group with "-" inverts the meaning of a group.
+ All projects are implicitly labelled with "default".
+
+ labels are resolved in order. In the example case of
+ project_groups: "default,group1,group2"
+ manifest_groups: "-group1,group2"
+ the project will be matched.
+ """
+ if self.groups is None:
+ return True
+ matched = False
+ for group in manifest_groups:
+ if group.startswith('-') and group[1:] in self.groups:
+ matched = False
+ elif group in self.groups:
+ matched = True
+
+ return matched
## Status Display ##
@@ -391,13 +675,18 @@
return False
- def PrintWorkTreeStatus(self):
+ def PrintWorkTreeStatus(self, output_redir=None):
"""Prints the status of the repository to stdout.
+
+ Args:
+ output: If specified, redirect the output to this object.
"""
if not os.path.isdir(self.worktree):
- print ''
- print 'project %s/' % self.relpath
- print ' missing (run "repo sync")'
+ if output_redir == None:
+ output_redir = sys.stdout
+ print >>output_redir, ''
+ print >>output_redir, 'project %s/' % self.relpath
+ print >>output_redir, ' missing (run "repo sync")'
return
self.work_git.update_index('-q',
@@ -408,10 +697,12 @@
di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
df = self.work_git.DiffZ('diff-files')
do = self.work_git.LsOthers()
- if not rb and not di and not df and not do:
+ if not rb and not di and not df and not do and not self.CurrentBranch:
return 'CLEAN'
out = StatusColoring(self.config)
+ if not output_redir == None:
+ out.redirect(output_redir)
out.project('project %-40s', self.relpath + '/')
branch = self.CurrentBranch
@@ -461,9 +752,10 @@
else:
out.write('%s', line)
out.nl()
+
return 'DIRTY'
- def PrintWorkTreeDiff(self):
+ def PrintWorkTreeDiff(self, absolute_paths=False):
"""Prints the status of the repository to stdout.
"""
out = DiffColoring(self.config)
@@ -471,6 +763,9 @@
if out.is_on:
cmd.append('--color')
cmd.append(HEAD)
+ if absolute_paths:
+ cmd.append('--src-prefix=a/%s/' % self.relpath)
+ cmd.append('--dst-prefix=b/%s/' % self.relpath)
cmd.append('--')
p = GitCommand(self,
cmd,
@@ -524,7 +819,7 @@
if R_HEADS + n not in heads:
self.bare_git.DeleteRef(name, id)
- def GetUploadableBranches(self):
+ def GetUploadableBranches(self, selected_branch=None):
"""List any branches which can be uploaded for review.
"""
heads = {}
@@ -540,6 +835,8 @@
for branch, id in heads.iteritems():
if branch in pubed and pubed[branch] == id:
continue
+ if selected_branch and branch != selected_branch:
+ continue
rb = self.GetUploadableBranch(branch)
if rb:
@@ -559,7 +856,8 @@
def UploadForReview(self, branch=None,
people=([],[]),
- auto_topic=False):
+ auto_topic=False,
+ draft=False):
"""Uploads the named branch for code review.
"""
if branch is None:
@@ -581,31 +879,36 @@
branch.remote.projectname = self.name
branch.remote.Save()
- if branch.remote.ReviewProtocol == 'ssh':
- if dest_branch.startswith(R_HEADS):
- dest_branch = dest_branch[len(R_HEADS):]
+ url = branch.remote.ReviewUrl(self.UserEmail)
+ if url is None:
+ raise UploadError('review not configured')
+ cmd = ['push']
+ if url.startswith('ssh://'):
rp = ['gerrit receive-pack']
for e in people[0]:
rp.append('--reviewer=%s' % sq(e))
for e in people[1]:
rp.append('--cc=%s' % sq(e))
-
- ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)
- if auto_topic:
- ref_spec = ref_spec + '/' + branch.name
-
- cmd = ['push']
cmd.append('--receive-pack=%s' % " ".join(rp))
- cmd.append(branch.remote.SshReviewUrl(self.UserEmail))
- cmd.append(ref_spec)
- if GitCommand(self, cmd, bare = True).Wait() != 0:
- raise UploadError('Upload failed')
+ cmd.append(url)
- else:
- raise UploadError('Unsupported protocol %s' \
- % branch.remote.review)
+ if dest_branch.startswith(R_HEADS):
+ dest_branch = dest_branch[len(R_HEADS):]
+
+ upload_type = 'for'
+ if draft:
+ upload_type = 'drafts'
+
+ ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
+ dest_branch)
+ if auto_topic:
+ ref_spec = ref_spec + '/' + branch.name
+ cmd.append(ref_spec)
+
+ if GitCommand(self, cmd, bare = True).Wait() != 0:
+ raise UploadError('Upload failed')
msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
self.bare_git.UpdateRef(R_PUB + branch.name,
@@ -615,35 +918,53 @@
## Sync ##
- def Sync_NetworkHalf(self, quiet=False):
+ def Sync_NetworkHalf(self,
+ quiet=False,
+ is_new=None,
+ current_branch_only=False,
+ clone_bundle=True):
"""Perform only the network IO portion of the sync process.
Local working directory/branch state is not affected.
"""
- is_new = not self.Exists
+ if is_new is None:
+ is_new = not self.Exists
if is_new:
- if not quiet:
- print >>sys.stderr
- print >>sys.stderr, 'Initializing project %s ...' % self.name
self._InitGitDir()
-
self._InitRemote()
- if not self._RemoteFetch(initial=is_new, quiet=quiet):
+
+ if is_new:
+ alt = os.path.join(self.gitdir, 'objects/info/alternates')
+ try:
+ fd = open(alt, 'rb')
+ try:
+ alt_dir = fd.readline().rstrip()
+ finally:
+ fd.close()
+ except IOError:
+ alt_dir = None
+ else:
+ alt_dir = None
+
+ if clone_bundle \
+ and alt_dir is None \
+ and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
+ is_new = False
+
+ if not current_branch_only:
+ if self.sync_c:
+ current_branch_only = True
+ elif not self.manifest._loaded:
+ # Manifest cannot check defaults until it syncs.
+ current_branch_only = False
+ elif self.manifest.default.sync_c:
+ current_branch_only = True
+
+ if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+ current_branch_only=current_branch_only):
return False
- #Check that the requested ref was found after fetch
- #
- try:
- self.GetRevisionId()
- except ManifestInvalidRevisionError:
- # if the ref is a tag. We can try fetching
- # the tag manually as a last resort
- #
- rev = self.revisionExpr
- if rev.startswith(R_TAGS):
- self._RemoteFetch(None, rev[len(R_TAGS):], quiet=quiet)
-
if self.worktree:
- self.manifest.SetMRefs(self)
+ self._InitMRef()
else:
self._InitMirrorHead()
try:
@@ -680,11 +1001,11 @@
"""Perform only the local IO portion of the sync process.
Network access is not required.
"""
- self._InitWorkTree()
all = self.bare_ref.all
self.CleanPublishedCache(all)
-
revid = self.GetRevisionId(all)
+
+ self._InitWorkTree()
head = self.work_git.GetHead()
if head.startswith(R_HEADS):
branch = head[len(R_HEADS):]
@@ -705,12 +1026,15 @@
if head == revid:
# No changes; don't do anything further.
+ # Except if the head needs to be detached
#
- return
+ if not syncbuf.detach_head:
+ return
+ else:
+ lost = self._revlist(not_rev(revid), HEAD)
+ if lost:
+ syncbuf.info(self, "discarding %d commits", len(lost))
- lost = self._revlist(not_rev(revid), HEAD)
- if lost:
- syncbuf.info(self, "discarding %d commits", len(lost))
try:
self._Checkout(revid, quiet=True)
except GitError, e:
@@ -728,7 +1052,7 @@
if not branch.LocalMerge:
# The current branch has no tracking configuration.
- # Jump off it to a deatched HEAD.
+ # Jump off it to a detached HEAD.
#
syncbuf.info(self,
"leaving %s; does not track upstream",
@@ -806,10 +1130,12 @@
len(local_changes) - cnt_mine)
branch.remote = self.GetRemote(self.remote.name)
- branch.merge = self.revisionExpr
+ if not ID_RE.match(self.revisionExpr):
+ # in case of manifest sync the revisionExpr might be a SHA1
+ branch.merge = self.revisionExpr
branch.Save()
- if cnt_mine > 0:
+ if cnt_mine > 0 and self.rebase:
def _dorebase():
self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
self._CopyFiles()
@@ -833,6 +1159,9 @@
abssrc = os.path.join(self.worktree, src)
self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
+ def AddAnnotation(self, name, value, keep):
+ self.annotations.append(_Annotation(name, value, keep))
+
def DownloadPatchSet(self, change_id, patch_id):
"""Download a single patch set of a single change to FETCH_HEAD.
"""
@@ -900,6 +1229,13 @@
def CheckoutBranch(self, name):
"""Checkout a local topic branch.
+
+ Args:
+ name: The name of the branch to checkout.
+
+ Returns:
+ True if the checkout succeeded; False if it didn't; None if the branch
+ didn't exist.
"""
rev = R_HEADS + name
head = self.work_git.GetHead()
@@ -914,7 +1250,7 @@
except KeyError:
# Branch does not exist in this project
#
- return False
+ return None
if head.startswith(R_HEADS):
try:
@@ -937,13 +1273,19 @@
def AbandonBranch(self, name):
"""Destroy a local topic branch.
+
+ Args:
+ name: The name of the branch to abandon.
+
+ Returns:
+ True if the abandon succeeded; False if it didn't; None if the branch
+ didn't exist.
"""
rev = R_HEADS + name
all = self.bare_ref.all
if rev not in all:
- # Doesn't exist; assume already abandoned.
- #
- return True
+ # Doesn't exist
+ return None
head = self.work_git.GetHead()
if head == rev:
@@ -1023,31 +1365,43 @@
## Direct Git Commands ##
- def _RemoteFetch(self, name=None, tag=None,
+ def _RemoteFetch(self, name=None,
+ current_branch_only=False,
initial=False,
- quiet=False):
+ quiet=False,
+ alt_dir=None):
+
+ is_sha1 = False
+ tag_name = None
+
+ if current_branch_only:
+ if ID_RE.match(self.revisionExpr) is not None:
+ is_sha1 = True
+ elif self.revisionExpr.startswith(R_TAGS):
+ # this is a tag and its sha1 value should never change
+ tag_name = self.revisionExpr[len(R_TAGS):]
+
+ if is_sha1 or tag_name is not None:
+ try:
+ # if revision (sha or tag) is not present then following function
+ # throws an error.
+ self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
+ return True
+ except GitError:
+ # There is no such persistent revision. We have to fetch it.
+ pass
+
if not name:
name = self.remote.name
ssh_proxy = False
- if self.GetRemote(name).PreConnectFetch():
+ remote = self.GetRemote(name)
+ if remote.PreConnectFetch():
ssh_proxy = True
if initial:
- alt = os.path.join(self.gitdir, 'objects/info/alternates')
- try:
- fd = open(alt, 'rb')
- try:
- ref_dir = fd.readline()
- if ref_dir and ref_dir.endswith('\n'):
- ref_dir = ref_dir[:-1]
- finally:
- fd.close()
- except IOError, e:
- ref_dir = None
-
- if ref_dir and 'objects' == os.path.basename(ref_dir):
- ref_dir = os.path.dirname(ref_dir)
+ if alt_dir and 'objects' == os.path.basename(alt_dir):
+ ref_dir = os.path.dirname(alt_dir)
packed_refs = os.path.join(self.gitdir, 'packed-refs')
remote = self.GetRemote(name)
@@ -1083,35 +1437,130 @@
old_packed += line
_lwrite(packed_refs, tmp_packed)
-
else:
- ref_dir = None
+ alt_dir = None
+
+ cmd = ['fetch']
+
+ # The --depth option only affects the initial fetch; after that we'll do
+ # full fetches of changes.
+ depth = self.manifest.manifestProject.config.GetString('repo.depth')
+ if depth and initial:
+ cmd.append('--depth=%s' % depth)
+
+ if quiet:
+ cmd.append('--quiet')
+ if not self.worktree:
+ cmd.append('--update-head-ok')
+ cmd.append(name)
+
+ if not current_branch_only or is_sha1:
+ # Fetch whole repo
+ cmd.append('--tags')
+ cmd.append((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*'))
+ elif tag_name is not None:
+ cmd.append('tag')
+ cmd.append(tag_name)
+ else:
+ branch = self.revisionExpr
+ if branch.startswith(R_HEADS):
+ branch = branch[len(R_HEADS):]
+ cmd.append((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch))
+
+ ok = False
+ for i in range(2):
+ if GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait() == 0:
+ ok = True
+ break
+ time.sleep(random.randint(30, 45))
+
+ if initial:
+ if alt_dir:
+ if old_packed != '':
+ _lwrite(packed_refs, old_packed)
+ else:
+ os.remove(packed_refs)
+ self.bare_git.pack_refs('--all', '--prune')
+ return ok
+
+ def _ApplyCloneBundle(self, initial=False, quiet=False):
+ if initial and self.manifest.manifestProject.config.GetString('repo.depth'):
+ return False
+
+ remote = self.GetRemote(self.remote.name)
+ bundle_url = remote.url + '/clone.bundle'
+ bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
+ if GetSchemeFromUrl(bundle_url) in ('persistent-http', 'persistent-https'):
+ bundle_url = bundle_url[len('persistent-'):]
+ if GetSchemeFromUrl(bundle_url) not in ('http', 'https'):
+ return False
+
+ bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
+ bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
+
+ exist_dst = os.path.exists(bundle_dst)
+ exist_tmp = os.path.exists(bundle_tmp)
+
+ if not initial and not exist_dst and not exist_tmp:
+ return False
+
+ if not exist_dst:
+ exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
+ if not exist_dst:
+ return False
cmd = ['fetch']
if quiet:
cmd.append('--quiet')
if not self.worktree:
cmd.append('--update-head-ok')
- cmd.append(name)
- if tag is not None:
- cmd.append('tag')
- cmd.append(tag)
+ cmd.append(bundle_dst)
+ for f in remote.fetch:
+ cmd.append(str(f))
+ cmd.append('refs/tags/*:refs/tags/*')
- ok = GitCommand(self,
- cmd,
- bare = True,
- ssh_proxy = ssh_proxy).Wait() == 0
-
- if initial:
- if ref_dir:
- if old_packed != '':
- _lwrite(packed_refs, old_packed)
- else:
- os.remove(packed_refs)
- self.bare_git.pack_refs('--all', '--prune')
-
+ ok = GitCommand(self, cmd, bare=True).Wait() == 0
+ if os.path.exists(bundle_dst):
+ os.remove(bundle_dst)
+ if os.path.exists(bundle_tmp):
+ os.remove(bundle_tmp)
return ok
+ def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
+ if os.path.exists(dstPath):
+ os.remove(dstPath)
+
+ cmd = ['curl', '--output', tmpPath, '--netrc', '--location']
+ if quiet:
+ cmd += ['--silent']
+ if os.path.exists(tmpPath):
+ size = os.stat(tmpPath).st_size
+ if size >= 1024:
+ cmd += ['--continue-at', '%d' % (size,)]
+ else:
+ os.remove(tmpPath)
+ if 'http_proxy' in os.environ and 'darwin' == sys.platform:
+ cmd += ['--proxy', os.environ['http_proxy']]
+ cmd += [srcUrl]
+
+ if IsTrace():
+ Trace('%s', ' '.join(cmd))
+ try:
+ proc = subprocess.Popen(cmd)
+ except OSError:
+ return False
+
+ ok = proc.wait() == 0
+ if os.path.exists(tmpPath):
+ if ok and os.stat(tmpPath).st_size > 16:
+ os.rename(tmpPath, dstPath)
+ return True
+ else:
+ os.remove(tmpPath)
+ return False
+ else:
+ return False
+
def _Checkout(self, rev, quiet=False):
cmd = ['checkout']
if quiet:
@@ -1122,6 +1571,23 @@
if self._allrefs:
raise GitError('%s checkout %s ' % (self.name, rev))
+ def _CherryPick(self, rev, quiet=False):
+ cmd = ['cherry-pick']
+ cmd.append(rev)
+ cmd.append('--')
+ if GitCommand(self, cmd).Wait() != 0:
+ if self._allrefs:
+ raise GitError('%s cherry-pick %s ' % (self.name, rev))
+
+ def _Revert(self, rev, quiet=False):
+ cmd = ['revert']
+ cmd.append('--no-edit')
+ cmd.append(rev)
+ cmd.append('--')
+ if GitCommand(self, cmd).Wait() != 0:
+ if self._allrefs:
+ raise GitError('%s revert %s ' % (self.name, rev))
+
def _ResetHard(self, rev, quiet=True):
cmd = ['reset', '--hard']
if quiet:
@@ -1138,8 +1604,10 @@
if GitCommand(self, cmd).Wait() != 0:
raise GitError('%s rebase %s ' % (self.name, upstream))
- def _FastForward(self, head):
+ def _FastForward(self, head, ffonly=False):
cmd = ['merge', head]
+ if ffonly:
+ cmd.append("--ff-only")
if GitCommand(self, cmd).Wait() != 0:
raise GitError('%s merge %s ' % (self.name, head))
@@ -1192,13 +1660,16 @@
hooks = self._gitdir_path('hooks')
if not os.path.exists(hooks):
os.makedirs(hooks)
- for stock_hook in repo_hooks():
+ for stock_hook in _ProjectHooks():
name = os.path.basename(stock_hook)
- if name in ('commit-msg') and not self.remote.review:
+ if name in ('commit-msg',) and not self.remote.review \
+ and not self is self.manifest.manifestProject:
# Don't install a Gerrit Code Review hook if this
# project does not appear to use it for reviews.
#
+ # Since the manifest project is one of those, but also
+ # managed through gerrit, it's excluded
continue
dst = os.path.join(hooks, name)
@@ -1211,7 +1682,7 @@
_error("%s: Not replacing %s hook", self.relpath, name)
continue
try:
- os.symlink(relpath(stock_hook, dst), dst)
+ os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
except OSError, e:
if e.errno == errno.EPERM:
raise GitError('filesystem must support symlinks')
@@ -1231,6 +1702,10 @@
remote.ResetFetch(mirror=True)
remote.Save()
+ def _InitMRef(self):
+ if self.manifest.branch:
+ self._InitAnyMRef(R_M + self.manifest.branch)
+
def _InitMirrorHead(self):
self._InitAnyMRef(HEAD)
@@ -1249,40 +1724,33 @@
msg = 'manifest set to %s' % self.revisionExpr
self.bare_git.symbolic_ref('-m', msg, ref, dst)
- def _LinkWorkTree(self, relink=False):
- dotgit = os.path.join(self.worktree, '.git')
- if not relink:
- os.makedirs(dotgit)
-
- for name in ['config',
- 'description',
- 'hooks',
- 'info',
- 'logs',
- 'objects',
- 'packed-refs',
- 'refs',
- 'rr-cache',
- 'svn']:
- try:
- src = os.path.join(self.gitdir, name)
- dst = os.path.join(dotgit, name)
- if relink:
- os.remove(dst)
- if os.path.islink(dst) or not os.path.exists(dst):
- os.symlink(relpath(src, dst), dst)
- else:
- raise GitError('cannot overwrite a local work tree')
- except OSError, e:
- if e.errno == errno.EPERM:
- raise GitError('filesystem must support symlinks')
- else:
- raise
-
def _InitWorkTree(self):
dotgit = os.path.join(self.worktree, '.git')
if not os.path.exists(dotgit):
- self._LinkWorkTree()
+ os.makedirs(dotgit)
+
+ for name in ['config',
+ 'description',
+ 'hooks',
+ 'info',
+ 'logs',
+ 'objects',
+ 'packed-refs',
+ 'refs',
+ 'rr-cache',
+ 'svn']:
+ try:
+ src = os.path.join(self.gitdir, name)
+ dst = os.path.join(dotgit, name)
+ if os.path.islink(dst) or not os.path.exists(dst):
+ os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
+ else:
+ raise GitError('cannot overwrite a local work tree')
+ except OSError, e:
+ if e.errno == errno.EPERM:
+ raise GitError('filesystem must support symlinks')
+ else:
+ raise
_lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
@@ -1291,6 +1759,11 @@
cmd.append(HEAD)
if GitCommand(self, cmd).Wait() != 0:
raise GitError("cannot initialize work tree")
+
+ rr_cache = os.path.join(self.gitdir, 'rr-cache')
+ if not os.path.exists(rr_cache):
+ os.makedirs(rr_cache)
+
self._CopyFiles()
def _gitdir_path(self, path):
@@ -1449,6 +1922,22 @@
return r
def __getattr__(self, name):
+ """Allow arbitrary git commands using pythonic syntax.
+
+ This allows you to do things like:
+ git_obj.rev_parse('HEAD')
+
+ Since we don't have a 'rev_parse' method defined, the __getattr__ will
+ run. We'll replace the '_' with a '-' and try to run a git command.
+ Any other arguments will be passed to the git command.
+
+ Args:
+ name: The name of the git command to call. Any '_' characters will
+ be replaced with '-'.
+
+ Returns:
+ A callable object that will try to call git with the named command.
+ """
name = name.replace('_', '-')
def runner(*args):
cmdv = [name]
@@ -1580,30 +2069,43 @@
class MetaProject(Project):
"""A special project housed under .repo.
"""
- def __init__(self, manifest, name, gitdir, worktree, relpath=None):
+ def __init__(self, manifest, name, gitdir, worktree):
repodir = manifest.repodir
- if relpath is None:
- relpath = '.repo/%s' % name
Project.__init__(self,
manifest = manifest,
name = name,
gitdir = gitdir,
worktree = worktree,
remote = RemoteSpec('origin'),
- relpath = relpath,
+ relpath = '.repo/%s' % name,
revisionExpr = 'refs/heads/master',
- revisionId = None)
+ revisionId = None,
+ groups = None)
def PreSync(self):
if self.Exists:
cb = self.CurrentBranch
if cb:
- cb = self.GetBranch(cb)
- if cb.merge:
- self.revisionExpr = cb.merge
+ base = self.GetBranch(cb).merge
+ if base:
+ self.revisionExpr = base
self.revisionId = None
- if cb.remote and cb.remote.name:
- self.remote.name = cb.remote.name
+
+ def MetaBranchSwitch(self, target):
+ """ Prepare MetaProject for manifest branch switch
+ """
+
+ # detach and delete manifest branch, allowing a new
+ # branch to take over
+ syncbuf = SyncBuffer(self.config, detach_head = True)
+ self.Sync_LocalHalf(syncbuf)
+ syncbuf.Finish()
+
+ return GitCommand(self,
+ ['update-ref', '-d', 'refs/heads/default'],
+ capture_stdout = True,
+ capture_stderr = True).Wait() == 0
+
@property
def LastFetch(self):
diff --git a/repo b/repo
index 773ad82..d6b46c8 100755
--- a/repo
+++ b/repo
@@ -2,7 +2,7 @@
## repo default configuration
##
-REPO_URL='git://android.git.kernel.org/tools/repo.git'
+REPO_URL='https://gerrit.googlesource.com/git-repo'
REPO_REV='stable'
# Copyright (C) 2008 Google Inc.
@@ -28,7 +28,7 @@
del magic
# increment this whenever we make important changes to this script
-VERSION = (1, 10)
+VERSION = (1, 17)
# increment this if the MAINTAINER_KEYS block is modified
KEYRING_VERSION = (1,0)
@@ -91,6 +91,7 @@
import readline
import subprocess
import sys
+import urllib2
home_dot_repo = os.path.expanduser('~/.repoconfig')
gpg_dir = os.path.join(home_dot_repo, 'gnupg')
@@ -109,23 +110,31 @@
group.add_option('-u', '--manifest-url',
dest='manifest_url',
help='manifest repository location', metavar='URL')
-group.add_option('-o', '--origin',
- dest='manifest_origin',
- help="use REMOTE instead of 'origin' to track upstream",
- metavar='REMOTE')
group.add_option('-b', '--manifest-branch',
dest='manifest_branch',
help='manifest branch or revision', metavar='REVISION')
group.add_option('-m', '--manifest-name',
dest='manifest_name',
- help='initial manifest file (deprecated)',
- metavar='NAME.xml')
+ help='initial manifest file', metavar='NAME.xml')
group.add_option('--mirror',
dest='mirror', action='store_true',
help='mirror the forrest')
group.add_option('--reference',
dest='reference',
help='location of mirror directory', metavar='DIR')
+group.add_option('--depth', type='int', default=None,
+ dest='depth',
+ help='create a shallow clone with given depth; see git clone')
+group.add_option('-g', '--groups',
+ dest='groups', default='default',
+ help='restrict manifest projects to ones with a specified group',
+ metavar='GROUP')
+group.add_option('-p', '--platform',
+ dest='platform', default="auto",
+ help='restrict manifest projects to ones with a specified'
+ 'platform group [auto|all|none|linux|darwin|...]',
+ metavar='PLATFORM')
+
# Tool
group = init_optparse.add_option_group('repo Version options')
@@ -139,6 +148,11 @@
dest='no_repo_verify', action='store_true',
help='do not verify repo source code')
+# Other
+group = init_optparse.add_option_group('Other options')
+group.add_option('--config-name',
+ dest='config_name', action="store_true", default=False,
+ help='Always prompt for name/e-mail')
class CloneFailure(Exception):
"""Indicate the remote clone of repo itself failed.
@@ -149,7 +163,7 @@
"""Installs repo by cloning it over the network.
"""
opt, args = init_optparse.parse_args(args)
- if args or not opt.manifest_url:
+ if args:
init_optparse.print_usage()
sys.exit(1)
@@ -188,10 +202,6 @@
else:
can_verify = True
- if not opt.quiet:
- print >>sys.stderr, 'Getting repo ...'
- print >>sys.stderr, ' from %s' % url
-
dst = os.path.abspath(os.path.join(repodir, S_repo))
_Clone(url, dst, opt.quiet)
@@ -210,7 +220,17 @@
def _CheckGitVersion():
cmd = [GIT, '--version']
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ try:
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ except OSError, e:
+ print >>sys.stderr
+ print >>sys.stderr, "fatal: '%s' is not available" % GIT
+ print >>sys.stderr, 'fatal: %s' % e
+ print >>sys.stderr
+ print >>sys.stderr, 'Please make sure %s is installed'\
+ ' and in your path.' % GIT
+ raise CloneFailure()
+
ver_str = proc.stdout.read().strip()
proc.stdout.close()
proc.wait()
@@ -301,15 +321,43 @@
raise CloneFailure()
-def _Fetch(local, quiet, *args):
+def _InitHttp():
+ handlers = []
+
+ mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ try:
+ import netrc
+ n = netrc.netrc()
+ for host in n.hosts:
+ p = n.hosts[host]
+ mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
+ mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
+ except:
+ pass
+ handlers.append(urllib2.HTTPBasicAuthHandler(mgr))
+ handlers.append(urllib2.HTTPDigestAuthHandler(mgr))
+
+ if 'http_proxy' in os.environ:
+ url = os.environ['http_proxy']
+ handlers.append(urllib2.ProxyHandler({'http': url, 'https': url}))
+ if 'REPO_CURL_VERBOSE' in os.environ:
+ handlers.append(urllib2.HTTPHandler(debuglevel=1))
+ handlers.append(urllib2.HTTPSHandler(debuglevel=1))
+ urllib2.install_opener(urllib2.build_opener(*handlers))
+
+def _Fetch(url, local, src, quiet):
+ if not quiet:
+ print >>sys.stderr, 'Get %s' % url
+
cmd = [GIT, 'fetch']
if quiet:
cmd.append('--quiet')
err = subprocess.PIPE
else:
err = None
- cmd.extend(args)
- cmd.append('origin')
+ cmd.append(src)
+ cmd.append('+refs/heads/*:refs/remotes/origin/*')
+ cmd.append('refs/tags/*:refs/tags/*')
proc = subprocess.Popen(cmd, cwd = local, stderr = err)
if err:
@@ -318,6 +366,62 @@
if proc.wait() != 0:
raise CloneFailure()
+def _DownloadBundle(url, local, quiet):
+ if not url.endswith('/'):
+ url += '/'
+ url += 'clone.bundle'
+
+ proc = subprocess.Popen(
+ [GIT, 'config', '--get-regexp', 'url.*.insteadof'],
+ cwd = local,
+ stdout = subprocess.PIPE)
+ for line in proc.stdout:
+ m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
+ if m:
+ new_url = m.group(1)
+ old_url = m.group(2)
+ if url.startswith(old_url):
+ url = new_url + url[len(old_url):]
+ break
+ proc.stdout.close()
+ proc.wait()
+
+ if not url.startswith('http:') and not url.startswith('https:'):
+ return False
+
+ dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b')
+ try:
+ try:
+ r = urllib2.urlopen(url)
+ except urllib2.HTTPError, e:
+ if e.code == 404:
+ return False
+ print >>sys.stderr, 'fatal: Cannot get %s' % url
+ print >>sys.stderr, 'fatal: HTTP error %s' % e.code
+ raise CloneFailure()
+ except urllib2.URLError, e:
+ print >>sys.stderr, 'fatal: Cannot get %s' % url
+ print >>sys.stderr, 'fatal: error %s' % e.reason
+ raise CloneFailure()
+ try:
+ if not quiet:
+ print >>sys.stderr, 'Get %s' % url
+ while True:
+ buf = r.read(8192)
+ if buf == '':
+ return True
+ dest.write(buf)
+ finally:
+ r.close()
+ finally:
+ dest.close()
+
+def _ImportBundle(local):
+ path = os.path.join(local, '.git', 'clone.bundle')
+ try:
+ _Fetch(local, local, path, True)
+ finally:
+ os.remove(path)
def _Clone(url, local, quiet):
"""Clones a git repository to a new subdirectory of repodir
@@ -345,11 +449,14 @@
print >>sys.stderr, 'fatal: could not create %s' % local
raise CloneFailure()
+ _InitHttp()
_SetConfig(local, 'remote.origin.url', url)
_SetConfig(local, 'remote.origin.fetch',
'+refs/heads/*:refs/remotes/origin/*')
- _Fetch(local, quiet)
- _Fetch(local, quiet, '--tags')
+ if _DownloadBundle(url, local, quiet):
+ _ImportBundle(local)
+ else:
+ _Fetch(url, local, 'origin', quiet)
def _Verify(cwd, branch, quiet):
@@ -601,4 +708,3 @@
if __name__ == '__main__':
main(sys.argv[1:])
-
diff --git a/subcmds/abandon.py b/subcmds/abandon.py
index 8af6132..42abb2f 100644
--- a/subcmds/abandon.py
+++ b/subcmds/abandon.py
@@ -41,21 +41,30 @@
nb = args[0]
err = []
+ success = []
all = self.GetProjects(args[1:])
pm = Progress('Abandon %s' % nb, len(all))
for project in all:
pm.update()
- if not project.AbandonBranch(nb):
- err.append(project)
+
+ status = project.AbandonBranch(nb)
+ if status is not None:
+ if status:
+ success.append(project)
+ else:
+ err.append(project)
pm.end()
if err:
- if len(err) == len(all):
- print >>sys.stderr, 'error: no project has branch %s' % nb
- else:
- for p in err:
- print >>sys.stderr,\
- "error: %s/: cannot abandon %s" \
- % (p.relpath, nb)
+ for p in err:
+ print >>sys.stderr,\
+ "error: %s/: cannot abandon %s" \
+ % (p.relpath, nb)
sys.exit(1)
+ elif not success:
+ print >>sys.stderr, 'error: no project has branch %s' % nb
+ sys.exit(1)
+ else:
+ print >>sys.stderr, 'Abandoned in %d project(s):\n %s' % (
+ len(success), '\n '.join(p.relpath for p in success))
diff --git a/subcmds/checkout.py b/subcmds/checkout.py
index 4198acd..533d20e 100644
--- a/subcmds/checkout.py
+++ b/subcmds/checkout.py
@@ -38,21 +38,27 @@
nb = args[0]
err = []
+ success = []
all = self.GetProjects(args[1:])
pm = Progress('Checkout %s' % nb, len(all))
for project in all:
pm.update()
- if not project.CheckoutBranch(nb):
- err.append(project)
+
+ status = project.CheckoutBranch(nb)
+ if status is not None:
+ if status:
+ success.append(project)
+ else:
+ err.append(project)
pm.end()
if err:
- if len(err) == len(all):
- print >>sys.stderr, 'error: no project has branch %s' % nb
- else:
- for p in err:
- print >>sys.stderr,\
- "error: %s/: cannot checkout %s" \
- % (p.relpath, nb)
+ for p in err:
+ print >>sys.stderr,\
+ "error: %s/: cannot checkout %s" \
+ % (p.relpath, nb)
+ sys.exit(1)
+ elif not success:
+ print >>sys.stderr, 'error: no project has branch %s' % nb
sys.exit(1)
diff --git a/subcmds/cherry_pick.py b/subcmds/cherry_pick.py
new file mode 100644
index 0000000..8da3a75
--- /dev/null
+++ b/subcmds/cherry_pick.py
@@ -0,0 +1,114 @@
+#
+# Copyright (C) 2010 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys, re, string, random, os
+from command import Command
+from git_command import GitCommand
+
+CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$')
+
+class CherryPick(Command):
+ common = True
+ helpSummary = "Cherry-pick a change."
+ helpUsage = """
+%prog <sha1>
+"""
+ helpDescription = """
+'%prog' cherry-picks a change from one branch to another.
+The change id will be updated, and a reference to the old
+change id will be added.
+"""
+
+ def _Options(self, p):
+ pass
+
+ def Execute(self, opt, args):
+ if len(args) != 1:
+ self.Usage()
+
+ reference = args[0]
+
+ p = GitCommand(None,
+ ['rev-parse', '--verify', reference],
+ capture_stdout = True,
+ capture_stderr = True)
+ if p.Wait() != 0:
+ print >>sys.stderr, p.stderr
+ sys.exit(1)
+ sha1 = p.stdout.strip()
+
+ p = GitCommand(None, ['cat-file', 'commit', sha1], capture_stdout=True)
+ if p.Wait() != 0:
+ print >>sys.stderr, "error: Failed to retrieve old commit message"
+ sys.exit(1)
+ old_msg = self._StripHeader(p.stdout)
+
+ p = GitCommand(None,
+ ['cherry-pick', sha1],
+ capture_stdout = True,
+ capture_stderr = True)
+ status = p.Wait()
+
+ print >>sys.stdout, p.stdout
+ print >>sys.stderr, p.stderr
+
+ if status == 0:
+ # The cherry-pick was applied correctly. We just need to edit the
+ # commit message.
+ new_msg = self._Reformat(old_msg, sha1)
+
+ p = GitCommand(None, ['commit', '--amend', '-F', '-'],
+ provide_stdin = True,
+ capture_stdout = True,
+ capture_stderr = True)
+ p.stdin.write(new_msg)
+ if p.Wait() != 0:
+ print >>sys.stderr, "error: Failed to update commit message"
+ sys.exit(1)
+
+ else:
+ print >>sys.stderr, """\
+NOTE: When committing (please see above) and editing the commit message,
+please remove the old Change-Id-line and add:
+"""
+ print >>sys.stderr, self._GetReference(sha1)
+ print >>sys.stderr
+
+ def _IsChangeId(self, line):
+ return CHANGE_ID_RE.match(line)
+
+ def _GetReference(self, sha1):
+ return "(cherry picked from commit %s)" % sha1
+
+ def _StripHeader(self, commit_msg):
+ lines = commit_msg.splitlines()
+ return "\n".join(lines[lines.index("")+1:])
+
+ def _Reformat(self, old_msg, sha1):
+ new_msg = []
+
+ for line in old_msg.splitlines():
+ if not self._IsChangeId(line):
+ new_msg.append(line)
+
+ # Add a blank line between the message and the change id/reference
+ try:
+ if new_msg[-1].strip() != "":
+ new_msg.append("")
+ except IndexError:
+ pass
+
+ new_msg.append(self._GetReference(sha1))
+ return "\n".join(new_msg)
diff --git a/subcmds/diff.py b/subcmds/diff.py
index e024714..f233f69 100644
--- a/subcmds/diff.py
+++ b/subcmds/diff.py
@@ -20,8 +20,21 @@
helpSummary = "Show changes between commit and working tree"
helpUsage = """
%prog [<project>...]
+
+The -u option causes '%prog' to generate diff output with file paths
+relative to the repository root, so the output can be applied
+to the Unix 'patch' command.
"""
+ def _Options(self, p):
+ def cmd(option, opt_str, value, parser):
+ setattr(parser.values, option.dest, list(parser.rargs))
+ while parser.rargs:
+ del parser.rargs[0]
+ p.add_option('-u', '--absolute',
+ dest='absolute', action='store_true',
+ help='Paths are relative to the repository root')
+
def Execute(self, opt, args):
for project in self.GetProjects(args):
- project.PrintWorkTreeDiff()
+ project.PrintWorkTreeDiff(opt.absolute)
diff --git a/subcmds/download.py b/subcmds/download.py
index 61eadd5..0ea45c3 100644
--- a/subcmds/download.py
+++ b/subcmds/download.py
@@ -33,7 +33,15 @@
"""
def _Options(self, p):
- pass
+ p.add_option('-c','--cherry-pick',
+ dest='cherrypick', action='store_true',
+ help="cherry-pick instead of checkout")
+ p.add_option('-r','--revert',
+ dest='revert', action='store_true',
+ help="revert instead of checkout")
+ p.add_option('-f','--ff-only',
+ dest='ffonly', action='store_true',
+ help="force fast-forward merge")
def _ParseChangeIds(self, args):
if not args:
@@ -66,7 +74,7 @@
% (project.name, change_id, ps_id)
sys.exit(1)
- if not dl.commits:
+ if not opt.revert and not dl.commits:
print >>sys.stderr, \
'[%s] change %d/%d has already been merged' \
% (project.name, change_id, ps_id)
@@ -78,4 +86,11 @@
% (project.name, change_id, ps_id, len(dl.commits))
for c in dl.commits:
print >>sys.stderr, ' %s' % (c)
- project._Checkout(dl.commit)
+ if opt.cherrypick:
+ project._CherryPick(dl.commit)
+ elif opt.revert:
+ project._Revert(dl.commit)
+ elif opt.ffonly:
+ project._FastForward(dl.commit, ffonly=True)
+ else:
+ project._Checkout(dl.commit)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index d3e70ae..9436f4e 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -82,6 +82,11 @@
REPO_RREV is the name of the revision from the manifest, exactly
as written in the manifest.
+REPO__* are any extra environment variables, specified by the
+"annotation" element under any project element. This can be useful
+for differentiating trees based on user-specific criteria, or simply
+annotating tree details.
+
shell positional arguments ($1, $2, .., $#) are set to any arguments
following <command>.
@@ -162,6 +167,8 @@
setenv('REPO_REMOTE', project.remote.name)
setenv('REPO_LREV', project.GetRevisionId())
setenv('REPO_RREV', project.revisionExpr)
+ for a in project.annotations:
+ setenv("REPO__%s" % (a.name), a.value)
if mirror:
setenv('GIT_DIR', project.gitdir)
diff --git a/subcmds/help.py b/subcmds/help.py
index e2f3074..0df3c14 100644
--- a/subcmds/help.py
+++ b/subcmds/help.py
@@ -165,7 +165,7 @@
print >>sys.stderr, "repo: '%s' is not a repo command." % name
sys.exit(1)
- cmd.repodir = self.repodir
+ cmd.manifest = self.manifest
self._PrintCommandHelp(cmd)
else:
diff --git a/subcmds/init.py b/subcmds/init.py
index 2ca4e16..a758fbb 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -14,16 +14,17 @@
# limitations under the License.
import os
+import platform
+import re
+import shutil
import sys
from color import Coloring
from command import InteractiveCommand, MirrorSafeCommand
from error import ManifestParseError
from project import SyncBuffer
+from git_config import GitConfig
from git_command import git_require, MIN_GIT_VERSION
-from manifest_submodule import SubmoduleManifest
-from manifest_xml import XmlManifest
-from subcmds.sync import _ReloadManifest
class Init(InteractiveCommand, MirrorSafeCommand):
common = True
@@ -75,21 +76,27 @@
g.add_option('-b', '--manifest-branch',
dest='manifest_branch',
help='manifest branch or revision', metavar='REVISION')
- g.add_option('-o', '--origin',
- dest='manifest_origin',
- help="use REMOTE instead of 'origin' to track upstream",
- metavar='REMOTE')
- if isinstance(self.manifest, XmlManifest) \
- or not self.manifest.manifestProject.Exists:
- g.add_option('-m', '--manifest-name',
- dest='manifest_name', default='default.xml',
- help='initial manifest file', metavar='NAME.xml')
+ g.add_option('-m', '--manifest-name',
+ dest='manifest_name', default='default.xml',
+ help='initial manifest file', metavar='NAME.xml')
g.add_option('--mirror',
dest='mirror', action='store_true',
help='mirror the forrest')
g.add_option('--reference',
dest='reference',
help='location of mirror directory', metavar='DIR')
+ g.add_option('--depth', type='int', default=None,
+ dest='depth',
+ help='create a shallow clone with given depth; see git clone')
+ g.add_option('-g', '--groups',
+ dest='groups', default='default',
+ help='restrict manifest projects to ones with a specified group',
+ metavar='GROUP')
+ g.add_option('-p', '--platform',
+ dest='platform', default='auto',
+ help='restrict manifest projects to ones with a specified'
+ 'platform group [auto|all|none|linux|darwin|...]',
+ metavar='PLATFORM')
# Tool
g = p.add_option_group('repo Version options')
@@ -103,26 +110,11 @@
dest='no_repo_verify', action='store_true',
help='do not verify repo source code')
- def _ApplyOptions(self, opt, is_new):
- m = self.manifest.manifestProject
-
- if is_new:
- if opt.manifest_origin:
- m.remote.name = opt.manifest_origin
-
- if opt.manifest_branch:
- m.revisionExpr = opt.manifest_branch
- else:
- m.revisionExpr = 'refs/heads/master'
- else:
- if opt.manifest_origin:
- print >>sys.stderr, 'fatal: cannot change origin name'
- sys.exit(1)
-
- if opt.manifest_branch:
- m.revisionExpr = opt.manifest_branch
- else:
- m.PreSync()
+ # Other
+ g = p.add_option_group('Other options')
+ g.add_option('--config-name',
+ dest='config_name', action="store_true", default=False,
+ help='Always prompt for name/e-mail')
def _SyncManifest(self, opt):
m = self.manifest.manifestProject
@@ -134,60 +126,78 @@
sys.exit(1)
if not opt.quiet:
- print >>sys.stderr, 'Getting manifest ...'
- print >>sys.stderr, ' from %s' % opt.manifest_url
+ print >>sys.stderr, 'Get %s' \
+ % GitConfig.ForUser().UrlInsteadOf(opt.manifest_url)
m._InitGitDir()
- self._ApplyOptions(opt, is_new)
+ if opt.manifest_branch:
+ m.revisionExpr = opt.manifest_branch
+ else:
+ m.revisionExpr = 'refs/heads/master'
+ else:
+ if opt.manifest_branch:
+ m.revisionExpr = opt.manifest_branch
+ else:
+ m.PreSync()
+
if opt.manifest_url:
r = m.GetRemote(m.remote.name)
r.url = opt.manifest_url
r.ResetFetch()
r.Save()
+ groups = re.split('[,\s]+', opt.groups)
+ all_platforms = ['linux', 'darwin']
+ platformize = lambda x: 'platform-' + x
+ if opt.platform == 'auto':
+ if (not opt.mirror and
+ not m.config.GetString('repo.mirror') == 'true'):
+ groups.append(platformize(platform.system().lower()))
+ elif opt.platform == 'all':
+ groups.extend(map(platformize, all_platforms))
+ elif opt.platform in all_platforms:
+ groups.extend(platformize(opt.platform))
+ elif opt.platform != 'none':
+ print >>sys.stderr, 'fatal: invalid platform flag'
+ sys.exit(1)
+
+ groups = [x for x in groups if x]
+ groupstr = ','.join(groups)
+ if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
+ groupstr = None
+ m.config.SetString('manifest.groups', groupstr)
+
if opt.reference:
m.config.SetString('repo.reference', opt.reference)
if opt.mirror:
if is_new:
m.config.SetString('repo.mirror', 'true')
- m.config.ClearCache()
else:
print >>sys.stderr, 'fatal: --mirror not supported on existing client'
sys.exit(1)
- if not m.Sync_NetworkHalf():
+ if not m.Sync_NetworkHalf(is_new=is_new):
r = m.GetRemote(m.remote.name)
print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
+
+ # Better delete the manifest git dir if we created it; otherwise next
+ # time (when user fixes problems) we won't go through the "is_new" logic.
+ if is_new:
+ shutil.rmtree(m.gitdir)
sys.exit(1)
- if is_new and SubmoduleManifest.IsBare(m):
- new = self.GetManifest(reparse=True, type=SubmoduleManifest)
- if m.gitdir != new.manifestProject.gitdir:
- os.rename(m.gitdir, new.manifestProject.gitdir)
- new = self.GetManifest(reparse=True, type=SubmoduleManifest)
- m = new.manifestProject
- self._ApplyOptions(opt, is_new)
-
- if not is_new:
- # Force the manifest to load if it exists, the old graph
- # may be needed inside of _ReloadManifest().
- #
- self.manifest.projects
+ if opt.manifest_branch:
+ m.MetaBranchSwitch(opt.manifest_branch)
syncbuf = SyncBuffer(m.config)
m.Sync_LocalHalf(syncbuf)
syncbuf.Finish()
- if isinstance(self.manifest, XmlManifest):
- self._LinkManifest(opt.manifest_name)
- _ReloadManifest(self)
-
- self._ApplyOptions(opt, is_new)
-
- if not self.manifest.InitBranch():
- print >>sys.stderr, 'fatal: cannot create branch in manifest'
- sys.exit(1)
+ if is_new or m.CurrentBranch is None:
+ if not m.StartBranch('default'):
+ print >>sys.stderr, 'fatal: cannot create default in manifest'
+ sys.exit(1)
def _LinkManifest(self, name):
if not name:
@@ -210,6 +220,24 @@
return value
return a
+ def _ShouldConfigureUser(self):
+ gc = self.manifest.globalConfig
+ mp = self.manifest.manifestProject
+
+ # If we don't have local settings, get from global.
+ if not mp.config.Has('user.name') or not mp.config.Has('user.email'):
+ if not gc.Has('user.name') or not gc.Has('user.email'):
+ return True
+
+ mp.config.SetString('user.name', gc.GetString('user.name'))
+ mp.config.SetString('user.email', gc.GetString('user.email'))
+
+ print ''
+ print 'Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
+ mp.config.GetString('user.email'))
+ print 'If you want to change this, please re-run \'repo init\' with --config-name'
+ return False
+
def _ConfigureUser(self):
mp = self.manifest.manifestProject
@@ -220,7 +248,7 @@
print ''
print 'Your identity is: %s <%s>' % (name, email)
- sys.stdout.write('is this correct [y/n]? ')
+ sys.stdout.write('is this correct [y/N]? ')
a = sys.stdin.readline().strip()
if a in ('yes', 'y', 't', 'true'):
break
@@ -262,19 +290,42 @@
out.printer(fg='black', attr=c)(' %-6s ', c)
out.nl()
- sys.stdout.write('Enable color display in this user account (y/n)? ')
+ sys.stdout.write('Enable color display in this user account (y/N)? ')
a = sys.stdin.readline().strip().lower()
if a in ('y', 'yes', 't', 'true', 'on'):
gc.SetString('color.ui', 'auto')
+ def _ConfigureDepth(self, opt):
+ """Configure the depth we'll sync down.
+
+ Args:
+ opt: Options from optparse. We care about opt.depth.
+ """
+ # Opt.depth will be non-None if user actually passed --depth to repo init.
+ if opt.depth is not None:
+ if opt.depth > 0:
+ # Positive values will set the depth.
+ depth = str(opt.depth)
+ else:
+ # Negative numbers will clear the depth; passing None to SetString
+ # will do that.
+ depth = None
+
+ # We store the depth in the main manifest project.
+ self.manifest.manifestProject.config.SetString('repo.depth', depth)
+
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION, fail=True)
self._SyncManifest(opt)
+ self._LinkManifest(opt.manifest_name)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
- self._ConfigureUser()
+ if opt.config_name or self._ShouldConfigureUser():
+ self._ConfigureUser()
self._ConfigureColor()
+ self._ConfigureDepth(opt)
+
if self.manifest.IsMirror:
type = 'mirror '
else:
diff --git a/subcmds/list.py b/subcmds/list.py
new file mode 100644
index 0000000..2be8257
--- /dev/null
+++ b/subcmds/list.py
@@ -0,0 +1,48 @@
+#
+# Copyright (C) 2011 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from command import Command, MirrorSafeCommand
+
+class List(Command, MirrorSafeCommand):
+ common = True
+ helpSummary = "List projects and their associated directories"
+ helpUsage = """
+%prog [<project>...]
+"""
+ helpDescription = """
+List all projects; pass '.' to list the project for the cwd.
+
+This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
+"""
+
+ def Execute(self, opt, args):
+ """List all projects and the associated directories.
+
+ This may be possible to do with 'repo forall', but repo newbies have
+ trouble figuring that out. The idea here is that it should be more
+ discoverable.
+
+ Args:
+ opt: The options. We don't take any.
+ args: Positional args. Can be a list of projects to list, or empty.
+ """
+ projects = self.GetProjects(args)
+
+ lines = []
+ for project in projects:
+ lines.append("%s : %s" % (project.relpath, project.name))
+
+ lines.sort()
+ print '\n'.join(lines)
diff --git a/subcmds/manifest.py b/subcmds/manifest.py
index dcd3df1..4374a9d 100644
--- a/subcmds/manifest.py
+++ b/subcmds/manifest.py
@@ -17,25 +17,14 @@
import sys
from command import PagedCommand
-from manifest_submodule import SubmoduleManifest
-from manifest_xml import XmlManifest
-
-def _doc(name):
- r = os.path.dirname(__file__)
- r = os.path.dirname(r)
- fd = open(os.path.join(r, 'docs', name))
- try:
- return fd.read()
- finally:
- fd.close()
class Manifest(PagedCommand):
common = False
helpSummary = "Manifest inspection utility"
helpUsage = """
-%prog [options]
+%prog [-o {-|NAME.xml} [-r]]
"""
- _xmlHelp = """
+ _helpDescription = """
With the -o option, exports the current manifest for inspection.
The manifest and (if present) local_manifest.xml are combined
@@ -46,30 +35,23 @@
@property
def helpDescription(self):
- help = ''
- if isinstance(self.manifest, XmlManifest):
- help += self._xmlHelp + '\n' + _doc('manifest_xml.txt')
- if isinstance(self.manifest, SubmoduleManifest):
- help += _doc('manifest_submodule.txt')
+ help = self._helpDescription + '\n'
+ r = os.path.dirname(__file__)
+ r = os.path.dirname(r)
+ fd = open(os.path.join(r, 'docs', 'manifest-format.txt'))
+ for line in fd:
+ help += line
+ fd.close()
return help
def _Options(self, p):
- if isinstance(self.manifest, XmlManifest):
- p.add_option('--upgrade',
- dest='upgrade', action='store_true',
- help='Upgrade XML manifest to submodule')
- p.add_option('-r', '--revision-as-HEAD',
- dest='peg_rev', action='store_true',
- help='Save revisions as current HEAD')
- p.add_option('-o', '--output-file',
- dest='output_file',
- help='File to save the manifest to',
- metavar='-|NAME.xml')
-
- def WantPager(self, opt):
- if isinstance(self.manifest, XmlManifest) and opt.upgrade:
- return False
- return True
+ p.add_option('-r', '--revision-as-HEAD',
+ dest='peg_rev', action='store_true',
+ help='Save revisions as current HEAD')
+ p.add_option('-o', '--output-file',
+ dest='output_file',
+ help='File to save the manifest to',
+ metavar='-|NAME.xml')
def _Output(self, opt):
if opt.output_file == '-':
@@ -82,38 +64,13 @@
if opt.output_file != '-':
print >>sys.stderr, 'Saved manifest to %s' % opt.output_file
- def _Upgrade(self):
- old = self.manifest
-
- if isinstance(old, SubmoduleManifest):
- print >>sys.stderr, 'error: already upgraded'
- sys.exit(1)
-
- old._Load()
- for p in old.projects.values():
- if not os.path.exists(p.gitdir) \
- or not os.path.exists(p.worktree):
- print >>sys.stderr, 'fatal: project "%s" missing' % p.relpath
- sys.exit(1)
-
- new = SubmoduleManifest(old.repodir)
- new.FromXml_Local_1(old, checkout=False)
- new.FromXml_Definition(old)
- new.FromXml_Local_2(old)
- print >>sys.stderr, 'upgraded manifest; commit result manually'
-
def Execute(self, opt, args):
if args:
self.Usage()
- if isinstance(self.manifest, XmlManifest):
- if opt.upgrade:
- self._Upgrade()
- return
-
- if opt.output_file is not None:
- self._Output(opt)
- return
+ if opt.output_file is not None:
+ self._Output(opt)
+ return
print >>sys.stderr, 'error: no operation to perform'
print >>sys.stderr, 'error: see repo help manifest'
diff --git a/subcmds/overview.py b/subcmds/overview.py
new file mode 100644
index 0000000..96fa93d
--- /dev/null
+++ b/subcmds/overview.py
@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2012 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from color import Coloring
+from command import PagedCommand
+
+
+class Overview(PagedCommand):
+ common = True
+ helpSummary = "Display overview of unmerged project branches"
+ helpUsage = """
+%prog [--current-branch] [<project>...]
+"""
+ helpDescription = """
+The '%prog' command is used to display an overview of the projects branches,
+and list any local commits that have not yet been merged into the project.
+
+The -b/--current-branch option can be used to restrict the output to only
+branches currently checked out in each project. By default, all branches
+are displayed.
+"""
+
+ def _Options(self, p):
+ p.add_option('-b', '--current-branch',
+ dest="current_branch", action="store_true",
+ help="Consider only checked out branches")
+
+ def Execute(self, opt, args):
+ all = []
+ for project in self.GetProjects(args):
+ br = [project.GetUploadableBranch(x)
+ for x in project.GetBranches().keys()]
+ br = [x for x in br if x]
+ if opt.current_branch:
+ br = [x for x in br if x.name == project.CurrentBranch]
+ all.extend(br)
+
+ if not all:
+ return
+
+ class Report(Coloring):
+ def __init__(self, config):
+ Coloring.__init__(self, config, 'status')
+ self.project = self.printer('header', attr='bold')
+
+ out = Report(all[0].project.config)
+ out.project('Projects Overview')
+ out.nl()
+
+ project = None
+
+ for branch in all:
+ if project != branch.project:
+ project = branch.project
+ out.nl()
+ out.project('project %s/' % project.relpath)
+ out.nl()
+
+ commits = branch.commits
+ date = branch.date
+ print '%s %-33s (%2d commit%s, %s)' % (
+ branch.name == project.CurrentBranch and '*' or ' ',
+ branch.name,
+ len(commits),
+ len(commits) != 1 and 's' or ' ',
+ date)
+ for commit in commits:
+ print '%-35s - %s' % ('', commit)
diff --git a/subcmds/rebase.py b/subcmds/rebase.py
index e341296..20662b1 100644
--- a/subcmds/rebase.py
+++ b/subcmds/rebase.py
@@ -17,7 +17,7 @@
from command import Command
from git_command import GitCommand
-from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB
+from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
from error import GitError
class Rebase(Command):
@@ -52,6 +52,9 @@
p.add_option('--whitespace',
dest='whitespace', action='store', metavar='WS',
help='Pass --whitespace to git rebase')
+ p.add_option('--auto-stash',
+ dest='auto_stash', action='store_true',
+ help='Stash local modifications before starting')
def Execute(self, opt, args):
all = self.GetProjects(args)
@@ -103,5 +106,23 @@
print >>sys.stderr, '# %s: rebasing %s -> %s' % \
(project.relpath, cb, upbranch.LocalMerge)
+ needs_stash = False
+ if opt.auto_stash:
+ stash_args = ["update-index", "--refresh", "-q"]
+
+ if GitCommand(project, stash_args).Wait() != 0:
+ needs_stash = True
+ # Dirty index, requires stash...
+ stash_args = ["stash"]
+
+ if GitCommand(project, stash_args).Wait() != 0:
+ return -1
+
if GitCommand(project, args).Wait() != 0:
return -1
+
+ if needs_stash:
+ stash_args.append('pop')
+ stash_args.append('--quiet')
+ if GitCommand(project, stash_args).Wait() != 0:
+ return -1
diff --git a/subcmds/start.py b/subcmds/start.py
index ae2985d..0088507 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -15,6 +15,7 @@
import sys
from command import Command
+from git_config import IsId
from git_command import git
from progress import Progress
@@ -56,6 +57,10 @@
pm = Progress('Starting %s' % nb, len(all))
for project in all:
pm.update()
+ # If the current revision is a specific SHA1 then we can't push back
+ # to it so substitute the manifest default revision instead.
+ if IsId(project.revisionExpr):
+ project.revisionExpr = self.manifest.default.revisionExpr
if not project.StartBranch(nb):
err.append(project)
pm.end()
diff --git a/subcmds/status.py b/subcmds/status.py
index b0d419a..69e2dbf 100644
--- a/subcmds/status.py
+++ b/subcmds/status.py
@@ -15,6 +15,15 @@
from command import PagedCommand
+try:
+ import threading as _threading
+except ImportError:
+ import dummy_threading as _threading
+
+import itertools
+import sys
+import StringIO
+
class Status(PagedCommand):
common = True
helpSummary = "Show the working tree status"
@@ -27,6 +36,9 @@
specified. A summary is displayed, one line per file where there
is a difference between these three states.
+The -j/--jobs option can be used to run multiple status queries
+in parallel.
+
Status Display
--------------
@@ -60,26 +72,60 @@
"""
+ def _Options(self, p):
+ p.add_option('-j', '--jobs',
+ dest='jobs', action='store', type='int', default=2,
+ help="number of projects to check simultaneously")
+
+ def _StatusHelper(self, project, clean_counter, sem, output):
+ """Obtains the status for a specific project.
+
+ Obtains the status for a project, redirecting the output to
+ the specified object. It will release the semaphore
+ when done.
+
+ Args:
+ project: Project to get status of.
+ clean_counter: Counter for clean projects.
+ sem: Semaphore, will call release() when complete.
+ output: Where to output the status.
+ """
+ try:
+ state = project.PrintWorkTreeStatus(output)
+ if state == 'CLEAN':
+ clean_counter.next()
+ finally:
+ sem.release()
+
def Execute(self, opt, args):
all = self.GetProjects(args)
- clean = 0
+ counter = itertools.count()
- on = {}
- for project in all:
- cb = project.CurrentBranch
- if cb:
- if cb not in on:
- on[cb] = []
- on[cb].append(project)
+ if opt.jobs == 1:
+ for project in all:
+ state = project.PrintWorkTreeStatus()
+ if state == 'CLEAN':
+ counter.next()
+ else:
+ sem = _threading.Semaphore(opt.jobs)
+ threads_and_output = []
+ for project in all:
+ sem.acquire()
- branch_names = list(on.keys())
- branch_names.sort()
- for cb in branch_names:
- print '# on branch %s' % cb
+ class BufList(StringIO.StringIO):
+ def dump(self, ostream):
+ for entry in self.buflist:
+ ostream.write(entry)
- for project in all:
- state = project.PrintWorkTreeStatus()
- if state == 'CLEAN':
- clean += 1
- if len(all) == clean:
+ output = BufList()
+
+ t = _threading.Thread(target=self._StatusHelper,
+ args=(project, counter, sem, output))
+ threads_and_output.append((t, output))
+ t.start()
+ for (t, output) in threads_and_output:
+ t.join()
+ output.dump(sys.stdout)
+ output.close()
+ if len(all) == counter.next():
print 'nothing to commit (working directory clean)'
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 16f1d18..bfe146b 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -28,6 +28,14 @@
except ImportError:
import dummy_threading as _threading
+try:
+ import resource
+ def _rlimit_nofile():
+ return resource.getrlimit(resource.RLIMIT_NOFILE)
+except ImportError:
+ def _rlimit_nofile():
+ return (256, 256)
+
from git_command import GIT
from git_refs import R_HEADS
from project import HEAD
@@ -39,6 +47,10 @@
from project import SyncBuffer
from progress import Progress
+class _FetchError(Exception):
+ """Internal error thrown in _FetchHelper() when we don't want stack trace."""
+ pass
+
class Sync(Command, MirrorSafeCommand):
jobs = 1
common = True
@@ -68,11 +80,18 @@
The -s/--smart-sync option can be used to sync to a known good
build as specified by the manifest-server element in the current
-manifest.
+manifest. The -t/--smart-tag option is similar and allows you to
+specify a custom tag/label.
The -f/--force-broken option can be used to proceed with syncing
other projects if a project sync fails.
+The --no-clone-bundle option disables any attempt to use
+$URL/clone.bundle to bootstrap a new Git repository from a
+resumeable bundle file on a content delivery network. This
+may be necessary if there are problems with the local Python
+HTTP client or proxy configuration, but the Git binary works.
+
SSH Connections
---------------
@@ -104,6 +123,8 @@
"""
def _Options(self, p, show_smart=True):
+ self.jobs = self.manifest.default.sync_j
+
p.add_option('-f', '--force-broken',
dest='force_broken', action='store_true',
help="continue sync even if a project fails to sync")
@@ -116,16 +137,28 @@
p.add_option('-d','--detach',
dest='detach_head', action='store_true',
help='detach projects back to manifest revision')
+ p.add_option('-c','--current-branch',
+ dest='current_branch_only', action='store_true',
+ help='fetch only current branch from server')
p.add_option('-q','--quiet',
dest='quiet', action='store_true',
help='be more quiet')
p.add_option('-j','--jobs',
dest='jobs', action='store', type='int',
- help="number of projects to fetch simultaneously")
+ help="projects to fetch simultaneously (default %d)" % self.jobs)
+ p.add_option('-m', '--manifest-name',
+ dest='manifest_name',
+ help='temporary manifest to use for this sync', metavar='NAME.xml')
+ p.add_option('--no-clone-bundle',
+ dest='no_clone_bundle', action='store_true',
+ help='disable use of /clone.bundle on HTTP/HTTPS')
if show_smart:
p.add_option('-s', '--smart-sync',
dest='smart_sync', action='store_true',
help='smart sync using manifest from a known good build')
+ p.add_option('-t', '--smart-tag',
+ dest='smart_tag', action='store',
+ help='smart sync using manifest from a known tag')
g = p.add_option_group('repo Version options')
g.add_option('--no-repo-verify',
@@ -135,20 +168,60 @@
dest='repo_upgraded', action='store_true',
help=SUPPRESS_HELP)
- def _FetchHelper(self, opt, project, lock, fetched, pm, sem):
- if not project.Sync_NetworkHalf(quiet=opt.quiet):
- print >>sys.stderr, 'error: Cannot fetch %s' % project.name
- if opt.force_broken:
- print >>sys.stderr, 'warn: --force-broken, continuing to sync'
- else:
- sem.release()
- sys.exit(1)
+ def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
+ """Main function of the fetch threads when jobs are > 1.
- lock.acquire()
- fetched.add(project.gitdir)
- pm.update()
- lock.release()
- sem.release()
+ Args:
+ opt: Program options returned from optparse. See _Options().
+ project: Project object for the project to fetch.
+ lock: Lock for accessing objects that are shared amongst multiple
+ _FetchHelper() threads.
+ fetched: set object that we will add project.gitdir to when we're done
+ (with our lock held).
+ pm: Instance of a Project object. We will call pm.update() (with our
+ lock held).
+ sem: We'll release() this semaphore when we exit so that another thread
+ can be started up.
+ err_event: We'll set this event in the case of an error (after printing
+ out info about the error).
+ """
+ # We'll set to true once we've locked the lock.
+ did_lock = False
+
+ # Encapsulate everything in a try/except/finally so that:
+ # - We always set err_event in the case of an exception.
+ # - We always make sure we call sem.release().
+ # - We always make sure we unlock the lock if we locked it.
+ try:
+ try:
+ success = project.Sync_NetworkHalf(
+ quiet=opt.quiet,
+ current_branch_only=opt.current_branch_only,
+ clone_bundle=not opt.no_clone_bundle)
+
+ # Lock around all the rest of the code, since printing, updating a set
+ # and Progress.update() are not thread safe.
+ lock.acquire()
+ did_lock = True
+
+ if not success:
+ print >>sys.stderr, 'error: Cannot fetch %s' % project.name
+ if opt.force_broken:
+ print >>sys.stderr, 'warn: --force-broken, continuing to sync'
+ else:
+ raise _FetchError()
+
+ fetched.add(project.gitdir)
+ pm.update()
+ except _FetchError:
+ err_event.set()
+ except:
+ err_event.set()
+ raise
+ finally:
+ if did_lock:
+ lock.release()
+ sem.release()
def _Fetch(self, projects, opt):
fetched = set()
@@ -157,7 +230,10 @@
if self.jobs == 1:
for project in projects:
pm.update()
- if project.Sync_NetworkHalf(quiet=opt.quiet):
+ if project.Sync_NetworkHalf(
+ quiet=opt.quiet,
+ current_branch_only=opt.current_branch_only,
+ clone_bundle=not opt.no_clone_bundle):
fetched.add(project.gitdir)
else:
print >>sys.stderr, 'error: Cannot fetch %s' % project.name
@@ -169,7 +245,13 @@
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
+ err_event = _threading.Event()
for project in projects:
+ # Check for any errors before starting any new threads.
+ # ...we'll let existing threads finish, though.
+ if err_event.isSet():
+ break
+
sem.acquire()
t = _threading.Thread(target = self._FetchHelper,
args = (opt,
@@ -177,13 +259,19 @@
lock,
fetched,
pm,
- sem))
+ sem,
+ err_event))
threads.add(t)
t.start()
for t in threads:
t.join()
+ # If we saw an error, exit with code 1 so that other scripts can check.
+ if err_event.isSet():
+ print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
+ sys.exit(1)
+
pm.end()
for project in projects:
project.bare_git.gc('--auto')
@@ -191,7 +279,7 @@
def UpdateProjectList(self):
new_project_paths = []
- for project in self.manifest.projects.values():
+ for project in self.GetProjects(None, missing_ok=True):
if project.relpath:
new_project_paths.append(project.relpath)
file_name = 'project.list'
@@ -220,7 +308,8 @@
worktree = os.path.join(self.manifest.topdir, path),
relpath = path,
revisionExpr = 'HEAD',
- revisionId = None)
+ revisionId = None,
+ groups = None)
if project.IsDirty():
print >>sys.stderr, 'error: Cannot remove project "%s": \
@@ -251,34 +340,51 @@
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
+ if self.jobs > 1:
+ soft_limit, _ = _rlimit_nofile()
+ self.jobs = min(self.jobs, (soft_limit - 5) / 3)
+
if opt.network_only and opt.detach_head:
print >>sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1)
if opt.network_only and opt.local_only:
print >>sys.stderr, 'error: cannot combine -n and -l'
sys.exit(1)
+ if opt.manifest_name and opt.smart_sync:
+ print >>sys.stderr, 'error: cannot combine -m and -s'
+ sys.exit(1)
+ if opt.manifest_name and opt.smart_tag:
+ print >>sys.stderr, 'error: cannot combine -m and -t'
+ sys.exit(1)
- if opt.smart_sync:
+ if opt.manifest_name:
+ self.manifest.Override(opt.manifest_name)
+
+ if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
print >>sys.stderr, \
'error: cannot smart sync: no manifest server defined in manifest'
sys.exit(1)
try:
server = xmlrpclib.Server(self.manifest.manifest_server)
- p = self.manifest.manifestProject
- b = p.GetBranch(p.CurrentBranch)
- branch = b.merge
- if branch.startswith(R_HEADS):
- branch = branch[len(R_HEADS):]
+ if opt.smart_sync:
+ p = self.manifest.manifestProject
+ b = p.GetBranch(p.CurrentBranch)
+ branch = b.merge
+ if branch.startswith(R_HEADS):
+ branch = branch[len(R_HEADS):]
- env = os.environ.copy()
- if (env.has_key('TARGET_PRODUCT') and
- env.has_key('TARGET_BUILD_VARIANT')):
- target = '%s-%s' % (env['TARGET_PRODUCT'],
- env['TARGET_BUILD_VARIANT'])
- [success, manifest_str] = server.GetApprovedManifest(branch, target)
+ env = os.environ.copy()
+ if (env.has_key('TARGET_PRODUCT') and
+ env.has_key('TARGET_BUILD_VARIANT')):
+ target = '%s-%s' % (env['TARGET_PRODUCT'],
+ env['TARGET_BUILD_VARIANT'])
+ [success, manifest_str] = server.GetApprovedManifest(branch, target)
+ else:
+ [success, manifest_str] = server.GetApprovedManifest(branch)
else:
- [success, manifest_str] = server.GetApprovedManifest(branch)
+ assert(opt.smart_tag)
+ [success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
manifest_name = "smart_sync_override.xml"
@@ -313,7 +419,8 @@
_PostRepoUpgrade(self.manifest)
if not opt.local_only:
- mp.Sync_NetworkHalf(quiet=opt.quiet)
+ mp.Sync_NetworkHalf(quiet=opt.quiet,
+ current_branch_only=opt.current_branch_only)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
@@ -321,6 +428,8 @@
if not syncbuf.Finish():
sys.exit(1)
self.manifest._Unload()
+ if opt.jobs is None:
+ self.jobs = self.manifest.default.sync_j
all = self.GetProjects(args, missing_ok=True)
if not opt.local_only:
@@ -336,14 +445,7 @@
# bail out now; the rest touches the working tree
return
- if mp.HasChanges:
- syncbuf = SyncBuffer(mp.config)
- mp.Sync_LocalHalf(syncbuf)
- if not syncbuf.Finish():
- sys.exit(1)
- _ReloadManifest(self)
- mp = self.manifest.manifestProject
-
+ self.manifest._Unload()
all = self.GetProjects(args, missing_ok=True)
missing = []
for project in all:
@@ -370,16 +472,10 @@
if not syncbuf.Finish():
sys.exit(1)
-def _ReloadManifest(cmd):
- old = cmd.manifest
- new = cmd.GetManifest(reparse=True)
-
- if old.__class__ != new.__class__:
- print >>sys.stderr, 'NOTICE: manifest format has changed ***'
- new.Upgrade_Local(old)
- else:
- if new.notice:
- print new.notice
+ # If there's a notice that's supposed to print at the end of the sync, print
+ # it now...
+ if self.manifest.notice:
+ print self.manifest.notice
def _PostRepoUpgrade(manifest):
for project in manifest.projects.values():
diff --git a/subcmds/upload.py b/subcmds/upload.py
index 2082209..c931297 100644
--- a/subcmds/upload.py
+++ b/subcmds/upload.py
@@ -19,7 +19,8 @@
from command import InteractiveCommand
from editor import Editor
-from error import UploadError
+from error import HookError, UploadError
+from project import RepoHook
UNUSUAL_COMMIT_THRESHOLD = 5
@@ -72,7 +73,7 @@
review.URL.autoupload:
-To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+To disable the "Upload ... (y/N)?" prompt, you can set a per-project
or global Git configuration option. If review.URL.autoupload is set
to "true" then repo will assume you always answer "y" at the prompt,
and will not prompt you further. If it is set to "false" then repo
@@ -102,6 +103,14 @@
autoupload = true
autocopy = johndoe@company.com,my-team-alias@company.com
+review.URL.uploadtopic:
+
+To add a topic branch whenever uploading a commit, you can set a
+per-project or global Git option to do so. If review.URL.uploadtopic
+is set to "true" then repo will assume you always want the equivalent
+of the -t option to the repo command. If unset or set to "false" then
+repo will make use of only the command line option.
+
References
----------
@@ -119,6 +128,38 @@
p.add_option('--cc',
type='string', action='append', dest='cc',
help='Also send email to these email addresses.')
+ p.add_option('--br',
+ type='string', action='store', dest='branch',
+ help='Branch to upload.')
+ p.add_option('--cbr', '--current-branch',
+ dest='current_branch', action='store_true',
+ help='Upload current git branch.')
+ p.add_option('-d', '--draft',
+ action='store_true', dest='draft', default=False,
+ help='If specified, upload as a draft.')
+
+ # Options relating to upload hook. Note that verify and no-verify are NOT
+ # opposites of each other, which is why they store to different locations.
+ # We are using them to match 'git commit' syntax.
+ #
+ # Combinations:
+ # - no-verify=False, verify=False (DEFAULT):
+ # If stdout is a tty, can prompt about running upload hooks if needed.
+ # If user denies running hooks, the upload is cancelled. If stdout is
+ # not a tty and we would need to prompt about upload hooks, upload is
+ # cancelled.
+ # - no-verify=False, verify=True:
+ # Always run upload hooks with no prompt.
+ # - no-verify=True, verify=False:
+ # Never run upload hooks, but upload anyway (AKA bypass hooks).
+ # - no-verify=True, verify=True:
+ # Invalid
+ p.add_option('--no-verify',
+ dest='bypass_hooks', action='store_true',
+ help='Do not run the upload hook.')
+ p.add_option('--verify',
+ dest='allow_all_hooks', action='store_true',
+ help='Run the upload hook without prompting.')
def _SingleBranch(self, opt, branch, people):
project = branch.project
@@ -135,7 +176,7 @@
date = branch.date
list = branch.commits
- print 'Upload project %s/:' % project.relpath
+ print 'Upload project %s/ to remote branch %s:' % (project.relpath, project.revisionExpr)
print ' branch %s (%2d commit%s, %s):' % (
name,
len(list),
@@ -144,7 +185,7 @@
for commit in list:
print ' %s' % commit
- sys.stdout.write('to %s (y/n)? ' % remote.review)
+ sys.stdout.write('to %s (y/N)? ' % remote.review)
answer = sys.stdin.readline().strip()
answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
@@ -175,11 +216,12 @@
if b:
script.append('#')
- script.append('# branch %s (%2d commit%s, %s):' % (
+ script.append('# branch %s (%2d commit%s, %s) to remote branch %s:' % (
name,
len(list),
len(list) != 1 and 's' or '',
- date))
+ date,
+ project.revisionExpr))
for commit in list:
script.append('# %s' % commit)
b[name] = branch
@@ -188,6 +230,11 @@
branches[project.name] = b
script.append('')
+ script = [ x.encode('utf-8')
+ if issubclass(type(x), unicode)
+ else x
+ for x in script ]
+
script = Editor.EditString("\n".join(script)).split("\n")
project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$')
@@ -267,7 +314,7 @@
# if they want to auto upload, let's not ask because it could be automated
if answer is None:
- sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/n) ')
+ sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
a = sys.stdin.readline().strip().lower()
if a not in ('y', 'yes', 't', 'true', 'on'):
print >>sys.stderr, "skipping upload"
@@ -275,7 +322,12 @@
branch.error = 'User aborted'
continue
- branch.UploadForReview(people, auto_topic=opt.auto_topic)
+ # Check if topic branches should be sent to the server during upload
+ if opt.auto_topic is not True:
+ key = 'review.%s.uploadtopic' % branch.project.remote.review
+ opt.auto_topic = branch.project.config.GetBoolean(key)
+
+ branch.UploadForReview(people, auto_topic=opt.auto_topic, draft=opt.draft)
branch.uploaded = True
except UploadError, e:
branch.error = e
@@ -312,6 +364,29 @@
pending = []
reviewers = []
cc = []
+ branch = None
+
+ if opt.branch:
+ branch = opt.branch
+
+ for project in project_list:
+ if opt.current_branch:
+ cbr = project.CurrentBranch
+ avail = [project.GetUploadableBranch(cbr)] if cbr else None
+ else:
+ avail = project.GetUploadableBranches(branch)
+ if avail:
+ pending.append((project, avail))
+
+ if pending and (not opt.bypass_hooks):
+ hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
+ self.manifest.topdir, abort_if_user_denies=True)
+ pending_proj_names = [project.name for (project, avail) in pending]
+ try:
+ hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
+ except HookError, e:
+ print >>sys.stderr, "ERROR: %s" % str(e)
+ return
if opt.reviewers:
reviewers = _SplitEmails(opt.reviewers)
@@ -319,11 +394,6 @@
cc = _SplitEmails(opt.cc)
people = (reviewers,cc)
- for project in project_list:
- avail = project.GetUploadableBranches()
- if avail:
- pending.append((project, avail))
-
if not pending:
print >>sys.stdout, "no branches ready for upload"
elif len(pending) == 1 and len(pending[0][1]) == 1:
diff --git a/subcmds/version.py b/subcmds/version.py
index 83e77d0..03195f8 100644
--- a/subcmds/version.py
+++ b/subcmds/version.py
@@ -19,6 +19,9 @@
from project import HEAD
class Version(Command, MirrorSafeCommand):
+ wrapper_version = None
+ wrapper_path = None
+
common = False
helpSummary = "Display the version of repo"
helpUsage = """
@@ -31,5 +34,10 @@
print 'repo version %s' % rp.work_git.describe(HEAD)
print ' (from %s)' % rem.url
+
+ if Version.wrapper_path is not None:
+ print 'repo launcher version %s' % Version.wrapper_version
+ print ' (from %s)' % Version.wrapper_path
+
print git.version().strip()
print 'Python %s' % sys.version