Merge "Fail if gitdir does not point to objdir during sync"
diff --git a/error.py b/error.py
index ff948f9..f2a7c4e 100644
--- a/error.py
+++ b/error.py
@@ -80,7 +80,7 @@
self.name = name
def __str__(self):
- if self.Name is None:
+ if self.name is None:
return 'in current directory'
return self.name
@@ -93,7 +93,7 @@
self.name = name
def __str__(self):
- if self.Name is None:
+ if self.name is None:
return 'in current directory'
return self.name
diff --git a/git_command.py b/git_command.py
index 259fb02..0893bff 100644
--- a/git_command.py
+++ b/git_command.py
@@ -92,7 +92,10 @@
def version(self):
p = GitCommand(None, ['--version'], capture_stdout=True)
if p.Wait() == 0:
- return p.stdout.decode('utf-8')
+ if hasattr(p.stdout, 'decode'):
+ return p.stdout.decode('utf-8')
+ else:
+ return p.stdout
return None
def version_tuple(self):
@@ -263,6 +266,8 @@
if not buf:
s_in.remove(s)
continue
+ if not hasattr(buf, 'encode'):
+ buf = buf.decode()
if s.std_name == 'stdout':
self.stdout += buf
else:
diff --git a/git_config.py b/git_config.py
index c4c31e1..8ded7c2 100644
--- a/git_config.py
+++ b/git_config.py
@@ -280,7 +280,7 @@
finally:
fd.close()
except (IOError, TypeError):
- if os.path.exists(self.json):
+ if os.path.exists(self._json):
os.remove(self._json)
def _ReadGit(self):
diff --git a/main.py b/main.py
index 47f083d..6736abc 100755
--- a/main.py
+++ b/main.py
@@ -45,6 +45,7 @@
from subcmds.version import Version
from editor import Editor
from error import DownloadError
+from error import InvalidProjectGroupsError
from error import ManifestInvalidRevisionError
from error import ManifestParseError
from error import NoManifestException
@@ -173,6 +174,12 @@
else:
print('error: no project in current directory', file=sys.stderr)
result = 1
+ except InvalidProjectGroupsError as e:
+ if e.name:
+ print('error: project group must be enabled for project %s' % e.name, file=sys.stderr)
+ else:
+ print('error: project group must be enabled for the project in the current directory', file=sys.stderr)
+ result = 1
finally:
elapsed = time.time() - start
hours, remainder = divmod(elapsed, 3600)
diff --git a/manifest_xml.py b/manifest_xml.py
index cfbd9ef..130e17c 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -253,11 +253,13 @@
else:
value = p.work_git.rev_parse(HEAD + '^0')
e.setAttribute('revision', value)
- if peg_rev_upstream and value != p.revisionExpr:
- # Only save the origin if the origin is not a sha1, and the default
- # isn't our value, and the if the default doesn't already have that
- # covered.
- e.setAttribute('upstream', p.revisionExpr)
+ if peg_rev_upstream:
+ if p.upstream:
+ e.setAttribute('upstream', p.upstream)
+ elif value != p.revisionExpr:
+ # Only save the origin if the origin is not a sha1, and the default
+ # isn't our value
+ e.setAttribute('upstream', p.revisionExpr)
else:
revision = self.remotes[remoteName].revision or d.revisionExpr
if not revision or revision != p.revisionExpr:
diff --git a/project.py b/project.py
index 003489a..d0d3b6e 100644
--- a/project.py
+++ b/project.py
@@ -16,6 +16,7 @@
import contextlib
import errno
import filecmp
+import glob
import os
import random
import re
@@ -233,28 +234,60 @@
_error('Cannot copy file %s to %s', src, dest)
class _LinkFile(object):
- def __init__(self, src, dest, abssrc, absdest):
+ def __init__(self, git_worktree, src, dest, relsrc, absdest):
+ self.git_worktree = git_worktree
self.src = src
self.dest = dest
- self.abs_src = abssrc
+ self.src_rel_to_dest = relsrc
self.abs_dest = absdest
- def _Link(self):
- src = self.abs_src
- dest = self.abs_dest
+ def __linkIt(self, relSrc, absDest):
# link file if it does not exist or is out of date
- if not os.path.islink(dest) or os.readlink(dest) != src:
+ if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc):
try:
# remove existing file first, since it might be read-only
- if os.path.exists(dest):
- os.remove(dest)
+ if os.path.exists(absDest):
+ os.remove(absDest)
else:
- dest_dir = os.path.dirname(dest)
+ dest_dir = os.path.dirname(absDest)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
- os.symlink(src, dest)
+ os.symlink(relSrc, absDest)
except IOError:
- _error('Cannot link file %s to %s', src, dest)
+ _error('Cannot link file %s to %s', relSrc, absDest)
+
+ def _Link(self):
+ """Link the self.rel_src_to_dest and self.abs_dest. Handles wild cards
+ on the src linking all of the files in the source in to the destination
+ directory.
+ """
+ # We use the absSrc to handle the situation where the current directory
+ # is not the root of the repo
+ absSrc = os.path.join(self.git_worktree, self.src)
+ if os.path.exists(absSrc):
+ # Entity exists so just a simple one to one link operation
+ self.__linkIt(self.src_rel_to_dest, self.abs_dest)
+ else:
+ # Entity doesn't exist assume there is a wild card
+ absDestDir = self.abs_dest
+ if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
+ _error('Link error: src with wildcard, %s must be a directory',
+ absDestDir)
+ else:
+ absSrcFiles = glob.glob(absSrc)
+ for absSrcFile in absSrcFiles:
+ # Create a releative path from source dir to destination dir
+ absSrcDir = os.path.dirname(absSrcFile)
+ relSrcDir = os.path.relpath(absSrcDir, absDestDir)
+
+ # Get the source file name
+ srcFile = os.path.basename(absSrcFile)
+
+ # Now form the final full paths to srcFile. They will be
+ # absolute for the desintaiton and relative for the srouce.
+ absDest = os.path.join(absDestDir, srcFile)
+ relSrc = os.path.join(relSrcDir, srcFile)
+ self.__linkIt(relSrc, absDest)
class RemoteSpec(object):
def __init__(self,
@@ -535,7 +568,8 @@
upstream=None,
parent=None,
is_derived=False,
- dest_branch=None):
+ dest_branch=None,
+ optimized_fetch=False):
"""Init a Project object.
Args:
@@ -557,6 +591,8 @@
is_derived: False if the project was explicitly defined in the manifest;
True if the project is a discovered submodule.
dest_branch: The branch to which to push changes for review by default.
+ optimized_fetch: If True, when a project is set to a sha1 revision, only
+ fetch from the remote if the sha1 is not present locally.
"""
self.manifest = manifest
self.name = name
@@ -585,6 +621,7 @@
self.upstream = upstream
self.parent = parent
self.is_derived = is_derived
+ self.optimized_fetch = optimized_fetch
self.subprojects = []
self.snapshots = {}
@@ -1066,7 +1103,8 @@
current_branch_only=False,
clone_bundle=True,
no_tags=False,
- archive=False):
+ archive=False,
+ optimized_fetch=False):
"""Perform only the network IO portion of the sync process.
Local working directory/branch state is not affected.
"""
@@ -1134,8 +1172,9 @@
elif self.manifest.default.sync_c:
current_branch_only = True
- has_sha1 = ID_RE.match(self.revisionExpr) and self._CheckForSha1()
- if (not has_sha1 #Need to fetch since we don't already have this revision
+ need_to_fetch = not (optimized_fetch and \
+ (ID_RE.match(self.revisionExpr) and self._CheckForSha1()))
+ if (need_to_fetch
and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
current_branch_only=current_branch_only,
no_tags=no_tags)):
@@ -1358,9 +1397,10 @@
def AddLinkFile(self, src, dest, absdest):
# dest should already be an absolute path, but src is project relative
- # make src an absolute path
- abssrc = os.path.join(self.worktree, src)
- self.linkfiles.append(_LinkFile(src, dest, abssrc, absdest))
+ # make src relative path to dest
+ absdestdir = os.path.dirname(absdest)
+ relsrc = os.path.relpath(os.path.join(self.worktree, src), absdestdir)
+ self.linkfiles.append(_LinkFile(self.worktree, src, dest, relsrc, absdest))
def AddAnnotation(self, name, value, keep):
self.annotations.append(_Annotation(name, value, keep))
@@ -1401,7 +1441,7 @@
branch = self.GetBranch(name)
branch.remote = self.GetRemote(self.remote.name)
branch.merge = self.revisionExpr
- if not branch.merge.startswith('refs/'):
+ if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr):
branch.merge = R_HEADS + self.revisionExpr
revid = self.GetRevisionId(all_refs)
@@ -1841,6 +1881,8 @@
cmd.append('--quiet')
if not self.worktree:
cmd.append('--update-head-ok')
+ if self.manifest.IsMirror:
+ cmd.append('--prune')
cmd.append(name)
# If using depth then we should not get all the tags since they may
@@ -1860,7 +1902,7 @@
if not self.manifest.IsMirror:
branch = self.revisionExpr
- if is_sha1 and depth:
+ if is_sha1 and depth and git_require((1, 8, 3)):
# Shallow checkout of a specific commit, fetch from that commit and not
# the heads only as the commit might be deeper in the history.
spec.append(branch)
@@ -1905,6 +1947,9 @@
# mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
# abort the optimization attempt and do a full sync.
break
+ elif ret < 0:
+ # Git died with a signal, exit immediately
+ break
time.sleep(random.randint(30, 45))
if initial:
@@ -1920,8 +1965,15 @@
# got what we wanted, else trigger a second run of all
# refs.
if not self._CheckForSha1():
- return self._RemoteFetch(name=name, current_branch_only=False,
- initial=False, quiet=quiet, alt_dir=alt_dir)
+ if not depth:
+ # Avoid infinite recursion when depth is True (since depth implies
+ # current_branch_only)
+ return self._RemoteFetch(name=name, current_branch_only=False,
+ initial=False, quiet=quiet, alt_dir=alt_dir)
+ if self.clone_depth:
+ self.clone_depth = None
+ return self._RemoteFetch(name=name, current_branch_only=current_branch_only,
+ initial=False, quiet=quiet, alt_dir=alt_dir)
return ok
diff --git a/subcmds/cherry_pick.py b/subcmds/cherry_pick.py
index 520e4c3..1f7dffd 100644
--- a/subcmds/cherry_pick.py
+++ b/subcmds/cherry_pick.py
@@ -76,6 +76,7 @@
capture_stdout = True,
capture_stderr = True)
p.stdin.write(new_msg)
+ p.stdin.close()
if p.Wait() != 0:
print("error: Failed to update commit message", file=sys.stderr)
sys.exit(1)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 88b23fb..b93cd6d 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -20,6 +20,7 @@
import re
import os
import select
+import signal
import sys
import subprocess
@@ -150,11 +151,15 @@
attributes that we need.
"""
+ if not self.manifest.IsMirror:
+ lrev = project.GetRevisionId()
+ else:
+ lrev = None
return {
'name': project.name,
'relpath': project.relpath,
'remote_name': project.remote.name,
- 'lrev': project.GetRevisionId(),
+ 'lrev': lrev,
'rrev': project.revisionExpr,
'annotations': dict((a.name, a.value) for a in project.annotations),
'gitdir': project.gitdir,
@@ -200,6 +205,13 @@
mirror = self.manifest.IsMirror
rc = 0
+ smart_sync_manifest_name = "smart_sync_override.xml"
+ smart_sync_manifest_path = os.path.join(
+ self.manifest.manifestProject.worktree, smart_sync_manifest_name)
+
+ if os.path.isfile(smart_sync_manifest_path):
+ self.manifest.Override(smart_sync_manifest_path)
+
if not opt.regex:
projects = self.GetProjects(args)
else:
@@ -207,14 +219,12 @@
os.environ['REPO_COUNT'] = str(len(projects))
- pool = multiprocessing.Pool(opt.jobs)
+ pool = multiprocessing.Pool(opt.jobs, InitWorker)
try:
config = self.manifest.manifestProject.config
results_it = pool.imap(
DoWorkWrapper,
- ([mirror, opt, cmd, shell, cnt, config, self._SerializeProject(p)]
- for cnt, p in enumerate(projects))
- )
+ self.ProjectArgs(projects, mirror, opt, cmd, shell, config))
pool.close()
for r in results_it:
rc = rc or r
@@ -236,12 +246,28 @@
if rc != 0:
sys.exit(rc)
+ def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
+ for cnt, p in enumerate(projects):
+ try:
+ project = self._SerializeProject(p)
+ except Exception as e:
+ print('Project list error: %r' % e,
+ file=sys.stderr)
+ return
+ except KeyboardInterrupt:
+ print('Project list interrupted',
+ file=sys.stderr)
+ return
+ yield [mirror, opt, cmd, shell, cnt, config, project]
class WorkerKeyboardInterrupt(Exception):
""" Keyboard interrupt exception for worker processes. """
pass
+def InitWorker():
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+
def DoWorkWrapper(args):
""" A wrapper around the DoWork() method.
@@ -263,7 +289,9 @@
def setenv(name, val):
if val is None:
val = ''
- env[name] = val.encode()
+ if hasattr(val, 'encode'):
+ val = val.encode()
+ env[name] = val
setenv('REPO_PROJECT', project['name'])
setenv('REPO_PATH', project['relpath'])
diff --git a/subcmds/init.py b/subcmds/init.py
index b73de71..dbb6ddd 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -27,7 +27,7 @@
import imp
import urlparse
urllib = imp.new_module('urllib')
- urllib.parse = urlparse.urlparse
+ urllib.parse = urlparse
from color import Coloring
from command import InteractiveCommand, MirrorSafeCommand
@@ -153,7 +153,7 @@
# server where this git is located, so let's save that here.
mirrored_manifest_git = None
if opt.reference:
- manifest_git_path = urllib.parse(opt.manifest_url).path[1:]
+ manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
if not mirrored_manifest_git.endswith(".git"):
mirrored_manifest_git += ".git"
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 2bdab3a..ec333ae 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -131,6 +131,10 @@
The -c/--current-branch option can be used to only fetch objects that
are on the branch specified by a project's revision.
+The --optimized-fetch option can be used to only fetch projects that
+are fixed to a sha1 revision if the sha1 revision does not already
+exist locally.
+
SSH Connections
---------------
@@ -206,6 +210,9 @@
p.add_option('--no-tags',
dest='no_tags', action='store_true',
help="don't fetch tags")
+ p.add_option('--optimized-fetch',
+ dest='optimized_fetch', action='store_true',
+ help='only fetch projects fixed to sha1 if revision does not exist locally')
if show_smart:
p.add_option('-s', '--smart-sync',
dest='smart_sync', action='store_true',
@@ -275,7 +282,8 @@
quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
clone_bundle=not opt.no_clone_bundle,
- no_tags=opt.no_tags, archive=self.manifest.IsArchive)
+ no_tags=opt.no_tags, archive=self.manifest.IsArchive,
+ optimized_fetch=opt.optimized_fetch)
self._fetch_times.Set(project, time.time() - start)
# Lock around all the rest of the code, since printing, updating a set
@@ -509,6 +517,9 @@
self.manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
+ smart_sync_manifest_name = "smart_sync_override.xml"
+ smart_sync_manifest_path = os.path.join(
+ self.manifest.manifestProject.worktree, smart_sync_manifest_name)
if opt.smart_sync or opt.smart_tag:
if not self.manifest.manifest_server:
@@ -575,17 +586,16 @@
[success, manifest_str] = server.GetManifest(opt.smart_tag)
if success:
- manifest_name = "smart_sync_override.xml"
- manifest_path = os.path.join(self.manifest.manifestProject.worktree,
- manifest_name)
+ manifest_name = smart_sync_manifest_name
try:
- f = open(manifest_path, 'w')
+ f = open(smart_sync_manifest_path, 'w')
try:
f.write(manifest_str)
finally:
f.close()
- except IOError:
- print('error: cannot write manifest to %s' % manifest_path,
+ except IOError as e:
+ print('error: cannot write manifest to %s:\n%s'
+ % (smart_sync_manifest_path, e),
file=sys.stderr)
sys.exit(1)
self._ReloadManifest(manifest_name)
@@ -602,6 +612,13 @@
% (self.manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
+ else: # Not smart sync or smart tag mode
+ if os.path.isfile(smart_sync_manifest_path):
+ try:
+ os.remove(smart_sync_manifest_path)
+ except OSError as e:
+ print('error: failed to remove existing smart sync override manifest: %s' %
+ e, file=sys.stderr)
rp = self.manifest.repoProject
rp.PreSync()
@@ -615,7 +632,8 @@
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,
- no_tags=opt.no_tags)
+ no_tags=opt.no_tags,
+ optimized_fetch=opt.optimized_fetch)
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)