diff --git a/.pylintrc b/.pylintrc
index 2ed0940..c6be743 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -61,9 +61,6 @@
 # (visual studio) and html
 output-format=text
 
-# Include message's id in output
-include-ids=yes
-
 # Put messages in a separate file for each module / package specified on the
 # command line instead of printing them on stdout. Reports (if any) will be
 # written in a file name "pylint_global.[txt|html]".
diff --git a/color.py b/color.py
index 7970198..0218aab 100644
--- a/color.py
+++ b/color.py
@@ -18,41 +18,43 @@
 
 import pager
 
-COLORS = {None     :-1,
-          'normal' :-1,
-          'black'  : 0,
-          'red'    : 1,
-          'green'  : 2,
-          'yellow' : 3,
-          'blue'   : 4,
+COLORS = {None: -1,
+          'normal': -1,
+          'black': 0,
+          'red': 1,
+          'green': 2,
+          'yellow': 3,
+          'blue': 4,
           'magenta': 5,
-          'cyan'   : 6,
-          'white'  : 7}
+          'cyan': 6,
+          'white': 7}
 
-ATTRS = {None     :-1,
-         'bold'   : 1,
-         'dim'    : 2,
-         'ul'     : 4,
-         'blink'  : 5,
+ATTRS = {None: -1,
+         'bold': 1,
+         'dim': 2,
+         'ul': 4,
+         'blink': 5,
          'reverse': 7}
 
-RESET = "\033[m"  # pylint: disable=W1401
-                  # backslash is not anomalous
+RESET = "\033[m"
+
 
 def is_color(s):
   return s in COLORS
 
+
 def is_attr(s):
   return s in ATTRS
 
-def _Color(fg = None, bg = None, attr = None):
+
+def _Color(fg=None, bg=None, attr=None):
   fg = COLORS[fg]
   bg = COLORS[bg]
   attr = ATTRS[attr]
 
   if attr >= 0 or fg >= 0 or bg >= 0:
     need_sep = False
-    code = "\033["  #pylint: disable=W1401
+    code = "\033["
 
     if attr >= 0:
       code += chr(ord('0') + attr)
@@ -71,7 +73,6 @@
     if bg >= 0:
       if need_sep:
         code += ';'
-      need_sep = True
 
       if bg < 8:
         code += '4%c' % (ord('0') + bg)
@@ -82,6 +83,27 @@
     code = ''
   return code
 
+DEFAULT = None
+
+
+def SetDefaultColoring(state):
+  """Set coloring behavior to |state|.
+
+  This is useful for overriding config options via the command line.
+  """
+  if state is None:
+    # Leave it alone -- return quick!
+    return
+
+  global DEFAULT
+  state = state.lower()
+  if state in ('auto',):
+    DEFAULT = state
+  elif state in ('always', 'yes', 'true', True):
+    DEFAULT = 'always'
+  elif state in ('never', 'no', 'false', False):
+    DEFAULT = 'never'
+
 
 class Coloring(object):
   def __init__(self, config, section_type):
@@ -89,9 +111,11 @@
     self._config = config
     self._out = sys.stdout
 
-    on = self._config.GetString(self._section)
+    on = DEFAULT
     if on is None:
-      on = self._config.GetString('color.ui')
+      on = self._config.GetString(self._section)
+      if on is None:
+        on = self._config.GetString('color.ui')
 
     if on == 'auto':
       if pager.active or os.isatty(1):
@@ -122,6 +146,7 @@
   def printer(self, opt=None, fg=None, bg=None, attr=None):
     s = self
     c = self.colorer(opt, fg, bg, attr)
+
     def f(fmt, *args):
       s._out.write(c(fmt, *args))
     return f
@@ -129,6 +154,7 @@
   def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
     s = self
     c = self.nofmt_colorer(opt, fg, bg, attr)
+
     def f(fmt):
       s._out.write(c(fmt))
     return f
@@ -136,11 +162,13 @@
   def colorer(self, opt=None, fg=None, bg=None, attr=None):
     if self._on:
       c = self._parse(opt, fg, bg, attr)
+
       def f(fmt, *args):
         output = fmt % args
         return ''.join([c, output, RESET])
       return f
     else:
+
       def f(fmt, *args):
         return fmt % args
       return f
@@ -148,6 +176,7 @@
   def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
     if self._on:
       c = self._parse(opt, fg, bg, attr)
+
       def f(fmt):
         return ''.join([c, fmt, RESET])
       return f
diff --git a/docs/manifest-format.txt b/docs/manifest-format.txt
index e48b75f..1aa9396 100644
--- a/docs/manifest-format.txt
+++ b/docs/manifest-format.txt
@@ -26,6 +26,7 @@
                         manifest-server?,
                         remove-project*,
                         project*,
+                        extend-project*,
                         repo-hooks?)>
 
     <!ELEMENT notice (#PCDATA)>
@@ -35,6 +36,7 @@
     <!ATTLIST remote alias        CDATA #IMPLIED>
     <!ATTLIST remote fetch        CDATA #REQUIRED>
     <!ATTLIST remote review       CDATA #IMPLIED>
+    <!ATTLIST remote revision     CDATA #IMPLIED>
 
     <!ELEMENT default (EMPTY)>
     <!ATTLIST default remote      IDREF #IMPLIED>
@@ -66,6 +68,11 @@
     <!ATTLIST annotation value CDATA #REQUIRED>
     <!ATTLIST annotation keep  CDATA "true">
 
+    <!ELEMENT extend-project>
+    <!ATTLIST extend-project name CDATA #REQUIRED>
+    <!ATTLIST extend-project path CDATA #IMPLIED>
+    <!ATTLIST extend-project groups CDATA #IMPLIED>
+
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
 
@@ -112,6 +119,10 @@
 are uploaded to by `repo upload`.  This attribute is optional;
 if not specified then `repo upload` will not function.
 
+Attribute `revision`: Name of a Git branch (e.g. `master` or
+`refs/heads/master`). Remotes with their own revision will override
+the default revision.
+
 Element default
 ---------------
 
@@ -132,14 +143,14 @@
 this value. If this value is not set, projects will use `revision`
 by default instead.
 
-Attribute `sync_j`: Number of parallel jobs to use when synching.
+Attribute `sync-j`: Number of parallel jobs to use when synching.
 
-Attribute `sync_c`: Set to true to only sync the given Git
+Attribute `sync-c`: Set to true to only sync the given Git
 branch (specified in the `revision` attribute) rather than the
-whole ref space.  Project elements lacking a sync_c element of
+whole ref space.  Project elements lacking a sync-c element of
 their own will use this value.
 
-Attribute `sync_s`: Set to true to also sync sub-projects.
+Attribute `sync-s`: Set to true to also sync sub-projects.
 
 
 Element manifest-server
@@ -208,7 +219,8 @@
 (e.g. just "master") or absolute (e.g. "refs/heads/master").
 Tags and/or explicit SHA-1s should work in theory, but have not
 been extensively tested.  If not supplied the revision given by
-the default element is used.
+the remote element is used if applicable, else the default
+element is used.
 
 Attribute `dest-branch`: Name of a Git branch (e.g. `master`).
 When using `repo upload`, changes will be submitted for code
@@ -226,13 +238,13 @@
 If the project has a parent element, the `name` and `path` here
 are the prefixed ones.
 
-Attribute `sync_c`: Set to true to only sync the given Git
+Attribute `sync-c`: Set to true to only sync the given Git
 branch (specified in the `revision` attribute) rather than the
 whole ref space.
 
-Attribute `sync_s`: Set to true to also sync sub-projects.
+Attribute `sync-s`: Set to true to also sync sub-projects.
 
-Attribute `upstream`: Name of the Git branch in which a sha1
+Attribute `upstream`: Name of the Git ref in which a sha1
 can be found.  Used when syncing a revision locked manifest in
 -c mode to avoid having to sync the entire ref space.
 
@@ -246,6 +258,22 @@
 local mirrors syncing, it will be ignored when syncing the projects in a
 client working directory.
 
+Element extend-project
+----------------------
+
+Modify the attributes of the named project.
+
+This element is mostly useful in a local manifest file, to modify the
+attributes of an existing project without completely replacing the
+existing project definition.  This makes the local manifest more robust
+against changes to the original manifest.
+
+Attribute `path`: If specified, limit the change to projects checked out
+at the specified path, rather than all projects with the given name.
+
+Attribute `groups`: List of additional groups to which this project
+belongs.  Same syntax as the corresponding element of `project`.
+
 Element annotation
 ------------------
 
diff --git a/error.py b/error.py
index ff948f9..f2a7c4e 100644
--- a/error.py
+++ b/error.py
@@ -80,7 +80,7 @@
     self.name = name
 
   def __str__(self):
-    if self.Name is None:
+    if self.name is None:
       return 'in current directory'
     return self.name
 
@@ -93,7 +93,7 @@
     self.name = name
 
   def __str__(self):
-    if self.Name is None:
+    if self.name is None:
       return 'in current directory'
     return self.name
 
diff --git a/git_command.py b/git_command.py
index 354fc71..0893bff 100644
--- a/git_command.py
+++ b/git_command.py
@@ -14,7 +14,9 @@
 # limitations under the License.
 
 from __future__ import print_function
+import fcntl
 import os
+import select
 import sys
 import subprocess
 import tempfile
@@ -76,17 +78,30 @@
 
 _git_version = None
 
+class _sfd(object):
+  """select file descriptor class"""
+  def __init__(self, fd, dest, std_name):
+    assert std_name in ('stdout', 'stderr')
+    self.fd = fd
+    self.dest = dest
+    self.std_name = std_name
+  def fileno(self):
+    return self.fd.fileno()
+
 class _GitCall(object):
   def version(self):
     p = GitCommand(None, ['--version'], capture_stdout=True)
     if p.Wait() == 0:
-      return p.stdout
+      if hasattr(p.stdout, 'decode'):
+        return p.stdout.decode('utf-8')
+      else:
+        return p.stdout
     return None
 
   def version_tuple(self):
     global _git_version
     if _git_version is None:
-      ver_str = git.version().decode('utf-8')
+      ver_str = git.version()
       _git_version = Wrapper().ParseGitVersion(ver_str)
       if _git_version is None:
         print('fatal: "%s" unsupported' % ver_str, file=sys.stderr)
@@ -139,6 +154,9 @@
       if key in env:
         del env[key]
 
+    # If we are not capturing std* then need to print it.
+    self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr}
+
     if disable_editor:
       _setenv(env, 'GIT_EDITOR', ':')
     if ssh_proxy:
@@ -162,22 +180,21 @@
       if gitdir:
         _setenv(env, GIT_DIR, gitdir)
       cwd = None
-    command.extend(cmdv)
+    command.append(cmdv[0])
+    # Need to use the --progress flag for fetch/clone so output will be
+    # displayed as by default git only does progress output if stderr is a TTY.
+    if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'):
+      if '--progress' not in cmdv and '--quiet' not in cmdv:
+        command.append('--progress')
+    command.extend(cmdv[1:])
 
     if provide_stdin:
       stdin = subprocess.PIPE
     else:
       stdin = None
 
-    if capture_stdout:
-      stdout = subprocess.PIPE
-    else:
-      stdout = None
-
-    if capture_stderr:
-      stderr = subprocess.PIPE
-    else:
-      stderr = None
+    stdout = subprocess.PIPE
+    stderr = subprocess.PIPE
 
     if IsTrace():
       global LAST_CWD
@@ -226,8 +243,36 @@
   def Wait(self):
     try:
       p = self.process
-      (self.stdout, self.stderr) = p.communicate()
-      rc = p.returncode
+      rc = self._CaptureOutput()
     finally:
       _remove_ssh_client(p)
     return rc
+
+  def _CaptureOutput(self):
+    p = self.process
+    s_in = [_sfd(p.stdout, sys.stdout, 'stdout'),
+            _sfd(p.stderr, sys.stderr, 'stderr')]
+    self.stdout = ''
+    self.stderr = ''
+
+    for s in s_in:
+      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    while s_in:
+      in_ready, _, _ = select.select(s_in, [], [])
+      for s in in_ready:
+        buf = s.fd.read(4096)
+        if not buf:
+          s_in.remove(s)
+          continue
+        if not hasattr(buf, 'encode'):
+          buf = buf.decode()
+        if s.std_name == 'stdout':
+          self.stdout += buf
+        else:
+          self.stderr += buf
+        if self.tee[s.std_name]:
+          s.dest.write(buf)
+          s.dest.flush()
+    return p.wait()
diff --git a/git_config.py b/git_config.py
index 32879ec..8ded7c2 100644
--- a/git_config.py
+++ b/git_config.py
@@ -15,8 +15,8 @@
 
 from __future__ import print_function
 
+import json
 import os
-import pickle
 import re
 import subprocess
 import sys
@@ -80,7 +80,7 @@
     return cls(configfile = os.path.join(gitdir, 'config'),
                defaults = defaults)
 
-  def __init__(self, configfile, defaults=None, pickleFile=None):
+  def __init__(self, configfile, defaults=None, jsonFile=None):
     self.file = configfile
     self.defaults = defaults
     self._cache_dict = None
@@ -88,12 +88,11 @@
     self._remotes = {}
     self._branches = {}
 
-    if pickleFile is None:
-      self._pickle = os.path.join(
+    self._json = jsonFile
+    if self._json is None:
+      self._json = os.path.join(
         os.path.dirname(self.file),
-        '.repopickle_' + os.path.basename(self.file))
-    else:
-      self._pickle = pickleFile
+        '.repo_' + os.path.basename(self.file) + '.json')
 
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
@@ -217,9 +216,9 @@
     """Resolve any url.*.insteadof references.
     """
     for new_url in self.GetSubSections('url'):
-      old_url = self.GetString('url.%s.insteadof' % new_url)
-      if old_url is not None and url.startswith(old_url):
-        return new_url + url[len(old_url):]
+      for old_url in self.GetString('url.%s.insteadof' % new_url, True):
+        if old_url is not None and url.startswith(old_url):
+          return new_url + url[len(old_url):]
     return url
 
   @property
@@ -248,50 +247,41 @@
     return self._cache_dict
 
   def _Read(self):
-    d = self._ReadPickle()
+    d = self._ReadJson()
     if d is None:
       d = self._ReadGit()
-      self._SavePickle(d)
+      self._SaveJson(d)
     return d
 
-  def _ReadPickle(self):
+  def _ReadJson(self):
     try:
-      if os.path.getmtime(self._pickle) \
+      if os.path.getmtime(self._json) \
       <= os.path.getmtime(self.file):
-        os.remove(self._pickle)
+        os.remove(self._json)
         return None
     except OSError:
       return None
     try:
-      Trace(': unpickle %s', self.file)
-      fd = open(self._pickle, 'rb')
+      Trace(': parsing %s', self.file)
+      fd = open(self._json)
       try:
-        return pickle.load(fd)
+        return json.load(fd)
       finally:
         fd.close()
-    except EOFError:
-      os.remove(self._pickle)
-      return None
-    except IOError:
-      os.remove(self._pickle)
-      return None
-    except pickle.PickleError:
-      os.remove(self._pickle)
+    except (IOError, ValueError):
+      os.remove(self._json)
       return None
 
-  def _SavePickle(self, cache):
+  def _SaveJson(self, cache):
     try:
-      fd = open(self._pickle, 'wb')
+      fd = open(self._json, 'w')
       try:
-        pickle.dump(cache, fd, pickle.HIGHEST_PROTOCOL)
+        json.dump(cache, fd, indent=2)
       finally:
         fd.close()
-    except IOError:
-      if os.path.exists(self._pickle):
-        os.remove(self._pickle)
-    except pickle.PickleError:
-      if os.path.exists(self._pickle):
-        os.remove(self._pickle)
+    except (IOError, TypeError):
+      if os.path.exists(self._json):
+        os.remove(self._json)
 
   def _ReadGit(self):
     """
@@ -576,6 +566,8 @@
         return None
 
       u = self.review
+      if u.startswith('persistent-'):
+        u = u[len('persistent-'):]
       if u.split(':')[0] not in ('http', 'https', 'sso'):
         u = 'http://%s' % u
       if u.endswith('/Gerrit'):
@@ -627,9 +619,7 @@
   def ToLocal(self, rev):
     """Convert a remote revision string to something we have locally.
     """
-    if IsId(rev):
-      return rev
-    if rev.startswith(R_TAGS):
+    if self.name == '.' or IsId(rev):
       return rev
 
     if not rev.startswith('refs/'):
@@ -638,6 +628,10 @@
     for spec in self.fetch:
       if spec.SourceMatches(rev):
         return spec.MapSource(rev)
+
+    if not rev.startswith(R_HEADS):
+      return rev
+
     raise GitError('remote %s does not have %s' % (self.name, rev))
 
   def WritesTo(self, ref):
@@ -707,7 +701,7 @@
       self._Set('merge', self.merge)
 
     else:
-      fd = open(self._config.file, 'ab')
+      fd = open(self._config.file, 'a')
       try:
         fd.write('[branch "%s"]\n' % self.name)
         if self.remote:
diff --git a/hooks/commit-msg b/hooks/commit-msg
index 5ca2b11..d8f009b 100755
--- a/hooks/commit-msg
+++ b/hooks/commit-msg
@@ -1,5 +1,4 @@
 #!/bin/sh
-# From Gerrit Code Review 2.6
 #
 # Part of Gerrit Code Review (http://code.google.com/p/gerrit/)
 #
@@ -27,7 +26,7 @@
 #
 add_ChangeId() {
 	clean_message=`sed -e '
-		/^diff --git a\/.*/{
+		/^diff --git .*/{
 			s///
 			q
 		}
@@ -39,6 +38,11 @@
 		return
 	fi
 
+	if test "false" = "`git config --bool --get gerrit.createChangeId`"
+	then
+		return
+	fi
+
 	# Does Change-Id: already exist? if so, exit (no change).
 	if grep -i '^Change-Id:' "$MSG" >/dev/null
 	then
@@ -77,7 +81,7 @@
 	# Skip the line starting with the diff command and everything after it,
 	# up to the end of the file, assuming it is only patch data.
 	# If more than one line before the diff was empty, strip all but one.
-	/^diff --git a/ {
+	/^diff --git / {
 		blankLines = 0
 		while (getline) { }
 		next
diff --git a/main.py b/main.py
index 72fb39b..6736abc 100755
--- a/main.py
+++ b/main.py
@@ -36,6 +36,7 @@
 except ImportError:
   kerberos = None
 
+from color import SetDefaultColoring
 from trace import SetTrace
 from git_command import git, GitCommand
 from git_config import init_ssh, close_ssh
@@ -44,6 +45,7 @@
 from subcmds.version import Version
 from editor import Editor
 from error import DownloadError
+from error import InvalidProjectGroupsError
 from error import ManifestInvalidRevisionError
 from error import ManifestParseError
 from error import NoManifestException
@@ -69,6 +71,9 @@
 global_options.add_option('--no-pager',
                           dest='no_pager', action='store_true',
                           help='disable the pager')
+global_options.add_option('--color',
+                          choices=('auto', 'always', 'never'), default=None,
+                          help='control color usage: auto, always, never')
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution')
@@ -113,6 +118,8 @@
         print('fatal: invalid usage of --version', file=sys.stderr)
         return 1
 
+    SetDefaultColoring(gopts.color)
+
     try:
       cmd = self.commands[name]
     except KeyError:
@@ -167,6 +174,12 @@
       else:
         print('error: no project in current directory', file=sys.stderr)
       result = 1
+    except InvalidProjectGroupsError as e:
+      if e.name:
+        print('error: project group must be enabled for project %s' % e.name, file=sys.stderr)
+      else:
+        print('error: project group must be enabled for the project in the current directory', file=sys.stderr)
+      result = 1
     finally:
       elapsed = time.time() - start
       hours, remainder = divmod(elapsed, 3600)
diff --git a/manifest_xml.py b/manifest_xml.py
index 3c8fadd..130e17c 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -38,8 +38,9 @@
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
 
-urllib.parse.uses_relative.extend(['ssh', 'git'])
-urllib.parse.uses_netloc.extend(['ssh', 'git'])
+# urljoin gets confused if the scheme is not known.
+urllib.parse.uses_relative.extend(['ssh', 'git', 'persistent-https', 'rpc'])
+urllib.parse.uses_netloc.extend(['ssh', 'git', 'persistent-https', 'rpc'])
 
 class _Default(object):
   """Project defaults within the manifest."""
@@ -63,12 +64,14 @@
                alias=None,
                fetch=None,
                manifestUrl=None,
-               review=None):
+               review=None,
+               revision=None):
     self.name = name
     self.fetchUrl = fetch
     self.manifestUrl = manifestUrl
     self.remoteAlias = alias
     self.reviewUrl = review
+    self.revision = revision
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
   def __eq__(self, other):
@@ -83,17 +86,14 @@
     # urljoin will gets confused over quite a few things.  The ones we care
     # about here are:
     # * no scheme in the base url, like <hostname:port>
-    # * persistent-https://
-    # We handle this by replacing these with obscure protocols
-    # and then replacing them with the original when we are done.
-    # gopher -> <none>
-    # wais -> persistent-https
+    # We handle no scheme by replacing it with an obscure protocol, gopher
+    # and then replacing it with the original when we are done.
+
     if manifestUrl.find(':') != manifestUrl.find('/') - 1:
-      manifestUrl = 'gopher://' + manifestUrl
-    manifestUrl = re.sub(r'^persistent-https://', 'wais://', manifestUrl)
-    url = urllib.parse.urljoin(manifestUrl, url)
-    url = re.sub(r'^gopher://', '', url)
-    url = re.sub(r'^wais://', 'persistent-https://', url)
+      url = urllib.parse.urljoin('gopher://' + manifestUrl, url)
+      url = re.sub(r'^gopher://', '', url)
+    else:
+      url = urllib.parse.urljoin(manifestUrl, url)
     return url
 
   def ToRemoteSpec(self, projectName):
@@ -159,6 +159,11 @@
       e.setAttribute('alias', r.remoteAlias)
     if r.reviewUrl is not None:
       e.setAttribute('review', r.reviewUrl)
+    if r.revision is not None:
+      e.setAttribute('revision', r.revision)
+
+  def _ParseGroups(self, groups):
+    return [x for x in re.split(r'[,\s]+', groups) if x]
 
   def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
     """Write the current manifest out to the given file descriptor.
@@ -167,7 +172,7 @@
 
     groups = mp.config.GetString('manifest.groups')
     if groups:
-      groups = [x for x in re.split(r'[,\s]+', groups) if x]
+      groups = self._ParseGroups(groups)
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
@@ -240,20 +245,27 @@
       if d.remote:
         remoteName = d.remote.remoteAlias or d.remote.name
       if not d.remote or p.remote.name != remoteName:
-        e.setAttribute('remote', p.remote.name)
+        remoteName = p.remote.name
+        e.setAttribute('remote', remoteName)
       if peg_rev:
         if self.IsMirror:
           value = p.bare_git.rev_parse(p.revisionExpr + '^0')
         else:
           value = p.work_git.rev_parse(HEAD + '^0')
         e.setAttribute('revision', value)
-        if peg_rev_upstream and value != p.revisionExpr:
-          # Only save the origin if the origin is not a sha1, and the default
-          # isn't our value, and the if the default doesn't already have that
-          # covered.
-          e.setAttribute('upstream', p.revisionExpr)
-      elif not d.revisionExpr or p.revisionExpr != d.revisionExpr:
-        e.setAttribute('revision', p.revisionExpr)
+        if peg_rev_upstream:
+          if p.upstream:
+            e.setAttribute('upstream', p.upstream)
+          elif value != p.revisionExpr:
+            # Only save the origin if the origin is not a sha1, and the default
+            # isn't our value
+            e.setAttribute('upstream', p.revisionExpr)
+      else:
+        revision = self.remotes[remoteName].revision or d.revisionExpr
+        if not revision or revision != p.revisionExpr:
+          e.setAttribute('revision', p.revisionExpr)
+        if p.upstream and p.upstream != p.revisionExpr:
+          e.setAttribute('upstream', p.upstream)
 
       for c in p.copyfiles:
         ce = doc.createElement('copyfile')
@@ -261,6 +273,12 @@
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+      for l in p.linkfiles:
+        le = doc.createElement('linkfile')
+        le.setAttribute('src', l.src)
+        le.setAttribute('dest', l.dest)
+        e.appendChild(le)
+
       default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
       egroups = [g for g in p.groups if g not in default_groups]
       if egroups:
@@ -304,7 +322,7 @@
   @property
   def projects(self):
     self._Load()
-    return self._paths.values()
+    return list(self._paths.values())
 
   @property
   def remotes(self):
@@ -492,6 +510,23 @@
       if node.nodeName == 'project':
         project = self._ParseProject(node)
         recursively_add_projects(project)
+      if node.nodeName == 'extend-project':
+        name = self._reqatt(node, 'name')
+
+        if name not in self._projects:
+          raise ManifestParseError('extend-project element specifies non-existent '
+                                   'project: %s' % name)
+
+        path = node.getAttribute('path')
+        groups = node.getAttribute('groups')
+        if groups:
+          groups = self._ParseGroups(groups)
+
+        for p in self._projects[name]:
+          if path and p.relpath != path:
+            continue
+          if groups:
+            p.groups.extend(groups)
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -586,8 +621,11 @@
     review = node.getAttribute('review')
     if review == '':
       review = None
+    revision = node.getAttribute('revision')
+    if revision == '':
+      revision = None
     manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-    return _XmlRemote(name, alias, fetch, manifestUrl, review)
+    return _XmlRemote(name, alias, fetch, manifestUrl, review, revision)
 
   def _ParseDefault(self, node):
     """
@@ -680,7 +718,7 @@
       raise ManifestParseError("no remote for project %s within %s" %
             (name, self.manifestFile))
 
-    revisionExpr = node.getAttribute('revision')
+    revisionExpr = node.getAttribute('revision') or remote.revision
     if not revisionExpr:
       revisionExpr = self._default.revisionExpr
     if not revisionExpr:
@@ -729,7 +767,7 @@
     groups = ''
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
-    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+    groups = self._ParseGroups(groups)
 
     if parent is None:
       relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
@@ -765,6 +803,8 @@
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
+      if n.nodeName == 'linkfile':
+        self._ParseLinkFile(project, n)
       if n.nodeName == 'annotation':
         self._ParseAnnotation(project, n)
       if n.nodeName == 'project':
@@ -814,6 +854,14 @@
       # dest is relative to the top of the tree
       project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
+  def _ParseLinkFile(self, project, node):
+    src = self._reqatt(node, 'src')
+    dest = self._reqatt(node, 'dest')
+    if not self.IsMirror:
+      # src is project relative;
+      # dest is relative to the top of the tree
+      project.AddLinkFile(src, dest, os.path.join(self.topdir, dest))
+
   def _ParseAnnotation(self, project, node):
     name = self._reqatt(node, 'name')
     value = self._reqatt(node, 'value')
@@ -856,10 +904,8 @@
     fromProjects = self.paths
     toProjects = manifest.paths
 
-    fromKeys = fromProjects.keys()
-    fromKeys.sort()
-    toKeys = toProjects.keys()
-    toKeys.sort()
+    fromKeys = sorted(fromProjects.keys())
+    toKeys = sorted(toProjects.keys())
 
     diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
 
diff --git a/project.py b/project.py
index 023cf73..868425c 100644
--- a/project.py
+++ b/project.py
@@ -13,9 +13,10 @@
 # limitations under the License.
 
 from __future__ import print_function
-import traceback
+import contextlib
 import errno
 import filecmp
+import glob
 import os
 import random
 import re
@@ -26,11 +27,12 @@
 import tarfile
 import tempfile
 import time
+import traceback
 
 from color import Coloring
 from git_command import GitCommand, git_require
 from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
-from error import GitError, HookError, UploadError
+from error import GitError, HookError, UploadError, DownloadError
 from error import ManifestInvalidRevisionError
 from error import NoManifestException
 from trace import IsTrace, Trace
@@ -46,7 +48,7 @@
 def _lwrite(path, content):
   lock = '%s.lock' % path
 
-  fd = open(lock, 'wb')
+  fd = open(lock, 'w')
   try:
     fd.write(content)
   finally:
@@ -84,7 +86,7 @@
   global _project_hook_list
   if _project_hook_list is None:
     d = os.path.realpath(os.path.abspath(os.path.dirname(__file__)))
-    d = os.path.join(d , 'hooks')
+    d = os.path.join(d, 'hooks')
     _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
   return _project_hook_list
 
@@ -182,28 +184,28 @@
 class StatusColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'status')
-    self.project   = self.printer('header',    attr = 'bold')
-    self.branch    = self.printer('header',    attr = 'bold')
-    self.nobranch  = self.printer('nobranch',  fg = 'red')
-    self.important = self.printer('important', fg = 'red')
+    self.project = self.printer('header', attr='bold')
+    self.branch = self.printer('header', attr='bold')
+    self.nobranch = self.printer('nobranch', fg='red')
+    self.important = self.printer('important', fg='red')
 
-    self.added     = self.printer('added',     fg = 'green')
-    self.changed   = self.printer('changed',   fg = 'red')
-    self.untracked = self.printer('untracked', fg = 'red')
+    self.added = self.printer('added', fg='green')
+    self.changed = self.printer('changed', fg='red')
+    self.untracked = self.printer('untracked', fg='red')
 
 
 class DiffColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'diff')
-    self.project   = self.printer('header',    attr = 'bold')
+    self.project = self.printer('header', attr='bold')
 
-class _Annotation:
+class _Annotation(object):
   def __init__(self, name, value, keep):
     self.name = name
     self.value = value
     self.keep = keep
 
-class _CopyFile:
+class _CopyFile(object):
   def __init__(self, src, dest, abssrc, absdest):
     self.src = src
     self.dest = dest
@@ -231,14 +233,72 @@
       except IOError:
         _error('Cannot copy file %s to %s', src, dest)
 
+class _LinkFile(object):
+  def __init__(self, git_worktree, src, dest, relsrc, absdest):
+    self.git_worktree = git_worktree
+    self.src = src
+    self.dest = dest
+    self.src_rel_to_dest = relsrc
+    self.abs_dest = absdest
+
+  def __linkIt(self, relSrc, absDest):
+    # link file if it does not exist or is out of date
+    if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc):
+      try:
+        # remove existing file first, since it might be read-only
+        if os.path.exists(absDest):
+          os.remove(absDest)
+        else:
+          dest_dir = os.path.dirname(absDest)
+          if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir)
+        os.symlink(relSrc, absDest)
+      except IOError:
+        _error('Cannot link file %s to %s', relSrc, absDest)
+
+  def _Link(self):
+    """Link the self.rel_src_to_dest and self.abs_dest. Handles wild cards
+    on the src linking all of the files in the source in to the destination
+    directory.
+    """
+    # We use the absSrc to handle the situation where the current directory
+    # is not the root of the repo
+    absSrc = os.path.join(self.git_worktree, self.src)
+    if os.path.exists(absSrc):
+      # Entity exists so just a simple one to one link operation
+      self.__linkIt(self.src_rel_to_dest, self.abs_dest)
+    else:
+      # Entity doesn't exist assume there is a wild card
+      absDestDir = self.abs_dest
+      if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
+        _error('Link error: src with wildcard, %s must be a directory',
+            absDestDir)
+      else:
+        absSrcFiles = glob.glob(absSrc)
+        for absSrcFile in absSrcFiles:
+          # Create a releative path from source dir to destination dir
+          absSrcDir = os.path.dirname(absSrcFile)
+          relSrcDir = os.path.relpath(absSrcDir, absDestDir)
+
+          # Get the source file name
+          srcFile = os.path.basename(absSrcFile)
+
+          # Now form the final full paths to srcFile. They will be
+          # absolute for the desintaiton and relative for the srouce.
+          absDest = os.path.join(absDestDir, srcFile)
+          relSrc = os.path.join(relSrcDir, srcFile)
+          self.__linkIt(relSrc, absDest)
+
 class RemoteSpec(object):
   def __init__(self,
                name,
-               url = None,
-               review = None):
+               url=None,
+               review=None,
+               revision=None):
     self.name = name
     self.url = url
     self.review = review
+    self.revision = revision
 
 class RepoHook(object):
   """A RepoHook contains information about a script to run as a hook.
@@ -414,7 +474,8 @@
       # and  convert to a HookError w/ just the failing traceback.
       context = {}
       try:
-        execfile(self._script_fullpath, context)
+        exec(compile(open(self._script_fullpath).read(),
+                     self._script_fullpath, 'exec'), context)
       except Exception:
         raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
                         traceback.format_exc(), self._hook_type))
@@ -483,6 +544,12 @@
 
 
 class Project(object):
+  # These objects can be shared between several working trees.
+  shareable_files = ['description', 'info']
+  shareable_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
+  # These objects can only be used by a single working tree.
+  working_tree_files = ['config', 'packed-refs', 'shallow']
+  working_tree_dirs = ['logs', 'refs']
   def __init__(self,
                manifest,
                name,
@@ -493,15 +560,16 @@
                relpath,
                revisionExpr,
                revisionId,
-               rebase = True,
-               groups = None,
-               sync_c = False,
-               sync_s = False,
-               clone_depth = None,
-               upstream = None,
-               parent = None,
-               is_derived = False,
-               dest_branch = None):
+               rebase=True,
+               groups=None,
+               sync_c=False,
+               sync_s=False,
+               clone_depth=None,
+               upstream=None,
+               parent=None,
+               is_derived=False,
+               dest_branch=None,
+               optimized_fetch=False):
     """Init a Project object.
 
     Args:
@@ -523,6 +591,8 @@
       is_derived: False if the project was explicitly defined in the manifest;
                   True if the project is a discovered submodule.
       dest_branch: The branch to which to push changes for review by default.
+      optimized_fetch: If True, when a project is set to a sha1 revision, only
+                       fetch from the remote if the sha1 is not present locally.
     """
     self.manifest = manifest
     self.name = name
@@ -551,14 +621,16 @@
     self.upstream = upstream
     self.parent = parent
     self.is_derived = is_derived
+    self.optimized_fetch = optimized_fetch
     self.subprojects = []
 
     self.snapshots = {}
     self.copyfiles = []
+    self.linkfiles = []
     self.annotations = []
     self.config = GitConfig.ForRepository(
-                    gitdir = self.gitdir,
-                    defaults =  self.manifest.globalConfig)
+                    gitdir=self.gitdir,
+                    defaults=self.manifest.globalConfig)
 
     if self.worktree:
       self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
@@ -579,7 +651,7 @@
 
   @property
   def Exists(self):
-    return os.path.isdir(self.gitdir)
+    return os.path.isdir(self.gitdir) and os.path.isdir(self.objdir)
 
   @property
   def CurrentBranch(self):
@@ -708,27 +780,49 @@
     return matched
 
 ## Status Display ##
+  def UncommitedFiles(self, get_all=True):
+    """Returns a list of strings, uncommitted files in the git tree.
 
-  def HasChanges(self):
-    """Returns true if there are uncommitted changes.
+    Args:
+      get_all: a boolean, if True - get information about all different
+               uncommitted files. If False - return as soon as any kind of
+               uncommitted files is detected.
     """
+    details = []
     self.work_git.update_index('-q',
                                '--unmerged',
                                '--ignore-missing',
                                '--refresh')
     if self.IsRebaseInProgress():
-      return True
+      details.append("rebase in progress")
+      if not get_all:
+        return details
 
-    if self.work_git.DiffZ('diff-index', '--cached', HEAD):
-      return True
+    changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
+    if changes:
+      details.extend(changes)
+      if not get_all:
+        return details
 
-    if self.work_git.DiffZ('diff-files'):
-      return True
+    changes = self.work_git.DiffZ('diff-files').keys()
+    if changes:
+      details.extend(changes)
+      if not get_all:
+        return details
 
-    if self.work_git.LsOthers():
-      return True
+    changes = self.work_git.LsOthers()
+    if changes:
+      details.extend(changes)
 
-    return False
+    return details
+
+  def HasChanges(self):
+    """Returns true if there are uncommitted changes.
+    """
+    if self.UncommitedFiles(get_all=False):
+      return True
+    else:
+      return False
 
   def PrintWorkTreeStatus(self, output_redir=None):
     """Prints the status of the repository to stdout.
@@ -758,7 +852,7 @@
     out = StatusColoring(self.config)
     if not output_redir == None:
       out.redirect(output_redir)
-    out.project('project %-40s', self.relpath + '/')
+    out.project('project %-40s', self.relpath + '/ ')
 
     branch = self.CurrentBranch
     if branch is None:
@@ -829,8 +923,8 @@
     cmd.append('--')
     p = GitCommand(self,
                    cmd,
-                   capture_stdout = True,
-                   capture_stderr = True)
+                   capture_stdout=True,
+                   capture_stderr=True)
     has_diff = False
     for line in p.process.stdout:
       if not has_diff:
@@ -915,7 +1009,7 @@
     return None
 
   def UploadForReview(self, branch=None,
-                      people=([],[]),
+                      people=([], []),
                       auto_topic=False,
                       draft=False,
                       dest_branch=None):
@@ -976,13 +1070,13 @@
         ref_spec = ref_spec + '%' + ','.join(rp)
     cmd.append(ref_spec)
 
-    if GitCommand(self, cmd, bare = True).Wait() != 0:
+    if GitCommand(self, cmd, bare=True).Wait() != 0:
       raise UploadError('Upload failed')
 
     msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
     self.bare_git.UpdateRef(R_PUB + branch.name,
                             R_HEADS + branch.name,
-                            message = msg)
+                            message=msg)
 
 
 ## Sync ##
@@ -1007,9 +1101,11 @@
       quiet=False,
       is_new=None,
       current_branch_only=False,
+      force_sync=False,
       clone_bundle=True,
       no_tags=False,
-      archive=False):
+      archive=False,
+      optimized_fetch=False):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
@@ -1040,13 +1136,12 @@
       except OSError as e:
         print("warn: Cannot remove archive %s: "
               "%s" % (tarpath, str(e)), file=sys.stderr)
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
       return True
-
     if is_new is None:
       is_new = not self.Exists
     if is_new:
-      self._InitGitDir()
+      self._InitGitDir(force_sync=force_sync)
     else:
       self._UpdateHooks()
     self._InitRemote()
@@ -1078,16 +1173,12 @@
       elif self.manifest.default.sync_c:
         current_branch_only = True
 
-    is_sha1 = False
-    if ID_RE.match(self.revisionExpr) is not None:
-      is_sha1 = True
-    if is_sha1 and self._CheckForSha1():
-      # Don't need to fetch since we already have this revision
-      return True
-
-    if not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
-                             current_branch_only=current_branch_only,
-                             no_tags=no_tags):
+    need_to_fetch = not (optimized_fetch and \
+      (ID_RE.match(self.revisionExpr) and self._CheckForSha1()))
+    if (need_to_fetch
+        and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
+                                  current_branch_only=current_branch_only,
+                                  no_tags=no_tags)):
       return False
 
     if self.worktree:
@@ -1103,9 +1194,11 @@
   def PostRepoUpgrade(self):
     self._InitHooks()
 
-  def _CopyFiles(self):
+  def _CopyAndLinkFiles(self):
     for copyfile in self.copyfiles:
       copyfile._Copy()
+    for linkfile in self.linkfiles:
+      linkfile._Link()
 
   def GetCommitRevisionId(self):
     """Get revisionId of a commit.
@@ -1141,18 +1234,18 @@
         'revision %s in %s not found' % (self.revisionExpr,
                                          self.name))
 
-  def Sync_LocalHalf(self, syncbuf):
+  def Sync_LocalHalf(self, syncbuf, force_sync=False):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
-    self._InitWorkTree()
+    self._InitWorkTree(force_sync=force_sync)
     all_refs = self.bare_ref.all
     self.CleanPublishedCache(all_refs)
     revid = self.GetRevisionId(all_refs)
 
     def _doff():
       self._FastForward(revid)
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
 
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
@@ -1188,7 +1281,7 @@
       except GitError as e:
         syncbuf.fail(self, e)
         return
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
       return
 
     if head == revid:
@@ -1210,7 +1303,7 @@
       except GitError as e:
         syncbuf.fail(self, e)
         return
-      self._CopyFiles()
+      self._CopyAndLinkFiles()
       return
 
     upstream_gain = self._revlist(not_rev(HEAD), revid)
@@ -1278,17 +1371,19 @@
     if not ID_RE.match(self.revisionExpr):
       # in case of manifest sync the revisionExpr might be a SHA1
       branch.merge = self.revisionExpr
+      if not branch.merge.startswith('refs/'):
+        branch.merge = R_HEADS + branch.merge
     branch.Save()
 
     if cnt_mine > 0 and self.rebase:
       def _dorebase():
-        self._Rebase(upstream = '%s^1' % last_mine, onto = revid)
-        self._CopyFiles()
+        self._Rebase(upstream='%s^1' % last_mine, onto=revid)
+        self._CopyAndLinkFiles()
       syncbuf.later2(self, _dorebase)
     elif local_changes:
       try:
         self._ResetHard(revid)
-        self._CopyFiles()
+        self._CopyAndLinkFiles()
       except GitError as e:
         syncbuf.fail(self, e)
         return
@@ -1301,6 +1396,13 @@
     abssrc = os.path.join(self.worktree, src)
     self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
 
+  def AddLinkFile(self, src, dest, absdest):
+    # dest should already be an absolute path, but src is project relative
+    # make src relative path to dest
+    absdestdir = os.path.dirname(absdest)
+    relsrc = os.path.relpath(os.path.join(self.worktree, src), absdestdir)
+    self.linkfiles.append(_LinkFile(self.worktree, src, dest, relsrc, absdest))
+
   def AddAnnotation(self, name, value, keep):
     self.annotations.append(_Annotation(name, value, keep))
 
@@ -1331,15 +1433,17 @@
       return True
 
     all_refs = self.bare_ref.all
-    if (R_HEADS + name) in all_refs:
+    if R_HEADS + name in all_refs:
       return GitCommand(self,
                         ['checkout', name, '--'],
-                        capture_stdout = True,
-                        capture_stderr = True).Wait() == 0
+                        capture_stdout=True,
+                        capture_stderr=True).Wait() == 0
 
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
     branch.merge = self.revisionExpr
+    if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr):
+      branch.merge = R_HEADS + self.revisionExpr
     revid = self.GetRevisionId(all_refs)
 
     if head.startswith(R_HEADS):
@@ -1362,8 +1466,8 @@
 
     if GitCommand(self,
                   ['checkout', '-b', branch.name, revid],
-                  capture_stdout = True,
-                  capture_stderr = True).Wait() == 0:
+                  capture_stdout=True,
+                  capture_stderr=True).Wait() == 0:
       branch.Save()
       return True
     return False
@@ -1409,8 +1513,8 @@
 
     return GitCommand(self,
                       ['checkout', name, '--'],
-                      capture_stdout = True,
-                      capture_stderr = True).Wait() == 0
+                      capture_stdout=True,
+                      capture_stderr=True).Wait() == 0
 
   def AbandonBranch(self, name):
     """Destroy a local topic branch.
@@ -1444,8 +1548,8 @@
 
     return GitCommand(self,
                       ['branch', '-D', name],
-                      capture_stdout = True,
-                      capture_stderr = True).Wait() == 0
+                      capture_stdout=True,
+                      capture_stderr=True).Wait() == 0
 
   def PruneHeads(self):
     """Prune any topic branches already merged into upstream.
@@ -1462,7 +1566,7 @@
     rev = self.GetRevisionId(left)
     if cb is not None \
        and not self._revlist(HEAD + '...' + rev) \
-       and not self.IsDirty(consider_untracked = False):
+       and not self.IsDirty(consider_untracked=False):
       self.work_git.DetachHead(HEAD)
       kill.append(cb)
 
@@ -1495,7 +1599,7 @@
 
     kept = []
     for branch in kill:
-      if (R_HEADS + branch) in left:
+      if R_HEADS + branch in left:
         branch = self.GetBranch(branch)
         base = branch.LocalMerge
         if not base:
@@ -1545,8 +1649,8 @@
     def parse_gitmodules(gitdir, rev):
       cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
       try:
-        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
-                       bare = True, gitdir = gitdir)
+        p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
+                       bare=True, gitdir=gitdir)
       except GitError:
         return [], []
       if p.Wait() != 0:
@@ -1558,8 +1662,8 @@
         os.write(fd, p.stdout)
         os.close(fd)
         cmd = ['config', '--file', temp_gitmodules_path, '--list']
-        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
-                       bare = True, gitdir = gitdir)
+        p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
+                       bare=True, gitdir=gitdir)
         if p.Wait() != 0:
           return [], []
         gitmodules_lines = p.stdout.split('\n')
@@ -1592,8 +1696,8 @@
       cmd = ['ls-tree', rev, '--']
       cmd.extend(paths)
       try:
-        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
-                       bare = True, gitdir = gitdir)
+        p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
+                       bare=True, gitdir=gitdir)
       except GitError:
         return []
       if p.Wait() != 0:
@@ -1628,23 +1732,24 @@
         continue
 
       remote = RemoteSpec(self.remote.name,
-                          url = url,
-                          review = self.remote.review)
-      subproject = Project(manifest = self.manifest,
-                           name = name,
-                           remote = remote,
-                           gitdir = gitdir,
-                           objdir = objdir,
-                           worktree = worktree,
-                           relpath = relpath,
-                           revisionExpr = self.revisionExpr,
-                           revisionId = rev,
-                           rebase = self.rebase,
-                           groups = self.groups,
-                           sync_c = self.sync_c,
-                           sync_s = self.sync_s,
-                           parent = self,
-                           is_derived = True)
+                          url=url,
+                          review=self.remote.review,
+                          revision=self.remote.revision)
+      subproject = Project(manifest=self.manifest,
+                           name=name,
+                           remote=remote,
+                           gitdir=gitdir,
+                           objdir=objdir,
+                           worktree=worktree,
+                           relpath=relpath,
+                           revisionExpr=self.revisionExpr,
+                           revisionId=rev,
+                           rebase=self.rebase,
+                           groups=self.groups,
+                           sync_c=self.sync_c,
+                           sync_s=self.sync_s,
+                           parent=self,
+                           is_derived=True)
       result.append(subproject)
       result.extend(subproject.GetDerivedSubprojects())
     return result
@@ -1674,6 +1779,7 @@
     if command.Wait() != 0:
       raise GitError('git archive %s: %s' % (self.name, command.stderr))
 
+
   def _RemoteFetch(self, name=None,
                    current_branch_only=False,
                    initial=False,
@@ -1683,26 +1789,43 @@
 
     is_sha1 = False
     tag_name = None
+    depth = None
 
-    if self.clone_depth:
-      depth = self.clone_depth
-    else:
-      depth = self.manifest.manifestProject.config.GetString('repo.depth')
+    # The depth should not be used when fetching to a mirror because
+    # it will result in a shallow repository that cannot be cloned or
+    # fetched from.
+    if not self.manifest.IsMirror:
+      if self.clone_depth:
+        depth = self.clone_depth
+      else:
+        depth = self.manifest.manifestProject.config.GetString('repo.depth')
+      # The repo project should never be synced with partial depth
+      if self.relpath == '.repo/repo':
+        depth = None
+
     if depth:
       current_branch_only = True
 
+    if ID_RE.match(self.revisionExpr) is not None:
+      is_sha1 = True
+
     if current_branch_only:
-      if ID_RE.match(self.revisionExpr) is not None:
-        is_sha1 = True
-      elif self.revisionExpr.startswith(R_TAGS):
+      if self.revisionExpr.startswith(R_TAGS):
         # this is a tag and its sha1 value should never change
         tag_name = self.revisionExpr[len(R_TAGS):]
 
       if is_sha1 or tag_name is not None:
         if self._CheckForSha1():
           return True
-      if is_sha1 and (not self.upstream or ID_RE.match(self.upstream)):
-        current_branch_only = False
+      if is_sha1 and not depth:
+        # When syncing a specific commit and --depth is not set:
+        # * if upstream is explicitly specified and is not a sha1, fetch only
+        #   upstream as users expect only upstream to be fetch.
+        #   Note: The commit might not be in upstream in which case the sync
+        #   will fail.
+        # * otherwise, fetch all branches to make sure we end up with the
+        #   specific commit.
+        current_branch_only = self.upstream and not ID_RE.match(self.upstream)
 
     if not name:
       name = self.remote.name
@@ -1752,9 +1875,7 @@
 
     cmd = ['fetch']
 
-    # The --depth option only affects the initial fetch; after that we'll do
-    # full fetches of changes.
-    if depth and initial:
+    if depth:
       cmd.append('--depth=%s' % depth)
 
     if quiet:
@@ -1763,46 +1884,74 @@
       cmd.append('--update-head-ok')
     cmd.append(name)
 
+    # If using depth then we should not get all the tags since they may
+    # be outside of the depth.
+    if no_tags or depth:
+      cmd.append('--no-tags')
+    else:
+      cmd.append('--tags')
+
+    spec = []
     if not current_branch_only:
       # Fetch whole repo
-      # If using depth then we should not get all the tags since they may
-      # be outside of the depth.
-      if no_tags or depth:
-        cmd.append('--no-tags')
-      else:
-        cmd.append('--tags')
-
-      cmd.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
+      spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
     elif tag_name is not None:
-      cmd.append('tag')
-      cmd.append(tag_name)
-    else:
+      spec.append('tag')
+      spec.append(tag_name)
+
+    if not self.manifest.IsMirror:
       branch = self.revisionExpr
-      if is_sha1:
-        branch = self.upstream
-      if branch.startswith(R_HEADS):
-        branch = branch[len(R_HEADS):]
-      cmd.append(str((u'+refs/heads/%s:' % branch) + remote.ToLocal('refs/heads/%s' % branch)))
+      if is_sha1 and depth and git_require((1, 8, 3)):
+        # Shallow checkout of a specific commit, fetch from that commit and not
+        # the heads only as the commit might be deeper in the history.
+        spec.append(branch)
+      else:
+        if is_sha1:
+          branch = self.upstream
+        if branch is not None and branch.strip():
+          if not branch.startswith('refs/'):
+            branch = R_HEADS + branch
+          spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
+    cmd.extend(spec)
+
+    shallowfetch = self.config.GetString('repo.shallowfetch')
+    if shallowfetch and shallowfetch != ' '.join(spec):
+      GitCommand(self, ['fetch', '--depth=2147483647', name]
+                 + shallowfetch.split(),
+                 bare=True, ssh_proxy=ssh_proxy).Wait()
+    if depth:
+      self.config.SetString('repo.shallowfetch', ' '.join(spec))
+    else:
+      self.config.SetString('repo.shallowfetch', None)
 
     ok = False
     for _i in range(2):
-      ret = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy).Wait()
+      gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy)
+      ret = gitcmd.Wait()
       if ret == 0:
         ok = True
         break
+      # If needed, run the 'git remote prune' the first time through the loop
+      elif (not _i and
+            "error:" in gitcmd.stderr and
+            "git remote prune" in gitcmd.stderr):
+        prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
+                              ssh_proxy=ssh_proxy)
+        ret = prunecmd.Wait()
+        if ret:
+          break
+        continue
       elif current_branch_only and is_sha1 and ret == 128:
         # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1
         # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
         # abort the optimization attempt and do a full sync.
         break
+      elif ret < 0:
+        # Git died with a signal, exit immediately
+        break
       time.sleep(random.randint(30, 45))
 
     if initial:
-      # Ensure that some refs exist.  Otherwise, we probably aren't looking
-      # at a real git repository and may have a bad url.
-      if not self.bare_ref.all:
-          ok = False
-
       if alt_dir:
         if old_packed != '':
           _lwrite(packed_refs, old_packed)
@@ -1815,8 +1964,15 @@
       # got what we wanted, else trigger a second run of all
       # refs.
       if not self._CheckForSha1():
-        return self._RemoteFetch(name=name, current_branch_only=False,
-                                 initial=False, quiet=quiet, alt_dir=alt_dir)
+        if not depth:
+          # Avoid infinite recursion when depth is True (since depth implies
+          # current_branch_only)
+          return self._RemoteFetch(name=name, current_branch_only=False,
+                                   initial=False, quiet=quiet, alt_dir=alt_dir)
+        if self.clone_depth:
+          self.clone_depth = None
+          return self._RemoteFetch(name=name, current_branch_only=current_branch_only,
+                                   initial=False, quiet=quiet, alt_dir=alt_dir)
 
     return ok
 
@@ -1877,34 +2033,34 @@
         os.remove(tmpPath)
     if 'http_proxy' in os.environ and 'darwin' == sys.platform:
       cmd += ['--proxy', os.environ['http_proxy']]
-    cookiefile = self._GetBundleCookieFile(srcUrl)
-    if cookiefile:
-      cmd += ['--cookie', cookiefile]
-    if srcUrl.startswith('persistent-'):
-      srcUrl = srcUrl[len('persistent-'):]
-    cmd += [srcUrl]
+    with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile:
+      if cookiefile:
+        cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
+      if srcUrl.startswith('persistent-'):
+        srcUrl = srcUrl[len('persistent-'):]
+      cmd += [srcUrl]
 
-    if IsTrace():
-      Trace('%s', ' '.join(cmd))
-    try:
-      proc = subprocess.Popen(cmd)
-    except OSError:
-      return False
+      if IsTrace():
+        Trace('%s', ' '.join(cmd))
+      try:
+        proc = subprocess.Popen(cmd)
+      except OSError:
+        return False
 
-    curlret = proc.wait()
+      curlret = proc.wait()
 
-    if curlret == 22:
-      # From curl man page:
-      # 22: HTTP page not retrieved. The requested url was not found or
-      # returned another error with the HTTP error code being 400 or above.
-      # This return code only appears if -f, --fail is used.
-      if not quiet:
-        print("Server does not provide clone.bundle; ignoring.",
-              file=sys.stderr)
-      return False
+      if curlret == 22:
+        # From curl man page:
+        # 22: HTTP page not retrieved. The requested url was not found or
+        # returned another error with the HTTP error code being 400 or above.
+        # This return code only appears if -f, --fail is used.
+        if not quiet:
+          print("Server does not provide clone.bundle; ignoring.",
+                file=sys.stderr)
+        return False
 
     if os.path.exists(tmpPath):
-      if curlret == 0 and self._IsValidBundle(tmpPath):
+      if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
         os.rename(tmpPath, dstPath)
         return True
       else:
@@ -1913,45 +2069,51 @@
     else:
       return False
 
-  def _IsValidBundle(self, path):
+  def _IsValidBundle(self, path, quiet):
     try:
       with open(path) as f:
         if f.read(16) == '# v2 git bundle\n':
           return True
         else:
-          print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
+          if not quiet:
+            print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
           return False
     except OSError:
       return False
 
-  def _GetBundleCookieFile(self, url):
+  @contextlib.contextmanager
+  def _GetBundleCookieFile(self, url, quiet):
     if url.startswith('persistent-'):
       try:
         p = subprocess.Popen(
             ['git-remote-persistent-https', '-print_config', url],
             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.PIPE)
-        p.stdin.close()  # Tell subprocess it's ok to close.
-        prefix = 'http.cookiefile='
-        cookiefile = None
-        for line in p.stdout:
-          line = line.strip()
-          if line.startswith(prefix):
-            cookiefile = line[len(prefix):]
-            break
-        if p.wait():
-          err_msg = p.stderr.read()
-          if ' -print_config' in err_msg:
-            pass  # Persistent proxy doesn't support -print_config.
-          else:
-            print(err_msg, file=sys.stderr)
-        if cookiefile:
-          return cookiefile
+        try:
+          prefix = 'http.cookiefile='
+          cookiefile = None
+          for line in p.stdout:
+            line = line.strip()
+            if line.startswith(prefix):
+              cookiefile = line[len(prefix):]
+              break
+          # Leave subprocess open, as cookie file may be transient.
+          if cookiefile:
+            yield cookiefile
+            return
+        finally:
+          p.stdin.close()
+          if p.wait():
+            err_msg = p.stderr.read()
+            if ' -print_config' in err_msg:
+              pass  # Persistent proxy doesn't support -print_config.
+            elif not quiet:
+              print(err_msg, file=sys.stderr)
       except OSError as e:
         if e.errno == errno.ENOENT:
           pass  # No persistent proxy.
         raise
-    return GitConfig.ForUser().GetString('http.cookiefile')
+    yield GitConfig.ForUser().GetString('http.cookiefile')
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -1963,7 +2125,7 @@
       if self._allrefs:
         raise GitError('%s checkout %s ' % (self.name, rev))
 
-  def _CherryPick(self, rev, quiet=False):
+  def _CherryPick(self, rev):
     cmd = ['cherry-pick']
     cmd.append(rev)
     cmd.append('--')
@@ -1971,7 +2133,7 @@
       if self._allrefs:
         raise GitError('%s cherry-pick %s ' % (self.name, rev))
 
-  def _Revert(self, rev, quiet=False):
+  def _Revert(self, rev):
     cmd = ['revert']
     cmd.append('--no-edit')
     cmd.append(rev)
@@ -1988,7 +2150,7 @@
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s reset --hard %s ' % (self.name, rev))
 
-  def _Rebase(self, upstream, onto = None):
+  def _Rebase(self, upstream, onto=None):
     cmd = ['rebase']
     if onto is not None:
       cmd.extend(['--onto', onto])
@@ -2003,64 +2165,80 @@
     if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s merge %s ' % (self.name, head))
 
-  def _InitGitDir(self, mirror_git=None):
-    if not os.path.exists(self.gitdir):
-
+  def _InitGitDir(self, mirror_git=None, force_sync=False):
+    init_git_dir = not os.path.exists(self.gitdir)
+    init_obj_dir = not os.path.exists(self.objdir)
+    try:
       # Initialize the bare repository, which contains all of the objects.
-      if not os.path.exists(self.objdir):
+      if init_obj_dir:
         os.makedirs(self.objdir)
         self.bare_objdir.init()
 
       # If we have a separate directory to hold refs, initialize it as well.
       if self.objdir != self.gitdir:
-        os.makedirs(self.gitdir)
-        self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
-                              copy_all=True)
+        if init_git_dir:
+          os.makedirs(self.gitdir)
 
-      mp = self.manifest.manifestProject
-      ref_dir = mp.config.GetString('repo.reference') or ''
+        if init_obj_dir or init_git_dir:
+          self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
+                                copy_all=True)
+        try:
+          self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
+        except GitError as e:
+          print("Retrying clone after deleting %s" % force_sync, file=sys.stderr)
+          if force_sync:
+            try:
+              shutil.rmtree(os.path.realpath(self.gitdir))
+              if self.worktree and os.path.exists(
+                  os.path.realpath(self.worktree)):
+                shutil.rmtree(os.path.realpath(self.worktree))
+              return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
+            except:
+              raise e
+          raise e
 
-      if ref_dir or mirror_git:
-        if not mirror_git:
-          mirror_git = os.path.join(ref_dir, self.name + '.git')
-        repo_git = os.path.join(ref_dir, '.repo', 'projects',
-                                self.relpath + '.git')
+      if init_git_dir:
+        mp = self.manifest.manifestProject
+        ref_dir = mp.config.GetString('repo.reference') or ''
 
-        if os.path.exists(mirror_git):
-          ref_dir = mirror_git
+        if ref_dir or mirror_git:
+          if not mirror_git:
+            mirror_git = os.path.join(ref_dir, self.name + '.git')
+          repo_git = os.path.join(ref_dir, '.repo', 'projects',
+                                  self.relpath + '.git')
 
-        elif os.path.exists(repo_git):
-          ref_dir = repo_git
+          if os.path.exists(mirror_git):
+            ref_dir = mirror_git
 
+          elif os.path.exists(repo_git):
+            ref_dir = repo_git
+
+          else:
+            ref_dir = None
+
+          if ref_dir:
+            _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
+                    os.path.join(ref_dir, 'objects') + '\n')
+
+        self._UpdateHooks()
+
+        m = self.manifest.manifestProject.config
+        for key in ['user.name', 'user.email']:
+          if m.Has(key, include_defaults=False):
+            self.config.SetString(key, m.GetString(key))
+        if self.manifest.IsMirror:
+          self.config.SetString('core.bare', 'true')
         else:
-          ref_dir = None
-
-        if ref_dir:
-          _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
-                  os.path.join(ref_dir, 'objects') + '\n')
-
-      self._UpdateHooks()
-
-      m = self.manifest.manifestProject.config
-      for key in ['user.name', 'user.email']:
-        if m.Has(key, include_defaults = False):
-          self.config.SetString(key, m.GetString(key))
-      if self.manifest.IsMirror:
-        self.config.SetString('core.bare', 'true')
-      else:
-        self.config.SetString('core.bare', None)
+          self.config.SetString('core.bare', None)
+    except Exception:
+      if init_obj_dir and os.path.exists(self.objdir):
+        shutil.rmtree(self.objdir)
+      if init_git_dir and os.path.exists(self.gitdir):
+        shutil.rmtree(self.gitdir)
+      raise
 
   def _UpdateHooks(self):
     if os.path.exists(self.gitdir):
-      # Always recreate hooks since they can have been changed
-      # since the latest update.
-      hooks = self._gitdir_path('hooks')
-      try:
-        to_rm = os.listdir(hooks)
-      except OSError:
-        to_rm = []
-      for old_hook in to_rm:
-        os.remove(os.path.join(hooks, old_hook))
       self._InitHooks()
 
   def _InitHooks(self):
@@ -2123,7 +2301,7 @@
       if cur != '' or self.bare_ref.get(ref) != self.revisionId:
         msg = 'manifest set to %s' % self.revisionId
         dst = self.revisionId + '^0'
-        self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
+        self.bare_git.UpdateRef(ref, dst, message=msg, detach=True)
     else:
       remote = self.GetRemote(self.remote.name)
       dst = remote.ToLocal(self.revisionExpr)
@@ -2131,6 +2309,22 @@
         msg = 'manifest set to %s' % self.revisionExpr
         self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
+  def _CheckDirReference(self, srcdir, destdir, share_refs):
+    symlink_files = self.shareable_files
+    symlink_dirs = self.shareable_dirs
+    if share_refs:
+      symlink_files += self.working_tree_files
+      symlink_dirs += self.working_tree_dirs
+    to_symlink = symlink_files + symlink_dirs
+    for name in set(to_symlink):
+      dst = os.path.realpath(os.path.join(destdir, name))
+      if os.path.lexists(dst):
+        src = os.path.realpath(os.path.join(srcdir, name))
+        # Fail if the links are pointing to the wrong place
+        if src != dst:
+          raise GitError('--force-sync not enabled; cannot overwrite a local '
+                         'work tree')
+
   def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
     """Update |dotgit| to reference |gitdir|, using symlinks where possible.
 
@@ -2142,13 +2336,11 @@
       copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
           This saves you the effort of initializing |dotgit| yourself.
     """
-    # These objects can be shared between several working trees.
-    symlink_files = ['description', 'info']
-    symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
+    symlink_files = self.shareable_files
+    symlink_dirs = self.shareable_dirs
     if share_refs:
-      # These objects can only be used by a single working tree.
-      symlink_files += ['config', 'packed-refs']
-      symlink_dirs += ['logs', 'refs']
+      symlink_files += self.working_tree_files
+      symlink_dirs += self.working_tree_dirs
     to_symlink = symlink_files + symlink_dirs
 
     to_copy = []
@@ -2160,13 +2352,21 @@
         src = os.path.realpath(os.path.join(gitdir, name))
         dst = os.path.realpath(os.path.join(dotgit, name))
 
-        if os.path.lexists(dst) and not os.path.islink(dst):
-          raise GitError('cannot overwrite a local work tree')
+        if os.path.lexists(dst):
+          continue
 
         # If the source dir doesn't exist, create an empty dir.
         if name in symlink_dirs and not os.path.lexists(src):
           os.makedirs(src)
 
+        # If the source file doesn't exist, ensure the destination
+        # file doesn't either.
+        if name in symlink_files and not os.path.lexists(src):
+          try:
+            os.remove(dst)
+          except OSError:
+            pass
+
         if name in to_symlink:
           os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
         elif copy_all and not os.path.islink(dst):
@@ -2176,26 +2376,44 @@
             shutil.copy(src, dst)
       except OSError as e:
         if e.errno == errno.EPERM:
-          raise GitError('filesystem must support symlinks')
+          raise DownloadError('filesystem must support symlinks')
         else:
           raise
 
-  def _InitWorkTree(self):
+  def _InitWorkTree(self, force_sync=False):
     dotgit = os.path.join(self.worktree, '.git')
-    if not os.path.exists(dotgit):
-      os.makedirs(dotgit)
-      self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
-                            copy_all=False)
+    init_dotgit = not os.path.exists(dotgit)
+    try:
+      if init_dotgit:
+        os.makedirs(dotgit)
+        self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
+                              copy_all=False)
 
-      _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
+      try:
+        self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
+      except GitError as e:
+        if force_sync:
+          try:
+            shutil.rmtree(dotgit)
+            return self._InitWorkTree(force_sync=False)
+          except:
+            raise e
+        raise e
 
-      cmd = ['read-tree', '--reset', '-u']
-      cmd.append('-v')
-      cmd.append(HEAD)
-      if GitCommand(self, cmd).Wait() != 0:
-        raise GitError("cannot initialize work tree")
+      if init_dotgit:
+        _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
 
-      self._CopyFiles()
+        cmd = ['read-tree', '--reset', '-u']
+        cmd.append('-v')
+        cmd.append(HEAD)
+        if GitCommand(self, cmd).Wait() != 0:
+          raise GitError("cannot initialize work tree")
+
+        self._CopyAndLinkFiles()
+    except Exception:
+      if init_dotgit:
+        shutil.rmtree(dotgit)
+      raise
 
   def _gitdir_path(self, path):
     return os.path.realpath(os.path.join(self.gitdir, path))
@@ -2259,10 +2477,10 @@
                       '-z',
                       '--others',
                       '--exclude-standard'],
-                     bare = False,
+                     bare=False,
                      gitdir=self._gitdir,
-                     capture_stdout = True,
-                     capture_stderr = True)
+                     capture_stdout=True,
+                     capture_stderr=True)
       if p.Wait() == 0:
         out = p.stdout
         if out:
@@ -2277,9 +2495,9 @@
       p = GitCommand(self._project,
                      cmd,
                      gitdir=self._gitdir,
-                     bare = False,
-                     capture_stdout = True,
-                     capture_stderr = True)
+                     bare=False,
+                     capture_stdout=True,
+                     capture_stderr=True)
       try:
         out = p.process.stdout.read()
         r = {}
@@ -2287,8 +2505,8 @@
           out = iter(out[:-1].split('\0'))  # pylint: disable=W1401
           while out:
             try:
-              info = out.next()
-              path = out.next()
+              info = next(out)
+              path = next(out)
             except StopIteration:
               break
 
@@ -2314,7 +2532,7 @@
             info = _Info(path, *info)
             if info.status in ('R', 'C'):
               info.src_path = info.path
-              info.path = out.next()
+              info.path = next(out)
             r[info.path] = info
         return r
       finally:
@@ -2385,10 +2603,10 @@
       cmdv.extend(args)
       p = GitCommand(self._project,
                      cmdv,
-                     bare = self._bare,
+                     bare=self._bare,
                      gitdir=self._gitdir,
-                     capture_stdout = True,
-                     capture_stderr = True)
+                     capture_stdout=True,
+                     capture_stderr=True)
       r = []
       for line in p.process.stdout:
         if line[-1] == '\n':
@@ -2438,10 +2656,10 @@
         cmdv.extend(args)
         p = GitCommand(self._project,
                        cmdv,
-                       bare = self._bare,
+                       bare=self._bare,
                        gitdir=self._gitdir,
-                       capture_stdout = True,
-                       capture_stderr = True)
+                       capture_stdout=True,
+                       capture_stderr=True)
         if p.Wait() != 0:
           raise GitError('%s %s: %s' % (
                          self._project.name,
@@ -2506,9 +2724,9 @@
 class _SyncColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'reposync')
-    self.project   = self.printer('header', attr = 'bold')
-    self.info      = self.printer('info')
-    self.fail      = self.printer('fail', fg='red')
+    self.project = self.printer('header', attr='bold')
+    self.info = self.printer('info')
+    self.fail = self.printer('fail', fg='red')
 
 class SyncBuffer(object):
   def __init__(self, config, detach_head=False):
@@ -2570,16 +2788,16 @@
   """
   def __init__(self, manifest, name, gitdir, worktree):
     Project.__init__(self,
-                     manifest = manifest,
-                     name = name,
-                     gitdir = gitdir,
-                     objdir = gitdir,
-                     worktree = worktree,
-                     remote = RemoteSpec('origin'),
-                     relpath = '.repo/%s' % name,
-                     revisionExpr = 'refs/heads/master',
-                     revisionId = None,
-                     groups = None)
+                     manifest=manifest,
+                     name=name,
+                     gitdir=gitdir,
+                     objdir=gitdir,
+                     worktree=worktree,
+                     remote=RemoteSpec('origin'),
+                     relpath='.repo/%s' % name,
+                     revisionExpr='refs/heads/master',
+                     revisionId=None,
+                     groups=None)
 
   def PreSync(self):
     if self.Exists:
@@ -2590,20 +2808,20 @@
           self.revisionExpr = base
           self.revisionId = None
 
-  def MetaBranchSwitch(self, target):
+  def MetaBranchSwitch(self):
     """ Prepare MetaProject for manifest branch switch
     """
 
     # detach and delete manifest branch, allowing a new
     # branch to take over
-    syncbuf = SyncBuffer(self.config, detach_head = True)
+    syncbuf = SyncBuffer(self.config, detach_head=True)
     self.Sync_LocalHalf(syncbuf)
     syncbuf.Finish()
 
     return GitCommand(self,
                         ['update-ref', '-d', 'refs/heads/default'],
-                        capture_stdout = True,
-                        capture_stderr = True).Wait() == 0
+                        capture_stdout=True,
+                        capture_stderr=True).Wait() == 0
 
 
   @property
diff --git a/repo b/repo
index 768f11f..f12354a 100755
--- a/repo
+++ b/repo
@@ -114,6 +114,7 @@
 import optparse
 import os
 import re
+import shutil
 import stat
 import subprocess
 import sys
@@ -138,10 +139,6 @@
 
 # Python version check
 ver = sys.version_info
-if ver[0] == 3:
-  _print('warning: Python 3 support is currently experimental. YMMV.\n'
-         'Please use Python 2.6 - 2.7 instead.',
-         file=sys.stderr)
 if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
   _print('error: Python version %s unsupported.\n'
          'Please use Python 2.6 - 2.7 instead.'
@@ -465,7 +462,7 @@
     try:
       r = urllib.request.urlopen(url)
     except urllib.error.HTTPError as e:
-      if e.code in [403, 404]:
+      if e.code in [401, 403, 404]:
         return False
       _print('fatal: Cannot get %s' % url, file=sys.stderr)
       _print('fatal: HTTP error %s' % e.code, file=sys.stderr)
@@ -741,12 +738,7 @@
       try:
         _Init(args)
       except CloneFailure:
-        for root, dirs, files in os.walk(repodir, topdown=False):
-          for name in files:
-            os.remove(os.path.join(root, name))
-          for name in dirs:
-            os.rmdir(os.path.join(root, name))
-        os.rmdir(repodir)
+        shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:
@@ -772,4 +764,8 @@
 
 
 if __name__ == '__main__':
+  if ver[0] == 3:
+    _print('warning: Python 3 support is currently experimental. YMMV.\n'
+           'Please use Python 2.6 - 2.7 instead.',
+           file=sys.stderr)
   main(sys.argv[1:])
diff --git a/subcmds/branches.py b/subcmds/branches.py
index f714c1e..2902684 100644
--- a/subcmds/branches.py
+++ b/subcmds/branches.py
@@ -47,6 +47,10 @@
     return self.current > 0
 
   @property
+  def IsSplitCurrent(self):
+    return self.current != 0 and self.current != len(self.projects)
+
+  @property
   def IsPublished(self):
     return self.published > 0
 
@@ -139,10 +143,14 @@
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
-        if in_cnt < project_cnt - in_cnt:
+        non_cur_paths = []
+        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
-            paths.append(b.project.relpath)
+            if not i.IsSplitCurrent or b.current:
+              paths.append(b.project.relpath)
+            else:
+              non_cur_paths.append(b.project.relpath)
         else:
           fmt = out.notinproject
           in_type = 'not in'
@@ -154,13 +162,19 @@
               paths.append(p.relpath)
 
         s = ' %s %s' % (in_type, ', '.join(paths))
-        if width + 7 + len(s) < 80:
+        if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
+          fmt = out.current if i.IsCurrent else fmt
           fmt(s)
         else:
           fmt(' %s:' % in_type)
+          fmt = out.current if i.IsCurrent else out.write
           for p in paths:
             out.nl()
             fmt(width*' ' + '          %s' % p)
+          fmt = out.write
+          for p in non_cur_paths:
+            out.nl()
+            fmt(width*' ' + '          %s' % p)
       else:
         out.write(' in all projects')
       out.nl()
diff --git a/subcmds/cherry_pick.py b/subcmds/cherry_pick.py
index 520e4c3..1f7dffd 100644
--- a/subcmds/cherry_pick.py
+++ b/subcmds/cherry_pick.py
@@ -76,6 +76,7 @@
                      capture_stdout = True,
                      capture_stderr = True)
       p.stdin.write(new_msg)
+      p.stdin.close()
       if p.Wait() != 0:
         print("error: Failed to update commit message", file=sys.stderr)
         sys.exit(1)
diff --git a/subcmds/download.py b/subcmds/download.py
index 098d8b4..a029462 100644
--- a/subcmds/download.py
+++ b/subcmds/download.py
@@ -93,6 +93,7 @@
         except GitError:
           print('[%s] Could not complete the cherry-pick of %s' \
                 % (project.name, dl.commit), file=sys.stderr)
+          sys.exit(1)
 
       elif opt.revert:
         project._Revert(dl.commit)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index e2a420a..b93cd6d 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -14,10 +14,13 @@
 # limitations under the License.
 
 from __future__ import print_function
+import errno
 import fcntl
+import multiprocessing
 import re
 import os
 import select
+import signal
 import sys
 import subprocess
 
@@ -31,6 +34,7 @@
   'log',
 ]
 
+
 class ForallColoring(Coloring):
   def __init__(self, config):
     Coloring.__init__(self, config, 'forall')
@@ -87,6 +91,12 @@
 REPO_RREV is the name of the revision from the manifest, exactly
 as written in the manifest.
 
+REPO_COUNT is the total number of projects being iterated.
+
+REPO_I is the current (1-based) iteration count. Can be used in
+conjunction with REPO_COUNT to add a simple progress indicator to your
+command.
+
 REPO__* are any extra environment variables, specified by the
 "annotation" element under any project element.  This can be useful
 for differentiating trees based on user-specific criteria, or simply
@@ -126,9 +136,35 @@
     g.add_option('-v', '--verbose',
                  dest='verbose', action='store_true',
                  help='Show command error messages')
+    g.add_option('-j', '--jobs',
+                 dest='jobs', action='store', type='int', default=1,
+                 help='number of commands to execute simultaneously')
 
   def WantPager(self, opt):
-    return opt.project_header
+    return opt.project_header and opt.jobs == 1
+
+  def _SerializeProject(self, project):
+    """ Serialize a project._GitGetByExec instance.
+
+    project._GitGetByExec is not pickle-able. Instead of trying to pass it
+    around between processes, make a dict ourselves containing only the
+    attributes that we need.
+
+    """
+    if not self.manifest.IsMirror:
+      lrev = project.GetRevisionId()
+    else:
+      lrev = None
+    return {
+      'name': project.name,
+      'relpath': project.relpath,
+      'remote_name': project.remote.name,
+      'lrev': lrev,
+      'rrev': project.revisionExpr,
+      'annotations': dict((a.name, a.value) for a in project.annotations),
+      'gitdir': project.gitdir,
+      'worktree': project.worktree,
+    }
 
   def Execute(self, opt, args):
     if not opt.command:
@@ -167,123 +203,188 @@
       # pylint: enable=W0631
 
     mirror = self.manifest.IsMirror
-    out = ForallColoring(self.manifest.manifestProject.config)
-    out.redirect(sys.stdout)
-
     rc = 0
-    first = True
+
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
+
+    if os.path.isfile(smart_sync_manifest_path):
+      self.manifest.Override(smart_sync_manifest_path)
 
     if not opt.regex:
       projects = self.GetProjects(args)
     else:
       projects = self.FindProjects(args)
 
-    for project in projects:
-      env = os.environ.copy()
-      def setenv(name, val):
-        if val is None:
-          val = ''
-        env[name] = val.encode()
+    os.environ['REPO_COUNT'] = str(len(projects))
 
-      setenv('REPO_PROJECT', project.name)
-      setenv('REPO_PATH', project.relpath)
-      setenv('REPO_REMOTE', project.remote.name)
-      setenv('REPO_LREV', project.GetRevisionId())
-      setenv('REPO_RREV', project.revisionExpr)
-      for a in project.annotations:
-        setenv("REPO__%s" % (a.name), a.value)
-
-      if mirror:
-        setenv('GIT_DIR', project.gitdir)
-        cwd = project.gitdir
-      else:
-        cwd = project.worktree
-
-      if not os.path.exists(cwd):
-        if (opt.project_header and opt.verbose) \
-        or not opt.project_header:
-          print('skipping %s/' % project.relpath, file=sys.stderr)
-        continue
-
-      if opt.project_header:
-        stdin = subprocess.PIPE
-        stdout = subprocess.PIPE
-        stderr = subprocess.PIPE
-      else:
-        stdin = None
-        stdout = None
-        stderr = None
-
-      p = subprocess.Popen(cmd,
-                           cwd = cwd,
-                           shell = shell,
-                           env = env,
-                           stdin = stdin,
-                           stdout = stdout,
-                           stderr = stderr)
-
-      if opt.project_header:
-        class sfd(object):
-          def __init__(self, fd, dest):
-            self.fd = fd
-            self.dest = dest
-          def fileno(self):
-            return self.fd.fileno()
-
-        empty = True
-        errbuf = ''
-
-        p.stdin.close()
-        s_in = [sfd(p.stdout, sys.stdout),
-                sfd(p.stderr, sys.stderr)]
-
-        for s in s_in:
-          flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
-          fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
-        while s_in:
-          in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
-          for s in in_ready:
-            buf = s.fd.read(4096)
-            if not buf:
-              s.fd.close()
-              s_in.remove(s)
-              continue
-
-            if not opt.verbose:
-              if s.fd != p.stdout:
-                errbuf += buf
-                continue
-
-            if empty:
-              if first:
-                first = False
-              else:
-                out.nl()
-
-              if mirror:
-                project_header_path = project.name
-              else:
-                project_header_path = project.relpath
-              out.project('project %s/', project_header_path)
-              out.nl()
-              out.flush()
-              if errbuf:
-                sys.stderr.write(errbuf)
-                sys.stderr.flush()
-                errbuf = ''
-              empty = False
-
-            s.dest.write(buf)
-            s.dest.flush()
-
-      r = p.wait()
-      if r != 0:
-        if r != rc:
-          rc = r
-        if opt.abort_on_errors:
-          print("error: %s: Aborting due to previous error" % project.relpath,
-                file=sys.stderr)
-          sys.exit(r)
+    pool = multiprocessing.Pool(opt.jobs, InitWorker)
+    try:
+      config = self.manifest.manifestProject.config
+      results_it = pool.imap(
+         DoWorkWrapper,
+         self.ProjectArgs(projects, mirror, opt, cmd, shell, config))
+      pool.close()
+      for r in results_it:
+        rc = rc or r
+        if r != 0 and opt.abort_on_errors:
+          raise Exception('Aborting due to previous error')
+    except (KeyboardInterrupt, WorkerKeyboardInterrupt):
+      # Catch KeyboardInterrupt raised inside and outside of workers
+      print('Interrupted - terminating the pool')
+      pool.terminate()
+      rc = rc or errno.EINTR
+    except Exception as e:
+      # Catch any other exceptions raised
+      print('Got an error, terminating the pool: %r' % e,
+            file=sys.stderr)
+      pool.terminate()
+      rc = rc or getattr(e, 'errno', 1)
+    finally:
+      pool.join()
     if rc != 0:
       sys.exit(rc)
+
+  def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
+    for cnt, p in enumerate(projects):
+      try:
+        project = self._SerializeProject(p)
+      except Exception as e:
+        print('Project list error: %r' % e,
+              file=sys.stderr)
+        return
+      except KeyboardInterrupt:
+        print('Project list interrupted',
+              file=sys.stderr)
+        return
+      yield [mirror, opt, cmd, shell, cnt, config, project]
+
+class WorkerKeyboardInterrupt(Exception):
+  """ Keyboard interrupt exception for worker processes. """
+  pass
+
+
+def InitWorker():
+  signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+def DoWorkWrapper(args):
+  """ A wrapper around the DoWork() method.
+
+  Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
+  ``Exception``-based exception to stop it flooding the console with stacktraces
+  and making the parent hang indefinitely.
+
+  """
+  project = args.pop()
+  try:
+    return DoWork(project, *args)
+  except KeyboardInterrupt:
+    print('%s: Worker interrupted' % project['name'])
+    raise WorkerKeyboardInterrupt()
+
+
+def DoWork(project, mirror, opt, cmd, shell, cnt, config):
+  env = os.environ.copy()
+  def setenv(name, val):
+    if val is None:
+      val = ''
+    if hasattr(val, 'encode'):
+      val = val.encode()
+    env[name] = val
+
+  setenv('REPO_PROJECT', project['name'])
+  setenv('REPO_PATH', project['relpath'])
+  setenv('REPO_REMOTE', project['remote_name'])
+  setenv('REPO_LREV', project['lrev'])
+  setenv('REPO_RREV', project['rrev'])
+  setenv('REPO_I', str(cnt + 1))
+  for name in project['annotations']:
+    setenv("REPO__%s" % (name), project['annotations'][name])
+
+  if mirror:
+    setenv('GIT_DIR', project['gitdir'])
+    cwd = project['gitdir']
+  else:
+    cwd = project['worktree']
+
+  if not os.path.exists(cwd):
+    if (opt.project_header and opt.verbose) \
+    or not opt.project_header:
+      print('skipping %s/' % project['relpath'], file=sys.stderr)
+    return
+
+  if opt.project_header:
+    stdin = subprocess.PIPE
+    stdout = subprocess.PIPE
+    stderr = subprocess.PIPE
+  else:
+    stdin = None
+    stdout = None
+    stderr = None
+
+  p = subprocess.Popen(cmd,
+                       cwd=cwd,
+                       shell=shell,
+                       env=env,
+                       stdin=stdin,
+                       stdout=stdout,
+                       stderr=stderr)
+
+  if opt.project_header:
+    out = ForallColoring(config)
+    out.redirect(sys.stdout)
+    class sfd(object):
+      def __init__(self, fd, dest):
+        self.fd = fd
+        self.dest = dest
+      def fileno(self):
+        return self.fd.fileno()
+
+    empty = True
+    errbuf = ''
+
+    p.stdin.close()
+    s_in = [sfd(p.stdout, sys.stdout),
+            sfd(p.stderr, sys.stderr)]
+
+    for s in s_in:
+      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    while s_in:
+      in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
+      for s in in_ready:
+        buf = s.fd.read(4096)
+        if not buf:
+          s.fd.close()
+          s_in.remove(s)
+          continue
+
+        if not opt.verbose:
+          if s.fd != p.stdout:
+            errbuf += buf
+            continue
+
+        if empty and out:
+          if not cnt == 0:
+            out.nl()
+
+          if mirror:
+            project_header_path = project['name']
+          else:
+            project_header_path = project['relpath']
+          out.project('project %s/', project_header_path)
+          out.nl()
+          out.flush()
+          if errbuf:
+            sys.stderr.write(errbuf)
+            sys.stderr.flush()
+            errbuf = ''
+          empty = False
+
+        s.dest.write(buf)
+        s.dest.flush()
+
+  r = p.wait()
+  return r
diff --git a/subcmds/info.py b/subcmds/info.py
index d42860a..ed196e9 100644
--- a/subcmds/info.py
+++ b/subcmds/info.py
@@ -59,7 +59,8 @@
                       or 'all,-notdefault')
 
     self.heading("Manifest branch: ")
-    self.headtext(self.manifest.default.revisionExpr)
+    if self.manifest.default.revisionExpr:
+        self.headtext(self.manifest.default.revisionExpr)
     self.out.nl()
     self.heading("Manifest merge branch: ")
     self.headtext(mergeBranch)
diff --git a/subcmds/init.py b/subcmds/init.py
index b1fcb69..dbb6ddd 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -27,7 +27,7 @@
   import imp
   import urlparse
   urllib = imp.new_module('urllib')
-  urllib.parse = urlparse.urlparse
+  urllib.parse = urlparse
 
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
@@ -153,7 +153,7 @@
       # server where this git is located, so let's save that here.
       mirrored_manifest_git = None
       if opt.reference:
-        manifest_git_path = urllib.parse(opt.manifest_url).path[1:]
+        manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
         mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
         if not mirrored_manifest_git.endswith(".git"):
           mirrored_manifest_git += ".git"
@@ -233,7 +233,7 @@
       sys.exit(1)
 
     if opt.manifest_branch:
-      m.MetaBranchSwitch(opt.manifest_branch)
+      m.MetaBranchSwitch()
 
     syncbuf = SyncBuffer(m.config)
     m.Sync_LocalHalf(syncbuf)
diff --git a/subcmds/start.py b/subcmds/start.py
index 2d723fc..60ad41e 100644
--- a/subcmds/start.py
+++ b/subcmds/start.py
@@ -59,9 +59,13 @@
     for project in all_projects:
       pm.update()
       # If the current revision is a specific SHA1 then we can't push back
-      # to it so substitute the manifest default revision instead.
+      # to it; so substitute with dest_branch if defined, or with manifest
+      # default revision instead.
       if IsId(project.revisionExpr):
-        project.revisionExpr = self.manifest.default.revisionExpr
+        if project.dest_branch:
+          project.revisionExpr = project.dest_branch
+        else:
+          project.revisionExpr = self.manifest.default.revisionExpr
       if not project.StartBranch(nb):
         err.append(project)
     pm.end()
diff --git a/subcmds/status.py b/subcmds/status.py
index 41c4429..38c229b 100644
--- a/subcmds/status.py
+++ b/subcmds/status.py
@@ -22,15 +22,8 @@
 
 import glob
 
-from pyversion import is_python3
-if is_python3():
-  import io
-else:
-  import StringIO as io
-
 import itertools
 import os
-import sys
 
 from color import Coloring
 
@@ -97,7 +90,7 @@
                  dest='orphans', action='store_true',
                  help="include objects in working directory outside of repo projects")
 
-  def _StatusHelper(self, project, clean_counter, sem, output):
+  def _StatusHelper(self, project, clean_counter, sem):
     """Obtains the status for a specific project.
 
     Obtains the status for a project, redirecting the output to
@@ -111,9 +104,9 @@
       output: Where to output the status.
     """
     try:
-      state = project.PrintWorkTreeStatus(output)
+      state = project.PrintWorkTreeStatus()
       if state == 'CLEAN':
-        clean_counter.next()
+        next(clean_counter)
     finally:
       sem.release()
 
@@ -122,16 +115,16 @@
     status_header = ' --\t'
     for item in dirs:
       if not os.path.isdir(item):
-        outstring.write(''.join([status_header, item]))
+        outstring.append(''.join([status_header, item]))
         continue
       if item in proj_dirs:
         continue
       if item in proj_dirs_parents:
-        self._FindOrphans(glob.glob('%s/.*' % item) + \
-            glob.glob('%s/*' % item), \
+        self._FindOrphans(glob.glob('%s/.*' % item) +
+            glob.glob('%s/*' % item),
             proj_dirs, proj_dirs_parents, outstring)
         continue
-      outstring.write(''.join([status_header, item, '/']))
+      outstring.append(''.join([status_header, item, '/']))
 
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
@@ -141,30 +134,21 @@
       for project in all_projects:
         state = project.PrintWorkTreeStatus()
         if state == 'CLEAN':
-          counter.next()
+          next(counter)
     else:
       sem = _threading.Semaphore(opt.jobs)
-      threads_and_output = []
+      threads = []
       for project in all_projects:
         sem.acquire()
 
-        class BufList(io.StringIO):
-          def dump(self, ostream):
-            for entry in self.buflist:
-              ostream.write(entry)
-
-        output = BufList()
-
         t = _threading.Thread(target=self._StatusHelper,
-                              args=(project, counter, sem, output))
-        threads_and_output.append((t, output))
+                              args=(project, counter, sem))
+        threads.append(t)
         t.daemon = True
         t.start()
-      for (t, output) in threads_and_output:
+      for t in threads:
         t.join()
-        output.dump(sys.stdout)
-        output.close()
-    if len(all_projects) == counter.next():
+    if len(all_projects) == next(counter):
       print('nothing to commit (working directory clean)')
 
     if opt.orphans:
@@ -188,23 +172,21 @@
       try:
         os.chdir(self.manifest.topdir)
 
-        outstring = io.StringIO()
-        self._FindOrphans(glob.glob('.*') + \
-            glob.glob('*'), \
+        outstring = []
+        self._FindOrphans(glob.glob('.*') +
+            glob.glob('*'),
             proj_dirs, proj_dirs_parents, outstring)
 
-        if outstring.buflist:
+        if outstring:
           output = StatusColoring(self.manifest.globalConfig)
           output.project('Objects not within a project (orphans)')
           output.nl()
-          for entry in outstring.buflist:
+          for entry in outstring:
             output.untracked(entry)
             output.nl()
         else:
           print('No orphan files or directories')
 
-        outstring.close()
-
       finally:
         # Restore CWD.
         os.chdir(orig_path)
diff --git a/subcmds/sync.py b/subcmds/sync.py
index b50df09..43d450b 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -14,10 +14,10 @@
 # limitations under the License.
 
 from __future__ import print_function
+import json
 import netrc
 from optparse import SUPPRESS_HELP
 import os
-import pickle
 import re
 import shutil
 import socket
@@ -119,6 +119,11 @@
 The -f/--force-broken option can be used to proceed with syncing
 other projects if a project sync fails.
 
+The --force-sync option can be used to overwrite existing git
+directories if they have previously been linked to a different
+object direcotry. WARNING: This may cause data to be lost since
+refs may be removed when overwriting.
+
 The --no-clone-bundle option disables any attempt to use
 $URL/clone.bundle to bootstrap a new Git repository from a
 resumeable bundle file on a content delivery network. This
@@ -128,6 +133,13 @@
 The --fetch-submodules option enables fetching Git submodules
 of a project from server.
 
+The -c/--current-branch option can be used to only fetch objects that
+are on the branch specified by a project's revision.
+
+The --optimized-fetch option can be used to only fetch projects that
+are fixed to a sha1 revision if the sha1 revision does not already
+exist locally.
+
 SSH Connections
 ---------------
 
@@ -167,6 +179,11 @@
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help="continue sync even if a project fails to sync")
+    p.add_option('--force-sync',
+                 dest='force_sync', action='store_true',
+                 help="overwrite an existing git directory if it needs to "
+                      "point to a different object directory. WARNING: this "
+                      "may cause loss of data")
     p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
@@ -203,6 +220,9 @@
     p.add_option('--no-tags',
                  dest='no_tags', action='store_true',
                  help="don't fetch tags")
+    p.add_option('--optimized-fetch',
+                 dest='optimized_fetch', action='store_true',
+                 help='only fetch projects fixed to sha1 if revision does not exist locally')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -271,8 +291,10 @@
         success = project.Sync_NetworkHalf(
           quiet=opt.quiet,
           current_branch_only=opt.current_branch_only,
+          force_sync=opt.force_sync,
           clone_bundle=not opt.no_clone_bundle,
-          no_tags=opt.no_tags, archive=self.manifest.IsArchive)
+          no_tags=opt.no_tags, archive=self.manifest.IsArchive,
+          optimized_fetch=opt.optimized_fetch)
         self._fetch_times.Set(project, time.time() - start)
 
         # Lock around all the rest of the code, since printing, updating a set
@@ -508,6 +530,9 @@
       self.manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
 
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
@@ -560,7 +585,10 @@
             branch = branch[len(R_HEADS):]
 
           env = os.environ.copy()
-          if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
+          if 'SYNC_TARGET' in env:
+            target = env['SYNC_TARGET']
+            [success, manifest_str] = server.GetApprovedManifest(branch, target)
+          elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
             target = '%s-%s' % (env['TARGET_PRODUCT'],
                                 env['TARGET_BUILD_VARIANT'])
             [success, manifest_str] = server.GetApprovedManifest(branch, target)
@@ -571,17 +599,16 @@
           [success, manifest_str] = server.GetManifest(opt.smart_tag)
 
         if success:
-          manifest_name = "smart_sync_override.xml"
-          manifest_path = os.path.join(self.manifest.manifestProject.worktree,
-                                       manifest_name)
+          manifest_name = smart_sync_manifest_name
           try:
-            f = open(manifest_path, 'w')
+            f = open(smart_sync_manifest_path, 'w')
             try:
               f.write(manifest_str)
             finally:
               f.close()
-          except IOError:
-            print('error: cannot write manifest to %s' % manifest_path,
+          except IOError as e:
+            print('error: cannot write manifest to %s:\n%s'
+                  % (smart_sync_manifest_path, e),
                   file=sys.stderr)
             sys.exit(1)
           self._ReloadManifest(manifest_name)
@@ -598,6 +625,13 @@
               % (self.manifest.manifest_server, e.errcode, e.errmsg),
               file=sys.stderr)
         sys.exit(1)
+    else:  # Not smart sync or smart tag mode
+      if os.path.isfile(smart_sync_manifest_path):
+        try:
+          os.remove(smart_sync_manifest_path)
+        except OSError as e:
+          print('error: failed to remove existing smart sync override manifest: %s' %
+                e, file=sys.stderr)
 
     rp = self.manifest.repoProject
     rp.PreSync()
@@ -611,7 +645,8 @@
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
                           current_branch_only=opt.current_branch_only,
-                          no_tags=opt.no_tags)
+                          no_tags=opt.no_tags,
+                          optimized_fetch=opt.optimized_fetch)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
@@ -674,7 +709,7 @@
     for project in all_projects:
       pm.update()
       if project.worktree:
-        project.Sync_LocalHalf(syncbuf)
+        project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
     pm.end()
     print(file=sys.stderr)
     if not syncbuf.Finish():
@@ -762,7 +797,7 @@
   _ALPHA = 0.5
 
   def __init__(self, manifest):
-    self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
+    self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
     self._times = None
     self._seen = set()
 
@@ -781,22 +816,17 @@
   def _Load(self):
     if self._times is None:
       try:
-        f = open(self._path, 'rb')
-      except IOError:
-        self._times = {}
-        return self._times
-      try:
+        f = open(self._path)
         try:
-          self._times = pickle.load(f)
-        except IOError:
-          try:
-            os.remove(self._path)
-          except OSError:
-            pass
-          self._times = {}
-      finally:
-        f.close()
-    return self._times
+          self._times = json.load(f)
+        finally:
+          f.close()
+      except (IOError, ValueError):
+        try:
+          os.remove(self._path)
+        except OSError:
+          pass
+        self._times = {}
 
   def Save(self):
     if self._times is None:
@@ -810,13 +840,13 @@
       del self._times[name]
 
     try:
-      f = open(self._path, 'wb')
+      f = open(self._path, 'w')
       try:
-        pickle.dump(self._times, f)
-      except (IOError, OSError, pickle.PickleError):
-        try:
-          os.remove(self._path)
-        except OSError:
-          pass
-    finally:
-      f.close()
+        json.dump(self._times, f, indent=2)
+      finally:
+        f.close()
+    except (IOError, TypeError):
+      try:
+        os.remove(self._path)
+      except OSError:
+        pass
diff --git a/subcmds/upload.py b/subcmds/upload.py
index e2fa261..674fc17 100644
--- a/subcmds/upload.py
+++ b/subcmds/upload.py
@@ -25,10 +25,12 @@
 from project import RepoHook
 
 from pyversion import is_python3
+# pylint:disable=W0622
 if not is_python3():
-  # pylint:disable=W0622
   input = raw_input
-  # pylint:enable=W0622
+else:
+  unicode = str
+# pylint:enable=W0622
 
 UNUSUAL_COMMIT_THRESHOLD = 5
 
@@ -337,13 +339,17 @@
         self._AppendAutoList(branch, people)
 
         # Check if there are local changes that may have been forgotten
-        if branch.project.HasChanges():
+        changes = branch.project.UncommitedFiles()
+        if changes:
           key = 'review.%s.autoupload' % branch.project.remote.review
           answer = branch.project.config.GetBoolean(key)
 
           # if they want to auto upload, let's not ask because it could be automated
           if answer is None:
-            sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/N) ')
+            sys.stdout.write('Uncommitted changes in ' + branch.project.name)
+            sys.stdout.write(' (did you forget to amend?):\n')
+            sys.stdout.write('\n'.join(changes) + '\n')
+            sys.stdout.write('Continue uploading? (y/N) ')
             a = sys.stdin.readline().strip().lower()
             if a not in ('y', 'yes', 't', 'true', 'on'):
               print("skipping upload", file=sys.stderr)