blob: 019f29d4b44e4112f19c61a421c5182ec2d34daa [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Sarah Owenscecd1d82012-11-01 22:59:27 -070015from __future__ import print_function
Shawn O. Pearce438ee1c2008-11-03 09:59:36 -080016import errno
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import filecmp
Wink Saville4c426ef2015-06-03 08:05:17 -070018import glob
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070020import random
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
22import shutil
23import stat
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Julien Campergue335f5ef2013-10-16 11:02:35 +020026import tarfile
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +080027import tempfile
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070028import time
Dave Borowitz137d0132015-01-02 11:12:54 -080029import traceback
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -070030
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070031from color import Coloring
Dave Borowitzb42b4742012-10-31 12:27:27 -070032from git_command import GitCommand, git_require
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070033from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
34 ID_RE
Kevin Degiabaa7f32014-11-12 11:27:45 -070035from error import GitError, HookError, UploadError, DownloadError
Shawn O. Pearce559b8462009-03-02 12:56:08 -080036from error import ManifestInvalidRevisionError
Conley Owens75ee0572012-11-15 17:33:11 -080037from error import NoManifestException
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070038from trace import IsTrace, Trace
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070039
Shawn O. Pearced237b692009-04-17 18:49:50 -070040from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070041
David Pursehouse59bbb582013-05-17 10:49:33 +090042from pyversion import is_python3
Mike Frysinger40252c22016-08-15 21:23:44 -040043if is_python3():
44 import urllib.parse
45else:
46 import imp
47 import urlparse
48 urllib = imp.new_module('urllib')
49 urllib.parse = urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090050 # pylint:disable=W0622
Chirayu Desai217ea7d2013-03-01 19:14:38 +053051 input = raw_input
David Pursehouse59bbb582013-05-17 10:49:33 +090052 # pylint:enable=W0622
Chirayu Desai217ea7d2013-03-01 19:14:38 +053053
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070054
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070055def _lwrite(path, content):
56 lock = '%s.lock' % path
57
Chirayu Desai303a82f2014-08-19 22:57:17 +053058 fd = open(lock, 'w')
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070059 try:
60 fd.write(content)
61 finally:
62 fd.close()
63
64 try:
65 os.rename(lock, path)
66 except OSError:
67 os.remove(lock)
68 raise
69
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070070
Shawn O. Pearce48244782009-04-16 08:25:57 -070071def _error(fmt, *args):
72 msg = fmt % args
Sarah Owenscecd1d82012-11-01 22:59:27 -070073 print('error: %s' % msg, file=sys.stderr)
Shawn O. Pearce48244782009-04-16 08:25:57 -070074
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070075
David Pursehousef33929d2015-08-24 14:39:14 +090076def _warn(fmt, *args):
77 msg = fmt % args
78 print('warn: %s' % msg, file=sys.stderr)
79
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070080
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070081def not_rev(r):
82 return '^' + r
83
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070084
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080085def sq(r):
86 return "'" + r.replace("'", "'\''") + "'"
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -080087
Jonathan Nieder93719792015-03-17 11:29:58 -070088_project_hook_list = None
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070089
90
Jonathan Nieder93719792015-03-17 11:29:58 -070091def _ProjectHooks():
92 """List the hooks present in the 'hooks' directory.
93
94 These hooks are project hooks and are copied to the '.git/hooks' directory
95 of all subprojects.
96
97 This function caches the list of hooks (based on the contents of the
98 'repo/hooks' directory) on the first call.
99
100 Returns:
101 A list of absolute paths to all of the files in the hooks directory.
102 """
103 global _project_hook_list
104 if _project_hook_list is None:
105 d = os.path.realpath(os.path.abspath(os.path.dirname(__file__)))
106 d = os.path.join(d, 'hooks')
107 _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
108 return _project_hook_list
109
110
Shawn O. Pearce632768b2008-10-23 11:58:52 -0700111class DownloadedChange(object):
112 _commit_cache = None
113
114 def __init__(self, project, base, change_id, ps_id, commit):
115 self.project = project
116 self.base = base
117 self.change_id = change_id
118 self.ps_id = ps_id
119 self.commit = commit
120
121 @property
122 def commits(self):
123 if self._commit_cache is None:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700124 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
125 '--abbrev-commit',
126 '--pretty=oneline',
127 '--reverse',
128 '--date-order',
129 not_rev(self.base),
130 self.commit,
131 '--')
Shawn O. Pearce632768b2008-10-23 11:58:52 -0700132 return self._commit_cache
133
134
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700135class ReviewableBranch(object):
136 _commit_cache = None
137
138 def __init__(self, project, branch, base):
139 self.project = project
140 self.branch = branch
141 self.base = base
142
143 @property
144 def name(self):
145 return self.branch.name
146
147 @property
148 def commits(self):
149 if self._commit_cache is None:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700150 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
151 '--abbrev-commit',
152 '--pretty=oneline',
153 '--reverse',
154 '--date-order',
155 not_rev(self.base),
156 R_HEADS + self.name,
157 '--')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700158 return self._commit_cache
159
160 @property
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800161 def unabbrev_commits(self):
162 r = dict()
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700163 for commit in self.project.bare_git.rev_list(not_rev(self.base),
164 R_HEADS + self.name,
165 '--'):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800166 r[commit[0:8]] = commit
167 return r
168
169 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700170 def date(self):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700171 return self.project.bare_git.log('--pretty=format:%cd',
172 '-n', '1',
173 R_HEADS + self.name,
174 '--')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700175
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700176 def UploadForReview(self, people,
177 auto_topic=False,
178 draft=False,
179 dest_branch=None):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800180 self.project.UploadForReview(self.name,
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -0700181 people,
Brian Harring435370c2012-07-28 15:37:04 -0700182 auto_topic=auto_topic,
Bryan Jacobsf609f912013-05-06 13:36:24 -0400183 draft=draft,
184 dest_branch=dest_branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700185
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700186 def GetPublishedRefs(self):
187 refs = {}
188 output = self.project.bare_git.ls_remote(
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700189 self.branch.remote.SshReviewUrl(self.project.UserEmail),
190 'refs/changes/*')
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700191 for line in output.split('\n'):
192 try:
193 (sha, ref) = line.split()
194 refs[sha] = ref
195 except ValueError:
196 pass
197
198 return refs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700199
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700200
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700201class StatusColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700202
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700203 def __init__(self, config):
204 Coloring.__init__(self, config, 'status')
Anthony King7bdac712014-07-16 12:56:40 +0100205 self.project = self.printer('header', attr='bold')
206 self.branch = self.printer('header', attr='bold')
207 self.nobranch = self.printer('nobranch', fg='red')
208 self.important = self.printer('important', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700209
Anthony King7bdac712014-07-16 12:56:40 +0100210 self.added = self.printer('added', fg='green')
211 self.changed = self.printer('changed', fg='red')
212 self.untracked = self.printer('untracked', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700213
214
215class DiffColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700216
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700217 def __init__(self, config):
218 Coloring.__init__(self, config, 'diff')
Anthony King7bdac712014-07-16 12:56:40 +0100219 self.project = self.printer('header', attr='bold')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700220
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700221
Anthony King7bdac712014-07-16 12:56:40 +0100222class _Annotation(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700223
James W. Mills24c13082012-04-12 15:04:13 -0500224 def __init__(self, name, value, keep):
225 self.name = name
226 self.value = value
227 self.keep = keep
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700228
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700229
Anthony King7bdac712014-07-16 12:56:40 +0100230class _CopyFile(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700231
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800232 def __init__(self, src, dest, abssrc, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700233 self.src = src
234 self.dest = dest
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800235 self.abs_src = abssrc
236 self.abs_dest = absdest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700237
238 def _Copy(self):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800239 src = self.abs_src
240 dest = self.abs_dest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700241 # copy file if it does not exist or is out of date
242 if not os.path.exists(dest) or not filecmp.cmp(src, dest):
243 try:
244 # remove existing file first, since it might be read-only
245 if os.path.exists(dest):
246 os.remove(dest)
Matthew Buckett2daf6672009-07-11 09:43:47 -0400247 else:
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200248 dest_dir = os.path.dirname(dest)
249 if not os.path.isdir(dest_dir):
250 os.makedirs(dest_dir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700251 shutil.copy(src, dest)
252 # make the file read-only
253 mode = os.stat(dest)[stat.ST_MODE]
254 mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
255 os.chmod(dest, mode)
256 except IOError:
Shawn O. Pearce48244782009-04-16 08:25:57 -0700257 _error('Cannot copy file %s to %s', src, dest)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700258
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700259
Anthony King7bdac712014-07-16 12:56:40 +0100260class _LinkFile(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700261
Wink Saville4c426ef2015-06-03 08:05:17 -0700262 def __init__(self, git_worktree, src, dest, relsrc, absdest):
263 self.git_worktree = git_worktree
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500264 self.src = src
265 self.dest = dest
Colin Cross0184dcc2015-05-05 00:24:54 -0700266 self.src_rel_to_dest = relsrc
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500267 self.abs_dest = absdest
268
Wink Saville4c426ef2015-06-03 08:05:17 -0700269 def __linkIt(self, relSrc, absDest):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500270 # link file if it does not exist or is out of date
Wink Saville4c426ef2015-06-03 08:05:17 -0700271 if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500272 try:
273 # remove existing file first, since it might be read-only
Dan Willemsene1e0bd12015-11-18 16:49:38 -0800274 if os.path.lexists(absDest):
Wink Saville4c426ef2015-06-03 08:05:17 -0700275 os.remove(absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500276 else:
Wink Saville4c426ef2015-06-03 08:05:17 -0700277 dest_dir = os.path.dirname(absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500278 if not os.path.isdir(dest_dir):
279 os.makedirs(dest_dir)
Wink Saville4c426ef2015-06-03 08:05:17 -0700280 os.symlink(relSrc, absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500281 except IOError:
Wink Saville4c426ef2015-06-03 08:05:17 -0700282 _error('Cannot link file %s to %s', relSrc, absDest)
283
284 def _Link(self):
285 """Link the self.rel_src_to_dest and self.abs_dest. Handles wild cards
286 on the src linking all of the files in the source in to the destination
287 directory.
288 """
289 # We use the absSrc to handle the situation where the current directory
290 # is not the root of the repo
291 absSrc = os.path.join(self.git_worktree, self.src)
292 if os.path.exists(absSrc):
293 # Entity exists so just a simple one to one link operation
294 self.__linkIt(self.src_rel_to_dest, self.abs_dest)
295 else:
296 # Entity doesn't exist assume there is a wild card
297 absDestDir = self.abs_dest
298 if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
299 _error('Link error: src with wildcard, %s must be a directory',
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700300 absDestDir)
Wink Saville4c426ef2015-06-03 08:05:17 -0700301 else:
302 absSrcFiles = glob.glob(absSrc)
303 for absSrcFile in absSrcFiles:
304 # Create a releative path from source dir to destination dir
305 absSrcDir = os.path.dirname(absSrcFile)
306 relSrcDir = os.path.relpath(absSrcDir, absDestDir)
307
308 # Get the source file name
309 srcFile = os.path.basename(absSrcFile)
310
311 # Now form the final full paths to srcFile. They will be
312 # absolute for the desintaiton and relative for the srouce.
313 absDest = os.path.join(absDestDir, srcFile)
314 relSrc = os.path.join(relSrcDir, srcFile)
315 self.__linkIt(relSrc, absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500316
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700317
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700318class RemoteSpec(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700319
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700320 def __init__(self,
321 name,
Anthony King7bdac712014-07-16 12:56:40 +0100322 url=None,
323 review=None,
Dan Willemsen96c2d652016-04-06 16:03:54 -0700324 revision=None,
325 orig_name=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700326 self.name = name
327 self.url = url
328 self.review = review
Anthony King36ea2fb2014-05-06 11:54:01 +0100329 self.revision = revision
Dan Willemsen96c2d652016-04-06 16:03:54 -0700330 self.orig_name = orig_name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700331
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700332
Doug Anderson37282b42011-03-04 11:54:18 -0800333class RepoHook(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700334
Doug Anderson37282b42011-03-04 11:54:18 -0800335 """A RepoHook contains information about a script to run as a hook.
336
337 Hooks are used to run a python script before running an upload (for instance,
338 to run presubmit checks). Eventually, we may have hooks for other actions.
339
340 This shouldn't be confused with files in the 'repo/hooks' directory. Those
341 files are copied into each '.git/hooks' folder for each project. Repo-level
342 hooks are associated instead with repo actions.
343
344 Hooks are always python. When a hook is run, we will load the hook into the
345 interpreter and execute its main() function.
346 """
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700347
Doug Anderson37282b42011-03-04 11:54:18 -0800348 def __init__(self,
349 hook_type,
350 hooks_project,
351 topdir,
Mike Frysinger40252c22016-08-15 21:23:44 -0400352 manifest_url,
Doug Anderson37282b42011-03-04 11:54:18 -0800353 abort_if_user_denies=False):
354 """RepoHook constructor.
355
356 Params:
357 hook_type: A string representing the type of hook. This is also used
358 to figure out the name of the file containing the hook. For
359 example: 'pre-upload'.
360 hooks_project: The project containing the repo hooks. If you have a
361 manifest, this is manifest.repo_hooks_project. OK if this is None,
362 which will make the hook a no-op.
363 topdir: Repo's top directory (the one containing the .repo directory).
364 Scripts will run with CWD as this directory. If you have a manifest,
365 this is manifest.topdir
Mike Frysinger40252c22016-08-15 21:23:44 -0400366 manifest_url: The URL to the manifest git repo.
Doug Anderson37282b42011-03-04 11:54:18 -0800367 abort_if_user_denies: If True, we'll throw a HookError() if the user
368 doesn't allow us to run the hook.
369 """
370 self._hook_type = hook_type
371 self._hooks_project = hooks_project
Mike Frysinger40252c22016-08-15 21:23:44 -0400372 self._manifest_url = manifest_url
Doug Anderson37282b42011-03-04 11:54:18 -0800373 self._topdir = topdir
374 self._abort_if_user_denies = abort_if_user_denies
375
376 # Store the full path to the script for convenience.
377 if self._hooks_project:
378 self._script_fullpath = os.path.join(self._hooks_project.worktree,
379 self._hook_type + '.py')
380 else:
381 self._script_fullpath = None
382
383 def _GetHash(self):
384 """Return a hash of the contents of the hooks directory.
385
386 We'll just use git to do this. This hash has the property that if anything
387 changes in the directory we will return a different has.
388
389 SECURITY CONSIDERATION:
390 This hash only represents the contents of files in the hook directory, not
391 any other files imported or called by hooks. Changes to imported files
392 can change the script behavior without affecting the hash.
393
394 Returns:
395 A string representing the hash. This will always be ASCII so that it can
396 be printed to the user easily.
397 """
398 assert self._hooks_project, "Must have hooks to calculate their hash."
399
400 # We will use the work_git object rather than just calling GetRevisionId().
401 # That gives us a hash of the latest checked in version of the files that
402 # the user will actually be executing. Specifically, GetRevisionId()
403 # doesn't appear to change even if a user checks out a different version
404 # of the hooks repo (via git checkout) nor if a user commits their own revs.
405 #
406 # NOTE: Local (non-committed) changes will not be factored into this hash.
407 # I think this is OK, since we're really only worried about warning the user
408 # about upstream changes.
409 return self._hooks_project.work_git.rev_parse('HEAD')
410
411 def _GetMustVerb(self):
412 """Return 'must' if the hook is required; 'should' if not."""
413 if self._abort_if_user_denies:
414 return 'must'
415 else:
416 return 'should'
417
418 def _CheckForHookApproval(self):
419 """Check to see whether this hook has been approved.
420
Mike Frysinger40252c22016-08-15 21:23:44 -0400421 We'll accept approval of manifest URLs if they're using secure transports.
422 This way the user can say they trust the manifest hoster. For insecure
423 hosts, we fall back to checking the hash of the hooks repo.
Doug Anderson37282b42011-03-04 11:54:18 -0800424
425 Note that we ask permission for each individual hook even though we use
426 the hash of all hooks when detecting changes. We'd like the user to be
427 able to approve / deny each hook individually. We only use the hash of all
428 hooks because there is no other easy way to detect changes to local imports.
429
430 Returns:
431 True if this hook is approved to run; False otherwise.
432
433 Raises:
434 HookError: Raised if the user doesn't approve and abort_if_user_denies
435 was passed to the consturctor.
436 """
Mike Frysinger40252c22016-08-15 21:23:44 -0400437 if self._ManifestUrlHasSecureScheme():
438 return self._CheckForHookApprovalManifest()
439 else:
440 return self._CheckForHookApprovalHash()
441
442 def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt,
443 changed_prompt):
444 """Check for approval for a particular attribute and hook.
445
446 Args:
447 subkey: The git config key under [repo.hooks.<hook_type>] to store the
448 last approved string.
449 new_val: The new value to compare against the last approved one.
450 main_prompt: Message to display to the user to ask for approval.
451 changed_prompt: Message explaining why we're re-asking for approval.
452
453 Returns:
454 True if this hook is approved to run; False otherwise.
455
456 Raises:
457 HookError: Raised if the user doesn't approve and abort_if_user_denies
458 was passed to the consturctor.
459 """
Doug Anderson37282b42011-03-04 11:54:18 -0800460 hooks_config = self._hooks_project.config
Mike Frysinger40252c22016-08-15 21:23:44 -0400461 git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey)
Doug Anderson37282b42011-03-04 11:54:18 -0800462
Mike Frysinger40252c22016-08-15 21:23:44 -0400463 # Get the last value that the user approved for this hook; may be None.
464 old_val = hooks_config.GetString(git_approval_key)
Doug Anderson37282b42011-03-04 11:54:18 -0800465
Mike Frysinger40252c22016-08-15 21:23:44 -0400466 if old_val is not None:
Doug Anderson37282b42011-03-04 11:54:18 -0800467 # User previously approved hook and asked not to be prompted again.
Mike Frysinger40252c22016-08-15 21:23:44 -0400468 if new_val == old_val:
Doug Anderson37282b42011-03-04 11:54:18 -0800469 # Approval matched. We're done.
470 return True
471 else:
472 # Give the user a reason why we're prompting, since they last told
473 # us to "never ask again".
Mike Frysinger40252c22016-08-15 21:23:44 -0400474 prompt = 'WARNING: %s\n\n' % (changed_prompt,)
Doug Anderson37282b42011-03-04 11:54:18 -0800475 else:
476 prompt = ''
477
478 # Prompt the user if we're not on a tty; on a tty we'll assume "no".
479 if sys.stdout.isatty():
Mike Frysinger40252c22016-08-15 21:23:44 -0400480 prompt += main_prompt + ' (yes/always/NO)? '
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530481 response = input(prompt).lower()
David Pursehouse98ffba12012-11-14 11:18:00 +0900482 print()
Doug Anderson37282b42011-03-04 11:54:18 -0800483
484 # User is doing a one-time approval.
485 if response in ('y', 'yes'):
486 return True
Mike Frysinger40252c22016-08-15 21:23:44 -0400487 elif response == 'always':
488 hooks_config.SetString(git_approval_key, new_val)
Doug Anderson37282b42011-03-04 11:54:18 -0800489 return True
490
491 # For anything else, we'll assume no approval.
492 if self._abort_if_user_denies:
493 raise HookError('You must allow the %s hook or use --no-verify.' %
494 self._hook_type)
495
496 return False
497
Mike Frysinger40252c22016-08-15 21:23:44 -0400498 def _ManifestUrlHasSecureScheme(self):
499 """Check if the URI for the manifest is a secure transport."""
500 secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc')
501 parse_results = urllib.parse.urlparse(self._manifest_url)
502 return parse_results.scheme in secure_schemes
503
504 def _CheckForHookApprovalManifest(self):
505 """Check whether the user has approved this manifest host.
506
507 Returns:
508 True if this hook is approved to run; False otherwise.
509 """
510 return self._CheckForHookApprovalHelper(
511 'approvedmanifest',
512 self._manifest_url,
513 'Run hook scripts from %s' % (self._manifest_url,),
514 'Manifest URL has changed since %s was allowed.' % (self._hook_type,))
515
516 def _CheckForHookApprovalHash(self):
517 """Check whether the user has approved the hooks repo.
518
519 Returns:
520 True if this hook is approved to run; False otherwise.
521 """
522 prompt = ('Repo %s run the script:\n'
523 ' %s\n'
524 '\n'
525 'Do you want to allow this script to run '
526 '(yes/yes-never-ask-again/NO)? ') % (self._GetMustVerb(),
527 self._script_fullpath)
528 return self._CheckForHookApprovalHelper(
529 'approvedhash',
530 self._GetHash(),
531 prompt,
532 'Scripts have changed since %s was allowed.' % (self._hook_type,))
533
Doug Anderson37282b42011-03-04 11:54:18 -0800534 def _ExecuteHook(self, **kwargs):
535 """Actually execute the given hook.
536
537 This will run the hook's 'main' function in our python interpreter.
538
539 Args:
540 kwargs: Keyword arguments to pass to the hook. These are often specific
541 to the hook type. For instance, pre-upload hooks will contain
542 a project_list.
543 """
544 # Keep sys.path and CWD stashed away so that we can always restore them
545 # upon function exit.
546 orig_path = os.getcwd()
547 orig_syspath = sys.path
548
549 try:
550 # Always run hooks with CWD as topdir.
551 os.chdir(self._topdir)
552
553 # Put the hook dir as the first item of sys.path so hooks can do
554 # relative imports. We want to replace the repo dir as [0] so
555 # hooks can't import repo files.
556 sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
557
558 # Exec, storing global context in the context dict. We catch exceptions
559 # and convert to a HookError w/ just the failing traceback.
Mike Frysinger4aa4b212016-03-04 15:03:00 -0500560 context = {'__file__': self._script_fullpath}
Doug Anderson37282b42011-03-04 11:54:18 -0800561 try:
Anthony King70f68902014-05-05 21:15:34 +0100562 exec(compile(open(self._script_fullpath).read(),
563 self._script_fullpath, 'exec'), context)
Doug Anderson37282b42011-03-04 11:54:18 -0800564 except Exception:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700565 raise HookError('%s\nFailed to import %s hook; see traceback above.' %
566 (traceback.format_exc(), self._hook_type))
Doug Anderson37282b42011-03-04 11:54:18 -0800567
568 # Running the script should have defined a main() function.
569 if 'main' not in context:
570 raise HookError('Missing main() in: "%s"' % self._script_fullpath)
571
Doug Anderson37282b42011-03-04 11:54:18 -0800572 # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
573 # We don't actually want hooks to define their main with this argument--
574 # it's there to remind them that their hook should always take **kwargs.
575 # For instance, a pre-upload hook should be defined like:
576 # def main(project_list, **kwargs):
577 #
578 # This allows us to later expand the API without breaking old hooks.
579 kwargs = kwargs.copy()
580 kwargs['hook_should_take_kwargs'] = True
581
582 # Call the main function in the hook. If the hook should cause the
583 # build to fail, it will raise an Exception. We'll catch that convert
584 # to a HookError w/ just the failing traceback.
585 try:
586 context['main'](**kwargs)
587 except Exception:
588 raise HookError('%s\nFailed to run main() for %s hook; see traceback '
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700589 'above.' % (traceback.format_exc(),
590 self._hook_type))
Doug Anderson37282b42011-03-04 11:54:18 -0800591 finally:
592 # Restore sys.path and CWD.
593 sys.path = orig_syspath
594 os.chdir(orig_path)
595
596 def Run(self, user_allows_all_hooks, **kwargs):
597 """Run the hook.
598
599 If the hook doesn't exist (because there is no hooks project or because
600 this particular hook is not enabled), this is a no-op.
601
602 Args:
603 user_allows_all_hooks: If True, we will never prompt about running the
604 hook--we'll just assume it's OK to run it.
605 kwargs: Keyword arguments to pass to the hook. These are often specific
606 to the hook type. For instance, pre-upload hooks will contain
607 a project_list.
608
609 Raises:
610 HookError: If there was a problem finding the hook or the user declined
611 to run a required hook (from _CheckForHookApproval).
612 """
613 # No-op if there is no hooks project or if hook is disabled.
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700614 if ((not self._hooks_project) or (self._hook_type not in
615 self._hooks_project.enabled_repo_hooks)):
Doug Anderson37282b42011-03-04 11:54:18 -0800616 return
617
618 # Bail with a nice error if we can't find the hook.
619 if not os.path.isfile(self._script_fullpath):
620 raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
621
622 # Make sure the user is OK with running the hook.
623 if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
624 return
625
626 # Run the hook with the same version of python we're using.
627 self._ExecuteHook(**kwargs)
628
629
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700630class Project(object):
Kevin Degi384b3c52014-10-16 16:02:58 -0600631 # These objects can be shared between several working trees.
632 shareable_files = ['description', 'info']
633 shareable_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
634 # These objects can only be used by a single working tree.
635 working_tree_files = ['config', 'packed-refs', 'shallow']
636 working_tree_dirs = ['logs', 'refs']
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700637
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700638 def __init__(self,
639 manifest,
640 name,
641 remote,
642 gitdir,
David James8d201162013-10-11 17:03:19 -0700643 objdir,
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700644 worktree,
645 relpath,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700646 revisionExpr,
Mike Pontillod3153822012-02-28 11:53:24 -0800647 revisionId,
Anthony King7bdac712014-07-16 12:56:40 +0100648 rebase=True,
649 groups=None,
650 sync_c=False,
651 sync_s=False,
652 clone_depth=None,
653 upstream=None,
654 parent=None,
655 is_derived=False,
David Pursehouseb1553542014-09-04 21:28:09 +0900656 dest_branch=None,
Simran Basib9a1b732015-08-20 12:19:28 -0700657 optimized_fetch=False,
658 old_revision=None):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800659 """Init a Project object.
660
661 Args:
662 manifest: The XmlManifest object.
663 name: The `name` attribute of manifest.xml's project element.
664 remote: RemoteSpec object specifying its remote's properties.
665 gitdir: Absolute path of git directory.
David James8d201162013-10-11 17:03:19 -0700666 objdir: Absolute path of directory to store git objects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800667 worktree: Absolute path of git working tree.
668 relpath: Relative path of git working tree to repo's top directory.
669 revisionExpr: The `revision` attribute of manifest.xml's project element.
670 revisionId: git commit id for checking out.
671 rebase: The `rebase` attribute of manifest.xml's project element.
672 groups: The `groups` attribute of manifest.xml's project element.
673 sync_c: The `sync-c` attribute of manifest.xml's project element.
674 sync_s: The `sync-s` attribute of manifest.xml's project element.
675 upstream: The `upstream` attribute of manifest.xml's project element.
676 parent: The parent Project object.
677 is_derived: False if the project was explicitly defined in the manifest;
678 True if the project is a discovered submodule.
Bryan Jacobsf609f912013-05-06 13:36:24 -0400679 dest_branch: The branch to which to push changes for review by default.
David Pursehouseb1553542014-09-04 21:28:09 +0900680 optimized_fetch: If True, when a project is set to a sha1 revision, only
681 fetch from the remote if the sha1 is not present locally.
Simran Basib9a1b732015-08-20 12:19:28 -0700682 old_revision: saved git commit id for open GITC projects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800683 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700684 self.manifest = manifest
685 self.name = name
686 self.remote = remote
Anthony Newnamdf14a702011-01-09 17:31:57 -0800687 self.gitdir = gitdir.replace('\\', '/')
David James8d201162013-10-11 17:03:19 -0700688 self.objdir = objdir.replace('\\', '/')
Shawn O. Pearce0ce6ca92011-01-10 13:26:01 -0800689 if worktree:
Mark E. Hamiltonf9fe3e12016-02-23 18:10:42 -0700690 self.worktree = os.path.normpath(worktree.replace('\\', '/'))
Shawn O. Pearce0ce6ca92011-01-10 13:26:01 -0800691 else:
692 self.worktree = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700693 self.relpath = relpath
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700694 self.revisionExpr = revisionExpr
695
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700696 if revisionId is None \
697 and revisionExpr \
698 and IsId(revisionExpr):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700699 self.revisionId = revisionExpr
700 else:
701 self.revisionId = revisionId
702
Mike Pontillod3153822012-02-28 11:53:24 -0800703 self.rebase = rebase
Colin Cross5acde752012-03-28 20:15:45 -0700704 self.groups = groups
Anatol Pomazau79770d22012-04-20 14:41:59 -0700705 self.sync_c = sync_c
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800706 self.sync_s = sync_s
David Pursehouseede7f122012-11-27 22:25:30 +0900707 self.clone_depth = clone_depth
Brian Harring14a66742012-09-28 20:21:57 -0700708 self.upstream = upstream
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800709 self.parent = parent
710 self.is_derived = is_derived
David Pursehouseb1553542014-09-04 21:28:09 +0900711 self.optimized_fetch = optimized_fetch
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800712 self.subprojects = []
Mike Pontillod3153822012-02-28 11:53:24 -0800713
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700714 self.snapshots = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700715 self.copyfiles = []
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500716 self.linkfiles = []
James W. Mills24c13082012-04-12 15:04:13 -0500717 self.annotations = []
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700718 self.config = GitConfig.ForRepository(gitdir=self.gitdir,
719 defaults=self.manifest.globalConfig)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700720
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800721 if self.worktree:
David James8d201162013-10-11 17:03:19 -0700722 self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800723 else:
724 self.work_git = None
David James8d201162013-10-11 17:03:19 -0700725 self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
Shawn O. Pearced237b692009-04-17 18:49:50 -0700726 self.bare_ref = GitRefs(gitdir)
David James8d201162013-10-11 17:03:19 -0700727 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
Bryan Jacobsf609f912013-05-06 13:36:24 -0400728 self.dest_branch = dest_branch
Simran Basib9a1b732015-08-20 12:19:28 -0700729 self.old_revision = old_revision
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700730
Doug Anderson37282b42011-03-04 11:54:18 -0800731 # This will be filled in if a project is later identified to be the
732 # project containing repo hooks.
733 self.enabled_repo_hooks = []
734
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700735 @property
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800736 def Derived(self):
737 return self.is_derived
738
739 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700740 def Exists(self):
Kevin Degi384b3c52014-10-16 16:02:58 -0600741 return os.path.isdir(self.gitdir) and os.path.isdir(self.objdir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700742
743 @property
744 def CurrentBranch(self):
745 """Obtain the name of the currently checked out branch.
746 The branch name omits the 'refs/heads/' prefix.
747 None is returned if the project is on a detached HEAD.
748 """
Shawn O. Pearce5b23f242009-04-17 18:43:33 -0700749 b = self.work_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700750 if b.startswith(R_HEADS):
751 return b[len(R_HEADS):]
752 return None
753
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700754 def IsRebaseInProgress(self):
755 w = self.worktree
756 g = os.path.join(w, '.git')
757 return os.path.exists(os.path.join(g, 'rebase-apply')) \
758 or os.path.exists(os.path.join(g, 'rebase-merge')) \
759 or os.path.exists(os.path.join(w, '.dotest'))
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200760
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700761 def IsDirty(self, consider_untracked=True):
762 """Is the working directory modified in some way?
763 """
764 self.work_git.update_index('-q',
765 '--unmerged',
766 '--ignore-missing',
767 '--refresh')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900768 if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700769 return True
770 if self.work_git.DiffZ('diff-files'):
771 return True
772 if consider_untracked and self.work_git.LsOthers():
773 return True
774 return False
775
776 _userident_name = None
777 _userident_email = None
778
779 @property
780 def UserName(self):
781 """Obtain the user's personal name.
782 """
783 if self._userident_name is None:
784 self._LoadUserIdentity()
785 return self._userident_name
786
787 @property
788 def UserEmail(self):
789 """Obtain the user's email address. This is very likely
790 to be their Gerrit login.
791 """
792 if self._userident_email is None:
793 self._LoadUserIdentity()
794 return self._userident_email
795
796 def _LoadUserIdentity(self):
David Pursehousec1b86a22012-11-14 11:36:51 +0900797 u = self.bare_git.var('GIT_COMMITTER_IDENT')
798 m = re.compile("^(.*) <([^>]*)> ").match(u)
799 if m:
800 self._userident_name = m.group(1)
801 self._userident_email = m.group(2)
802 else:
803 self._userident_name = ''
804 self._userident_email = ''
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700805
806 def GetRemote(self, name):
807 """Get the configuration for a single remote.
808 """
809 return self.config.GetRemote(name)
810
811 def GetBranch(self, name):
812 """Get the configuration for a single branch.
813 """
814 return self.config.GetBranch(name)
815
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700816 def GetBranches(self):
817 """Get all existing local branches.
818 """
819 current = self.CurrentBranch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900820 all_refs = self._allrefs
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700821 heads = {}
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700822
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530823 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700824 if name.startswith(R_HEADS):
825 name = name[len(R_HEADS):]
826 b = self.GetBranch(name)
827 b.current = name == current
828 b.published = None
David Pursehouse8a68ff92012-09-24 12:15:13 +0900829 b.revision = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700830 heads[name] = b
831
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530832 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700833 if name.startswith(R_PUB):
834 name = name[len(R_PUB):]
835 b = heads.get(name)
836 if b:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900837 b.published = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700838
839 return heads
840
Colin Cross5acde752012-03-28 20:15:45 -0700841 def MatchesGroups(self, manifest_groups):
842 """Returns true if the manifest groups specified at init should cause
843 this project to be synced.
844 Prefixing a manifest group with "-" inverts the meaning of a group.
Conley Owensbb1b5f52012-08-13 13:11:18 -0700845 All projects are implicitly labelled with "all".
Conley Owens971de8e2012-04-16 10:36:08 -0700846
847 labels are resolved in order. In the example case of
Conley Owensbb1b5f52012-08-13 13:11:18 -0700848 project_groups: "all,group1,group2"
Conley Owens971de8e2012-04-16 10:36:08 -0700849 manifest_groups: "-group1,group2"
850 the project will be matched.
David Holmer0a1c6a12012-11-14 19:19:00 -0500851
852 The special manifest group "default" will match any project that
853 does not have the special project group "notdefault"
Colin Cross5acde752012-03-28 20:15:45 -0700854 """
David Holmer0a1c6a12012-11-14 19:19:00 -0500855 expanded_manifest_groups = manifest_groups or ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700856 expanded_project_groups = ['all'] + (self.groups or [])
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700857 if 'notdefault' not in expanded_project_groups:
David Holmer0a1c6a12012-11-14 19:19:00 -0500858 expanded_project_groups += ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700859
Conley Owens971de8e2012-04-16 10:36:08 -0700860 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700861 for group in expanded_manifest_groups:
862 if group.startswith('-') and group[1:] in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700863 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700864 elif group in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700865 matched = True
Colin Cross5acde752012-03-28 20:15:45 -0700866
Conley Owens971de8e2012-04-16 10:36:08 -0700867 return matched
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700868
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700869# Status Display ##
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700870 def UncommitedFiles(self, get_all=True):
871 """Returns a list of strings, uncommitted files in the git tree.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700872
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700873 Args:
874 get_all: a boolean, if True - get information about all different
875 uncommitted files. If False - return as soon as any kind of
876 uncommitted files is detected.
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500877 """
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700878 details = []
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500879 self.work_git.update_index('-q',
880 '--unmerged',
881 '--ignore-missing',
882 '--refresh')
883 if self.IsRebaseInProgress():
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700884 details.append("rebase in progress")
885 if not get_all:
886 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500887
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700888 changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
889 if changes:
890 details.extend(changes)
891 if not get_all:
892 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500893
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700894 changes = self.work_git.DiffZ('diff-files').keys()
895 if changes:
896 details.extend(changes)
897 if not get_all:
898 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500899
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700900 changes = self.work_git.LsOthers()
901 if changes:
902 details.extend(changes)
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500903
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700904 return details
905
906 def HasChanges(self):
907 """Returns true if there are uncommitted changes.
908 """
909 if self.UncommitedFiles(get_all=False):
910 return True
911 else:
912 return False
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500913
Terence Haddock4655e812011-03-31 12:33:34 +0200914 def PrintWorkTreeStatus(self, output_redir=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700915 """Prints the status of the repository to stdout.
Terence Haddock4655e812011-03-31 12:33:34 +0200916
917 Args:
918 output: If specified, redirect the output to this object.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700919 """
920 if not os.path.isdir(self.worktree):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700921 if output_redir is None:
Terence Haddock4655e812011-03-31 12:33:34 +0200922 output_redir = sys.stdout
Sarah Owenscecd1d82012-11-01 22:59:27 -0700923 print(file=output_redir)
924 print('project %s/' % self.relpath, file=output_redir)
925 print(' missing (run "repo sync")', file=output_redir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700926 return
927
928 self.work_git.update_index('-q',
929 '--unmerged',
930 '--ignore-missing',
931 '--refresh')
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700932 rb = self.IsRebaseInProgress()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700933 di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
934 df = self.work_git.DiffZ('diff-files')
935 do = self.work_git.LsOthers()
Ali Utku Selen76abcc12012-01-25 10:51:12 +0100936 if not rb and not di and not df and not do and not self.CurrentBranch:
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700937 return 'CLEAN'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700938
939 out = StatusColoring(self.config)
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700940 if output_redir is not None:
Terence Haddock4655e812011-03-31 12:33:34 +0200941 out.redirect(output_redir)
Jakub Vrana0402cd82014-09-09 15:39:15 -0700942 out.project('project %-40s', self.relpath + '/ ')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700943
944 branch = self.CurrentBranch
945 if branch is None:
946 out.nobranch('(*** NO BRANCH ***)')
947 else:
948 out.branch('branch %s', branch)
949 out.nl()
950
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700951 if rb:
952 out.important('prior sync failed; rebase still in progress')
953 out.nl()
954
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700955 paths = list()
956 paths.extend(di.keys())
957 paths.extend(df.keys())
958 paths.extend(do)
959
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530960 for p in sorted(set(paths)):
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900961 try:
962 i = di[p]
963 except KeyError:
964 i = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700965
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900966 try:
967 f = df[p]
968 except KeyError:
969 f = None
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200970
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900971 if i:
972 i_status = i.status.upper()
973 else:
974 i_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700975
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900976 if f:
977 f_status = f.status.lower()
978 else:
979 f_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700980
981 if i and i.src_path:
Shawn O. Pearcefe086752009-03-03 13:49:48 -0800982 line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700983 i.src_path, p, i.level)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700984 else:
985 line = ' %s%s\t%s' % (i_status, f_status, p)
986
987 if i and not f:
988 out.added('%s', line)
989 elif (i and f) or (not i and f):
990 out.changed('%s', line)
991 elif not i and not f:
992 out.untracked('%s', line)
993 else:
994 out.write('%s', line)
995 out.nl()
Terence Haddock4655e812011-03-31 12:33:34 +0200996
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700997 return 'DIRTY'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700998
pelyad67872d2012-03-28 14:49:58 +0300999 def PrintWorkTreeDiff(self, absolute_paths=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001000 """Prints the status of the repository to stdout.
1001 """
1002 out = DiffColoring(self.config)
1003 cmd = ['diff']
1004 if out.is_on:
1005 cmd.append('--color')
1006 cmd.append(HEAD)
pelyad67872d2012-03-28 14:49:58 +03001007 if absolute_paths:
1008 cmd.append('--src-prefix=a/%s/' % self.relpath)
1009 cmd.append('--dst-prefix=b/%s/' % self.relpath)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001010 cmd.append('--')
1011 p = GitCommand(self,
1012 cmd,
Anthony King7bdac712014-07-16 12:56:40 +01001013 capture_stdout=True,
1014 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001015 has_diff = False
1016 for line in p.process.stdout:
1017 if not has_diff:
1018 out.nl()
1019 out.project('project %s/' % self.relpath)
1020 out.nl()
1021 has_diff = True
Sarah Owenscecd1d82012-11-01 22:59:27 -07001022 print(line[:-1])
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001023 p.Wait()
1024
1025
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001026# Publish / Upload ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001027
David Pursehouse8a68ff92012-09-24 12:15:13 +09001028 def WasPublished(self, branch, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001029 """Was the branch published (uploaded) for code review?
1030 If so, returns the SHA-1 hash of the last published
1031 state for the branch.
1032 """
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001033 key = R_PUB + branch
David Pursehouse8a68ff92012-09-24 12:15:13 +09001034 if all_refs is None:
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001035 try:
1036 return self.bare_git.rev_parse(key)
1037 except GitError:
1038 return None
1039 else:
1040 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001041 return all_refs[key]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001042 except KeyError:
1043 return None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001044
David Pursehouse8a68ff92012-09-24 12:15:13 +09001045 def CleanPublishedCache(self, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001046 """Prunes any stale published refs.
1047 """
David Pursehouse8a68ff92012-09-24 12:15:13 +09001048 if all_refs is None:
1049 all_refs = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001050 heads = set()
1051 canrm = {}
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301052 for name, ref_id in all_refs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001053 if name.startswith(R_HEADS):
1054 heads.add(name)
1055 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001056 canrm[name] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001057
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301058 for name, ref_id in canrm.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001059 n = name[len(R_PUB):]
1060 if R_HEADS + n not in heads:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001061 self.bare_git.DeleteRef(name, ref_id)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001062
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -07001063 def GetUploadableBranches(self, selected_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001064 """List any branches which can be uploaded for review.
1065 """
1066 heads = {}
1067 pubed = {}
1068
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301069 for name, ref_id in self._allrefs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001070 if name.startswith(R_HEADS):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001071 heads[name[len(R_HEADS):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001072 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001073 pubed[name[len(R_PUB):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001074
1075 ready = []
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301076 for branch, ref_id in heads.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09001077 if branch in pubed and pubed[branch] == ref_id:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001078 continue
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -07001079 if selected_branch and branch != selected_branch:
1080 continue
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001081
Shawn O. Pearce35f25962008-11-11 17:03:13 -08001082 rb = self.GetUploadableBranch(branch)
1083 if rb:
1084 ready.append(rb)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001085 return ready
1086
Shawn O. Pearce35f25962008-11-11 17:03:13 -08001087 def GetUploadableBranch(self, branch_name):
1088 """Get a single uploadable branch, or None.
1089 """
1090 branch = self.GetBranch(branch_name)
1091 base = branch.LocalMerge
1092 if branch.LocalMerge:
1093 rb = ReviewableBranch(self, branch, base)
1094 if rb.commits:
1095 return rb
1096 return None
1097
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001098 def UploadForReview(self, branch=None,
Anthony King7bdac712014-07-16 12:56:40 +01001099 people=([], []),
Brian Harring435370c2012-07-28 15:37:04 -07001100 auto_topic=False,
Bryan Jacobsf609f912013-05-06 13:36:24 -04001101 draft=False,
1102 dest_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001103 """Uploads the named branch for code review.
1104 """
1105 if branch is None:
1106 branch = self.CurrentBranch
1107 if branch is None:
1108 raise GitError('not currently on a branch')
1109
1110 branch = self.GetBranch(branch)
1111 if not branch.LocalMerge:
1112 raise GitError('branch %s does not track a remote' % branch.name)
1113 if not branch.remote.review:
1114 raise GitError('remote %s has no review url' % branch.remote.name)
1115
Bryan Jacobsf609f912013-05-06 13:36:24 -04001116 if dest_branch is None:
1117 dest_branch = self.dest_branch
1118 if dest_branch is None:
1119 dest_branch = branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001120 if not dest_branch.startswith(R_HEADS):
1121 dest_branch = R_HEADS + dest_branch
1122
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -08001123 if not branch.remote.projectname:
1124 branch.remote.projectname = self.name
1125 branch.remote.Save()
1126
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001127 url = branch.remote.ReviewUrl(self.UserEmail)
1128 if url is None:
1129 raise UploadError('review not configured')
1130 cmd = ['push']
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001131
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001132 if url.startswith('ssh://'):
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001133 rp = ['gerrit receive-pack']
1134 for e in people[0]:
1135 rp.append('--reviewer=%s' % sq(e))
1136 for e in people[1]:
1137 rp.append('--cc=%s' % sq(e))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001138 cmd.append('--receive-pack=%s' % " ".join(rp))
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001139
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001140 cmd.append(url)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001141
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001142 if dest_branch.startswith(R_HEADS):
1143 dest_branch = dest_branch[len(R_HEADS):]
Brian Harring435370c2012-07-28 15:37:04 -07001144
1145 upload_type = 'for'
1146 if draft:
1147 upload_type = 'drafts'
1148
1149 ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
1150 dest_branch)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001151 if auto_topic:
1152 ref_spec = ref_spec + '/' + branch.name
Shawn Pearce45d21682013-02-28 00:35:51 -08001153 if not url.startswith('ssh://'):
1154 rp = ['r=%s' % p for p in people[0]] + \
1155 ['cc=%s' % p for p in people[1]]
1156 if rp:
1157 ref_spec = ref_spec + '%' + ','.join(rp)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001158 cmd.append(ref_spec)
1159
Anthony King7bdac712014-07-16 12:56:40 +01001160 if GitCommand(self, cmd, bare=True).Wait() != 0:
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001161 raise UploadError('Upload failed')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001162
1163 msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
1164 self.bare_git.UpdateRef(R_PUB + branch.name,
1165 R_HEADS + branch.name,
Anthony King7bdac712014-07-16 12:56:40 +01001166 message=msg)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001167
1168
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001169# Sync ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001170
Julien Campergue335f5ef2013-10-16 11:02:35 +02001171 def _ExtractArchive(self, tarpath, path=None):
1172 """Extract the given tar on its current location
1173
1174 Args:
1175 - tarpath: The path to the actual tar file
1176
1177 """
1178 try:
1179 with tarfile.open(tarpath, 'r') as tar:
1180 tar.extractall(path=path)
1181 return True
1182 except (IOError, tarfile.TarError) as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001183 _error("Cannot extract archive %s: %s", tarpath, str(e))
Julien Campergue335f5ef2013-10-16 11:02:35 +02001184 return False
1185
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001186 def Sync_NetworkHalf(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001187 quiet=False,
1188 is_new=None,
1189 current_branch_only=False,
1190 force_sync=False,
1191 clone_bundle=True,
1192 no_tags=False,
1193 archive=False,
1194 optimized_fetch=False,
1195 prune=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001196 """Perform only the network IO portion of the sync process.
1197 Local working directory/branch state is not affected.
1198 """
Julien Campergue335f5ef2013-10-16 11:02:35 +02001199 if archive and not isinstance(self, MetaProject):
1200 if self.remote.url.startswith(('http://', 'https://')):
David Pursehousef33929d2015-08-24 14:39:14 +09001201 _error("%s: Cannot fetch archives from http/https remotes.", self.name)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001202 return False
1203
1204 name = self.relpath.replace('\\', '/')
1205 name = name.replace('/', '_')
1206 tarpath = '%s.tar' % name
1207 topdir = self.manifest.topdir
1208
1209 try:
1210 self._FetchArchive(tarpath, cwd=topdir)
1211 except GitError as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001212 _error('%s', e)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001213 return False
1214
1215 # From now on, we only need absolute tarpath
1216 tarpath = os.path.join(topdir, tarpath)
1217
1218 if not self._ExtractArchive(tarpath, path=topdir):
1219 return False
1220 try:
1221 os.remove(tarpath)
1222 except OSError as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001223 _warn("Cannot remove archive %s: %s", tarpath, str(e))
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001224 self._CopyAndLinkFiles()
Julien Campergue335f5ef2013-10-16 11:02:35 +02001225 return True
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001226 if is_new is None:
1227 is_new = not self.Exists
Shawn O. Pearce88443382010-10-08 10:02:09 +02001228 if is_new:
Kevin Degiabaa7f32014-11-12 11:27:45 -07001229 self._InitGitDir(force_sync=force_sync)
Jimmie Westera0444582012-10-24 13:44:42 +02001230 else:
1231 self._UpdateHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001232 self._InitRemote()
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001233
1234 if is_new:
1235 alt = os.path.join(self.gitdir, 'objects/info/alternates')
1236 try:
1237 fd = open(alt, 'rb')
1238 try:
1239 alt_dir = fd.readline().rstrip()
1240 finally:
1241 fd.close()
1242 except IOError:
1243 alt_dir = None
1244 else:
1245 alt_dir = None
1246
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001247 if clone_bundle \
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001248 and alt_dir is None \
1249 and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001250 is_new = False
1251
Shawn O. Pearce6ba6ba02012-05-24 09:46:50 -07001252 if not current_branch_only:
1253 if self.sync_c:
1254 current_branch_only = True
1255 elif not self.manifest._loaded:
1256 # Manifest cannot check defaults until it syncs.
1257 current_branch_only = False
1258 elif self.manifest.default.sync_c:
1259 current_branch_only = True
1260
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001261 need_to_fetch = not (optimized_fetch and
1262 (ID_RE.match(self.revisionExpr) and
1263 self._CheckForSha1()))
1264 if (need_to_fetch and
1265 not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
1266 current_branch_only=current_branch_only,
1267 no_tags=no_tags, prune=prune)):
Anthony King7bdac712014-07-16 12:56:40 +01001268 return False
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001269
1270 if self.worktree:
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001271 self._InitMRef()
1272 else:
1273 self._InitMirrorHead()
1274 try:
1275 os.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
1276 except OSError:
1277 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001278 return True
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08001279
1280 def PostRepoUpgrade(self):
1281 self._InitHooks()
1282
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001283 def _CopyAndLinkFiles(self):
Simran Basib9a1b732015-08-20 12:19:28 -07001284 if self.manifest.isGitcClient:
1285 return
David Pursehouse8a68ff92012-09-24 12:15:13 +09001286 for copyfile in self.copyfiles:
1287 copyfile._Copy()
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001288 for linkfile in self.linkfiles:
1289 linkfile._Link()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001290
Julien Camperguedd654222014-01-09 16:21:37 +01001291 def GetCommitRevisionId(self):
1292 """Get revisionId of a commit.
1293
1294 Use this method instead of GetRevisionId to get the id of the commit rather
1295 than the id of the current git object (for example, a tag)
1296
1297 """
1298 if not self.revisionExpr.startswith(R_TAGS):
1299 return self.GetRevisionId(self._allrefs)
1300
1301 try:
1302 return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
1303 except GitError:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001304 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1305 (self.revisionExpr, self.name))
Julien Camperguedd654222014-01-09 16:21:37 +01001306
David Pursehouse8a68ff92012-09-24 12:15:13 +09001307 def GetRevisionId(self, all_refs=None):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001308 if self.revisionId:
1309 return self.revisionId
1310
1311 rem = self.GetRemote(self.remote.name)
1312 rev = rem.ToLocal(self.revisionExpr)
1313
David Pursehouse8a68ff92012-09-24 12:15:13 +09001314 if all_refs is not None and rev in all_refs:
1315 return all_refs[rev]
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001316
1317 try:
1318 return self.bare_git.rev_parse('--verify', '%s^0' % rev)
1319 except GitError:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001320 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1321 (self.revisionExpr, self.name))
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001322
Kevin Degiabaa7f32014-11-12 11:27:45 -07001323 def Sync_LocalHalf(self, syncbuf, force_sync=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001324 """Perform only the local IO portion of the sync process.
1325 Network access is not required.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001326 """
Kevin Degiabaa7f32014-11-12 11:27:45 -07001327 self._InitWorkTree(force_sync=force_sync)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001328 all_refs = self.bare_ref.all
1329 self.CleanPublishedCache(all_refs)
1330 revid = self.GetRevisionId(all_refs)
Skyler Kaufman835cd682011-03-08 12:14:41 -08001331
David Pursehouse1d947b32012-10-25 12:23:11 +09001332 def _doff():
1333 self._FastForward(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001334 self._CopyAndLinkFiles()
David Pursehouse1d947b32012-10-25 12:23:11 +09001335
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001336 head = self.work_git.GetHead()
1337 if head.startswith(R_HEADS):
1338 branch = head[len(R_HEADS):]
1339 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001340 head = all_refs[head]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001341 except KeyError:
1342 head = None
1343 else:
1344 branch = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001345
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001346 if branch is None or syncbuf.detach_head:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001347 # Currently on a detached HEAD. The user is assumed to
1348 # not have any local modifications worth worrying about.
1349 #
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -07001350 if self.IsRebaseInProgress():
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001351 syncbuf.fail(self, _PriorSyncFailedError())
1352 return
1353
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001354 if head == revid:
1355 # No changes; don't do anything further.
Florian Vallee7cf1b362012-06-07 17:11:42 +02001356 # Except if the head needs to be detached
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001357 #
Florian Vallee7cf1b362012-06-07 17:11:42 +02001358 if not syncbuf.detach_head:
Dan Willemsen029eaf32015-09-03 12:52:28 -07001359 # The copy/linkfile config may have changed.
1360 self._CopyAndLinkFiles()
Florian Vallee7cf1b362012-06-07 17:11:42 +02001361 return
1362 else:
1363 lost = self._revlist(not_rev(revid), HEAD)
1364 if lost:
1365 syncbuf.info(self, "discarding %d commits", len(lost))
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001366
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001367 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001368 self._Checkout(revid, quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001369 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001370 syncbuf.fail(self, e)
1371 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001372 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001373 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001374
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001375 if head == revid:
1376 # No changes; don't do anything further.
1377 #
Dan Willemsen029eaf32015-09-03 12:52:28 -07001378 # The copy/linkfile config may have changed.
1379 self._CopyAndLinkFiles()
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001380 return
1381
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001382 branch = self.GetBranch(branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001383
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001384 if not branch.LocalMerge:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001385 # The current branch has no tracking configuration.
Anatol Pomazau2a32f6a2011-08-30 10:52:33 -07001386 # Jump off it to a detached HEAD.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001387 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001388 syncbuf.info(self,
1389 "leaving %s; does not track upstream",
1390 branch.name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001391 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001392 self._Checkout(revid, quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001393 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001394 syncbuf.fail(self, e)
1395 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001396 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001397 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001398
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001399 upstream_gain = self._revlist(not_rev(HEAD), revid)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001400 pub = self.WasPublished(branch.name, all_refs)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001401 if pub:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001402 not_merged = self._revlist(not_rev(revid), pub)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001403 if not_merged:
1404 if upstream_gain:
1405 # The user has published this branch and some of those
1406 # commits are not yet merged upstream. We do not want
1407 # to rewrite the published commits so we punt.
1408 #
Daniel Sandler4c50dee2010-03-02 15:38:03 -05001409 syncbuf.fail(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001410 "branch %s is published (but not merged) and is now "
1411 "%d commits behind" % (branch.name, len(upstream_gain)))
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001412 return
Shawn O. Pearce05f66b62009-04-21 08:26:32 -07001413 elif pub == head:
1414 # All published commits are merged, and thus we are a
1415 # strict subset. We can fast-forward safely.
Shawn O. Pearcea54c5272008-10-30 11:03:00 -07001416 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001417 syncbuf.later1(self, _doff)
1418 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001419
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001420 # Examine the local commits not in the remote. Find the
1421 # last one attributed to this user, if any.
1422 #
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001423 local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001424 last_mine = None
1425 cnt_mine = 0
1426 for commit in local_changes:
Chirayu Desai0eb35cb2013-11-19 18:46:29 +05301427 commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001428 if committer_email == self.UserEmail:
1429 last_mine = commit_id
1430 cnt_mine += 1
1431
Shawn O. Pearceda88ff42009-06-03 11:09:12 -07001432 if not upstream_gain and cnt_mine == len(local_changes):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001433 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001434
1435 if self.IsDirty(consider_untracked=False):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001436 syncbuf.fail(self, _DirtyError())
1437 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001438
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001439 # If the upstream switched on us, warn the user.
1440 #
1441 if branch.merge != self.revisionExpr:
1442 if branch.merge and self.revisionExpr:
1443 syncbuf.info(self,
1444 'manifest switched %s...%s',
1445 branch.merge,
1446 self.revisionExpr)
1447 elif branch.merge:
1448 syncbuf.info(self,
1449 'manifest no longer tracks %s',
1450 branch.merge)
1451
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001452 if cnt_mine < len(local_changes):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001453 # Upstream rebased. Not everything in HEAD
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001454 # was created by this user.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001455 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001456 syncbuf.info(self,
1457 "discarding %d commits removed from upstream",
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001458 len(local_changes) - cnt_mine)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001459
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001460 branch.remote = self.GetRemote(self.remote.name)
Anatol Pomazaucd7c5de2012-03-20 13:45:00 -07001461 if not ID_RE.match(self.revisionExpr):
1462 # in case of manifest sync the revisionExpr might be a SHA1
1463 branch.merge = self.revisionExpr
Conley Owens04f2f0e2014-10-01 17:22:46 -07001464 if not branch.merge.startswith('refs/'):
1465 branch.merge = R_HEADS + branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001466 branch.Save()
1467
Mike Pontillod3153822012-02-28 11:53:24 -08001468 if cnt_mine > 0 and self.rebase:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001469 def _dorebase():
Anthony King7bdac712014-07-16 12:56:40 +01001470 self._Rebase(upstream='%s^1' % last_mine, onto=revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001471 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001472 syncbuf.later2(self, _dorebase)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001473 elif local_changes:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001474 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001475 self._ResetHard(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001476 self._CopyAndLinkFiles()
Sarah Owensa5be53f2012-09-09 15:37:57 -07001477 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001478 syncbuf.fail(self, e)
1479 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001480 else:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001481 syncbuf.later1(self, _doff)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001482
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001483 def AddCopyFile(self, src, dest, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001484 # dest should already be an absolute path, but src is project relative
1485 # make src an absolute path
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001486 abssrc = os.path.join(self.worktree, src)
1487 self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001488
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001489 def AddLinkFile(self, src, dest, absdest):
1490 # dest should already be an absolute path, but src is project relative
Colin Cross0184dcc2015-05-05 00:24:54 -07001491 # make src relative path to dest
1492 absdestdir = os.path.dirname(absdest)
1493 relsrc = os.path.relpath(os.path.join(self.worktree, src), absdestdir)
Wink Saville4c426ef2015-06-03 08:05:17 -07001494 self.linkfiles.append(_LinkFile(self.worktree, src, dest, relsrc, absdest))
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001495
James W. Mills24c13082012-04-12 15:04:13 -05001496 def AddAnnotation(self, name, value, keep):
1497 self.annotations.append(_Annotation(name, value, keep))
1498
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001499 def DownloadPatchSet(self, change_id, patch_id):
1500 """Download a single patch set of a single change to FETCH_HEAD.
1501 """
1502 remote = self.GetRemote(self.remote.name)
1503
1504 cmd = ['fetch', remote.name]
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001505 cmd.append('refs/changes/%2.2d/%d/%d'
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001506 % (change_id % 100, change_id, patch_id))
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001507 if GitCommand(self, cmd, bare=True).Wait() != 0:
1508 return None
1509 return DownloadedChange(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001510 self.GetRevisionId(),
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001511 change_id,
1512 patch_id,
1513 self.bare_git.rev_parse('FETCH_HEAD'))
1514
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001515
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001516# Branch Management ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001517
Simran Basib9a1b732015-08-20 12:19:28 -07001518 def StartBranch(self, name, branch_merge=''):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001519 """Create a new branch off the manifest's revision.
1520 """
Simran Basib9a1b732015-08-20 12:19:28 -07001521 if not branch_merge:
1522 branch_merge = self.revisionExpr
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001523 head = self.work_git.GetHead()
1524 if head == (R_HEADS + name):
1525 return True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001526
David Pursehouse8a68ff92012-09-24 12:15:13 +09001527 all_refs = self.bare_ref.all
Anthony King7bdac712014-07-16 12:56:40 +01001528 if R_HEADS + name in all_refs:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001529 return GitCommand(self,
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001530 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001531 capture_stdout=True,
1532 capture_stderr=True).Wait() == 0
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001533
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001534 branch = self.GetBranch(name)
1535 branch.remote = self.GetRemote(self.remote.name)
Simran Basib9a1b732015-08-20 12:19:28 -07001536 branch.merge = branch_merge
1537 if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
1538 branch.merge = R_HEADS + branch_merge
David Pursehouse8a68ff92012-09-24 12:15:13 +09001539 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001540
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001541 if head.startswith(R_HEADS):
1542 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001543 head = all_refs[head]
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001544 except KeyError:
1545 head = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001546 if revid and head and revid == head:
1547 ref = os.path.join(self.gitdir, R_HEADS + name)
1548 try:
1549 os.makedirs(os.path.dirname(ref))
1550 except OSError:
1551 pass
1552 _lwrite(ref, '%s\n' % revid)
1553 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1554 'ref: %s%s\n' % (R_HEADS, name))
1555 branch.Save()
1556 return True
1557
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001558 if GitCommand(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001559 ['checkout', '-b', branch.name, revid],
Anthony King7bdac712014-07-16 12:56:40 +01001560 capture_stdout=True,
1561 capture_stderr=True).Wait() == 0:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001562 branch.Save()
1563 return True
1564 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001565
Wink Saville02d79452009-04-10 13:01:24 -07001566 def CheckoutBranch(self, name):
1567 """Checkout a local topic branch.
Doug Anderson3ba5f952011-04-07 12:51:04 -07001568
1569 Args:
1570 name: The name of the branch to checkout.
1571
1572 Returns:
1573 True if the checkout succeeded; False if it didn't; None if the branch
1574 didn't exist.
Wink Saville02d79452009-04-10 13:01:24 -07001575 """
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001576 rev = R_HEADS + name
1577 head = self.work_git.GetHead()
1578 if head == rev:
1579 # Already on the branch
1580 #
1581 return True
Wink Saville02d79452009-04-10 13:01:24 -07001582
David Pursehouse8a68ff92012-09-24 12:15:13 +09001583 all_refs = self.bare_ref.all
Wink Saville02d79452009-04-10 13:01:24 -07001584 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001585 revid = all_refs[rev]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001586 except KeyError:
1587 # Branch does not exist in this project
1588 #
Doug Anderson3ba5f952011-04-07 12:51:04 -07001589 return None
Wink Saville02d79452009-04-10 13:01:24 -07001590
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001591 if head.startswith(R_HEADS):
1592 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001593 head = all_refs[head]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001594 except KeyError:
1595 head = None
1596
1597 if head == revid:
1598 # Same revision; just update HEAD to point to the new
1599 # target branch, but otherwise take no other action.
1600 #
1601 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1602 'ref: %s%s\n' % (R_HEADS, name))
1603 return True
1604
1605 return GitCommand(self,
1606 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001607 capture_stdout=True,
1608 capture_stderr=True).Wait() == 0
Wink Saville02d79452009-04-10 13:01:24 -07001609
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001610 def AbandonBranch(self, name):
1611 """Destroy a local topic branch.
Doug Andersondafb1d62011-04-07 11:46:59 -07001612
1613 Args:
1614 name: The name of the branch to abandon.
1615
1616 Returns:
1617 True if the abandon succeeded; False if it didn't; None if the branch
1618 didn't exist.
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001619 """
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001620 rev = R_HEADS + name
David Pursehouse8a68ff92012-09-24 12:15:13 +09001621 all_refs = self.bare_ref.all
1622 if rev not in all_refs:
Doug Andersondafb1d62011-04-07 11:46:59 -07001623 # Doesn't exist
1624 return None
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001625
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001626 head = self.work_git.GetHead()
1627 if head == rev:
1628 # We can't destroy the branch while we are sitting
1629 # on it. Switch to a detached HEAD.
1630 #
David Pursehouse8a68ff92012-09-24 12:15:13 +09001631 head = all_refs[head]
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001632
David Pursehouse8a68ff92012-09-24 12:15:13 +09001633 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001634 if head == revid:
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001635 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1636 '%s\n' % revid)
1637 else:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001638 self._Checkout(revid, quiet=True)
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001639
1640 return GitCommand(self,
1641 ['branch', '-D', name],
Anthony King7bdac712014-07-16 12:56:40 +01001642 capture_stdout=True,
1643 capture_stderr=True).Wait() == 0
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001644
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001645 def PruneHeads(self):
1646 """Prune any topic branches already merged into upstream.
1647 """
1648 cb = self.CurrentBranch
1649 kill = []
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001650 left = self._allrefs
1651 for name in left.keys():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001652 if name.startswith(R_HEADS):
1653 name = name[len(R_HEADS):]
1654 if cb is None or name != cb:
1655 kill.append(name)
1656
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001657 rev = self.GetRevisionId(left)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001658 if cb is not None \
1659 and not self._revlist(HEAD + '...' + rev) \
Anthony King7bdac712014-07-16 12:56:40 +01001660 and not self.IsDirty(consider_untracked=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001661 self.work_git.DetachHead(HEAD)
1662 kill.append(cb)
1663
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001664 if kill:
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07001665 old = self.bare_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001666
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001667 try:
1668 self.bare_git.DetachHead(rev)
1669
1670 b = ['branch', '-d']
1671 b.extend(kill)
1672 b = GitCommand(self, b, bare=True,
1673 capture_stdout=True,
1674 capture_stderr=True)
1675 b.Wait()
1676 finally:
Dan Willemsen1a799d12015-12-15 13:40:05 -08001677 if ID_RE.match(old):
1678 self.bare_git.DetachHead(old)
1679 else:
1680 self.bare_git.SetHead(old)
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001681 left = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001682
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001683 for branch in kill:
1684 if (R_HEADS + branch) not in left:
1685 self.CleanPublishedCache()
1686 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001687
1688 if cb and cb not in kill:
1689 kill.append(cb)
Shawn O. Pearce7c6c64d2009-03-02 12:38:13 -08001690 kill.sort()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001691
1692 kept = []
1693 for branch in kill:
Anthony King7bdac712014-07-16 12:56:40 +01001694 if R_HEADS + branch in left:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001695 branch = self.GetBranch(branch)
1696 base = branch.LocalMerge
1697 if not base:
1698 base = rev
1699 kept.append(ReviewableBranch(self, branch, base))
1700 return kept
1701
1702
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001703# Submodule Management ##
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001704
1705 def GetRegisteredSubprojects(self):
1706 result = []
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001707
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001708 def rec(subprojects):
1709 if not subprojects:
1710 return
1711 result.extend(subprojects)
1712 for p in subprojects:
1713 rec(p.subprojects)
1714 rec(self.subprojects)
1715 return result
1716
1717 def _GetSubmodules(self):
1718 # Unfortunately we cannot call `git submodule status --recursive` here
1719 # because the working tree might not exist yet, and it cannot be used
1720 # without a working tree in its current implementation.
1721
1722 def get_submodules(gitdir, rev):
1723 # Parse .gitmodules for submodule sub_paths and sub_urls
1724 sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
1725 if not sub_paths:
1726 return []
1727 # Run `git ls-tree` to read SHAs of submodule object, which happen to be
1728 # revision of submodule repository
1729 sub_revs = git_ls_tree(gitdir, rev, sub_paths)
1730 submodules = []
1731 for sub_path, sub_url in zip(sub_paths, sub_urls):
1732 try:
1733 sub_rev = sub_revs[sub_path]
1734 except KeyError:
1735 # Ignore non-exist submodules
1736 continue
1737 submodules.append((sub_rev, sub_path, sub_url))
1738 return submodules
1739
1740 re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
1741 re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001742
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001743 def parse_gitmodules(gitdir, rev):
1744 cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
1745 try:
Anthony King7bdac712014-07-16 12:56:40 +01001746 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1747 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001748 except GitError:
1749 return [], []
1750 if p.Wait() != 0:
1751 return [], []
1752
1753 gitmodules_lines = []
1754 fd, temp_gitmodules_path = tempfile.mkstemp()
1755 try:
1756 os.write(fd, p.stdout)
1757 os.close(fd)
1758 cmd = ['config', '--file', temp_gitmodules_path, '--list']
Anthony King7bdac712014-07-16 12:56:40 +01001759 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1760 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001761 if p.Wait() != 0:
1762 return [], []
1763 gitmodules_lines = p.stdout.split('\n')
1764 except GitError:
1765 return [], []
1766 finally:
1767 os.remove(temp_gitmodules_path)
1768
1769 names = set()
1770 paths = {}
1771 urls = {}
1772 for line in gitmodules_lines:
1773 if not line:
1774 continue
1775 m = re_path.match(line)
1776 if m:
1777 names.add(m.group(1))
1778 paths[m.group(1)] = m.group(2)
1779 continue
1780 m = re_url.match(line)
1781 if m:
1782 names.add(m.group(1))
1783 urls[m.group(1)] = m.group(2)
1784 continue
1785 names = sorted(names)
1786 return ([paths.get(name, '') for name in names],
1787 [urls.get(name, '') for name in names])
1788
1789 def git_ls_tree(gitdir, rev, paths):
1790 cmd = ['ls-tree', rev, '--']
1791 cmd.extend(paths)
1792 try:
Anthony King7bdac712014-07-16 12:56:40 +01001793 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1794 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001795 except GitError:
1796 return []
1797 if p.Wait() != 0:
1798 return []
1799 objects = {}
1800 for line in p.stdout.split('\n'):
1801 if not line.strip():
1802 continue
1803 object_rev, object_path = line.split()[2:4]
1804 objects[object_path] = object_rev
1805 return objects
1806
1807 try:
1808 rev = self.GetRevisionId()
1809 except GitError:
1810 return []
1811 return get_submodules(self.gitdir, rev)
1812
1813 def GetDerivedSubprojects(self):
1814 result = []
1815 if not self.Exists:
1816 # If git repo does not exist yet, querying its submodules will
1817 # mess up its states; so return here.
1818 return result
1819 for rev, path, url in self._GetSubmodules():
1820 name = self.manifest.GetSubprojectName(self, path)
David James8d201162013-10-11 17:03:19 -07001821 relpath, worktree, gitdir, objdir = \
1822 self.manifest.GetSubprojectPaths(self, name, path)
1823 project = self.manifest.paths.get(relpath)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001824 if project:
1825 result.extend(project.GetDerivedSubprojects())
1826 continue
David James8d201162013-10-11 17:03:19 -07001827
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001828 remote = RemoteSpec(self.remote.name,
Anthony King7bdac712014-07-16 12:56:40 +01001829 url=url,
1830 review=self.remote.review,
1831 revision=self.remote.revision)
1832 subproject = Project(manifest=self.manifest,
1833 name=name,
1834 remote=remote,
1835 gitdir=gitdir,
1836 objdir=objdir,
1837 worktree=worktree,
1838 relpath=relpath,
1839 revisionExpr=self.revisionExpr,
1840 revisionId=rev,
1841 rebase=self.rebase,
1842 groups=self.groups,
1843 sync_c=self.sync_c,
1844 sync_s=self.sync_s,
1845 parent=self,
1846 is_derived=True)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001847 result.append(subproject)
1848 result.extend(subproject.GetDerivedSubprojects())
1849 return result
1850
1851
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001852# Direct Git Commands ##
Chris AtLee2fb64662014-01-16 21:32:33 -05001853 def _CheckForSha1(self):
1854 try:
1855 # if revision (sha or tag) is not present then following function
1856 # throws an error.
1857 self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
1858 return True
1859 except GitError:
1860 # There is no such persistent revision. We have to fetch it.
1861 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001862
Julien Campergue335f5ef2013-10-16 11:02:35 +02001863 def _FetchArchive(self, tarpath, cwd=None):
1864 cmd = ['archive', '-v', '-o', tarpath]
1865 cmd.append('--remote=%s' % self.remote.url)
1866 cmd.append('--prefix=%s/' % self.relpath)
1867 cmd.append(self.revisionExpr)
1868
1869 command = GitCommand(self, cmd, cwd=cwd,
1870 capture_stdout=True,
1871 capture_stderr=True)
1872
1873 if command.Wait() != 0:
1874 raise GitError('git archive %s: %s' % (self.name, command.stderr))
1875
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001876 def _RemoteFetch(self, name=None,
1877 current_branch_only=False,
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001878 initial=False,
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001879 quiet=False,
Mitchel Humpherys597868b2012-10-29 10:18:34 -07001880 alt_dir=None,
David Pursehouse74cfd272015-10-14 10:50:15 +09001881 no_tags=False,
1882 prune=False):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001883
1884 is_sha1 = False
1885 tag_name = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001886 depth = None
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001887
David Pursehouse9bc422f2014-04-15 10:28:56 +09001888 # The depth should not be used when fetching to a mirror because
1889 # it will result in a shallow repository that cannot be cloned or
1890 # fetched from.
1891 if not self.manifest.IsMirror:
1892 if self.clone_depth:
1893 depth = self.clone_depth
1894 else:
1895 depth = self.manifest.manifestProject.config.GetString('repo.depth')
Conley Owense4978cf2015-02-03 18:06:16 -08001896 # The repo project should never be synced with partial depth
1897 if self.relpath == '.repo/repo':
1898 depth = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001899
Shawn Pearce69e04d82014-01-29 12:48:54 -08001900 if depth:
1901 current_branch_only = True
1902
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001903 if ID_RE.match(self.revisionExpr) is not None:
1904 is_sha1 = True
1905
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001906 if current_branch_only:
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001907 if self.revisionExpr.startswith(R_TAGS):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001908 # this is a tag and its sha1 value should never change
1909 tag_name = self.revisionExpr[len(R_TAGS):]
1910
1911 if is_sha1 or tag_name is not None:
Chris AtLee2fb64662014-01-16 21:32:33 -05001912 if self._CheckForSha1():
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001913 return True
Bertrand SIMONNET3000cda2014-11-25 16:19:29 -08001914 if is_sha1 and not depth:
1915 # When syncing a specific commit and --depth is not set:
1916 # * if upstream is explicitly specified and is not a sha1, fetch only
1917 # upstream as users expect only upstream to be fetch.
1918 # Note: The commit might not be in upstream in which case the sync
1919 # will fail.
1920 # * otherwise, fetch all branches to make sure we end up with the
1921 # specific commit.
Aymen Bouaziz037040f2016-06-28 12:27:23 +02001922 if self.upstream:
1923 current_branch_only = not ID_RE.match(self.upstream)
1924 else:
1925 current_branch_only = False
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001926
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001927 if not name:
1928 name = self.remote.name
Shawn O. Pearcefb231612009-04-10 18:53:46 -07001929
1930 ssh_proxy = False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001931 remote = self.GetRemote(name)
1932 if remote.PreConnectFetch():
Shawn O. Pearcefb231612009-04-10 18:53:46 -07001933 ssh_proxy = True
1934
Shawn O. Pearce88443382010-10-08 10:02:09 +02001935 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001936 if alt_dir and 'objects' == os.path.basename(alt_dir):
1937 ref_dir = os.path.dirname(alt_dir)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001938 packed_refs = os.path.join(self.gitdir, 'packed-refs')
1939 remote = self.GetRemote(name)
1940
David Pursehouse8a68ff92012-09-24 12:15:13 +09001941 all_refs = self.bare_ref.all
1942 ids = set(all_refs.values())
Shawn O. Pearce88443382010-10-08 10:02:09 +02001943 tmp = set()
1944
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301945 for r, ref_id in GitRefs(ref_dir).all.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09001946 if r not in all_refs:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001947 if r.startswith(R_TAGS) or remote.WritesTo(r):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001948 all_refs[r] = ref_id
1949 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001950 continue
1951
David Pursehouse8a68ff92012-09-24 12:15:13 +09001952 if ref_id in ids:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001953 continue
1954
David Pursehouse8a68ff92012-09-24 12:15:13 +09001955 r = 'refs/_alt/%s' % ref_id
1956 all_refs[r] = ref_id
1957 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001958 tmp.add(r)
1959
Shawn O. Pearce88443382010-10-08 10:02:09 +02001960 tmp_packed = ''
1961 old_packed = ''
1962
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301963 for r in sorted(all_refs):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001964 line = '%s %s\n' % (all_refs[r], r)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001965 tmp_packed += line
1966 if r not in tmp:
1967 old_packed += line
1968
1969 _lwrite(packed_refs, tmp_packed)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001970 else:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001971 alt_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02001972
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001973 cmd = ['fetch']
Doug Anderson30d45292011-05-04 15:01:04 -07001974
Conley Owensf97e8382015-01-21 11:12:46 -08001975 if depth:
Doug Anderson30d45292011-05-04 15:01:04 -07001976 cmd.append('--depth=%s' % depth)
Dan Willemseneeab6862015-08-03 13:11:53 -07001977 else:
1978 # If this repo has shallow objects, then we don't know which refs have
1979 # shallow objects or not. Tell git to unshallow all fetched refs. Don't
1980 # do this with projects that don't have shallow objects, since it is less
1981 # efficient.
1982 if os.path.exists(os.path.join(self.gitdir, 'shallow')):
1983 cmd.append('--depth=2147483647')
Doug Anderson30d45292011-05-04 15:01:04 -07001984
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001985 if quiet:
1986 cmd.append('--quiet')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001987 if not self.worktree:
1988 cmd.append('--update-head-ok')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001989 cmd.append(name)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001990
Mitchel Humpherys26c45a72014-03-10 14:21:59 -07001991 # If using depth then we should not get all the tags since they may
1992 # be outside of the depth.
1993 if no_tags or depth:
1994 cmd.append('--no-tags')
1995 else:
1996 cmd.append('--tags')
1997
David Pursehouse74cfd272015-10-14 10:50:15 +09001998 if prune:
1999 cmd.append('--prune')
2000
Conley Owens80b87fe2014-05-09 17:13:44 -07002001 spec = []
Brian Harring14a66742012-09-28 20:21:57 -07002002 if not current_branch_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002003 # Fetch whole repo
Conley Owens80b87fe2014-05-09 17:13:44 -07002004 spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002005 elif tag_name is not None:
Conley Owens80b87fe2014-05-09 17:13:44 -07002006 spec.append('tag')
2007 spec.append(tag_name)
Nasser Grainawi04e52d62014-09-30 13:34:52 -06002008
David Pursehouse403b64e2015-04-27 10:41:33 +09002009 if not self.manifest.IsMirror:
2010 branch = self.revisionExpr
Kevin Degi679bac42015-06-22 15:31:26 -06002011 if is_sha1 and depth and git_require((1, 8, 3)):
David Pursehouse403b64e2015-04-27 10:41:33 +09002012 # Shallow checkout of a specific commit, fetch from that commit and not
2013 # the heads only as the commit might be deeper in the history.
2014 spec.append(branch)
2015 else:
2016 if is_sha1:
2017 branch = self.upstream
2018 if branch is not None and branch.strip():
2019 if not branch.startswith('refs/'):
2020 branch = R_HEADS + branch
2021 spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
Conley Owens80b87fe2014-05-09 17:13:44 -07002022 cmd.extend(spec)
2023
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002024 ok = False
David Pursehouse8a68ff92012-09-24 12:15:13 +09002025 for _i in range(2):
John L. Villalovos9c76f672015-03-16 20:49:10 -07002026 gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy)
John L. Villalovos126e2982015-01-29 21:58:12 -08002027 ret = gitcmd.Wait()
Brian Harring14a66742012-09-28 20:21:57 -07002028 if ret == 0:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002029 ok = True
2030 break
John L. Villalovos126e2982015-01-29 21:58:12 -08002031 # If needed, run the 'git remote prune' the first time through the loop
2032 elif (not _i and
2033 "error:" in gitcmd.stderr and
2034 "git remote prune" in gitcmd.stderr):
2035 prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
John L. Villalovos9c76f672015-03-16 20:49:10 -07002036 ssh_proxy=ssh_proxy)
John L. Villalovose30f46b2015-02-25 14:27:02 -08002037 ret = prunecmd.Wait()
John L. Villalovose30f46b2015-02-25 14:27:02 -08002038 if ret:
John L. Villalovos126e2982015-01-29 21:58:12 -08002039 break
2040 continue
Brian Harring14a66742012-09-28 20:21:57 -07002041 elif current_branch_only and is_sha1 and ret == 128:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002042 # Exit code 128 means "couldn't find the ref you asked for"; if we're
2043 # in sha1 mode, we just tried sync'ing from the upstream field; it
2044 # doesn't exist, thus abort the optimization attempt and do a full sync.
Brian Harring14a66742012-09-28 20:21:57 -07002045 break
Colin Crossc4b301f2015-05-13 00:10:02 -07002046 elif ret < 0:
2047 # Git died with a signal, exit immediately
2048 break
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002049 time.sleep(random.randint(30, 45))
Shawn O. Pearce88443382010-10-08 10:02:09 +02002050
2051 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002052 if alt_dir:
Shawn O. Pearce88443382010-10-08 10:02:09 +02002053 if old_packed != '':
2054 _lwrite(packed_refs, old_packed)
2055 else:
2056 os.remove(packed_refs)
2057 self.bare_git.pack_refs('--all', '--prune')
Brian Harring14a66742012-09-28 20:21:57 -07002058
2059 if is_sha1 and current_branch_only and self.upstream:
2060 # We just synced the upstream given branch; verify we
2061 # got what we wanted, else trigger a second run of all
2062 # refs.
Chris AtLee2fb64662014-01-16 21:32:33 -05002063 if not self._CheckForSha1():
Kevin Degi679bac42015-06-22 15:31:26 -06002064 if not depth:
2065 # Avoid infinite recursion when depth is True (since depth implies
2066 # current_branch_only)
2067 return self._RemoteFetch(name=name, current_branch_only=False,
2068 initial=False, quiet=quiet, alt_dir=alt_dir)
2069 if self.clone_depth:
2070 self.clone_depth = None
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002071 return self._RemoteFetch(name=name,
2072 current_branch_only=current_branch_only,
Kevin Degi679bac42015-06-22 15:31:26 -06002073 initial=False, quiet=quiet, alt_dir=alt_dir)
Brian Harring14a66742012-09-28 20:21:57 -07002074
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002075 return ok
Shawn O. Pearce88443382010-10-08 10:02:09 +02002076
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002077 def _ApplyCloneBundle(self, initial=False, quiet=False):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002078 if initial and \
2079 (self.manifest.manifestProject.config.GetString('repo.depth') or
2080 self.clone_depth):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002081 return False
2082
2083 remote = self.GetRemote(self.remote.name)
2084 bundle_url = remote.url + '/clone.bundle'
2085 bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002086 if GetSchemeFromUrl(bundle_url) not in ('http', 'https',
2087 'persistent-http',
2088 'persistent-https'):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002089 return False
2090
2091 bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
2092 bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
2093
2094 exist_dst = os.path.exists(bundle_dst)
2095 exist_tmp = os.path.exists(bundle_tmp)
2096
2097 if not initial and not exist_dst and not exist_tmp:
2098 return False
2099
2100 if not exist_dst:
2101 exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
2102 if not exist_dst:
2103 return False
2104
2105 cmd = ['fetch']
2106 if quiet:
2107 cmd.append('--quiet')
2108 if not self.worktree:
2109 cmd.append('--update-head-ok')
2110 cmd.append(bundle_dst)
2111 for f in remote.fetch:
2112 cmd.append(str(f))
2113 cmd.append('refs/tags/*:refs/tags/*')
2114
2115 ok = GitCommand(self, cmd, bare=True).Wait() == 0
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002116 if os.path.exists(bundle_dst):
2117 os.remove(bundle_dst)
2118 if os.path.exists(bundle_tmp):
2119 os.remove(bundle_tmp)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002120 return ok
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002121
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002122 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002123 if os.path.exists(dstPath):
2124 os.remove(dstPath)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002125
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002126 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002127 if quiet:
2128 cmd += ['--silent']
2129 if os.path.exists(tmpPath):
2130 size = os.stat(tmpPath).st_size
2131 if size >= 1024:
2132 cmd += ['--continue-at', '%d' % (size,)]
2133 else:
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002134 os.remove(tmpPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002135 if 'http_proxy' in os.environ and 'darwin' == sys.platform:
2136 cmd += ['--proxy', os.environ['http_proxy']]
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002137 with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy):
Dave Borowitz137d0132015-01-02 11:12:54 -08002138 if cookiefile:
Dave Borowitz4abf8e62015-01-02 11:39:04 -08002139 cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
Dave Borowitz137d0132015-01-02 11:12:54 -08002140 if srcUrl.startswith('persistent-'):
2141 srcUrl = srcUrl[len('persistent-'):]
2142 cmd += [srcUrl]
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002143
Dave Borowitz137d0132015-01-02 11:12:54 -08002144 if IsTrace():
2145 Trace('%s', ' '.join(cmd))
2146 try:
2147 proc = subprocess.Popen(cmd)
2148 except OSError:
2149 return False
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002150
Dave Borowitz137d0132015-01-02 11:12:54 -08002151 curlret = proc.wait()
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002152
Dave Borowitz137d0132015-01-02 11:12:54 -08002153 if curlret == 22:
2154 # From curl man page:
2155 # 22: HTTP page not retrieved. The requested url was not found or
2156 # returned another error with the HTTP error code being 400 or above.
2157 # This return code only appears if -f, --fail is used.
2158 if not quiet:
2159 print("Server does not provide clone.bundle; ignoring.",
2160 file=sys.stderr)
2161 return False
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002162
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002163 if os.path.exists(tmpPath):
Kris Giesingc8d882a2014-12-23 13:02:32 -08002164 if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002165 os.rename(tmpPath, dstPath)
2166 return True
2167 else:
2168 os.remove(tmpPath)
2169 return False
2170 else:
2171 return False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002172
Kris Giesingc8d882a2014-12-23 13:02:32 -08002173 def _IsValidBundle(self, path, quiet):
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002174 try:
2175 with open(path) as f:
2176 if f.read(16) == '# v2 git bundle\n':
2177 return True
2178 else:
Kris Giesingc8d882a2014-12-23 13:02:32 -08002179 if not quiet:
2180 print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002181 return False
2182 except OSError:
2183 return False
2184
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002185 def _Checkout(self, rev, quiet=False):
2186 cmd = ['checkout']
2187 if quiet:
2188 cmd.append('-q')
2189 cmd.append(rev)
2190 cmd.append('--')
2191 if GitCommand(self, cmd).Wait() != 0:
2192 if self._allrefs:
2193 raise GitError('%s checkout %s ' % (self.name, rev))
2194
Anthony King7bdac712014-07-16 12:56:40 +01002195 def _CherryPick(self, rev):
Pierre Tardye5a21222011-03-24 16:28:18 +01002196 cmd = ['cherry-pick']
2197 cmd.append(rev)
2198 cmd.append('--')
2199 if GitCommand(self, cmd).Wait() != 0:
2200 if self._allrefs:
2201 raise GitError('%s cherry-pick %s ' % (self.name, rev))
2202
Anthony King7bdac712014-07-16 12:56:40 +01002203 def _Revert(self, rev):
Erwan Mahea94f1622011-08-19 13:56:09 +02002204 cmd = ['revert']
2205 cmd.append('--no-edit')
2206 cmd.append(rev)
2207 cmd.append('--')
2208 if GitCommand(self, cmd).Wait() != 0:
2209 if self._allrefs:
2210 raise GitError('%s revert %s ' % (self.name, rev))
2211
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002212 def _ResetHard(self, rev, quiet=True):
2213 cmd = ['reset', '--hard']
2214 if quiet:
2215 cmd.append('-q')
2216 cmd.append(rev)
2217 if GitCommand(self, cmd).Wait() != 0:
2218 raise GitError('%s reset --hard %s ' % (self.name, rev))
2219
Anthony King7bdac712014-07-16 12:56:40 +01002220 def _Rebase(self, upstream, onto=None):
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002221 cmd = ['rebase']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002222 if onto is not None:
2223 cmd.extend(['--onto', onto])
2224 cmd.append(upstream)
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002225 if GitCommand(self, cmd).Wait() != 0:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002226 raise GitError('%s rebase %s ' % (self.name, upstream))
2227
Pierre Tardy3d125942012-05-04 12:18:12 +02002228 def _FastForward(self, head, ffonly=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002229 cmd = ['merge', head]
Pierre Tardy3d125942012-05-04 12:18:12 +02002230 if ffonly:
2231 cmd.append("--ff-only")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002232 if GitCommand(self, cmd).Wait() != 0:
2233 raise GitError('%s merge %s ' % (self.name, head))
2234
Kevin Degiabaa7f32014-11-12 11:27:45 -07002235 def _InitGitDir(self, mirror_git=None, force_sync=False):
Kevin Degi384b3c52014-10-16 16:02:58 -06002236 init_git_dir = not os.path.exists(self.gitdir)
2237 init_obj_dir = not os.path.exists(self.objdir)
Kevin Degib1a07b82015-07-27 13:33:43 -06002238 try:
2239 # Initialize the bare repository, which contains all of the objects.
2240 if init_obj_dir:
2241 os.makedirs(self.objdir)
2242 self.bare_objdir.init()
David James8d201162013-10-11 17:03:19 -07002243
Kevin Degib1a07b82015-07-27 13:33:43 -06002244 # If we have a separate directory to hold refs, initialize it as well.
2245 if self.objdir != self.gitdir:
2246 if init_git_dir:
2247 os.makedirs(self.gitdir)
2248
2249 if init_obj_dir or init_git_dir:
2250 self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
2251 copy_all=True)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002252 try:
2253 self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
2254 except GitError as e:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002255 if force_sync:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002256 print("Retrying clone after deleting %s" %
2257 self.gitdir, file=sys.stderr)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002258 try:
2259 shutil.rmtree(os.path.realpath(self.gitdir))
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002260 if self.worktree and os.path.exists(os.path.realpath
2261 (self.worktree)):
Kevin Degiabaa7f32014-11-12 11:27:45 -07002262 shutil.rmtree(os.path.realpath(self.worktree))
2263 return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
2264 except:
2265 raise e
2266 raise e
Kevin Degib1a07b82015-07-27 13:33:43 -06002267
Kevin Degi384b3c52014-10-16 16:02:58 -06002268 if init_git_dir:
Kevin Degib1a07b82015-07-27 13:33:43 -06002269 mp = self.manifest.manifestProject
2270 ref_dir = mp.config.GetString('repo.reference') or ''
Kevin Degi384b3c52014-10-16 16:02:58 -06002271
Kevin Degib1a07b82015-07-27 13:33:43 -06002272 if ref_dir or mirror_git:
2273 if not mirror_git:
2274 mirror_git = os.path.join(ref_dir, self.name + '.git')
2275 repo_git = os.path.join(ref_dir, '.repo', 'projects',
2276 self.relpath + '.git')
Shawn O. Pearce2816d4f2009-03-03 17:53:18 -08002277
Kevin Degib1a07b82015-07-27 13:33:43 -06002278 if os.path.exists(mirror_git):
2279 ref_dir = mirror_git
Shawn O. Pearce88443382010-10-08 10:02:09 +02002280
Kevin Degib1a07b82015-07-27 13:33:43 -06002281 elif os.path.exists(repo_git):
2282 ref_dir = repo_git
Shawn O. Pearce88443382010-10-08 10:02:09 +02002283
Kevin Degib1a07b82015-07-27 13:33:43 -06002284 else:
2285 ref_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02002286
Kevin Degib1a07b82015-07-27 13:33:43 -06002287 if ref_dir:
2288 _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
2289 os.path.join(ref_dir, 'objects') + '\n')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002290
Kevin Degib1a07b82015-07-27 13:33:43 -06002291 self._UpdateHooks()
2292
2293 m = self.manifest.manifestProject.config
2294 for key in ['user.name', 'user.email']:
2295 if m.Has(key, include_defaults=False):
2296 self.config.SetString(key, m.GetString(key))
David Pursehouse76a4a9d2016-08-16 12:11:12 +09002297 self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
Kevin Degib1a07b82015-07-27 13:33:43 -06002298 if self.manifest.IsMirror:
2299 self.config.SetString('core.bare', 'true')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002300 else:
Kevin Degib1a07b82015-07-27 13:33:43 -06002301 self.config.SetString('core.bare', None)
2302 except Exception:
2303 if init_obj_dir and os.path.exists(self.objdir):
2304 shutil.rmtree(self.objdir)
2305 if init_git_dir and os.path.exists(self.gitdir):
2306 shutil.rmtree(self.gitdir)
2307 raise
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002308
Jimmie Westera0444582012-10-24 13:44:42 +02002309 def _UpdateHooks(self):
2310 if os.path.exists(self.gitdir):
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002311 self._InitHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002312
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002313 def _InitHooks(self):
Jesse Hall672cc492013-11-27 11:17:13 -08002314 hooks = os.path.realpath(self._gitdir_path('hooks'))
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002315 if not os.path.exists(hooks):
2316 os.makedirs(hooks)
Jonathan Nieder93719792015-03-17 11:29:58 -07002317 for stock_hook in _ProjectHooks():
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002318 name = os.path.basename(stock_hook)
2319
Victor Boivie65e0f352011-04-18 11:23:29 +02002320 if name in ('commit-msg',) and not self.remote.review \
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002321 and self is not self.manifest.manifestProject:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002322 # Don't install a Gerrit Code Review hook if this
2323 # project does not appear to use it for reviews.
2324 #
Victor Boivie65e0f352011-04-18 11:23:29 +02002325 # Since the manifest project is one of those, but also
2326 # managed through gerrit, it's excluded
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002327 continue
2328
2329 dst = os.path.join(hooks, name)
2330 if os.path.islink(dst):
2331 continue
2332 if os.path.exists(dst):
2333 if filecmp.cmp(stock_hook, dst, shallow=False):
2334 os.remove(dst)
2335 else:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002336 _warn("%s: Not replacing locally modified %s hook",
2337 self.relpath, name)
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002338 continue
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002339 try:
Mickaël Salaünb9477bc2012-08-05 13:39:26 +02002340 os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
Sarah Owensa5be53f2012-09-09 15:37:57 -07002341 except OSError as e:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002342 if e.errno == errno.EPERM:
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002343 raise GitError('filesystem must support symlinks')
2344 else:
2345 raise
2346
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002347 def _InitRemote(self):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002348 if self.remote.url:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002349 remote = self.GetRemote(self.remote.name)
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002350 remote.url = self.remote.url
2351 remote.review = self.remote.review
2352 remote.projectname = self.name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002353
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002354 if self.worktree:
2355 remote.ResetFetch(mirror=False)
2356 else:
2357 remote.ResetFetch(mirror=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002358 remote.Save()
2359
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002360 def _InitMRef(self):
2361 if self.manifest.branch:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002362 self._InitAnyMRef(R_M + self.manifest.branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002363
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002364 def _InitMirrorHead(self):
Shawn O. Pearcefe200ee2009-06-01 15:28:21 -07002365 self._InitAnyMRef(HEAD)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002366
2367 def _InitAnyMRef(self, ref):
2368 cur = self.bare_ref.symref(ref)
2369
2370 if self.revisionId:
2371 if cur != '' or self.bare_ref.get(ref) != self.revisionId:
2372 msg = 'manifest set to %s' % self.revisionId
2373 dst = self.revisionId + '^0'
Anthony King7bdac712014-07-16 12:56:40 +01002374 self.bare_git.UpdateRef(ref, dst, message=msg, detach=True)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002375 else:
2376 remote = self.GetRemote(self.remote.name)
2377 dst = remote.ToLocal(self.revisionExpr)
2378 if cur != dst:
2379 msg = 'manifest set to %s' % self.revisionExpr
2380 self.bare_git.symbolic_ref('-m', msg, ref, dst)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002381
Kevin Degi384b3c52014-10-16 16:02:58 -06002382 def _CheckDirReference(self, srcdir, destdir, share_refs):
Dan Willemsenbdb866e2016-04-05 17:22:02 -07002383 symlink_files = self.shareable_files[:]
2384 symlink_dirs = self.shareable_dirs[:]
Kevin Degi384b3c52014-10-16 16:02:58 -06002385 if share_refs:
2386 symlink_files += self.working_tree_files
2387 symlink_dirs += self.working_tree_dirs
2388 to_symlink = symlink_files + symlink_dirs
2389 for name in set(to_symlink):
2390 dst = os.path.realpath(os.path.join(destdir, name))
2391 if os.path.lexists(dst):
2392 src = os.path.realpath(os.path.join(srcdir, name))
2393 # Fail if the links are pointing to the wrong place
2394 if src != dst:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002395 raise GitError('--force-sync not enabled; cannot overwrite a local '
Simon Ruggierf9b76832015-07-31 17:18:34 -04002396 'work tree. If you\'re comfortable with the '
2397 'possibility of losing the work tree\'s git metadata,'
2398 ' use `repo sync --force-sync {0}` to '
2399 'proceed.'.format(self.relpath))
Kevin Degi384b3c52014-10-16 16:02:58 -06002400
David James8d201162013-10-11 17:03:19 -07002401 def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
2402 """Update |dotgit| to reference |gitdir|, using symlinks where possible.
2403
2404 Args:
2405 gitdir: The bare git repository. Must already be initialized.
2406 dotgit: The repository you would like to initialize.
2407 share_refs: If true, |dotgit| will store its refs under |gitdir|.
2408 Only one work tree can store refs under a given |gitdir|.
2409 copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
2410 This saves you the effort of initializing |dotgit| yourself.
2411 """
Dan Willemsenbdb866e2016-04-05 17:22:02 -07002412 symlink_files = self.shareable_files[:]
2413 symlink_dirs = self.shareable_dirs[:]
David James8d201162013-10-11 17:03:19 -07002414 if share_refs:
Kevin Degi384b3c52014-10-16 16:02:58 -06002415 symlink_files += self.working_tree_files
2416 symlink_dirs += self.working_tree_dirs
David James8d201162013-10-11 17:03:19 -07002417 to_symlink = symlink_files + symlink_dirs
2418
2419 to_copy = []
2420 if copy_all:
2421 to_copy = os.listdir(gitdir)
2422
Dan Willemsen2a3e1522015-07-30 20:43:33 -07002423 dotgit = os.path.realpath(dotgit)
David James8d201162013-10-11 17:03:19 -07002424 for name in set(to_copy).union(to_symlink):
2425 try:
2426 src = os.path.realpath(os.path.join(gitdir, name))
Dan Willemsen2a3e1522015-07-30 20:43:33 -07002427 dst = os.path.join(dotgit, name)
David James8d201162013-10-11 17:03:19 -07002428
Kevin Degi384b3c52014-10-16 16:02:58 -06002429 if os.path.lexists(dst):
2430 continue
David James8d201162013-10-11 17:03:19 -07002431
2432 # If the source dir doesn't exist, create an empty dir.
2433 if name in symlink_dirs and not os.path.lexists(src):
2434 os.makedirs(src)
2435
Conley Owens80b87fe2014-05-09 17:13:44 -07002436 # If the source file doesn't exist, ensure the destination
2437 # file doesn't either.
2438 if name in symlink_files and not os.path.lexists(src):
2439 try:
2440 os.remove(dst)
2441 except OSError:
2442 pass
2443
David James8d201162013-10-11 17:03:19 -07002444 if name in to_symlink:
2445 os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
2446 elif copy_all and not os.path.islink(dst):
2447 if os.path.isdir(src):
2448 shutil.copytree(src, dst)
2449 elif os.path.isfile(src):
2450 shutil.copy(src, dst)
2451 except OSError as e:
2452 if e.errno == errno.EPERM:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002453 raise DownloadError('filesystem must support symlinks')
David James8d201162013-10-11 17:03:19 -07002454 else:
2455 raise
2456
Kevin Degiabaa7f32014-11-12 11:27:45 -07002457 def _InitWorkTree(self, force_sync=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002458 dotgit = os.path.join(self.worktree, '.git')
Kevin Degi384b3c52014-10-16 16:02:58 -06002459 init_dotgit = not os.path.exists(dotgit)
Kevin Degib1a07b82015-07-27 13:33:43 -06002460 try:
2461 if init_dotgit:
2462 os.makedirs(dotgit)
2463 self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
2464 copy_all=False)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002465
Kevin Degiabaa7f32014-11-12 11:27:45 -07002466 try:
2467 self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
2468 except GitError as e:
2469 if force_sync:
2470 try:
2471 shutil.rmtree(dotgit)
2472 return self._InitWorkTree(force_sync=False)
2473 except:
2474 raise e
2475 raise e
Kevin Degi384b3c52014-10-16 16:02:58 -06002476
Kevin Degib1a07b82015-07-27 13:33:43 -06002477 if init_dotgit:
2478 _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002479
Kevin Degib1a07b82015-07-27 13:33:43 -06002480 cmd = ['read-tree', '--reset', '-u']
2481 cmd.append('-v')
2482 cmd.append(HEAD)
2483 if GitCommand(self, cmd).Wait() != 0:
2484 raise GitError("cannot initialize work tree")
Victor Boivie0960b5b2010-11-26 13:42:13 +01002485
Kevin Degib1a07b82015-07-27 13:33:43 -06002486 self._CopyAndLinkFiles()
2487 except Exception:
2488 if init_dotgit:
2489 shutil.rmtree(dotgit)
2490 raise
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002491
2492 def _gitdir_path(self, path):
David James8d201162013-10-11 17:03:19 -07002493 return os.path.realpath(os.path.join(self.gitdir, path))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002494
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002495 def _revlist(self, *args, **kw):
2496 a = []
2497 a.extend(args)
2498 a.append('--')
2499 return self.work_git.rev_list(*a, **kw)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002500
2501 @property
2502 def _allrefs(self):
Shawn O. Pearced237b692009-04-17 18:49:50 -07002503 return self.bare_ref.all
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002504
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002505 def _getLogs(self, rev1, rev2, oneline=False, color=True, pretty_format=None):
Julien Camperguedd654222014-01-09 16:21:37 +01002506 """Get logs between two revisions of this project."""
2507 comp = '..'
2508 if rev1:
2509 revs = [rev1]
2510 if rev2:
2511 revs.extend([comp, rev2])
2512 cmd = ['log', ''.join(revs)]
2513 out = DiffColoring(self.config)
2514 if out.is_on and color:
2515 cmd.append('--color')
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002516 if pretty_format is not None:
2517 cmd.append('--pretty=format:%s' % pretty_format)
Julien Camperguedd654222014-01-09 16:21:37 +01002518 if oneline:
2519 cmd.append('--oneline')
2520
2521 try:
2522 log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
2523 if log.Wait() == 0:
2524 return log.stdout
2525 except GitError:
2526 # worktree may not exist if groups changed for example. In that case,
2527 # try in gitdir instead.
2528 if not os.path.exists(self.worktree):
2529 return self.bare_git.log(*cmd[1:])
2530 else:
2531 raise
2532 return None
2533
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002534 def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True,
2535 pretty_format=None):
Julien Camperguedd654222014-01-09 16:21:37 +01002536 """Get the list of logs from this revision to given revisionId"""
2537 logs = {}
2538 selfId = self.GetRevisionId(self._allrefs)
2539 toId = toProject.GetRevisionId(toProject._allrefs)
2540
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002541 logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color,
2542 pretty_format=pretty_format)
2543 logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color,
2544 pretty_format=pretty_format)
Julien Camperguedd654222014-01-09 16:21:37 +01002545 return logs
2546
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002547 class _GitGetByExec(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002548
David James8d201162013-10-11 17:03:19 -07002549 def __init__(self, project, bare, gitdir):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002550 self._project = project
2551 self._bare = bare
David James8d201162013-10-11 17:03:19 -07002552 self._gitdir = gitdir
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002553
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002554 def LsOthers(self):
2555 p = GitCommand(self._project,
2556 ['ls-files',
2557 '-z',
2558 '--others',
2559 '--exclude-standard'],
Anthony King7bdac712014-07-16 12:56:40 +01002560 bare=False,
David James8d201162013-10-11 17:03:19 -07002561 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002562 capture_stdout=True,
2563 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002564 if p.Wait() == 0:
2565 out = p.stdout
2566 if out:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002567 # Backslash is not anomalous
David Pursehouse1d947b32012-10-25 12:23:11 +09002568 return out[:-1].split('\0') # pylint: disable=W1401
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002569 return []
2570
2571 def DiffZ(self, name, *args):
2572 cmd = [name]
2573 cmd.append('-z')
2574 cmd.extend(args)
2575 p = GitCommand(self._project,
2576 cmd,
David James8d201162013-10-11 17:03:19 -07002577 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002578 bare=False,
2579 capture_stdout=True,
2580 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002581 try:
2582 out = p.process.stdout.read()
2583 r = {}
2584 if out:
David Pursehouse1d947b32012-10-25 12:23:11 +09002585 out = iter(out[:-1].split('\0')) # pylint: disable=W1401
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002586 while out:
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002587 try:
Anthony King2cd1f042014-05-05 21:24:05 +01002588 info = next(out)
2589 path = next(out)
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002590 except StopIteration:
2591 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002592
2593 class _Info(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002594
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002595 def __init__(self, path, omode, nmode, oid, nid, state):
2596 self.path = path
2597 self.src_path = None
2598 self.old_mode = omode
2599 self.new_mode = nmode
2600 self.old_id = oid
2601 self.new_id = nid
2602
2603 if len(state) == 1:
2604 self.status = state
2605 self.level = None
2606 else:
2607 self.status = state[:1]
2608 self.level = state[1:]
2609 while self.level.startswith('0'):
2610 self.level = self.level[1:]
2611
2612 info = info[1:].split(' ')
David Pursehouse8f62fb72012-11-14 12:09:38 +09002613 info = _Info(path, *info)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002614 if info.status in ('R', 'C'):
2615 info.src_path = info.path
Anthony King2cd1f042014-05-05 21:24:05 +01002616 info.path = next(out)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002617 r[info.path] = info
2618 return r
2619 finally:
2620 p.Wait()
2621
2622 def GetHead(self):
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002623 if self._bare:
2624 path = os.path.join(self._project.gitdir, HEAD)
2625 else:
2626 path = os.path.join(self._project.worktree, '.git', HEAD)
Conley Owens75ee0572012-11-15 17:33:11 -08002627 try:
2628 fd = open(path, 'rb')
Dan Sandler53e902a2014-03-09 13:20:02 -04002629 except IOError as e:
2630 raise NoManifestException(path, str(e))
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -07002631 try:
2632 line = fd.read()
2633 finally:
2634 fd.close()
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302635 try:
2636 line = line.decode()
2637 except AttributeError:
2638 pass
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002639 if line.startswith('ref: '):
2640 return line[5:-1]
2641 return line[:-1]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002642
2643 def SetHead(self, ref, message=None):
2644 cmdv = []
2645 if message is not None:
2646 cmdv.extend(['-m', message])
2647 cmdv.append(HEAD)
2648 cmdv.append(ref)
2649 self.symbolic_ref(*cmdv)
2650
2651 def DetachHead(self, new, message=None):
2652 cmdv = ['--no-deref']
2653 if message is not None:
2654 cmdv.extend(['-m', message])
2655 cmdv.append(HEAD)
2656 cmdv.append(new)
2657 self.update_ref(*cmdv)
2658
2659 def UpdateRef(self, name, new, old=None,
2660 message=None,
2661 detach=False):
2662 cmdv = []
2663 if message is not None:
2664 cmdv.extend(['-m', message])
2665 if detach:
2666 cmdv.append('--no-deref')
2667 cmdv.append(name)
2668 cmdv.append(new)
2669 if old is not None:
2670 cmdv.append(old)
2671 self.update_ref(*cmdv)
2672
2673 def DeleteRef(self, name, old=None):
2674 if not old:
2675 old = self.rev_parse(name)
2676 self.update_ref('-d', name, old)
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07002677 self._project.bare_ref.deleted(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002678
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002679 def rev_list(self, *args, **kw):
2680 if 'format' in kw:
2681 cmdv = ['log', '--pretty=format:%s' % kw['format']]
2682 else:
2683 cmdv = ['rev-list']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002684 cmdv.extend(args)
2685 p = GitCommand(self._project,
2686 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002687 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002688 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002689 capture_stdout=True,
2690 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002691 r = []
2692 for line in p.process.stdout:
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002693 if line[-1] == '\n':
2694 line = line[:-1]
2695 r.append(line)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002696 if p.Wait() != 0:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002697 raise GitError('%s rev-list %s: %s' %
2698 (self._project.name, str(args), p.stderr))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002699 return r
2700
2701 def __getattr__(self, name):
Doug Anderson37282b42011-03-04 11:54:18 -08002702 """Allow arbitrary git commands using pythonic syntax.
2703
2704 This allows you to do things like:
2705 git_obj.rev_parse('HEAD')
2706
2707 Since we don't have a 'rev_parse' method defined, the __getattr__ will
2708 run. We'll replace the '_' with a '-' and try to run a git command.
Dave Borowitz091f8932012-10-23 17:01:04 -07002709 Any other positional arguments will be passed to the git command, and the
2710 following keyword arguments are supported:
2711 config: An optional dict of git config options to be passed with '-c'.
Doug Anderson37282b42011-03-04 11:54:18 -08002712
2713 Args:
2714 name: The name of the git command to call. Any '_' characters will
2715 be replaced with '-'.
2716
2717 Returns:
2718 A callable object that will try to call git with the named command.
2719 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002720 name = name.replace('_', '-')
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002721
Dave Borowitz091f8932012-10-23 17:01:04 -07002722 def runner(*args, **kwargs):
2723 cmdv = []
2724 config = kwargs.pop('config', None)
2725 for k in kwargs:
2726 raise TypeError('%s() got an unexpected keyword argument %r'
2727 % (name, k))
2728 if config is not None:
Dave Borowitzb42b4742012-10-31 12:27:27 -07002729 if not git_require((1, 7, 2)):
2730 raise ValueError('cannot set config on command line for %s()'
2731 % name)
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302732 for k, v in config.items():
Dave Borowitz091f8932012-10-23 17:01:04 -07002733 cmdv.append('-c')
2734 cmdv.append('%s=%s' % (k, v))
2735 cmdv.append(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002736 cmdv.extend(args)
2737 p = GitCommand(self._project,
2738 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002739 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002740 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002741 capture_stdout=True,
2742 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002743 if p.Wait() != 0:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002744 raise GitError('%s %s: %s' %
2745 (self._project.name, name, p.stderr))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002746 r = p.stdout
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302747 try:
Conley Owensedd01512013-09-26 12:59:58 -07002748 r = r.decode('utf-8')
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302749 except AttributeError:
2750 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002751 if r.endswith('\n') and r.index('\n') == len(r) - 1:
2752 return r[:-1]
2753 return r
2754 return runner
2755
2756
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002757class _PriorSyncFailedError(Exception):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002758
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002759 def __str__(self):
2760 return 'prior sync failed; rebase still in progress'
2761
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002762
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002763class _DirtyError(Exception):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002764
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002765 def __str__(self):
2766 return 'contains uncommitted changes'
2767
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002768
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002769class _InfoMessage(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002770
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002771 def __init__(self, project, text):
2772 self.project = project
2773 self.text = text
2774
2775 def Print(self, syncbuf):
2776 syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
2777 syncbuf.out.nl()
2778
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002779
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002780class _Failure(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002781
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002782 def __init__(self, project, why):
2783 self.project = project
2784 self.why = why
2785
2786 def Print(self, syncbuf):
2787 syncbuf.out.fail('error: %s/: %s',
2788 self.project.relpath,
2789 str(self.why))
2790 syncbuf.out.nl()
2791
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002792
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002793class _Later(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002794
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002795 def __init__(self, project, action):
2796 self.project = project
2797 self.action = action
2798
2799 def Run(self, syncbuf):
2800 out = syncbuf.out
2801 out.project('project %s/', self.project.relpath)
2802 out.nl()
2803 try:
2804 self.action()
2805 out.nl()
2806 return True
David Pursehouse8a68ff92012-09-24 12:15:13 +09002807 except GitError:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002808 out.nl()
2809 return False
2810
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002811
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002812class _SyncColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002813
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002814 def __init__(self, config):
2815 Coloring.__init__(self, config, 'reposync')
Anthony King7bdac712014-07-16 12:56:40 +01002816 self.project = self.printer('header', attr='bold')
2817 self.info = self.printer('info')
2818 self.fail = self.printer('fail', fg='red')
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002819
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002820
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002821class SyncBuffer(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002822
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002823 def __init__(self, config, detach_head=False):
2824 self._messages = []
2825 self._failures = []
2826 self._later_queue1 = []
2827 self._later_queue2 = []
2828
2829 self.out = _SyncColoring(config)
2830 self.out.redirect(sys.stderr)
2831
2832 self.detach_head = detach_head
2833 self.clean = True
2834
2835 def info(self, project, fmt, *args):
2836 self._messages.append(_InfoMessage(project, fmt % args))
2837
2838 def fail(self, project, err=None):
2839 self._failures.append(_Failure(project, err))
2840 self.clean = False
2841
2842 def later1(self, project, what):
2843 self._later_queue1.append(_Later(project, what))
2844
2845 def later2(self, project, what):
2846 self._later_queue2.append(_Later(project, what))
2847
2848 def Finish(self):
2849 self._PrintMessages()
2850 self._RunLater()
2851 self._PrintMessages()
2852 return self.clean
2853
2854 def _RunLater(self):
2855 for q in ['_later_queue1', '_later_queue2']:
2856 if not self._RunQueue(q):
2857 return
2858
2859 def _RunQueue(self, queue):
2860 for m in getattr(self, queue):
2861 if not m.Run(self):
2862 self.clean = False
2863 return False
2864 setattr(self, queue, [])
2865 return True
2866
2867 def _PrintMessages(self):
2868 for m in self._messages:
2869 m.Print(self)
2870 for m in self._failures:
2871 m.Print(self)
2872
2873 self._messages = []
2874 self._failures = []
2875
2876
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002877class MetaProject(Project):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002878
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002879 """A special project housed under .repo.
2880 """
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002881
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002882 def __init__(self, manifest, name, gitdir, worktree):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002883 Project.__init__(self,
Anthony King7bdac712014-07-16 12:56:40 +01002884 manifest=manifest,
2885 name=name,
2886 gitdir=gitdir,
2887 objdir=gitdir,
2888 worktree=worktree,
2889 remote=RemoteSpec('origin'),
2890 relpath='.repo/%s' % name,
2891 revisionExpr='refs/heads/master',
2892 revisionId=None,
2893 groups=None)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002894
2895 def PreSync(self):
2896 if self.Exists:
2897 cb = self.CurrentBranch
2898 if cb:
2899 base = self.GetBranch(cb).merge
2900 if base:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002901 self.revisionExpr = base
2902 self.revisionId = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002903
Anthony King7bdac712014-07-16 12:56:40 +01002904 def MetaBranchSwitch(self):
Florian Vallee5d016502012-06-07 17:19:26 +02002905 """ Prepare MetaProject for manifest branch switch
2906 """
2907
2908 # detach and delete manifest branch, allowing a new
2909 # branch to take over
Anthony King7bdac712014-07-16 12:56:40 +01002910 syncbuf = SyncBuffer(self.config, detach_head=True)
Florian Vallee5d016502012-06-07 17:19:26 +02002911 self.Sync_LocalHalf(syncbuf)
2912 syncbuf.Finish()
2913
2914 return GitCommand(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002915 ['update-ref', '-d', 'refs/heads/default'],
2916 capture_stdout=True,
2917 capture_stderr=True).Wait() == 0
Florian Vallee5d016502012-06-07 17:19:26 +02002918
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002919 @property
Shawn O. Pearcef6906872009-04-18 10:49:00 -07002920 def LastFetch(self):
2921 try:
2922 fh = os.path.join(self.gitdir, 'FETCH_HEAD')
2923 return os.path.getmtime(fh)
2924 except OSError:
2925 return 0
2926
2927 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002928 def HasChanges(self):
2929 """Has the remote received new commits not yet checked out?
2930 """
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002931 if not self.remote or not self.revisionExpr:
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002932 return False
2933
David Pursehouse8a68ff92012-09-24 12:15:13 +09002934 all_refs = self.bare_ref.all
2935 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002936 head = self.work_git.GetHead()
2937 if head.startswith(R_HEADS):
2938 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09002939 head = all_refs[head]
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002940 except KeyError:
2941 head = None
2942
2943 if revid == head:
2944 return False
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002945 elif self._revlist(not_rev(HEAD), revid):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002946 return True
2947 return False