blob: 918ee09c95c111f46a2e0fde44d55a36030af2ce [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Sarah Owenscecd1d82012-11-01 22:59:27 -070015from __future__ import print_function
Shawn O. Pearce438ee1c2008-11-03 09:59:36 -080016import errno
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import filecmp
Wink Saville4c426ef2015-06-03 08:05:17 -070018import glob
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070020import random
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
22import shutil
23import stat
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Julien Campergue335f5ef2013-10-16 11:02:35 +020026import tarfile
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +080027import tempfile
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070028import time
Dave Borowitz137d0132015-01-02 11:12:54 -080029import traceback
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -070030
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070031from color import Coloring
Dave Borowitzb42b4742012-10-31 12:27:27 -070032from git_command import GitCommand, git_require
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070033from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
34 ID_RE
Kevin Degiabaa7f32014-11-12 11:27:45 -070035from error import GitError, HookError, UploadError, DownloadError
Shawn O. Pearce559b8462009-03-02 12:56:08 -080036from error import ManifestInvalidRevisionError
Conley Owens75ee0572012-11-15 17:33:11 -080037from error import NoManifestException
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070038from trace import IsTrace, Trace
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070039
Shawn O. Pearced237b692009-04-17 18:49:50 -070040from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070041
David Pursehouse59bbb582013-05-17 10:49:33 +090042from pyversion import is_python3
43if not is_python3():
44 # pylint:disable=W0622
Chirayu Desai217ea7d2013-03-01 19:14:38 +053045 input = raw_input
David Pursehouse59bbb582013-05-17 10:49:33 +090046 # pylint:enable=W0622
Chirayu Desai217ea7d2013-03-01 19:14:38 +053047
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070048
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070049def _lwrite(path, content):
50 lock = '%s.lock' % path
51
Chirayu Desai303a82f2014-08-19 22:57:17 +053052 fd = open(lock, 'w')
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070053 try:
54 fd.write(content)
55 finally:
56 fd.close()
57
58 try:
59 os.rename(lock, path)
60 except OSError:
61 os.remove(lock)
62 raise
63
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070064
Shawn O. Pearce48244782009-04-16 08:25:57 -070065def _error(fmt, *args):
66 msg = fmt % args
Sarah Owenscecd1d82012-11-01 22:59:27 -070067 print('error: %s' % msg, file=sys.stderr)
Shawn O. Pearce48244782009-04-16 08:25:57 -070068
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070069
David Pursehousef33929d2015-08-24 14:39:14 +090070def _warn(fmt, *args):
71 msg = fmt % args
72 print('warn: %s' % msg, file=sys.stderr)
73
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070074
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070075def not_rev(r):
76 return '^' + r
77
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070078
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080079def sq(r):
80 return "'" + r.replace("'", "'\''") + "'"
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -080081
Jonathan Nieder93719792015-03-17 11:29:58 -070082_project_hook_list = None
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070083
84
Jonathan Nieder93719792015-03-17 11:29:58 -070085def _ProjectHooks():
86 """List the hooks present in the 'hooks' directory.
87
88 These hooks are project hooks and are copied to the '.git/hooks' directory
89 of all subprojects.
90
91 This function caches the list of hooks (based on the contents of the
92 'repo/hooks' directory) on the first call.
93
94 Returns:
95 A list of absolute paths to all of the files in the hooks directory.
96 """
97 global _project_hook_list
98 if _project_hook_list is None:
99 d = os.path.realpath(os.path.abspath(os.path.dirname(__file__)))
100 d = os.path.join(d, 'hooks')
101 _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
102 return _project_hook_list
103
104
Shawn O. Pearce632768b2008-10-23 11:58:52 -0700105class DownloadedChange(object):
106 _commit_cache = None
107
108 def __init__(self, project, base, change_id, ps_id, commit):
109 self.project = project
110 self.base = base
111 self.change_id = change_id
112 self.ps_id = ps_id
113 self.commit = commit
114
115 @property
116 def commits(self):
117 if self._commit_cache is None:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700118 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
119 '--abbrev-commit',
120 '--pretty=oneline',
121 '--reverse',
122 '--date-order',
123 not_rev(self.base),
124 self.commit,
125 '--')
Shawn O. Pearce632768b2008-10-23 11:58:52 -0700126 return self._commit_cache
127
128
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700129class ReviewableBranch(object):
130 _commit_cache = None
131
132 def __init__(self, project, branch, base):
133 self.project = project
134 self.branch = branch
135 self.base = base
136
137 @property
138 def name(self):
139 return self.branch.name
140
141 @property
142 def commits(self):
143 if self._commit_cache is None:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700144 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
145 '--abbrev-commit',
146 '--pretty=oneline',
147 '--reverse',
148 '--date-order',
149 not_rev(self.base),
150 R_HEADS + self.name,
151 '--')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700152 return self._commit_cache
153
154 @property
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800155 def unabbrev_commits(self):
156 r = dict()
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700157 for commit in self.project.bare_git.rev_list(not_rev(self.base),
158 R_HEADS + self.name,
159 '--'):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800160 r[commit[0:8]] = commit
161 return r
162
163 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700164 def date(self):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700165 return self.project.bare_git.log('--pretty=format:%cd',
166 '-n', '1',
167 R_HEADS + self.name,
168 '--')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700169
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700170 def UploadForReview(self, people,
171 auto_topic=False,
172 draft=False,
173 dest_branch=None):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800174 self.project.UploadForReview(self.name,
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -0700175 people,
Brian Harring435370c2012-07-28 15:37:04 -0700176 auto_topic=auto_topic,
Bryan Jacobsf609f912013-05-06 13:36:24 -0400177 draft=draft,
178 dest_branch=dest_branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700179
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700180 def GetPublishedRefs(self):
181 refs = {}
182 output = self.project.bare_git.ls_remote(
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700183 self.branch.remote.SshReviewUrl(self.project.UserEmail),
184 'refs/changes/*')
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700185 for line in output.split('\n'):
186 try:
187 (sha, ref) = line.split()
188 refs[sha] = ref
189 except ValueError:
190 pass
191
192 return refs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700193
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700194
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700195class StatusColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700196
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700197 def __init__(self, config):
198 Coloring.__init__(self, config, 'status')
Anthony King7bdac712014-07-16 12:56:40 +0100199 self.project = self.printer('header', attr='bold')
200 self.branch = self.printer('header', attr='bold')
201 self.nobranch = self.printer('nobranch', fg='red')
202 self.important = self.printer('important', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700203
Anthony King7bdac712014-07-16 12:56:40 +0100204 self.added = self.printer('added', fg='green')
205 self.changed = self.printer('changed', fg='red')
206 self.untracked = self.printer('untracked', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700207
208
209class DiffColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700210
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700211 def __init__(self, config):
212 Coloring.__init__(self, config, 'diff')
Anthony King7bdac712014-07-16 12:56:40 +0100213 self.project = self.printer('header', attr='bold')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700214
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700215
Anthony King7bdac712014-07-16 12:56:40 +0100216class _Annotation(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700217
James W. Mills24c13082012-04-12 15:04:13 -0500218 def __init__(self, name, value, keep):
219 self.name = name
220 self.value = value
221 self.keep = keep
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700222
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700223
Anthony King7bdac712014-07-16 12:56:40 +0100224class _CopyFile(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700225
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800226 def __init__(self, src, dest, abssrc, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700227 self.src = src
228 self.dest = dest
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800229 self.abs_src = abssrc
230 self.abs_dest = absdest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700231
232 def _Copy(self):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800233 src = self.abs_src
234 dest = self.abs_dest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700235 # copy file if it does not exist or is out of date
236 if not os.path.exists(dest) or not filecmp.cmp(src, dest):
237 try:
238 # remove existing file first, since it might be read-only
239 if os.path.exists(dest):
240 os.remove(dest)
Matthew Buckett2daf6672009-07-11 09:43:47 -0400241 else:
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200242 dest_dir = os.path.dirname(dest)
243 if not os.path.isdir(dest_dir):
244 os.makedirs(dest_dir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700245 shutil.copy(src, dest)
246 # make the file read-only
247 mode = os.stat(dest)[stat.ST_MODE]
248 mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
249 os.chmod(dest, mode)
250 except IOError:
Shawn O. Pearce48244782009-04-16 08:25:57 -0700251 _error('Cannot copy file %s to %s', src, dest)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700252
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700253
Anthony King7bdac712014-07-16 12:56:40 +0100254class _LinkFile(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700255
Wink Saville4c426ef2015-06-03 08:05:17 -0700256 def __init__(self, git_worktree, src, dest, relsrc, absdest):
257 self.git_worktree = git_worktree
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500258 self.src = src
259 self.dest = dest
Colin Cross0184dcc2015-05-05 00:24:54 -0700260 self.src_rel_to_dest = relsrc
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500261 self.abs_dest = absdest
262
Wink Saville4c426ef2015-06-03 08:05:17 -0700263 def __linkIt(self, relSrc, absDest):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500264 # link file if it does not exist or is out of date
Wink Saville4c426ef2015-06-03 08:05:17 -0700265 if not os.path.islink(absDest) or (os.readlink(absDest) != relSrc):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500266 try:
267 # remove existing file first, since it might be read-only
Dan Willemsene1e0bd12015-11-18 16:49:38 -0800268 if os.path.lexists(absDest):
Wink Saville4c426ef2015-06-03 08:05:17 -0700269 os.remove(absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500270 else:
Wink Saville4c426ef2015-06-03 08:05:17 -0700271 dest_dir = os.path.dirname(absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500272 if not os.path.isdir(dest_dir):
273 os.makedirs(dest_dir)
Wink Saville4c426ef2015-06-03 08:05:17 -0700274 os.symlink(relSrc, absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500275 except IOError:
Wink Saville4c426ef2015-06-03 08:05:17 -0700276 _error('Cannot link file %s to %s', relSrc, absDest)
277
278 def _Link(self):
279 """Link the self.rel_src_to_dest and self.abs_dest. Handles wild cards
280 on the src linking all of the files in the source in to the destination
281 directory.
282 """
283 # We use the absSrc to handle the situation where the current directory
284 # is not the root of the repo
285 absSrc = os.path.join(self.git_worktree, self.src)
286 if os.path.exists(absSrc):
287 # Entity exists so just a simple one to one link operation
288 self.__linkIt(self.src_rel_to_dest, self.abs_dest)
289 else:
290 # Entity doesn't exist assume there is a wild card
291 absDestDir = self.abs_dest
292 if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
293 _error('Link error: src with wildcard, %s must be a directory',
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700294 absDestDir)
Wink Saville4c426ef2015-06-03 08:05:17 -0700295 else:
296 absSrcFiles = glob.glob(absSrc)
297 for absSrcFile in absSrcFiles:
298 # Create a releative path from source dir to destination dir
299 absSrcDir = os.path.dirname(absSrcFile)
300 relSrcDir = os.path.relpath(absSrcDir, absDestDir)
301
302 # Get the source file name
303 srcFile = os.path.basename(absSrcFile)
304
305 # Now form the final full paths to srcFile. They will be
306 # absolute for the desintaiton and relative for the srouce.
307 absDest = os.path.join(absDestDir, srcFile)
308 relSrc = os.path.join(relSrcDir, srcFile)
309 self.__linkIt(relSrc, absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500310
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700311
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700312class RemoteSpec(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700313
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700314 def __init__(self,
315 name,
Anthony King7bdac712014-07-16 12:56:40 +0100316 url=None,
317 review=None,
Dan Willemsen96c2d652016-04-06 16:03:54 -0700318 revision=None,
319 orig_name=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700320 self.name = name
321 self.url = url
322 self.review = review
Anthony King36ea2fb2014-05-06 11:54:01 +0100323 self.revision = revision
Dan Willemsen96c2d652016-04-06 16:03:54 -0700324 self.orig_name = orig_name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700325
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700326
Doug Anderson37282b42011-03-04 11:54:18 -0800327class RepoHook(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700328
Doug Anderson37282b42011-03-04 11:54:18 -0800329 """A RepoHook contains information about a script to run as a hook.
330
331 Hooks are used to run a python script before running an upload (for instance,
332 to run presubmit checks). Eventually, we may have hooks for other actions.
333
334 This shouldn't be confused with files in the 'repo/hooks' directory. Those
335 files are copied into each '.git/hooks' folder for each project. Repo-level
336 hooks are associated instead with repo actions.
337
338 Hooks are always python. When a hook is run, we will load the hook into the
339 interpreter and execute its main() function.
340 """
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700341
Doug Anderson37282b42011-03-04 11:54:18 -0800342 def __init__(self,
343 hook_type,
344 hooks_project,
345 topdir,
346 abort_if_user_denies=False):
347 """RepoHook constructor.
348
349 Params:
350 hook_type: A string representing the type of hook. This is also used
351 to figure out the name of the file containing the hook. For
352 example: 'pre-upload'.
353 hooks_project: The project containing the repo hooks. If you have a
354 manifest, this is manifest.repo_hooks_project. OK if this is None,
355 which will make the hook a no-op.
356 topdir: Repo's top directory (the one containing the .repo directory).
357 Scripts will run with CWD as this directory. If you have a manifest,
358 this is manifest.topdir
359 abort_if_user_denies: If True, we'll throw a HookError() if the user
360 doesn't allow us to run the hook.
361 """
362 self._hook_type = hook_type
363 self._hooks_project = hooks_project
364 self._topdir = topdir
365 self._abort_if_user_denies = abort_if_user_denies
366
367 # Store the full path to the script for convenience.
368 if self._hooks_project:
369 self._script_fullpath = os.path.join(self._hooks_project.worktree,
370 self._hook_type + '.py')
371 else:
372 self._script_fullpath = None
373
374 def _GetHash(self):
375 """Return a hash of the contents of the hooks directory.
376
377 We'll just use git to do this. This hash has the property that if anything
378 changes in the directory we will return a different has.
379
380 SECURITY CONSIDERATION:
381 This hash only represents the contents of files in the hook directory, not
382 any other files imported or called by hooks. Changes to imported files
383 can change the script behavior without affecting the hash.
384
385 Returns:
386 A string representing the hash. This will always be ASCII so that it can
387 be printed to the user easily.
388 """
389 assert self._hooks_project, "Must have hooks to calculate their hash."
390
391 # We will use the work_git object rather than just calling GetRevisionId().
392 # That gives us a hash of the latest checked in version of the files that
393 # the user will actually be executing. Specifically, GetRevisionId()
394 # doesn't appear to change even if a user checks out a different version
395 # of the hooks repo (via git checkout) nor if a user commits their own revs.
396 #
397 # NOTE: Local (non-committed) changes will not be factored into this hash.
398 # I think this is OK, since we're really only worried about warning the user
399 # about upstream changes.
400 return self._hooks_project.work_git.rev_parse('HEAD')
401
402 def _GetMustVerb(self):
403 """Return 'must' if the hook is required; 'should' if not."""
404 if self._abort_if_user_denies:
405 return 'must'
406 else:
407 return 'should'
408
409 def _CheckForHookApproval(self):
410 """Check to see whether this hook has been approved.
411
412 We'll look at the hash of all of the hooks. If this matches the hash that
413 the user last approved, we're done. If it doesn't, we'll ask the user
414 about approval.
415
416 Note that we ask permission for each individual hook even though we use
417 the hash of all hooks when detecting changes. We'd like the user to be
418 able to approve / deny each hook individually. We only use the hash of all
419 hooks because there is no other easy way to detect changes to local imports.
420
421 Returns:
422 True if this hook is approved to run; False otherwise.
423
424 Raises:
425 HookError: Raised if the user doesn't approve and abort_if_user_denies
426 was passed to the consturctor.
427 """
Doug Anderson37282b42011-03-04 11:54:18 -0800428 hooks_config = self._hooks_project.config
429 git_approval_key = 'repo.hooks.%s.approvedhash' % self._hook_type
430
431 # Get the last hash that the user approved for this hook; may be None.
432 old_hash = hooks_config.GetString(git_approval_key)
433
434 # Get the current hash so we can tell if scripts changed since approval.
435 new_hash = self._GetHash()
436
437 if old_hash is not None:
438 # User previously approved hook and asked not to be prompted again.
439 if new_hash == old_hash:
440 # Approval matched. We're done.
441 return True
442 else:
443 # Give the user a reason why we're prompting, since they last told
444 # us to "never ask again".
445 prompt = 'WARNING: Scripts have changed since %s was allowed.\n\n' % (
446 self._hook_type)
447 else:
448 prompt = ''
449
450 # Prompt the user if we're not on a tty; on a tty we'll assume "no".
451 if sys.stdout.isatty():
452 prompt += ('Repo %s run the script:\n'
453 ' %s\n'
454 '\n'
455 'Do you want to allow this script to run '
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700456 '(yes/yes-never-ask-again/NO)? ') % (self._GetMustVerb(),
457 self._script_fullpath)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530458 response = input(prompt).lower()
David Pursehouse98ffba12012-11-14 11:18:00 +0900459 print()
Doug Anderson37282b42011-03-04 11:54:18 -0800460
461 # User is doing a one-time approval.
462 if response in ('y', 'yes'):
463 return True
464 elif response == 'yes-never-ask-again':
465 hooks_config.SetString(git_approval_key, new_hash)
466 return True
467
468 # For anything else, we'll assume no approval.
469 if self._abort_if_user_denies:
470 raise HookError('You must allow the %s hook or use --no-verify.' %
471 self._hook_type)
472
473 return False
474
475 def _ExecuteHook(self, **kwargs):
476 """Actually execute the given hook.
477
478 This will run the hook's 'main' function in our python interpreter.
479
480 Args:
481 kwargs: Keyword arguments to pass to the hook. These are often specific
482 to the hook type. For instance, pre-upload hooks will contain
483 a project_list.
484 """
485 # Keep sys.path and CWD stashed away so that we can always restore them
486 # upon function exit.
487 orig_path = os.getcwd()
488 orig_syspath = sys.path
489
490 try:
491 # Always run hooks with CWD as topdir.
492 os.chdir(self._topdir)
493
494 # Put the hook dir as the first item of sys.path so hooks can do
495 # relative imports. We want to replace the repo dir as [0] so
496 # hooks can't import repo files.
497 sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
498
499 # Exec, storing global context in the context dict. We catch exceptions
500 # and convert to a HookError w/ just the failing traceback.
Mike Frysinger4aa4b212016-03-04 15:03:00 -0500501 context = {'__file__': self._script_fullpath}
Doug Anderson37282b42011-03-04 11:54:18 -0800502 try:
Anthony King70f68902014-05-05 21:15:34 +0100503 exec(compile(open(self._script_fullpath).read(),
504 self._script_fullpath, 'exec'), context)
Doug Anderson37282b42011-03-04 11:54:18 -0800505 except Exception:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700506 raise HookError('%s\nFailed to import %s hook; see traceback above.' %
507 (traceback.format_exc(), self._hook_type))
Doug Anderson37282b42011-03-04 11:54:18 -0800508
509 # Running the script should have defined a main() function.
510 if 'main' not in context:
511 raise HookError('Missing main() in: "%s"' % self._script_fullpath)
512
Doug Anderson37282b42011-03-04 11:54:18 -0800513 # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
514 # We don't actually want hooks to define their main with this argument--
515 # it's there to remind them that their hook should always take **kwargs.
516 # For instance, a pre-upload hook should be defined like:
517 # def main(project_list, **kwargs):
518 #
519 # This allows us to later expand the API without breaking old hooks.
520 kwargs = kwargs.copy()
521 kwargs['hook_should_take_kwargs'] = True
522
523 # Call the main function in the hook. If the hook should cause the
524 # build to fail, it will raise an Exception. We'll catch that convert
525 # to a HookError w/ just the failing traceback.
526 try:
527 context['main'](**kwargs)
528 except Exception:
529 raise HookError('%s\nFailed to run main() for %s hook; see traceback '
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700530 'above.' % (traceback.format_exc(),
531 self._hook_type))
Doug Anderson37282b42011-03-04 11:54:18 -0800532 finally:
533 # Restore sys.path and CWD.
534 sys.path = orig_syspath
535 os.chdir(orig_path)
536
537 def Run(self, user_allows_all_hooks, **kwargs):
538 """Run the hook.
539
540 If the hook doesn't exist (because there is no hooks project or because
541 this particular hook is not enabled), this is a no-op.
542
543 Args:
544 user_allows_all_hooks: If True, we will never prompt about running the
545 hook--we'll just assume it's OK to run it.
546 kwargs: Keyword arguments to pass to the hook. These are often specific
547 to the hook type. For instance, pre-upload hooks will contain
548 a project_list.
549
550 Raises:
551 HookError: If there was a problem finding the hook or the user declined
552 to run a required hook (from _CheckForHookApproval).
553 """
554 # No-op if there is no hooks project or if hook is disabled.
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700555 if ((not self._hooks_project) or (self._hook_type not in
556 self._hooks_project.enabled_repo_hooks)):
Doug Anderson37282b42011-03-04 11:54:18 -0800557 return
558
559 # Bail with a nice error if we can't find the hook.
560 if not os.path.isfile(self._script_fullpath):
561 raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
562
563 # Make sure the user is OK with running the hook.
564 if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
565 return
566
567 # Run the hook with the same version of python we're using.
568 self._ExecuteHook(**kwargs)
569
570
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700571class Project(object):
Kevin Degi384b3c52014-10-16 16:02:58 -0600572 # These objects can be shared between several working trees.
573 shareable_files = ['description', 'info']
574 shareable_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
575 # These objects can only be used by a single working tree.
576 working_tree_files = ['config', 'packed-refs', 'shallow']
577 working_tree_dirs = ['logs', 'refs']
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700578
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700579 def __init__(self,
580 manifest,
581 name,
582 remote,
583 gitdir,
David James8d201162013-10-11 17:03:19 -0700584 objdir,
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700585 worktree,
586 relpath,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700587 revisionExpr,
Mike Pontillod3153822012-02-28 11:53:24 -0800588 revisionId,
Anthony King7bdac712014-07-16 12:56:40 +0100589 rebase=True,
590 groups=None,
591 sync_c=False,
592 sync_s=False,
593 clone_depth=None,
594 upstream=None,
595 parent=None,
596 is_derived=False,
David Pursehouseb1553542014-09-04 21:28:09 +0900597 dest_branch=None,
Simran Basib9a1b732015-08-20 12:19:28 -0700598 optimized_fetch=False,
599 old_revision=None):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800600 """Init a Project object.
601
602 Args:
603 manifest: The XmlManifest object.
604 name: The `name` attribute of manifest.xml's project element.
605 remote: RemoteSpec object specifying its remote's properties.
606 gitdir: Absolute path of git directory.
David James8d201162013-10-11 17:03:19 -0700607 objdir: Absolute path of directory to store git objects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800608 worktree: Absolute path of git working tree.
609 relpath: Relative path of git working tree to repo's top directory.
610 revisionExpr: The `revision` attribute of manifest.xml's project element.
611 revisionId: git commit id for checking out.
612 rebase: The `rebase` attribute of manifest.xml's project element.
613 groups: The `groups` attribute of manifest.xml's project element.
614 sync_c: The `sync-c` attribute of manifest.xml's project element.
615 sync_s: The `sync-s` attribute of manifest.xml's project element.
616 upstream: The `upstream` attribute of manifest.xml's project element.
617 parent: The parent Project object.
618 is_derived: False if the project was explicitly defined in the manifest;
619 True if the project is a discovered submodule.
Bryan Jacobsf609f912013-05-06 13:36:24 -0400620 dest_branch: The branch to which to push changes for review by default.
David Pursehouseb1553542014-09-04 21:28:09 +0900621 optimized_fetch: If True, when a project is set to a sha1 revision, only
622 fetch from the remote if the sha1 is not present locally.
Simran Basib9a1b732015-08-20 12:19:28 -0700623 old_revision: saved git commit id for open GITC projects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800624 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700625 self.manifest = manifest
626 self.name = name
627 self.remote = remote
Anthony Newnamdf14a702011-01-09 17:31:57 -0800628 self.gitdir = gitdir.replace('\\', '/')
David James8d201162013-10-11 17:03:19 -0700629 self.objdir = objdir.replace('\\', '/')
Shawn O. Pearce0ce6ca92011-01-10 13:26:01 -0800630 if worktree:
Mark E. Hamiltonf9fe3e12016-02-23 18:10:42 -0700631 self.worktree = os.path.normpath(worktree.replace('\\', '/'))
Shawn O. Pearce0ce6ca92011-01-10 13:26:01 -0800632 else:
633 self.worktree = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700634 self.relpath = relpath
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700635 self.revisionExpr = revisionExpr
636
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700637 if revisionId is None \
638 and revisionExpr \
639 and IsId(revisionExpr):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700640 self.revisionId = revisionExpr
641 else:
642 self.revisionId = revisionId
643
Mike Pontillod3153822012-02-28 11:53:24 -0800644 self.rebase = rebase
Colin Cross5acde752012-03-28 20:15:45 -0700645 self.groups = groups
Anatol Pomazau79770d22012-04-20 14:41:59 -0700646 self.sync_c = sync_c
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800647 self.sync_s = sync_s
David Pursehouseede7f122012-11-27 22:25:30 +0900648 self.clone_depth = clone_depth
Brian Harring14a66742012-09-28 20:21:57 -0700649 self.upstream = upstream
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800650 self.parent = parent
651 self.is_derived = is_derived
David Pursehouseb1553542014-09-04 21:28:09 +0900652 self.optimized_fetch = optimized_fetch
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800653 self.subprojects = []
Mike Pontillod3153822012-02-28 11:53:24 -0800654
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700655 self.snapshots = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700656 self.copyfiles = []
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500657 self.linkfiles = []
James W. Mills24c13082012-04-12 15:04:13 -0500658 self.annotations = []
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700659 self.config = GitConfig.ForRepository(gitdir=self.gitdir,
660 defaults=self.manifest.globalConfig)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700661
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800662 if self.worktree:
David James8d201162013-10-11 17:03:19 -0700663 self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800664 else:
665 self.work_git = None
David James8d201162013-10-11 17:03:19 -0700666 self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
Shawn O. Pearced237b692009-04-17 18:49:50 -0700667 self.bare_ref = GitRefs(gitdir)
David James8d201162013-10-11 17:03:19 -0700668 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
Bryan Jacobsf609f912013-05-06 13:36:24 -0400669 self.dest_branch = dest_branch
Simran Basib9a1b732015-08-20 12:19:28 -0700670 self.old_revision = old_revision
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700671
Doug Anderson37282b42011-03-04 11:54:18 -0800672 # This will be filled in if a project is later identified to be the
673 # project containing repo hooks.
674 self.enabled_repo_hooks = []
675
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700676 @property
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800677 def Derived(self):
678 return self.is_derived
679
680 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700681 def Exists(self):
Kevin Degi384b3c52014-10-16 16:02:58 -0600682 return os.path.isdir(self.gitdir) and os.path.isdir(self.objdir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700683
684 @property
685 def CurrentBranch(self):
686 """Obtain the name of the currently checked out branch.
687 The branch name omits the 'refs/heads/' prefix.
688 None is returned if the project is on a detached HEAD.
689 """
Shawn O. Pearce5b23f242009-04-17 18:43:33 -0700690 b = self.work_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700691 if b.startswith(R_HEADS):
692 return b[len(R_HEADS):]
693 return None
694
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700695 def IsRebaseInProgress(self):
696 w = self.worktree
697 g = os.path.join(w, '.git')
698 return os.path.exists(os.path.join(g, 'rebase-apply')) \
699 or os.path.exists(os.path.join(g, 'rebase-merge')) \
700 or os.path.exists(os.path.join(w, '.dotest'))
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200701
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700702 def IsDirty(self, consider_untracked=True):
703 """Is the working directory modified in some way?
704 """
705 self.work_git.update_index('-q',
706 '--unmerged',
707 '--ignore-missing',
708 '--refresh')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900709 if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700710 return True
711 if self.work_git.DiffZ('diff-files'):
712 return True
713 if consider_untracked and self.work_git.LsOthers():
714 return True
715 return False
716
717 _userident_name = None
718 _userident_email = None
719
720 @property
721 def UserName(self):
722 """Obtain the user's personal name.
723 """
724 if self._userident_name is None:
725 self._LoadUserIdentity()
726 return self._userident_name
727
728 @property
729 def UserEmail(self):
730 """Obtain the user's email address. This is very likely
731 to be their Gerrit login.
732 """
733 if self._userident_email is None:
734 self._LoadUserIdentity()
735 return self._userident_email
736
737 def _LoadUserIdentity(self):
David Pursehousec1b86a22012-11-14 11:36:51 +0900738 u = self.bare_git.var('GIT_COMMITTER_IDENT')
739 m = re.compile("^(.*) <([^>]*)> ").match(u)
740 if m:
741 self._userident_name = m.group(1)
742 self._userident_email = m.group(2)
743 else:
744 self._userident_name = ''
745 self._userident_email = ''
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700746
747 def GetRemote(self, name):
748 """Get the configuration for a single remote.
749 """
750 return self.config.GetRemote(name)
751
752 def GetBranch(self, name):
753 """Get the configuration for a single branch.
754 """
755 return self.config.GetBranch(name)
756
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700757 def GetBranches(self):
758 """Get all existing local branches.
759 """
760 current = self.CurrentBranch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900761 all_refs = self._allrefs
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700762 heads = {}
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700763
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530764 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700765 if name.startswith(R_HEADS):
766 name = name[len(R_HEADS):]
767 b = self.GetBranch(name)
768 b.current = name == current
769 b.published = None
David Pursehouse8a68ff92012-09-24 12:15:13 +0900770 b.revision = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700771 heads[name] = b
772
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530773 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700774 if name.startswith(R_PUB):
775 name = name[len(R_PUB):]
776 b = heads.get(name)
777 if b:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900778 b.published = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700779
780 return heads
781
Colin Cross5acde752012-03-28 20:15:45 -0700782 def MatchesGroups(self, manifest_groups):
783 """Returns true if the manifest groups specified at init should cause
784 this project to be synced.
785 Prefixing a manifest group with "-" inverts the meaning of a group.
Conley Owensbb1b5f52012-08-13 13:11:18 -0700786 All projects are implicitly labelled with "all".
Conley Owens971de8e2012-04-16 10:36:08 -0700787
788 labels are resolved in order. In the example case of
Conley Owensbb1b5f52012-08-13 13:11:18 -0700789 project_groups: "all,group1,group2"
Conley Owens971de8e2012-04-16 10:36:08 -0700790 manifest_groups: "-group1,group2"
791 the project will be matched.
David Holmer0a1c6a12012-11-14 19:19:00 -0500792
793 The special manifest group "default" will match any project that
794 does not have the special project group "notdefault"
Colin Cross5acde752012-03-28 20:15:45 -0700795 """
David Holmer0a1c6a12012-11-14 19:19:00 -0500796 expanded_manifest_groups = manifest_groups or ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700797 expanded_project_groups = ['all'] + (self.groups or [])
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700798 if 'notdefault' not in expanded_project_groups:
David Holmer0a1c6a12012-11-14 19:19:00 -0500799 expanded_project_groups += ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700800
Conley Owens971de8e2012-04-16 10:36:08 -0700801 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700802 for group in expanded_manifest_groups:
803 if group.startswith('-') and group[1:] in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700804 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700805 elif group in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700806 matched = True
Colin Cross5acde752012-03-28 20:15:45 -0700807
Conley Owens971de8e2012-04-16 10:36:08 -0700808 return matched
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700809
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700810# Status Display ##
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700811 def UncommitedFiles(self, get_all=True):
812 """Returns a list of strings, uncommitted files in the git tree.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700813
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700814 Args:
815 get_all: a boolean, if True - get information about all different
816 uncommitted files. If False - return as soon as any kind of
817 uncommitted files is detected.
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500818 """
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700819 details = []
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500820 self.work_git.update_index('-q',
821 '--unmerged',
822 '--ignore-missing',
823 '--refresh')
824 if self.IsRebaseInProgress():
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700825 details.append("rebase in progress")
826 if not get_all:
827 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500828
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700829 changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
830 if changes:
831 details.extend(changes)
832 if not get_all:
833 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500834
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700835 changes = self.work_git.DiffZ('diff-files').keys()
836 if changes:
837 details.extend(changes)
838 if not get_all:
839 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500840
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700841 changes = self.work_git.LsOthers()
842 if changes:
843 details.extend(changes)
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500844
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700845 return details
846
847 def HasChanges(self):
848 """Returns true if there are uncommitted changes.
849 """
850 if self.UncommitedFiles(get_all=False):
851 return True
852 else:
853 return False
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500854
Terence Haddock4655e812011-03-31 12:33:34 +0200855 def PrintWorkTreeStatus(self, output_redir=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700856 """Prints the status of the repository to stdout.
Terence Haddock4655e812011-03-31 12:33:34 +0200857
858 Args:
859 output: If specified, redirect the output to this object.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700860 """
861 if not os.path.isdir(self.worktree):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700862 if output_redir is None:
Terence Haddock4655e812011-03-31 12:33:34 +0200863 output_redir = sys.stdout
Sarah Owenscecd1d82012-11-01 22:59:27 -0700864 print(file=output_redir)
865 print('project %s/' % self.relpath, file=output_redir)
866 print(' missing (run "repo sync")', file=output_redir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700867 return
868
869 self.work_git.update_index('-q',
870 '--unmerged',
871 '--ignore-missing',
872 '--refresh')
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700873 rb = self.IsRebaseInProgress()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700874 di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
875 df = self.work_git.DiffZ('diff-files')
876 do = self.work_git.LsOthers()
Ali Utku Selen76abcc12012-01-25 10:51:12 +0100877 if not rb and not di and not df and not do and not self.CurrentBranch:
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700878 return 'CLEAN'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700879
880 out = StatusColoring(self.config)
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700881 if output_redir is not None:
Terence Haddock4655e812011-03-31 12:33:34 +0200882 out.redirect(output_redir)
Jakub Vrana0402cd82014-09-09 15:39:15 -0700883 out.project('project %-40s', self.relpath + '/ ')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700884
885 branch = self.CurrentBranch
886 if branch is None:
887 out.nobranch('(*** NO BRANCH ***)')
888 else:
889 out.branch('branch %s', branch)
890 out.nl()
891
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700892 if rb:
893 out.important('prior sync failed; rebase still in progress')
894 out.nl()
895
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700896 paths = list()
897 paths.extend(di.keys())
898 paths.extend(df.keys())
899 paths.extend(do)
900
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530901 for p in sorted(set(paths)):
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900902 try:
903 i = di[p]
904 except KeyError:
905 i = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700906
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900907 try:
908 f = df[p]
909 except KeyError:
910 f = None
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200911
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900912 if i:
913 i_status = i.status.upper()
914 else:
915 i_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700916
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900917 if f:
918 f_status = f.status.lower()
919 else:
920 f_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700921
922 if i and i.src_path:
Shawn O. Pearcefe086752009-03-03 13:49:48 -0800923 line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700924 i.src_path, p, i.level)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700925 else:
926 line = ' %s%s\t%s' % (i_status, f_status, p)
927
928 if i and not f:
929 out.added('%s', line)
930 elif (i and f) or (not i and f):
931 out.changed('%s', line)
932 elif not i and not f:
933 out.untracked('%s', line)
934 else:
935 out.write('%s', line)
936 out.nl()
Terence Haddock4655e812011-03-31 12:33:34 +0200937
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700938 return 'DIRTY'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700939
pelyad67872d2012-03-28 14:49:58 +0300940 def PrintWorkTreeDiff(self, absolute_paths=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700941 """Prints the status of the repository to stdout.
942 """
943 out = DiffColoring(self.config)
944 cmd = ['diff']
945 if out.is_on:
946 cmd.append('--color')
947 cmd.append(HEAD)
pelyad67872d2012-03-28 14:49:58 +0300948 if absolute_paths:
949 cmd.append('--src-prefix=a/%s/' % self.relpath)
950 cmd.append('--dst-prefix=b/%s/' % self.relpath)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700951 cmd.append('--')
952 p = GitCommand(self,
953 cmd,
Anthony King7bdac712014-07-16 12:56:40 +0100954 capture_stdout=True,
955 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700956 has_diff = False
957 for line in p.process.stdout:
958 if not has_diff:
959 out.nl()
960 out.project('project %s/' % self.relpath)
961 out.nl()
962 has_diff = True
Sarah Owenscecd1d82012-11-01 22:59:27 -0700963 print(line[:-1])
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700964 p.Wait()
965
966
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700967# Publish / Upload ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700968
David Pursehouse8a68ff92012-09-24 12:15:13 +0900969 def WasPublished(self, branch, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700970 """Was the branch published (uploaded) for code review?
971 If so, returns the SHA-1 hash of the last published
972 state for the branch.
973 """
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700974 key = R_PUB + branch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900975 if all_refs is None:
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700976 try:
977 return self.bare_git.rev_parse(key)
978 except GitError:
979 return None
980 else:
981 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900982 return all_refs[key]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700983 except KeyError:
984 return None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700985
David Pursehouse8a68ff92012-09-24 12:15:13 +0900986 def CleanPublishedCache(self, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700987 """Prunes any stale published refs.
988 """
David Pursehouse8a68ff92012-09-24 12:15:13 +0900989 if all_refs is None:
990 all_refs = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700991 heads = set()
992 canrm = {}
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530993 for name, ref_id in all_refs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700994 if name.startswith(R_HEADS):
995 heads.add(name)
996 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900997 canrm[name] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700998
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530999 for name, ref_id in canrm.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001000 n = name[len(R_PUB):]
1001 if R_HEADS + n not in heads:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001002 self.bare_git.DeleteRef(name, ref_id)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001003
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -07001004 def GetUploadableBranches(self, selected_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001005 """List any branches which can be uploaded for review.
1006 """
1007 heads = {}
1008 pubed = {}
1009
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301010 for name, ref_id in self._allrefs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001011 if name.startswith(R_HEADS):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001012 heads[name[len(R_HEADS):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001013 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001014 pubed[name[len(R_PUB):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001015
1016 ready = []
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301017 for branch, ref_id in heads.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09001018 if branch in pubed and pubed[branch] == ref_id:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001019 continue
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -07001020 if selected_branch and branch != selected_branch:
1021 continue
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001022
Shawn O. Pearce35f25962008-11-11 17:03:13 -08001023 rb = self.GetUploadableBranch(branch)
1024 if rb:
1025 ready.append(rb)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001026 return ready
1027
Shawn O. Pearce35f25962008-11-11 17:03:13 -08001028 def GetUploadableBranch(self, branch_name):
1029 """Get a single uploadable branch, or None.
1030 """
1031 branch = self.GetBranch(branch_name)
1032 base = branch.LocalMerge
1033 if branch.LocalMerge:
1034 rb = ReviewableBranch(self, branch, base)
1035 if rb.commits:
1036 return rb
1037 return None
1038
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001039 def UploadForReview(self, branch=None,
Anthony King7bdac712014-07-16 12:56:40 +01001040 people=([], []),
Brian Harring435370c2012-07-28 15:37:04 -07001041 auto_topic=False,
Bryan Jacobsf609f912013-05-06 13:36:24 -04001042 draft=False,
1043 dest_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001044 """Uploads the named branch for code review.
1045 """
1046 if branch is None:
1047 branch = self.CurrentBranch
1048 if branch is None:
1049 raise GitError('not currently on a branch')
1050
1051 branch = self.GetBranch(branch)
1052 if not branch.LocalMerge:
1053 raise GitError('branch %s does not track a remote' % branch.name)
1054 if not branch.remote.review:
1055 raise GitError('remote %s has no review url' % branch.remote.name)
1056
Bryan Jacobsf609f912013-05-06 13:36:24 -04001057 if dest_branch is None:
1058 dest_branch = self.dest_branch
1059 if dest_branch is None:
1060 dest_branch = branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001061 if not dest_branch.startswith(R_HEADS):
1062 dest_branch = R_HEADS + dest_branch
1063
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -08001064 if not branch.remote.projectname:
1065 branch.remote.projectname = self.name
1066 branch.remote.Save()
1067
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001068 url = branch.remote.ReviewUrl(self.UserEmail)
1069 if url is None:
1070 raise UploadError('review not configured')
1071 cmd = ['push']
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001072
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001073 if url.startswith('ssh://'):
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001074 rp = ['gerrit receive-pack']
1075 for e in people[0]:
1076 rp.append('--reviewer=%s' % sq(e))
1077 for e in people[1]:
1078 rp.append('--cc=%s' % sq(e))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001079 cmd.append('--receive-pack=%s' % " ".join(rp))
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001080
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001081 cmd.append(url)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001082
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001083 if dest_branch.startswith(R_HEADS):
1084 dest_branch = dest_branch[len(R_HEADS):]
Brian Harring435370c2012-07-28 15:37:04 -07001085
1086 upload_type = 'for'
1087 if draft:
1088 upload_type = 'drafts'
1089
1090 ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
1091 dest_branch)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001092 if auto_topic:
1093 ref_spec = ref_spec + '/' + branch.name
Shawn Pearce45d21682013-02-28 00:35:51 -08001094 if not url.startswith('ssh://'):
1095 rp = ['r=%s' % p for p in people[0]] + \
1096 ['cc=%s' % p for p in people[1]]
1097 if rp:
1098 ref_spec = ref_spec + '%' + ','.join(rp)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001099 cmd.append(ref_spec)
1100
Anthony King7bdac712014-07-16 12:56:40 +01001101 if GitCommand(self, cmd, bare=True).Wait() != 0:
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001102 raise UploadError('Upload failed')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001103
1104 msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
1105 self.bare_git.UpdateRef(R_PUB + branch.name,
1106 R_HEADS + branch.name,
Anthony King7bdac712014-07-16 12:56:40 +01001107 message=msg)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001108
1109
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001110# Sync ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001111
Julien Campergue335f5ef2013-10-16 11:02:35 +02001112 def _ExtractArchive(self, tarpath, path=None):
1113 """Extract the given tar on its current location
1114
1115 Args:
1116 - tarpath: The path to the actual tar file
1117
1118 """
1119 try:
1120 with tarfile.open(tarpath, 'r') as tar:
1121 tar.extractall(path=path)
1122 return True
1123 except (IOError, tarfile.TarError) as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001124 _error("Cannot extract archive %s: %s", tarpath, str(e))
Julien Campergue335f5ef2013-10-16 11:02:35 +02001125 return False
1126
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001127 def Sync_NetworkHalf(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001128 quiet=False,
1129 is_new=None,
1130 current_branch_only=False,
1131 force_sync=False,
1132 clone_bundle=True,
1133 no_tags=False,
1134 archive=False,
1135 optimized_fetch=False,
1136 prune=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001137 """Perform only the network IO portion of the sync process.
1138 Local working directory/branch state is not affected.
1139 """
Julien Campergue335f5ef2013-10-16 11:02:35 +02001140 if archive and not isinstance(self, MetaProject):
1141 if self.remote.url.startswith(('http://', 'https://')):
David Pursehousef33929d2015-08-24 14:39:14 +09001142 _error("%s: Cannot fetch archives from http/https remotes.", self.name)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001143 return False
1144
1145 name = self.relpath.replace('\\', '/')
1146 name = name.replace('/', '_')
1147 tarpath = '%s.tar' % name
1148 topdir = self.manifest.topdir
1149
1150 try:
1151 self._FetchArchive(tarpath, cwd=topdir)
1152 except GitError as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001153 _error('%s', e)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001154 return False
1155
1156 # From now on, we only need absolute tarpath
1157 tarpath = os.path.join(topdir, tarpath)
1158
1159 if not self._ExtractArchive(tarpath, path=topdir):
1160 return False
1161 try:
1162 os.remove(tarpath)
1163 except OSError as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001164 _warn("Cannot remove archive %s: %s", tarpath, str(e))
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001165 self._CopyAndLinkFiles()
Julien Campergue335f5ef2013-10-16 11:02:35 +02001166 return True
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001167 if is_new is None:
1168 is_new = not self.Exists
Shawn O. Pearce88443382010-10-08 10:02:09 +02001169 if is_new:
Kevin Degiabaa7f32014-11-12 11:27:45 -07001170 self._InitGitDir(force_sync=force_sync)
Jimmie Westera0444582012-10-24 13:44:42 +02001171 else:
1172 self._UpdateHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001173 self._InitRemote()
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001174
1175 if is_new:
1176 alt = os.path.join(self.gitdir, 'objects/info/alternates')
1177 try:
1178 fd = open(alt, 'rb')
1179 try:
1180 alt_dir = fd.readline().rstrip()
1181 finally:
1182 fd.close()
1183 except IOError:
1184 alt_dir = None
1185 else:
1186 alt_dir = None
1187
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001188 if clone_bundle \
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001189 and alt_dir is None \
1190 and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001191 is_new = False
1192
Shawn O. Pearce6ba6ba02012-05-24 09:46:50 -07001193 if not current_branch_only:
1194 if self.sync_c:
1195 current_branch_only = True
1196 elif not self.manifest._loaded:
1197 # Manifest cannot check defaults until it syncs.
1198 current_branch_only = False
1199 elif self.manifest.default.sync_c:
1200 current_branch_only = True
1201
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001202 need_to_fetch = not (optimized_fetch and
1203 (ID_RE.match(self.revisionExpr) and
1204 self._CheckForSha1()))
1205 if (need_to_fetch and
1206 not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
1207 current_branch_only=current_branch_only,
1208 no_tags=no_tags, prune=prune)):
Anthony King7bdac712014-07-16 12:56:40 +01001209 return False
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001210
1211 if self.worktree:
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001212 self._InitMRef()
1213 else:
1214 self._InitMirrorHead()
1215 try:
1216 os.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
1217 except OSError:
1218 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001219 return True
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08001220
1221 def PostRepoUpgrade(self):
1222 self._InitHooks()
1223
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001224 def _CopyAndLinkFiles(self):
Simran Basib9a1b732015-08-20 12:19:28 -07001225 if self.manifest.isGitcClient:
1226 return
David Pursehouse8a68ff92012-09-24 12:15:13 +09001227 for copyfile in self.copyfiles:
1228 copyfile._Copy()
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001229 for linkfile in self.linkfiles:
1230 linkfile._Link()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001231
Julien Camperguedd654222014-01-09 16:21:37 +01001232 def GetCommitRevisionId(self):
1233 """Get revisionId of a commit.
1234
1235 Use this method instead of GetRevisionId to get the id of the commit rather
1236 than the id of the current git object (for example, a tag)
1237
1238 """
1239 if not self.revisionExpr.startswith(R_TAGS):
1240 return self.GetRevisionId(self._allrefs)
1241
1242 try:
1243 return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
1244 except GitError:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001245 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1246 (self.revisionExpr, self.name))
Julien Camperguedd654222014-01-09 16:21:37 +01001247
David Pursehouse8a68ff92012-09-24 12:15:13 +09001248 def GetRevisionId(self, all_refs=None):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001249 if self.revisionId:
1250 return self.revisionId
1251
1252 rem = self.GetRemote(self.remote.name)
1253 rev = rem.ToLocal(self.revisionExpr)
1254
David Pursehouse8a68ff92012-09-24 12:15:13 +09001255 if all_refs is not None and rev in all_refs:
1256 return all_refs[rev]
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001257
1258 try:
1259 return self.bare_git.rev_parse('--verify', '%s^0' % rev)
1260 except GitError:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001261 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1262 (self.revisionExpr, self.name))
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001263
Kevin Degiabaa7f32014-11-12 11:27:45 -07001264 def Sync_LocalHalf(self, syncbuf, force_sync=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001265 """Perform only the local IO portion of the sync process.
1266 Network access is not required.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001267 """
Kevin Degiabaa7f32014-11-12 11:27:45 -07001268 self._InitWorkTree(force_sync=force_sync)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001269 all_refs = self.bare_ref.all
1270 self.CleanPublishedCache(all_refs)
1271 revid = self.GetRevisionId(all_refs)
Skyler Kaufman835cd682011-03-08 12:14:41 -08001272
David Pursehouse1d947b32012-10-25 12:23:11 +09001273 def _doff():
1274 self._FastForward(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001275 self._CopyAndLinkFiles()
David Pursehouse1d947b32012-10-25 12:23:11 +09001276
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001277 head = self.work_git.GetHead()
1278 if head.startswith(R_HEADS):
1279 branch = head[len(R_HEADS):]
1280 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001281 head = all_refs[head]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001282 except KeyError:
1283 head = None
1284 else:
1285 branch = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001286
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001287 if branch is None or syncbuf.detach_head:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001288 # Currently on a detached HEAD. The user is assumed to
1289 # not have any local modifications worth worrying about.
1290 #
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -07001291 if self.IsRebaseInProgress():
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001292 syncbuf.fail(self, _PriorSyncFailedError())
1293 return
1294
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001295 if head == revid:
1296 # No changes; don't do anything further.
Florian Vallee7cf1b362012-06-07 17:11:42 +02001297 # Except if the head needs to be detached
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001298 #
Florian Vallee7cf1b362012-06-07 17:11:42 +02001299 if not syncbuf.detach_head:
Dan Willemsen029eaf32015-09-03 12:52:28 -07001300 # The copy/linkfile config may have changed.
1301 self._CopyAndLinkFiles()
Florian Vallee7cf1b362012-06-07 17:11:42 +02001302 return
1303 else:
1304 lost = self._revlist(not_rev(revid), HEAD)
1305 if lost:
1306 syncbuf.info(self, "discarding %d commits", len(lost))
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001307
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001308 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001309 self._Checkout(revid, quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001310 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001311 syncbuf.fail(self, e)
1312 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001313 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001314 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001315
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001316 if head == revid:
1317 # No changes; don't do anything further.
1318 #
Dan Willemsen029eaf32015-09-03 12:52:28 -07001319 # The copy/linkfile config may have changed.
1320 self._CopyAndLinkFiles()
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001321 return
1322
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001323 branch = self.GetBranch(branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001324
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001325 if not branch.LocalMerge:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001326 # The current branch has no tracking configuration.
Anatol Pomazau2a32f6a2011-08-30 10:52:33 -07001327 # Jump off it to a detached HEAD.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001328 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001329 syncbuf.info(self,
1330 "leaving %s; does not track upstream",
1331 branch.name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001332 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001333 self._Checkout(revid, quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001334 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001335 syncbuf.fail(self, e)
1336 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001337 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001338 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001339
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001340 upstream_gain = self._revlist(not_rev(HEAD), revid)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001341 pub = self.WasPublished(branch.name, all_refs)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001342 if pub:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001343 not_merged = self._revlist(not_rev(revid), pub)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001344 if not_merged:
1345 if upstream_gain:
1346 # The user has published this branch and some of those
1347 # commits are not yet merged upstream. We do not want
1348 # to rewrite the published commits so we punt.
1349 #
Daniel Sandler4c50dee2010-03-02 15:38:03 -05001350 syncbuf.fail(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001351 "branch %s is published (but not merged) and is now "
1352 "%d commits behind" % (branch.name, len(upstream_gain)))
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001353 return
Shawn O. Pearce05f66b62009-04-21 08:26:32 -07001354 elif pub == head:
1355 # All published commits are merged, and thus we are a
1356 # strict subset. We can fast-forward safely.
Shawn O. Pearcea54c5272008-10-30 11:03:00 -07001357 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001358 syncbuf.later1(self, _doff)
1359 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001360
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001361 # Examine the local commits not in the remote. Find the
1362 # last one attributed to this user, if any.
1363 #
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001364 local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001365 last_mine = None
1366 cnt_mine = 0
1367 for commit in local_changes:
Chirayu Desai0eb35cb2013-11-19 18:46:29 +05301368 commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001369 if committer_email == self.UserEmail:
1370 last_mine = commit_id
1371 cnt_mine += 1
1372
Shawn O. Pearceda88ff42009-06-03 11:09:12 -07001373 if not upstream_gain and cnt_mine == len(local_changes):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001374 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001375
1376 if self.IsDirty(consider_untracked=False):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001377 syncbuf.fail(self, _DirtyError())
1378 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001379
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001380 # If the upstream switched on us, warn the user.
1381 #
1382 if branch.merge != self.revisionExpr:
1383 if branch.merge and self.revisionExpr:
1384 syncbuf.info(self,
1385 'manifest switched %s...%s',
1386 branch.merge,
1387 self.revisionExpr)
1388 elif branch.merge:
1389 syncbuf.info(self,
1390 'manifest no longer tracks %s',
1391 branch.merge)
1392
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001393 if cnt_mine < len(local_changes):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001394 # Upstream rebased. Not everything in HEAD
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001395 # was created by this user.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001396 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001397 syncbuf.info(self,
1398 "discarding %d commits removed from upstream",
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001399 len(local_changes) - cnt_mine)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001400
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001401 branch.remote = self.GetRemote(self.remote.name)
Anatol Pomazaucd7c5de2012-03-20 13:45:00 -07001402 if not ID_RE.match(self.revisionExpr):
1403 # in case of manifest sync the revisionExpr might be a SHA1
1404 branch.merge = self.revisionExpr
Conley Owens04f2f0e2014-10-01 17:22:46 -07001405 if not branch.merge.startswith('refs/'):
1406 branch.merge = R_HEADS + branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001407 branch.Save()
1408
Mike Pontillod3153822012-02-28 11:53:24 -08001409 if cnt_mine > 0 and self.rebase:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001410 def _dorebase():
Anthony King7bdac712014-07-16 12:56:40 +01001411 self._Rebase(upstream='%s^1' % last_mine, onto=revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001412 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001413 syncbuf.later2(self, _dorebase)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001414 elif local_changes:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001415 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001416 self._ResetHard(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001417 self._CopyAndLinkFiles()
Sarah Owensa5be53f2012-09-09 15:37:57 -07001418 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001419 syncbuf.fail(self, e)
1420 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001421 else:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001422 syncbuf.later1(self, _doff)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001423
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001424 def AddCopyFile(self, src, dest, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001425 # dest should already be an absolute path, but src is project relative
1426 # make src an absolute path
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001427 abssrc = os.path.join(self.worktree, src)
1428 self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001429
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001430 def AddLinkFile(self, src, dest, absdest):
1431 # dest should already be an absolute path, but src is project relative
Colin Cross0184dcc2015-05-05 00:24:54 -07001432 # make src relative path to dest
1433 absdestdir = os.path.dirname(absdest)
1434 relsrc = os.path.relpath(os.path.join(self.worktree, src), absdestdir)
Wink Saville4c426ef2015-06-03 08:05:17 -07001435 self.linkfiles.append(_LinkFile(self.worktree, src, dest, relsrc, absdest))
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001436
James W. Mills24c13082012-04-12 15:04:13 -05001437 def AddAnnotation(self, name, value, keep):
1438 self.annotations.append(_Annotation(name, value, keep))
1439
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001440 def DownloadPatchSet(self, change_id, patch_id):
1441 """Download a single patch set of a single change to FETCH_HEAD.
1442 """
1443 remote = self.GetRemote(self.remote.name)
1444
1445 cmd = ['fetch', remote.name]
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001446 cmd.append('refs/changes/%2.2d/%d/%d'
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001447 % (change_id % 100, change_id, patch_id))
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001448 if GitCommand(self, cmd, bare=True).Wait() != 0:
1449 return None
1450 return DownloadedChange(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001451 self.GetRevisionId(),
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001452 change_id,
1453 patch_id,
1454 self.bare_git.rev_parse('FETCH_HEAD'))
1455
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001456
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001457# Branch Management ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001458
Simran Basib9a1b732015-08-20 12:19:28 -07001459 def StartBranch(self, name, branch_merge=''):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001460 """Create a new branch off the manifest's revision.
1461 """
Simran Basib9a1b732015-08-20 12:19:28 -07001462 if not branch_merge:
1463 branch_merge = self.revisionExpr
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001464 head = self.work_git.GetHead()
1465 if head == (R_HEADS + name):
1466 return True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001467
David Pursehouse8a68ff92012-09-24 12:15:13 +09001468 all_refs = self.bare_ref.all
Anthony King7bdac712014-07-16 12:56:40 +01001469 if R_HEADS + name in all_refs:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001470 return GitCommand(self,
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001471 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001472 capture_stdout=True,
1473 capture_stderr=True).Wait() == 0
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001474
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001475 branch = self.GetBranch(name)
1476 branch.remote = self.GetRemote(self.remote.name)
Simran Basib9a1b732015-08-20 12:19:28 -07001477 branch.merge = branch_merge
1478 if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
1479 branch.merge = R_HEADS + branch_merge
David Pursehouse8a68ff92012-09-24 12:15:13 +09001480 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001481
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001482 if head.startswith(R_HEADS):
1483 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001484 head = all_refs[head]
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001485 except KeyError:
1486 head = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001487 if revid and head and revid == head:
1488 ref = os.path.join(self.gitdir, R_HEADS + name)
1489 try:
1490 os.makedirs(os.path.dirname(ref))
1491 except OSError:
1492 pass
1493 _lwrite(ref, '%s\n' % revid)
1494 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1495 'ref: %s%s\n' % (R_HEADS, name))
1496 branch.Save()
1497 return True
1498
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001499 if GitCommand(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001500 ['checkout', '-b', branch.name, revid],
Anthony King7bdac712014-07-16 12:56:40 +01001501 capture_stdout=True,
1502 capture_stderr=True).Wait() == 0:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001503 branch.Save()
1504 return True
1505 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001506
Wink Saville02d79452009-04-10 13:01:24 -07001507 def CheckoutBranch(self, name):
1508 """Checkout a local topic branch.
Doug Anderson3ba5f952011-04-07 12:51:04 -07001509
1510 Args:
1511 name: The name of the branch to checkout.
1512
1513 Returns:
1514 True if the checkout succeeded; False if it didn't; None if the branch
1515 didn't exist.
Wink Saville02d79452009-04-10 13:01:24 -07001516 """
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001517 rev = R_HEADS + name
1518 head = self.work_git.GetHead()
1519 if head == rev:
1520 # Already on the branch
1521 #
1522 return True
Wink Saville02d79452009-04-10 13:01:24 -07001523
David Pursehouse8a68ff92012-09-24 12:15:13 +09001524 all_refs = self.bare_ref.all
Wink Saville02d79452009-04-10 13:01:24 -07001525 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001526 revid = all_refs[rev]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001527 except KeyError:
1528 # Branch does not exist in this project
1529 #
Doug Anderson3ba5f952011-04-07 12:51:04 -07001530 return None
Wink Saville02d79452009-04-10 13:01:24 -07001531
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001532 if head.startswith(R_HEADS):
1533 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001534 head = all_refs[head]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001535 except KeyError:
1536 head = None
1537
1538 if head == revid:
1539 # Same revision; just update HEAD to point to the new
1540 # target branch, but otherwise take no other action.
1541 #
1542 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1543 'ref: %s%s\n' % (R_HEADS, name))
1544 return True
1545
1546 return GitCommand(self,
1547 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001548 capture_stdout=True,
1549 capture_stderr=True).Wait() == 0
Wink Saville02d79452009-04-10 13:01:24 -07001550
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001551 def AbandonBranch(self, name):
1552 """Destroy a local topic branch.
Doug Andersondafb1d62011-04-07 11:46:59 -07001553
1554 Args:
1555 name: The name of the branch to abandon.
1556
1557 Returns:
1558 True if the abandon succeeded; False if it didn't; None if the branch
1559 didn't exist.
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001560 """
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001561 rev = R_HEADS + name
David Pursehouse8a68ff92012-09-24 12:15:13 +09001562 all_refs = self.bare_ref.all
1563 if rev not in all_refs:
Doug Andersondafb1d62011-04-07 11:46:59 -07001564 # Doesn't exist
1565 return None
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001566
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001567 head = self.work_git.GetHead()
1568 if head == rev:
1569 # We can't destroy the branch while we are sitting
1570 # on it. Switch to a detached HEAD.
1571 #
David Pursehouse8a68ff92012-09-24 12:15:13 +09001572 head = all_refs[head]
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001573
David Pursehouse8a68ff92012-09-24 12:15:13 +09001574 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001575 if head == revid:
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001576 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1577 '%s\n' % revid)
1578 else:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001579 self._Checkout(revid, quiet=True)
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001580
1581 return GitCommand(self,
1582 ['branch', '-D', name],
Anthony King7bdac712014-07-16 12:56:40 +01001583 capture_stdout=True,
1584 capture_stderr=True).Wait() == 0
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001585
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001586 def PruneHeads(self):
1587 """Prune any topic branches already merged into upstream.
1588 """
1589 cb = self.CurrentBranch
1590 kill = []
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001591 left = self._allrefs
1592 for name in left.keys():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001593 if name.startswith(R_HEADS):
1594 name = name[len(R_HEADS):]
1595 if cb is None or name != cb:
1596 kill.append(name)
1597
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001598 rev = self.GetRevisionId(left)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001599 if cb is not None \
1600 and not self._revlist(HEAD + '...' + rev) \
Anthony King7bdac712014-07-16 12:56:40 +01001601 and not self.IsDirty(consider_untracked=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001602 self.work_git.DetachHead(HEAD)
1603 kill.append(cb)
1604
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001605 if kill:
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07001606 old = self.bare_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001607
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001608 try:
1609 self.bare_git.DetachHead(rev)
1610
1611 b = ['branch', '-d']
1612 b.extend(kill)
1613 b = GitCommand(self, b, bare=True,
1614 capture_stdout=True,
1615 capture_stderr=True)
1616 b.Wait()
1617 finally:
Dan Willemsen1a799d12015-12-15 13:40:05 -08001618 if ID_RE.match(old):
1619 self.bare_git.DetachHead(old)
1620 else:
1621 self.bare_git.SetHead(old)
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001622 left = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001623
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001624 for branch in kill:
1625 if (R_HEADS + branch) not in left:
1626 self.CleanPublishedCache()
1627 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001628
1629 if cb and cb not in kill:
1630 kill.append(cb)
Shawn O. Pearce7c6c64d2009-03-02 12:38:13 -08001631 kill.sort()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001632
1633 kept = []
1634 for branch in kill:
Anthony King7bdac712014-07-16 12:56:40 +01001635 if R_HEADS + branch in left:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001636 branch = self.GetBranch(branch)
1637 base = branch.LocalMerge
1638 if not base:
1639 base = rev
1640 kept.append(ReviewableBranch(self, branch, base))
1641 return kept
1642
1643
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001644# Submodule Management ##
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001645
1646 def GetRegisteredSubprojects(self):
1647 result = []
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001648
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001649 def rec(subprojects):
1650 if not subprojects:
1651 return
1652 result.extend(subprojects)
1653 for p in subprojects:
1654 rec(p.subprojects)
1655 rec(self.subprojects)
1656 return result
1657
1658 def _GetSubmodules(self):
1659 # Unfortunately we cannot call `git submodule status --recursive` here
1660 # because the working tree might not exist yet, and it cannot be used
1661 # without a working tree in its current implementation.
1662
1663 def get_submodules(gitdir, rev):
1664 # Parse .gitmodules for submodule sub_paths and sub_urls
1665 sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
1666 if not sub_paths:
1667 return []
1668 # Run `git ls-tree` to read SHAs of submodule object, which happen to be
1669 # revision of submodule repository
1670 sub_revs = git_ls_tree(gitdir, rev, sub_paths)
1671 submodules = []
1672 for sub_path, sub_url in zip(sub_paths, sub_urls):
1673 try:
1674 sub_rev = sub_revs[sub_path]
1675 except KeyError:
1676 # Ignore non-exist submodules
1677 continue
1678 submodules.append((sub_rev, sub_path, sub_url))
1679 return submodules
1680
1681 re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
1682 re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001683
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001684 def parse_gitmodules(gitdir, rev):
1685 cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
1686 try:
Anthony King7bdac712014-07-16 12:56:40 +01001687 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1688 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001689 except GitError:
1690 return [], []
1691 if p.Wait() != 0:
1692 return [], []
1693
1694 gitmodules_lines = []
1695 fd, temp_gitmodules_path = tempfile.mkstemp()
1696 try:
1697 os.write(fd, p.stdout)
1698 os.close(fd)
1699 cmd = ['config', '--file', temp_gitmodules_path, '--list']
Anthony King7bdac712014-07-16 12:56:40 +01001700 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1701 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001702 if p.Wait() != 0:
1703 return [], []
1704 gitmodules_lines = p.stdout.split('\n')
1705 except GitError:
1706 return [], []
1707 finally:
1708 os.remove(temp_gitmodules_path)
1709
1710 names = set()
1711 paths = {}
1712 urls = {}
1713 for line in gitmodules_lines:
1714 if not line:
1715 continue
1716 m = re_path.match(line)
1717 if m:
1718 names.add(m.group(1))
1719 paths[m.group(1)] = m.group(2)
1720 continue
1721 m = re_url.match(line)
1722 if m:
1723 names.add(m.group(1))
1724 urls[m.group(1)] = m.group(2)
1725 continue
1726 names = sorted(names)
1727 return ([paths.get(name, '') for name in names],
1728 [urls.get(name, '') for name in names])
1729
1730 def git_ls_tree(gitdir, rev, paths):
1731 cmd = ['ls-tree', rev, '--']
1732 cmd.extend(paths)
1733 try:
Anthony King7bdac712014-07-16 12:56:40 +01001734 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1735 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001736 except GitError:
1737 return []
1738 if p.Wait() != 0:
1739 return []
1740 objects = {}
1741 for line in p.stdout.split('\n'):
1742 if not line.strip():
1743 continue
1744 object_rev, object_path = line.split()[2:4]
1745 objects[object_path] = object_rev
1746 return objects
1747
1748 try:
1749 rev = self.GetRevisionId()
1750 except GitError:
1751 return []
1752 return get_submodules(self.gitdir, rev)
1753
1754 def GetDerivedSubprojects(self):
1755 result = []
1756 if not self.Exists:
1757 # If git repo does not exist yet, querying its submodules will
1758 # mess up its states; so return here.
1759 return result
1760 for rev, path, url in self._GetSubmodules():
1761 name = self.manifest.GetSubprojectName(self, path)
David James8d201162013-10-11 17:03:19 -07001762 relpath, worktree, gitdir, objdir = \
1763 self.manifest.GetSubprojectPaths(self, name, path)
1764 project = self.manifest.paths.get(relpath)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001765 if project:
1766 result.extend(project.GetDerivedSubprojects())
1767 continue
David James8d201162013-10-11 17:03:19 -07001768
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001769 remote = RemoteSpec(self.remote.name,
Anthony King7bdac712014-07-16 12:56:40 +01001770 url=url,
1771 review=self.remote.review,
1772 revision=self.remote.revision)
1773 subproject = Project(manifest=self.manifest,
1774 name=name,
1775 remote=remote,
1776 gitdir=gitdir,
1777 objdir=objdir,
1778 worktree=worktree,
1779 relpath=relpath,
1780 revisionExpr=self.revisionExpr,
1781 revisionId=rev,
1782 rebase=self.rebase,
1783 groups=self.groups,
1784 sync_c=self.sync_c,
1785 sync_s=self.sync_s,
1786 parent=self,
1787 is_derived=True)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001788 result.append(subproject)
1789 result.extend(subproject.GetDerivedSubprojects())
1790 return result
1791
1792
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001793# Direct Git Commands ##
Chris AtLee2fb64662014-01-16 21:32:33 -05001794 def _CheckForSha1(self):
1795 try:
1796 # if revision (sha or tag) is not present then following function
1797 # throws an error.
1798 self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
1799 return True
1800 except GitError:
1801 # There is no such persistent revision. We have to fetch it.
1802 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001803
Julien Campergue335f5ef2013-10-16 11:02:35 +02001804 def _FetchArchive(self, tarpath, cwd=None):
1805 cmd = ['archive', '-v', '-o', tarpath]
1806 cmd.append('--remote=%s' % self.remote.url)
1807 cmd.append('--prefix=%s/' % self.relpath)
1808 cmd.append(self.revisionExpr)
1809
1810 command = GitCommand(self, cmd, cwd=cwd,
1811 capture_stdout=True,
1812 capture_stderr=True)
1813
1814 if command.Wait() != 0:
1815 raise GitError('git archive %s: %s' % (self.name, command.stderr))
1816
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001817 def _RemoteFetch(self, name=None,
1818 current_branch_only=False,
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001819 initial=False,
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001820 quiet=False,
Mitchel Humpherys597868b2012-10-29 10:18:34 -07001821 alt_dir=None,
David Pursehouse74cfd272015-10-14 10:50:15 +09001822 no_tags=False,
1823 prune=False):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001824
1825 is_sha1 = False
1826 tag_name = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001827 depth = None
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001828
David Pursehouse9bc422f2014-04-15 10:28:56 +09001829 # The depth should not be used when fetching to a mirror because
1830 # it will result in a shallow repository that cannot be cloned or
1831 # fetched from.
1832 if not self.manifest.IsMirror:
1833 if self.clone_depth:
1834 depth = self.clone_depth
1835 else:
1836 depth = self.manifest.manifestProject.config.GetString('repo.depth')
Conley Owense4978cf2015-02-03 18:06:16 -08001837 # The repo project should never be synced with partial depth
1838 if self.relpath == '.repo/repo':
1839 depth = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001840
Shawn Pearce69e04d82014-01-29 12:48:54 -08001841 if depth:
1842 current_branch_only = True
1843
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001844 if ID_RE.match(self.revisionExpr) is not None:
1845 is_sha1 = True
1846
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001847 if current_branch_only:
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001848 if self.revisionExpr.startswith(R_TAGS):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001849 # this is a tag and its sha1 value should never change
1850 tag_name = self.revisionExpr[len(R_TAGS):]
1851
1852 if is_sha1 or tag_name is not None:
Chris AtLee2fb64662014-01-16 21:32:33 -05001853 if self._CheckForSha1():
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001854 return True
Bertrand SIMONNET3000cda2014-11-25 16:19:29 -08001855 if is_sha1 and not depth:
1856 # When syncing a specific commit and --depth is not set:
1857 # * if upstream is explicitly specified and is not a sha1, fetch only
1858 # upstream as users expect only upstream to be fetch.
1859 # Note: The commit might not be in upstream in which case the sync
1860 # will fail.
1861 # * otherwise, fetch all branches to make sure we end up with the
1862 # specific commit.
Aymen Bouaziz037040f2016-06-28 12:27:23 +02001863 if self.upstream:
1864 current_branch_only = not ID_RE.match(self.upstream)
1865 else:
1866 current_branch_only = False
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001867
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001868 if not name:
1869 name = self.remote.name
Shawn O. Pearcefb231612009-04-10 18:53:46 -07001870
1871 ssh_proxy = False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001872 remote = self.GetRemote(name)
1873 if remote.PreConnectFetch():
Shawn O. Pearcefb231612009-04-10 18:53:46 -07001874 ssh_proxy = True
1875
Shawn O. Pearce88443382010-10-08 10:02:09 +02001876 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001877 if alt_dir and 'objects' == os.path.basename(alt_dir):
1878 ref_dir = os.path.dirname(alt_dir)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001879 packed_refs = os.path.join(self.gitdir, 'packed-refs')
1880 remote = self.GetRemote(name)
1881
David Pursehouse8a68ff92012-09-24 12:15:13 +09001882 all_refs = self.bare_ref.all
1883 ids = set(all_refs.values())
Shawn O. Pearce88443382010-10-08 10:02:09 +02001884 tmp = set()
1885
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301886 for r, ref_id in GitRefs(ref_dir).all.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09001887 if r not in all_refs:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001888 if r.startswith(R_TAGS) or remote.WritesTo(r):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001889 all_refs[r] = ref_id
1890 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001891 continue
1892
David Pursehouse8a68ff92012-09-24 12:15:13 +09001893 if ref_id in ids:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001894 continue
1895
David Pursehouse8a68ff92012-09-24 12:15:13 +09001896 r = 'refs/_alt/%s' % ref_id
1897 all_refs[r] = ref_id
1898 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001899 tmp.add(r)
1900
Shawn O. Pearce88443382010-10-08 10:02:09 +02001901 tmp_packed = ''
1902 old_packed = ''
1903
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301904 for r in sorted(all_refs):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001905 line = '%s %s\n' % (all_refs[r], r)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001906 tmp_packed += line
1907 if r not in tmp:
1908 old_packed += line
1909
1910 _lwrite(packed_refs, tmp_packed)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001911 else:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001912 alt_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02001913
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001914 cmd = ['fetch']
Doug Anderson30d45292011-05-04 15:01:04 -07001915
Conley Owensf97e8382015-01-21 11:12:46 -08001916 if depth:
Doug Anderson30d45292011-05-04 15:01:04 -07001917 cmd.append('--depth=%s' % depth)
Dan Willemseneeab6862015-08-03 13:11:53 -07001918 else:
1919 # If this repo has shallow objects, then we don't know which refs have
1920 # shallow objects or not. Tell git to unshallow all fetched refs. Don't
1921 # do this with projects that don't have shallow objects, since it is less
1922 # efficient.
1923 if os.path.exists(os.path.join(self.gitdir, 'shallow')):
1924 cmd.append('--depth=2147483647')
Doug Anderson30d45292011-05-04 15:01:04 -07001925
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001926 if quiet:
1927 cmd.append('--quiet')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001928 if not self.worktree:
1929 cmd.append('--update-head-ok')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001930 cmd.append(name)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001931
Mitchel Humpherys26c45a72014-03-10 14:21:59 -07001932 # If using depth then we should not get all the tags since they may
1933 # be outside of the depth.
1934 if no_tags or depth:
1935 cmd.append('--no-tags')
1936 else:
1937 cmd.append('--tags')
1938
David Pursehouse74cfd272015-10-14 10:50:15 +09001939 if prune:
1940 cmd.append('--prune')
1941
Conley Owens80b87fe2014-05-09 17:13:44 -07001942 spec = []
Brian Harring14a66742012-09-28 20:21:57 -07001943 if not current_branch_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001944 # Fetch whole repo
Conley Owens80b87fe2014-05-09 17:13:44 -07001945 spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001946 elif tag_name is not None:
Conley Owens80b87fe2014-05-09 17:13:44 -07001947 spec.append('tag')
1948 spec.append(tag_name)
Nasser Grainawi04e52d62014-09-30 13:34:52 -06001949
David Pursehouse403b64e2015-04-27 10:41:33 +09001950 if not self.manifest.IsMirror:
1951 branch = self.revisionExpr
Kevin Degi679bac42015-06-22 15:31:26 -06001952 if is_sha1 and depth and git_require((1, 8, 3)):
David Pursehouse403b64e2015-04-27 10:41:33 +09001953 # Shallow checkout of a specific commit, fetch from that commit and not
1954 # the heads only as the commit might be deeper in the history.
1955 spec.append(branch)
1956 else:
1957 if is_sha1:
1958 branch = self.upstream
1959 if branch is not None and branch.strip():
1960 if not branch.startswith('refs/'):
1961 branch = R_HEADS + branch
1962 spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
Conley Owens80b87fe2014-05-09 17:13:44 -07001963 cmd.extend(spec)
1964
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001965 ok = False
David Pursehouse8a68ff92012-09-24 12:15:13 +09001966 for _i in range(2):
John L. Villalovos9c76f672015-03-16 20:49:10 -07001967 gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy)
John L. Villalovos126e2982015-01-29 21:58:12 -08001968 ret = gitcmd.Wait()
Brian Harring14a66742012-09-28 20:21:57 -07001969 if ret == 0:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001970 ok = True
1971 break
John L. Villalovos126e2982015-01-29 21:58:12 -08001972 # If needed, run the 'git remote prune' the first time through the loop
1973 elif (not _i and
1974 "error:" in gitcmd.stderr and
1975 "git remote prune" in gitcmd.stderr):
1976 prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
John L. Villalovos9c76f672015-03-16 20:49:10 -07001977 ssh_proxy=ssh_proxy)
John L. Villalovose30f46b2015-02-25 14:27:02 -08001978 ret = prunecmd.Wait()
John L. Villalovose30f46b2015-02-25 14:27:02 -08001979 if ret:
John L. Villalovos126e2982015-01-29 21:58:12 -08001980 break
1981 continue
Brian Harring14a66742012-09-28 20:21:57 -07001982 elif current_branch_only and is_sha1 and ret == 128:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001983 # Exit code 128 means "couldn't find the ref you asked for"; if we're
1984 # in sha1 mode, we just tried sync'ing from the upstream field; it
1985 # doesn't exist, thus abort the optimization attempt and do a full sync.
Brian Harring14a66742012-09-28 20:21:57 -07001986 break
Colin Crossc4b301f2015-05-13 00:10:02 -07001987 elif ret < 0:
1988 # Git died with a signal, exit immediately
1989 break
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001990 time.sleep(random.randint(30, 45))
Shawn O. Pearce88443382010-10-08 10:02:09 +02001991
1992 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001993 if alt_dir:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001994 if old_packed != '':
1995 _lwrite(packed_refs, old_packed)
1996 else:
1997 os.remove(packed_refs)
1998 self.bare_git.pack_refs('--all', '--prune')
Brian Harring14a66742012-09-28 20:21:57 -07001999
2000 if is_sha1 and current_branch_only and self.upstream:
2001 # We just synced the upstream given branch; verify we
2002 # got what we wanted, else trigger a second run of all
2003 # refs.
Chris AtLee2fb64662014-01-16 21:32:33 -05002004 if not self._CheckForSha1():
Kevin Degi679bac42015-06-22 15:31:26 -06002005 if not depth:
2006 # Avoid infinite recursion when depth is True (since depth implies
2007 # current_branch_only)
2008 return self._RemoteFetch(name=name, current_branch_only=False,
2009 initial=False, quiet=quiet, alt_dir=alt_dir)
2010 if self.clone_depth:
2011 self.clone_depth = None
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002012 return self._RemoteFetch(name=name,
2013 current_branch_only=current_branch_only,
Kevin Degi679bac42015-06-22 15:31:26 -06002014 initial=False, quiet=quiet, alt_dir=alt_dir)
Brian Harring14a66742012-09-28 20:21:57 -07002015
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002016 return ok
Shawn O. Pearce88443382010-10-08 10:02:09 +02002017
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002018 def _ApplyCloneBundle(self, initial=False, quiet=False):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002019 if initial and \
2020 (self.manifest.manifestProject.config.GetString('repo.depth') or
2021 self.clone_depth):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002022 return False
2023
2024 remote = self.GetRemote(self.remote.name)
2025 bundle_url = remote.url + '/clone.bundle'
2026 bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002027 if GetSchemeFromUrl(bundle_url) not in ('http', 'https',
2028 'persistent-http',
2029 'persistent-https'):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002030 return False
2031
2032 bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
2033 bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
2034
2035 exist_dst = os.path.exists(bundle_dst)
2036 exist_tmp = os.path.exists(bundle_tmp)
2037
2038 if not initial and not exist_dst and not exist_tmp:
2039 return False
2040
2041 if not exist_dst:
2042 exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
2043 if not exist_dst:
2044 return False
2045
2046 cmd = ['fetch']
2047 if quiet:
2048 cmd.append('--quiet')
2049 if not self.worktree:
2050 cmd.append('--update-head-ok')
2051 cmd.append(bundle_dst)
2052 for f in remote.fetch:
2053 cmd.append(str(f))
2054 cmd.append('refs/tags/*:refs/tags/*')
2055
2056 ok = GitCommand(self, cmd, bare=True).Wait() == 0
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002057 if os.path.exists(bundle_dst):
2058 os.remove(bundle_dst)
2059 if os.path.exists(bundle_tmp):
2060 os.remove(bundle_tmp)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002061 return ok
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002062
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002063 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002064 if os.path.exists(dstPath):
2065 os.remove(dstPath)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002066
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002067 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002068 if quiet:
2069 cmd += ['--silent']
2070 if os.path.exists(tmpPath):
2071 size = os.stat(tmpPath).st_size
2072 if size >= 1024:
2073 cmd += ['--continue-at', '%d' % (size,)]
2074 else:
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002075 os.remove(tmpPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002076 if 'http_proxy' in os.environ and 'darwin' == sys.platform:
2077 cmd += ['--proxy', os.environ['http_proxy']]
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002078 with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy):
Dave Borowitz137d0132015-01-02 11:12:54 -08002079 if cookiefile:
Dave Borowitz4abf8e62015-01-02 11:39:04 -08002080 cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
Dave Borowitz137d0132015-01-02 11:12:54 -08002081 if srcUrl.startswith('persistent-'):
2082 srcUrl = srcUrl[len('persistent-'):]
2083 cmd += [srcUrl]
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002084
Dave Borowitz137d0132015-01-02 11:12:54 -08002085 if IsTrace():
2086 Trace('%s', ' '.join(cmd))
2087 try:
2088 proc = subprocess.Popen(cmd)
2089 except OSError:
2090 return False
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002091
Dave Borowitz137d0132015-01-02 11:12:54 -08002092 curlret = proc.wait()
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002093
Dave Borowitz137d0132015-01-02 11:12:54 -08002094 if curlret == 22:
2095 # From curl man page:
2096 # 22: HTTP page not retrieved. The requested url was not found or
2097 # returned another error with the HTTP error code being 400 or above.
2098 # This return code only appears if -f, --fail is used.
2099 if not quiet:
2100 print("Server does not provide clone.bundle; ignoring.",
2101 file=sys.stderr)
2102 return False
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002103
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002104 if os.path.exists(tmpPath):
Kris Giesingc8d882a2014-12-23 13:02:32 -08002105 if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002106 os.rename(tmpPath, dstPath)
2107 return True
2108 else:
2109 os.remove(tmpPath)
2110 return False
2111 else:
2112 return False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002113
Kris Giesingc8d882a2014-12-23 13:02:32 -08002114 def _IsValidBundle(self, path, quiet):
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002115 try:
2116 with open(path) as f:
2117 if f.read(16) == '# v2 git bundle\n':
2118 return True
2119 else:
Kris Giesingc8d882a2014-12-23 13:02:32 -08002120 if not quiet:
2121 print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002122 return False
2123 except OSError:
2124 return False
2125
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002126 def _Checkout(self, rev, quiet=False):
2127 cmd = ['checkout']
2128 if quiet:
2129 cmd.append('-q')
2130 cmd.append(rev)
2131 cmd.append('--')
2132 if GitCommand(self, cmd).Wait() != 0:
2133 if self._allrefs:
2134 raise GitError('%s checkout %s ' % (self.name, rev))
2135
Anthony King7bdac712014-07-16 12:56:40 +01002136 def _CherryPick(self, rev):
Pierre Tardye5a21222011-03-24 16:28:18 +01002137 cmd = ['cherry-pick']
2138 cmd.append(rev)
2139 cmd.append('--')
2140 if GitCommand(self, cmd).Wait() != 0:
2141 if self._allrefs:
2142 raise GitError('%s cherry-pick %s ' % (self.name, rev))
2143
Anthony King7bdac712014-07-16 12:56:40 +01002144 def _Revert(self, rev):
Erwan Mahea94f1622011-08-19 13:56:09 +02002145 cmd = ['revert']
2146 cmd.append('--no-edit')
2147 cmd.append(rev)
2148 cmd.append('--')
2149 if GitCommand(self, cmd).Wait() != 0:
2150 if self._allrefs:
2151 raise GitError('%s revert %s ' % (self.name, rev))
2152
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002153 def _ResetHard(self, rev, quiet=True):
2154 cmd = ['reset', '--hard']
2155 if quiet:
2156 cmd.append('-q')
2157 cmd.append(rev)
2158 if GitCommand(self, cmd).Wait() != 0:
2159 raise GitError('%s reset --hard %s ' % (self.name, rev))
2160
Anthony King7bdac712014-07-16 12:56:40 +01002161 def _Rebase(self, upstream, onto=None):
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002162 cmd = ['rebase']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002163 if onto is not None:
2164 cmd.extend(['--onto', onto])
2165 cmd.append(upstream)
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002166 if GitCommand(self, cmd).Wait() != 0:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002167 raise GitError('%s rebase %s ' % (self.name, upstream))
2168
Pierre Tardy3d125942012-05-04 12:18:12 +02002169 def _FastForward(self, head, ffonly=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002170 cmd = ['merge', head]
Pierre Tardy3d125942012-05-04 12:18:12 +02002171 if ffonly:
2172 cmd.append("--ff-only")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002173 if GitCommand(self, cmd).Wait() != 0:
2174 raise GitError('%s merge %s ' % (self.name, head))
2175
Kevin Degiabaa7f32014-11-12 11:27:45 -07002176 def _InitGitDir(self, mirror_git=None, force_sync=False):
Kevin Degi384b3c52014-10-16 16:02:58 -06002177 init_git_dir = not os.path.exists(self.gitdir)
2178 init_obj_dir = not os.path.exists(self.objdir)
Kevin Degib1a07b82015-07-27 13:33:43 -06002179 try:
2180 # Initialize the bare repository, which contains all of the objects.
2181 if init_obj_dir:
2182 os.makedirs(self.objdir)
2183 self.bare_objdir.init()
David James8d201162013-10-11 17:03:19 -07002184
Kevin Degib1a07b82015-07-27 13:33:43 -06002185 # If we have a separate directory to hold refs, initialize it as well.
2186 if self.objdir != self.gitdir:
2187 if init_git_dir:
2188 os.makedirs(self.gitdir)
2189
2190 if init_obj_dir or init_git_dir:
2191 self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
2192 copy_all=True)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002193 try:
2194 self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
2195 except GitError as e:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002196 if force_sync:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002197 print("Retrying clone after deleting %s" %
2198 self.gitdir, file=sys.stderr)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002199 try:
2200 shutil.rmtree(os.path.realpath(self.gitdir))
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002201 if self.worktree and os.path.exists(os.path.realpath
2202 (self.worktree)):
Kevin Degiabaa7f32014-11-12 11:27:45 -07002203 shutil.rmtree(os.path.realpath(self.worktree))
2204 return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
2205 except:
2206 raise e
2207 raise e
Kevin Degib1a07b82015-07-27 13:33:43 -06002208
Kevin Degi384b3c52014-10-16 16:02:58 -06002209 if init_git_dir:
Kevin Degib1a07b82015-07-27 13:33:43 -06002210 mp = self.manifest.manifestProject
2211 ref_dir = mp.config.GetString('repo.reference') or ''
Kevin Degi384b3c52014-10-16 16:02:58 -06002212
Kevin Degib1a07b82015-07-27 13:33:43 -06002213 if ref_dir or mirror_git:
2214 if not mirror_git:
2215 mirror_git = os.path.join(ref_dir, self.name + '.git')
2216 repo_git = os.path.join(ref_dir, '.repo', 'projects',
2217 self.relpath + '.git')
Shawn O. Pearce2816d4f2009-03-03 17:53:18 -08002218
Kevin Degib1a07b82015-07-27 13:33:43 -06002219 if os.path.exists(mirror_git):
2220 ref_dir = mirror_git
Shawn O. Pearce88443382010-10-08 10:02:09 +02002221
Kevin Degib1a07b82015-07-27 13:33:43 -06002222 elif os.path.exists(repo_git):
2223 ref_dir = repo_git
Shawn O. Pearce88443382010-10-08 10:02:09 +02002224
Kevin Degib1a07b82015-07-27 13:33:43 -06002225 else:
2226 ref_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02002227
Kevin Degib1a07b82015-07-27 13:33:43 -06002228 if ref_dir:
2229 _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
2230 os.path.join(ref_dir, 'objects') + '\n')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002231
Kevin Degib1a07b82015-07-27 13:33:43 -06002232 self._UpdateHooks()
2233
2234 m = self.manifest.manifestProject.config
2235 for key in ['user.name', 'user.email']:
2236 if m.Has(key, include_defaults=False):
2237 self.config.SetString(key, m.GetString(key))
2238 if self.manifest.IsMirror:
2239 self.config.SetString('core.bare', 'true')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002240 else:
Kevin Degib1a07b82015-07-27 13:33:43 -06002241 self.config.SetString('core.bare', None)
2242 except Exception:
2243 if init_obj_dir and os.path.exists(self.objdir):
2244 shutil.rmtree(self.objdir)
2245 if init_git_dir and os.path.exists(self.gitdir):
2246 shutil.rmtree(self.gitdir)
2247 raise
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002248
Jimmie Westera0444582012-10-24 13:44:42 +02002249 def _UpdateHooks(self):
2250 if os.path.exists(self.gitdir):
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002251 self._InitHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002252
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002253 def _InitHooks(self):
Jesse Hall672cc492013-11-27 11:17:13 -08002254 hooks = os.path.realpath(self._gitdir_path('hooks'))
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002255 if not os.path.exists(hooks):
2256 os.makedirs(hooks)
Jonathan Nieder93719792015-03-17 11:29:58 -07002257 for stock_hook in _ProjectHooks():
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002258 name = os.path.basename(stock_hook)
2259
Victor Boivie65e0f352011-04-18 11:23:29 +02002260 if name in ('commit-msg',) and not self.remote.review \
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002261 and self is not self.manifest.manifestProject:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002262 # Don't install a Gerrit Code Review hook if this
2263 # project does not appear to use it for reviews.
2264 #
Victor Boivie65e0f352011-04-18 11:23:29 +02002265 # Since the manifest project is one of those, but also
2266 # managed through gerrit, it's excluded
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002267 continue
2268
2269 dst = os.path.join(hooks, name)
2270 if os.path.islink(dst):
2271 continue
2272 if os.path.exists(dst):
2273 if filecmp.cmp(stock_hook, dst, shallow=False):
2274 os.remove(dst)
2275 else:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002276 _warn("%s: Not replacing locally modified %s hook",
2277 self.relpath, name)
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002278 continue
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002279 try:
Mickaël Salaünb9477bc2012-08-05 13:39:26 +02002280 os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
Sarah Owensa5be53f2012-09-09 15:37:57 -07002281 except OSError as e:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002282 if e.errno == errno.EPERM:
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002283 raise GitError('filesystem must support symlinks')
2284 else:
2285 raise
2286
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002287 def _InitRemote(self):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002288 if self.remote.url:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002289 remote = self.GetRemote(self.remote.name)
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002290 remote.url = self.remote.url
2291 remote.review = self.remote.review
2292 remote.projectname = self.name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002293
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002294 if self.worktree:
2295 remote.ResetFetch(mirror=False)
2296 else:
2297 remote.ResetFetch(mirror=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002298 remote.Save()
2299
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002300 def _InitMRef(self):
2301 if self.manifest.branch:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002302 self._InitAnyMRef(R_M + self.manifest.branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002303
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002304 def _InitMirrorHead(self):
Shawn O. Pearcefe200ee2009-06-01 15:28:21 -07002305 self._InitAnyMRef(HEAD)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002306
2307 def _InitAnyMRef(self, ref):
2308 cur = self.bare_ref.symref(ref)
2309
2310 if self.revisionId:
2311 if cur != '' or self.bare_ref.get(ref) != self.revisionId:
2312 msg = 'manifest set to %s' % self.revisionId
2313 dst = self.revisionId + '^0'
Anthony King7bdac712014-07-16 12:56:40 +01002314 self.bare_git.UpdateRef(ref, dst, message=msg, detach=True)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002315 else:
2316 remote = self.GetRemote(self.remote.name)
2317 dst = remote.ToLocal(self.revisionExpr)
2318 if cur != dst:
2319 msg = 'manifest set to %s' % self.revisionExpr
2320 self.bare_git.symbolic_ref('-m', msg, ref, dst)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002321
Kevin Degi384b3c52014-10-16 16:02:58 -06002322 def _CheckDirReference(self, srcdir, destdir, share_refs):
Dan Willemsenbdb866e2016-04-05 17:22:02 -07002323 symlink_files = self.shareable_files[:]
2324 symlink_dirs = self.shareable_dirs[:]
Kevin Degi384b3c52014-10-16 16:02:58 -06002325 if share_refs:
2326 symlink_files += self.working_tree_files
2327 symlink_dirs += self.working_tree_dirs
2328 to_symlink = symlink_files + symlink_dirs
2329 for name in set(to_symlink):
2330 dst = os.path.realpath(os.path.join(destdir, name))
2331 if os.path.lexists(dst):
2332 src = os.path.realpath(os.path.join(srcdir, name))
2333 # Fail if the links are pointing to the wrong place
2334 if src != dst:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002335 raise GitError('--force-sync not enabled; cannot overwrite a local '
Simon Ruggierf9b76832015-07-31 17:18:34 -04002336 'work tree. If you\'re comfortable with the '
2337 'possibility of losing the work tree\'s git metadata,'
2338 ' use `repo sync --force-sync {0}` to '
2339 'proceed.'.format(self.relpath))
Kevin Degi384b3c52014-10-16 16:02:58 -06002340
David James8d201162013-10-11 17:03:19 -07002341 def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
2342 """Update |dotgit| to reference |gitdir|, using symlinks where possible.
2343
2344 Args:
2345 gitdir: The bare git repository. Must already be initialized.
2346 dotgit: The repository you would like to initialize.
2347 share_refs: If true, |dotgit| will store its refs under |gitdir|.
2348 Only one work tree can store refs under a given |gitdir|.
2349 copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
2350 This saves you the effort of initializing |dotgit| yourself.
2351 """
Dan Willemsenbdb866e2016-04-05 17:22:02 -07002352 symlink_files = self.shareable_files[:]
2353 symlink_dirs = self.shareable_dirs[:]
David James8d201162013-10-11 17:03:19 -07002354 if share_refs:
Kevin Degi384b3c52014-10-16 16:02:58 -06002355 symlink_files += self.working_tree_files
2356 symlink_dirs += self.working_tree_dirs
David James8d201162013-10-11 17:03:19 -07002357 to_symlink = symlink_files + symlink_dirs
2358
2359 to_copy = []
2360 if copy_all:
2361 to_copy = os.listdir(gitdir)
2362
Dan Willemsen2a3e1522015-07-30 20:43:33 -07002363 dotgit = os.path.realpath(dotgit)
David James8d201162013-10-11 17:03:19 -07002364 for name in set(to_copy).union(to_symlink):
2365 try:
2366 src = os.path.realpath(os.path.join(gitdir, name))
Dan Willemsen2a3e1522015-07-30 20:43:33 -07002367 dst = os.path.join(dotgit, name)
David James8d201162013-10-11 17:03:19 -07002368
Kevin Degi384b3c52014-10-16 16:02:58 -06002369 if os.path.lexists(dst):
2370 continue
David James8d201162013-10-11 17:03:19 -07002371
2372 # If the source dir doesn't exist, create an empty dir.
2373 if name in symlink_dirs and not os.path.lexists(src):
2374 os.makedirs(src)
2375
Conley Owens80b87fe2014-05-09 17:13:44 -07002376 # If the source file doesn't exist, ensure the destination
2377 # file doesn't either.
2378 if name in symlink_files and not os.path.lexists(src):
2379 try:
2380 os.remove(dst)
2381 except OSError:
2382 pass
2383
David James8d201162013-10-11 17:03:19 -07002384 if name in to_symlink:
2385 os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
2386 elif copy_all and not os.path.islink(dst):
2387 if os.path.isdir(src):
2388 shutil.copytree(src, dst)
2389 elif os.path.isfile(src):
2390 shutil.copy(src, dst)
2391 except OSError as e:
2392 if e.errno == errno.EPERM:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002393 raise DownloadError('filesystem must support symlinks')
David James8d201162013-10-11 17:03:19 -07002394 else:
2395 raise
2396
Kevin Degiabaa7f32014-11-12 11:27:45 -07002397 def _InitWorkTree(self, force_sync=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002398 dotgit = os.path.join(self.worktree, '.git')
Kevin Degi384b3c52014-10-16 16:02:58 -06002399 init_dotgit = not os.path.exists(dotgit)
Kevin Degib1a07b82015-07-27 13:33:43 -06002400 try:
2401 if init_dotgit:
2402 os.makedirs(dotgit)
2403 self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
2404 copy_all=False)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002405
Kevin Degiabaa7f32014-11-12 11:27:45 -07002406 try:
2407 self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
2408 except GitError as e:
2409 if force_sync:
2410 try:
2411 shutil.rmtree(dotgit)
2412 return self._InitWorkTree(force_sync=False)
2413 except:
2414 raise e
2415 raise e
Kevin Degi384b3c52014-10-16 16:02:58 -06002416
Kevin Degib1a07b82015-07-27 13:33:43 -06002417 if init_dotgit:
2418 _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002419
Kevin Degib1a07b82015-07-27 13:33:43 -06002420 cmd = ['read-tree', '--reset', '-u']
2421 cmd.append('-v')
2422 cmd.append(HEAD)
2423 if GitCommand(self, cmd).Wait() != 0:
2424 raise GitError("cannot initialize work tree")
Victor Boivie0960b5b2010-11-26 13:42:13 +01002425
Kevin Degib1a07b82015-07-27 13:33:43 -06002426 self._CopyAndLinkFiles()
2427 except Exception:
2428 if init_dotgit:
2429 shutil.rmtree(dotgit)
2430 raise
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002431
2432 def _gitdir_path(self, path):
David James8d201162013-10-11 17:03:19 -07002433 return os.path.realpath(os.path.join(self.gitdir, path))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002434
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002435 def _revlist(self, *args, **kw):
2436 a = []
2437 a.extend(args)
2438 a.append('--')
2439 return self.work_git.rev_list(*a, **kw)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002440
2441 @property
2442 def _allrefs(self):
Shawn O. Pearced237b692009-04-17 18:49:50 -07002443 return self.bare_ref.all
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002444
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002445 def _getLogs(self, rev1, rev2, oneline=False, color=True, pretty_format=None):
Julien Camperguedd654222014-01-09 16:21:37 +01002446 """Get logs between two revisions of this project."""
2447 comp = '..'
2448 if rev1:
2449 revs = [rev1]
2450 if rev2:
2451 revs.extend([comp, rev2])
2452 cmd = ['log', ''.join(revs)]
2453 out = DiffColoring(self.config)
2454 if out.is_on and color:
2455 cmd.append('--color')
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002456 if pretty_format is not None:
2457 cmd.append('--pretty=format:%s' % pretty_format)
Julien Camperguedd654222014-01-09 16:21:37 +01002458 if oneline:
2459 cmd.append('--oneline')
2460
2461 try:
2462 log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
2463 if log.Wait() == 0:
2464 return log.stdout
2465 except GitError:
2466 # worktree may not exist if groups changed for example. In that case,
2467 # try in gitdir instead.
2468 if not os.path.exists(self.worktree):
2469 return self.bare_git.log(*cmd[1:])
2470 else:
2471 raise
2472 return None
2473
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002474 def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True,
2475 pretty_format=None):
Julien Camperguedd654222014-01-09 16:21:37 +01002476 """Get the list of logs from this revision to given revisionId"""
2477 logs = {}
2478 selfId = self.GetRevisionId(self._allrefs)
2479 toId = toProject.GetRevisionId(toProject._allrefs)
2480
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002481 logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color,
2482 pretty_format=pretty_format)
2483 logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color,
2484 pretty_format=pretty_format)
Julien Camperguedd654222014-01-09 16:21:37 +01002485 return logs
2486
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002487 class _GitGetByExec(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002488
David James8d201162013-10-11 17:03:19 -07002489 def __init__(self, project, bare, gitdir):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002490 self._project = project
2491 self._bare = bare
David James8d201162013-10-11 17:03:19 -07002492 self._gitdir = gitdir
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002493
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002494 def LsOthers(self):
2495 p = GitCommand(self._project,
2496 ['ls-files',
2497 '-z',
2498 '--others',
2499 '--exclude-standard'],
Anthony King7bdac712014-07-16 12:56:40 +01002500 bare=False,
David James8d201162013-10-11 17:03:19 -07002501 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002502 capture_stdout=True,
2503 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002504 if p.Wait() == 0:
2505 out = p.stdout
2506 if out:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002507 # Backslash is not anomalous
David Pursehouse1d947b32012-10-25 12:23:11 +09002508 return out[:-1].split('\0') # pylint: disable=W1401
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002509 return []
2510
2511 def DiffZ(self, name, *args):
2512 cmd = [name]
2513 cmd.append('-z')
2514 cmd.extend(args)
2515 p = GitCommand(self._project,
2516 cmd,
David James8d201162013-10-11 17:03:19 -07002517 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002518 bare=False,
2519 capture_stdout=True,
2520 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002521 try:
2522 out = p.process.stdout.read()
2523 r = {}
2524 if out:
David Pursehouse1d947b32012-10-25 12:23:11 +09002525 out = iter(out[:-1].split('\0')) # pylint: disable=W1401
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002526 while out:
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002527 try:
Anthony King2cd1f042014-05-05 21:24:05 +01002528 info = next(out)
2529 path = next(out)
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002530 except StopIteration:
2531 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002532
2533 class _Info(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002534
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002535 def __init__(self, path, omode, nmode, oid, nid, state):
2536 self.path = path
2537 self.src_path = None
2538 self.old_mode = omode
2539 self.new_mode = nmode
2540 self.old_id = oid
2541 self.new_id = nid
2542
2543 if len(state) == 1:
2544 self.status = state
2545 self.level = None
2546 else:
2547 self.status = state[:1]
2548 self.level = state[1:]
2549 while self.level.startswith('0'):
2550 self.level = self.level[1:]
2551
2552 info = info[1:].split(' ')
David Pursehouse8f62fb72012-11-14 12:09:38 +09002553 info = _Info(path, *info)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002554 if info.status in ('R', 'C'):
2555 info.src_path = info.path
Anthony King2cd1f042014-05-05 21:24:05 +01002556 info.path = next(out)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002557 r[info.path] = info
2558 return r
2559 finally:
2560 p.Wait()
2561
2562 def GetHead(self):
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002563 if self._bare:
2564 path = os.path.join(self._project.gitdir, HEAD)
2565 else:
2566 path = os.path.join(self._project.worktree, '.git', HEAD)
Conley Owens75ee0572012-11-15 17:33:11 -08002567 try:
2568 fd = open(path, 'rb')
Dan Sandler53e902a2014-03-09 13:20:02 -04002569 except IOError as e:
2570 raise NoManifestException(path, str(e))
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -07002571 try:
2572 line = fd.read()
2573 finally:
2574 fd.close()
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302575 try:
2576 line = line.decode()
2577 except AttributeError:
2578 pass
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002579 if line.startswith('ref: '):
2580 return line[5:-1]
2581 return line[:-1]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002582
2583 def SetHead(self, ref, message=None):
2584 cmdv = []
2585 if message is not None:
2586 cmdv.extend(['-m', message])
2587 cmdv.append(HEAD)
2588 cmdv.append(ref)
2589 self.symbolic_ref(*cmdv)
2590
2591 def DetachHead(self, new, message=None):
2592 cmdv = ['--no-deref']
2593 if message is not None:
2594 cmdv.extend(['-m', message])
2595 cmdv.append(HEAD)
2596 cmdv.append(new)
2597 self.update_ref(*cmdv)
2598
2599 def UpdateRef(self, name, new, old=None,
2600 message=None,
2601 detach=False):
2602 cmdv = []
2603 if message is not None:
2604 cmdv.extend(['-m', message])
2605 if detach:
2606 cmdv.append('--no-deref')
2607 cmdv.append(name)
2608 cmdv.append(new)
2609 if old is not None:
2610 cmdv.append(old)
2611 self.update_ref(*cmdv)
2612
2613 def DeleteRef(self, name, old=None):
2614 if not old:
2615 old = self.rev_parse(name)
2616 self.update_ref('-d', name, old)
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07002617 self._project.bare_ref.deleted(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002618
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002619 def rev_list(self, *args, **kw):
2620 if 'format' in kw:
2621 cmdv = ['log', '--pretty=format:%s' % kw['format']]
2622 else:
2623 cmdv = ['rev-list']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002624 cmdv.extend(args)
2625 p = GitCommand(self._project,
2626 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002627 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002628 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002629 capture_stdout=True,
2630 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002631 r = []
2632 for line in p.process.stdout:
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002633 if line[-1] == '\n':
2634 line = line[:-1]
2635 r.append(line)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002636 if p.Wait() != 0:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002637 raise GitError('%s rev-list %s: %s' %
2638 (self._project.name, str(args), p.stderr))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002639 return r
2640
2641 def __getattr__(self, name):
Doug Anderson37282b42011-03-04 11:54:18 -08002642 """Allow arbitrary git commands using pythonic syntax.
2643
2644 This allows you to do things like:
2645 git_obj.rev_parse('HEAD')
2646
2647 Since we don't have a 'rev_parse' method defined, the __getattr__ will
2648 run. We'll replace the '_' with a '-' and try to run a git command.
Dave Borowitz091f8932012-10-23 17:01:04 -07002649 Any other positional arguments will be passed to the git command, and the
2650 following keyword arguments are supported:
2651 config: An optional dict of git config options to be passed with '-c'.
Doug Anderson37282b42011-03-04 11:54:18 -08002652
2653 Args:
2654 name: The name of the git command to call. Any '_' characters will
2655 be replaced with '-'.
2656
2657 Returns:
2658 A callable object that will try to call git with the named command.
2659 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002660 name = name.replace('_', '-')
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002661
Dave Borowitz091f8932012-10-23 17:01:04 -07002662 def runner(*args, **kwargs):
2663 cmdv = []
2664 config = kwargs.pop('config', None)
2665 for k in kwargs:
2666 raise TypeError('%s() got an unexpected keyword argument %r'
2667 % (name, k))
2668 if config is not None:
Dave Borowitzb42b4742012-10-31 12:27:27 -07002669 if not git_require((1, 7, 2)):
2670 raise ValueError('cannot set config on command line for %s()'
2671 % name)
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302672 for k, v in config.items():
Dave Borowitz091f8932012-10-23 17:01:04 -07002673 cmdv.append('-c')
2674 cmdv.append('%s=%s' % (k, v))
2675 cmdv.append(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002676 cmdv.extend(args)
2677 p = GitCommand(self._project,
2678 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002679 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002680 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002681 capture_stdout=True,
2682 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002683 if p.Wait() != 0:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002684 raise GitError('%s %s: %s' %
2685 (self._project.name, name, p.stderr))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002686 r = p.stdout
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302687 try:
Conley Owensedd01512013-09-26 12:59:58 -07002688 r = r.decode('utf-8')
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302689 except AttributeError:
2690 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002691 if r.endswith('\n') and r.index('\n') == len(r) - 1:
2692 return r[:-1]
2693 return r
2694 return runner
2695
2696
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002697class _PriorSyncFailedError(Exception):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002698
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002699 def __str__(self):
2700 return 'prior sync failed; rebase still in progress'
2701
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002702
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002703class _DirtyError(Exception):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002704
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002705 def __str__(self):
2706 return 'contains uncommitted changes'
2707
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002708
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002709class _InfoMessage(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002710
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002711 def __init__(self, project, text):
2712 self.project = project
2713 self.text = text
2714
2715 def Print(self, syncbuf):
2716 syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
2717 syncbuf.out.nl()
2718
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002719
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002720class _Failure(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002721
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002722 def __init__(self, project, why):
2723 self.project = project
2724 self.why = why
2725
2726 def Print(self, syncbuf):
2727 syncbuf.out.fail('error: %s/: %s',
2728 self.project.relpath,
2729 str(self.why))
2730 syncbuf.out.nl()
2731
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002732
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002733class _Later(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002734
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002735 def __init__(self, project, action):
2736 self.project = project
2737 self.action = action
2738
2739 def Run(self, syncbuf):
2740 out = syncbuf.out
2741 out.project('project %s/', self.project.relpath)
2742 out.nl()
2743 try:
2744 self.action()
2745 out.nl()
2746 return True
David Pursehouse8a68ff92012-09-24 12:15:13 +09002747 except GitError:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002748 out.nl()
2749 return False
2750
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002751
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002752class _SyncColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002753
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002754 def __init__(self, config):
2755 Coloring.__init__(self, config, 'reposync')
Anthony King7bdac712014-07-16 12:56:40 +01002756 self.project = self.printer('header', attr='bold')
2757 self.info = self.printer('info')
2758 self.fail = self.printer('fail', fg='red')
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002759
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002760
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002761class SyncBuffer(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002762
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002763 def __init__(self, config, detach_head=False):
2764 self._messages = []
2765 self._failures = []
2766 self._later_queue1 = []
2767 self._later_queue2 = []
2768
2769 self.out = _SyncColoring(config)
2770 self.out.redirect(sys.stderr)
2771
2772 self.detach_head = detach_head
2773 self.clean = True
2774
2775 def info(self, project, fmt, *args):
2776 self._messages.append(_InfoMessage(project, fmt % args))
2777
2778 def fail(self, project, err=None):
2779 self._failures.append(_Failure(project, err))
2780 self.clean = False
2781
2782 def later1(self, project, what):
2783 self._later_queue1.append(_Later(project, what))
2784
2785 def later2(self, project, what):
2786 self._later_queue2.append(_Later(project, what))
2787
2788 def Finish(self):
2789 self._PrintMessages()
2790 self._RunLater()
2791 self._PrintMessages()
2792 return self.clean
2793
2794 def _RunLater(self):
2795 for q in ['_later_queue1', '_later_queue2']:
2796 if not self._RunQueue(q):
2797 return
2798
2799 def _RunQueue(self, queue):
2800 for m in getattr(self, queue):
2801 if not m.Run(self):
2802 self.clean = False
2803 return False
2804 setattr(self, queue, [])
2805 return True
2806
2807 def _PrintMessages(self):
2808 for m in self._messages:
2809 m.Print(self)
2810 for m in self._failures:
2811 m.Print(self)
2812
2813 self._messages = []
2814 self._failures = []
2815
2816
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002817class MetaProject(Project):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002818
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002819 """A special project housed under .repo.
2820 """
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002821
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002822 def __init__(self, manifest, name, gitdir, worktree):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002823 Project.__init__(self,
Anthony King7bdac712014-07-16 12:56:40 +01002824 manifest=manifest,
2825 name=name,
2826 gitdir=gitdir,
2827 objdir=gitdir,
2828 worktree=worktree,
2829 remote=RemoteSpec('origin'),
2830 relpath='.repo/%s' % name,
2831 revisionExpr='refs/heads/master',
2832 revisionId=None,
2833 groups=None)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002834
2835 def PreSync(self):
2836 if self.Exists:
2837 cb = self.CurrentBranch
2838 if cb:
2839 base = self.GetBranch(cb).merge
2840 if base:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002841 self.revisionExpr = base
2842 self.revisionId = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002843
Anthony King7bdac712014-07-16 12:56:40 +01002844 def MetaBranchSwitch(self):
Florian Vallee5d016502012-06-07 17:19:26 +02002845 """ Prepare MetaProject for manifest branch switch
2846 """
2847
2848 # detach and delete manifest branch, allowing a new
2849 # branch to take over
Anthony King7bdac712014-07-16 12:56:40 +01002850 syncbuf = SyncBuffer(self.config, detach_head=True)
Florian Vallee5d016502012-06-07 17:19:26 +02002851 self.Sync_LocalHalf(syncbuf)
2852 syncbuf.Finish()
2853
2854 return GitCommand(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002855 ['update-ref', '-d', 'refs/heads/default'],
2856 capture_stdout=True,
2857 capture_stderr=True).Wait() == 0
Florian Vallee5d016502012-06-07 17:19:26 +02002858
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002859 @property
Shawn O. Pearcef6906872009-04-18 10:49:00 -07002860 def LastFetch(self):
2861 try:
2862 fh = os.path.join(self.gitdir, 'FETCH_HEAD')
2863 return os.path.getmtime(fh)
2864 except OSError:
2865 return 0
2866
2867 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002868 def HasChanges(self):
2869 """Has the remote received new commits not yet checked out?
2870 """
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002871 if not self.remote or not self.revisionExpr:
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002872 return False
2873
David Pursehouse8a68ff92012-09-24 12:15:13 +09002874 all_refs = self.bare_ref.all
2875 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002876 head = self.work_git.GetHead()
2877 if head.startswith(R_HEADS):
2878 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09002879 head = all_refs[head]
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002880 except KeyError:
2881 head = None
2882
2883 if revid == head:
2884 return False
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002885 elif self._revlist(not_rev(HEAD), revid):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002886 return True
2887 return False