blob: 68d7421fc34454e47a5e9ed9b773743eee8c0a95 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Sarah Owenscecd1d82012-11-01 22:59:27 -070015from __future__ import print_function
Dave Borowitz137d0132015-01-02 11:12:54 -080016import contextlib
Shawn O. Pearce438ee1c2008-11-03 09:59:36 -080017import errno
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import filecmp
19import os
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070020import random
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
22import shutil
23import stat
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Julien Campergue335f5ef2013-10-16 11:02:35 +020026import tarfile
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +080027import tempfile
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070028import time
Dave Borowitz137d0132015-01-02 11:12:54 -080029import traceback
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -070030
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070031from color import Coloring
Dave Borowitzb42b4742012-10-31 12:27:27 -070032from git_command import GitCommand, git_require
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -070033from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
David Pursehousee15c65a2012-08-22 10:46:11 +090034from error import GitError, HookError, UploadError
Shawn O. Pearce559b8462009-03-02 12:56:08 -080035from error import ManifestInvalidRevisionError
Conley Owens75ee0572012-11-15 17:33:11 -080036from error import NoManifestException
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070037from trace import IsTrace, Trace
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070038
Shawn O. Pearced237b692009-04-17 18:49:50 -070039from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
David Pursehouse59bbb582013-05-17 10:49:33 +090041from pyversion import is_python3
42if not is_python3():
43 # pylint:disable=W0622
Chirayu Desai217ea7d2013-03-01 19:14:38 +053044 input = raw_input
David Pursehouse59bbb582013-05-17 10:49:33 +090045 # pylint:enable=W0622
Chirayu Desai217ea7d2013-03-01 19:14:38 +053046
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070047def _lwrite(path, content):
48 lock = '%s.lock' % path
49
Chirayu Desai303a82f2014-08-19 22:57:17 +053050 fd = open(lock, 'w')
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070051 try:
52 fd.write(content)
53 finally:
54 fd.close()
55
56 try:
57 os.rename(lock, path)
58 except OSError:
59 os.remove(lock)
60 raise
61
Shawn O. Pearce48244782009-04-16 08:25:57 -070062def _error(fmt, *args):
63 msg = fmt % args
Sarah Owenscecd1d82012-11-01 22:59:27 -070064 print('error: %s' % msg, file=sys.stderr)
Shawn O. Pearce48244782009-04-16 08:25:57 -070065
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070066def not_rev(r):
67 return '^' + r
68
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080069def sq(r):
70 return "'" + r.replace("'", "'\''") + "'"
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -080071
Shawn O. Pearce632768b2008-10-23 11:58:52 -070072class DownloadedChange(object):
73 _commit_cache = None
74
75 def __init__(self, project, base, change_id, ps_id, commit):
76 self.project = project
77 self.base = base
78 self.change_id = change_id
79 self.ps_id = ps_id
80 self.commit = commit
81
82 @property
83 def commits(self):
84 if self._commit_cache is None:
85 self._commit_cache = self.project.bare_git.rev_list(
86 '--abbrev=8',
87 '--abbrev-commit',
88 '--pretty=oneline',
89 '--reverse',
90 '--date-order',
91 not_rev(self.base),
92 self.commit,
93 '--')
94 return self._commit_cache
95
96
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070097class ReviewableBranch(object):
98 _commit_cache = None
99
100 def __init__(self, project, branch, base):
101 self.project = project
102 self.branch = branch
103 self.base = base
104
105 @property
106 def name(self):
107 return self.branch.name
108
109 @property
110 def commits(self):
111 if self._commit_cache is None:
112 self._commit_cache = self.project.bare_git.rev_list(
113 '--abbrev=8',
114 '--abbrev-commit',
115 '--pretty=oneline',
116 '--reverse',
117 '--date-order',
118 not_rev(self.base),
119 R_HEADS + self.name,
120 '--')
121 return self._commit_cache
122
123 @property
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800124 def unabbrev_commits(self):
125 r = dict()
126 for commit in self.project.bare_git.rev_list(
127 not_rev(self.base),
128 R_HEADS + self.name,
129 '--'):
130 r[commit[0:8]] = commit
131 return r
132
133 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700134 def date(self):
135 return self.project.bare_git.log(
136 '--pretty=format:%cd',
137 '-n', '1',
138 R_HEADS + self.name,
139 '--')
140
Bryan Jacobsf609f912013-05-06 13:36:24 -0400141 def UploadForReview(self, people, auto_topic=False, draft=False, dest_branch=None):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800142 self.project.UploadForReview(self.name,
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -0700143 people,
Brian Harring435370c2012-07-28 15:37:04 -0700144 auto_topic=auto_topic,
Bryan Jacobsf609f912013-05-06 13:36:24 -0400145 draft=draft,
146 dest_branch=dest_branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700147
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700148 def GetPublishedRefs(self):
149 refs = {}
150 output = self.project.bare_git.ls_remote(
151 self.branch.remote.SshReviewUrl(self.project.UserEmail),
152 'refs/changes/*')
153 for line in output.split('\n'):
154 try:
155 (sha, ref) = line.split()
156 refs[sha] = ref
157 except ValueError:
158 pass
159
160 return refs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700161
162class StatusColoring(Coloring):
163 def __init__(self, config):
164 Coloring.__init__(self, config, 'status')
Anthony King7bdac712014-07-16 12:56:40 +0100165 self.project = self.printer('header', attr='bold')
166 self.branch = self.printer('header', attr='bold')
167 self.nobranch = self.printer('nobranch', fg='red')
168 self.important = self.printer('important', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700169
Anthony King7bdac712014-07-16 12:56:40 +0100170 self.added = self.printer('added', fg='green')
171 self.changed = self.printer('changed', fg='red')
172 self.untracked = self.printer('untracked', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700173
174
175class DiffColoring(Coloring):
176 def __init__(self, config):
177 Coloring.__init__(self, config, 'diff')
Anthony King7bdac712014-07-16 12:56:40 +0100178 self.project = self.printer('header', attr='bold')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700179
Anthony King7bdac712014-07-16 12:56:40 +0100180class _Annotation(object):
James W. Mills24c13082012-04-12 15:04:13 -0500181 def __init__(self, name, value, keep):
182 self.name = name
183 self.value = value
184 self.keep = keep
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700185
Anthony King7bdac712014-07-16 12:56:40 +0100186class _CopyFile(object):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800187 def __init__(self, src, dest, abssrc, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700188 self.src = src
189 self.dest = dest
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800190 self.abs_src = abssrc
191 self.abs_dest = absdest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700192
193 def _Copy(self):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800194 src = self.abs_src
195 dest = self.abs_dest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700196 # copy file if it does not exist or is out of date
197 if not os.path.exists(dest) or not filecmp.cmp(src, dest):
198 try:
199 # remove existing file first, since it might be read-only
200 if os.path.exists(dest):
201 os.remove(dest)
Matthew Buckett2daf6672009-07-11 09:43:47 -0400202 else:
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200203 dest_dir = os.path.dirname(dest)
204 if not os.path.isdir(dest_dir):
205 os.makedirs(dest_dir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700206 shutil.copy(src, dest)
207 # make the file read-only
208 mode = os.stat(dest)[stat.ST_MODE]
209 mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
210 os.chmod(dest, mode)
211 except IOError:
Shawn O. Pearce48244782009-04-16 08:25:57 -0700212 _error('Cannot copy file %s to %s', src, dest)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700213
Anthony King7bdac712014-07-16 12:56:40 +0100214class _LinkFile(object):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500215 def __init__(self, src, dest, abssrc, absdest):
216 self.src = src
217 self.dest = dest
218 self.abs_src = abssrc
219 self.abs_dest = absdest
220
221 def _Link(self):
222 src = self.abs_src
223 dest = self.abs_dest
224 # link file if it does not exist or is out of date
225 if not os.path.islink(dest) or os.readlink(dest) != src:
226 try:
227 # remove existing file first, since it might be read-only
228 if os.path.exists(dest):
229 os.remove(dest)
230 else:
231 dest_dir = os.path.dirname(dest)
232 if not os.path.isdir(dest_dir):
233 os.makedirs(dest_dir)
234 os.symlink(src, dest)
235 except IOError:
236 _error('Cannot link file %s to %s', src, dest)
237
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700238class RemoteSpec(object):
239 def __init__(self,
240 name,
Anthony King7bdac712014-07-16 12:56:40 +0100241 url=None,
242 review=None,
243 revision=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700244 self.name = name
245 self.url = url
246 self.review = review
Anthony King36ea2fb2014-05-06 11:54:01 +0100247 self.revision = revision
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700248
Doug Anderson37282b42011-03-04 11:54:18 -0800249class RepoHook(object):
250 """A RepoHook contains information about a script to run as a hook.
251
252 Hooks are used to run a python script before running an upload (for instance,
253 to run presubmit checks). Eventually, we may have hooks for other actions.
254
255 This shouldn't be confused with files in the 'repo/hooks' directory. Those
256 files are copied into each '.git/hooks' folder for each project. Repo-level
257 hooks are associated instead with repo actions.
258
259 Hooks are always python. When a hook is run, we will load the hook into the
260 interpreter and execute its main() function.
261 """
262 def __init__(self,
263 hook_type,
264 hooks_project,
265 topdir,
266 abort_if_user_denies=False):
267 """RepoHook constructor.
268
269 Params:
270 hook_type: A string representing the type of hook. This is also used
271 to figure out the name of the file containing the hook. For
272 example: 'pre-upload'.
273 hooks_project: The project containing the repo hooks. If you have a
274 manifest, this is manifest.repo_hooks_project. OK if this is None,
275 which will make the hook a no-op.
276 topdir: Repo's top directory (the one containing the .repo directory).
277 Scripts will run with CWD as this directory. If you have a manifest,
278 this is manifest.topdir
279 abort_if_user_denies: If True, we'll throw a HookError() if the user
280 doesn't allow us to run the hook.
281 """
282 self._hook_type = hook_type
283 self._hooks_project = hooks_project
284 self._topdir = topdir
285 self._abort_if_user_denies = abort_if_user_denies
286
287 # Store the full path to the script for convenience.
288 if self._hooks_project:
289 self._script_fullpath = os.path.join(self._hooks_project.worktree,
290 self._hook_type + '.py')
291 else:
292 self._script_fullpath = None
293
294 def _GetHash(self):
295 """Return a hash of the contents of the hooks directory.
296
297 We'll just use git to do this. This hash has the property that if anything
298 changes in the directory we will return a different has.
299
300 SECURITY CONSIDERATION:
301 This hash only represents the contents of files in the hook directory, not
302 any other files imported or called by hooks. Changes to imported files
303 can change the script behavior without affecting the hash.
304
305 Returns:
306 A string representing the hash. This will always be ASCII so that it can
307 be printed to the user easily.
308 """
309 assert self._hooks_project, "Must have hooks to calculate their hash."
310
311 # We will use the work_git object rather than just calling GetRevisionId().
312 # That gives us a hash of the latest checked in version of the files that
313 # the user will actually be executing. Specifically, GetRevisionId()
314 # doesn't appear to change even if a user checks out a different version
315 # of the hooks repo (via git checkout) nor if a user commits their own revs.
316 #
317 # NOTE: Local (non-committed) changes will not be factored into this hash.
318 # I think this is OK, since we're really only worried about warning the user
319 # about upstream changes.
320 return self._hooks_project.work_git.rev_parse('HEAD')
321
322 def _GetMustVerb(self):
323 """Return 'must' if the hook is required; 'should' if not."""
324 if self._abort_if_user_denies:
325 return 'must'
326 else:
327 return 'should'
328
329 def _CheckForHookApproval(self):
330 """Check to see whether this hook has been approved.
331
332 We'll look at the hash of all of the hooks. If this matches the hash that
333 the user last approved, we're done. If it doesn't, we'll ask the user
334 about approval.
335
336 Note that we ask permission for each individual hook even though we use
337 the hash of all hooks when detecting changes. We'd like the user to be
338 able to approve / deny each hook individually. We only use the hash of all
339 hooks because there is no other easy way to detect changes to local imports.
340
341 Returns:
342 True if this hook is approved to run; False otherwise.
343
344 Raises:
345 HookError: Raised if the user doesn't approve and abort_if_user_denies
346 was passed to the consturctor.
347 """
Doug Anderson37282b42011-03-04 11:54:18 -0800348 hooks_config = self._hooks_project.config
349 git_approval_key = 'repo.hooks.%s.approvedhash' % self._hook_type
350
351 # Get the last hash that the user approved for this hook; may be None.
352 old_hash = hooks_config.GetString(git_approval_key)
353
354 # Get the current hash so we can tell if scripts changed since approval.
355 new_hash = self._GetHash()
356
357 if old_hash is not None:
358 # User previously approved hook and asked not to be prompted again.
359 if new_hash == old_hash:
360 # Approval matched. We're done.
361 return True
362 else:
363 # Give the user a reason why we're prompting, since they last told
364 # us to "never ask again".
365 prompt = 'WARNING: Scripts have changed since %s was allowed.\n\n' % (
366 self._hook_type)
367 else:
368 prompt = ''
369
370 # Prompt the user if we're not on a tty; on a tty we'll assume "no".
371 if sys.stdout.isatty():
372 prompt += ('Repo %s run the script:\n'
373 ' %s\n'
374 '\n'
375 'Do you want to allow this script to run '
376 '(yes/yes-never-ask-again/NO)? ') % (
377 self._GetMustVerb(), self._script_fullpath)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530378 response = input(prompt).lower()
David Pursehouse98ffba12012-11-14 11:18:00 +0900379 print()
Doug Anderson37282b42011-03-04 11:54:18 -0800380
381 # User is doing a one-time approval.
382 if response in ('y', 'yes'):
383 return True
384 elif response == 'yes-never-ask-again':
385 hooks_config.SetString(git_approval_key, new_hash)
386 return True
387
388 # For anything else, we'll assume no approval.
389 if self._abort_if_user_denies:
390 raise HookError('You must allow the %s hook or use --no-verify.' %
391 self._hook_type)
392
393 return False
394
395 def _ExecuteHook(self, **kwargs):
396 """Actually execute the given hook.
397
398 This will run the hook's 'main' function in our python interpreter.
399
400 Args:
401 kwargs: Keyword arguments to pass to the hook. These are often specific
402 to the hook type. For instance, pre-upload hooks will contain
403 a project_list.
404 """
405 # Keep sys.path and CWD stashed away so that we can always restore them
406 # upon function exit.
407 orig_path = os.getcwd()
408 orig_syspath = sys.path
409
410 try:
411 # Always run hooks with CWD as topdir.
412 os.chdir(self._topdir)
413
414 # Put the hook dir as the first item of sys.path so hooks can do
415 # relative imports. We want to replace the repo dir as [0] so
416 # hooks can't import repo files.
417 sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
418
419 # Exec, storing global context in the context dict. We catch exceptions
420 # and convert to a HookError w/ just the failing traceback.
421 context = {}
422 try:
Anthony King70f68902014-05-05 21:15:34 +0100423 exec(compile(open(self._script_fullpath).read(),
424 self._script_fullpath, 'exec'), context)
Doug Anderson37282b42011-03-04 11:54:18 -0800425 except Exception:
426 raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
427 traceback.format_exc(), self._hook_type))
428
429 # Running the script should have defined a main() function.
430 if 'main' not in context:
431 raise HookError('Missing main() in: "%s"' % self._script_fullpath)
432
433
434 # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
435 # We don't actually want hooks to define their main with this argument--
436 # it's there to remind them that their hook should always take **kwargs.
437 # For instance, a pre-upload hook should be defined like:
438 # def main(project_list, **kwargs):
439 #
440 # This allows us to later expand the API without breaking old hooks.
441 kwargs = kwargs.copy()
442 kwargs['hook_should_take_kwargs'] = True
443
444 # Call the main function in the hook. If the hook should cause the
445 # build to fail, it will raise an Exception. We'll catch that convert
446 # to a HookError w/ just the failing traceback.
447 try:
448 context['main'](**kwargs)
449 except Exception:
450 raise HookError('%s\nFailed to run main() for %s hook; see traceback '
451 'above.' % (
452 traceback.format_exc(), self._hook_type))
453 finally:
454 # Restore sys.path and CWD.
455 sys.path = orig_syspath
456 os.chdir(orig_path)
457
458 def Run(self, user_allows_all_hooks, **kwargs):
459 """Run the hook.
460
461 If the hook doesn't exist (because there is no hooks project or because
462 this particular hook is not enabled), this is a no-op.
463
464 Args:
465 user_allows_all_hooks: If True, we will never prompt about running the
466 hook--we'll just assume it's OK to run it.
467 kwargs: Keyword arguments to pass to the hook. These are often specific
468 to the hook type. For instance, pre-upload hooks will contain
469 a project_list.
470
471 Raises:
472 HookError: If there was a problem finding the hook or the user declined
473 to run a required hook (from _CheckForHookApproval).
474 """
475 # No-op if there is no hooks project or if hook is disabled.
476 if ((not self._hooks_project) or
477 (self._hook_type not in self._hooks_project.enabled_repo_hooks)):
478 return
479
480 # Bail with a nice error if we can't find the hook.
481 if not os.path.isfile(self._script_fullpath):
482 raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
483
484 # Make sure the user is OK with running the hook.
485 if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
486 return
487
488 # Run the hook with the same version of python we're using.
489 self._ExecuteHook(**kwargs)
490
491
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700492class Project(object):
493 def __init__(self,
494 manifest,
495 name,
496 remote,
497 gitdir,
David James8d201162013-10-11 17:03:19 -0700498 objdir,
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700499 worktree,
500 relpath,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700501 revisionExpr,
Mike Pontillod3153822012-02-28 11:53:24 -0800502 revisionId,
Anthony King7bdac712014-07-16 12:56:40 +0100503 rebase=True,
504 groups=None,
505 sync_c=False,
506 sync_s=False,
507 clone_depth=None,
508 upstream=None,
509 parent=None,
510 is_derived=False,
511 dest_branch=None):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800512 """Init a Project object.
513
514 Args:
515 manifest: The XmlManifest object.
516 name: The `name` attribute of manifest.xml's project element.
517 remote: RemoteSpec object specifying its remote's properties.
518 gitdir: Absolute path of git directory.
David James8d201162013-10-11 17:03:19 -0700519 objdir: Absolute path of directory to store git objects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800520 worktree: Absolute path of git working tree.
521 relpath: Relative path of git working tree to repo's top directory.
522 revisionExpr: The `revision` attribute of manifest.xml's project element.
523 revisionId: git commit id for checking out.
524 rebase: The `rebase` attribute of manifest.xml's project element.
525 groups: The `groups` attribute of manifest.xml's project element.
526 sync_c: The `sync-c` attribute of manifest.xml's project element.
527 sync_s: The `sync-s` attribute of manifest.xml's project element.
528 upstream: The `upstream` attribute of manifest.xml's project element.
529 parent: The parent Project object.
530 is_derived: False if the project was explicitly defined in the manifest;
531 True if the project is a discovered submodule.
Bryan Jacobsf609f912013-05-06 13:36:24 -0400532 dest_branch: The branch to which to push changes for review by default.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800533 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700534 self.manifest = manifest
535 self.name = name
536 self.remote = remote
Anthony Newnamdf14a702011-01-09 17:31:57 -0800537 self.gitdir = gitdir.replace('\\', '/')
David James8d201162013-10-11 17:03:19 -0700538 self.objdir = objdir.replace('\\', '/')
Shawn O. Pearce0ce6ca92011-01-10 13:26:01 -0800539 if worktree:
540 self.worktree = worktree.replace('\\', '/')
541 else:
542 self.worktree = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700543 self.relpath = relpath
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700544 self.revisionExpr = revisionExpr
545
546 if revisionId is None \
547 and revisionExpr \
548 and IsId(revisionExpr):
549 self.revisionId = revisionExpr
550 else:
551 self.revisionId = revisionId
552
Mike Pontillod3153822012-02-28 11:53:24 -0800553 self.rebase = rebase
Colin Cross5acde752012-03-28 20:15:45 -0700554 self.groups = groups
Anatol Pomazau79770d22012-04-20 14:41:59 -0700555 self.sync_c = sync_c
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800556 self.sync_s = sync_s
David Pursehouseede7f122012-11-27 22:25:30 +0900557 self.clone_depth = clone_depth
Brian Harring14a66742012-09-28 20:21:57 -0700558 self.upstream = upstream
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800559 self.parent = parent
560 self.is_derived = is_derived
561 self.subprojects = []
Mike Pontillod3153822012-02-28 11:53:24 -0800562
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700563 self.snapshots = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700564 self.copyfiles = []
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500565 self.linkfiles = []
James W. Mills24c13082012-04-12 15:04:13 -0500566 self.annotations = []
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700567 self.config = GitConfig.ForRepository(
Anthony King7bdac712014-07-16 12:56:40 +0100568 gitdir=self.gitdir,
569 defaults=self.manifest.globalConfig)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700570
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800571 if self.worktree:
David James8d201162013-10-11 17:03:19 -0700572 self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800573 else:
574 self.work_git = None
David James8d201162013-10-11 17:03:19 -0700575 self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
Shawn O. Pearced237b692009-04-17 18:49:50 -0700576 self.bare_ref = GitRefs(gitdir)
David James8d201162013-10-11 17:03:19 -0700577 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
Bryan Jacobsf609f912013-05-06 13:36:24 -0400578 self.dest_branch = dest_branch
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700579
Doug Anderson37282b42011-03-04 11:54:18 -0800580 # This will be filled in if a project is later identified to be the
581 # project containing repo hooks.
582 self.enabled_repo_hooks = []
583
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700584 @property
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800585 def Derived(self):
586 return self.is_derived
587
588 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700589 def Exists(self):
590 return os.path.isdir(self.gitdir)
591
592 @property
593 def CurrentBranch(self):
594 """Obtain the name of the currently checked out branch.
595 The branch name omits the 'refs/heads/' prefix.
596 None is returned if the project is on a detached HEAD.
597 """
Shawn O. Pearce5b23f242009-04-17 18:43:33 -0700598 b = self.work_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700599 if b.startswith(R_HEADS):
600 return b[len(R_HEADS):]
601 return None
602
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700603 def IsRebaseInProgress(self):
604 w = self.worktree
605 g = os.path.join(w, '.git')
606 return os.path.exists(os.path.join(g, 'rebase-apply')) \
607 or os.path.exists(os.path.join(g, 'rebase-merge')) \
608 or os.path.exists(os.path.join(w, '.dotest'))
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200609
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700610 def IsDirty(self, consider_untracked=True):
611 """Is the working directory modified in some way?
612 """
613 self.work_git.update_index('-q',
614 '--unmerged',
615 '--ignore-missing',
616 '--refresh')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900617 if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700618 return True
619 if self.work_git.DiffZ('diff-files'):
620 return True
621 if consider_untracked and self.work_git.LsOthers():
622 return True
623 return False
624
625 _userident_name = None
626 _userident_email = None
627
628 @property
629 def UserName(self):
630 """Obtain the user's personal name.
631 """
632 if self._userident_name is None:
633 self._LoadUserIdentity()
634 return self._userident_name
635
636 @property
637 def UserEmail(self):
638 """Obtain the user's email address. This is very likely
639 to be their Gerrit login.
640 """
641 if self._userident_email is None:
642 self._LoadUserIdentity()
643 return self._userident_email
644
645 def _LoadUserIdentity(self):
David Pursehousec1b86a22012-11-14 11:36:51 +0900646 u = self.bare_git.var('GIT_COMMITTER_IDENT')
647 m = re.compile("^(.*) <([^>]*)> ").match(u)
648 if m:
649 self._userident_name = m.group(1)
650 self._userident_email = m.group(2)
651 else:
652 self._userident_name = ''
653 self._userident_email = ''
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700654
655 def GetRemote(self, name):
656 """Get the configuration for a single remote.
657 """
658 return self.config.GetRemote(name)
659
660 def GetBranch(self, name):
661 """Get the configuration for a single branch.
662 """
663 return self.config.GetBranch(name)
664
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700665 def GetBranches(self):
666 """Get all existing local branches.
667 """
668 current = self.CurrentBranch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900669 all_refs = self._allrefs
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700670 heads = {}
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700671
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530672 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700673 if name.startswith(R_HEADS):
674 name = name[len(R_HEADS):]
675 b = self.GetBranch(name)
676 b.current = name == current
677 b.published = None
David Pursehouse8a68ff92012-09-24 12:15:13 +0900678 b.revision = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700679 heads[name] = b
680
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530681 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700682 if name.startswith(R_PUB):
683 name = name[len(R_PUB):]
684 b = heads.get(name)
685 if b:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900686 b.published = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700687
688 return heads
689
Colin Cross5acde752012-03-28 20:15:45 -0700690 def MatchesGroups(self, manifest_groups):
691 """Returns true if the manifest groups specified at init should cause
692 this project to be synced.
693 Prefixing a manifest group with "-" inverts the meaning of a group.
Conley Owensbb1b5f52012-08-13 13:11:18 -0700694 All projects are implicitly labelled with "all".
Conley Owens971de8e2012-04-16 10:36:08 -0700695
696 labels are resolved in order. In the example case of
Conley Owensbb1b5f52012-08-13 13:11:18 -0700697 project_groups: "all,group1,group2"
Conley Owens971de8e2012-04-16 10:36:08 -0700698 manifest_groups: "-group1,group2"
699 the project will be matched.
David Holmer0a1c6a12012-11-14 19:19:00 -0500700
701 The special manifest group "default" will match any project that
702 does not have the special project group "notdefault"
Colin Cross5acde752012-03-28 20:15:45 -0700703 """
David Holmer0a1c6a12012-11-14 19:19:00 -0500704 expanded_manifest_groups = manifest_groups or ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700705 expanded_project_groups = ['all'] + (self.groups or [])
David Holmer0a1c6a12012-11-14 19:19:00 -0500706 if not 'notdefault' in expanded_project_groups:
707 expanded_project_groups += ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700708
Conley Owens971de8e2012-04-16 10:36:08 -0700709 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700710 for group in expanded_manifest_groups:
711 if group.startswith('-') and group[1:] in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700712 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700713 elif group in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700714 matched = True
Colin Cross5acde752012-03-28 20:15:45 -0700715
Conley Owens971de8e2012-04-16 10:36:08 -0700716 return matched
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700717
718## Status Display ##
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700719 def UncommitedFiles(self, get_all=True):
720 """Returns a list of strings, uncommitted files in the git tree.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700721
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700722 Args:
723 get_all: a boolean, if True - get information about all different
724 uncommitted files. If False - return as soon as any kind of
725 uncommitted files is detected.
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500726 """
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700727 details = []
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500728 self.work_git.update_index('-q',
729 '--unmerged',
730 '--ignore-missing',
731 '--refresh')
732 if self.IsRebaseInProgress():
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700733 details.append("rebase in progress")
734 if not get_all:
735 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500736
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700737 changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
738 if changes:
739 details.extend(changes)
740 if not get_all:
741 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500742
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700743 changes = self.work_git.DiffZ('diff-files').keys()
744 if changes:
745 details.extend(changes)
746 if not get_all:
747 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500748
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700749 changes = self.work_git.LsOthers()
750 if changes:
751 details.extend(changes)
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500752
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700753 return details
754
755 def HasChanges(self):
756 """Returns true if there are uncommitted changes.
757 """
758 if self.UncommitedFiles(get_all=False):
759 return True
760 else:
761 return False
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500762
Terence Haddock4655e812011-03-31 12:33:34 +0200763 def PrintWorkTreeStatus(self, output_redir=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700764 """Prints the status of the repository to stdout.
Terence Haddock4655e812011-03-31 12:33:34 +0200765
766 Args:
767 output: If specified, redirect the output to this object.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700768 """
769 if not os.path.isdir(self.worktree):
Terence Haddock4655e812011-03-31 12:33:34 +0200770 if output_redir == None:
771 output_redir = sys.stdout
Sarah Owenscecd1d82012-11-01 22:59:27 -0700772 print(file=output_redir)
773 print('project %s/' % self.relpath, file=output_redir)
774 print(' missing (run "repo sync")', file=output_redir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700775 return
776
777 self.work_git.update_index('-q',
778 '--unmerged',
779 '--ignore-missing',
780 '--refresh')
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700781 rb = self.IsRebaseInProgress()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700782 di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
783 df = self.work_git.DiffZ('diff-files')
784 do = self.work_git.LsOthers()
Ali Utku Selen76abcc12012-01-25 10:51:12 +0100785 if not rb and not di and not df and not do and not self.CurrentBranch:
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700786 return 'CLEAN'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700787
788 out = StatusColoring(self.config)
Terence Haddock4655e812011-03-31 12:33:34 +0200789 if not output_redir == None:
790 out.redirect(output_redir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700791 out.project('project %-40s', self.relpath + '/')
792
793 branch = self.CurrentBranch
794 if branch is None:
795 out.nobranch('(*** NO BRANCH ***)')
796 else:
797 out.branch('branch %s', branch)
798 out.nl()
799
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700800 if rb:
801 out.important('prior sync failed; rebase still in progress')
802 out.nl()
803
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700804 paths = list()
805 paths.extend(di.keys())
806 paths.extend(df.keys())
807 paths.extend(do)
808
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530809 for p in sorted(set(paths)):
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900810 try:
811 i = di[p]
812 except KeyError:
813 i = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700814
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900815 try:
816 f = df[p]
817 except KeyError:
818 f = None
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200819
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900820 if i:
821 i_status = i.status.upper()
822 else:
823 i_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700824
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900825 if f:
826 f_status = f.status.lower()
827 else:
828 f_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700829
830 if i and i.src_path:
Shawn O. Pearcefe086752009-03-03 13:49:48 -0800831 line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700832 i.src_path, p, i.level)
833 else:
834 line = ' %s%s\t%s' % (i_status, f_status, p)
835
836 if i and not f:
837 out.added('%s', line)
838 elif (i and f) or (not i and f):
839 out.changed('%s', line)
840 elif not i and not f:
841 out.untracked('%s', line)
842 else:
843 out.write('%s', line)
844 out.nl()
Terence Haddock4655e812011-03-31 12:33:34 +0200845
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700846 return 'DIRTY'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700847
pelyad67872d2012-03-28 14:49:58 +0300848 def PrintWorkTreeDiff(self, absolute_paths=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700849 """Prints the status of the repository to stdout.
850 """
851 out = DiffColoring(self.config)
852 cmd = ['diff']
853 if out.is_on:
854 cmd.append('--color')
855 cmd.append(HEAD)
pelyad67872d2012-03-28 14:49:58 +0300856 if absolute_paths:
857 cmd.append('--src-prefix=a/%s/' % self.relpath)
858 cmd.append('--dst-prefix=b/%s/' % self.relpath)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700859 cmd.append('--')
860 p = GitCommand(self,
861 cmd,
Anthony King7bdac712014-07-16 12:56:40 +0100862 capture_stdout=True,
863 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700864 has_diff = False
865 for line in p.process.stdout:
866 if not has_diff:
867 out.nl()
868 out.project('project %s/' % self.relpath)
869 out.nl()
870 has_diff = True
Sarah Owenscecd1d82012-11-01 22:59:27 -0700871 print(line[:-1])
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700872 p.Wait()
873
874
875## Publish / Upload ##
876
David Pursehouse8a68ff92012-09-24 12:15:13 +0900877 def WasPublished(self, branch, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700878 """Was the branch published (uploaded) for code review?
879 If so, returns the SHA-1 hash of the last published
880 state for the branch.
881 """
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700882 key = R_PUB + branch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900883 if all_refs is None:
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700884 try:
885 return self.bare_git.rev_parse(key)
886 except GitError:
887 return None
888 else:
889 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900890 return all_refs[key]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700891 except KeyError:
892 return None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700893
David Pursehouse8a68ff92012-09-24 12:15:13 +0900894 def CleanPublishedCache(self, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700895 """Prunes any stale published refs.
896 """
David Pursehouse8a68ff92012-09-24 12:15:13 +0900897 if all_refs is None:
898 all_refs = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700899 heads = set()
900 canrm = {}
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530901 for name, ref_id in all_refs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700902 if name.startswith(R_HEADS):
903 heads.add(name)
904 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900905 canrm[name] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700906
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530907 for name, ref_id in canrm.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700908 n = name[len(R_PUB):]
909 if R_HEADS + n not in heads:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900910 self.bare_git.DeleteRef(name, ref_id)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700911
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -0700912 def GetUploadableBranches(self, selected_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700913 """List any branches which can be uploaded for review.
914 """
915 heads = {}
916 pubed = {}
917
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530918 for name, ref_id in self._allrefs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700919 if name.startswith(R_HEADS):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900920 heads[name[len(R_HEADS):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700921 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900922 pubed[name[len(R_PUB):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700923
924 ready = []
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530925 for branch, ref_id in heads.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +0900926 if branch in pubed and pubed[branch] == ref_id:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700927 continue
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -0700928 if selected_branch and branch != selected_branch:
929 continue
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700930
Shawn O. Pearce35f25962008-11-11 17:03:13 -0800931 rb = self.GetUploadableBranch(branch)
932 if rb:
933 ready.append(rb)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700934 return ready
935
Shawn O. Pearce35f25962008-11-11 17:03:13 -0800936 def GetUploadableBranch(self, branch_name):
937 """Get a single uploadable branch, or None.
938 """
939 branch = self.GetBranch(branch_name)
940 base = branch.LocalMerge
941 if branch.LocalMerge:
942 rb = ReviewableBranch(self, branch, base)
943 if rb.commits:
944 return rb
945 return None
946
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -0700947 def UploadForReview(self, branch=None,
Anthony King7bdac712014-07-16 12:56:40 +0100948 people=([], []),
Brian Harring435370c2012-07-28 15:37:04 -0700949 auto_topic=False,
Bryan Jacobsf609f912013-05-06 13:36:24 -0400950 draft=False,
951 dest_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700952 """Uploads the named branch for code review.
953 """
954 if branch is None:
955 branch = self.CurrentBranch
956 if branch is None:
957 raise GitError('not currently on a branch')
958
959 branch = self.GetBranch(branch)
960 if not branch.LocalMerge:
961 raise GitError('branch %s does not track a remote' % branch.name)
962 if not branch.remote.review:
963 raise GitError('remote %s has no review url' % branch.remote.name)
964
Bryan Jacobsf609f912013-05-06 13:36:24 -0400965 if dest_branch is None:
966 dest_branch = self.dest_branch
967 if dest_branch is None:
968 dest_branch = branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700969 if not dest_branch.startswith(R_HEADS):
970 dest_branch = R_HEADS + dest_branch
971
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800972 if not branch.remote.projectname:
973 branch.remote.projectname = self.name
974 branch.remote.Save()
975
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800976 url = branch.remote.ReviewUrl(self.UserEmail)
977 if url is None:
978 raise UploadError('review not configured')
979 cmd = ['push']
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800980
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800981 if url.startswith('ssh://'):
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800982 rp = ['gerrit receive-pack']
983 for e in people[0]:
984 rp.append('--reviewer=%s' % sq(e))
985 for e in people[1]:
986 rp.append('--cc=%s' % sq(e))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800987 cmd.append('--receive-pack=%s' % " ".join(rp))
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -0700988
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800989 cmd.append(url)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800990
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800991 if dest_branch.startswith(R_HEADS):
992 dest_branch = dest_branch[len(R_HEADS):]
Brian Harring435370c2012-07-28 15:37:04 -0700993
994 upload_type = 'for'
995 if draft:
996 upload_type = 'drafts'
997
998 ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
999 dest_branch)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001000 if auto_topic:
1001 ref_spec = ref_spec + '/' + branch.name
Shawn Pearce45d21682013-02-28 00:35:51 -08001002 if not url.startswith('ssh://'):
1003 rp = ['r=%s' % p for p in people[0]] + \
1004 ['cc=%s' % p for p in people[1]]
1005 if rp:
1006 ref_spec = ref_spec + '%' + ','.join(rp)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001007 cmd.append(ref_spec)
1008
Anthony King7bdac712014-07-16 12:56:40 +01001009 if GitCommand(self, cmd, bare=True).Wait() != 0:
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001010 raise UploadError('Upload failed')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001011
1012 msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
1013 self.bare_git.UpdateRef(R_PUB + branch.name,
1014 R_HEADS + branch.name,
Anthony King7bdac712014-07-16 12:56:40 +01001015 message=msg)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001016
1017
1018## Sync ##
1019
Julien Campergue335f5ef2013-10-16 11:02:35 +02001020 def _ExtractArchive(self, tarpath, path=None):
1021 """Extract the given tar on its current location
1022
1023 Args:
1024 - tarpath: The path to the actual tar file
1025
1026 """
1027 try:
1028 with tarfile.open(tarpath, 'r') as tar:
1029 tar.extractall(path=path)
1030 return True
1031 except (IOError, tarfile.TarError) as e:
1032 print("error: Cannot extract archive %s: "
1033 "%s" % (tarpath, str(e)), file=sys.stderr)
1034 return False
1035
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001036 def Sync_NetworkHalf(self,
1037 quiet=False,
1038 is_new=None,
1039 current_branch_only=False,
Mitchel Humpherys597868b2012-10-29 10:18:34 -07001040 clone_bundle=True,
Julien Campergue335f5ef2013-10-16 11:02:35 +02001041 no_tags=False,
1042 archive=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001043 """Perform only the network IO portion of the sync process.
1044 Local working directory/branch state is not affected.
1045 """
Julien Campergue335f5ef2013-10-16 11:02:35 +02001046 if archive and not isinstance(self, MetaProject):
1047 if self.remote.url.startswith(('http://', 'https://')):
1048 print("error: %s: Cannot fetch archives from http/https "
1049 "remotes." % self.name, file=sys.stderr)
1050 return False
1051
1052 name = self.relpath.replace('\\', '/')
1053 name = name.replace('/', '_')
1054 tarpath = '%s.tar' % name
1055 topdir = self.manifest.topdir
1056
1057 try:
1058 self._FetchArchive(tarpath, cwd=topdir)
1059 except GitError as e:
1060 print('error: %s' % str(e), file=sys.stderr)
1061 return False
1062
1063 # From now on, we only need absolute tarpath
1064 tarpath = os.path.join(topdir, tarpath)
1065
1066 if not self._ExtractArchive(tarpath, path=topdir):
1067 return False
1068 try:
1069 os.remove(tarpath)
1070 except OSError as e:
1071 print("warn: Cannot remove archive %s: "
1072 "%s" % (tarpath, str(e)), file=sys.stderr)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001073 self._CopyAndLinkFiles()
Julien Campergue335f5ef2013-10-16 11:02:35 +02001074 return True
1075
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001076 if is_new is None:
1077 is_new = not self.Exists
Shawn O. Pearce88443382010-10-08 10:02:09 +02001078 if is_new:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001079 self._InitGitDir()
Jimmie Westera0444582012-10-24 13:44:42 +02001080 else:
1081 self._UpdateHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001082 self._InitRemote()
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001083
1084 if is_new:
1085 alt = os.path.join(self.gitdir, 'objects/info/alternates')
1086 try:
1087 fd = open(alt, 'rb')
1088 try:
1089 alt_dir = fd.readline().rstrip()
1090 finally:
1091 fd.close()
1092 except IOError:
1093 alt_dir = None
1094 else:
1095 alt_dir = None
1096
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001097 if clone_bundle \
1098 and alt_dir is None \
1099 and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001100 is_new = False
1101
Shawn O. Pearce6ba6ba02012-05-24 09:46:50 -07001102 if not current_branch_only:
1103 if self.sync_c:
1104 current_branch_only = True
1105 elif not self.manifest._loaded:
1106 # Manifest cannot check defaults until it syncs.
1107 current_branch_only = False
1108 elif self.manifest.default.sync_c:
1109 current_branch_only = True
1110
Conley Owens666d5342014-05-01 13:09:57 -07001111 has_sha1 = ID_RE.match(self.revisionExpr) and self._CheckForSha1()
1112 if (not has_sha1 #Need to fetch since we don't already have this revision
1113 and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
1114 current_branch_only=current_branch_only,
1115 no_tags=no_tags)):
Anthony King7bdac712014-07-16 12:56:40 +01001116 return False
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001117
1118 if self.worktree:
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001119 self._InitMRef()
1120 else:
1121 self._InitMirrorHead()
1122 try:
1123 os.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
1124 except OSError:
1125 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001126 return True
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08001127
1128 def PostRepoUpgrade(self):
1129 self._InitHooks()
1130
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001131 def _CopyAndLinkFiles(self):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001132 for copyfile in self.copyfiles:
1133 copyfile._Copy()
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001134 for linkfile in self.linkfiles:
1135 linkfile._Link()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001136
Julien Camperguedd654222014-01-09 16:21:37 +01001137 def GetCommitRevisionId(self):
1138 """Get revisionId of a commit.
1139
1140 Use this method instead of GetRevisionId to get the id of the commit rather
1141 than the id of the current git object (for example, a tag)
1142
1143 """
1144 if not self.revisionExpr.startswith(R_TAGS):
1145 return self.GetRevisionId(self._allrefs)
1146
1147 try:
1148 return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
1149 except GitError:
1150 raise ManifestInvalidRevisionError(
1151 'revision %s in %s not found' % (self.revisionExpr,
1152 self.name))
1153
David Pursehouse8a68ff92012-09-24 12:15:13 +09001154 def GetRevisionId(self, all_refs=None):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001155 if self.revisionId:
1156 return self.revisionId
1157
1158 rem = self.GetRemote(self.remote.name)
1159 rev = rem.ToLocal(self.revisionExpr)
1160
David Pursehouse8a68ff92012-09-24 12:15:13 +09001161 if all_refs is not None and rev in all_refs:
1162 return all_refs[rev]
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001163
1164 try:
1165 return self.bare_git.rev_parse('--verify', '%s^0' % rev)
1166 except GitError:
1167 raise ManifestInvalidRevisionError(
1168 'revision %s in %s not found' % (self.revisionExpr,
1169 self.name))
1170
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001171 def Sync_LocalHalf(self, syncbuf):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001172 """Perform only the local IO portion of the sync process.
1173 Network access is not required.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001174 """
David James8d201162013-10-11 17:03:19 -07001175 self._InitWorkTree()
David Pursehouse8a68ff92012-09-24 12:15:13 +09001176 all_refs = self.bare_ref.all
1177 self.CleanPublishedCache(all_refs)
1178 revid = self.GetRevisionId(all_refs)
Skyler Kaufman835cd682011-03-08 12:14:41 -08001179
David Pursehouse1d947b32012-10-25 12:23:11 +09001180 def _doff():
1181 self._FastForward(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001182 self._CopyAndLinkFiles()
David Pursehouse1d947b32012-10-25 12:23:11 +09001183
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001184 head = self.work_git.GetHead()
1185 if head.startswith(R_HEADS):
1186 branch = head[len(R_HEADS):]
1187 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001188 head = all_refs[head]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001189 except KeyError:
1190 head = None
1191 else:
1192 branch = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001193
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001194 if branch is None or syncbuf.detach_head:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001195 # Currently on a detached HEAD. The user is assumed to
1196 # not have any local modifications worth worrying about.
1197 #
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -07001198 if self.IsRebaseInProgress():
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001199 syncbuf.fail(self, _PriorSyncFailedError())
1200 return
1201
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001202 if head == revid:
1203 # No changes; don't do anything further.
Florian Vallee7cf1b362012-06-07 17:11:42 +02001204 # Except if the head needs to be detached
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001205 #
Florian Vallee7cf1b362012-06-07 17:11:42 +02001206 if not syncbuf.detach_head:
1207 return
1208 else:
1209 lost = self._revlist(not_rev(revid), HEAD)
1210 if lost:
1211 syncbuf.info(self, "discarding %d commits", len(lost))
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001212
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001213 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001214 self._Checkout(revid, quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001215 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001216 syncbuf.fail(self, e)
1217 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001218 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001219 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001220
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001221 if head == revid:
1222 # No changes; don't do anything further.
1223 #
1224 return
1225
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001226 branch = self.GetBranch(branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001227
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001228 if not branch.LocalMerge:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001229 # The current branch has no tracking configuration.
Anatol Pomazau2a32f6a2011-08-30 10:52:33 -07001230 # Jump off it to a detached HEAD.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001231 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001232 syncbuf.info(self,
1233 "leaving %s; does not track upstream",
1234 branch.name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001235 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001236 self._Checkout(revid, quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001237 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001238 syncbuf.fail(self, e)
1239 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001240 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001241 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001242
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001243 upstream_gain = self._revlist(not_rev(HEAD), revid)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001244 pub = self.WasPublished(branch.name, all_refs)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001245 if pub:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001246 not_merged = self._revlist(not_rev(revid), pub)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001247 if not_merged:
1248 if upstream_gain:
1249 # The user has published this branch and some of those
1250 # commits are not yet merged upstream. We do not want
1251 # to rewrite the published commits so we punt.
1252 #
Daniel Sandler4c50dee2010-03-02 15:38:03 -05001253 syncbuf.fail(self,
1254 "branch %s is published (but not merged) and is now %d commits behind"
1255 % (branch.name, len(upstream_gain)))
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001256 return
Shawn O. Pearce05f66b62009-04-21 08:26:32 -07001257 elif pub == head:
1258 # All published commits are merged, and thus we are a
1259 # strict subset. We can fast-forward safely.
Shawn O. Pearcea54c5272008-10-30 11:03:00 -07001260 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001261 syncbuf.later1(self, _doff)
1262 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001263
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001264 # Examine the local commits not in the remote. Find the
1265 # last one attributed to this user, if any.
1266 #
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001267 local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001268 last_mine = None
1269 cnt_mine = 0
1270 for commit in local_changes:
Chirayu Desai0eb35cb2013-11-19 18:46:29 +05301271 commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001272 if committer_email == self.UserEmail:
1273 last_mine = commit_id
1274 cnt_mine += 1
1275
Shawn O. Pearceda88ff42009-06-03 11:09:12 -07001276 if not upstream_gain and cnt_mine == len(local_changes):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001277 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001278
1279 if self.IsDirty(consider_untracked=False):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001280 syncbuf.fail(self, _DirtyError())
1281 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001282
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001283 # If the upstream switched on us, warn the user.
1284 #
1285 if branch.merge != self.revisionExpr:
1286 if branch.merge and self.revisionExpr:
1287 syncbuf.info(self,
1288 'manifest switched %s...%s',
1289 branch.merge,
1290 self.revisionExpr)
1291 elif branch.merge:
1292 syncbuf.info(self,
1293 'manifest no longer tracks %s',
1294 branch.merge)
1295
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001296 if cnt_mine < len(local_changes):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001297 # Upstream rebased. Not everything in HEAD
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001298 # was created by this user.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001299 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001300 syncbuf.info(self,
1301 "discarding %d commits removed from upstream",
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001302 len(local_changes) - cnt_mine)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001303
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001304 branch.remote = self.GetRemote(self.remote.name)
Anatol Pomazaucd7c5de2012-03-20 13:45:00 -07001305 if not ID_RE.match(self.revisionExpr):
1306 # in case of manifest sync the revisionExpr might be a SHA1
1307 branch.merge = self.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001308 branch.Save()
1309
Mike Pontillod3153822012-02-28 11:53:24 -08001310 if cnt_mine > 0 and self.rebase:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001311 def _dorebase():
Anthony King7bdac712014-07-16 12:56:40 +01001312 self._Rebase(upstream='%s^1' % last_mine, onto=revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001313 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001314 syncbuf.later2(self, _dorebase)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001315 elif local_changes:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001316 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001317 self._ResetHard(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001318 self._CopyAndLinkFiles()
Sarah Owensa5be53f2012-09-09 15:37:57 -07001319 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001320 syncbuf.fail(self, e)
1321 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001322 else:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001323 syncbuf.later1(self, _doff)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001324
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001325 def AddCopyFile(self, src, dest, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001326 # dest should already be an absolute path, but src is project relative
1327 # make src an absolute path
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001328 abssrc = os.path.join(self.worktree, src)
1329 self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001330
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001331 def AddLinkFile(self, src, dest, absdest):
1332 # dest should already be an absolute path, but src is project relative
1333 # make src an absolute path
1334 abssrc = os.path.join(self.worktree, src)
1335 self.linkfiles.append(_LinkFile(src, dest, abssrc, absdest))
1336
James W. Mills24c13082012-04-12 15:04:13 -05001337 def AddAnnotation(self, name, value, keep):
1338 self.annotations.append(_Annotation(name, value, keep))
1339
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001340 def DownloadPatchSet(self, change_id, patch_id):
1341 """Download a single patch set of a single change to FETCH_HEAD.
1342 """
1343 remote = self.GetRemote(self.remote.name)
1344
1345 cmd = ['fetch', remote.name]
1346 cmd.append('refs/changes/%2.2d/%d/%d' \
1347 % (change_id % 100, change_id, patch_id))
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001348 if GitCommand(self, cmd, bare=True).Wait() != 0:
1349 return None
1350 return DownloadedChange(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001351 self.GetRevisionId(),
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001352 change_id,
1353 patch_id,
1354 self.bare_git.rev_parse('FETCH_HEAD'))
1355
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001356
1357## Branch Management ##
1358
1359 def StartBranch(self, name):
1360 """Create a new branch off the manifest's revision.
1361 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001362 head = self.work_git.GetHead()
1363 if head == (R_HEADS + name):
1364 return True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001365
David Pursehouse8a68ff92012-09-24 12:15:13 +09001366 all_refs = self.bare_ref.all
Anthony King7bdac712014-07-16 12:56:40 +01001367 if R_HEADS + name in all_refs:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001368 return GitCommand(self,
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001369 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001370 capture_stdout=True,
1371 capture_stderr=True).Wait() == 0
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001372
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001373 branch = self.GetBranch(name)
1374 branch.remote = self.GetRemote(self.remote.name)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001375 branch.merge = self.revisionExpr
David Pursehouse8a68ff92012-09-24 12:15:13 +09001376 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001377
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001378 if head.startswith(R_HEADS):
1379 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001380 head = all_refs[head]
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001381 except KeyError:
1382 head = None
1383
1384 if revid and head and revid == head:
1385 ref = os.path.join(self.gitdir, R_HEADS + name)
1386 try:
1387 os.makedirs(os.path.dirname(ref))
1388 except OSError:
1389 pass
1390 _lwrite(ref, '%s\n' % revid)
1391 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1392 'ref: %s%s\n' % (R_HEADS, name))
1393 branch.Save()
1394 return True
1395
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001396 if GitCommand(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001397 ['checkout', '-b', branch.name, revid],
Anthony King7bdac712014-07-16 12:56:40 +01001398 capture_stdout=True,
1399 capture_stderr=True).Wait() == 0:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001400 branch.Save()
1401 return True
1402 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001403
Wink Saville02d79452009-04-10 13:01:24 -07001404 def CheckoutBranch(self, name):
1405 """Checkout a local topic branch.
Doug Anderson3ba5f952011-04-07 12:51:04 -07001406
1407 Args:
1408 name: The name of the branch to checkout.
1409
1410 Returns:
1411 True if the checkout succeeded; False if it didn't; None if the branch
1412 didn't exist.
Wink Saville02d79452009-04-10 13:01:24 -07001413 """
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001414 rev = R_HEADS + name
1415 head = self.work_git.GetHead()
1416 if head == rev:
1417 # Already on the branch
1418 #
1419 return True
Wink Saville02d79452009-04-10 13:01:24 -07001420
David Pursehouse8a68ff92012-09-24 12:15:13 +09001421 all_refs = self.bare_ref.all
Wink Saville02d79452009-04-10 13:01:24 -07001422 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001423 revid = all_refs[rev]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001424 except KeyError:
1425 # Branch does not exist in this project
1426 #
Doug Anderson3ba5f952011-04-07 12:51:04 -07001427 return None
Wink Saville02d79452009-04-10 13:01:24 -07001428
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001429 if head.startswith(R_HEADS):
1430 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001431 head = all_refs[head]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001432 except KeyError:
1433 head = None
1434
1435 if head == revid:
1436 # Same revision; just update HEAD to point to the new
1437 # target branch, but otherwise take no other action.
1438 #
1439 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1440 'ref: %s%s\n' % (R_HEADS, name))
1441 return True
1442
1443 return GitCommand(self,
1444 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001445 capture_stdout=True,
1446 capture_stderr=True).Wait() == 0
Wink Saville02d79452009-04-10 13:01:24 -07001447
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001448 def AbandonBranch(self, name):
1449 """Destroy a local topic branch.
Doug Andersondafb1d62011-04-07 11:46:59 -07001450
1451 Args:
1452 name: The name of the branch to abandon.
1453
1454 Returns:
1455 True if the abandon succeeded; False if it didn't; None if the branch
1456 didn't exist.
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001457 """
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001458 rev = R_HEADS + name
David Pursehouse8a68ff92012-09-24 12:15:13 +09001459 all_refs = self.bare_ref.all
1460 if rev not in all_refs:
Doug Andersondafb1d62011-04-07 11:46:59 -07001461 # Doesn't exist
1462 return None
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001463
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001464 head = self.work_git.GetHead()
1465 if head == rev:
1466 # We can't destroy the branch while we are sitting
1467 # on it. Switch to a detached HEAD.
1468 #
David Pursehouse8a68ff92012-09-24 12:15:13 +09001469 head = all_refs[head]
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001470
David Pursehouse8a68ff92012-09-24 12:15:13 +09001471 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001472 if head == revid:
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001473 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1474 '%s\n' % revid)
1475 else:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001476 self._Checkout(revid, quiet=True)
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001477
1478 return GitCommand(self,
1479 ['branch', '-D', name],
Anthony King7bdac712014-07-16 12:56:40 +01001480 capture_stdout=True,
1481 capture_stderr=True).Wait() == 0
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001482
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001483 def PruneHeads(self):
1484 """Prune any topic branches already merged into upstream.
1485 """
1486 cb = self.CurrentBranch
1487 kill = []
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001488 left = self._allrefs
1489 for name in left.keys():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001490 if name.startswith(R_HEADS):
1491 name = name[len(R_HEADS):]
1492 if cb is None or name != cb:
1493 kill.append(name)
1494
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001495 rev = self.GetRevisionId(left)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001496 if cb is not None \
1497 and not self._revlist(HEAD + '...' + rev) \
Anthony King7bdac712014-07-16 12:56:40 +01001498 and not self.IsDirty(consider_untracked=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001499 self.work_git.DetachHead(HEAD)
1500 kill.append(cb)
1501
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001502 if kill:
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07001503 old = self.bare_git.GetHead()
1504 if old is None:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001505 old = 'refs/heads/please_never_use_this_as_a_branch_name'
1506
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001507 try:
1508 self.bare_git.DetachHead(rev)
1509
1510 b = ['branch', '-d']
1511 b.extend(kill)
1512 b = GitCommand(self, b, bare=True,
1513 capture_stdout=True,
1514 capture_stderr=True)
1515 b.Wait()
1516 finally:
1517 self.bare_git.SetHead(old)
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001518 left = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001519
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001520 for branch in kill:
1521 if (R_HEADS + branch) not in left:
1522 self.CleanPublishedCache()
1523 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001524
1525 if cb and cb not in kill:
1526 kill.append(cb)
Shawn O. Pearce7c6c64d2009-03-02 12:38:13 -08001527 kill.sort()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001528
1529 kept = []
1530 for branch in kill:
Anthony King7bdac712014-07-16 12:56:40 +01001531 if R_HEADS + branch in left:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001532 branch = self.GetBranch(branch)
1533 base = branch.LocalMerge
1534 if not base:
1535 base = rev
1536 kept.append(ReviewableBranch(self, branch, base))
1537 return kept
1538
1539
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001540## Submodule Management ##
1541
1542 def GetRegisteredSubprojects(self):
1543 result = []
1544 def rec(subprojects):
1545 if not subprojects:
1546 return
1547 result.extend(subprojects)
1548 for p in subprojects:
1549 rec(p.subprojects)
1550 rec(self.subprojects)
1551 return result
1552
1553 def _GetSubmodules(self):
1554 # Unfortunately we cannot call `git submodule status --recursive` here
1555 # because the working tree might not exist yet, and it cannot be used
1556 # without a working tree in its current implementation.
1557
1558 def get_submodules(gitdir, rev):
1559 # Parse .gitmodules for submodule sub_paths and sub_urls
1560 sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
1561 if not sub_paths:
1562 return []
1563 # Run `git ls-tree` to read SHAs of submodule object, which happen to be
1564 # revision of submodule repository
1565 sub_revs = git_ls_tree(gitdir, rev, sub_paths)
1566 submodules = []
1567 for sub_path, sub_url in zip(sub_paths, sub_urls):
1568 try:
1569 sub_rev = sub_revs[sub_path]
1570 except KeyError:
1571 # Ignore non-exist submodules
1572 continue
1573 submodules.append((sub_rev, sub_path, sub_url))
1574 return submodules
1575
1576 re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
1577 re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
1578 def parse_gitmodules(gitdir, rev):
1579 cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
1580 try:
Anthony King7bdac712014-07-16 12:56:40 +01001581 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1582 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001583 except GitError:
1584 return [], []
1585 if p.Wait() != 0:
1586 return [], []
1587
1588 gitmodules_lines = []
1589 fd, temp_gitmodules_path = tempfile.mkstemp()
1590 try:
1591 os.write(fd, p.stdout)
1592 os.close(fd)
1593 cmd = ['config', '--file', temp_gitmodules_path, '--list']
Anthony King7bdac712014-07-16 12:56:40 +01001594 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1595 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001596 if p.Wait() != 0:
1597 return [], []
1598 gitmodules_lines = p.stdout.split('\n')
1599 except GitError:
1600 return [], []
1601 finally:
1602 os.remove(temp_gitmodules_path)
1603
1604 names = set()
1605 paths = {}
1606 urls = {}
1607 for line in gitmodules_lines:
1608 if not line:
1609 continue
1610 m = re_path.match(line)
1611 if m:
1612 names.add(m.group(1))
1613 paths[m.group(1)] = m.group(2)
1614 continue
1615 m = re_url.match(line)
1616 if m:
1617 names.add(m.group(1))
1618 urls[m.group(1)] = m.group(2)
1619 continue
1620 names = sorted(names)
1621 return ([paths.get(name, '') for name in names],
1622 [urls.get(name, '') for name in names])
1623
1624 def git_ls_tree(gitdir, rev, paths):
1625 cmd = ['ls-tree', rev, '--']
1626 cmd.extend(paths)
1627 try:
Anthony King7bdac712014-07-16 12:56:40 +01001628 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1629 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001630 except GitError:
1631 return []
1632 if p.Wait() != 0:
1633 return []
1634 objects = {}
1635 for line in p.stdout.split('\n'):
1636 if not line.strip():
1637 continue
1638 object_rev, object_path = line.split()[2:4]
1639 objects[object_path] = object_rev
1640 return objects
1641
1642 try:
1643 rev = self.GetRevisionId()
1644 except GitError:
1645 return []
1646 return get_submodules(self.gitdir, rev)
1647
1648 def GetDerivedSubprojects(self):
1649 result = []
1650 if not self.Exists:
1651 # If git repo does not exist yet, querying its submodules will
1652 # mess up its states; so return here.
1653 return result
1654 for rev, path, url in self._GetSubmodules():
1655 name = self.manifest.GetSubprojectName(self, path)
David James8d201162013-10-11 17:03:19 -07001656 relpath, worktree, gitdir, objdir = \
1657 self.manifest.GetSubprojectPaths(self, name, path)
1658 project = self.manifest.paths.get(relpath)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001659 if project:
1660 result.extend(project.GetDerivedSubprojects())
1661 continue
David James8d201162013-10-11 17:03:19 -07001662
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001663 remote = RemoteSpec(self.remote.name,
Anthony King7bdac712014-07-16 12:56:40 +01001664 url=url,
1665 review=self.remote.review,
1666 revision=self.remote.revision)
1667 subproject = Project(manifest=self.manifest,
1668 name=name,
1669 remote=remote,
1670 gitdir=gitdir,
1671 objdir=objdir,
1672 worktree=worktree,
1673 relpath=relpath,
1674 revisionExpr=self.revisionExpr,
1675 revisionId=rev,
1676 rebase=self.rebase,
1677 groups=self.groups,
1678 sync_c=self.sync_c,
1679 sync_s=self.sync_s,
1680 parent=self,
1681 is_derived=True)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001682 result.append(subproject)
1683 result.extend(subproject.GetDerivedSubprojects())
1684 return result
1685
1686
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001687## Direct Git Commands ##
Chris AtLee2fb64662014-01-16 21:32:33 -05001688 def _CheckForSha1(self):
1689 try:
1690 # if revision (sha or tag) is not present then following function
1691 # throws an error.
1692 self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
1693 return True
1694 except GitError:
1695 # There is no such persistent revision. We have to fetch it.
1696 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001697
Julien Campergue335f5ef2013-10-16 11:02:35 +02001698 def _FetchArchive(self, tarpath, cwd=None):
1699 cmd = ['archive', '-v', '-o', tarpath]
1700 cmd.append('--remote=%s' % self.remote.url)
1701 cmd.append('--prefix=%s/' % self.relpath)
1702 cmd.append(self.revisionExpr)
1703
1704 command = GitCommand(self, cmd, cwd=cwd,
1705 capture_stdout=True,
1706 capture_stderr=True)
1707
1708 if command.Wait() != 0:
1709 raise GitError('git archive %s: %s' % (self.name, command.stderr))
1710
Conley Owens80b87fe2014-05-09 17:13:44 -07001711
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001712 def _RemoteFetch(self, name=None,
1713 current_branch_only=False,
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001714 initial=False,
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001715 quiet=False,
Mitchel Humpherys597868b2012-10-29 10:18:34 -07001716 alt_dir=None,
1717 no_tags=False):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001718
1719 is_sha1 = False
1720 tag_name = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001721 depth = None
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001722
David Pursehouse9bc422f2014-04-15 10:28:56 +09001723 # The depth should not be used when fetching to a mirror because
1724 # it will result in a shallow repository that cannot be cloned or
1725 # fetched from.
1726 if not self.manifest.IsMirror:
1727 if self.clone_depth:
1728 depth = self.clone_depth
1729 else:
1730 depth = self.manifest.manifestProject.config.GetString('repo.depth')
Conley Owense4978cf2015-02-03 18:06:16 -08001731 # The repo project should never be synced with partial depth
1732 if self.relpath == '.repo/repo':
1733 depth = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001734
Shawn Pearce69e04d82014-01-29 12:48:54 -08001735 if depth:
1736 current_branch_only = True
1737
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001738 if ID_RE.match(self.revisionExpr) is not None:
1739 is_sha1 = True
1740
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001741 if current_branch_only:
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001742 if self.revisionExpr.startswith(R_TAGS):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001743 # this is a tag and its sha1 value should never change
1744 tag_name = self.revisionExpr[len(R_TAGS):]
1745
1746 if is_sha1 or tag_name is not None:
Chris AtLee2fb64662014-01-16 21:32:33 -05001747 if self._CheckForSha1():
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001748 return True
Bertrand SIMONNET3000cda2014-11-25 16:19:29 -08001749 if is_sha1 and not depth:
1750 # When syncing a specific commit and --depth is not set:
1751 # * if upstream is explicitly specified and is not a sha1, fetch only
1752 # upstream as users expect only upstream to be fetch.
1753 # Note: The commit might not be in upstream in which case the sync
1754 # will fail.
1755 # * otherwise, fetch all branches to make sure we end up with the
1756 # specific commit.
1757 current_branch_only = self.upstream and not ID_RE.match(self.upstream)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001758
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001759 if not name:
1760 name = self.remote.name
Shawn O. Pearcefb231612009-04-10 18:53:46 -07001761
1762 ssh_proxy = False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001763 remote = self.GetRemote(name)
1764 if remote.PreConnectFetch():
Shawn O. Pearcefb231612009-04-10 18:53:46 -07001765 ssh_proxy = True
1766
Shawn O. Pearce88443382010-10-08 10:02:09 +02001767 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001768 if alt_dir and 'objects' == os.path.basename(alt_dir):
1769 ref_dir = os.path.dirname(alt_dir)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001770 packed_refs = os.path.join(self.gitdir, 'packed-refs')
1771 remote = self.GetRemote(name)
1772
David Pursehouse8a68ff92012-09-24 12:15:13 +09001773 all_refs = self.bare_ref.all
1774 ids = set(all_refs.values())
Shawn O. Pearce88443382010-10-08 10:02:09 +02001775 tmp = set()
1776
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301777 for r, ref_id in GitRefs(ref_dir).all.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09001778 if r not in all_refs:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001779 if r.startswith(R_TAGS) or remote.WritesTo(r):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001780 all_refs[r] = ref_id
1781 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001782 continue
1783
David Pursehouse8a68ff92012-09-24 12:15:13 +09001784 if ref_id in ids:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001785 continue
1786
David Pursehouse8a68ff92012-09-24 12:15:13 +09001787 r = 'refs/_alt/%s' % ref_id
1788 all_refs[r] = ref_id
1789 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001790 tmp.add(r)
1791
Shawn O. Pearce88443382010-10-08 10:02:09 +02001792 tmp_packed = ''
1793 old_packed = ''
1794
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301795 for r in sorted(all_refs):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001796 line = '%s %s\n' % (all_refs[r], r)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001797 tmp_packed += line
1798 if r not in tmp:
1799 old_packed += line
1800
1801 _lwrite(packed_refs, tmp_packed)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001802 else:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001803 alt_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02001804
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001805 cmd = ['fetch']
Doug Anderson30d45292011-05-04 15:01:04 -07001806
Conley Owensf97e8382015-01-21 11:12:46 -08001807 if depth:
Doug Anderson30d45292011-05-04 15:01:04 -07001808 cmd.append('--depth=%s' % depth)
1809
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001810 if quiet:
1811 cmd.append('--quiet')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001812 if not self.worktree:
1813 cmd.append('--update-head-ok')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001814 cmd.append(name)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001815
Mitchel Humpherys26c45a72014-03-10 14:21:59 -07001816 # If using depth then we should not get all the tags since they may
1817 # be outside of the depth.
1818 if no_tags or depth:
1819 cmd.append('--no-tags')
1820 else:
1821 cmd.append('--tags')
1822
Conley Owens80b87fe2014-05-09 17:13:44 -07001823 spec = []
Brian Harring14a66742012-09-28 20:21:57 -07001824 if not current_branch_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001825 # Fetch whole repo
Conley Owens80b87fe2014-05-09 17:13:44 -07001826 spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001827 elif tag_name is not None:
Conley Owens80b87fe2014-05-09 17:13:44 -07001828 spec.append('tag')
1829 spec.append(tag_name)
Nasser Grainawi04e52d62014-09-30 13:34:52 -06001830
1831 branch = self.revisionExpr
Bertrand SIMONNET3000cda2014-11-25 16:19:29 -08001832 if is_sha1 and depth:
1833 # Shallow checkout of a specific commit, fetch from that commit and not
1834 # the heads only as the commit might be deeper in the history.
1835 spec.append(branch)
1836 else:
1837 if is_sha1:
1838 branch = self.upstream
1839 if branch is not None and branch.strip():
1840 if not branch.startswith('refs/'):
1841 branch = R_HEADS + branch
1842 spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
Conley Owens80b87fe2014-05-09 17:13:44 -07001843 cmd.extend(spec)
1844
1845 shallowfetch = self.config.GetString('repo.shallowfetch')
1846 if shallowfetch and shallowfetch != ' '.join(spec):
1847 GitCommand(self, ['fetch', '--unshallow', name] + shallowfetch.split(),
1848 bare=True, ssh_proxy=ssh_proxy).Wait()
1849 if depth:
Anthony King7bdac712014-07-16 12:56:40 +01001850 self.config.SetString('repo.shallowfetch', ' '.join(spec))
Conley Owens80b87fe2014-05-09 17:13:44 -07001851 else:
Anthony King7bdac712014-07-16 12:56:40 +01001852 self.config.SetString('repo.shallowfetch', None)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001853
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001854 ok = False
David Pursehouse8a68ff92012-09-24 12:15:13 +09001855 for _i in range(2):
John L. Villalovos126e2982015-01-29 21:58:12 -08001856 gitcmd = GitCommand(self, cmd, bare=True, capture_stderr=True,
1857 ssh_proxy=ssh_proxy)
1858 ret = gitcmd.Wait()
John L. Villalovose30f46b2015-02-25 14:27:02 -08001859 print(gitcmd.stderr, file=sys.stderr, end='')
Brian Harring14a66742012-09-28 20:21:57 -07001860 if ret == 0:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001861 ok = True
1862 break
John L. Villalovos126e2982015-01-29 21:58:12 -08001863 # If needed, run the 'git remote prune' the first time through the loop
1864 elif (not _i and
1865 "error:" in gitcmd.stderr and
1866 "git remote prune" in gitcmd.stderr):
1867 prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
1868 capture_stderr=True, ssh_proxy=ssh_proxy)
John L. Villalovose30f46b2015-02-25 14:27:02 -08001869 ret = prunecmd.Wait()
1870 print(prunecmd.stderr, file=sys.stderr, end='')
1871 if ret:
John L. Villalovos126e2982015-01-29 21:58:12 -08001872 break
1873 continue
Brian Harring14a66742012-09-28 20:21:57 -07001874 elif current_branch_only and is_sha1 and ret == 128:
1875 # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1
1876 # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
1877 # abort the optimization attempt and do a full sync.
1878 break
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001879 time.sleep(random.randint(30, 45))
Shawn O. Pearce88443382010-10-08 10:02:09 +02001880
1881 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001882 if alt_dir:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001883 if old_packed != '':
1884 _lwrite(packed_refs, old_packed)
1885 else:
1886 os.remove(packed_refs)
1887 self.bare_git.pack_refs('--all', '--prune')
Brian Harring14a66742012-09-28 20:21:57 -07001888
1889 if is_sha1 and current_branch_only and self.upstream:
1890 # We just synced the upstream given branch; verify we
1891 # got what we wanted, else trigger a second run of all
1892 # refs.
Chris AtLee2fb64662014-01-16 21:32:33 -05001893 if not self._CheckForSha1():
Brian Harring14a66742012-09-28 20:21:57 -07001894 return self._RemoteFetch(name=name, current_branch_only=False,
1895 initial=False, quiet=quiet, alt_dir=alt_dir)
1896
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001897 return ok
Shawn O. Pearce88443382010-10-08 10:02:09 +02001898
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001899 def _ApplyCloneBundle(self, initial=False, quiet=False):
David Pursehouseede7f122012-11-27 22:25:30 +09001900 if initial and (self.manifest.manifestProject.config.GetString('repo.depth') or self.clone_depth):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001901 return False
1902
1903 remote = self.GetRemote(self.remote.name)
1904 bundle_url = remote.url + '/clone.bundle'
1905 bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
Dave Borowitz74c1f3d2013-06-03 15:05:07 -07001906 if GetSchemeFromUrl(bundle_url) not in (
1907 'http', 'https', 'persistent-http', 'persistent-https'):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001908 return False
1909
1910 bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
1911 bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
1912
1913 exist_dst = os.path.exists(bundle_dst)
1914 exist_tmp = os.path.exists(bundle_tmp)
1915
1916 if not initial and not exist_dst and not exist_tmp:
1917 return False
1918
1919 if not exist_dst:
1920 exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
1921 if not exist_dst:
1922 return False
1923
1924 cmd = ['fetch']
1925 if quiet:
1926 cmd.append('--quiet')
1927 if not self.worktree:
1928 cmd.append('--update-head-ok')
1929 cmd.append(bundle_dst)
1930 for f in remote.fetch:
1931 cmd.append(str(f))
1932 cmd.append('refs/tags/*:refs/tags/*')
1933
1934 ok = GitCommand(self, cmd, bare=True).Wait() == 0
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001935 if os.path.exists(bundle_dst):
1936 os.remove(bundle_dst)
1937 if os.path.exists(bundle_tmp):
1938 os.remove(bundle_tmp)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001939 return ok
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001940
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001941 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001942 if os.path.exists(dstPath):
1943 os.remove(dstPath)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001944
Matt Gumbel2dc810c2012-08-30 09:39:36 -07001945 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001946 if quiet:
1947 cmd += ['--silent']
1948 if os.path.exists(tmpPath):
1949 size = os.stat(tmpPath).st_size
1950 if size >= 1024:
1951 cmd += ['--continue-at', '%d' % (size,)]
1952 else:
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001953 os.remove(tmpPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001954 if 'http_proxy' in os.environ and 'darwin' == sys.platform:
1955 cmd += ['--proxy', os.environ['http_proxy']]
Dave Borowitz497bde42015-01-02 13:58:05 -08001956 with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile:
Dave Borowitz137d0132015-01-02 11:12:54 -08001957 if cookiefile:
Dave Borowitz4abf8e62015-01-02 11:39:04 -08001958 cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
Dave Borowitz137d0132015-01-02 11:12:54 -08001959 if srcUrl.startswith('persistent-'):
1960 srcUrl = srcUrl[len('persistent-'):]
1961 cmd += [srcUrl]
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001962
Dave Borowitz137d0132015-01-02 11:12:54 -08001963 if IsTrace():
1964 Trace('%s', ' '.join(cmd))
1965 try:
1966 proc = subprocess.Popen(cmd)
1967 except OSError:
1968 return False
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001969
Dave Borowitz137d0132015-01-02 11:12:54 -08001970 curlret = proc.wait()
Matt Gumbel2dc810c2012-08-30 09:39:36 -07001971
Dave Borowitz137d0132015-01-02 11:12:54 -08001972 if curlret == 22:
1973 # From curl man page:
1974 # 22: HTTP page not retrieved. The requested url was not found or
1975 # returned another error with the HTTP error code being 400 or above.
1976 # This return code only appears if -f, --fail is used.
1977 if not quiet:
1978 print("Server does not provide clone.bundle; ignoring.",
1979 file=sys.stderr)
1980 return False
Matt Gumbel2dc810c2012-08-30 09:39:36 -07001981
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001982 if os.path.exists(tmpPath):
Kris Giesingc8d882a2014-12-23 13:02:32 -08001983 if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001984 os.rename(tmpPath, dstPath)
1985 return True
1986 else:
1987 os.remove(tmpPath)
1988 return False
1989 else:
1990 return False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001991
Kris Giesingc8d882a2014-12-23 13:02:32 -08001992 def _IsValidBundle(self, path, quiet):
Dave Borowitz91f3ba52013-06-03 12:15:23 -07001993 try:
1994 with open(path) as f:
1995 if f.read(16) == '# v2 git bundle\n':
1996 return True
1997 else:
Kris Giesingc8d882a2014-12-23 13:02:32 -08001998 if not quiet:
1999 print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002000 return False
2001 except OSError:
2002 return False
2003
Dave Borowitz137d0132015-01-02 11:12:54 -08002004 @contextlib.contextmanager
Dave Borowitz497bde42015-01-02 13:58:05 -08002005 def _GetBundleCookieFile(self, url, quiet):
Dave Borowitz74c1f3d2013-06-03 15:05:07 -07002006 if url.startswith('persistent-'):
2007 try:
2008 p = subprocess.Popen(
2009 ['git-remote-persistent-https', '-print_config', url],
2010 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2011 stderr=subprocess.PIPE)
Dave Borowitz137d0132015-01-02 11:12:54 -08002012 try:
2013 prefix = 'http.cookiefile='
2014 cookiefile = None
2015 for line in p.stdout:
2016 line = line.strip()
2017 if line.startswith(prefix):
2018 cookiefile = line[len(prefix):]
2019 break
2020 # Leave subprocess open, as cookie file may be transient.
2021 if cookiefile:
2022 yield cookiefile
2023 return
2024 finally:
2025 p.stdin.close()
2026 if p.wait():
2027 err_msg = p.stderr.read()
2028 if ' -print_config' in err_msg:
2029 pass # Persistent proxy doesn't support -print_config.
Dave Borowitz497bde42015-01-02 13:58:05 -08002030 elif not quiet:
Dave Borowitz137d0132015-01-02 11:12:54 -08002031 print(err_msg, file=sys.stderr)
Dave Borowitz74c1f3d2013-06-03 15:05:07 -07002032 except OSError as e:
2033 if e.errno == errno.ENOENT:
2034 pass # No persistent proxy.
2035 raise
Dave Borowitz137d0132015-01-02 11:12:54 -08002036 yield GitConfig.ForUser().GetString('http.cookiefile')
Dave Borowitz74c1f3d2013-06-03 15:05:07 -07002037
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002038 def _Checkout(self, rev, quiet=False):
2039 cmd = ['checkout']
2040 if quiet:
2041 cmd.append('-q')
2042 cmd.append(rev)
2043 cmd.append('--')
2044 if GitCommand(self, cmd).Wait() != 0:
2045 if self._allrefs:
2046 raise GitError('%s checkout %s ' % (self.name, rev))
2047
Anthony King7bdac712014-07-16 12:56:40 +01002048 def _CherryPick(self, rev):
Pierre Tardye5a21222011-03-24 16:28:18 +01002049 cmd = ['cherry-pick']
2050 cmd.append(rev)
2051 cmd.append('--')
2052 if GitCommand(self, cmd).Wait() != 0:
2053 if self._allrefs:
2054 raise GitError('%s cherry-pick %s ' % (self.name, rev))
2055
Anthony King7bdac712014-07-16 12:56:40 +01002056 def _Revert(self, rev):
Erwan Mahea94f1622011-08-19 13:56:09 +02002057 cmd = ['revert']
2058 cmd.append('--no-edit')
2059 cmd.append(rev)
2060 cmd.append('--')
2061 if GitCommand(self, cmd).Wait() != 0:
2062 if self._allrefs:
2063 raise GitError('%s revert %s ' % (self.name, rev))
2064
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002065 def _ResetHard(self, rev, quiet=True):
2066 cmd = ['reset', '--hard']
2067 if quiet:
2068 cmd.append('-q')
2069 cmd.append(rev)
2070 if GitCommand(self, cmd).Wait() != 0:
2071 raise GitError('%s reset --hard %s ' % (self.name, rev))
2072
Anthony King7bdac712014-07-16 12:56:40 +01002073 def _Rebase(self, upstream, onto=None):
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002074 cmd = ['rebase']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002075 if onto is not None:
2076 cmd.extend(['--onto', onto])
2077 cmd.append(upstream)
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002078 if GitCommand(self, cmd).Wait() != 0:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002079 raise GitError('%s rebase %s ' % (self.name, upstream))
2080
Pierre Tardy3d125942012-05-04 12:18:12 +02002081 def _FastForward(self, head, ffonly=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002082 cmd = ['merge', head]
Pierre Tardy3d125942012-05-04 12:18:12 +02002083 if ffonly:
2084 cmd.append("--ff-only")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002085 if GitCommand(self, cmd).Wait() != 0:
2086 raise GitError('%s merge %s ' % (self.name, head))
2087
Jimmie Wester38e43872012-10-24 14:35:05 +02002088 def _InitGitDir(self, mirror_git=None, MirrorOverride=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002089 if not os.path.exists(self.gitdir):
David James8d201162013-10-11 17:03:19 -07002090
2091 # Initialize the bare repository, which contains all of the objects.
2092 if not os.path.exists(self.objdir):
2093 os.makedirs(self.objdir)
2094 self.bare_objdir.init()
2095
2096 # If we have a separate directory to hold refs, initialize it as well.
2097 if self.objdir != self.gitdir:
2098 os.makedirs(self.gitdir)
2099 self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
2100 copy_all=True)
Shawn O. Pearce2816d4f2009-03-03 17:53:18 -08002101
Shawn O. Pearce88443382010-10-08 10:02:09 +02002102 mp = self.manifest.manifestProject
Victor Boivie2b30e3a2012-10-05 12:37:58 +02002103 ref_dir = mp.config.GetString('repo.reference') or ''
Shawn O. Pearce88443382010-10-08 10:02:09 +02002104
Victor Boivie2b30e3a2012-10-05 12:37:58 +02002105 if ref_dir or mirror_git:
2106 if not mirror_git:
2107 mirror_git = os.path.join(ref_dir, self.name + '.git')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002108 repo_git = os.path.join(ref_dir, '.repo', 'projects',
2109 self.relpath + '.git')
2110
2111 if os.path.exists(mirror_git):
2112 ref_dir = mirror_git
2113
2114 elif os.path.exists(repo_git):
2115 ref_dir = repo_git
2116
2117 else:
2118 ref_dir = None
2119
2120 if ref_dir:
2121 _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
2122 os.path.join(ref_dir, 'objects') + '\n')
2123
Jimmie Westera0444582012-10-24 13:44:42 +02002124 self._UpdateHooks()
2125
2126 m = self.manifest.manifestProject.config
2127 for key in ['user.name', 'user.email']:
Anthony King7bdac712014-07-16 12:56:40 +01002128 if m.Has(key, include_defaults=False):
Jimmie Westera0444582012-10-24 13:44:42 +02002129 self.config.SetString(key, m.GetString(key))
Jimmie Wester38e43872012-10-24 14:35:05 +02002130 if self.manifest.IsMirror and not MirrorOverride:
Shawn O. Pearce2816d4f2009-03-03 17:53:18 -08002131 self.config.SetString('core.bare', 'true')
2132 else:
2133 self.config.SetString('core.bare', None)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002134
Jimmie Wester38e43872012-10-24 14:35:05 +02002135 def _ProjectHooks(self, remote, repodir):
2136 """List the hooks present in the 'hooks' directory.
2137
2138 These hooks are project hooks and are copied to the '.git/hooks' directory
2139 of all subprojects.
2140
2141 The remote projecthooks supplement/overrule any stockhook making it possible to
2142 have a combination of hooks both from the remote projecthook and
2143 .repo/hooks directories.
2144
2145 Returns:
2146 A list of absolute paths to all of the files in the hooks directory and
2147 projecthooks files, excluding the .git folder.
2148 """
2149 hooks = {}
2150 d = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'hooks')
2151 hooks = dict([(x, os.path.join(d, x)) for x in os.listdir(d)])
2152 if remote is not None:
2153 if remote.projecthookName is not None:
2154 d = os.path.abspath('%s/projecthooks/%s/%s' % (repodir, remote.name, remote.projecthookName))
2155 if os.path.isdir(d):
2156 hooks.update(dict([(x, os.path.join(d, x)) for x in os.listdir(d)]))
2157
2158 if hooks.has_key('.git'):
2159 del hooks['.git']
2160 return hooks.values()
2161
Jimmie Westera0444582012-10-24 13:44:42 +02002162 def _UpdateHooks(self):
2163 if os.path.exists(self.gitdir):
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002164 self._InitHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002165
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002166 def _InitHooks(self):
Jesse Hall672cc492013-11-27 11:17:13 -08002167 hooks = os.path.realpath(self._gitdir_path('hooks'))
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002168 if not os.path.exists(hooks):
2169 os.makedirs(hooks)
Jimmie Wester38e43872012-10-24 14:35:05 +02002170 pr = None
2171 if self is not self.manifest.manifestProject:
2172 pr = self.manifest.remotes.get(self.remote.name)
2173 for stock_hook in self._ProjectHooks(pr, self.manifest.repodir):
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002174 name = os.path.basename(stock_hook)
2175
Victor Boivie65e0f352011-04-18 11:23:29 +02002176 if name in ('commit-msg',) and not self.remote.review \
2177 and not self is self.manifest.manifestProject:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002178 # Don't install a Gerrit Code Review hook if this
2179 # project does not appear to use it for reviews.
2180 #
Victor Boivie65e0f352011-04-18 11:23:29 +02002181 # Since the manifest project is one of those, but also
2182 # managed through gerrit, it's excluded
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002183 continue
2184
2185 dst = os.path.join(hooks, name)
2186 if os.path.islink(dst):
2187 continue
2188 if os.path.exists(dst):
2189 if filecmp.cmp(stock_hook, dst, shallow=False):
2190 os.remove(dst)
2191 else:
2192 _error("%s: Not replacing %s hook", self.relpath, name)
2193 continue
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002194 try:
Mickaël Salaünb9477bc2012-08-05 13:39:26 +02002195 os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
Sarah Owensa5be53f2012-09-09 15:37:57 -07002196 except OSError as e:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002197 if e.errno == errno.EPERM:
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002198 raise GitError('filesystem must support symlinks')
2199 else:
2200 raise
2201
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002202 def _InitRemote(self):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002203 if self.remote.url:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002204 remote = self.GetRemote(self.remote.name)
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002205 remote.url = self.remote.url
2206 remote.review = self.remote.review
2207 remote.projectname = self.name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002208
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002209 if self.worktree:
2210 remote.ResetFetch(mirror=False)
2211 else:
2212 remote.ResetFetch(mirror=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002213 remote.Save()
2214
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002215 def _InitMRef(self):
2216 if self.manifest.branch:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002217 self._InitAnyMRef(R_M + self.manifest.branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002218
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002219 def _InitMirrorHead(self):
Shawn O. Pearcefe200ee2009-06-01 15:28:21 -07002220 self._InitAnyMRef(HEAD)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002221
2222 def _InitAnyMRef(self, ref):
2223 cur = self.bare_ref.symref(ref)
2224
2225 if self.revisionId:
2226 if cur != '' or self.bare_ref.get(ref) != self.revisionId:
2227 msg = 'manifest set to %s' % self.revisionId
2228 dst = self.revisionId + '^0'
Anthony King7bdac712014-07-16 12:56:40 +01002229 self.bare_git.UpdateRef(ref, dst, message=msg, detach=True)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002230 else:
2231 remote = self.GetRemote(self.remote.name)
2232 dst = remote.ToLocal(self.revisionExpr)
2233 if cur != dst:
2234 msg = 'manifest set to %s' % self.revisionExpr
2235 self.bare_git.symbolic_ref('-m', msg, ref, dst)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002236
David James8d201162013-10-11 17:03:19 -07002237 def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
2238 """Update |dotgit| to reference |gitdir|, using symlinks where possible.
2239
2240 Args:
2241 gitdir: The bare git repository. Must already be initialized.
2242 dotgit: The repository you would like to initialize.
2243 share_refs: If true, |dotgit| will store its refs under |gitdir|.
2244 Only one work tree can store refs under a given |gitdir|.
2245 copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
2246 This saves you the effort of initializing |dotgit| yourself.
2247 """
2248 # These objects can be shared between several working trees.
2249 symlink_files = ['description', 'info']
2250 symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
2251 if share_refs:
2252 # These objects can only be used by a single working tree.
Conley Owensf2af7562014-04-30 11:31:01 -07002253 symlink_files += ['config', 'packed-refs', 'shallow']
David James8d201162013-10-11 17:03:19 -07002254 symlink_dirs += ['logs', 'refs']
2255 to_symlink = symlink_files + symlink_dirs
2256
2257 to_copy = []
2258 if copy_all:
2259 to_copy = os.listdir(gitdir)
2260
2261 for name in set(to_copy).union(to_symlink):
2262 try:
2263 src = os.path.realpath(os.path.join(gitdir, name))
2264 dst = os.path.realpath(os.path.join(dotgit, name))
2265
2266 if os.path.lexists(dst) and not os.path.islink(dst):
2267 raise GitError('cannot overwrite a local work tree')
2268
2269 # If the source dir doesn't exist, create an empty dir.
2270 if name in symlink_dirs and not os.path.lexists(src):
2271 os.makedirs(src)
2272
Conley Owens80b87fe2014-05-09 17:13:44 -07002273 # If the source file doesn't exist, ensure the destination
2274 # file doesn't either.
2275 if name in symlink_files and not os.path.lexists(src):
2276 try:
2277 os.remove(dst)
2278 except OSError:
2279 pass
2280
David James8d201162013-10-11 17:03:19 -07002281 if name in to_symlink:
2282 os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
2283 elif copy_all and not os.path.islink(dst):
2284 if os.path.isdir(src):
2285 shutil.copytree(src, dst)
2286 elif os.path.isfile(src):
2287 shutil.copy(src, dst)
2288 except OSError as e:
2289 if e.errno == errno.EPERM:
2290 raise GitError('filesystem must support symlinks')
2291 else:
2292 raise
2293
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002294 def _InitWorkTree(self):
2295 dotgit = os.path.join(self.worktree, '.git')
2296 if not os.path.exists(dotgit):
2297 os.makedirs(dotgit)
David James8d201162013-10-11 17:03:19 -07002298 self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
2299 copy_all=False)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002300
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002301 _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002302
2303 cmd = ['read-tree', '--reset', '-u']
2304 cmd.append('-v')
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002305 cmd.append(HEAD)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002306 if GitCommand(self, cmd).Wait() != 0:
2307 raise GitError("cannot initialize work tree")
Victor Boivie0960b5b2010-11-26 13:42:13 +01002308
Jeff Hamiltone0df2322014-04-21 17:10:59 -05002309 self._CopyAndLinkFiles()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002310
2311 def _gitdir_path(self, path):
David James8d201162013-10-11 17:03:19 -07002312 return os.path.realpath(os.path.join(self.gitdir, path))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002313
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002314 def _revlist(self, *args, **kw):
2315 a = []
2316 a.extend(args)
2317 a.append('--')
2318 return self.work_git.rev_list(*a, **kw)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002319
2320 @property
2321 def _allrefs(self):
Shawn O. Pearced237b692009-04-17 18:49:50 -07002322 return self.bare_ref.all
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002323
Julien Camperguedd654222014-01-09 16:21:37 +01002324 def _getLogs(self, rev1, rev2, oneline=False, color=True):
2325 """Get logs between two revisions of this project."""
2326 comp = '..'
2327 if rev1:
2328 revs = [rev1]
2329 if rev2:
2330 revs.extend([comp, rev2])
2331 cmd = ['log', ''.join(revs)]
2332 out = DiffColoring(self.config)
2333 if out.is_on and color:
2334 cmd.append('--color')
2335 if oneline:
2336 cmd.append('--oneline')
2337
2338 try:
2339 log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
2340 if log.Wait() == 0:
2341 return log.stdout
2342 except GitError:
2343 # worktree may not exist if groups changed for example. In that case,
2344 # try in gitdir instead.
2345 if not os.path.exists(self.worktree):
2346 return self.bare_git.log(*cmd[1:])
2347 else:
2348 raise
2349 return None
2350
2351 def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True):
2352 """Get the list of logs from this revision to given revisionId"""
2353 logs = {}
2354 selfId = self.GetRevisionId(self._allrefs)
2355 toId = toProject.GetRevisionId(toProject._allrefs)
2356
2357 logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color)
2358 logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color)
2359 return logs
2360
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002361 class _GitGetByExec(object):
David James8d201162013-10-11 17:03:19 -07002362 def __init__(self, project, bare, gitdir):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002363 self._project = project
2364 self._bare = bare
David James8d201162013-10-11 17:03:19 -07002365 self._gitdir = gitdir
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002366
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002367 def LsOthers(self):
2368 p = GitCommand(self._project,
2369 ['ls-files',
2370 '-z',
2371 '--others',
2372 '--exclude-standard'],
Anthony King7bdac712014-07-16 12:56:40 +01002373 bare=False,
David James8d201162013-10-11 17:03:19 -07002374 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002375 capture_stdout=True,
2376 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002377 if p.Wait() == 0:
2378 out = p.stdout
2379 if out:
David Pursehouse1d947b32012-10-25 12:23:11 +09002380 return out[:-1].split('\0') # pylint: disable=W1401
2381 # Backslash is not anomalous
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002382 return []
2383
2384 def DiffZ(self, name, *args):
2385 cmd = [name]
2386 cmd.append('-z')
2387 cmd.extend(args)
2388 p = GitCommand(self._project,
2389 cmd,
David James8d201162013-10-11 17:03:19 -07002390 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002391 bare=False,
2392 capture_stdout=True,
2393 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002394 try:
2395 out = p.process.stdout.read()
2396 r = {}
2397 if out:
David Pursehouse1d947b32012-10-25 12:23:11 +09002398 out = iter(out[:-1].split('\0')) # pylint: disable=W1401
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002399 while out:
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002400 try:
Anthony King2cd1f042014-05-05 21:24:05 +01002401 info = next(out)
2402 path = next(out)
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002403 except StopIteration:
2404 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002405
2406 class _Info(object):
2407 def __init__(self, path, omode, nmode, oid, nid, state):
2408 self.path = path
2409 self.src_path = None
2410 self.old_mode = omode
2411 self.new_mode = nmode
2412 self.old_id = oid
2413 self.new_id = nid
2414
2415 if len(state) == 1:
2416 self.status = state
2417 self.level = None
2418 else:
2419 self.status = state[:1]
2420 self.level = state[1:]
2421 while self.level.startswith('0'):
2422 self.level = self.level[1:]
2423
2424 info = info[1:].split(' ')
David Pursehouse8f62fb72012-11-14 12:09:38 +09002425 info = _Info(path, *info)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002426 if info.status in ('R', 'C'):
2427 info.src_path = info.path
Anthony King2cd1f042014-05-05 21:24:05 +01002428 info.path = next(out)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002429 r[info.path] = info
2430 return r
2431 finally:
2432 p.Wait()
2433
2434 def GetHead(self):
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002435 if self._bare:
2436 path = os.path.join(self._project.gitdir, HEAD)
2437 else:
2438 path = os.path.join(self._project.worktree, '.git', HEAD)
Conley Owens75ee0572012-11-15 17:33:11 -08002439 try:
2440 fd = open(path, 'rb')
Dan Sandler53e902a2014-03-09 13:20:02 -04002441 except IOError as e:
2442 raise NoManifestException(path, str(e))
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -07002443 try:
2444 line = fd.read()
2445 finally:
2446 fd.close()
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302447 try:
2448 line = line.decode()
2449 except AttributeError:
2450 pass
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002451 if line.startswith('ref: '):
2452 return line[5:-1]
2453 return line[:-1]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002454
2455 def SetHead(self, ref, message=None):
2456 cmdv = []
2457 if message is not None:
2458 cmdv.extend(['-m', message])
2459 cmdv.append(HEAD)
2460 cmdv.append(ref)
2461 self.symbolic_ref(*cmdv)
2462
2463 def DetachHead(self, new, message=None):
2464 cmdv = ['--no-deref']
2465 if message is not None:
2466 cmdv.extend(['-m', message])
2467 cmdv.append(HEAD)
2468 cmdv.append(new)
2469 self.update_ref(*cmdv)
2470
2471 def UpdateRef(self, name, new, old=None,
2472 message=None,
2473 detach=False):
2474 cmdv = []
2475 if message is not None:
2476 cmdv.extend(['-m', message])
2477 if detach:
2478 cmdv.append('--no-deref')
2479 cmdv.append(name)
2480 cmdv.append(new)
2481 if old is not None:
2482 cmdv.append(old)
2483 self.update_ref(*cmdv)
2484
2485 def DeleteRef(self, name, old=None):
2486 if not old:
2487 old = self.rev_parse(name)
2488 self.update_ref('-d', name, old)
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07002489 self._project.bare_ref.deleted(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002490
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002491 def rev_list(self, *args, **kw):
2492 if 'format' in kw:
2493 cmdv = ['log', '--pretty=format:%s' % kw['format']]
2494 else:
2495 cmdv = ['rev-list']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002496 cmdv.extend(args)
2497 p = GitCommand(self._project,
2498 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002499 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002500 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002501 capture_stdout=True,
2502 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002503 r = []
2504 for line in p.process.stdout:
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002505 if line[-1] == '\n':
2506 line = line[:-1]
2507 r.append(line)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002508 if p.Wait() != 0:
2509 raise GitError('%s rev-list %s: %s' % (
2510 self._project.name,
2511 str(args),
2512 p.stderr))
2513 return r
2514
2515 def __getattr__(self, name):
Doug Anderson37282b42011-03-04 11:54:18 -08002516 """Allow arbitrary git commands using pythonic syntax.
2517
2518 This allows you to do things like:
2519 git_obj.rev_parse('HEAD')
2520
2521 Since we don't have a 'rev_parse' method defined, the __getattr__ will
2522 run. We'll replace the '_' with a '-' and try to run a git command.
Dave Borowitz091f8932012-10-23 17:01:04 -07002523 Any other positional arguments will be passed to the git command, and the
2524 following keyword arguments are supported:
2525 config: An optional dict of git config options to be passed with '-c'.
Doug Anderson37282b42011-03-04 11:54:18 -08002526
2527 Args:
2528 name: The name of the git command to call. Any '_' characters will
2529 be replaced with '-'.
2530
2531 Returns:
2532 A callable object that will try to call git with the named command.
2533 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002534 name = name.replace('_', '-')
Dave Borowitz091f8932012-10-23 17:01:04 -07002535 def runner(*args, **kwargs):
2536 cmdv = []
2537 config = kwargs.pop('config', None)
2538 for k in kwargs:
2539 raise TypeError('%s() got an unexpected keyword argument %r'
2540 % (name, k))
2541 if config is not None:
Dave Borowitzb42b4742012-10-31 12:27:27 -07002542 if not git_require((1, 7, 2)):
2543 raise ValueError('cannot set config on command line for %s()'
2544 % name)
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302545 for k, v in config.items():
Dave Borowitz091f8932012-10-23 17:01:04 -07002546 cmdv.append('-c')
2547 cmdv.append('%s=%s' % (k, v))
2548 cmdv.append(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002549 cmdv.extend(args)
2550 p = GitCommand(self._project,
2551 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002552 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002553 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002554 capture_stdout=True,
2555 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002556 if p.Wait() != 0:
2557 raise GitError('%s %s: %s' % (
2558 self._project.name,
2559 name,
2560 p.stderr))
2561 r = p.stdout
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302562 try:
Conley Owensedd01512013-09-26 12:59:58 -07002563 r = r.decode('utf-8')
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302564 except AttributeError:
2565 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002566 if r.endswith('\n') and r.index('\n') == len(r) - 1:
2567 return r[:-1]
2568 return r
2569 return runner
2570
2571
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002572class _PriorSyncFailedError(Exception):
2573 def __str__(self):
2574 return 'prior sync failed; rebase still in progress'
2575
2576class _DirtyError(Exception):
2577 def __str__(self):
2578 return 'contains uncommitted changes'
2579
2580class _InfoMessage(object):
2581 def __init__(self, project, text):
2582 self.project = project
2583 self.text = text
2584
2585 def Print(self, syncbuf):
2586 syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
2587 syncbuf.out.nl()
2588
2589class _Failure(object):
2590 def __init__(self, project, why):
2591 self.project = project
2592 self.why = why
2593
2594 def Print(self, syncbuf):
2595 syncbuf.out.fail('error: %s/: %s',
2596 self.project.relpath,
2597 str(self.why))
2598 syncbuf.out.nl()
2599
2600class _Later(object):
2601 def __init__(self, project, action):
2602 self.project = project
2603 self.action = action
2604
2605 def Run(self, syncbuf):
2606 out = syncbuf.out
2607 out.project('project %s/', self.project.relpath)
2608 out.nl()
2609 try:
2610 self.action()
2611 out.nl()
2612 return True
David Pursehouse8a68ff92012-09-24 12:15:13 +09002613 except GitError:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002614 out.nl()
2615 return False
2616
2617class _SyncColoring(Coloring):
2618 def __init__(self, config):
2619 Coloring.__init__(self, config, 'reposync')
Anthony King7bdac712014-07-16 12:56:40 +01002620 self.project = self.printer('header', attr='bold')
2621 self.info = self.printer('info')
2622 self.fail = self.printer('fail', fg='red')
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002623
2624class SyncBuffer(object):
2625 def __init__(self, config, detach_head=False):
2626 self._messages = []
2627 self._failures = []
2628 self._later_queue1 = []
2629 self._later_queue2 = []
2630
2631 self.out = _SyncColoring(config)
2632 self.out.redirect(sys.stderr)
2633
2634 self.detach_head = detach_head
2635 self.clean = True
2636
2637 def info(self, project, fmt, *args):
2638 self._messages.append(_InfoMessage(project, fmt % args))
2639
2640 def fail(self, project, err=None):
2641 self._failures.append(_Failure(project, err))
2642 self.clean = False
2643
2644 def later1(self, project, what):
2645 self._later_queue1.append(_Later(project, what))
2646
2647 def later2(self, project, what):
2648 self._later_queue2.append(_Later(project, what))
2649
2650 def Finish(self):
2651 self._PrintMessages()
2652 self._RunLater()
2653 self._PrintMessages()
2654 return self.clean
2655
2656 def _RunLater(self):
2657 for q in ['_later_queue1', '_later_queue2']:
2658 if not self._RunQueue(q):
2659 return
2660
2661 def _RunQueue(self, queue):
2662 for m in getattr(self, queue):
2663 if not m.Run(self):
2664 self.clean = False
2665 return False
2666 setattr(self, queue, [])
2667 return True
2668
2669 def _PrintMessages(self):
2670 for m in self._messages:
2671 m.Print(self)
2672 for m in self._failures:
2673 m.Print(self)
2674
2675 self._messages = []
2676 self._failures = []
2677
2678
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002679class MetaProject(Project):
2680 """A special project housed under .repo.
2681 """
2682 def __init__(self, manifest, name, gitdir, worktree):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002683 Project.__init__(self,
Anthony King7bdac712014-07-16 12:56:40 +01002684 manifest=manifest,
2685 name=name,
2686 gitdir=gitdir,
2687 objdir=gitdir,
2688 worktree=worktree,
2689 remote=RemoteSpec('origin'),
2690 relpath='.repo/%s' % name,
2691 revisionExpr='refs/heads/master',
2692 revisionId=None,
2693 groups=None)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002694
2695 def PreSync(self):
2696 if self.Exists:
2697 cb = self.CurrentBranch
2698 if cb:
2699 base = self.GetBranch(cb).merge
2700 if base:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002701 self.revisionExpr = base
2702 self.revisionId = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002703
Anthony King7bdac712014-07-16 12:56:40 +01002704 def MetaBranchSwitch(self):
Florian Vallee5d016502012-06-07 17:19:26 +02002705 """ Prepare MetaProject for manifest branch switch
2706 """
2707
2708 # detach and delete manifest branch, allowing a new
2709 # branch to take over
Anthony King7bdac712014-07-16 12:56:40 +01002710 syncbuf = SyncBuffer(self.config, detach_head=True)
Florian Vallee5d016502012-06-07 17:19:26 +02002711 self.Sync_LocalHalf(syncbuf)
2712 syncbuf.Finish()
2713
2714 return GitCommand(self,
Torne (Richard Coles)e8f75fa2012-07-20 15:32:19 +01002715 ['update-ref', '-d', 'refs/heads/default'],
Anthony King7bdac712014-07-16 12:56:40 +01002716 capture_stdout=True,
2717 capture_stderr=True).Wait() == 0
Florian Vallee5d016502012-06-07 17:19:26 +02002718
2719
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002720 @property
Shawn O. Pearcef6906872009-04-18 10:49:00 -07002721 def LastFetch(self):
2722 try:
2723 fh = os.path.join(self.gitdir, 'FETCH_HEAD')
2724 return os.path.getmtime(fh)
2725 except OSError:
2726 return 0
2727
2728 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002729 def HasChanges(self):
2730 """Has the remote received new commits not yet checked out?
2731 """
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002732 if not self.remote or not self.revisionExpr:
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002733 return False
2734
David Pursehouse8a68ff92012-09-24 12:15:13 +09002735 all_refs = self.bare_ref.all
2736 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002737 head = self.work_git.GetHead()
2738 if head.startswith(R_HEADS):
2739 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09002740 head = all_refs[head]
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002741 except KeyError:
2742 head = None
2743
2744 if revid == head:
2745 return False
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002746 elif self._revlist(not_rev(HEAD), revid):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002747 return True
2748 return False