blob: 00e41ada4553580590d3f45be898628d3d467468 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Sarah Owenscecd1d82012-11-01 22:59:27 -070015from __future__ import print_function
Dave Borowitz137d0132015-01-02 11:12:54 -080016import contextlib
Shawn O. Pearce438ee1c2008-11-03 09:59:36 -080017import errno
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import filecmp
19import os
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070020import random
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
22import shutil
23import stat
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Julien Campergue335f5ef2013-10-16 11:02:35 +020026import tarfile
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +080027import tempfile
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070028import time
Dave Borowitz137d0132015-01-02 11:12:54 -080029import traceback
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -070030
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070031from color import Coloring
Dave Borowitzb42b4742012-10-31 12:27:27 -070032from git_command import GitCommand, git_require
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -070033from git_config import GitConfig, IsId, GetSchemeFromUrl, ID_RE
David Pursehousee15c65a2012-08-22 10:46:11 +090034from error import GitError, HookError, UploadError
Shawn O. Pearce559b8462009-03-02 12:56:08 -080035from error import ManifestInvalidRevisionError
Conley Owens75ee0572012-11-15 17:33:11 -080036from error import NoManifestException
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070037from trace import IsTrace, Trace
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070038
Shawn O. Pearced237b692009-04-17 18:49:50 -070039from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
David Pursehouse59bbb582013-05-17 10:49:33 +090041from pyversion import is_python3
42if not is_python3():
43 # pylint:disable=W0622
Chirayu Desai217ea7d2013-03-01 19:14:38 +053044 input = raw_input
David Pursehouse59bbb582013-05-17 10:49:33 +090045 # pylint:enable=W0622
Chirayu Desai217ea7d2013-03-01 19:14:38 +053046
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070047def _lwrite(path, content):
48 lock = '%s.lock' % path
49
Chirayu Desai303a82f2014-08-19 22:57:17 +053050 fd = open(lock, 'w')
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070051 try:
52 fd.write(content)
53 finally:
54 fd.close()
55
56 try:
57 os.rename(lock, path)
58 except OSError:
59 os.remove(lock)
60 raise
61
Shawn O. Pearce48244782009-04-16 08:25:57 -070062def _error(fmt, *args):
63 msg = fmt % args
Sarah Owenscecd1d82012-11-01 22:59:27 -070064 print('error: %s' % msg, file=sys.stderr)
Shawn O. Pearce48244782009-04-16 08:25:57 -070065
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070066def not_rev(r):
67 return '^' + r
68
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080069def sq(r):
70 return "'" + r.replace("'", "'\''") + "'"
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -080071
Jonathan Nieder93719792015-03-17 11:29:58 -070072_project_hook_list = None
73def _ProjectHooks():
74 """List the hooks present in the 'hooks' directory.
75
76 These hooks are project hooks and are copied to the '.git/hooks' directory
77 of all subprojects.
78
79 This function caches the list of hooks (based on the contents of the
80 'repo/hooks' directory) on the first call.
81
82 Returns:
83 A list of absolute paths to all of the files in the hooks directory.
84 """
85 global _project_hook_list
86 if _project_hook_list is None:
87 d = os.path.realpath(os.path.abspath(os.path.dirname(__file__)))
88 d = os.path.join(d, 'hooks')
89 _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
90 return _project_hook_list
91
92
Shawn O. Pearce632768b2008-10-23 11:58:52 -070093class DownloadedChange(object):
94 _commit_cache = None
95
96 def __init__(self, project, base, change_id, ps_id, commit):
97 self.project = project
98 self.base = base
99 self.change_id = change_id
100 self.ps_id = ps_id
101 self.commit = commit
102
103 @property
104 def commits(self):
105 if self._commit_cache is None:
106 self._commit_cache = self.project.bare_git.rev_list(
107 '--abbrev=8',
108 '--abbrev-commit',
109 '--pretty=oneline',
110 '--reverse',
111 '--date-order',
112 not_rev(self.base),
113 self.commit,
114 '--')
115 return self._commit_cache
116
117
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700118class ReviewableBranch(object):
119 _commit_cache = None
120
121 def __init__(self, project, branch, base):
122 self.project = project
123 self.branch = branch
124 self.base = base
125
126 @property
127 def name(self):
128 return self.branch.name
129
130 @property
131 def commits(self):
132 if self._commit_cache is None:
133 self._commit_cache = self.project.bare_git.rev_list(
134 '--abbrev=8',
135 '--abbrev-commit',
136 '--pretty=oneline',
137 '--reverse',
138 '--date-order',
139 not_rev(self.base),
140 R_HEADS + self.name,
141 '--')
142 return self._commit_cache
143
144 @property
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800145 def unabbrev_commits(self):
146 r = dict()
147 for commit in self.project.bare_git.rev_list(
148 not_rev(self.base),
149 R_HEADS + self.name,
150 '--'):
151 r[commit[0:8]] = commit
152 return r
153
154 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700155 def date(self):
156 return self.project.bare_git.log(
157 '--pretty=format:%cd',
158 '-n', '1',
159 R_HEADS + self.name,
160 '--')
161
Bryan Jacobsf609f912013-05-06 13:36:24 -0400162 def UploadForReview(self, people, auto_topic=False, draft=False, dest_branch=None):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800163 self.project.UploadForReview(self.name,
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -0700164 people,
Brian Harring435370c2012-07-28 15:37:04 -0700165 auto_topic=auto_topic,
Bryan Jacobsf609f912013-05-06 13:36:24 -0400166 draft=draft,
167 dest_branch=dest_branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700168
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700169 def GetPublishedRefs(self):
170 refs = {}
171 output = self.project.bare_git.ls_remote(
172 self.branch.remote.SshReviewUrl(self.project.UserEmail),
173 'refs/changes/*')
174 for line in output.split('\n'):
175 try:
176 (sha, ref) = line.split()
177 refs[sha] = ref
178 except ValueError:
179 pass
180
181 return refs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700182
183class StatusColoring(Coloring):
184 def __init__(self, config):
185 Coloring.__init__(self, config, 'status')
Anthony King7bdac712014-07-16 12:56:40 +0100186 self.project = self.printer('header', attr='bold')
187 self.branch = self.printer('header', attr='bold')
188 self.nobranch = self.printer('nobranch', fg='red')
189 self.important = self.printer('important', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700190
Anthony King7bdac712014-07-16 12:56:40 +0100191 self.added = self.printer('added', fg='green')
192 self.changed = self.printer('changed', fg='red')
193 self.untracked = self.printer('untracked', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700194
195
196class DiffColoring(Coloring):
197 def __init__(self, config):
198 Coloring.__init__(self, config, 'diff')
Anthony King7bdac712014-07-16 12:56:40 +0100199 self.project = self.printer('header', attr='bold')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700200
Anthony King7bdac712014-07-16 12:56:40 +0100201class _Annotation(object):
James W. Mills24c13082012-04-12 15:04:13 -0500202 def __init__(self, name, value, keep):
203 self.name = name
204 self.value = value
205 self.keep = keep
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700206
Anthony King7bdac712014-07-16 12:56:40 +0100207class _CopyFile(object):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800208 def __init__(self, src, dest, abssrc, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700209 self.src = src
210 self.dest = dest
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800211 self.abs_src = abssrc
212 self.abs_dest = absdest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700213
214 def _Copy(self):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800215 src = self.abs_src
216 dest = self.abs_dest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700217 # copy file if it does not exist or is out of date
218 if not os.path.exists(dest) or not filecmp.cmp(src, dest):
219 try:
220 # remove existing file first, since it might be read-only
221 if os.path.exists(dest):
222 os.remove(dest)
Matthew Buckett2daf6672009-07-11 09:43:47 -0400223 else:
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200224 dest_dir = os.path.dirname(dest)
225 if not os.path.isdir(dest_dir):
226 os.makedirs(dest_dir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700227 shutil.copy(src, dest)
228 # make the file read-only
229 mode = os.stat(dest)[stat.ST_MODE]
230 mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
231 os.chmod(dest, mode)
232 except IOError:
Shawn O. Pearce48244782009-04-16 08:25:57 -0700233 _error('Cannot copy file %s to %s', src, dest)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700234
Anthony King7bdac712014-07-16 12:56:40 +0100235class _LinkFile(object):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500236 def __init__(self, src, dest, abssrc, absdest):
237 self.src = src
238 self.dest = dest
239 self.abs_src = abssrc
240 self.abs_dest = absdest
241
242 def _Link(self):
243 src = self.abs_src
244 dest = self.abs_dest
245 # link file if it does not exist or is out of date
246 if not os.path.islink(dest) or os.readlink(dest) != src:
247 try:
248 # remove existing file first, since it might be read-only
249 if os.path.exists(dest):
250 os.remove(dest)
251 else:
252 dest_dir = os.path.dirname(dest)
253 if not os.path.isdir(dest_dir):
254 os.makedirs(dest_dir)
255 os.symlink(src, dest)
256 except IOError:
257 _error('Cannot link file %s to %s', src, dest)
258
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700259class RemoteSpec(object):
260 def __init__(self,
261 name,
Anthony King7bdac712014-07-16 12:56:40 +0100262 url=None,
263 review=None,
264 revision=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700265 self.name = name
266 self.url = url
267 self.review = review
Anthony King36ea2fb2014-05-06 11:54:01 +0100268 self.revision = revision
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700269
Doug Anderson37282b42011-03-04 11:54:18 -0800270class RepoHook(object):
271 """A RepoHook contains information about a script to run as a hook.
272
273 Hooks are used to run a python script before running an upload (for instance,
274 to run presubmit checks). Eventually, we may have hooks for other actions.
275
276 This shouldn't be confused with files in the 'repo/hooks' directory. Those
277 files are copied into each '.git/hooks' folder for each project. Repo-level
278 hooks are associated instead with repo actions.
279
280 Hooks are always python. When a hook is run, we will load the hook into the
281 interpreter and execute its main() function.
282 """
283 def __init__(self,
284 hook_type,
285 hooks_project,
286 topdir,
287 abort_if_user_denies=False):
288 """RepoHook constructor.
289
290 Params:
291 hook_type: A string representing the type of hook. This is also used
292 to figure out the name of the file containing the hook. For
293 example: 'pre-upload'.
294 hooks_project: The project containing the repo hooks. If you have a
295 manifest, this is manifest.repo_hooks_project. OK if this is None,
296 which will make the hook a no-op.
297 topdir: Repo's top directory (the one containing the .repo directory).
298 Scripts will run with CWD as this directory. If you have a manifest,
299 this is manifest.topdir
300 abort_if_user_denies: If True, we'll throw a HookError() if the user
301 doesn't allow us to run the hook.
302 """
303 self._hook_type = hook_type
304 self._hooks_project = hooks_project
305 self._topdir = topdir
306 self._abort_if_user_denies = abort_if_user_denies
307
308 # Store the full path to the script for convenience.
309 if self._hooks_project:
310 self._script_fullpath = os.path.join(self._hooks_project.worktree,
311 self._hook_type + '.py')
312 else:
313 self._script_fullpath = None
314
315 def _GetHash(self):
316 """Return a hash of the contents of the hooks directory.
317
318 We'll just use git to do this. This hash has the property that if anything
319 changes in the directory we will return a different has.
320
321 SECURITY CONSIDERATION:
322 This hash only represents the contents of files in the hook directory, not
323 any other files imported or called by hooks. Changes to imported files
324 can change the script behavior without affecting the hash.
325
326 Returns:
327 A string representing the hash. This will always be ASCII so that it can
328 be printed to the user easily.
329 """
330 assert self._hooks_project, "Must have hooks to calculate their hash."
331
332 # We will use the work_git object rather than just calling GetRevisionId().
333 # That gives us a hash of the latest checked in version of the files that
334 # the user will actually be executing. Specifically, GetRevisionId()
335 # doesn't appear to change even if a user checks out a different version
336 # of the hooks repo (via git checkout) nor if a user commits their own revs.
337 #
338 # NOTE: Local (non-committed) changes will not be factored into this hash.
339 # I think this is OK, since we're really only worried about warning the user
340 # about upstream changes.
341 return self._hooks_project.work_git.rev_parse('HEAD')
342
343 def _GetMustVerb(self):
344 """Return 'must' if the hook is required; 'should' if not."""
345 if self._abort_if_user_denies:
346 return 'must'
347 else:
348 return 'should'
349
350 def _CheckForHookApproval(self):
351 """Check to see whether this hook has been approved.
352
353 We'll look at the hash of all of the hooks. If this matches the hash that
354 the user last approved, we're done. If it doesn't, we'll ask the user
355 about approval.
356
357 Note that we ask permission for each individual hook even though we use
358 the hash of all hooks when detecting changes. We'd like the user to be
359 able to approve / deny each hook individually. We only use the hash of all
360 hooks because there is no other easy way to detect changes to local imports.
361
362 Returns:
363 True if this hook is approved to run; False otherwise.
364
365 Raises:
366 HookError: Raised if the user doesn't approve and abort_if_user_denies
367 was passed to the consturctor.
368 """
Doug Anderson37282b42011-03-04 11:54:18 -0800369 hooks_config = self._hooks_project.config
370 git_approval_key = 'repo.hooks.%s.approvedhash' % self._hook_type
371
372 # Get the last hash that the user approved for this hook; may be None.
373 old_hash = hooks_config.GetString(git_approval_key)
374
375 # Get the current hash so we can tell if scripts changed since approval.
376 new_hash = self._GetHash()
377
378 if old_hash is not None:
379 # User previously approved hook and asked not to be prompted again.
380 if new_hash == old_hash:
381 # Approval matched. We're done.
382 return True
383 else:
384 # Give the user a reason why we're prompting, since they last told
385 # us to "never ask again".
386 prompt = 'WARNING: Scripts have changed since %s was allowed.\n\n' % (
387 self._hook_type)
388 else:
389 prompt = ''
390
391 # Prompt the user if we're not on a tty; on a tty we'll assume "no".
392 if sys.stdout.isatty():
393 prompt += ('Repo %s run the script:\n'
394 ' %s\n'
395 '\n'
396 'Do you want to allow this script to run '
397 '(yes/yes-never-ask-again/NO)? ') % (
398 self._GetMustVerb(), self._script_fullpath)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530399 response = input(prompt).lower()
David Pursehouse98ffba12012-11-14 11:18:00 +0900400 print()
Doug Anderson37282b42011-03-04 11:54:18 -0800401
402 # User is doing a one-time approval.
403 if response in ('y', 'yes'):
404 return True
405 elif response == 'yes-never-ask-again':
406 hooks_config.SetString(git_approval_key, new_hash)
407 return True
408
409 # For anything else, we'll assume no approval.
410 if self._abort_if_user_denies:
411 raise HookError('You must allow the %s hook or use --no-verify.' %
412 self._hook_type)
413
414 return False
415
416 def _ExecuteHook(self, **kwargs):
417 """Actually execute the given hook.
418
419 This will run the hook's 'main' function in our python interpreter.
420
421 Args:
422 kwargs: Keyword arguments to pass to the hook. These are often specific
423 to the hook type. For instance, pre-upload hooks will contain
424 a project_list.
425 """
426 # Keep sys.path and CWD stashed away so that we can always restore them
427 # upon function exit.
428 orig_path = os.getcwd()
429 orig_syspath = sys.path
430
431 try:
432 # Always run hooks with CWD as topdir.
433 os.chdir(self._topdir)
434
435 # Put the hook dir as the first item of sys.path so hooks can do
436 # relative imports. We want to replace the repo dir as [0] so
437 # hooks can't import repo files.
438 sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
439
440 # Exec, storing global context in the context dict. We catch exceptions
441 # and convert to a HookError w/ just the failing traceback.
442 context = {}
443 try:
Anthony King70f68902014-05-05 21:15:34 +0100444 exec(compile(open(self._script_fullpath).read(),
445 self._script_fullpath, 'exec'), context)
Doug Anderson37282b42011-03-04 11:54:18 -0800446 except Exception:
447 raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
448 traceback.format_exc(), self._hook_type))
449
450 # Running the script should have defined a main() function.
451 if 'main' not in context:
452 raise HookError('Missing main() in: "%s"' % self._script_fullpath)
453
454
455 # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
456 # We don't actually want hooks to define their main with this argument--
457 # it's there to remind them that their hook should always take **kwargs.
458 # For instance, a pre-upload hook should be defined like:
459 # def main(project_list, **kwargs):
460 #
461 # This allows us to later expand the API without breaking old hooks.
462 kwargs = kwargs.copy()
463 kwargs['hook_should_take_kwargs'] = True
464
465 # Call the main function in the hook. If the hook should cause the
466 # build to fail, it will raise an Exception. We'll catch that convert
467 # to a HookError w/ just the failing traceback.
468 try:
469 context['main'](**kwargs)
470 except Exception:
471 raise HookError('%s\nFailed to run main() for %s hook; see traceback '
472 'above.' % (
473 traceback.format_exc(), self._hook_type))
474 finally:
475 # Restore sys.path and CWD.
476 sys.path = orig_syspath
477 os.chdir(orig_path)
478
479 def Run(self, user_allows_all_hooks, **kwargs):
480 """Run the hook.
481
482 If the hook doesn't exist (because there is no hooks project or because
483 this particular hook is not enabled), this is a no-op.
484
485 Args:
486 user_allows_all_hooks: If True, we will never prompt about running the
487 hook--we'll just assume it's OK to run it.
488 kwargs: Keyword arguments to pass to the hook. These are often specific
489 to the hook type. For instance, pre-upload hooks will contain
490 a project_list.
491
492 Raises:
493 HookError: If there was a problem finding the hook or the user declined
494 to run a required hook (from _CheckForHookApproval).
495 """
496 # No-op if there is no hooks project or if hook is disabled.
497 if ((not self._hooks_project) or
498 (self._hook_type not in self._hooks_project.enabled_repo_hooks)):
499 return
500
501 # Bail with a nice error if we can't find the hook.
502 if not os.path.isfile(self._script_fullpath):
503 raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
504
505 # Make sure the user is OK with running the hook.
506 if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
507 return
508
509 # Run the hook with the same version of python we're using.
510 self._ExecuteHook(**kwargs)
511
512
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700513class Project(object):
514 def __init__(self,
515 manifest,
516 name,
517 remote,
518 gitdir,
David James8d201162013-10-11 17:03:19 -0700519 objdir,
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700520 worktree,
521 relpath,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700522 revisionExpr,
Mike Pontillod3153822012-02-28 11:53:24 -0800523 revisionId,
Anthony King7bdac712014-07-16 12:56:40 +0100524 rebase=True,
525 groups=None,
526 sync_c=False,
527 sync_s=False,
528 clone_depth=None,
529 upstream=None,
530 parent=None,
531 is_derived=False,
David Pursehouseb1553542014-09-04 21:28:09 +0900532 dest_branch=None,
533 optimized_fetch=False):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800534 """Init a Project object.
535
536 Args:
537 manifest: The XmlManifest object.
538 name: The `name` attribute of manifest.xml's project element.
539 remote: RemoteSpec object specifying its remote's properties.
540 gitdir: Absolute path of git directory.
David James8d201162013-10-11 17:03:19 -0700541 objdir: Absolute path of directory to store git objects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800542 worktree: Absolute path of git working tree.
543 relpath: Relative path of git working tree to repo's top directory.
544 revisionExpr: The `revision` attribute of manifest.xml's project element.
545 revisionId: git commit id for checking out.
546 rebase: The `rebase` attribute of manifest.xml's project element.
547 groups: The `groups` attribute of manifest.xml's project element.
548 sync_c: The `sync-c` attribute of manifest.xml's project element.
549 sync_s: The `sync-s` attribute of manifest.xml's project element.
550 upstream: The `upstream` attribute of manifest.xml's project element.
551 parent: The parent Project object.
552 is_derived: False if the project was explicitly defined in the manifest;
553 True if the project is a discovered submodule.
Bryan Jacobsf609f912013-05-06 13:36:24 -0400554 dest_branch: The branch to which to push changes for review by default.
David Pursehouseb1553542014-09-04 21:28:09 +0900555 optimized_fetch: If True, when a project is set to a sha1 revision, only
556 fetch from the remote if the sha1 is not present locally.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800557 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700558 self.manifest = manifest
559 self.name = name
560 self.remote = remote
Anthony Newnamdf14a702011-01-09 17:31:57 -0800561 self.gitdir = gitdir.replace('\\', '/')
David James8d201162013-10-11 17:03:19 -0700562 self.objdir = objdir.replace('\\', '/')
Shawn O. Pearce0ce6ca92011-01-10 13:26:01 -0800563 if worktree:
564 self.worktree = worktree.replace('\\', '/')
565 else:
566 self.worktree = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700567 self.relpath = relpath
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700568 self.revisionExpr = revisionExpr
569
570 if revisionId is None \
571 and revisionExpr \
572 and IsId(revisionExpr):
573 self.revisionId = revisionExpr
574 else:
575 self.revisionId = revisionId
576
Mike Pontillod3153822012-02-28 11:53:24 -0800577 self.rebase = rebase
Colin Cross5acde752012-03-28 20:15:45 -0700578 self.groups = groups
Anatol Pomazau79770d22012-04-20 14:41:59 -0700579 self.sync_c = sync_c
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800580 self.sync_s = sync_s
David Pursehouseede7f122012-11-27 22:25:30 +0900581 self.clone_depth = clone_depth
Brian Harring14a66742012-09-28 20:21:57 -0700582 self.upstream = upstream
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800583 self.parent = parent
584 self.is_derived = is_derived
David Pursehouseb1553542014-09-04 21:28:09 +0900585 self.optimized_fetch = optimized_fetch
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800586 self.subprojects = []
Mike Pontillod3153822012-02-28 11:53:24 -0800587
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700588 self.snapshots = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700589 self.copyfiles = []
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500590 self.linkfiles = []
James W. Mills24c13082012-04-12 15:04:13 -0500591 self.annotations = []
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700592 self.config = GitConfig.ForRepository(
Anthony King7bdac712014-07-16 12:56:40 +0100593 gitdir=self.gitdir,
594 defaults=self.manifest.globalConfig)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700595
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800596 if self.worktree:
David James8d201162013-10-11 17:03:19 -0700597 self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800598 else:
599 self.work_git = None
David James8d201162013-10-11 17:03:19 -0700600 self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
Shawn O. Pearced237b692009-04-17 18:49:50 -0700601 self.bare_ref = GitRefs(gitdir)
David James8d201162013-10-11 17:03:19 -0700602 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
Bryan Jacobsf609f912013-05-06 13:36:24 -0400603 self.dest_branch = dest_branch
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700604
Doug Anderson37282b42011-03-04 11:54:18 -0800605 # This will be filled in if a project is later identified to be the
606 # project containing repo hooks.
607 self.enabled_repo_hooks = []
608
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700609 @property
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800610 def Derived(self):
611 return self.is_derived
612
613 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700614 def Exists(self):
615 return os.path.isdir(self.gitdir)
616
617 @property
618 def CurrentBranch(self):
619 """Obtain the name of the currently checked out branch.
620 The branch name omits the 'refs/heads/' prefix.
621 None is returned if the project is on a detached HEAD.
622 """
Shawn O. Pearce5b23f242009-04-17 18:43:33 -0700623 b = self.work_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700624 if b.startswith(R_HEADS):
625 return b[len(R_HEADS):]
626 return None
627
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700628 def IsRebaseInProgress(self):
629 w = self.worktree
630 g = os.path.join(w, '.git')
631 return os.path.exists(os.path.join(g, 'rebase-apply')) \
632 or os.path.exists(os.path.join(g, 'rebase-merge')) \
633 or os.path.exists(os.path.join(w, '.dotest'))
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200634
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700635 def IsDirty(self, consider_untracked=True):
636 """Is the working directory modified in some way?
637 """
638 self.work_git.update_index('-q',
639 '--unmerged',
640 '--ignore-missing',
641 '--refresh')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900642 if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700643 return True
644 if self.work_git.DiffZ('diff-files'):
645 return True
646 if consider_untracked and self.work_git.LsOthers():
647 return True
648 return False
649
650 _userident_name = None
651 _userident_email = None
652
653 @property
654 def UserName(self):
655 """Obtain the user's personal name.
656 """
657 if self._userident_name is None:
658 self._LoadUserIdentity()
659 return self._userident_name
660
661 @property
662 def UserEmail(self):
663 """Obtain the user's email address. This is very likely
664 to be their Gerrit login.
665 """
666 if self._userident_email is None:
667 self._LoadUserIdentity()
668 return self._userident_email
669
670 def _LoadUserIdentity(self):
David Pursehousec1b86a22012-11-14 11:36:51 +0900671 u = self.bare_git.var('GIT_COMMITTER_IDENT')
672 m = re.compile("^(.*) <([^>]*)> ").match(u)
673 if m:
674 self._userident_name = m.group(1)
675 self._userident_email = m.group(2)
676 else:
677 self._userident_name = ''
678 self._userident_email = ''
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700679
680 def GetRemote(self, name):
681 """Get the configuration for a single remote.
682 """
683 return self.config.GetRemote(name)
684
685 def GetBranch(self, name):
686 """Get the configuration for a single branch.
687 """
688 return self.config.GetBranch(name)
689
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700690 def GetBranches(self):
691 """Get all existing local branches.
692 """
693 current = self.CurrentBranch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900694 all_refs = self._allrefs
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700695 heads = {}
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700696
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530697 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700698 if name.startswith(R_HEADS):
699 name = name[len(R_HEADS):]
700 b = self.GetBranch(name)
701 b.current = name == current
702 b.published = None
David Pursehouse8a68ff92012-09-24 12:15:13 +0900703 b.revision = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700704 heads[name] = b
705
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530706 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700707 if name.startswith(R_PUB):
708 name = name[len(R_PUB):]
709 b = heads.get(name)
710 if b:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900711 b.published = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700712
713 return heads
714
Colin Cross5acde752012-03-28 20:15:45 -0700715 def MatchesGroups(self, manifest_groups):
716 """Returns true if the manifest groups specified at init should cause
717 this project to be synced.
718 Prefixing a manifest group with "-" inverts the meaning of a group.
Conley Owensbb1b5f52012-08-13 13:11:18 -0700719 All projects are implicitly labelled with "all".
Conley Owens971de8e2012-04-16 10:36:08 -0700720
721 labels are resolved in order. In the example case of
Conley Owensbb1b5f52012-08-13 13:11:18 -0700722 project_groups: "all,group1,group2"
Conley Owens971de8e2012-04-16 10:36:08 -0700723 manifest_groups: "-group1,group2"
724 the project will be matched.
David Holmer0a1c6a12012-11-14 19:19:00 -0500725
726 The special manifest group "default" will match any project that
727 does not have the special project group "notdefault"
Colin Cross5acde752012-03-28 20:15:45 -0700728 """
David Holmer0a1c6a12012-11-14 19:19:00 -0500729 expanded_manifest_groups = manifest_groups or ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700730 expanded_project_groups = ['all'] + (self.groups or [])
David Holmer0a1c6a12012-11-14 19:19:00 -0500731 if not 'notdefault' in expanded_project_groups:
732 expanded_project_groups += ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700733
Conley Owens971de8e2012-04-16 10:36:08 -0700734 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700735 for group in expanded_manifest_groups:
736 if group.startswith('-') and group[1:] in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700737 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700738 elif group in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700739 matched = True
Colin Cross5acde752012-03-28 20:15:45 -0700740
Conley Owens971de8e2012-04-16 10:36:08 -0700741 return matched
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700742
743## Status Display ##
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700744 def UncommitedFiles(self, get_all=True):
745 """Returns a list of strings, uncommitted files in the git tree.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700746
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700747 Args:
748 get_all: a boolean, if True - get information about all different
749 uncommitted files. If False - return as soon as any kind of
750 uncommitted files is detected.
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500751 """
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700752 details = []
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500753 self.work_git.update_index('-q',
754 '--unmerged',
755 '--ignore-missing',
756 '--refresh')
757 if self.IsRebaseInProgress():
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700758 details.append("rebase in progress")
759 if not get_all:
760 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500761
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700762 changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
763 if changes:
764 details.extend(changes)
765 if not get_all:
766 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500767
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700768 changes = self.work_git.DiffZ('diff-files').keys()
769 if changes:
770 details.extend(changes)
771 if not get_all:
772 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500773
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700774 changes = self.work_git.LsOthers()
775 if changes:
776 details.extend(changes)
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500777
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700778 return details
779
780 def HasChanges(self):
781 """Returns true if there are uncommitted changes.
782 """
783 if self.UncommitedFiles(get_all=False):
784 return True
785 else:
786 return False
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500787
Terence Haddock4655e812011-03-31 12:33:34 +0200788 def PrintWorkTreeStatus(self, output_redir=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700789 """Prints the status of the repository to stdout.
Terence Haddock4655e812011-03-31 12:33:34 +0200790
791 Args:
792 output: If specified, redirect the output to this object.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700793 """
794 if not os.path.isdir(self.worktree):
Terence Haddock4655e812011-03-31 12:33:34 +0200795 if output_redir == None:
796 output_redir = sys.stdout
Sarah Owenscecd1d82012-11-01 22:59:27 -0700797 print(file=output_redir)
798 print('project %s/' % self.relpath, file=output_redir)
799 print(' missing (run "repo sync")', file=output_redir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700800 return
801
802 self.work_git.update_index('-q',
803 '--unmerged',
804 '--ignore-missing',
805 '--refresh')
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700806 rb = self.IsRebaseInProgress()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700807 di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
808 df = self.work_git.DiffZ('diff-files')
809 do = self.work_git.LsOthers()
Ali Utku Selen76abcc12012-01-25 10:51:12 +0100810 if not rb and not di and not df and not do and not self.CurrentBranch:
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700811 return 'CLEAN'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700812
813 out = StatusColoring(self.config)
Terence Haddock4655e812011-03-31 12:33:34 +0200814 if not output_redir == None:
815 out.redirect(output_redir)
Jakub Vrana0402cd82014-09-09 15:39:15 -0700816 out.project('project %-40s', self.relpath + '/ ')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700817
818 branch = self.CurrentBranch
819 if branch is None:
820 out.nobranch('(*** NO BRANCH ***)')
821 else:
822 out.branch('branch %s', branch)
823 out.nl()
824
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700825 if rb:
826 out.important('prior sync failed; rebase still in progress')
827 out.nl()
828
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700829 paths = list()
830 paths.extend(di.keys())
831 paths.extend(df.keys())
832 paths.extend(do)
833
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530834 for p in sorted(set(paths)):
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900835 try:
836 i = di[p]
837 except KeyError:
838 i = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700839
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900840 try:
841 f = df[p]
842 except KeyError:
843 f = None
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200844
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900845 if i:
846 i_status = i.status.upper()
847 else:
848 i_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700849
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900850 if f:
851 f_status = f.status.lower()
852 else:
853 f_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700854
855 if i and i.src_path:
Shawn O. Pearcefe086752009-03-03 13:49:48 -0800856 line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700857 i.src_path, p, i.level)
858 else:
859 line = ' %s%s\t%s' % (i_status, f_status, p)
860
861 if i and not f:
862 out.added('%s', line)
863 elif (i and f) or (not i and f):
864 out.changed('%s', line)
865 elif not i and not f:
866 out.untracked('%s', line)
867 else:
868 out.write('%s', line)
869 out.nl()
Terence Haddock4655e812011-03-31 12:33:34 +0200870
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700871 return 'DIRTY'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700872
pelyad67872d2012-03-28 14:49:58 +0300873 def PrintWorkTreeDiff(self, absolute_paths=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700874 """Prints the status of the repository to stdout.
875 """
876 out = DiffColoring(self.config)
877 cmd = ['diff']
878 if out.is_on:
879 cmd.append('--color')
880 cmd.append(HEAD)
pelyad67872d2012-03-28 14:49:58 +0300881 if absolute_paths:
882 cmd.append('--src-prefix=a/%s/' % self.relpath)
883 cmd.append('--dst-prefix=b/%s/' % self.relpath)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700884 cmd.append('--')
885 p = GitCommand(self,
886 cmd,
Anthony King7bdac712014-07-16 12:56:40 +0100887 capture_stdout=True,
888 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700889 has_diff = False
890 for line in p.process.stdout:
891 if not has_diff:
892 out.nl()
893 out.project('project %s/' % self.relpath)
894 out.nl()
895 has_diff = True
Sarah Owenscecd1d82012-11-01 22:59:27 -0700896 print(line[:-1])
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700897 p.Wait()
898
899
900## Publish / Upload ##
901
David Pursehouse8a68ff92012-09-24 12:15:13 +0900902 def WasPublished(self, branch, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700903 """Was the branch published (uploaded) for code review?
904 If so, returns the SHA-1 hash of the last published
905 state for the branch.
906 """
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700907 key = R_PUB + branch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900908 if all_refs is None:
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700909 try:
910 return self.bare_git.rev_parse(key)
911 except GitError:
912 return None
913 else:
914 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900915 return all_refs[key]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -0700916 except KeyError:
917 return None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700918
David Pursehouse8a68ff92012-09-24 12:15:13 +0900919 def CleanPublishedCache(self, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700920 """Prunes any stale published refs.
921 """
David Pursehouse8a68ff92012-09-24 12:15:13 +0900922 if all_refs is None:
923 all_refs = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700924 heads = set()
925 canrm = {}
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530926 for name, ref_id in all_refs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700927 if name.startswith(R_HEADS):
928 heads.add(name)
929 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900930 canrm[name] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700931
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530932 for name, ref_id in canrm.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700933 n = name[len(R_PUB):]
934 if R_HEADS + n not in heads:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900935 self.bare_git.DeleteRef(name, ref_id)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700936
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -0700937 def GetUploadableBranches(self, selected_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700938 """List any branches which can be uploaded for review.
939 """
940 heads = {}
941 pubed = {}
942
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530943 for name, ref_id in self._allrefs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700944 if name.startswith(R_HEADS):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900945 heads[name[len(R_HEADS):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700946 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900947 pubed[name[len(R_PUB):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700948
949 ready = []
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530950 for branch, ref_id in heads.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +0900951 if branch in pubed and pubed[branch] == ref_id:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700952 continue
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -0700953 if selected_branch and branch != selected_branch:
954 continue
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700955
Shawn O. Pearce35f25962008-11-11 17:03:13 -0800956 rb = self.GetUploadableBranch(branch)
957 if rb:
958 ready.append(rb)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700959 return ready
960
Shawn O. Pearce35f25962008-11-11 17:03:13 -0800961 def GetUploadableBranch(self, branch_name):
962 """Get a single uploadable branch, or None.
963 """
964 branch = self.GetBranch(branch_name)
965 base = branch.LocalMerge
966 if branch.LocalMerge:
967 rb = ReviewableBranch(self, branch, base)
968 if rb.commits:
969 return rb
970 return None
971
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -0700972 def UploadForReview(self, branch=None,
Anthony King7bdac712014-07-16 12:56:40 +0100973 people=([], []),
Brian Harring435370c2012-07-28 15:37:04 -0700974 auto_topic=False,
Bryan Jacobsf609f912013-05-06 13:36:24 -0400975 draft=False,
976 dest_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700977 """Uploads the named branch for code review.
978 """
979 if branch is None:
980 branch = self.CurrentBranch
981 if branch is None:
982 raise GitError('not currently on a branch')
983
984 branch = self.GetBranch(branch)
985 if not branch.LocalMerge:
986 raise GitError('branch %s does not track a remote' % branch.name)
987 if not branch.remote.review:
988 raise GitError('remote %s has no review url' % branch.remote.name)
989
Bryan Jacobsf609f912013-05-06 13:36:24 -0400990 if dest_branch is None:
991 dest_branch = self.dest_branch
992 if dest_branch is None:
993 dest_branch = branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700994 if not dest_branch.startswith(R_HEADS):
995 dest_branch = R_HEADS + dest_branch
996
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800997 if not branch.remote.projectname:
998 branch.remote.projectname = self.name
999 branch.remote.Save()
1000
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001001 url = branch.remote.ReviewUrl(self.UserEmail)
1002 if url is None:
1003 raise UploadError('review not configured')
1004 cmd = ['push']
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001005
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001006 if url.startswith('ssh://'):
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001007 rp = ['gerrit receive-pack']
1008 for e in people[0]:
1009 rp.append('--reviewer=%s' % sq(e))
1010 for e in people[1]:
1011 rp.append('--cc=%s' % sq(e))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001012 cmd.append('--receive-pack=%s' % " ".join(rp))
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001013
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001014 cmd.append(url)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001015
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001016 if dest_branch.startswith(R_HEADS):
1017 dest_branch = dest_branch[len(R_HEADS):]
Brian Harring435370c2012-07-28 15:37:04 -07001018
1019 upload_type = 'for'
1020 if draft:
1021 upload_type = 'drafts'
1022
1023 ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
1024 dest_branch)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001025 if auto_topic:
1026 ref_spec = ref_spec + '/' + branch.name
Shawn Pearce45d21682013-02-28 00:35:51 -08001027 if not url.startswith('ssh://'):
1028 rp = ['r=%s' % p for p in people[0]] + \
1029 ['cc=%s' % p for p in people[1]]
1030 if rp:
1031 ref_spec = ref_spec + '%' + ','.join(rp)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001032 cmd.append(ref_spec)
1033
Anthony King7bdac712014-07-16 12:56:40 +01001034 if GitCommand(self, cmd, bare=True).Wait() != 0:
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001035 raise UploadError('Upload failed')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001036
1037 msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
1038 self.bare_git.UpdateRef(R_PUB + branch.name,
1039 R_HEADS + branch.name,
Anthony King7bdac712014-07-16 12:56:40 +01001040 message=msg)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001041
1042
1043## Sync ##
1044
Julien Campergue335f5ef2013-10-16 11:02:35 +02001045 def _ExtractArchive(self, tarpath, path=None):
1046 """Extract the given tar on its current location
1047
1048 Args:
1049 - tarpath: The path to the actual tar file
1050
1051 """
1052 try:
1053 with tarfile.open(tarpath, 'r') as tar:
1054 tar.extractall(path=path)
1055 return True
1056 except (IOError, tarfile.TarError) as e:
1057 print("error: Cannot extract archive %s: "
1058 "%s" % (tarpath, str(e)), file=sys.stderr)
1059 return False
1060
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001061 def Sync_NetworkHalf(self,
1062 quiet=False,
1063 is_new=None,
1064 current_branch_only=False,
Mitchel Humpherys597868b2012-10-29 10:18:34 -07001065 clone_bundle=True,
Julien Campergue335f5ef2013-10-16 11:02:35 +02001066 no_tags=False,
David Pursehouseb1553542014-09-04 21:28:09 +09001067 archive=False,
1068 optimized_fetch=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001069 """Perform only the network IO portion of the sync process.
1070 Local working directory/branch state is not affected.
1071 """
Julien Campergue335f5ef2013-10-16 11:02:35 +02001072 if archive and not isinstance(self, MetaProject):
1073 if self.remote.url.startswith(('http://', 'https://')):
1074 print("error: %s: Cannot fetch archives from http/https "
1075 "remotes." % self.name, file=sys.stderr)
1076 return False
1077
1078 name = self.relpath.replace('\\', '/')
1079 name = name.replace('/', '_')
1080 tarpath = '%s.tar' % name
1081 topdir = self.manifest.topdir
1082
1083 try:
1084 self._FetchArchive(tarpath, cwd=topdir)
1085 except GitError as e:
1086 print('error: %s' % str(e), file=sys.stderr)
1087 return False
1088
1089 # From now on, we only need absolute tarpath
1090 tarpath = os.path.join(topdir, tarpath)
1091
1092 if not self._ExtractArchive(tarpath, path=topdir):
1093 return False
1094 try:
1095 os.remove(tarpath)
1096 except OSError as e:
1097 print("warn: Cannot remove archive %s: "
1098 "%s" % (tarpath, str(e)), file=sys.stderr)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001099 self._CopyAndLinkFiles()
Julien Campergue335f5ef2013-10-16 11:02:35 +02001100 return True
1101
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001102 if is_new is None:
1103 is_new = not self.Exists
Shawn O. Pearce88443382010-10-08 10:02:09 +02001104 if is_new:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001105 self._InitGitDir()
Jimmie Westera0444582012-10-24 13:44:42 +02001106 else:
1107 self._UpdateHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001108 self._InitRemote()
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001109
1110 if is_new:
1111 alt = os.path.join(self.gitdir, 'objects/info/alternates')
1112 try:
1113 fd = open(alt, 'rb')
1114 try:
1115 alt_dir = fd.readline().rstrip()
1116 finally:
1117 fd.close()
1118 except IOError:
1119 alt_dir = None
1120 else:
1121 alt_dir = None
1122
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001123 if clone_bundle \
1124 and alt_dir is None \
1125 and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001126 is_new = False
1127
Shawn O. Pearce6ba6ba02012-05-24 09:46:50 -07001128 if not current_branch_only:
1129 if self.sync_c:
1130 current_branch_only = True
1131 elif not self.manifest._loaded:
1132 # Manifest cannot check defaults until it syncs.
1133 current_branch_only = False
1134 elif self.manifest.default.sync_c:
1135 current_branch_only = True
1136
David Pursehouseb1553542014-09-04 21:28:09 +09001137 need_to_fetch = not (optimized_fetch and \
1138 (ID_RE.match(self.revisionExpr) and self._CheckForSha1()))
1139 if (need_to_fetch
Conley Owens666d5342014-05-01 13:09:57 -07001140 and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
1141 current_branch_only=current_branch_only,
1142 no_tags=no_tags)):
Anthony King7bdac712014-07-16 12:56:40 +01001143 return False
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001144
1145 if self.worktree:
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001146 self._InitMRef()
1147 else:
1148 self._InitMirrorHead()
1149 try:
1150 os.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
1151 except OSError:
1152 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001153 return True
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08001154
1155 def PostRepoUpgrade(self):
1156 self._InitHooks()
1157
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001158 def _CopyAndLinkFiles(self):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001159 for copyfile in self.copyfiles:
1160 copyfile._Copy()
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001161 for linkfile in self.linkfiles:
1162 linkfile._Link()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001163
Julien Camperguedd654222014-01-09 16:21:37 +01001164 def GetCommitRevisionId(self):
1165 """Get revisionId of a commit.
1166
1167 Use this method instead of GetRevisionId to get the id of the commit rather
1168 than the id of the current git object (for example, a tag)
1169
1170 """
1171 if not self.revisionExpr.startswith(R_TAGS):
1172 return self.GetRevisionId(self._allrefs)
1173
1174 try:
1175 return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
1176 except GitError:
1177 raise ManifestInvalidRevisionError(
1178 'revision %s in %s not found' % (self.revisionExpr,
1179 self.name))
1180
David Pursehouse8a68ff92012-09-24 12:15:13 +09001181 def GetRevisionId(self, all_refs=None):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001182 if self.revisionId:
1183 return self.revisionId
1184
1185 rem = self.GetRemote(self.remote.name)
1186 rev = rem.ToLocal(self.revisionExpr)
1187
David Pursehouse8a68ff92012-09-24 12:15:13 +09001188 if all_refs is not None and rev in all_refs:
1189 return all_refs[rev]
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001190
1191 try:
1192 return self.bare_git.rev_parse('--verify', '%s^0' % rev)
1193 except GitError:
1194 raise ManifestInvalidRevisionError(
1195 'revision %s in %s not found' % (self.revisionExpr,
1196 self.name))
1197
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001198 def Sync_LocalHalf(self, syncbuf):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001199 """Perform only the local IO portion of the sync process.
1200 Network access is not required.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001201 """
David James8d201162013-10-11 17:03:19 -07001202 self._InitWorkTree()
David Pursehouse8a68ff92012-09-24 12:15:13 +09001203 all_refs = self.bare_ref.all
1204 self.CleanPublishedCache(all_refs)
1205 revid = self.GetRevisionId(all_refs)
Skyler Kaufman835cd682011-03-08 12:14:41 -08001206
David Pursehouse1d947b32012-10-25 12:23:11 +09001207 def _doff():
1208 self._FastForward(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001209 self._CopyAndLinkFiles()
David Pursehouse1d947b32012-10-25 12:23:11 +09001210
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001211 head = self.work_git.GetHead()
1212 if head.startswith(R_HEADS):
1213 branch = head[len(R_HEADS):]
1214 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001215 head = all_refs[head]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001216 except KeyError:
1217 head = None
1218 else:
1219 branch = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001220
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001221 if branch is None or syncbuf.detach_head:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001222 # Currently on a detached HEAD. The user is assumed to
1223 # not have any local modifications worth worrying about.
1224 #
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -07001225 if self.IsRebaseInProgress():
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001226 syncbuf.fail(self, _PriorSyncFailedError())
1227 return
1228
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001229 if head == revid:
1230 # No changes; don't do anything further.
Florian Vallee7cf1b362012-06-07 17:11:42 +02001231 # Except if the head needs to be detached
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001232 #
Florian Vallee7cf1b362012-06-07 17:11:42 +02001233 if not syncbuf.detach_head:
1234 return
1235 else:
1236 lost = self._revlist(not_rev(revid), HEAD)
1237 if lost:
1238 syncbuf.info(self, "discarding %d commits", len(lost))
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001239
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001240 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001241 self._Checkout(revid, quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001242 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001243 syncbuf.fail(self, e)
1244 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001245 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001246 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001247
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001248 if head == revid:
1249 # No changes; don't do anything further.
1250 #
1251 return
1252
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001253 branch = self.GetBranch(branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001254
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001255 if not branch.LocalMerge:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001256 # The current branch has no tracking configuration.
Anatol Pomazau2a32f6a2011-08-30 10:52:33 -07001257 # Jump off it to a detached HEAD.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001258 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001259 syncbuf.info(self,
1260 "leaving %s; does not track upstream",
1261 branch.name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001262 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001263 self._Checkout(revid, quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001264 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001265 syncbuf.fail(self, e)
1266 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001267 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001268 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001269
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001270 upstream_gain = self._revlist(not_rev(HEAD), revid)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001271 pub = self.WasPublished(branch.name, all_refs)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001272 if pub:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001273 not_merged = self._revlist(not_rev(revid), pub)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001274 if not_merged:
1275 if upstream_gain:
1276 # The user has published this branch and some of those
1277 # commits are not yet merged upstream. We do not want
1278 # to rewrite the published commits so we punt.
1279 #
Daniel Sandler4c50dee2010-03-02 15:38:03 -05001280 syncbuf.fail(self,
1281 "branch %s is published (but not merged) and is now %d commits behind"
1282 % (branch.name, len(upstream_gain)))
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001283 return
Shawn O. Pearce05f66b62009-04-21 08:26:32 -07001284 elif pub == head:
1285 # All published commits are merged, and thus we are a
1286 # strict subset. We can fast-forward safely.
Shawn O. Pearcea54c5272008-10-30 11:03:00 -07001287 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001288 syncbuf.later1(self, _doff)
1289 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001290
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001291 # Examine the local commits not in the remote. Find the
1292 # last one attributed to this user, if any.
1293 #
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001294 local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001295 last_mine = None
1296 cnt_mine = 0
1297 for commit in local_changes:
Chirayu Desai0eb35cb2013-11-19 18:46:29 +05301298 commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001299 if committer_email == self.UserEmail:
1300 last_mine = commit_id
1301 cnt_mine += 1
1302
Shawn O. Pearceda88ff42009-06-03 11:09:12 -07001303 if not upstream_gain and cnt_mine == len(local_changes):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001304 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001305
1306 if self.IsDirty(consider_untracked=False):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001307 syncbuf.fail(self, _DirtyError())
1308 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001309
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001310 # If the upstream switched on us, warn the user.
1311 #
1312 if branch.merge != self.revisionExpr:
1313 if branch.merge and self.revisionExpr:
1314 syncbuf.info(self,
1315 'manifest switched %s...%s',
1316 branch.merge,
1317 self.revisionExpr)
1318 elif branch.merge:
1319 syncbuf.info(self,
1320 'manifest no longer tracks %s',
1321 branch.merge)
1322
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001323 if cnt_mine < len(local_changes):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001324 # Upstream rebased. Not everything in HEAD
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001325 # was created by this user.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001326 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001327 syncbuf.info(self,
1328 "discarding %d commits removed from upstream",
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001329 len(local_changes) - cnt_mine)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001330
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001331 branch.remote = self.GetRemote(self.remote.name)
Anatol Pomazaucd7c5de2012-03-20 13:45:00 -07001332 if not ID_RE.match(self.revisionExpr):
1333 # in case of manifest sync the revisionExpr might be a SHA1
1334 branch.merge = self.revisionExpr
Conley Owens04f2f0e2014-10-01 17:22:46 -07001335 if not branch.merge.startswith('refs/'):
1336 branch.merge = R_HEADS + branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001337 branch.Save()
1338
Mike Pontillod3153822012-02-28 11:53:24 -08001339 if cnt_mine > 0 and self.rebase:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001340 def _dorebase():
Anthony King7bdac712014-07-16 12:56:40 +01001341 self._Rebase(upstream='%s^1' % last_mine, onto=revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001342 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001343 syncbuf.later2(self, _dorebase)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001344 elif local_changes:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001345 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001346 self._ResetHard(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001347 self._CopyAndLinkFiles()
Sarah Owensa5be53f2012-09-09 15:37:57 -07001348 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001349 syncbuf.fail(self, e)
1350 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001351 else:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001352 syncbuf.later1(self, _doff)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001353
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001354 def AddCopyFile(self, src, dest, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001355 # dest should already be an absolute path, but src is project relative
1356 # make src an absolute path
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001357 abssrc = os.path.join(self.worktree, src)
1358 self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001359
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001360 def AddLinkFile(self, src, dest, absdest):
1361 # dest should already be an absolute path, but src is project relative
1362 # make src an absolute path
1363 abssrc = os.path.join(self.worktree, src)
1364 self.linkfiles.append(_LinkFile(src, dest, abssrc, absdest))
1365
James W. Mills24c13082012-04-12 15:04:13 -05001366 def AddAnnotation(self, name, value, keep):
1367 self.annotations.append(_Annotation(name, value, keep))
1368
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001369 def DownloadPatchSet(self, change_id, patch_id):
1370 """Download a single patch set of a single change to FETCH_HEAD.
1371 """
1372 remote = self.GetRemote(self.remote.name)
1373
1374 cmd = ['fetch', remote.name]
1375 cmd.append('refs/changes/%2.2d/%d/%d' \
1376 % (change_id % 100, change_id, patch_id))
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001377 if GitCommand(self, cmd, bare=True).Wait() != 0:
1378 return None
1379 return DownloadedChange(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001380 self.GetRevisionId(),
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001381 change_id,
1382 patch_id,
1383 self.bare_git.rev_parse('FETCH_HEAD'))
1384
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001385
1386## Branch Management ##
1387
1388 def StartBranch(self, name):
1389 """Create a new branch off the manifest's revision.
1390 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001391 head = self.work_git.GetHead()
1392 if head == (R_HEADS + name):
1393 return True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001394
David Pursehouse8a68ff92012-09-24 12:15:13 +09001395 all_refs = self.bare_ref.all
Anthony King7bdac712014-07-16 12:56:40 +01001396 if R_HEADS + name in all_refs:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001397 return GitCommand(self,
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001398 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001399 capture_stdout=True,
1400 capture_stderr=True).Wait() == 0
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001401
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001402 branch = self.GetBranch(name)
1403 branch.remote = self.GetRemote(self.remote.name)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001404 branch.merge = self.revisionExpr
Alexandre Boeglin38258272015-04-30 14:50:33 +02001405 if not branch.merge.startswith('refs/') and not ID_RE.match(self.revisionExpr):
Conley Owens04f2f0e2014-10-01 17:22:46 -07001406 branch.merge = R_HEADS + self.revisionExpr
David Pursehouse8a68ff92012-09-24 12:15:13 +09001407 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001408
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001409 if head.startswith(R_HEADS):
1410 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001411 head = all_refs[head]
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001412 except KeyError:
1413 head = None
1414
1415 if revid and head and revid == head:
1416 ref = os.path.join(self.gitdir, R_HEADS + name)
1417 try:
1418 os.makedirs(os.path.dirname(ref))
1419 except OSError:
1420 pass
1421 _lwrite(ref, '%s\n' % revid)
1422 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1423 'ref: %s%s\n' % (R_HEADS, name))
1424 branch.Save()
1425 return True
1426
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001427 if GitCommand(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001428 ['checkout', '-b', branch.name, revid],
Anthony King7bdac712014-07-16 12:56:40 +01001429 capture_stdout=True,
1430 capture_stderr=True).Wait() == 0:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001431 branch.Save()
1432 return True
1433 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001434
Wink Saville02d79452009-04-10 13:01:24 -07001435 def CheckoutBranch(self, name):
1436 """Checkout a local topic branch.
Doug Anderson3ba5f952011-04-07 12:51:04 -07001437
1438 Args:
1439 name: The name of the branch to checkout.
1440
1441 Returns:
1442 True if the checkout succeeded; False if it didn't; None if the branch
1443 didn't exist.
Wink Saville02d79452009-04-10 13:01:24 -07001444 """
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001445 rev = R_HEADS + name
1446 head = self.work_git.GetHead()
1447 if head == rev:
1448 # Already on the branch
1449 #
1450 return True
Wink Saville02d79452009-04-10 13:01:24 -07001451
David Pursehouse8a68ff92012-09-24 12:15:13 +09001452 all_refs = self.bare_ref.all
Wink Saville02d79452009-04-10 13:01:24 -07001453 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001454 revid = all_refs[rev]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001455 except KeyError:
1456 # Branch does not exist in this project
1457 #
Doug Anderson3ba5f952011-04-07 12:51:04 -07001458 return None
Wink Saville02d79452009-04-10 13:01:24 -07001459
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001460 if head.startswith(R_HEADS):
1461 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001462 head = all_refs[head]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001463 except KeyError:
1464 head = None
1465
1466 if head == revid:
1467 # Same revision; just update HEAD to point to the new
1468 # target branch, but otherwise take no other action.
1469 #
1470 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1471 'ref: %s%s\n' % (R_HEADS, name))
1472 return True
1473
1474 return GitCommand(self,
1475 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001476 capture_stdout=True,
1477 capture_stderr=True).Wait() == 0
Wink Saville02d79452009-04-10 13:01:24 -07001478
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001479 def AbandonBranch(self, name):
1480 """Destroy a local topic branch.
Doug Andersondafb1d62011-04-07 11:46:59 -07001481
1482 Args:
1483 name: The name of the branch to abandon.
1484
1485 Returns:
1486 True if the abandon succeeded; False if it didn't; None if the branch
1487 didn't exist.
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001488 """
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001489 rev = R_HEADS + name
David Pursehouse8a68ff92012-09-24 12:15:13 +09001490 all_refs = self.bare_ref.all
1491 if rev not in all_refs:
Doug Andersondafb1d62011-04-07 11:46:59 -07001492 # Doesn't exist
1493 return None
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001494
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001495 head = self.work_git.GetHead()
1496 if head == rev:
1497 # We can't destroy the branch while we are sitting
1498 # on it. Switch to a detached HEAD.
1499 #
David Pursehouse8a68ff92012-09-24 12:15:13 +09001500 head = all_refs[head]
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001501
David Pursehouse8a68ff92012-09-24 12:15:13 +09001502 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001503 if head == revid:
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001504 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1505 '%s\n' % revid)
1506 else:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001507 self._Checkout(revid, quiet=True)
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001508
1509 return GitCommand(self,
1510 ['branch', '-D', name],
Anthony King7bdac712014-07-16 12:56:40 +01001511 capture_stdout=True,
1512 capture_stderr=True).Wait() == 0
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001513
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001514 def PruneHeads(self):
1515 """Prune any topic branches already merged into upstream.
1516 """
1517 cb = self.CurrentBranch
1518 kill = []
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001519 left = self._allrefs
1520 for name in left.keys():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001521 if name.startswith(R_HEADS):
1522 name = name[len(R_HEADS):]
1523 if cb is None or name != cb:
1524 kill.append(name)
1525
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001526 rev = self.GetRevisionId(left)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001527 if cb is not None \
1528 and not self._revlist(HEAD + '...' + rev) \
Anthony King7bdac712014-07-16 12:56:40 +01001529 and not self.IsDirty(consider_untracked=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001530 self.work_git.DetachHead(HEAD)
1531 kill.append(cb)
1532
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001533 if kill:
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07001534 old = self.bare_git.GetHead()
1535 if old is None:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001536 old = 'refs/heads/please_never_use_this_as_a_branch_name'
1537
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001538 try:
1539 self.bare_git.DetachHead(rev)
1540
1541 b = ['branch', '-d']
1542 b.extend(kill)
1543 b = GitCommand(self, b, bare=True,
1544 capture_stdout=True,
1545 capture_stderr=True)
1546 b.Wait()
1547 finally:
1548 self.bare_git.SetHead(old)
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001549 left = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001550
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001551 for branch in kill:
1552 if (R_HEADS + branch) not in left:
1553 self.CleanPublishedCache()
1554 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001555
1556 if cb and cb not in kill:
1557 kill.append(cb)
Shawn O. Pearce7c6c64d2009-03-02 12:38:13 -08001558 kill.sort()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001559
1560 kept = []
1561 for branch in kill:
Anthony King7bdac712014-07-16 12:56:40 +01001562 if R_HEADS + branch in left:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001563 branch = self.GetBranch(branch)
1564 base = branch.LocalMerge
1565 if not base:
1566 base = rev
1567 kept.append(ReviewableBranch(self, branch, base))
1568 return kept
1569
1570
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001571## Submodule Management ##
1572
1573 def GetRegisteredSubprojects(self):
1574 result = []
1575 def rec(subprojects):
1576 if not subprojects:
1577 return
1578 result.extend(subprojects)
1579 for p in subprojects:
1580 rec(p.subprojects)
1581 rec(self.subprojects)
1582 return result
1583
1584 def _GetSubmodules(self):
1585 # Unfortunately we cannot call `git submodule status --recursive` here
1586 # because the working tree might not exist yet, and it cannot be used
1587 # without a working tree in its current implementation.
1588
1589 def get_submodules(gitdir, rev):
1590 # Parse .gitmodules for submodule sub_paths and sub_urls
1591 sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
1592 if not sub_paths:
1593 return []
1594 # Run `git ls-tree` to read SHAs of submodule object, which happen to be
1595 # revision of submodule repository
1596 sub_revs = git_ls_tree(gitdir, rev, sub_paths)
1597 submodules = []
1598 for sub_path, sub_url in zip(sub_paths, sub_urls):
1599 try:
1600 sub_rev = sub_revs[sub_path]
1601 except KeyError:
1602 # Ignore non-exist submodules
1603 continue
1604 submodules.append((sub_rev, sub_path, sub_url))
1605 return submodules
1606
1607 re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
1608 re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
1609 def parse_gitmodules(gitdir, rev):
1610 cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
1611 try:
Anthony King7bdac712014-07-16 12:56:40 +01001612 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1613 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001614 except GitError:
1615 return [], []
1616 if p.Wait() != 0:
1617 return [], []
1618
1619 gitmodules_lines = []
1620 fd, temp_gitmodules_path = tempfile.mkstemp()
1621 try:
1622 os.write(fd, p.stdout)
1623 os.close(fd)
1624 cmd = ['config', '--file', temp_gitmodules_path, '--list']
Anthony King7bdac712014-07-16 12:56:40 +01001625 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1626 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001627 if p.Wait() != 0:
1628 return [], []
1629 gitmodules_lines = p.stdout.split('\n')
1630 except GitError:
1631 return [], []
1632 finally:
1633 os.remove(temp_gitmodules_path)
1634
1635 names = set()
1636 paths = {}
1637 urls = {}
1638 for line in gitmodules_lines:
1639 if not line:
1640 continue
1641 m = re_path.match(line)
1642 if m:
1643 names.add(m.group(1))
1644 paths[m.group(1)] = m.group(2)
1645 continue
1646 m = re_url.match(line)
1647 if m:
1648 names.add(m.group(1))
1649 urls[m.group(1)] = m.group(2)
1650 continue
1651 names = sorted(names)
1652 return ([paths.get(name, '') for name in names],
1653 [urls.get(name, '') for name in names])
1654
1655 def git_ls_tree(gitdir, rev, paths):
1656 cmd = ['ls-tree', rev, '--']
1657 cmd.extend(paths)
1658 try:
Anthony King7bdac712014-07-16 12:56:40 +01001659 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1660 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001661 except GitError:
1662 return []
1663 if p.Wait() != 0:
1664 return []
1665 objects = {}
1666 for line in p.stdout.split('\n'):
1667 if not line.strip():
1668 continue
1669 object_rev, object_path = line.split()[2:4]
1670 objects[object_path] = object_rev
1671 return objects
1672
1673 try:
1674 rev = self.GetRevisionId()
1675 except GitError:
1676 return []
1677 return get_submodules(self.gitdir, rev)
1678
1679 def GetDerivedSubprojects(self):
1680 result = []
1681 if not self.Exists:
1682 # If git repo does not exist yet, querying its submodules will
1683 # mess up its states; so return here.
1684 return result
1685 for rev, path, url in self._GetSubmodules():
1686 name = self.manifest.GetSubprojectName(self, path)
David James8d201162013-10-11 17:03:19 -07001687 relpath, worktree, gitdir, objdir = \
1688 self.manifest.GetSubprojectPaths(self, name, path)
1689 project = self.manifest.paths.get(relpath)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001690 if project:
1691 result.extend(project.GetDerivedSubprojects())
1692 continue
David James8d201162013-10-11 17:03:19 -07001693
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001694 remote = RemoteSpec(self.remote.name,
Anthony King7bdac712014-07-16 12:56:40 +01001695 url=url,
1696 review=self.remote.review,
1697 revision=self.remote.revision)
1698 subproject = Project(manifest=self.manifest,
1699 name=name,
1700 remote=remote,
1701 gitdir=gitdir,
1702 objdir=objdir,
1703 worktree=worktree,
1704 relpath=relpath,
1705 revisionExpr=self.revisionExpr,
1706 revisionId=rev,
1707 rebase=self.rebase,
1708 groups=self.groups,
1709 sync_c=self.sync_c,
1710 sync_s=self.sync_s,
1711 parent=self,
1712 is_derived=True)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001713 result.append(subproject)
1714 result.extend(subproject.GetDerivedSubprojects())
1715 return result
1716
1717
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001718## Direct Git Commands ##
Chris AtLee2fb64662014-01-16 21:32:33 -05001719 def _CheckForSha1(self):
1720 try:
1721 # if revision (sha or tag) is not present then following function
1722 # throws an error.
1723 self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
1724 return True
1725 except GitError:
1726 # There is no such persistent revision. We have to fetch it.
1727 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001728
Julien Campergue335f5ef2013-10-16 11:02:35 +02001729 def _FetchArchive(self, tarpath, cwd=None):
1730 cmd = ['archive', '-v', '-o', tarpath]
1731 cmd.append('--remote=%s' % self.remote.url)
1732 cmd.append('--prefix=%s/' % self.relpath)
1733 cmd.append(self.revisionExpr)
1734
1735 command = GitCommand(self, cmd, cwd=cwd,
1736 capture_stdout=True,
1737 capture_stderr=True)
1738
1739 if command.Wait() != 0:
1740 raise GitError('git archive %s: %s' % (self.name, command.stderr))
1741
Conley Owens80b87fe2014-05-09 17:13:44 -07001742
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001743 def _RemoteFetch(self, name=None,
1744 current_branch_only=False,
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001745 initial=False,
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001746 quiet=False,
Mitchel Humpherys597868b2012-10-29 10:18:34 -07001747 alt_dir=None,
1748 no_tags=False):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001749
1750 is_sha1 = False
1751 tag_name = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001752 depth = None
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001753
David Pursehouse9bc422f2014-04-15 10:28:56 +09001754 # The depth should not be used when fetching to a mirror because
1755 # it will result in a shallow repository that cannot be cloned or
1756 # fetched from.
1757 if not self.manifest.IsMirror:
1758 if self.clone_depth:
1759 depth = self.clone_depth
1760 else:
1761 depth = self.manifest.manifestProject.config.GetString('repo.depth')
Conley Owense4978cf2015-02-03 18:06:16 -08001762 # The repo project should never be synced with partial depth
1763 if self.relpath == '.repo/repo':
1764 depth = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001765
Shawn Pearce69e04d82014-01-29 12:48:54 -08001766 if depth:
1767 current_branch_only = True
1768
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001769 if ID_RE.match(self.revisionExpr) is not None:
1770 is_sha1 = True
1771
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001772 if current_branch_only:
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001773 if self.revisionExpr.startswith(R_TAGS):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001774 # this is a tag and its sha1 value should never change
1775 tag_name = self.revisionExpr[len(R_TAGS):]
1776
1777 if is_sha1 or tag_name is not None:
Chris AtLee2fb64662014-01-16 21:32:33 -05001778 if self._CheckForSha1():
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001779 return True
Bertrand SIMONNET3000cda2014-11-25 16:19:29 -08001780 if is_sha1 and not depth:
1781 # When syncing a specific commit and --depth is not set:
1782 # * if upstream is explicitly specified and is not a sha1, fetch only
1783 # upstream as users expect only upstream to be fetch.
1784 # Note: The commit might not be in upstream in which case the sync
1785 # will fail.
1786 # * otherwise, fetch all branches to make sure we end up with the
1787 # specific commit.
1788 current_branch_only = self.upstream and not ID_RE.match(self.upstream)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001789
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001790 if not name:
1791 name = self.remote.name
Shawn O. Pearcefb231612009-04-10 18:53:46 -07001792
1793 ssh_proxy = False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001794 remote = self.GetRemote(name)
1795 if remote.PreConnectFetch():
Shawn O. Pearcefb231612009-04-10 18:53:46 -07001796 ssh_proxy = True
1797
Shawn O. Pearce88443382010-10-08 10:02:09 +02001798 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001799 if alt_dir and 'objects' == os.path.basename(alt_dir):
1800 ref_dir = os.path.dirname(alt_dir)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001801 packed_refs = os.path.join(self.gitdir, 'packed-refs')
1802 remote = self.GetRemote(name)
1803
David Pursehouse8a68ff92012-09-24 12:15:13 +09001804 all_refs = self.bare_ref.all
1805 ids = set(all_refs.values())
Shawn O. Pearce88443382010-10-08 10:02:09 +02001806 tmp = set()
1807
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301808 for r, ref_id in GitRefs(ref_dir).all.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09001809 if r not in all_refs:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001810 if r.startswith(R_TAGS) or remote.WritesTo(r):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001811 all_refs[r] = ref_id
1812 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001813 continue
1814
David Pursehouse8a68ff92012-09-24 12:15:13 +09001815 if ref_id in ids:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001816 continue
1817
David Pursehouse8a68ff92012-09-24 12:15:13 +09001818 r = 'refs/_alt/%s' % ref_id
1819 all_refs[r] = ref_id
1820 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001821 tmp.add(r)
1822
Shawn O. Pearce88443382010-10-08 10:02:09 +02001823 tmp_packed = ''
1824 old_packed = ''
1825
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301826 for r in sorted(all_refs):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001827 line = '%s %s\n' % (all_refs[r], r)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001828 tmp_packed += line
1829 if r not in tmp:
1830 old_packed += line
1831
1832 _lwrite(packed_refs, tmp_packed)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001833 else:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001834 alt_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02001835
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001836 cmd = ['fetch']
Doug Anderson30d45292011-05-04 15:01:04 -07001837
Conley Owensf97e8382015-01-21 11:12:46 -08001838 if depth:
Doug Anderson30d45292011-05-04 15:01:04 -07001839 cmd.append('--depth=%s' % depth)
1840
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001841 if quiet:
1842 cmd.append('--quiet')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001843 if not self.worktree:
1844 cmd.append('--update-head-ok')
David Pursehouseb4d43b92015-04-28 18:28:12 +09001845 if self.manifest.IsMirror:
1846 cmd.append('--prune')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001847 cmd.append(name)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001848
Mitchel Humpherys26c45a72014-03-10 14:21:59 -07001849 # If using depth then we should not get all the tags since they may
1850 # be outside of the depth.
1851 if no_tags or depth:
1852 cmd.append('--no-tags')
1853 else:
1854 cmd.append('--tags')
1855
Conley Owens80b87fe2014-05-09 17:13:44 -07001856 spec = []
Brian Harring14a66742012-09-28 20:21:57 -07001857 if not current_branch_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001858 # Fetch whole repo
Conley Owens80b87fe2014-05-09 17:13:44 -07001859 spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001860 elif tag_name is not None:
Conley Owens80b87fe2014-05-09 17:13:44 -07001861 spec.append('tag')
1862 spec.append(tag_name)
Nasser Grainawi04e52d62014-09-30 13:34:52 -06001863
David Pursehouse403b64e2015-04-27 10:41:33 +09001864 if not self.manifest.IsMirror:
1865 branch = self.revisionExpr
1866 if is_sha1 and depth:
1867 # Shallow checkout of a specific commit, fetch from that commit and not
1868 # the heads only as the commit might be deeper in the history.
1869 spec.append(branch)
1870 else:
1871 if is_sha1:
1872 branch = self.upstream
1873 if branch is not None and branch.strip():
1874 if not branch.startswith('refs/'):
1875 branch = R_HEADS + branch
1876 spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
Conley Owens80b87fe2014-05-09 17:13:44 -07001877 cmd.extend(spec)
1878
1879 shallowfetch = self.config.GetString('repo.shallowfetch')
1880 if shallowfetch and shallowfetch != ' '.join(spec):
Anthony King23ff7df2015-03-28 19:42:39 +00001881 GitCommand(self, ['fetch', '--depth=2147483647', name]
1882 + shallowfetch.split(),
Conley Owens80b87fe2014-05-09 17:13:44 -07001883 bare=True, ssh_proxy=ssh_proxy).Wait()
1884 if depth:
Anthony King7bdac712014-07-16 12:56:40 +01001885 self.config.SetString('repo.shallowfetch', ' '.join(spec))
Conley Owens80b87fe2014-05-09 17:13:44 -07001886 else:
Anthony King7bdac712014-07-16 12:56:40 +01001887 self.config.SetString('repo.shallowfetch', None)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001888
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001889 ok = False
David Pursehouse8a68ff92012-09-24 12:15:13 +09001890 for _i in range(2):
John L. Villalovos9c76f672015-03-16 20:49:10 -07001891 gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy)
John L. Villalovos126e2982015-01-29 21:58:12 -08001892 ret = gitcmd.Wait()
Brian Harring14a66742012-09-28 20:21:57 -07001893 if ret == 0:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001894 ok = True
1895 break
John L. Villalovos126e2982015-01-29 21:58:12 -08001896 # If needed, run the 'git remote prune' the first time through the loop
1897 elif (not _i and
1898 "error:" in gitcmd.stderr and
1899 "git remote prune" in gitcmd.stderr):
1900 prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
John L. Villalovos9c76f672015-03-16 20:49:10 -07001901 ssh_proxy=ssh_proxy)
John L. Villalovose30f46b2015-02-25 14:27:02 -08001902 ret = prunecmd.Wait()
John L. Villalovose30f46b2015-02-25 14:27:02 -08001903 if ret:
John L. Villalovos126e2982015-01-29 21:58:12 -08001904 break
1905 continue
Brian Harring14a66742012-09-28 20:21:57 -07001906 elif current_branch_only and is_sha1 and ret == 128:
1907 # Exit code 128 means "couldn't find the ref you asked for"; if we're in sha1
1908 # mode, we just tried sync'ing from the upstream field; it doesn't exist, thus
1909 # abort the optimization attempt and do a full sync.
1910 break
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001911 time.sleep(random.randint(30, 45))
Shawn O. Pearce88443382010-10-08 10:02:09 +02001912
1913 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001914 if alt_dir:
Shawn O. Pearce88443382010-10-08 10:02:09 +02001915 if old_packed != '':
1916 _lwrite(packed_refs, old_packed)
1917 else:
1918 os.remove(packed_refs)
1919 self.bare_git.pack_refs('--all', '--prune')
Brian Harring14a66742012-09-28 20:21:57 -07001920
1921 if is_sha1 and current_branch_only and self.upstream:
1922 # We just synced the upstream given branch; verify we
1923 # got what we wanted, else trigger a second run of all
1924 # refs.
Chris AtLee2fb64662014-01-16 21:32:33 -05001925 if not self._CheckForSha1():
Brian Harring14a66742012-09-28 20:21:57 -07001926 return self._RemoteFetch(name=name, current_branch_only=False,
1927 initial=False, quiet=quiet, alt_dir=alt_dir)
1928
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001929 return ok
Shawn O. Pearce88443382010-10-08 10:02:09 +02001930
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001931 def _ApplyCloneBundle(self, initial=False, quiet=False):
David Pursehouseede7f122012-11-27 22:25:30 +09001932 if initial and (self.manifest.manifestProject.config.GetString('repo.depth') or self.clone_depth):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001933 return False
1934
1935 remote = self.GetRemote(self.remote.name)
1936 bundle_url = remote.url + '/clone.bundle'
1937 bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
Dave Borowitz74c1f3d2013-06-03 15:05:07 -07001938 if GetSchemeFromUrl(bundle_url) not in (
1939 'http', 'https', 'persistent-http', 'persistent-https'):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001940 return False
1941
1942 bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
1943 bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
1944
1945 exist_dst = os.path.exists(bundle_dst)
1946 exist_tmp = os.path.exists(bundle_tmp)
1947
1948 if not initial and not exist_dst and not exist_tmp:
1949 return False
1950
1951 if not exist_dst:
1952 exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
1953 if not exist_dst:
1954 return False
1955
1956 cmd = ['fetch']
1957 if quiet:
1958 cmd.append('--quiet')
1959 if not self.worktree:
1960 cmd.append('--update-head-ok')
1961 cmd.append(bundle_dst)
1962 for f in remote.fetch:
1963 cmd.append(str(f))
1964 cmd.append('refs/tags/*:refs/tags/*')
1965
1966 ok = GitCommand(self, cmd, bare=True).Wait() == 0
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001967 if os.path.exists(bundle_dst):
1968 os.remove(bundle_dst)
1969 if os.path.exists(bundle_tmp):
1970 os.remove(bundle_tmp)
Shawn O. Pearce88443382010-10-08 10:02:09 +02001971 return ok
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001972
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001973 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001974 if os.path.exists(dstPath):
1975 os.remove(dstPath)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001976
Matt Gumbel2dc810c2012-08-30 09:39:36 -07001977 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001978 if quiet:
1979 cmd += ['--silent']
1980 if os.path.exists(tmpPath):
1981 size = os.stat(tmpPath).st_size
1982 if size >= 1024:
1983 cmd += ['--continue-at', '%d' % (size,)]
1984 else:
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001985 os.remove(tmpPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001986 if 'http_proxy' in os.environ and 'darwin' == sys.platform:
1987 cmd += ['--proxy', os.environ['http_proxy']]
Dave Borowitz497bde42015-01-02 13:58:05 -08001988 with self._GetBundleCookieFile(srcUrl, quiet) as cookiefile:
Dave Borowitz137d0132015-01-02 11:12:54 -08001989 if cookiefile:
Dave Borowitz4abf8e62015-01-02 11:39:04 -08001990 cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
Dave Borowitz137d0132015-01-02 11:12:54 -08001991 if srcUrl.startswith('persistent-'):
1992 srcUrl = srcUrl[len('persistent-'):]
1993 cmd += [srcUrl]
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07001994
Dave Borowitz137d0132015-01-02 11:12:54 -08001995 if IsTrace():
1996 Trace('%s', ' '.join(cmd))
1997 try:
1998 proc = subprocess.Popen(cmd)
1999 except OSError:
2000 return False
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002001
Dave Borowitz137d0132015-01-02 11:12:54 -08002002 curlret = proc.wait()
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002003
Dave Borowitz137d0132015-01-02 11:12:54 -08002004 if curlret == 22:
2005 # From curl man page:
2006 # 22: HTTP page not retrieved. The requested url was not found or
2007 # returned another error with the HTTP error code being 400 or above.
2008 # This return code only appears if -f, --fail is used.
2009 if not quiet:
2010 print("Server does not provide clone.bundle; ignoring.",
2011 file=sys.stderr)
2012 return False
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002013
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002014 if os.path.exists(tmpPath):
Kris Giesingc8d882a2014-12-23 13:02:32 -08002015 if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002016 os.rename(tmpPath, dstPath)
2017 return True
2018 else:
2019 os.remove(tmpPath)
2020 return False
2021 else:
2022 return False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002023
Kris Giesingc8d882a2014-12-23 13:02:32 -08002024 def _IsValidBundle(self, path, quiet):
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002025 try:
2026 with open(path) as f:
2027 if f.read(16) == '# v2 git bundle\n':
2028 return True
2029 else:
Kris Giesingc8d882a2014-12-23 13:02:32 -08002030 if not quiet:
2031 print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002032 return False
2033 except OSError:
2034 return False
2035
Dave Borowitz137d0132015-01-02 11:12:54 -08002036 @contextlib.contextmanager
Dave Borowitz497bde42015-01-02 13:58:05 -08002037 def _GetBundleCookieFile(self, url, quiet):
Dave Borowitz74c1f3d2013-06-03 15:05:07 -07002038 if url.startswith('persistent-'):
2039 try:
2040 p = subprocess.Popen(
2041 ['git-remote-persistent-https', '-print_config', url],
2042 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2043 stderr=subprocess.PIPE)
Dave Borowitz137d0132015-01-02 11:12:54 -08002044 try:
2045 prefix = 'http.cookiefile='
2046 cookiefile = None
2047 for line in p.stdout:
2048 line = line.strip()
2049 if line.startswith(prefix):
2050 cookiefile = line[len(prefix):]
2051 break
2052 # Leave subprocess open, as cookie file may be transient.
2053 if cookiefile:
2054 yield cookiefile
2055 return
2056 finally:
2057 p.stdin.close()
2058 if p.wait():
2059 err_msg = p.stderr.read()
2060 if ' -print_config' in err_msg:
2061 pass # Persistent proxy doesn't support -print_config.
Dave Borowitz497bde42015-01-02 13:58:05 -08002062 elif not quiet:
Dave Borowitz137d0132015-01-02 11:12:54 -08002063 print(err_msg, file=sys.stderr)
Dave Borowitz74c1f3d2013-06-03 15:05:07 -07002064 except OSError as e:
2065 if e.errno == errno.ENOENT:
2066 pass # No persistent proxy.
2067 raise
Dave Borowitz137d0132015-01-02 11:12:54 -08002068 yield GitConfig.ForUser().GetString('http.cookiefile')
Dave Borowitz74c1f3d2013-06-03 15:05:07 -07002069
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002070 def _Checkout(self, rev, quiet=False):
2071 cmd = ['checkout']
2072 if quiet:
2073 cmd.append('-q')
2074 cmd.append(rev)
2075 cmd.append('--')
2076 if GitCommand(self, cmd).Wait() != 0:
2077 if self._allrefs:
2078 raise GitError('%s checkout %s ' % (self.name, rev))
2079
Anthony King7bdac712014-07-16 12:56:40 +01002080 def _CherryPick(self, rev):
Pierre Tardye5a21222011-03-24 16:28:18 +01002081 cmd = ['cherry-pick']
2082 cmd.append(rev)
2083 cmd.append('--')
2084 if GitCommand(self, cmd).Wait() != 0:
2085 if self._allrefs:
2086 raise GitError('%s cherry-pick %s ' % (self.name, rev))
2087
Anthony King7bdac712014-07-16 12:56:40 +01002088 def _Revert(self, rev):
Erwan Mahea94f1622011-08-19 13:56:09 +02002089 cmd = ['revert']
2090 cmd.append('--no-edit')
2091 cmd.append(rev)
2092 cmd.append('--')
2093 if GitCommand(self, cmd).Wait() != 0:
2094 if self._allrefs:
2095 raise GitError('%s revert %s ' % (self.name, rev))
2096
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002097 def _ResetHard(self, rev, quiet=True):
2098 cmd = ['reset', '--hard']
2099 if quiet:
2100 cmd.append('-q')
2101 cmd.append(rev)
2102 if GitCommand(self, cmd).Wait() != 0:
2103 raise GitError('%s reset --hard %s ' % (self.name, rev))
2104
Anthony King7bdac712014-07-16 12:56:40 +01002105 def _Rebase(self, upstream, onto=None):
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002106 cmd = ['rebase']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002107 if onto is not None:
2108 cmd.extend(['--onto', onto])
2109 cmd.append(upstream)
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002110 if GitCommand(self, cmd).Wait() != 0:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002111 raise GitError('%s rebase %s ' % (self.name, upstream))
2112
Pierre Tardy3d125942012-05-04 12:18:12 +02002113 def _FastForward(self, head, ffonly=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002114 cmd = ['merge', head]
Pierre Tardy3d125942012-05-04 12:18:12 +02002115 if ffonly:
2116 cmd.append("--ff-only")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002117 if GitCommand(self, cmd).Wait() != 0:
2118 raise GitError('%s merge %s ' % (self.name, head))
2119
Jonathan Nieder93719792015-03-17 11:29:58 -07002120 def _InitGitDir(self, mirror_git=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002121 if not os.path.exists(self.gitdir):
David James8d201162013-10-11 17:03:19 -07002122
2123 # Initialize the bare repository, which contains all of the objects.
2124 if not os.path.exists(self.objdir):
2125 os.makedirs(self.objdir)
2126 self.bare_objdir.init()
2127
2128 # If we have a separate directory to hold refs, initialize it as well.
2129 if self.objdir != self.gitdir:
2130 os.makedirs(self.gitdir)
2131 self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
2132 copy_all=True)
Shawn O. Pearce2816d4f2009-03-03 17:53:18 -08002133
Shawn O. Pearce88443382010-10-08 10:02:09 +02002134 mp = self.manifest.manifestProject
Victor Boivie2b30e3a2012-10-05 12:37:58 +02002135 ref_dir = mp.config.GetString('repo.reference') or ''
Shawn O. Pearce88443382010-10-08 10:02:09 +02002136
Victor Boivie2b30e3a2012-10-05 12:37:58 +02002137 if ref_dir or mirror_git:
2138 if not mirror_git:
2139 mirror_git = os.path.join(ref_dir, self.name + '.git')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002140 repo_git = os.path.join(ref_dir, '.repo', 'projects',
2141 self.relpath + '.git')
2142
2143 if os.path.exists(mirror_git):
2144 ref_dir = mirror_git
2145
2146 elif os.path.exists(repo_git):
2147 ref_dir = repo_git
2148
2149 else:
2150 ref_dir = None
2151
2152 if ref_dir:
2153 _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
2154 os.path.join(ref_dir, 'objects') + '\n')
2155
Jimmie Westera0444582012-10-24 13:44:42 +02002156 self._UpdateHooks()
2157
2158 m = self.manifest.manifestProject.config
2159 for key in ['user.name', 'user.email']:
Anthony King7bdac712014-07-16 12:56:40 +01002160 if m.Has(key, include_defaults=False):
Jimmie Westera0444582012-10-24 13:44:42 +02002161 self.config.SetString(key, m.GetString(key))
Jonathan Nieder93719792015-03-17 11:29:58 -07002162 if self.manifest.IsMirror:
Shawn O. Pearce2816d4f2009-03-03 17:53:18 -08002163 self.config.SetString('core.bare', 'true')
2164 else:
2165 self.config.SetString('core.bare', None)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002166
Jimmie Westera0444582012-10-24 13:44:42 +02002167 def _UpdateHooks(self):
2168 if os.path.exists(self.gitdir):
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002169 self._InitHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002170
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002171 def _InitHooks(self):
Jesse Hall672cc492013-11-27 11:17:13 -08002172 hooks = os.path.realpath(self._gitdir_path('hooks'))
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002173 if not os.path.exists(hooks):
2174 os.makedirs(hooks)
Jonathan Nieder93719792015-03-17 11:29:58 -07002175 for stock_hook in _ProjectHooks():
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002176 name = os.path.basename(stock_hook)
2177
Victor Boivie65e0f352011-04-18 11:23:29 +02002178 if name in ('commit-msg',) and not self.remote.review \
2179 and not self is self.manifest.manifestProject:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002180 # Don't install a Gerrit Code Review hook if this
2181 # project does not appear to use it for reviews.
2182 #
Victor Boivie65e0f352011-04-18 11:23:29 +02002183 # Since the manifest project is one of those, but also
2184 # managed through gerrit, it's excluded
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002185 continue
2186
2187 dst = os.path.join(hooks, name)
2188 if os.path.islink(dst):
2189 continue
2190 if os.path.exists(dst):
2191 if filecmp.cmp(stock_hook, dst, shallow=False):
2192 os.remove(dst)
2193 else:
2194 _error("%s: Not replacing %s hook", self.relpath, name)
2195 continue
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002196 try:
Mickaël Salaünb9477bc2012-08-05 13:39:26 +02002197 os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
Sarah Owensa5be53f2012-09-09 15:37:57 -07002198 except OSError as e:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002199 if e.errno == errno.EPERM:
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002200 raise GitError('filesystem must support symlinks')
2201 else:
2202 raise
2203
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002204 def _InitRemote(self):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002205 if self.remote.url:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002206 remote = self.GetRemote(self.remote.name)
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002207 remote.url = self.remote.url
2208 remote.review = self.remote.review
2209 remote.projectname = self.name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002210
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002211 if self.worktree:
2212 remote.ResetFetch(mirror=False)
2213 else:
2214 remote.ResetFetch(mirror=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002215 remote.Save()
2216
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002217 def _InitMRef(self):
2218 if self.manifest.branch:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002219 self._InitAnyMRef(R_M + self.manifest.branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002220
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002221 def _InitMirrorHead(self):
Shawn O. Pearcefe200ee2009-06-01 15:28:21 -07002222 self._InitAnyMRef(HEAD)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002223
2224 def _InitAnyMRef(self, ref):
2225 cur = self.bare_ref.symref(ref)
2226
2227 if self.revisionId:
2228 if cur != '' or self.bare_ref.get(ref) != self.revisionId:
2229 msg = 'manifest set to %s' % self.revisionId
2230 dst = self.revisionId + '^0'
Anthony King7bdac712014-07-16 12:56:40 +01002231 self.bare_git.UpdateRef(ref, dst, message=msg, detach=True)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002232 else:
2233 remote = self.GetRemote(self.remote.name)
2234 dst = remote.ToLocal(self.revisionExpr)
2235 if cur != dst:
2236 msg = 'manifest set to %s' % self.revisionExpr
2237 self.bare_git.symbolic_ref('-m', msg, ref, dst)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002238
David James8d201162013-10-11 17:03:19 -07002239 def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
2240 """Update |dotgit| to reference |gitdir|, using symlinks where possible.
2241
2242 Args:
2243 gitdir: The bare git repository. Must already be initialized.
2244 dotgit: The repository you would like to initialize.
2245 share_refs: If true, |dotgit| will store its refs under |gitdir|.
2246 Only one work tree can store refs under a given |gitdir|.
2247 copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
2248 This saves you the effort of initializing |dotgit| yourself.
2249 """
2250 # These objects can be shared between several working trees.
2251 symlink_files = ['description', 'info']
2252 symlink_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
2253 if share_refs:
2254 # These objects can only be used by a single working tree.
Conley Owensf2af7562014-04-30 11:31:01 -07002255 symlink_files += ['config', 'packed-refs', 'shallow']
David James8d201162013-10-11 17:03:19 -07002256 symlink_dirs += ['logs', 'refs']
2257 to_symlink = symlink_files + symlink_dirs
2258
2259 to_copy = []
2260 if copy_all:
2261 to_copy = os.listdir(gitdir)
2262
2263 for name in set(to_copy).union(to_symlink):
2264 try:
2265 src = os.path.realpath(os.path.join(gitdir, name))
2266 dst = os.path.realpath(os.path.join(dotgit, name))
2267
2268 if os.path.lexists(dst) and not os.path.islink(dst):
2269 raise GitError('cannot overwrite a local work tree')
2270
2271 # If the source dir doesn't exist, create an empty dir.
2272 if name in symlink_dirs and not os.path.lexists(src):
2273 os.makedirs(src)
2274
Conley Owens80b87fe2014-05-09 17:13:44 -07002275 # If the source file doesn't exist, ensure the destination
2276 # file doesn't either.
2277 if name in symlink_files and not os.path.lexists(src):
2278 try:
2279 os.remove(dst)
2280 except OSError:
2281 pass
2282
David James8d201162013-10-11 17:03:19 -07002283 if name in to_symlink:
2284 os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
2285 elif copy_all and not os.path.islink(dst):
2286 if os.path.isdir(src):
2287 shutil.copytree(src, dst)
2288 elif os.path.isfile(src):
2289 shutil.copy(src, dst)
2290 except OSError as e:
2291 if e.errno == errno.EPERM:
2292 raise GitError('filesystem must support symlinks')
2293 else:
2294 raise
2295
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002296 def _InitWorkTree(self):
2297 dotgit = os.path.join(self.worktree, '.git')
2298 if not os.path.exists(dotgit):
2299 os.makedirs(dotgit)
David James8d201162013-10-11 17:03:19 -07002300 self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
2301 copy_all=False)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002302
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002303 _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002304
2305 cmd = ['read-tree', '--reset', '-u']
2306 cmd.append('-v')
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002307 cmd.append(HEAD)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002308 if GitCommand(self, cmd).Wait() != 0:
2309 raise GitError("cannot initialize work tree")
Victor Boivie0960b5b2010-11-26 13:42:13 +01002310
Jeff Hamiltone0df2322014-04-21 17:10:59 -05002311 self._CopyAndLinkFiles()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002312
2313 def _gitdir_path(self, path):
David James8d201162013-10-11 17:03:19 -07002314 return os.path.realpath(os.path.join(self.gitdir, path))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002315
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002316 def _revlist(self, *args, **kw):
2317 a = []
2318 a.extend(args)
2319 a.append('--')
2320 return self.work_git.rev_list(*a, **kw)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002321
2322 @property
2323 def _allrefs(self):
Shawn O. Pearced237b692009-04-17 18:49:50 -07002324 return self.bare_ref.all
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002325
Julien Camperguedd654222014-01-09 16:21:37 +01002326 def _getLogs(self, rev1, rev2, oneline=False, color=True):
2327 """Get logs between two revisions of this project."""
2328 comp = '..'
2329 if rev1:
2330 revs = [rev1]
2331 if rev2:
2332 revs.extend([comp, rev2])
2333 cmd = ['log', ''.join(revs)]
2334 out = DiffColoring(self.config)
2335 if out.is_on and color:
2336 cmd.append('--color')
2337 if oneline:
2338 cmd.append('--oneline')
2339
2340 try:
2341 log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
2342 if log.Wait() == 0:
2343 return log.stdout
2344 except GitError:
2345 # worktree may not exist if groups changed for example. In that case,
2346 # try in gitdir instead.
2347 if not os.path.exists(self.worktree):
2348 return self.bare_git.log(*cmd[1:])
2349 else:
2350 raise
2351 return None
2352
2353 def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True):
2354 """Get the list of logs from this revision to given revisionId"""
2355 logs = {}
2356 selfId = self.GetRevisionId(self._allrefs)
2357 toId = toProject.GetRevisionId(toProject._allrefs)
2358
2359 logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color)
2360 logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color)
2361 return logs
2362
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002363 class _GitGetByExec(object):
David James8d201162013-10-11 17:03:19 -07002364 def __init__(self, project, bare, gitdir):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002365 self._project = project
2366 self._bare = bare
David James8d201162013-10-11 17:03:19 -07002367 self._gitdir = gitdir
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002368
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002369 def LsOthers(self):
2370 p = GitCommand(self._project,
2371 ['ls-files',
2372 '-z',
2373 '--others',
2374 '--exclude-standard'],
Anthony King7bdac712014-07-16 12:56:40 +01002375 bare=False,
David James8d201162013-10-11 17:03:19 -07002376 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002377 capture_stdout=True,
2378 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002379 if p.Wait() == 0:
2380 out = p.stdout
2381 if out:
David Pursehouse1d947b32012-10-25 12:23:11 +09002382 return out[:-1].split('\0') # pylint: disable=W1401
2383 # Backslash is not anomalous
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002384 return []
2385
2386 def DiffZ(self, name, *args):
2387 cmd = [name]
2388 cmd.append('-z')
2389 cmd.extend(args)
2390 p = GitCommand(self._project,
2391 cmd,
David James8d201162013-10-11 17:03:19 -07002392 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002393 bare=False,
2394 capture_stdout=True,
2395 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002396 try:
2397 out = p.process.stdout.read()
2398 r = {}
2399 if out:
David Pursehouse1d947b32012-10-25 12:23:11 +09002400 out = iter(out[:-1].split('\0')) # pylint: disable=W1401
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002401 while out:
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002402 try:
Anthony King2cd1f042014-05-05 21:24:05 +01002403 info = next(out)
2404 path = next(out)
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002405 except StopIteration:
2406 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002407
2408 class _Info(object):
2409 def __init__(self, path, omode, nmode, oid, nid, state):
2410 self.path = path
2411 self.src_path = None
2412 self.old_mode = omode
2413 self.new_mode = nmode
2414 self.old_id = oid
2415 self.new_id = nid
2416
2417 if len(state) == 1:
2418 self.status = state
2419 self.level = None
2420 else:
2421 self.status = state[:1]
2422 self.level = state[1:]
2423 while self.level.startswith('0'):
2424 self.level = self.level[1:]
2425
2426 info = info[1:].split(' ')
David Pursehouse8f62fb72012-11-14 12:09:38 +09002427 info = _Info(path, *info)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002428 if info.status in ('R', 'C'):
2429 info.src_path = info.path
Anthony King2cd1f042014-05-05 21:24:05 +01002430 info.path = next(out)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002431 r[info.path] = info
2432 return r
2433 finally:
2434 p.Wait()
2435
2436 def GetHead(self):
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002437 if self._bare:
2438 path = os.path.join(self._project.gitdir, HEAD)
2439 else:
2440 path = os.path.join(self._project.worktree, '.git', HEAD)
Conley Owens75ee0572012-11-15 17:33:11 -08002441 try:
2442 fd = open(path, 'rb')
Dan Sandler53e902a2014-03-09 13:20:02 -04002443 except IOError as e:
2444 raise NoManifestException(path, str(e))
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -07002445 try:
2446 line = fd.read()
2447 finally:
2448 fd.close()
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302449 try:
2450 line = line.decode()
2451 except AttributeError:
2452 pass
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002453 if line.startswith('ref: '):
2454 return line[5:-1]
2455 return line[:-1]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002456
2457 def SetHead(self, ref, message=None):
2458 cmdv = []
2459 if message is not None:
2460 cmdv.extend(['-m', message])
2461 cmdv.append(HEAD)
2462 cmdv.append(ref)
2463 self.symbolic_ref(*cmdv)
2464
2465 def DetachHead(self, new, message=None):
2466 cmdv = ['--no-deref']
2467 if message is not None:
2468 cmdv.extend(['-m', message])
2469 cmdv.append(HEAD)
2470 cmdv.append(new)
2471 self.update_ref(*cmdv)
2472
2473 def UpdateRef(self, name, new, old=None,
2474 message=None,
2475 detach=False):
2476 cmdv = []
2477 if message is not None:
2478 cmdv.extend(['-m', message])
2479 if detach:
2480 cmdv.append('--no-deref')
2481 cmdv.append(name)
2482 cmdv.append(new)
2483 if old is not None:
2484 cmdv.append(old)
2485 self.update_ref(*cmdv)
2486
2487 def DeleteRef(self, name, old=None):
2488 if not old:
2489 old = self.rev_parse(name)
2490 self.update_ref('-d', name, old)
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07002491 self._project.bare_ref.deleted(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002492
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002493 def rev_list(self, *args, **kw):
2494 if 'format' in kw:
2495 cmdv = ['log', '--pretty=format:%s' % kw['format']]
2496 else:
2497 cmdv = ['rev-list']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002498 cmdv.extend(args)
2499 p = GitCommand(self._project,
2500 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002501 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002502 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002503 capture_stdout=True,
2504 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002505 r = []
2506 for line in p.process.stdout:
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002507 if line[-1] == '\n':
2508 line = line[:-1]
2509 r.append(line)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002510 if p.Wait() != 0:
2511 raise GitError('%s rev-list %s: %s' % (
2512 self._project.name,
2513 str(args),
2514 p.stderr))
2515 return r
2516
2517 def __getattr__(self, name):
Doug Anderson37282b42011-03-04 11:54:18 -08002518 """Allow arbitrary git commands using pythonic syntax.
2519
2520 This allows you to do things like:
2521 git_obj.rev_parse('HEAD')
2522
2523 Since we don't have a 'rev_parse' method defined, the __getattr__ will
2524 run. We'll replace the '_' with a '-' and try to run a git command.
Dave Borowitz091f8932012-10-23 17:01:04 -07002525 Any other positional arguments will be passed to the git command, and the
2526 following keyword arguments are supported:
2527 config: An optional dict of git config options to be passed with '-c'.
Doug Anderson37282b42011-03-04 11:54:18 -08002528
2529 Args:
2530 name: The name of the git command to call. Any '_' characters will
2531 be replaced with '-'.
2532
2533 Returns:
2534 A callable object that will try to call git with the named command.
2535 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002536 name = name.replace('_', '-')
Dave Borowitz091f8932012-10-23 17:01:04 -07002537 def runner(*args, **kwargs):
2538 cmdv = []
2539 config = kwargs.pop('config', None)
2540 for k in kwargs:
2541 raise TypeError('%s() got an unexpected keyword argument %r'
2542 % (name, k))
2543 if config is not None:
Dave Borowitzb42b4742012-10-31 12:27:27 -07002544 if not git_require((1, 7, 2)):
2545 raise ValueError('cannot set config on command line for %s()'
2546 % name)
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302547 for k, v in config.items():
Dave Borowitz091f8932012-10-23 17:01:04 -07002548 cmdv.append('-c')
2549 cmdv.append('%s=%s' % (k, v))
2550 cmdv.append(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002551 cmdv.extend(args)
2552 p = GitCommand(self._project,
2553 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002554 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002555 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002556 capture_stdout=True,
2557 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002558 if p.Wait() != 0:
2559 raise GitError('%s %s: %s' % (
2560 self._project.name,
2561 name,
2562 p.stderr))
2563 r = p.stdout
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302564 try:
Conley Owensedd01512013-09-26 12:59:58 -07002565 r = r.decode('utf-8')
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302566 except AttributeError:
2567 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002568 if r.endswith('\n') and r.index('\n') == len(r) - 1:
2569 return r[:-1]
2570 return r
2571 return runner
2572
2573
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002574class _PriorSyncFailedError(Exception):
2575 def __str__(self):
2576 return 'prior sync failed; rebase still in progress'
2577
2578class _DirtyError(Exception):
2579 def __str__(self):
2580 return 'contains uncommitted changes'
2581
2582class _InfoMessage(object):
2583 def __init__(self, project, text):
2584 self.project = project
2585 self.text = text
2586
2587 def Print(self, syncbuf):
2588 syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
2589 syncbuf.out.nl()
2590
2591class _Failure(object):
2592 def __init__(self, project, why):
2593 self.project = project
2594 self.why = why
2595
2596 def Print(self, syncbuf):
2597 syncbuf.out.fail('error: %s/: %s',
2598 self.project.relpath,
2599 str(self.why))
2600 syncbuf.out.nl()
2601
2602class _Later(object):
2603 def __init__(self, project, action):
2604 self.project = project
2605 self.action = action
2606
2607 def Run(self, syncbuf):
2608 out = syncbuf.out
2609 out.project('project %s/', self.project.relpath)
2610 out.nl()
2611 try:
2612 self.action()
2613 out.nl()
2614 return True
David Pursehouse8a68ff92012-09-24 12:15:13 +09002615 except GitError:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002616 out.nl()
2617 return False
2618
2619class _SyncColoring(Coloring):
2620 def __init__(self, config):
2621 Coloring.__init__(self, config, 'reposync')
Anthony King7bdac712014-07-16 12:56:40 +01002622 self.project = self.printer('header', attr='bold')
2623 self.info = self.printer('info')
2624 self.fail = self.printer('fail', fg='red')
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002625
2626class SyncBuffer(object):
2627 def __init__(self, config, detach_head=False):
2628 self._messages = []
2629 self._failures = []
2630 self._later_queue1 = []
2631 self._later_queue2 = []
2632
2633 self.out = _SyncColoring(config)
2634 self.out.redirect(sys.stderr)
2635
2636 self.detach_head = detach_head
2637 self.clean = True
2638
2639 def info(self, project, fmt, *args):
2640 self._messages.append(_InfoMessage(project, fmt % args))
2641
2642 def fail(self, project, err=None):
2643 self._failures.append(_Failure(project, err))
2644 self.clean = False
2645
2646 def later1(self, project, what):
2647 self._later_queue1.append(_Later(project, what))
2648
2649 def later2(self, project, what):
2650 self._later_queue2.append(_Later(project, what))
2651
2652 def Finish(self):
2653 self._PrintMessages()
2654 self._RunLater()
2655 self._PrintMessages()
2656 return self.clean
2657
2658 def _RunLater(self):
2659 for q in ['_later_queue1', '_later_queue2']:
2660 if not self._RunQueue(q):
2661 return
2662
2663 def _RunQueue(self, queue):
2664 for m in getattr(self, queue):
2665 if not m.Run(self):
2666 self.clean = False
2667 return False
2668 setattr(self, queue, [])
2669 return True
2670
2671 def _PrintMessages(self):
2672 for m in self._messages:
2673 m.Print(self)
2674 for m in self._failures:
2675 m.Print(self)
2676
2677 self._messages = []
2678 self._failures = []
2679
2680
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002681class MetaProject(Project):
2682 """A special project housed under .repo.
2683 """
2684 def __init__(self, manifest, name, gitdir, worktree):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002685 Project.__init__(self,
Anthony King7bdac712014-07-16 12:56:40 +01002686 manifest=manifest,
2687 name=name,
2688 gitdir=gitdir,
2689 objdir=gitdir,
2690 worktree=worktree,
2691 remote=RemoteSpec('origin'),
2692 relpath='.repo/%s' % name,
2693 revisionExpr='refs/heads/master',
2694 revisionId=None,
2695 groups=None)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002696
2697 def PreSync(self):
2698 if self.Exists:
2699 cb = self.CurrentBranch
2700 if cb:
2701 base = self.GetBranch(cb).merge
2702 if base:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002703 self.revisionExpr = base
2704 self.revisionId = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002705
Anthony King7bdac712014-07-16 12:56:40 +01002706 def MetaBranchSwitch(self):
Florian Vallee5d016502012-06-07 17:19:26 +02002707 """ Prepare MetaProject for manifest branch switch
2708 """
2709
2710 # detach and delete manifest branch, allowing a new
2711 # branch to take over
Anthony King7bdac712014-07-16 12:56:40 +01002712 syncbuf = SyncBuffer(self.config, detach_head=True)
Florian Vallee5d016502012-06-07 17:19:26 +02002713 self.Sync_LocalHalf(syncbuf)
2714 syncbuf.Finish()
2715
2716 return GitCommand(self,
Torne (Richard Coles)e8f75fa2012-07-20 15:32:19 +01002717 ['update-ref', '-d', 'refs/heads/default'],
Anthony King7bdac712014-07-16 12:56:40 +01002718 capture_stdout=True,
2719 capture_stderr=True).Wait() == 0
Florian Vallee5d016502012-06-07 17:19:26 +02002720
2721
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002722 @property
Shawn O. Pearcef6906872009-04-18 10:49:00 -07002723 def LastFetch(self):
2724 try:
2725 fh = os.path.join(self.gitdir, 'FETCH_HEAD')
2726 return os.path.getmtime(fh)
2727 except OSError:
2728 return 0
2729
2730 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002731 def HasChanges(self):
2732 """Has the remote received new commits not yet checked out?
2733 """
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002734 if not self.remote or not self.revisionExpr:
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002735 return False
2736
David Pursehouse8a68ff92012-09-24 12:15:13 +09002737 all_refs = self.bare_ref.all
2738 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002739 head = self.work_git.GetHead()
2740 if head.startswith(R_HEADS):
2741 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09002742 head = all_refs[head]
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07002743 except KeyError:
2744 head = None
2745
2746 if revid == head:
2747 return False
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002748 elif self._revlist(not_rev(HEAD), revid):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002749 return True
2750 return False