blob: b8147424a608a2bdf9f00e64ba5b253871802a1a [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Sarah Owenscecd1d82012-11-01 22:59:27 -070015from __future__ import print_function
Shawn O. Pearce438ee1c2008-11-03 09:59:36 -080016import errno
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import filecmp
Wink Saville4c426ef2015-06-03 08:05:17 -070018import glob
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070020import random
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
22import shutil
23import stat
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Julien Campergue335f5ef2013-10-16 11:02:35 +020026import tarfile
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +080027import tempfile
Shawn O. Pearcec325dc32011-10-03 08:30:24 -070028import time
Dave Borowitz137d0132015-01-02 11:12:54 -080029import traceback
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -070030
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070031from color import Coloring
Dave Borowitzb42b4742012-10-31 12:27:27 -070032from git_command import GitCommand, git_require
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070033from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
34 ID_RE
Kevin Degiabaa7f32014-11-12 11:27:45 -070035from error import GitError, HookError, UploadError, DownloadError
Shawn O. Pearce559b8462009-03-02 12:56:08 -080036from error import ManifestInvalidRevisionError
Conley Owens75ee0572012-11-15 17:33:11 -080037from error import NoManifestException
Renaud Paquayd5cec5e2016-11-01 11:24:03 -070038import platform_utils
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -070039from trace import IsTrace, Trace
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
Shawn O. Pearced237b692009-04-17 18:49:50 -070041from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070042
David Pursehouse59bbb582013-05-17 10:49:33 +090043from pyversion import is_python3
Mike Frysinger40252c22016-08-15 21:23:44 -040044if is_python3():
45 import urllib.parse
46else:
47 import imp
48 import urlparse
49 urllib = imp.new_module('urllib')
50 urllib.parse = urlparse
Chirayu Desai217ea7d2013-03-01 19:14:38 +053051 input = raw_input
Chirayu Desai217ea7d2013-03-01 19:14:38 +053052
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070053
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070054def _lwrite(path, content):
55 lock = '%s.lock' % path
56
Chirayu Desai303a82f2014-08-19 22:57:17 +053057 fd = open(lock, 'w')
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070058 try:
59 fd.write(content)
60 finally:
61 fd.close()
62
63 try:
Renaud Paquayad1abcb2016-11-01 11:34:55 -070064 platform_utils.rename(lock, path)
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070065 except OSError:
Renaud Paquay010fed72016-11-11 14:25:29 -080066 platform_utils.remove(lock)
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070067 raise
68
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070069
Shawn O. Pearce48244782009-04-16 08:25:57 -070070def _error(fmt, *args):
71 msg = fmt % args
Sarah Owenscecd1d82012-11-01 22:59:27 -070072 print('error: %s' % msg, file=sys.stderr)
Shawn O. Pearce48244782009-04-16 08:25:57 -070073
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070074
David Pursehousef33929d2015-08-24 14:39:14 +090075def _warn(fmt, *args):
76 msg = fmt % args
77 print('warn: %s' % msg, file=sys.stderr)
78
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070079
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070080def not_rev(r):
81 return '^' + r
82
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070083
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080084def sq(r):
85 return "'" + r.replace("'", "'\''") + "'"
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -080086
Jonathan Nieder93719792015-03-17 11:29:58 -070087_project_hook_list = None
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -070088
89
Jonathan Nieder93719792015-03-17 11:29:58 -070090def _ProjectHooks():
91 """List the hooks present in the 'hooks' directory.
92
93 These hooks are project hooks and are copied to the '.git/hooks' directory
94 of all subprojects.
95
96 This function caches the list of hooks (based on the contents of the
97 'repo/hooks' directory) on the first call.
98
99 Returns:
100 A list of absolute paths to all of the files in the hooks directory.
101 """
102 global _project_hook_list
103 if _project_hook_list is None:
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700104 d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__)))
Jonathan Nieder93719792015-03-17 11:29:58 -0700105 d = os.path.join(d, 'hooks')
Renaud Paquaybed8b622018-09-27 10:46:58 -0700106 _project_hook_list = [os.path.join(d, x) for x in platform_utils.listdir(d)]
Jonathan Nieder93719792015-03-17 11:29:58 -0700107 return _project_hook_list
108
109
Shawn O. Pearce632768b2008-10-23 11:58:52 -0700110class DownloadedChange(object):
111 _commit_cache = None
112
113 def __init__(self, project, base, change_id, ps_id, commit):
114 self.project = project
115 self.base = base
116 self.change_id = change_id
117 self.ps_id = ps_id
118 self.commit = commit
119
120 @property
121 def commits(self):
122 if self._commit_cache is None:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700123 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
124 '--abbrev-commit',
125 '--pretty=oneline',
126 '--reverse',
127 '--date-order',
128 not_rev(self.base),
129 self.commit,
130 '--')
Shawn O. Pearce632768b2008-10-23 11:58:52 -0700131 return self._commit_cache
132
133
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700134class ReviewableBranch(object):
135 _commit_cache = None
136
137 def __init__(self, project, branch, base):
138 self.project = project
139 self.branch = branch
140 self.base = base
141
142 @property
143 def name(self):
144 return self.branch.name
145
146 @property
147 def commits(self):
148 if self._commit_cache is None:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700149 self._commit_cache = self.project.bare_git.rev_list('--abbrev=8',
150 '--abbrev-commit',
151 '--pretty=oneline',
152 '--reverse',
153 '--date-order',
154 not_rev(self.base),
155 R_HEADS + self.name,
156 '--')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700157 return self._commit_cache
158
159 @property
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800160 def unabbrev_commits(self):
161 r = dict()
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700162 for commit in self.project.bare_git.rev_list(not_rev(self.base),
163 R_HEADS + self.name,
164 '--'):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800165 r[commit[0:8]] = commit
166 return r
167
168 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700169 def date(self):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700170 return self.project.bare_git.log('--pretty=format:%cd',
171 '-n', '1',
172 R_HEADS + self.name,
173 '--')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700174
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700175 def UploadForReview(self, people,
176 auto_topic=False,
Jonathan Niederc94d6eb2017-08-08 18:34:53 +0000177 draft=False,
Changcheng Xiao87984c62017-08-02 16:55:03 +0200178 private=False,
Vadim Bendeburybd8f6582018-10-31 13:48:01 -0700179 notify=None,
Changcheng Xiao87984c62017-08-02 16:55:03 +0200180 wip=False,
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200181 dest_branch=None,
Masaya Suzuki305a2d02017-11-13 10:48:34 -0800182 validate_certs=True,
183 push_options=None):
Shawn O. Pearcec99883f2008-11-11 17:12:43 -0800184 self.project.UploadForReview(self.name,
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -0700185 people,
Brian Harring435370c2012-07-28 15:37:04 -0700186 auto_topic=auto_topic,
Jonathan Niederc94d6eb2017-08-08 18:34:53 +0000187 draft=draft,
Changcheng Xiao87984c62017-08-02 16:55:03 +0200188 private=private,
Vadim Bendeburybd8f6582018-10-31 13:48:01 -0700189 notify=notify,
Changcheng Xiao87984c62017-08-02 16:55:03 +0200190 wip=wip,
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200191 dest_branch=dest_branch,
Masaya Suzuki305a2d02017-11-13 10:48:34 -0800192 validate_certs=validate_certs,
193 push_options=push_options)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700194
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700195 def GetPublishedRefs(self):
196 refs = {}
197 output = self.project.bare_git.ls_remote(
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700198 self.branch.remote.SshReviewUrl(self.project.UserEmail),
199 'refs/changes/*')
Ficus Kirkpatrickbc7ef672009-05-04 12:45:11 -0700200 for line in output.split('\n'):
201 try:
202 (sha, ref) = line.split()
203 refs[sha] = ref
204 except ValueError:
205 pass
206
207 return refs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700208
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700209
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700210class StatusColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700211
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700212 def __init__(self, config):
213 Coloring.__init__(self, config, 'status')
Anthony King7bdac712014-07-16 12:56:40 +0100214 self.project = self.printer('header', attr='bold')
215 self.branch = self.printer('header', attr='bold')
216 self.nobranch = self.printer('nobranch', fg='red')
217 self.important = self.printer('important', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700218
Anthony King7bdac712014-07-16 12:56:40 +0100219 self.added = self.printer('added', fg='green')
220 self.changed = self.printer('changed', fg='red')
221 self.untracked = self.printer('untracked', fg='red')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700222
223
224class DiffColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700225
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700226 def __init__(self, config):
227 Coloring.__init__(self, config, 'diff')
Anthony King7bdac712014-07-16 12:56:40 +0100228 self.project = self.printer('header', attr='bold')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700229
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700230
Anthony King7bdac712014-07-16 12:56:40 +0100231class _Annotation(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700232
James W. Mills24c13082012-04-12 15:04:13 -0500233 def __init__(self, name, value, keep):
234 self.name = name
235 self.value = value
236 self.keep = keep
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700237
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700238
Anthony King7bdac712014-07-16 12:56:40 +0100239class _CopyFile(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700240
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800241 def __init__(self, src, dest, abssrc, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700242 self.src = src
243 self.dest = dest
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800244 self.abs_src = abssrc
245 self.abs_dest = absdest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700246
247 def _Copy(self):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800248 src = self.abs_src
249 dest = self.abs_dest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700250 # copy file if it does not exist or is out of date
251 if not os.path.exists(dest) or not filecmp.cmp(src, dest):
252 try:
253 # remove existing file first, since it might be read-only
254 if os.path.exists(dest):
Renaud Paquay010fed72016-11-11 14:25:29 -0800255 platform_utils.remove(dest)
Matthew Buckett2daf6672009-07-11 09:43:47 -0400256 else:
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200257 dest_dir = os.path.dirname(dest)
Renaud Paquaybed8b622018-09-27 10:46:58 -0700258 if not platform_utils.isdir(dest_dir):
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200259 os.makedirs(dest_dir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700260 shutil.copy(src, dest)
261 # make the file read-only
262 mode = os.stat(dest)[stat.ST_MODE]
263 mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
264 os.chmod(dest, mode)
265 except IOError:
Shawn O. Pearce48244782009-04-16 08:25:57 -0700266 _error('Cannot copy file %s to %s', src, dest)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700267
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700268
Anthony King7bdac712014-07-16 12:56:40 +0100269class _LinkFile(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700270
Wink Saville4c426ef2015-06-03 08:05:17 -0700271 def __init__(self, git_worktree, src, dest, relsrc, absdest):
272 self.git_worktree = git_worktree
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500273 self.src = src
274 self.dest = dest
Colin Cross0184dcc2015-05-05 00:24:54 -0700275 self.src_rel_to_dest = relsrc
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500276 self.abs_dest = absdest
277
Wink Saville4c426ef2015-06-03 08:05:17 -0700278 def __linkIt(self, relSrc, absDest):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500279 # link file if it does not exist or is out of date
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700280 if not platform_utils.islink(absDest) or (platform_utils.readlink(absDest) != relSrc):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500281 try:
282 # remove existing file first, since it might be read-only
Dan Willemsene1e0bd12015-11-18 16:49:38 -0800283 if os.path.lexists(absDest):
Renaud Paquay010fed72016-11-11 14:25:29 -0800284 platform_utils.remove(absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500285 else:
Wink Saville4c426ef2015-06-03 08:05:17 -0700286 dest_dir = os.path.dirname(absDest)
Renaud Paquaybed8b622018-09-27 10:46:58 -0700287 if not platform_utils.isdir(dest_dir):
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500288 os.makedirs(dest_dir)
Renaud Paquayd5cec5e2016-11-01 11:24:03 -0700289 platform_utils.symlink(relSrc, absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500290 except IOError:
Wink Saville4c426ef2015-06-03 08:05:17 -0700291 _error('Cannot link file %s to %s', relSrc, absDest)
292
293 def _Link(self):
294 """Link the self.rel_src_to_dest and self.abs_dest. Handles wild cards
295 on the src linking all of the files in the source in to the destination
296 directory.
297 """
298 # We use the absSrc to handle the situation where the current directory
299 # is not the root of the repo
300 absSrc = os.path.join(self.git_worktree, self.src)
301 if os.path.exists(absSrc):
302 # Entity exists so just a simple one to one link operation
303 self.__linkIt(self.src_rel_to_dest, self.abs_dest)
304 else:
305 # Entity doesn't exist assume there is a wild card
306 absDestDir = self.abs_dest
Renaud Paquaybed8b622018-09-27 10:46:58 -0700307 if os.path.exists(absDestDir) and not platform_utils.isdir(absDestDir):
Wink Saville4c426ef2015-06-03 08:05:17 -0700308 _error('Link error: src with wildcard, %s must be a directory',
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700309 absDestDir)
Wink Saville4c426ef2015-06-03 08:05:17 -0700310 else:
311 absSrcFiles = glob.glob(absSrc)
312 for absSrcFile in absSrcFiles:
313 # Create a releative path from source dir to destination dir
314 absSrcDir = os.path.dirname(absSrcFile)
315 relSrcDir = os.path.relpath(absSrcDir, absDestDir)
316
317 # Get the source file name
318 srcFile = os.path.basename(absSrcFile)
319
320 # Now form the final full paths to srcFile. They will be
321 # absolute for the desintaiton and relative for the srouce.
322 absDest = os.path.join(absDestDir, srcFile)
323 relSrc = os.path.join(relSrcDir, srcFile)
324 self.__linkIt(relSrc, absDest)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500325
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700326
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700327class RemoteSpec(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700328
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700329 def __init__(self,
330 name,
Anthony King7bdac712014-07-16 12:56:40 +0100331 url=None,
Steve Raed6480452016-08-10 15:00:00 -0700332 pushUrl=None,
Anthony King7bdac712014-07-16 12:56:40 +0100333 review=None,
Dan Willemsen96c2d652016-04-06 16:03:54 -0700334 revision=None,
David Rileye0684ad2017-04-05 00:02:59 -0700335 orig_name=None,
336 fetchUrl=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700337 self.name = name
338 self.url = url
Steve Raed6480452016-08-10 15:00:00 -0700339 self.pushUrl = pushUrl
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700340 self.review = review
Anthony King36ea2fb2014-05-06 11:54:01 +0100341 self.revision = revision
Dan Willemsen96c2d652016-04-06 16:03:54 -0700342 self.orig_name = orig_name
David Rileye0684ad2017-04-05 00:02:59 -0700343 self.fetchUrl = fetchUrl
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700344
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700345
Doug Anderson37282b42011-03-04 11:54:18 -0800346class RepoHook(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700347
Doug Anderson37282b42011-03-04 11:54:18 -0800348 """A RepoHook contains information about a script to run as a hook.
349
350 Hooks are used to run a python script before running an upload (for instance,
351 to run presubmit checks). Eventually, we may have hooks for other actions.
352
353 This shouldn't be confused with files in the 'repo/hooks' directory. Those
354 files are copied into each '.git/hooks' folder for each project. Repo-level
355 hooks are associated instead with repo actions.
356
357 Hooks are always python. When a hook is run, we will load the hook into the
358 interpreter and execute its main() function.
359 """
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700360
Doug Anderson37282b42011-03-04 11:54:18 -0800361 def __init__(self,
362 hook_type,
363 hooks_project,
364 topdir,
Mike Frysinger40252c22016-08-15 21:23:44 -0400365 manifest_url,
Doug Anderson37282b42011-03-04 11:54:18 -0800366 abort_if_user_denies=False):
367 """RepoHook constructor.
368
369 Params:
370 hook_type: A string representing the type of hook. This is also used
371 to figure out the name of the file containing the hook. For
372 example: 'pre-upload'.
373 hooks_project: The project containing the repo hooks. If you have a
374 manifest, this is manifest.repo_hooks_project. OK if this is None,
375 which will make the hook a no-op.
376 topdir: Repo's top directory (the one containing the .repo directory).
377 Scripts will run with CWD as this directory. If you have a manifest,
378 this is manifest.topdir
Mike Frysinger40252c22016-08-15 21:23:44 -0400379 manifest_url: The URL to the manifest git repo.
Doug Anderson37282b42011-03-04 11:54:18 -0800380 abort_if_user_denies: If True, we'll throw a HookError() if the user
381 doesn't allow us to run the hook.
382 """
383 self._hook_type = hook_type
384 self._hooks_project = hooks_project
Mike Frysinger40252c22016-08-15 21:23:44 -0400385 self._manifest_url = manifest_url
Doug Anderson37282b42011-03-04 11:54:18 -0800386 self._topdir = topdir
387 self._abort_if_user_denies = abort_if_user_denies
388
389 # Store the full path to the script for convenience.
390 if self._hooks_project:
391 self._script_fullpath = os.path.join(self._hooks_project.worktree,
392 self._hook_type + '.py')
393 else:
394 self._script_fullpath = None
395
396 def _GetHash(self):
397 """Return a hash of the contents of the hooks directory.
398
399 We'll just use git to do this. This hash has the property that if anything
400 changes in the directory we will return a different has.
401
402 SECURITY CONSIDERATION:
403 This hash only represents the contents of files in the hook directory, not
404 any other files imported or called by hooks. Changes to imported files
405 can change the script behavior without affecting the hash.
406
407 Returns:
408 A string representing the hash. This will always be ASCII so that it can
409 be printed to the user easily.
410 """
411 assert self._hooks_project, "Must have hooks to calculate their hash."
412
413 # We will use the work_git object rather than just calling GetRevisionId().
414 # That gives us a hash of the latest checked in version of the files that
415 # the user will actually be executing. Specifically, GetRevisionId()
416 # doesn't appear to change even if a user checks out a different version
417 # of the hooks repo (via git checkout) nor if a user commits their own revs.
418 #
419 # NOTE: Local (non-committed) changes will not be factored into this hash.
420 # I think this is OK, since we're really only worried about warning the user
421 # about upstream changes.
422 return self._hooks_project.work_git.rev_parse('HEAD')
423
424 def _GetMustVerb(self):
425 """Return 'must' if the hook is required; 'should' if not."""
426 if self._abort_if_user_denies:
427 return 'must'
428 else:
429 return 'should'
430
431 def _CheckForHookApproval(self):
432 """Check to see whether this hook has been approved.
433
Mike Frysinger40252c22016-08-15 21:23:44 -0400434 We'll accept approval of manifest URLs if they're using secure transports.
435 This way the user can say they trust the manifest hoster. For insecure
436 hosts, we fall back to checking the hash of the hooks repo.
Doug Anderson37282b42011-03-04 11:54:18 -0800437
438 Note that we ask permission for each individual hook even though we use
439 the hash of all hooks when detecting changes. We'd like the user to be
440 able to approve / deny each hook individually. We only use the hash of all
441 hooks because there is no other easy way to detect changes to local imports.
442
443 Returns:
444 True if this hook is approved to run; False otherwise.
445
446 Raises:
447 HookError: Raised if the user doesn't approve and abort_if_user_denies
448 was passed to the consturctor.
449 """
Mike Frysinger40252c22016-08-15 21:23:44 -0400450 if self._ManifestUrlHasSecureScheme():
451 return self._CheckForHookApprovalManifest()
452 else:
453 return self._CheckForHookApprovalHash()
454
455 def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt,
456 changed_prompt):
457 """Check for approval for a particular attribute and hook.
458
459 Args:
460 subkey: The git config key under [repo.hooks.<hook_type>] to store the
461 last approved string.
462 new_val: The new value to compare against the last approved one.
463 main_prompt: Message to display to the user to ask for approval.
464 changed_prompt: Message explaining why we're re-asking for approval.
465
466 Returns:
467 True if this hook is approved to run; False otherwise.
468
469 Raises:
470 HookError: Raised if the user doesn't approve and abort_if_user_denies
471 was passed to the consturctor.
472 """
Doug Anderson37282b42011-03-04 11:54:18 -0800473 hooks_config = self._hooks_project.config
Mike Frysinger40252c22016-08-15 21:23:44 -0400474 git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey)
Doug Anderson37282b42011-03-04 11:54:18 -0800475
Mike Frysinger40252c22016-08-15 21:23:44 -0400476 # Get the last value that the user approved for this hook; may be None.
477 old_val = hooks_config.GetString(git_approval_key)
Doug Anderson37282b42011-03-04 11:54:18 -0800478
Mike Frysinger40252c22016-08-15 21:23:44 -0400479 if old_val is not None:
Doug Anderson37282b42011-03-04 11:54:18 -0800480 # User previously approved hook and asked not to be prompted again.
Mike Frysinger40252c22016-08-15 21:23:44 -0400481 if new_val == old_val:
Doug Anderson37282b42011-03-04 11:54:18 -0800482 # Approval matched. We're done.
483 return True
484 else:
485 # Give the user a reason why we're prompting, since they last told
486 # us to "never ask again".
Mike Frysinger40252c22016-08-15 21:23:44 -0400487 prompt = 'WARNING: %s\n\n' % (changed_prompt,)
Doug Anderson37282b42011-03-04 11:54:18 -0800488 else:
489 prompt = ''
490
491 # Prompt the user if we're not on a tty; on a tty we'll assume "no".
492 if sys.stdout.isatty():
Mike Frysinger40252c22016-08-15 21:23:44 -0400493 prompt += main_prompt + ' (yes/always/NO)? '
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530494 response = input(prompt).lower()
David Pursehouse98ffba12012-11-14 11:18:00 +0900495 print()
Doug Anderson37282b42011-03-04 11:54:18 -0800496
497 # User is doing a one-time approval.
498 if response in ('y', 'yes'):
499 return True
Mike Frysinger40252c22016-08-15 21:23:44 -0400500 elif response == 'always':
501 hooks_config.SetString(git_approval_key, new_val)
Doug Anderson37282b42011-03-04 11:54:18 -0800502 return True
503
504 # For anything else, we'll assume no approval.
505 if self._abort_if_user_denies:
506 raise HookError('You must allow the %s hook or use --no-verify.' %
507 self._hook_type)
508
509 return False
510
Mike Frysinger40252c22016-08-15 21:23:44 -0400511 def _ManifestUrlHasSecureScheme(self):
512 """Check if the URI for the manifest is a secure transport."""
513 secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc')
514 parse_results = urllib.parse.urlparse(self._manifest_url)
515 return parse_results.scheme in secure_schemes
516
517 def _CheckForHookApprovalManifest(self):
518 """Check whether the user has approved this manifest host.
519
520 Returns:
521 True if this hook is approved to run; False otherwise.
522 """
523 return self._CheckForHookApprovalHelper(
524 'approvedmanifest',
525 self._manifest_url,
526 'Run hook scripts from %s' % (self._manifest_url,),
527 'Manifest URL has changed since %s was allowed.' % (self._hook_type,))
528
529 def _CheckForHookApprovalHash(self):
530 """Check whether the user has approved the hooks repo.
531
532 Returns:
533 True if this hook is approved to run; False otherwise.
534 """
535 prompt = ('Repo %s run the script:\n'
536 ' %s\n'
537 '\n'
Jonathan Nieder71e4cea2016-08-16 12:05:09 -0700538 'Do you want to allow this script to run')
Mike Frysinger40252c22016-08-15 21:23:44 -0400539 return self._CheckForHookApprovalHelper(
540 'approvedhash',
541 self._GetHash(),
Jonathan Nieder71e4cea2016-08-16 12:05:09 -0700542 prompt % (self._GetMustVerb(), self._script_fullpath),
Mike Frysinger40252c22016-08-15 21:23:44 -0400543 'Scripts have changed since %s was allowed.' % (self._hook_type,))
544
Doug Anderson37282b42011-03-04 11:54:18 -0800545 def _ExecuteHook(self, **kwargs):
546 """Actually execute the given hook.
547
548 This will run the hook's 'main' function in our python interpreter.
549
550 Args:
551 kwargs: Keyword arguments to pass to the hook. These are often specific
552 to the hook type. For instance, pre-upload hooks will contain
553 a project_list.
554 """
555 # Keep sys.path and CWD stashed away so that we can always restore them
556 # upon function exit.
557 orig_path = os.getcwd()
558 orig_syspath = sys.path
559
560 try:
561 # Always run hooks with CWD as topdir.
562 os.chdir(self._topdir)
563
564 # Put the hook dir as the first item of sys.path so hooks can do
565 # relative imports. We want to replace the repo dir as [0] so
566 # hooks can't import repo files.
567 sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
568
569 # Exec, storing global context in the context dict. We catch exceptions
570 # and convert to a HookError w/ just the failing traceback.
Mike Frysinger4aa4b212016-03-04 15:03:00 -0500571 context = {'__file__': self._script_fullpath}
Doug Anderson37282b42011-03-04 11:54:18 -0800572 try:
Anthony King70f68902014-05-05 21:15:34 +0100573 exec(compile(open(self._script_fullpath).read(),
574 self._script_fullpath, 'exec'), context)
Doug Anderson37282b42011-03-04 11:54:18 -0800575 except Exception:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700576 raise HookError('%s\nFailed to import %s hook; see traceback above.' %
577 (traceback.format_exc(), self._hook_type))
Doug Anderson37282b42011-03-04 11:54:18 -0800578
579 # Running the script should have defined a main() function.
580 if 'main' not in context:
581 raise HookError('Missing main() in: "%s"' % self._script_fullpath)
582
Doug Anderson37282b42011-03-04 11:54:18 -0800583 # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
584 # We don't actually want hooks to define their main with this argument--
585 # it's there to remind them that their hook should always take **kwargs.
586 # For instance, a pre-upload hook should be defined like:
587 # def main(project_list, **kwargs):
588 #
589 # This allows us to later expand the API without breaking old hooks.
590 kwargs = kwargs.copy()
591 kwargs['hook_should_take_kwargs'] = True
592
593 # Call the main function in the hook. If the hook should cause the
594 # build to fail, it will raise an Exception. We'll catch that convert
595 # to a HookError w/ just the failing traceback.
596 try:
597 context['main'](**kwargs)
598 except Exception:
599 raise HookError('%s\nFailed to run main() for %s hook; see traceback '
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700600 'above.' % (traceback.format_exc(),
601 self._hook_type))
Doug Anderson37282b42011-03-04 11:54:18 -0800602 finally:
603 # Restore sys.path and CWD.
604 sys.path = orig_syspath
605 os.chdir(orig_path)
606
607 def Run(self, user_allows_all_hooks, **kwargs):
608 """Run the hook.
609
610 If the hook doesn't exist (because there is no hooks project or because
611 this particular hook is not enabled), this is a no-op.
612
613 Args:
614 user_allows_all_hooks: If True, we will never prompt about running the
615 hook--we'll just assume it's OK to run it.
616 kwargs: Keyword arguments to pass to the hook. These are often specific
617 to the hook type. For instance, pre-upload hooks will contain
618 a project_list.
619
620 Raises:
621 HookError: If there was a problem finding the hook or the user declined
622 to run a required hook (from _CheckForHookApproval).
623 """
624 # No-op if there is no hooks project or if hook is disabled.
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700625 if ((not self._hooks_project) or (self._hook_type not in
626 self._hooks_project.enabled_repo_hooks)):
Doug Anderson37282b42011-03-04 11:54:18 -0800627 return
628
629 # Bail with a nice error if we can't find the hook.
630 if not os.path.isfile(self._script_fullpath):
631 raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
632
633 # Make sure the user is OK with running the hook.
634 if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
635 return
636
637 # Run the hook with the same version of python we're using.
638 self._ExecuteHook(**kwargs)
639
640
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700641class Project(object):
Kevin Degi384b3c52014-10-16 16:02:58 -0600642 # These objects can be shared between several working trees.
643 shareable_files = ['description', 'info']
644 shareable_dirs = ['hooks', 'objects', 'rr-cache', 'svn']
645 # These objects can only be used by a single working tree.
646 working_tree_files = ['config', 'packed-refs', 'shallow']
647 working_tree_dirs = ['logs', 'refs']
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700648
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700649 def __init__(self,
650 manifest,
651 name,
652 remote,
653 gitdir,
David James8d201162013-10-11 17:03:19 -0700654 objdir,
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700655 worktree,
656 relpath,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700657 revisionExpr,
Mike Pontillod3153822012-02-28 11:53:24 -0800658 revisionId,
Anthony King7bdac712014-07-16 12:56:40 +0100659 rebase=True,
660 groups=None,
661 sync_c=False,
662 sync_s=False,
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900663 sync_tags=True,
Anthony King7bdac712014-07-16 12:56:40 +0100664 clone_depth=None,
665 upstream=None,
666 parent=None,
667 is_derived=False,
David Pursehouseb1553542014-09-04 21:28:09 +0900668 dest_branch=None,
Simran Basib9a1b732015-08-20 12:19:28 -0700669 optimized_fetch=False,
670 old_revision=None):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800671 """Init a Project object.
672
673 Args:
674 manifest: The XmlManifest object.
675 name: The `name` attribute of manifest.xml's project element.
676 remote: RemoteSpec object specifying its remote's properties.
677 gitdir: Absolute path of git directory.
David James8d201162013-10-11 17:03:19 -0700678 objdir: Absolute path of directory to store git objects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800679 worktree: Absolute path of git working tree.
680 relpath: Relative path of git working tree to repo's top directory.
681 revisionExpr: The `revision` attribute of manifest.xml's project element.
682 revisionId: git commit id for checking out.
683 rebase: The `rebase` attribute of manifest.xml's project element.
684 groups: The `groups` attribute of manifest.xml's project element.
685 sync_c: The `sync-c` attribute of manifest.xml's project element.
686 sync_s: The `sync-s` attribute of manifest.xml's project element.
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900687 sync_tags: The `sync-tags` attribute of manifest.xml's project element.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800688 upstream: The `upstream` attribute of manifest.xml's project element.
689 parent: The parent Project object.
690 is_derived: False if the project was explicitly defined in the manifest;
691 True if the project is a discovered submodule.
Bryan Jacobsf609f912013-05-06 13:36:24 -0400692 dest_branch: The branch to which to push changes for review by default.
David Pursehouseb1553542014-09-04 21:28:09 +0900693 optimized_fetch: If True, when a project is set to a sha1 revision, only
694 fetch from the remote if the sha1 is not present locally.
Simran Basib9a1b732015-08-20 12:19:28 -0700695 old_revision: saved git commit id for open GITC projects.
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800696 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700697 self.manifest = manifest
698 self.name = name
699 self.remote = remote
Anthony Newnamdf14a702011-01-09 17:31:57 -0800700 self.gitdir = gitdir.replace('\\', '/')
David James8d201162013-10-11 17:03:19 -0700701 self.objdir = objdir.replace('\\', '/')
Shawn O. Pearce0ce6ca92011-01-10 13:26:01 -0800702 if worktree:
Renaud Paquayfef9f212016-11-01 18:28:01 -0700703 self.worktree = os.path.normpath(worktree).replace('\\', '/')
Shawn O. Pearce0ce6ca92011-01-10 13:26:01 -0800704 else:
705 self.worktree = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700706 self.relpath = relpath
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700707 self.revisionExpr = revisionExpr
708
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700709 if revisionId is None \
710 and revisionExpr \
711 and IsId(revisionExpr):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700712 self.revisionId = revisionExpr
713 else:
714 self.revisionId = revisionId
715
Mike Pontillod3153822012-02-28 11:53:24 -0800716 self.rebase = rebase
Colin Cross5acde752012-03-28 20:15:45 -0700717 self.groups = groups
Anatol Pomazau79770d22012-04-20 14:41:59 -0700718 self.sync_c = sync_c
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800719 self.sync_s = sync_s
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900720 self.sync_tags = sync_tags
David Pursehouseede7f122012-11-27 22:25:30 +0900721 self.clone_depth = clone_depth
Brian Harring14a66742012-09-28 20:21:57 -0700722 self.upstream = upstream
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800723 self.parent = parent
724 self.is_derived = is_derived
David Pursehouseb1553542014-09-04 21:28:09 +0900725 self.optimized_fetch = optimized_fetch
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800726 self.subprojects = []
Mike Pontillod3153822012-02-28 11:53:24 -0800727
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700728 self.snapshots = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700729 self.copyfiles = []
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500730 self.linkfiles = []
James W. Mills24c13082012-04-12 15:04:13 -0500731 self.annotations = []
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700732 self.config = GitConfig.ForRepository(gitdir=self.gitdir,
733 defaults=self.manifest.globalConfig)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700734
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800735 if self.worktree:
David James8d201162013-10-11 17:03:19 -0700736 self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800737 else:
738 self.work_git = None
David James8d201162013-10-11 17:03:19 -0700739 self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
Shawn O. Pearced237b692009-04-17 18:49:50 -0700740 self.bare_ref = GitRefs(gitdir)
David James8d201162013-10-11 17:03:19 -0700741 self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
Bryan Jacobsf609f912013-05-06 13:36:24 -0400742 self.dest_branch = dest_branch
Simran Basib9a1b732015-08-20 12:19:28 -0700743 self.old_revision = old_revision
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700744
Doug Anderson37282b42011-03-04 11:54:18 -0800745 # This will be filled in if a project is later identified to be the
746 # project containing repo hooks.
747 self.enabled_repo_hooks = []
748
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700749 @property
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800750 def Derived(self):
751 return self.is_derived
752
753 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700754 def Exists(self):
Renaud Paquaybed8b622018-09-27 10:46:58 -0700755 return platform_utils.isdir(self.gitdir) and platform_utils.isdir(self.objdir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700756
757 @property
758 def CurrentBranch(self):
759 """Obtain the name of the currently checked out branch.
760 The branch name omits the 'refs/heads/' prefix.
761 None is returned if the project is on a detached HEAD.
762 """
Shawn O. Pearce5b23f242009-04-17 18:43:33 -0700763 b = self.work_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700764 if b.startswith(R_HEADS):
765 return b[len(R_HEADS):]
766 return None
767
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700768 def IsRebaseInProgress(self):
769 w = self.worktree
770 g = os.path.join(w, '.git')
771 return os.path.exists(os.path.join(g, 'rebase-apply')) \
772 or os.path.exists(os.path.join(g, 'rebase-merge')) \
773 or os.path.exists(os.path.join(w, '.dotest'))
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200774
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700775 def IsDirty(self, consider_untracked=True):
776 """Is the working directory modified in some way?
777 """
778 self.work_git.update_index('-q',
779 '--unmerged',
780 '--ignore-missing',
781 '--refresh')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900782 if self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700783 return True
784 if self.work_git.DiffZ('diff-files'):
785 return True
786 if consider_untracked and self.work_git.LsOthers():
787 return True
788 return False
789
790 _userident_name = None
791 _userident_email = None
792
793 @property
794 def UserName(self):
795 """Obtain the user's personal name.
796 """
797 if self._userident_name is None:
798 self._LoadUserIdentity()
799 return self._userident_name
800
801 @property
802 def UserEmail(self):
803 """Obtain the user's email address. This is very likely
804 to be their Gerrit login.
805 """
806 if self._userident_email is None:
807 self._LoadUserIdentity()
808 return self._userident_email
809
810 def _LoadUserIdentity(self):
David Pursehousec1b86a22012-11-14 11:36:51 +0900811 u = self.bare_git.var('GIT_COMMITTER_IDENT')
812 m = re.compile("^(.*) <([^>]*)> ").match(u)
813 if m:
814 self._userident_name = m.group(1)
815 self._userident_email = m.group(2)
816 else:
817 self._userident_name = ''
818 self._userident_email = ''
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700819
820 def GetRemote(self, name):
821 """Get the configuration for a single remote.
822 """
823 return self.config.GetRemote(name)
824
825 def GetBranch(self, name):
826 """Get the configuration for a single branch.
827 """
828 return self.config.GetBranch(name)
829
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700830 def GetBranches(self):
831 """Get all existing local branches.
832 """
833 current = self.CurrentBranch
David Pursehouse8a68ff92012-09-24 12:15:13 +0900834 all_refs = self._allrefs
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700835 heads = {}
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700836
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530837 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700838 if name.startswith(R_HEADS):
839 name = name[len(R_HEADS):]
840 b = self.GetBranch(name)
841 b.current = name == current
842 b.published = None
David Pursehouse8a68ff92012-09-24 12:15:13 +0900843 b.revision = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700844 heads[name] = b
845
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530846 for name, ref_id in all_refs.items():
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700847 if name.startswith(R_PUB):
848 name = name[len(R_PUB):]
849 b = heads.get(name)
850 if b:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900851 b.published = ref_id
Shawn O. Pearce27b07322009-04-10 16:02:48 -0700852
853 return heads
854
Colin Cross5acde752012-03-28 20:15:45 -0700855 def MatchesGroups(self, manifest_groups):
856 """Returns true if the manifest groups specified at init should cause
857 this project to be synced.
858 Prefixing a manifest group with "-" inverts the meaning of a group.
Conley Owensbb1b5f52012-08-13 13:11:18 -0700859 All projects are implicitly labelled with "all".
Conley Owens971de8e2012-04-16 10:36:08 -0700860
861 labels are resolved in order. In the example case of
Conley Owensbb1b5f52012-08-13 13:11:18 -0700862 project_groups: "all,group1,group2"
Conley Owens971de8e2012-04-16 10:36:08 -0700863 manifest_groups: "-group1,group2"
864 the project will be matched.
David Holmer0a1c6a12012-11-14 19:19:00 -0500865
866 The special manifest group "default" will match any project that
867 does not have the special project group "notdefault"
Colin Cross5acde752012-03-28 20:15:45 -0700868 """
David Holmer0a1c6a12012-11-14 19:19:00 -0500869 expanded_manifest_groups = manifest_groups or ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700870 expanded_project_groups = ['all'] + (self.groups or [])
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700871 if 'notdefault' not in expanded_project_groups:
David Holmer0a1c6a12012-11-14 19:19:00 -0500872 expanded_project_groups += ['default']
Conley Owensbb1b5f52012-08-13 13:11:18 -0700873
Conley Owens971de8e2012-04-16 10:36:08 -0700874 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700875 for group in expanded_manifest_groups:
876 if group.startswith('-') and group[1:] in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700877 matched = False
Conley Owensbb1b5f52012-08-13 13:11:18 -0700878 elif group in expanded_project_groups:
Conley Owens971de8e2012-04-16 10:36:08 -0700879 matched = True
Colin Cross5acde752012-03-28 20:15:45 -0700880
Conley Owens971de8e2012-04-16 10:36:08 -0700881 return matched
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700882
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700883# Status Display ##
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700884 def UncommitedFiles(self, get_all=True):
885 """Returns a list of strings, uncommitted files in the git tree.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700886
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700887 Args:
888 get_all: a boolean, if True - get information about all different
889 uncommitted files. If False - return as soon as any kind of
890 uncommitted files is detected.
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500891 """
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700892 details = []
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500893 self.work_git.update_index('-q',
894 '--unmerged',
895 '--ignore-missing',
896 '--refresh')
897 if self.IsRebaseInProgress():
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700898 details.append("rebase in progress")
899 if not get_all:
900 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500901
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700902 changes = self.work_git.DiffZ('diff-index', '--cached', HEAD).keys()
903 if changes:
904 details.extend(changes)
905 if not get_all:
906 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500907
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700908 changes = self.work_git.DiffZ('diff-files').keys()
909 if changes:
910 details.extend(changes)
911 if not get_all:
912 return details
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500913
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700914 changes = self.work_git.LsOthers()
915 if changes:
916 details.extend(changes)
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500917
Vadim Bendebury14e134d2014-10-05 15:40:30 -0700918 return details
919
920 def HasChanges(self):
921 """Returns true if there are uncommitted changes.
922 """
923 if self.UncommitedFiles(get_all=False):
924 return True
925 else:
926 return False
Anthony Newnamcc50bac2010-04-08 10:28:59 -0500927
Andrew Wheeler4d5bb682012-02-27 13:52:22 -0600928 def PrintWorkTreeStatus(self, output_redir=None, quiet=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700929 """Prints the status of the repository to stdout.
Terence Haddock4655e812011-03-31 12:33:34 +0200930
931 Args:
932 output: If specified, redirect the output to this object.
Andrew Wheeler4d5bb682012-02-27 13:52:22 -0600933 quiet: If True then only print the project name. Do not print
934 the modified files, branch name, etc.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700935 """
Renaud Paquaybed8b622018-09-27 10:46:58 -0700936 if not platform_utils.isdir(self.worktree):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700937 if output_redir is None:
Terence Haddock4655e812011-03-31 12:33:34 +0200938 output_redir = sys.stdout
Sarah Owenscecd1d82012-11-01 22:59:27 -0700939 print(file=output_redir)
940 print('project %s/' % self.relpath, file=output_redir)
941 print(' missing (run "repo sync")', file=output_redir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700942 return
943
944 self.work_git.update_index('-q',
945 '--unmerged',
946 '--ignore-missing',
947 '--refresh')
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700948 rb = self.IsRebaseInProgress()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700949 di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
950 df = self.work_git.DiffZ('diff-files')
951 do = self.work_git.LsOthers()
Ali Utku Selen76abcc12012-01-25 10:51:12 +0100952 if not rb and not di and not df and not do and not self.CurrentBranch:
Shawn O. Pearce161f4452009-04-10 17:41:44 -0700953 return 'CLEAN'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700954
955 out = StatusColoring(self.config)
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -0700956 if output_redir is not None:
Terence Haddock4655e812011-03-31 12:33:34 +0200957 out.redirect(output_redir)
Jakub Vrana0402cd82014-09-09 15:39:15 -0700958 out.project('project %-40s', self.relpath + '/ ')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700959
Andrew Wheeler4d5bb682012-02-27 13:52:22 -0600960 if quiet:
961 out.nl()
962 return 'DIRTY'
963
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700964 branch = self.CurrentBranch
965 if branch is None:
966 out.nobranch('(*** NO BRANCH ***)')
967 else:
968 out.branch('branch %s', branch)
969 out.nl()
970
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -0700971 if rb:
972 out.important('prior sync failed; rebase still in progress')
973 out.nl()
974
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700975 paths = list()
976 paths.extend(di.keys())
977 paths.extend(df.keys())
978 paths.extend(do)
979
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530980 for p in sorted(set(paths)):
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900981 try:
982 i = di[p]
983 except KeyError:
984 i = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700985
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900986 try:
987 f = df[p]
988 except KeyError:
989 f = None
Julius Gustavsson0cb1b3f2010-06-17 17:55:02 +0200990
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900991 if i:
992 i_status = i.status.upper()
993 else:
994 i_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700995
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900996 if f:
997 f_status = f.status.lower()
998 else:
999 f_status = '-'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001000
1001 if i and i.src_path:
Shawn O. Pearcefe086752009-03-03 13:49:48 -08001002 line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001003 i.src_path, p, i.level)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001004 else:
1005 line = ' %s%s\t%s' % (i_status, f_status, p)
1006
1007 if i and not f:
1008 out.added('%s', line)
1009 elif (i and f) or (not i and f):
1010 out.changed('%s', line)
1011 elif not i and not f:
1012 out.untracked('%s', line)
1013 else:
1014 out.write('%s', line)
1015 out.nl()
Terence Haddock4655e812011-03-31 12:33:34 +02001016
Shawn O. Pearce161f4452009-04-10 17:41:44 -07001017 return 'DIRTY'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001018
pelyad67872d2012-03-28 14:49:58 +03001019 def PrintWorkTreeDiff(self, absolute_paths=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001020 """Prints the status of the repository to stdout.
1021 """
1022 out = DiffColoring(self.config)
1023 cmd = ['diff']
1024 if out.is_on:
1025 cmd.append('--color')
1026 cmd.append(HEAD)
pelyad67872d2012-03-28 14:49:58 +03001027 if absolute_paths:
1028 cmd.append('--src-prefix=a/%s/' % self.relpath)
1029 cmd.append('--dst-prefix=b/%s/' % self.relpath)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001030 cmd.append('--')
1031 p = GitCommand(self,
1032 cmd,
Anthony King7bdac712014-07-16 12:56:40 +01001033 capture_stdout=True,
1034 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001035 has_diff = False
1036 for line in p.process.stdout:
1037 if not has_diff:
1038 out.nl()
1039 out.project('project %s/' % self.relpath)
1040 out.nl()
1041 has_diff = True
Sarah Owenscecd1d82012-11-01 22:59:27 -07001042 print(line[:-1])
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001043 p.Wait()
1044
1045
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001046# Publish / Upload ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001047
David Pursehouse8a68ff92012-09-24 12:15:13 +09001048 def WasPublished(self, branch, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001049 """Was the branch published (uploaded) for code review?
1050 If so, returns the SHA-1 hash of the last published
1051 state for the branch.
1052 """
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001053 key = R_PUB + branch
David Pursehouse8a68ff92012-09-24 12:15:13 +09001054 if all_refs is None:
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001055 try:
1056 return self.bare_git.rev_parse(key)
1057 except GitError:
1058 return None
1059 else:
1060 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001061 return all_refs[key]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001062 except KeyError:
1063 return None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001064
David Pursehouse8a68ff92012-09-24 12:15:13 +09001065 def CleanPublishedCache(self, all_refs=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001066 """Prunes any stale published refs.
1067 """
David Pursehouse8a68ff92012-09-24 12:15:13 +09001068 if all_refs is None:
1069 all_refs = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001070 heads = set()
1071 canrm = {}
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301072 for name, ref_id in all_refs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001073 if name.startswith(R_HEADS):
1074 heads.add(name)
1075 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001076 canrm[name] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001077
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301078 for name, ref_id in canrm.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001079 n = name[len(R_PUB):]
1080 if R_HEADS + n not in heads:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001081 self.bare_git.DeleteRef(name, ref_id)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001082
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -07001083 def GetUploadableBranches(self, selected_branch=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001084 """List any branches which can be uploaded for review.
1085 """
1086 heads = {}
1087 pubed = {}
1088
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301089 for name, ref_id in self._allrefs.items():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001090 if name.startswith(R_HEADS):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001091 heads[name[len(R_HEADS):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001092 elif name.startswith(R_PUB):
David Pursehouse8a68ff92012-09-24 12:15:13 +09001093 pubed[name[len(R_PUB):]] = ref_id
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001094
1095 ready = []
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301096 for branch, ref_id in heads.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09001097 if branch in pubed and pubed[branch] == ref_id:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001098 continue
Mandeep Singh Bainesd6c93a22011-05-26 10:34:11 -07001099 if selected_branch and branch != selected_branch:
1100 continue
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001101
Shawn O. Pearce35f25962008-11-11 17:03:13 -08001102 rb = self.GetUploadableBranch(branch)
1103 if rb:
1104 ready.append(rb)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001105 return ready
1106
Shawn O. Pearce35f25962008-11-11 17:03:13 -08001107 def GetUploadableBranch(self, branch_name):
1108 """Get a single uploadable branch, or None.
1109 """
1110 branch = self.GetBranch(branch_name)
1111 base = branch.LocalMerge
1112 if branch.LocalMerge:
1113 rb = ReviewableBranch(self, branch, base)
1114 if rb.commits:
1115 return rb
1116 return None
1117
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001118 def UploadForReview(self, branch=None,
Anthony King7bdac712014-07-16 12:56:40 +01001119 people=([], []),
Brian Harring435370c2012-07-28 15:37:04 -07001120 auto_topic=False,
Jonathan Niederc94d6eb2017-08-08 18:34:53 +00001121 draft=False,
Changcheng Xiao87984c62017-08-02 16:55:03 +02001122 private=False,
Vadim Bendeburybd8f6582018-10-31 13:48:01 -07001123 notify=None,
Changcheng Xiao87984c62017-08-02 16:55:03 +02001124 wip=False,
Łukasz Gardońbed59ce2017-08-08 10:18:11 +02001125 dest_branch=None,
Masaya Suzuki305a2d02017-11-13 10:48:34 -08001126 validate_certs=True,
1127 push_options=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001128 """Uploads the named branch for code review.
1129 """
1130 if branch is None:
1131 branch = self.CurrentBranch
1132 if branch is None:
1133 raise GitError('not currently on a branch')
1134
1135 branch = self.GetBranch(branch)
1136 if not branch.LocalMerge:
1137 raise GitError('branch %s does not track a remote' % branch.name)
1138 if not branch.remote.review:
1139 raise GitError('remote %s has no review url' % branch.remote.name)
1140
Bryan Jacobsf609f912013-05-06 13:36:24 -04001141 if dest_branch is None:
1142 dest_branch = self.dest_branch
1143 if dest_branch is None:
1144 dest_branch = branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001145 if not dest_branch.startswith(R_HEADS):
1146 dest_branch = R_HEADS + dest_branch
1147
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -08001148 if not branch.remote.projectname:
1149 branch.remote.projectname = self.name
1150 branch.remote.Save()
1151
Łukasz Gardońbed59ce2017-08-08 10:18:11 +02001152 url = branch.remote.ReviewUrl(self.UserEmail, validate_certs)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001153 if url is None:
1154 raise UploadError('review not configured')
1155 cmd = ['push']
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001156
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001157 if url.startswith('ssh://'):
Jonathan Nieder713c5872018-11-05 13:21:52 -08001158 cmd.append('--receive-pack=gerrit receive-pack')
Shawn O. Pearcea5ece0e2010-07-15 16:52:42 -07001159
Masaya Suzuki305a2d02017-11-13 10:48:34 -08001160 for push_option in (push_options or []):
1161 cmd.append('-o')
1162 cmd.append(push_option)
1163
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001164 cmd.append(url)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -08001165
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001166 if dest_branch.startswith(R_HEADS):
1167 dest_branch = dest_branch[len(R_HEADS):]
Brian Harring435370c2012-07-28 15:37:04 -07001168
Jonathan Niederc94d6eb2017-08-08 18:34:53 +00001169 upload_type = 'for'
1170 if draft:
1171 upload_type = 'drafts'
1172
1173 ref_spec = '%s:refs/%s/%s' % (R_HEADS + branch.name, upload_type,
1174 dest_branch)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001175 if auto_topic:
1176 ref_spec = ref_spec + '/' + branch.name
Changcheng Xiao87984c62017-08-02 16:55:03 +02001177
Jonathan Nieder713c5872018-11-05 13:21:52 -08001178 opts = ['r=%s' % p for p in people[0]]
1179 opts += ['cc=%s' % p for p in people[1]]
Vadim Bendeburybd8f6582018-10-31 13:48:01 -07001180 if notify:
1181 opts += ['notify=' + notify]
Jonathan Nieder713c5872018-11-05 13:21:52 -08001182 if private:
1183 opts += ['private']
1184 if wip:
1185 opts += ['wip']
1186 if opts:
1187 ref_spec = ref_spec + '%' + ','.join(opts)
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001188 cmd.append(ref_spec)
1189
Anthony King7bdac712014-07-16 12:56:40 +01001190 if GitCommand(self, cmd, bare=True).Wait() != 0:
Shawn O. Pearcec9571422012-01-11 14:58:54 -08001191 raise UploadError('Upload failed')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001192
1193 msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
1194 self.bare_git.UpdateRef(R_PUB + branch.name,
1195 R_HEADS + branch.name,
Anthony King7bdac712014-07-16 12:56:40 +01001196 message=msg)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001197
1198
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001199# Sync ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001200
Julien Campergue335f5ef2013-10-16 11:02:35 +02001201 def _ExtractArchive(self, tarpath, path=None):
1202 """Extract the given tar on its current location
1203
1204 Args:
1205 - tarpath: The path to the actual tar file
1206
1207 """
1208 try:
1209 with tarfile.open(tarpath, 'r') as tar:
1210 tar.extractall(path=path)
1211 return True
1212 except (IOError, tarfile.TarError) as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001213 _error("Cannot extract archive %s: %s", tarpath, str(e))
Julien Campergue335f5ef2013-10-16 11:02:35 +02001214 return False
1215
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001216 def Sync_NetworkHalf(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001217 quiet=False,
1218 is_new=None,
1219 current_branch_only=False,
1220 force_sync=False,
1221 clone_bundle=True,
1222 no_tags=False,
1223 archive=False,
1224 optimized_fetch=False,
Martin Kellye4e94d22017-03-21 16:05:12 -07001225 prune=False,
1226 submodules=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001227 """Perform only the network IO portion of the sync process.
1228 Local working directory/branch state is not affected.
1229 """
Julien Campergue335f5ef2013-10-16 11:02:35 +02001230 if archive and not isinstance(self, MetaProject):
1231 if self.remote.url.startswith(('http://', 'https://')):
David Pursehousef33929d2015-08-24 14:39:14 +09001232 _error("%s: Cannot fetch archives from http/https remotes.", self.name)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001233 return False
1234
1235 name = self.relpath.replace('\\', '/')
1236 name = name.replace('/', '_')
1237 tarpath = '%s.tar' % name
1238 topdir = self.manifest.topdir
1239
1240 try:
1241 self._FetchArchive(tarpath, cwd=topdir)
1242 except GitError as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001243 _error('%s', e)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001244 return False
1245
1246 # From now on, we only need absolute tarpath
1247 tarpath = os.path.join(topdir, tarpath)
1248
1249 if not self._ExtractArchive(tarpath, path=topdir):
1250 return False
1251 try:
Renaud Paquay010fed72016-11-11 14:25:29 -08001252 platform_utils.remove(tarpath)
Julien Campergue335f5ef2013-10-16 11:02:35 +02001253 except OSError as e:
David Pursehousef33929d2015-08-24 14:39:14 +09001254 _warn("Cannot remove archive %s: %s", tarpath, str(e))
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001255 self._CopyAndLinkFiles()
Julien Campergue335f5ef2013-10-16 11:02:35 +02001256 return True
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07001257 if is_new is None:
1258 is_new = not self.Exists
Shawn O. Pearce88443382010-10-08 10:02:09 +02001259 if is_new:
Kevin Degiabaa7f32014-11-12 11:27:45 -07001260 self._InitGitDir(force_sync=force_sync)
Jimmie Westera0444582012-10-24 13:44:42 +02001261 else:
1262 self._UpdateHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001263 self._InitRemote()
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001264
1265 if is_new:
1266 alt = os.path.join(self.gitdir, 'objects/info/alternates')
1267 try:
Renaud Paquay2a4be942016-11-01 13:48:15 -07001268 fd = open(alt)
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001269 try:
Samuel Hollandbaa00092018-01-22 10:57:29 -06001270 # This works for both absolute and relative alternate directories.
1271 alt_dir = os.path.join(self.objdir, 'objects', fd.readline().rstrip())
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001272 finally:
1273 fd.close()
1274 except IOError:
1275 alt_dir = None
1276 else:
1277 alt_dir = None
1278
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -07001279 if clone_bundle \
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001280 and alt_dir is None \
1281 and self._ApplyCloneBundle(initial=is_new, quiet=quiet):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001282 is_new = False
1283
Shawn O. Pearce6ba6ba02012-05-24 09:46:50 -07001284 if not current_branch_only:
1285 if self.sync_c:
1286 current_branch_only = True
1287 elif not self.manifest._loaded:
1288 # Manifest cannot check defaults until it syncs.
1289 current_branch_only = False
1290 elif self.manifest.default.sync_c:
1291 current_branch_only = True
1292
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +09001293 if not no_tags:
1294 if not self.sync_tags:
1295 no_tags = True
1296
Aymen Bouaziz6c594462016-10-25 18:03:51 +02001297 if self.clone_depth:
1298 depth = self.clone_depth
1299 else:
1300 depth = self.manifest.manifestProject.config.GetString('repo.depth')
1301
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001302 need_to_fetch = not (optimized_fetch and
1303 (ID_RE.match(self.revisionExpr) and
Zac Livingstone4332262017-06-16 08:56:09 -06001304 self._CheckForImmutableRevision()))
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001305 if (need_to_fetch and
1306 not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir,
1307 current_branch_only=current_branch_only,
Martin Kellye4e94d22017-03-21 16:05:12 -07001308 no_tags=no_tags, prune=prune, depth=depth,
1309 submodules=submodules)):
Anthony King7bdac712014-07-16 12:56:40 +01001310 return False
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001311
Nikolai Merinov09f0abb2018-10-19 15:07:05 +05001312 mp = self.manifest.manifestProject
1313 dissociate = mp.config.GetBoolean('repo.dissociate')
1314 if dissociate:
1315 alternates_file = os.path.join(self.gitdir, 'objects/info/alternates')
1316 if os.path.exists(alternates_file):
1317 cmd = ['repack', '-a', '-d']
1318 if GitCommand(self, cmd, bare=True).Wait() != 0:
1319 return False
1320 platform_utils.remove(alternates_file)
1321
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001322 if self.worktree:
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001323 self._InitMRef()
1324 else:
1325 self._InitMirrorHead()
1326 try:
Renaud Paquay010fed72016-11-11 14:25:29 -08001327 platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001328 except OSError:
1329 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001330 return True
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08001331
1332 def PostRepoUpgrade(self):
1333 self._InitHooks()
1334
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001335 def _CopyAndLinkFiles(self):
Simran Basib9a1b732015-08-20 12:19:28 -07001336 if self.manifest.isGitcClient:
1337 return
David Pursehouse8a68ff92012-09-24 12:15:13 +09001338 for copyfile in self.copyfiles:
1339 copyfile._Copy()
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001340 for linkfile in self.linkfiles:
1341 linkfile._Link()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001342
Julien Camperguedd654222014-01-09 16:21:37 +01001343 def GetCommitRevisionId(self):
1344 """Get revisionId of a commit.
1345
1346 Use this method instead of GetRevisionId to get the id of the commit rather
1347 than the id of the current git object (for example, a tag)
1348
1349 """
1350 if not self.revisionExpr.startswith(R_TAGS):
1351 return self.GetRevisionId(self._allrefs)
1352
1353 try:
1354 return self.bare_git.rev_list(self.revisionExpr, '-1')[0]
1355 except GitError:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001356 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1357 (self.revisionExpr, self.name))
Julien Camperguedd654222014-01-09 16:21:37 +01001358
David Pursehouse8a68ff92012-09-24 12:15:13 +09001359 def GetRevisionId(self, all_refs=None):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001360 if self.revisionId:
1361 return self.revisionId
1362
1363 rem = self.GetRemote(self.remote.name)
1364 rev = rem.ToLocal(self.revisionExpr)
1365
David Pursehouse8a68ff92012-09-24 12:15:13 +09001366 if all_refs is not None and rev in all_refs:
1367 return all_refs[rev]
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001368
1369 try:
1370 return self.bare_git.rev_parse('--verify', '%s^0' % rev)
1371 except GitError:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001372 raise ManifestInvalidRevisionError('revision %s in %s not found' %
1373 (self.revisionExpr, self.name))
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001374
Martin Kellye4e94d22017-03-21 16:05:12 -07001375 def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001376 """Perform only the local IO portion of the sync process.
1377 Network access is not required.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001378 """
Martin Kellye4e94d22017-03-21 16:05:12 -07001379 self._InitWorkTree(force_sync=force_sync, submodules=submodules)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001380 all_refs = self.bare_ref.all
1381 self.CleanPublishedCache(all_refs)
1382 revid = self.GetRevisionId(all_refs)
Skyler Kaufman835cd682011-03-08 12:14:41 -08001383
David Pursehouse1d947b32012-10-25 12:23:11 +09001384 def _doff():
1385 self._FastForward(revid)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001386 self._CopyAndLinkFiles()
David Pursehouse1d947b32012-10-25 12:23:11 +09001387
Martin Kellye4e94d22017-03-21 16:05:12 -07001388 def _dosubmodules():
1389 self._SyncSubmodules(quiet=True)
1390
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001391 head = self.work_git.GetHead()
1392 if head.startswith(R_HEADS):
1393 branch = head[len(R_HEADS):]
1394 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001395 head = all_refs[head]
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001396 except KeyError:
1397 head = None
1398 else:
1399 branch = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001400
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001401 if branch is None or syncbuf.detach_head:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001402 # Currently on a detached HEAD. The user is assumed to
1403 # not have any local modifications worth worrying about.
1404 #
Shawn O. Pearce3d2cdd02009-04-18 15:26:10 -07001405 if self.IsRebaseInProgress():
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001406 syncbuf.fail(self, _PriorSyncFailedError())
1407 return
1408
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001409 if head == revid:
1410 # No changes; don't do anything further.
Florian Vallee7cf1b362012-06-07 17:11:42 +02001411 # Except if the head needs to be detached
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001412 #
Florian Vallee7cf1b362012-06-07 17:11:42 +02001413 if not syncbuf.detach_head:
Dan Willemsen029eaf32015-09-03 12:52:28 -07001414 # The copy/linkfile config may have changed.
1415 self._CopyAndLinkFiles()
Florian Vallee7cf1b362012-06-07 17:11:42 +02001416 return
1417 else:
1418 lost = self._revlist(not_rev(revid), HEAD)
1419 if lost:
1420 syncbuf.info(self, "discarding %d commits", len(lost))
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001421
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001422 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001423 self._Checkout(revid, quiet=True)
Martin Kellye4e94d22017-03-21 16:05:12 -07001424 if submodules:
1425 self._SyncSubmodules(quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001426 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001427 syncbuf.fail(self, e)
1428 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001429 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001430 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001431
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001432 if head == revid:
1433 # No changes; don't do anything further.
1434 #
Dan Willemsen029eaf32015-09-03 12:52:28 -07001435 # The copy/linkfile config may have changed.
1436 self._CopyAndLinkFiles()
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07001437 return
1438
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001439 branch = self.GetBranch(branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001440
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001441 if not branch.LocalMerge:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001442 # The current branch has no tracking configuration.
Anatol Pomazau2a32f6a2011-08-30 10:52:33 -07001443 # Jump off it to a detached HEAD.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001444 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001445 syncbuf.info(self,
1446 "leaving %s; does not track upstream",
1447 branch.name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001448 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001449 self._Checkout(revid, quiet=True)
Martin Kellye4e94d22017-03-21 16:05:12 -07001450 if submodules:
1451 self._SyncSubmodules(quiet=True)
Sarah Owensa5be53f2012-09-09 15:37:57 -07001452 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001453 syncbuf.fail(self, e)
1454 return
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001455 self._CopyAndLinkFiles()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001456 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001457
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001458 upstream_gain = self._revlist(not_rev(HEAD), revid)
David Pursehouse8a68ff92012-09-24 12:15:13 +09001459 pub = self.WasPublished(branch.name, all_refs)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001460 if pub:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001461 not_merged = self._revlist(not_rev(revid), pub)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001462 if not_merged:
1463 if upstream_gain:
1464 # The user has published this branch and some of those
1465 # commits are not yet merged upstream. We do not want
1466 # to rewrite the published commits so we punt.
1467 #
Daniel Sandler4c50dee2010-03-02 15:38:03 -05001468 syncbuf.fail(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001469 "branch %s is published (but not merged) and is now "
1470 "%d commits behind" % (branch.name, len(upstream_gain)))
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001471 return
Shawn O. Pearce05f66b62009-04-21 08:26:32 -07001472 elif pub == head:
1473 # All published commits are merged, and thus we are a
1474 # strict subset. We can fast-forward safely.
Shawn O. Pearcea54c5272008-10-30 11:03:00 -07001475 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001476 syncbuf.later1(self, _doff)
Martin Kellye4e94d22017-03-21 16:05:12 -07001477 if submodules:
1478 syncbuf.later1(self, _dosubmodules)
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001479 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001480
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001481 # Examine the local commits not in the remote. Find the
1482 # last one attributed to this user, if any.
1483 #
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001484 local_changes = self._revlist(not_rev(revid), HEAD, format='%H %ce')
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001485 last_mine = None
1486 cnt_mine = 0
1487 for commit in local_changes:
Chirayu Desai0eb35cb2013-11-19 18:46:29 +05301488 commit_id, committer_email = commit.decode('utf-8').split(' ', 1)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001489 if committer_email == self.UserEmail:
1490 last_mine = commit_id
1491 cnt_mine += 1
1492
Shawn O. Pearceda88ff42009-06-03 11:09:12 -07001493 if not upstream_gain and cnt_mine == len(local_changes):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001494 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001495
1496 if self.IsDirty(consider_untracked=False):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001497 syncbuf.fail(self, _DirtyError())
1498 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001499
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001500 # If the upstream switched on us, warn the user.
1501 #
1502 if branch.merge != self.revisionExpr:
1503 if branch.merge and self.revisionExpr:
1504 syncbuf.info(self,
1505 'manifest switched %s...%s',
1506 branch.merge,
1507 self.revisionExpr)
1508 elif branch.merge:
1509 syncbuf.info(self,
1510 'manifest no longer tracks %s',
1511 branch.merge)
1512
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001513 if cnt_mine < len(local_changes):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001514 # Upstream rebased. Not everything in HEAD
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001515 # was created by this user.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001516 #
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001517 syncbuf.info(self,
1518 "discarding %d commits removed from upstream",
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001519 len(local_changes) - cnt_mine)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001520
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001521 branch.remote = self.GetRemote(self.remote.name)
Anatol Pomazaucd7c5de2012-03-20 13:45:00 -07001522 if not ID_RE.match(self.revisionExpr):
1523 # in case of manifest sync the revisionExpr might be a SHA1
1524 branch.merge = self.revisionExpr
Conley Owens04f2f0e2014-10-01 17:22:46 -07001525 if not branch.merge.startswith('refs/'):
1526 branch.merge = R_HEADS + branch.merge
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001527 branch.Save()
1528
Mike Pontillod3153822012-02-28 11:53:24 -08001529 if cnt_mine > 0 and self.rebase:
Martin Kellye4e94d22017-03-21 16:05:12 -07001530 def _docopyandlink():
1531 self._CopyAndLinkFiles()
1532
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001533 def _dorebase():
Anthony King7bdac712014-07-16 12:56:40 +01001534 self._Rebase(upstream='%s^1' % last_mine, onto=revid)
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001535 syncbuf.later2(self, _dorebase)
Martin Kellye4e94d22017-03-21 16:05:12 -07001536 if submodules:
1537 syncbuf.later2(self, _dosubmodules)
1538 syncbuf.later2(self, _docopyandlink)
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07001539 elif local_changes:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001540 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001541 self._ResetHard(revid)
Martin Kellye4e94d22017-03-21 16:05:12 -07001542 if submodules:
1543 self._SyncSubmodules(quiet=True)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001544 self._CopyAndLinkFiles()
Sarah Owensa5be53f2012-09-09 15:37:57 -07001545 except GitError as e:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001546 syncbuf.fail(self, e)
1547 return
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001548 else:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001549 syncbuf.later1(self, _doff)
Martin Kellye4e94d22017-03-21 16:05:12 -07001550 if submodules:
1551 syncbuf.later1(self, _dosubmodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001552
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001553 def AddCopyFile(self, src, dest, absdest):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001554 # dest should already be an absolute path, but src is project relative
1555 # make src an absolute path
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -08001556 abssrc = os.path.join(self.worktree, src)
1557 self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001558
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001559 def AddLinkFile(self, src, dest, absdest):
1560 # dest should already be an absolute path, but src is project relative
Colin Cross0184dcc2015-05-05 00:24:54 -07001561 # make src relative path to dest
1562 absdestdir = os.path.dirname(absdest)
1563 relsrc = os.path.relpath(os.path.join(self.worktree, src), absdestdir)
Wink Saville4c426ef2015-06-03 08:05:17 -07001564 self.linkfiles.append(_LinkFile(self.worktree, src, dest, relsrc, absdest))
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001565
James W. Mills24c13082012-04-12 15:04:13 -05001566 def AddAnnotation(self, name, value, keep):
1567 self.annotations.append(_Annotation(name, value, keep))
1568
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001569 def DownloadPatchSet(self, change_id, patch_id):
1570 """Download a single patch set of a single change to FETCH_HEAD.
1571 """
1572 remote = self.GetRemote(self.remote.name)
1573
1574 cmd = ['fetch', remote.name]
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001575 cmd.append('refs/changes/%2.2d/%d/%d'
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001576 % (change_id % 100, change_id, patch_id))
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001577 if GitCommand(self, cmd, bare=True).Wait() != 0:
1578 return None
1579 return DownloadedChange(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001580 self.GetRevisionId(),
Shawn O. Pearce632768b2008-10-23 11:58:52 -07001581 change_id,
1582 patch_id,
1583 self.bare_git.rev_parse('FETCH_HEAD'))
1584
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001585
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001586# Branch Management ##
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001587
Simran Basib9a1b732015-08-20 12:19:28 -07001588 def StartBranch(self, name, branch_merge=''):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001589 """Create a new branch off the manifest's revision.
1590 """
Simran Basib9a1b732015-08-20 12:19:28 -07001591 if not branch_merge:
1592 branch_merge = self.revisionExpr
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001593 head = self.work_git.GetHead()
1594 if head == (R_HEADS + name):
1595 return True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001596
David Pursehouse8a68ff92012-09-24 12:15:13 +09001597 all_refs = self.bare_ref.all
Anthony King7bdac712014-07-16 12:56:40 +01001598 if R_HEADS + name in all_refs:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001599 return GitCommand(self,
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001600 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001601 capture_stdout=True,
1602 capture_stderr=True).Wait() == 0
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001603
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001604 branch = self.GetBranch(name)
1605 branch.remote = self.GetRemote(self.remote.name)
Simran Basib9a1b732015-08-20 12:19:28 -07001606 branch.merge = branch_merge
1607 if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
1608 branch.merge = R_HEADS + branch_merge
David Pursehouse8a68ff92012-09-24 12:15:13 +09001609 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce0a389e92009-04-10 16:21:18 -07001610
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001611 if head.startswith(R_HEADS):
1612 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001613 head = all_refs[head]
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001614 except KeyError:
1615 head = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001616 if revid and head and revid == head:
1617 ref = os.path.join(self.gitdir, R_HEADS + name)
1618 try:
1619 os.makedirs(os.path.dirname(ref))
1620 except OSError:
1621 pass
1622 _lwrite(ref, '%s\n' % revid)
1623 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1624 'ref: %s%s\n' % (R_HEADS, name))
1625 branch.Save()
1626 return True
1627
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001628 if GitCommand(self,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001629 ['checkout', '-b', branch.name, revid],
Anthony King7bdac712014-07-16 12:56:40 +01001630 capture_stdout=True,
1631 capture_stderr=True).Wait() == 0:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -07001632 branch.Save()
1633 return True
1634 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001635
Wink Saville02d79452009-04-10 13:01:24 -07001636 def CheckoutBranch(self, name):
1637 """Checkout a local topic branch.
Doug Anderson3ba5f952011-04-07 12:51:04 -07001638
1639 Args:
1640 name: The name of the branch to checkout.
1641
1642 Returns:
1643 True if the checkout succeeded; False if it didn't; None if the branch
1644 didn't exist.
Wink Saville02d79452009-04-10 13:01:24 -07001645 """
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001646 rev = R_HEADS + name
1647 head = self.work_git.GetHead()
1648 if head == rev:
1649 # Already on the branch
1650 #
1651 return True
Wink Saville02d79452009-04-10 13:01:24 -07001652
David Pursehouse8a68ff92012-09-24 12:15:13 +09001653 all_refs = self.bare_ref.all
Wink Saville02d79452009-04-10 13:01:24 -07001654 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001655 revid = all_refs[rev]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001656 except KeyError:
1657 # Branch does not exist in this project
1658 #
Doug Anderson3ba5f952011-04-07 12:51:04 -07001659 return None
Wink Saville02d79452009-04-10 13:01:24 -07001660
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001661 if head.startswith(R_HEADS):
1662 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09001663 head = all_refs[head]
Shawn O. Pearce89e717d2009-04-18 15:04:41 -07001664 except KeyError:
1665 head = None
1666
1667 if head == revid:
1668 # Same revision; just update HEAD to point to the new
1669 # target branch, but otherwise take no other action.
1670 #
1671 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1672 'ref: %s%s\n' % (R_HEADS, name))
1673 return True
1674
1675 return GitCommand(self,
1676 ['checkout', name, '--'],
Anthony King7bdac712014-07-16 12:56:40 +01001677 capture_stdout=True,
1678 capture_stderr=True).Wait() == 0
Wink Saville02d79452009-04-10 13:01:24 -07001679
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001680 def AbandonBranch(self, name):
1681 """Destroy a local topic branch.
Doug Andersondafb1d62011-04-07 11:46:59 -07001682
1683 Args:
1684 name: The name of the branch to abandon.
1685
1686 Returns:
1687 True if the abandon succeeded; False if it didn't; None if the branch
1688 didn't exist.
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001689 """
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001690 rev = R_HEADS + name
David Pursehouse8a68ff92012-09-24 12:15:13 +09001691 all_refs = self.bare_ref.all
1692 if rev not in all_refs:
Doug Andersondafb1d62011-04-07 11:46:59 -07001693 # Doesn't exist
1694 return None
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001695
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001696 head = self.work_git.GetHead()
1697 if head == rev:
1698 # We can't destroy the branch while we are sitting
1699 # on it. Switch to a detached HEAD.
1700 #
David Pursehouse8a68ff92012-09-24 12:15:13 +09001701 head = all_refs[head]
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001702
David Pursehouse8a68ff92012-09-24 12:15:13 +09001703 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001704 if head == revid:
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001705 _lwrite(os.path.join(self.worktree, '.git', HEAD),
1706 '%s\n' % revid)
1707 else:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001708 self._Checkout(revid, quiet=True)
Shawn O. Pearce552ac892009-04-18 15:15:24 -07001709
1710 return GitCommand(self,
1711 ['branch', '-D', name],
Anthony King7bdac712014-07-16 12:56:40 +01001712 capture_stdout=True,
1713 capture_stderr=True).Wait() == 0
Shawn O. Pearce9fa44db2008-11-03 11:24:59 -08001714
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001715 def PruneHeads(self):
1716 """Prune any topic branches already merged into upstream.
1717 """
1718 cb = self.CurrentBranch
1719 kill = []
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001720 left = self._allrefs
1721 for name in left.keys():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001722 if name.startswith(R_HEADS):
1723 name = name[len(R_HEADS):]
1724 if cb is None or name != cb:
1725 kill.append(name)
1726
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001727 rev = self.GetRevisionId(left)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001728 if cb is not None \
1729 and not self._revlist(HEAD + '...' + rev) \
Anthony King7bdac712014-07-16 12:56:40 +01001730 and not self.IsDirty(consider_untracked=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001731 self.work_git.DetachHead(HEAD)
1732 kill.append(cb)
1733
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001734 if kill:
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07001735 old = self.bare_git.GetHead()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001736
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001737 try:
1738 self.bare_git.DetachHead(rev)
1739
1740 b = ['branch', '-d']
1741 b.extend(kill)
1742 b = GitCommand(self, b, bare=True,
1743 capture_stdout=True,
1744 capture_stderr=True)
1745 b.Wait()
1746 finally:
Dan Willemsen1a799d12015-12-15 13:40:05 -08001747 if ID_RE.match(old):
1748 self.bare_git.DetachHead(old)
1749 else:
1750 self.bare_git.SetHead(old)
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001751 left = self._allrefs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001752
Shawn O. Pearce3778f9d2009-03-02 12:30:50 -08001753 for branch in kill:
1754 if (R_HEADS + branch) not in left:
1755 self.CleanPublishedCache()
1756 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001757
1758 if cb and cb not in kill:
1759 kill.append(cb)
Shawn O. Pearce7c6c64d2009-03-02 12:38:13 -08001760 kill.sort()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001761
1762 kept = []
1763 for branch in kill:
Anthony King7bdac712014-07-16 12:56:40 +01001764 if R_HEADS + branch in left:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001765 branch = self.GetBranch(branch)
1766 base = branch.LocalMerge
1767 if not base:
1768 base = rev
1769 kept.append(ReviewableBranch(self, branch, base))
1770 return kept
1771
1772
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001773# Submodule Management ##
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001774
1775 def GetRegisteredSubprojects(self):
1776 result = []
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001777
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001778 def rec(subprojects):
1779 if not subprojects:
1780 return
1781 result.extend(subprojects)
1782 for p in subprojects:
1783 rec(p.subprojects)
1784 rec(self.subprojects)
1785 return result
1786
1787 def _GetSubmodules(self):
1788 # Unfortunately we cannot call `git submodule status --recursive` here
1789 # because the working tree might not exist yet, and it cannot be used
1790 # without a working tree in its current implementation.
1791
1792 def get_submodules(gitdir, rev):
1793 # Parse .gitmodules for submodule sub_paths and sub_urls
1794 sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
1795 if not sub_paths:
1796 return []
1797 # Run `git ls-tree` to read SHAs of submodule object, which happen to be
1798 # revision of submodule repository
1799 sub_revs = git_ls_tree(gitdir, rev, sub_paths)
1800 submodules = []
1801 for sub_path, sub_url in zip(sub_paths, sub_urls):
1802 try:
1803 sub_rev = sub_revs[sub_path]
1804 except KeyError:
1805 # Ignore non-exist submodules
1806 continue
1807 submodules.append((sub_rev, sub_path, sub_url))
1808 return submodules
1809
1810 re_path = re.compile(r'^submodule\.([^.]+)\.path=(.*)$')
1811 re_url = re.compile(r'^submodule\.([^.]+)\.url=(.*)$')
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001812
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001813 def parse_gitmodules(gitdir, rev):
1814 cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
1815 try:
Anthony King7bdac712014-07-16 12:56:40 +01001816 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1817 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001818 except GitError:
1819 return [], []
1820 if p.Wait() != 0:
1821 return [], []
1822
1823 gitmodules_lines = []
1824 fd, temp_gitmodules_path = tempfile.mkstemp()
1825 try:
1826 os.write(fd, p.stdout)
1827 os.close(fd)
1828 cmd = ['config', '--file', temp_gitmodules_path, '--list']
Anthony King7bdac712014-07-16 12:56:40 +01001829 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1830 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001831 if p.Wait() != 0:
1832 return [], []
1833 gitmodules_lines = p.stdout.split('\n')
1834 except GitError:
1835 return [], []
1836 finally:
Renaud Paquay010fed72016-11-11 14:25:29 -08001837 platform_utils.remove(temp_gitmodules_path)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001838
1839 names = set()
1840 paths = {}
1841 urls = {}
1842 for line in gitmodules_lines:
1843 if not line:
1844 continue
1845 m = re_path.match(line)
1846 if m:
1847 names.add(m.group(1))
1848 paths[m.group(1)] = m.group(2)
1849 continue
1850 m = re_url.match(line)
1851 if m:
1852 names.add(m.group(1))
1853 urls[m.group(1)] = m.group(2)
1854 continue
1855 names = sorted(names)
1856 return ([paths.get(name, '') for name in names],
1857 [urls.get(name, '') for name in names])
1858
1859 def git_ls_tree(gitdir, rev, paths):
1860 cmd = ['ls-tree', rev, '--']
1861 cmd.extend(paths)
1862 try:
Anthony King7bdac712014-07-16 12:56:40 +01001863 p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
1864 bare=True, gitdir=gitdir)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001865 except GitError:
1866 return []
1867 if p.Wait() != 0:
1868 return []
1869 objects = {}
1870 for line in p.stdout.split('\n'):
1871 if not line.strip():
1872 continue
1873 object_rev, object_path = line.split()[2:4]
1874 objects[object_path] = object_rev
1875 return objects
1876
1877 try:
1878 rev = self.GetRevisionId()
1879 except GitError:
1880 return []
1881 return get_submodules(self.gitdir, rev)
1882
1883 def GetDerivedSubprojects(self):
1884 result = []
1885 if not self.Exists:
1886 # If git repo does not exist yet, querying its submodules will
1887 # mess up its states; so return here.
1888 return result
1889 for rev, path, url in self._GetSubmodules():
1890 name = self.manifest.GetSubprojectName(self, path)
David James8d201162013-10-11 17:03:19 -07001891 relpath, worktree, gitdir, objdir = \
1892 self.manifest.GetSubprojectPaths(self, name, path)
1893 project = self.manifest.paths.get(relpath)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001894 if project:
1895 result.extend(project.GetDerivedSubprojects())
1896 continue
David James8d201162013-10-11 17:03:19 -07001897
Shouheng Zhang02c0ee62017-12-08 09:54:58 +08001898 if url.startswith('..'):
1899 url = urllib.parse.urljoin("%s/" % self.remote.url, url)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001900 remote = RemoteSpec(self.remote.name,
Anthony King7bdac712014-07-16 12:56:40 +01001901 url=url,
Steve Raed6480452016-08-10 15:00:00 -07001902 pushUrl=self.remote.pushUrl,
Anthony King7bdac712014-07-16 12:56:40 +01001903 review=self.remote.review,
1904 revision=self.remote.revision)
1905 subproject = Project(manifest=self.manifest,
1906 name=name,
1907 remote=remote,
1908 gitdir=gitdir,
1909 objdir=objdir,
1910 worktree=worktree,
1911 relpath=relpath,
Aymen Bouaziz2598ed02016-06-24 14:34:08 +02001912 revisionExpr=rev,
Anthony King7bdac712014-07-16 12:56:40 +01001913 revisionId=rev,
1914 rebase=self.rebase,
1915 groups=self.groups,
1916 sync_c=self.sync_c,
1917 sync_s=self.sync_s,
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +09001918 sync_tags=self.sync_tags,
Anthony King7bdac712014-07-16 12:56:40 +01001919 parent=self,
1920 is_derived=True)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001921 result.append(subproject)
1922 result.extend(subproject.GetDerivedSubprojects())
1923 return result
1924
1925
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07001926# Direct Git Commands ##
Zac Livingstone4332262017-06-16 08:56:09 -06001927 def _CheckForImmutableRevision(self):
Chris AtLee2fb64662014-01-16 21:32:33 -05001928 try:
1929 # if revision (sha or tag) is not present then following function
1930 # throws an error.
1931 self.bare_git.rev_parse('--verify', '%s^0' % self.revisionExpr)
1932 return True
1933 except GitError:
1934 # There is no such persistent revision. We have to fetch it.
1935 return False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001936
Julien Campergue335f5ef2013-10-16 11:02:35 +02001937 def _FetchArchive(self, tarpath, cwd=None):
1938 cmd = ['archive', '-v', '-o', tarpath]
1939 cmd.append('--remote=%s' % self.remote.url)
1940 cmd.append('--prefix=%s/' % self.relpath)
1941 cmd.append(self.revisionExpr)
1942
1943 command = GitCommand(self, cmd, cwd=cwd,
1944 capture_stdout=True,
1945 capture_stderr=True)
1946
1947 if command.Wait() != 0:
1948 raise GitError('git archive %s: %s' % (self.name, command.stderr))
1949
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001950 def _RemoteFetch(self, name=None,
1951 current_branch_only=False,
Shawn O. Pearce16614f82010-10-29 12:05:43 -07001952 initial=False,
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07001953 quiet=False,
Mitchel Humpherys597868b2012-10-29 10:18:34 -07001954 alt_dir=None,
David Pursehouse74cfd272015-10-14 10:50:15 +09001955 no_tags=False,
Aymen Bouaziz6c594462016-10-25 18:03:51 +02001956 prune=False,
Martin Kellye4e94d22017-03-21 16:05:12 -07001957 depth=None,
1958 submodules=False):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001959
1960 is_sha1 = False
1961 tag_name = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001962 # The depth should not be used when fetching to a mirror because
1963 # it will result in a shallow repository that cannot be cloned or
1964 # fetched from.
Aymen Bouaziz6c594462016-10-25 18:03:51 +02001965 # The repo project should also never be synced with partial depth.
1966 if self.manifest.IsMirror or self.relpath == '.repo/repo':
1967 depth = None
David Pursehouse9bc422f2014-04-15 10:28:56 +09001968
Shawn Pearce69e04d82014-01-29 12:48:54 -08001969 if depth:
1970 current_branch_only = True
1971
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001972 if ID_RE.match(self.revisionExpr) is not None:
1973 is_sha1 = True
1974
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001975 if current_branch_only:
Nasser Grainawi909d58b2014-09-19 12:13:04 -06001976 if self.revisionExpr.startswith(R_TAGS):
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001977 # this is a tag and its sha1 value should never change
1978 tag_name = self.revisionExpr[len(R_TAGS):]
1979
1980 if is_sha1 or tag_name is not None:
Zac Livingstone4332262017-06-16 08:56:09 -06001981 if self._CheckForImmutableRevision():
1982 print('Skipped fetching project %s (already have persistent ref)'
1983 % self.name)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001984 return True
Bertrand SIMONNET3000cda2014-11-25 16:19:29 -08001985 if is_sha1 and not depth:
1986 # When syncing a specific commit and --depth is not set:
1987 # * if upstream is explicitly specified and is not a sha1, fetch only
1988 # upstream as users expect only upstream to be fetch.
1989 # Note: The commit might not be in upstream in which case the sync
1990 # will fail.
1991 # * otherwise, fetch all branches to make sure we end up with the
1992 # specific commit.
Aymen Bouaziz037040f2016-06-28 12:27:23 +02001993 if self.upstream:
1994 current_branch_only = not ID_RE.match(self.upstream)
1995 else:
1996 current_branch_only = False
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07001997
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001998 if not name:
1999 name = self.remote.name
Shawn O. Pearcefb231612009-04-10 18:53:46 -07002000
2001 ssh_proxy = False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002002 remote = self.GetRemote(name)
2003 if remote.PreConnectFetch():
Shawn O. Pearcefb231612009-04-10 18:53:46 -07002004 ssh_proxy = True
2005
Shawn O. Pearce88443382010-10-08 10:02:09 +02002006 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002007 if alt_dir and 'objects' == os.path.basename(alt_dir):
2008 ref_dir = os.path.dirname(alt_dir)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002009 packed_refs = os.path.join(self.gitdir, 'packed-refs')
2010 remote = self.GetRemote(name)
2011
David Pursehouse8a68ff92012-09-24 12:15:13 +09002012 all_refs = self.bare_ref.all
2013 ids = set(all_refs.values())
Shawn O. Pearce88443382010-10-08 10:02:09 +02002014 tmp = set()
2015
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302016 for r, ref_id in GitRefs(ref_dir).all.items():
David Pursehouse8a68ff92012-09-24 12:15:13 +09002017 if r not in all_refs:
Shawn O. Pearce88443382010-10-08 10:02:09 +02002018 if r.startswith(R_TAGS) or remote.WritesTo(r):
David Pursehouse8a68ff92012-09-24 12:15:13 +09002019 all_refs[r] = ref_id
2020 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002021 continue
2022
David Pursehouse8a68ff92012-09-24 12:15:13 +09002023 if ref_id in ids:
Shawn O. Pearce88443382010-10-08 10:02:09 +02002024 continue
2025
David Pursehouse8a68ff92012-09-24 12:15:13 +09002026 r = 'refs/_alt/%s' % ref_id
2027 all_refs[r] = ref_id
2028 ids.add(ref_id)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002029 tmp.add(r)
2030
heping3d7bbc92017-04-12 19:51:47 +08002031 tmp_packed_lines = []
2032 old_packed_lines = []
Shawn O. Pearce88443382010-10-08 10:02:09 +02002033
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302034 for r in sorted(all_refs):
David Pursehouse8a68ff92012-09-24 12:15:13 +09002035 line = '%s %s\n' % (all_refs[r], r)
heping3d7bbc92017-04-12 19:51:47 +08002036 tmp_packed_lines.append(line)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002037 if r not in tmp:
heping3d7bbc92017-04-12 19:51:47 +08002038 old_packed_lines.append(line)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002039
heping3d7bbc92017-04-12 19:51:47 +08002040 tmp_packed = ''.join(tmp_packed_lines)
2041 old_packed = ''.join(old_packed_lines)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002042 _lwrite(packed_refs, tmp_packed)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002043 else:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002044 alt_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02002045
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002046 cmd = ['fetch']
Doug Anderson30d45292011-05-04 15:01:04 -07002047
Conley Owensf97e8382015-01-21 11:12:46 -08002048 if depth:
Doug Anderson30d45292011-05-04 15:01:04 -07002049 cmd.append('--depth=%s' % depth)
Dan Willemseneeab6862015-08-03 13:11:53 -07002050 else:
2051 # If this repo has shallow objects, then we don't know which refs have
2052 # shallow objects or not. Tell git to unshallow all fetched refs. Don't
2053 # do this with projects that don't have shallow objects, since it is less
2054 # efficient.
2055 if os.path.exists(os.path.join(self.gitdir, 'shallow')):
2056 cmd.append('--depth=2147483647')
Doug Anderson30d45292011-05-04 15:01:04 -07002057
Shawn O. Pearce16614f82010-10-29 12:05:43 -07002058 if quiet:
2059 cmd.append('--quiet')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002060 if not self.worktree:
2061 cmd.append('--update-head-ok')
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002062 cmd.append(name)
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002063
Mitchel Humpherys26c45a72014-03-10 14:21:59 -07002064 # If using depth then we should not get all the tags since they may
2065 # be outside of the depth.
2066 if no_tags or depth:
2067 cmd.append('--no-tags')
2068 else:
2069 cmd.append('--tags')
2070
David Pursehouse74cfd272015-10-14 10:50:15 +09002071 if prune:
2072 cmd.append('--prune')
2073
Martin Kellye4e94d22017-03-21 16:05:12 -07002074 if submodules:
2075 cmd.append('--recurse-submodules=on-demand')
2076
Conley Owens80b87fe2014-05-09 17:13:44 -07002077 spec = []
Brian Harring14a66742012-09-28 20:21:57 -07002078 if not current_branch_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002079 # Fetch whole repo
Conley Owens80b87fe2014-05-09 17:13:44 -07002080 spec.append(str((u'+refs/heads/*:') + remote.ToLocal('refs/heads/*')))
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -07002081 elif tag_name is not None:
Conley Owens80b87fe2014-05-09 17:13:44 -07002082 spec.append('tag')
2083 spec.append(tag_name)
Nasser Grainawi04e52d62014-09-30 13:34:52 -06002084
David Pursehouse403b64e2015-04-27 10:41:33 +09002085 if not self.manifest.IsMirror:
2086 branch = self.revisionExpr
Kevin Degi679bac42015-06-22 15:31:26 -06002087 if is_sha1 and depth and git_require((1, 8, 3)):
David Pursehouse403b64e2015-04-27 10:41:33 +09002088 # Shallow checkout of a specific commit, fetch from that commit and not
2089 # the heads only as the commit might be deeper in the history.
2090 spec.append(branch)
2091 else:
2092 if is_sha1:
2093 branch = self.upstream
2094 if branch is not None and branch.strip():
2095 if not branch.startswith('refs/'):
2096 branch = R_HEADS + branch
2097 spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
Conley Owens80b87fe2014-05-09 17:13:44 -07002098 cmd.extend(spec)
2099
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002100 ok = False
David Pursehouse8a68ff92012-09-24 12:15:13 +09002101 for _i in range(2):
John L. Villalovos9c76f672015-03-16 20:49:10 -07002102 gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy)
John L. Villalovos126e2982015-01-29 21:58:12 -08002103 ret = gitcmd.Wait()
Brian Harring14a66742012-09-28 20:21:57 -07002104 if ret == 0:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002105 ok = True
2106 break
John L. Villalovos126e2982015-01-29 21:58:12 -08002107 # If needed, run the 'git remote prune' the first time through the loop
2108 elif (not _i and
2109 "error:" in gitcmd.stderr and
2110 "git remote prune" in gitcmd.stderr):
2111 prunecmd = GitCommand(self, ['remote', 'prune', name], bare=True,
John L. Villalovos9c76f672015-03-16 20:49:10 -07002112 ssh_proxy=ssh_proxy)
John L. Villalovose30f46b2015-02-25 14:27:02 -08002113 ret = prunecmd.Wait()
John L. Villalovose30f46b2015-02-25 14:27:02 -08002114 if ret:
John L. Villalovos126e2982015-01-29 21:58:12 -08002115 break
2116 continue
Brian Harring14a66742012-09-28 20:21:57 -07002117 elif current_branch_only and is_sha1 and ret == 128:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002118 # Exit code 128 means "couldn't find the ref you asked for"; if we're
2119 # in sha1 mode, we just tried sync'ing from the upstream field; it
2120 # doesn't exist, thus abort the optimization attempt and do a full sync.
Brian Harring14a66742012-09-28 20:21:57 -07002121 break
Colin Crossc4b301f2015-05-13 00:10:02 -07002122 elif ret < 0:
2123 # Git died with a signal, exit immediately
2124 break
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002125 time.sleep(random.randint(30, 45))
Shawn O. Pearce88443382010-10-08 10:02:09 +02002126
2127 if initial:
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002128 if alt_dir:
Shawn O. Pearce88443382010-10-08 10:02:09 +02002129 if old_packed != '':
2130 _lwrite(packed_refs, old_packed)
2131 else:
Renaud Paquay010fed72016-11-11 14:25:29 -08002132 platform_utils.remove(packed_refs)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002133 self.bare_git.pack_refs('--all', '--prune')
Brian Harring14a66742012-09-28 20:21:57 -07002134
Aymen Bouaziz6c594462016-10-25 18:03:51 +02002135 if is_sha1 and current_branch_only:
Brian Harring14a66742012-09-28 20:21:57 -07002136 # We just synced the upstream given branch; verify we
2137 # got what we wanted, else trigger a second run of all
2138 # refs.
Zac Livingstone4332262017-06-16 08:56:09 -06002139 if not self._CheckForImmutableRevision():
Aymen Bouaziz6c594462016-10-25 18:03:51 +02002140 if current_branch_only and depth:
2141 # Sync the current branch only with depth set to None
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002142 return self._RemoteFetch(name=name,
2143 current_branch_only=current_branch_only,
Aymen Bouaziz6c594462016-10-25 18:03:51 +02002144 initial=False, quiet=quiet, alt_dir=alt_dir,
2145 depth=None)
2146 else:
2147 # Avoid infinite recursion: sync all branches with depth set to None
2148 return self._RemoteFetch(name=name, current_branch_only=False,
2149 initial=False, quiet=quiet, alt_dir=alt_dir,
2150 depth=None)
Brian Harring14a66742012-09-28 20:21:57 -07002151
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002152 return ok
Shawn O. Pearce88443382010-10-08 10:02:09 +02002153
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002154 def _ApplyCloneBundle(self, initial=False, quiet=False):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002155 if initial and \
2156 (self.manifest.manifestProject.config.GetString('repo.depth') or
2157 self.clone_depth):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002158 return False
2159
2160 remote = self.GetRemote(self.remote.name)
2161 bundle_url = remote.url + '/clone.bundle'
2162 bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002163 if GetSchemeFromUrl(bundle_url) not in ('http', 'https',
2164 'persistent-http',
2165 'persistent-https'):
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002166 return False
2167
2168 bundle_dst = os.path.join(self.gitdir, 'clone.bundle')
2169 bundle_tmp = os.path.join(self.gitdir, 'clone.bundle.tmp')
2170
2171 exist_dst = os.path.exists(bundle_dst)
2172 exist_tmp = os.path.exists(bundle_tmp)
2173
2174 if not initial and not exist_dst and not exist_tmp:
2175 return False
2176
2177 if not exist_dst:
2178 exist_dst = self._FetchBundle(bundle_url, bundle_tmp, bundle_dst, quiet)
2179 if not exist_dst:
2180 return False
2181
2182 cmd = ['fetch']
2183 if quiet:
2184 cmd.append('--quiet')
2185 if not self.worktree:
2186 cmd.append('--update-head-ok')
2187 cmd.append(bundle_dst)
2188 for f in remote.fetch:
2189 cmd.append(str(f))
2190 cmd.append('refs/tags/*:refs/tags/*')
2191
2192 ok = GitCommand(self, cmd, bare=True).Wait() == 0
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002193 if os.path.exists(bundle_dst):
Renaud Paquay010fed72016-11-11 14:25:29 -08002194 platform_utils.remove(bundle_dst)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002195 if os.path.exists(bundle_tmp):
Renaud Paquay010fed72016-11-11 14:25:29 -08002196 platform_utils.remove(bundle_tmp)
Shawn O. Pearce88443382010-10-08 10:02:09 +02002197 return ok
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002198
Shawn O. Pearcec325dc32011-10-03 08:30:24 -07002199 def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet):
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002200 if os.path.exists(dstPath):
Renaud Paquay010fed72016-11-11 14:25:29 -08002201 platform_utils.remove(dstPath)
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002202
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002203 cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002204 if quiet:
2205 cmd += ['--silent']
2206 if os.path.exists(tmpPath):
2207 size = os.stat(tmpPath).st_size
2208 if size >= 1024:
2209 cmd += ['--continue-at', '%d' % (size,)]
2210 else:
Renaud Paquay010fed72016-11-11 14:25:29 -08002211 platform_utils.remove(tmpPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002212 if 'http_proxy' in os.environ and 'darwin' == sys.platform:
2213 cmd += ['--proxy', os.environ['http_proxy']]
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002214 with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, _proxy):
Dave Borowitz137d0132015-01-02 11:12:54 -08002215 if cookiefile:
Dave Borowitz4abf8e62015-01-02 11:39:04 -08002216 cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
Dave Borowitz137d0132015-01-02 11:12:54 -08002217 if srcUrl.startswith('persistent-'):
2218 srcUrl = srcUrl[len('persistent-'):]
2219 cmd += [srcUrl]
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002220
Dave Borowitz137d0132015-01-02 11:12:54 -08002221 if IsTrace():
2222 Trace('%s', ' '.join(cmd))
2223 try:
2224 proc = subprocess.Popen(cmd)
2225 except OSError:
2226 return False
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002227
Dave Borowitz137d0132015-01-02 11:12:54 -08002228 curlret = proc.wait()
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002229
Dave Borowitz137d0132015-01-02 11:12:54 -08002230 if curlret == 22:
2231 # From curl man page:
2232 # 22: HTTP page not retrieved. The requested url was not found or
2233 # returned another error with the HTTP error code being 400 or above.
2234 # This return code only appears if -f, --fail is used.
2235 if not quiet:
2236 print("Server does not provide clone.bundle; ignoring.",
2237 file=sys.stderr)
2238 return False
Matt Gumbel2dc810c2012-08-30 09:39:36 -07002239
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002240 if os.path.exists(tmpPath):
Kris Giesingc8d882a2014-12-23 13:02:32 -08002241 if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
Renaud Paquayad1abcb2016-11-01 11:34:55 -07002242 platform_utils.rename(tmpPath, dstPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002243 return True
2244 else:
Renaud Paquay010fed72016-11-11 14:25:29 -08002245 platform_utils.remove(tmpPath)
Shawn O. Pearce5e7127d2012-08-02 14:57:37 -07002246 return False
2247 else:
2248 return False
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -07002249
Kris Giesingc8d882a2014-12-23 13:02:32 -08002250 def _IsValidBundle(self, path, quiet):
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002251 try:
2252 with open(path) as f:
2253 if f.read(16) == '# v2 git bundle\n':
2254 return True
2255 else:
Kris Giesingc8d882a2014-12-23 13:02:32 -08002256 if not quiet:
2257 print("Invalid clone.bundle file; ignoring.", file=sys.stderr)
Dave Borowitz91f3ba52013-06-03 12:15:23 -07002258 return False
2259 except OSError:
2260 return False
2261
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002262 def _Checkout(self, rev, quiet=False):
2263 cmd = ['checkout']
2264 if quiet:
2265 cmd.append('-q')
2266 cmd.append(rev)
2267 cmd.append('--')
2268 if GitCommand(self, cmd).Wait() != 0:
2269 if self._allrefs:
2270 raise GitError('%s checkout %s ' % (self.name, rev))
2271
Anthony King7bdac712014-07-16 12:56:40 +01002272 def _CherryPick(self, rev):
Pierre Tardye5a21222011-03-24 16:28:18 +01002273 cmd = ['cherry-pick']
2274 cmd.append(rev)
2275 cmd.append('--')
2276 if GitCommand(self, cmd).Wait() != 0:
2277 if self._allrefs:
2278 raise GitError('%s cherry-pick %s ' % (self.name, rev))
2279
Akshay Verma0f2e45a2018-03-24 12:27:05 +05302280 def _LsRemote(self, refs):
2281 cmd = ['ls-remote', self.remote.name, refs]
Akshay Vermacf7c0832018-03-15 21:56:30 +05302282 p = GitCommand(self, cmd, capture_stdout=True)
2283 if p.Wait() == 0:
2284 if hasattr(p.stdout, 'decode'):
2285 return p.stdout.decode('utf-8')
2286 else:
2287 return p.stdout
2288 return None
2289
Anthony King7bdac712014-07-16 12:56:40 +01002290 def _Revert(self, rev):
Erwan Mahea94f1622011-08-19 13:56:09 +02002291 cmd = ['revert']
2292 cmd.append('--no-edit')
2293 cmd.append(rev)
2294 cmd.append('--')
2295 if GitCommand(self, cmd).Wait() != 0:
2296 if self._allrefs:
2297 raise GitError('%s revert %s ' % (self.name, rev))
2298
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002299 def _ResetHard(self, rev, quiet=True):
2300 cmd = ['reset', '--hard']
2301 if quiet:
2302 cmd.append('-q')
2303 cmd.append(rev)
2304 if GitCommand(self, cmd).Wait() != 0:
2305 raise GitError('%s reset --hard %s ' % (self.name, rev))
2306
Martin Kellye4e94d22017-03-21 16:05:12 -07002307 def _SyncSubmodules(self, quiet=True):
2308 cmd = ['submodule', 'update', '--init', '--recursive']
2309 if quiet:
2310 cmd.append('-q')
2311 if GitCommand(self, cmd).Wait() != 0:
2312 raise GitError('%s submodule update --init --recursive %s ' % self.name)
2313
Anthony King7bdac712014-07-16 12:56:40 +01002314 def _Rebase(self, upstream, onto=None):
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002315 cmd = ['rebase']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002316 if onto is not None:
2317 cmd.extend(['--onto', onto])
2318 cmd.append(upstream)
Shawn O. Pearce19a83d82009-04-16 08:14:26 -07002319 if GitCommand(self, cmd).Wait() != 0:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002320 raise GitError('%s rebase %s ' % (self.name, upstream))
2321
Pierre Tardy3d125942012-05-04 12:18:12 +02002322 def _FastForward(self, head, ffonly=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002323 cmd = ['merge', head]
Pierre Tardy3d125942012-05-04 12:18:12 +02002324 if ffonly:
2325 cmd.append("--ff-only")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002326 if GitCommand(self, cmd).Wait() != 0:
2327 raise GitError('%s merge %s ' % (self.name, head))
2328
Kevin Degiabaa7f32014-11-12 11:27:45 -07002329 def _InitGitDir(self, mirror_git=None, force_sync=False):
Kevin Degi384b3c52014-10-16 16:02:58 -06002330 init_git_dir = not os.path.exists(self.gitdir)
2331 init_obj_dir = not os.path.exists(self.objdir)
Kevin Degib1a07b82015-07-27 13:33:43 -06002332 try:
2333 # Initialize the bare repository, which contains all of the objects.
2334 if init_obj_dir:
2335 os.makedirs(self.objdir)
2336 self.bare_objdir.init()
David James8d201162013-10-11 17:03:19 -07002337
Kevin Degib1a07b82015-07-27 13:33:43 -06002338 # If we have a separate directory to hold refs, initialize it as well.
2339 if self.objdir != self.gitdir:
2340 if init_git_dir:
2341 os.makedirs(self.gitdir)
2342
2343 if init_obj_dir or init_git_dir:
2344 self._ReferenceGitDir(self.objdir, self.gitdir, share_refs=False,
2345 copy_all=True)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002346 try:
2347 self._CheckDirReference(self.objdir, self.gitdir, share_refs=False)
2348 except GitError as e:
Kevin Degiabaa7f32014-11-12 11:27:45 -07002349 if force_sync:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002350 print("Retrying clone after deleting %s" %
2351 self.gitdir, file=sys.stderr)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002352 try:
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002353 platform_utils.rmtree(platform_utils.realpath(self.gitdir))
2354 if self.worktree and os.path.exists(platform_utils.realpath
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002355 (self.worktree)):
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002356 platform_utils.rmtree(platform_utils.realpath(self.worktree))
Kevin Degiabaa7f32014-11-12 11:27:45 -07002357 return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
2358 except:
2359 raise e
2360 raise e
Kevin Degib1a07b82015-07-27 13:33:43 -06002361
Kevin Degi384b3c52014-10-16 16:02:58 -06002362 if init_git_dir:
Kevin Degib1a07b82015-07-27 13:33:43 -06002363 mp = self.manifest.manifestProject
2364 ref_dir = mp.config.GetString('repo.reference') or ''
Kevin Degi384b3c52014-10-16 16:02:58 -06002365
Kevin Degib1a07b82015-07-27 13:33:43 -06002366 if ref_dir or mirror_git:
2367 if not mirror_git:
2368 mirror_git = os.path.join(ref_dir, self.name + '.git')
2369 repo_git = os.path.join(ref_dir, '.repo', 'projects',
2370 self.relpath + '.git')
Shawn O. Pearce2816d4f2009-03-03 17:53:18 -08002371
Kevin Degib1a07b82015-07-27 13:33:43 -06002372 if os.path.exists(mirror_git):
2373 ref_dir = mirror_git
Shawn O. Pearce88443382010-10-08 10:02:09 +02002374
Kevin Degib1a07b82015-07-27 13:33:43 -06002375 elif os.path.exists(repo_git):
2376 ref_dir = repo_git
Shawn O. Pearce88443382010-10-08 10:02:09 +02002377
Kevin Degib1a07b82015-07-27 13:33:43 -06002378 else:
2379 ref_dir = None
Shawn O. Pearce88443382010-10-08 10:02:09 +02002380
Kevin Degib1a07b82015-07-27 13:33:43 -06002381 if ref_dir:
Samuel Hollandbaa00092018-01-22 10:57:29 -06002382 if not os.path.isabs(ref_dir):
2383 # The alternate directory is relative to the object database.
2384 ref_dir = os.path.relpath(ref_dir,
2385 os.path.join(self.objdir, 'objects'))
Kevin Degib1a07b82015-07-27 13:33:43 -06002386 _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
2387 os.path.join(ref_dir, 'objects') + '\n')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002388
Kevin Degib1a07b82015-07-27 13:33:43 -06002389 self._UpdateHooks()
2390
2391 m = self.manifest.manifestProject.config
2392 for key in ['user.name', 'user.email']:
2393 if m.Has(key, include_defaults=False):
2394 self.config.SetString(key, m.GetString(key))
David Pursehouse76a4a9d2016-08-16 12:11:12 +09002395 self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
Kevin Degib1a07b82015-07-27 13:33:43 -06002396 if self.manifest.IsMirror:
2397 self.config.SetString('core.bare', 'true')
Shawn O. Pearce88443382010-10-08 10:02:09 +02002398 else:
Kevin Degib1a07b82015-07-27 13:33:43 -06002399 self.config.SetString('core.bare', None)
2400 except Exception:
2401 if init_obj_dir and os.path.exists(self.objdir):
Renaud Paquaya65adf72016-11-03 10:37:53 -07002402 platform_utils.rmtree(self.objdir)
Kevin Degib1a07b82015-07-27 13:33:43 -06002403 if init_git_dir and os.path.exists(self.gitdir):
Renaud Paquaya65adf72016-11-03 10:37:53 -07002404 platform_utils.rmtree(self.gitdir)
Kevin Degib1a07b82015-07-27 13:33:43 -06002405 raise
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002406
Jimmie Westera0444582012-10-24 13:44:42 +02002407 def _UpdateHooks(self):
2408 if os.path.exists(self.gitdir):
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002409 self._InitHooks()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002410
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002411 def _InitHooks(self):
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002412 hooks = platform_utils.realpath(self._gitdir_path('hooks'))
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002413 if not os.path.exists(hooks):
2414 os.makedirs(hooks)
Jonathan Nieder93719792015-03-17 11:29:58 -07002415 for stock_hook in _ProjectHooks():
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002416 name = os.path.basename(stock_hook)
2417
Victor Boivie65e0f352011-04-18 11:23:29 +02002418 if name in ('commit-msg',) and not self.remote.review \
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002419 and self is not self.manifest.manifestProject:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002420 # Don't install a Gerrit Code Review hook if this
2421 # project does not appear to use it for reviews.
2422 #
Victor Boivie65e0f352011-04-18 11:23:29 +02002423 # Since the manifest project is one of those, but also
2424 # managed through gerrit, it's excluded
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002425 continue
2426
2427 dst = os.path.join(hooks, name)
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002428 if platform_utils.islink(dst):
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002429 continue
2430 if os.path.exists(dst):
2431 if filecmp.cmp(stock_hook, dst, shallow=False):
Renaud Paquay010fed72016-11-11 14:25:29 -08002432 platform_utils.remove(dst)
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002433 else:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002434 _warn("%s: Not replacing locally modified %s hook",
2435 self.relpath, name)
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002436 continue
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002437 try:
Renaud Paquayd5cec5e2016-11-01 11:24:03 -07002438 platform_utils.symlink(
2439 os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
Sarah Owensa5be53f2012-09-09 15:37:57 -07002440 except OSError as e:
Shawn O. Pearce9452e4e2009-08-22 18:17:46 -07002441 if e.errno == errno.EPERM:
Renaud Paquay788e9622017-01-27 11:41:12 -08002442 raise GitError(self._get_symlink_error_message())
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -08002443 else:
2444 raise
2445
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002446 def _InitRemote(self):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002447 if self.remote.url:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002448 remote = self.GetRemote(self.remote.name)
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002449 remote.url = self.remote.url
Steve Raed6480452016-08-10 15:00:00 -07002450 remote.pushUrl = self.remote.pushUrl
Shawn O. Pearced1f70d92009-05-19 14:58:02 -07002451 remote.review = self.remote.review
2452 remote.projectname = self.name
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002453
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002454 if self.worktree:
2455 remote.ResetFetch(mirror=False)
2456 else:
2457 remote.ResetFetch(mirror=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002458 remote.Save()
2459
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002460 def _InitMRef(self):
2461 if self.manifest.branch:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002462 self._InitAnyMRef(R_M + self.manifest.branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002463
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002464 def _InitMirrorHead(self):
Shawn O. Pearcefe200ee2009-06-01 15:28:21 -07002465 self._InitAnyMRef(HEAD)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002466
2467 def _InitAnyMRef(self, ref):
2468 cur = self.bare_ref.symref(ref)
2469
2470 if self.revisionId:
2471 if cur != '' or self.bare_ref.get(ref) != self.revisionId:
2472 msg = 'manifest set to %s' % self.revisionId
2473 dst = self.revisionId + '^0'
Anthony King7bdac712014-07-16 12:56:40 +01002474 self.bare_git.UpdateRef(ref, dst, message=msg, detach=True)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07002475 else:
2476 remote = self.GetRemote(self.remote.name)
2477 dst = remote.ToLocal(self.revisionExpr)
2478 if cur != dst:
2479 msg = 'manifest set to %s' % self.revisionExpr
2480 self.bare_git.symbolic_ref('-m', msg, ref, dst)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08002481
Kevin Degi384b3c52014-10-16 16:02:58 -06002482 def _CheckDirReference(self, srcdir, destdir, share_refs):
Dan Willemsenbdb866e2016-04-05 17:22:02 -07002483 symlink_files = self.shareable_files[:]
2484 symlink_dirs = self.shareable_dirs[:]
Kevin Degi384b3c52014-10-16 16:02:58 -06002485 if share_refs:
2486 symlink_files += self.working_tree_files
2487 symlink_dirs += self.working_tree_dirs
2488 to_symlink = symlink_files + symlink_dirs
2489 for name in set(to_symlink):
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002490 dst = platform_utils.realpath(os.path.join(destdir, name))
Kevin Degi384b3c52014-10-16 16:02:58 -06002491 if os.path.lexists(dst):
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002492 src = platform_utils.realpath(os.path.join(srcdir, name))
Kevin Degi384b3c52014-10-16 16:02:58 -06002493 # Fail if the links are pointing to the wrong place
2494 if src != dst:
Marc Herbertec287902016-10-27 12:58:26 -07002495 _error('%s is different in %s vs %s', name, destdir, srcdir)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002496 raise GitError('--force-sync not enabled; cannot overwrite a local '
Simon Ruggierf9b76832015-07-31 17:18:34 -04002497 'work tree. If you\'re comfortable with the '
2498 'possibility of losing the work tree\'s git metadata,'
2499 ' use `repo sync --force-sync {0}` to '
2500 'proceed.'.format(self.relpath))
Kevin Degi384b3c52014-10-16 16:02:58 -06002501
David James8d201162013-10-11 17:03:19 -07002502 def _ReferenceGitDir(self, gitdir, dotgit, share_refs, copy_all):
2503 """Update |dotgit| to reference |gitdir|, using symlinks where possible.
2504
2505 Args:
2506 gitdir: The bare git repository. Must already be initialized.
2507 dotgit: The repository you would like to initialize.
2508 share_refs: If true, |dotgit| will store its refs under |gitdir|.
2509 Only one work tree can store refs under a given |gitdir|.
2510 copy_all: If true, copy all remaining files from |gitdir| -> |dotgit|.
2511 This saves you the effort of initializing |dotgit| yourself.
2512 """
Dan Willemsenbdb866e2016-04-05 17:22:02 -07002513 symlink_files = self.shareable_files[:]
2514 symlink_dirs = self.shareable_dirs[:]
David James8d201162013-10-11 17:03:19 -07002515 if share_refs:
Kevin Degi384b3c52014-10-16 16:02:58 -06002516 symlink_files += self.working_tree_files
2517 symlink_dirs += self.working_tree_dirs
David James8d201162013-10-11 17:03:19 -07002518 to_symlink = symlink_files + symlink_dirs
2519
2520 to_copy = []
2521 if copy_all:
Renaud Paquaybed8b622018-09-27 10:46:58 -07002522 to_copy = platform_utils.listdir(gitdir)
David James8d201162013-10-11 17:03:19 -07002523
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002524 dotgit = platform_utils.realpath(dotgit)
David James8d201162013-10-11 17:03:19 -07002525 for name in set(to_copy).union(to_symlink):
2526 try:
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002527 src = platform_utils.realpath(os.path.join(gitdir, name))
Dan Willemsen2a3e1522015-07-30 20:43:33 -07002528 dst = os.path.join(dotgit, name)
David James8d201162013-10-11 17:03:19 -07002529
Kevin Degi384b3c52014-10-16 16:02:58 -06002530 if os.path.lexists(dst):
2531 continue
David James8d201162013-10-11 17:03:19 -07002532
2533 # If the source dir doesn't exist, create an empty dir.
2534 if name in symlink_dirs and not os.path.lexists(src):
2535 os.makedirs(src)
2536
Cheuk Leung8ac0c962015-07-06 21:33:00 +02002537 if name in to_symlink:
Renaud Paquayd5cec5e2016-11-01 11:24:03 -07002538 platform_utils.symlink(
2539 os.path.relpath(src, os.path.dirname(dst)), dst)
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002540 elif copy_all and not platform_utils.islink(dst):
Renaud Paquaybed8b622018-09-27 10:46:58 -07002541 if platform_utils.isdir(src):
Cheuk Leung8ac0c962015-07-06 21:33:00 +02002542 shutil.copytree(src, dst)
2543 elif os.path.isfile(src):
2544 shutil.copy(src, dst)
2545
Conley Owens80b87fe2014-05-09 17:13:44 -07002546 # If the source file doesn't exist, ensure the destination
2547 # file doesn't either.
2548 if name in symlink_files and not os.path.lexists(src):
2549 try:
Renaud Paquay010fed72016-11-11 14:25:29 -08002550 platform_utils.remove(dst)
Conley Owens80b87fe2014-05-09 17:13:44 -07002551 except OSError:
2552 pass
2553
David James8d201162013-10-11 17:03:19 -07002554 except OSError as e:
2555 if e.errno == errno.EPERM:
Renaud Paquay788e9622017-01-27 11:41:12 -08002556 raise DownloadError(self._get_symlink_error_message())
David James8d201162013-10-11 17:03:19 -07002557 else:
2558 raise
2559
Martin Kellye4e94d22017-03-21 16:05:12 -07002560 def _InitWorkTree(self, force_sync=False, submodules=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002561 dotgit = os.path.join(self.worktree, '.git')
Kevin Degi384b3c52014-10-16 16:02:58 -06002562 init_dotgit = not os.path.exists(dotgit)
Kevin Degib1a07b82015-07-27 13:33:43 -06002563 try:
2564 if init_dotgit:
2565 os.makedirs(dotgit)
2566 self._ReferenceGitDir(self.gitdir, dotgit, share_refs=True,
2567 copy_all=False)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002568
Kevin Degiabaa7f32014-11-12 11:27:45 -07002569 try:
2570 self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
2571 except GitError as e:
2572 if force_sync:
2573 try:
Renaud Paquaya65adf72016-11-03 10:37:53 -07002574 platform_utils.rmtree(dotgit)
Martin Kellye4e94d22017-03-21 16:05:12 -07002575 return self._InitWorkTree(force_sync=False, submodules=submodules)
Kevin Degiabaa7f32014-11-12 11:27:45 -07002576 except:
2577 raise e
2578 raise e
Kevin Degi384b3c52014-10-16 16:02:58 -06002579
Kevin Degib1a07b82015-07-27 13:33:43 -06002580 if init_dotgit:
2581 _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002582
Kevin Degib1a07b82015-07-27 13:33:43 -06002583 cmd = ['read-tree', '--reset', '-u']
2584 cmd.append('-v')
2585 cmd.append(HEAD)
2586 if GitCommand(self, cmd).Wait() != 0:
2587 raise GitError("cannot initialize work tree")
Victor Boivie0960b5b2010-11-26 13:42:13 +01002588
Martin Kellye4e94d22017-03-21 16:05:12 -07002589 if submodules:
2590 self._SyncSubmodules(quiet=True)
Kevin Degib1a07b82015-07-27 13:33:43 -06002591 self._CopyAndLinkFiles()
2592 except Exception:
2593 if init_dotgit:
Renaud Paquaya65adf72016-11-03 10:37:53 -07002594 platform_utils.rmtree(dotgit)
Kevin Degib1a07b82015-07-27 13:33:43 -06002595 raise
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002596
Renaud Paquay788e9622017-01-27 11:41:12 -08002597 def _get_symlink_error_message(self):
2598 if platform_utils.isWindows():
2599 return ('Unable to create symbolic link. Please re-run the command as '
2600 'Administrator, or see '
2601 'https://github.com/git-for-windows/git/wiki/Symbolic-Links '
2602 'for other options.')
2603 return 'filesystem must support symlinks'
2604
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002605 def _gitdir_path(self, path):
Renaud Paquay227ad2e2016-11-01 14:37:13 -07002606 return platform_utils.realpath(os.path.join(self.gitdir, path))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002607
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002608 def _revlist(self, *args, **kw):
2609 a = []
2610 a.extend(args)
2611 a.append('--')
2612 return self.work_git.rev_list(*a, **kw)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002613
2614 @property
2615 def _allrefs(self):
Shawn O. Pearced237b692009-04-17 18:49:50 -07002616 return self.bare_ref.all
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002617
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002618 def _getLogs(self, rev1, rev2, oneline=False, color=True, pretty_format=None):
Julien Camperguedd654222014-01-09 16:21:37 +01002619 """Get logs between two revisions of this project."""
2620 comp = '..'
2621 if rev1:
2622 revs = [rev1]
2623 if rev2:
2624 revs.extend([comp, rev2])
2625 cmd = ['log', ''.join(revs)]
2626 out = DiffColoring(self.config)
2627 if out.is_on and color:
2628 cmd.append('--color')
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002629 if pretty_format is not None:
2630 cmd.append('--pretty=format:%s' % pretty_format)
Julien Camperguedd654222014-01-09 16:21:37 +01002631 if oneline:
2632 cmd.append('--oneline')
2633
2634 try:
2635 log = GitCommand(self, cmd, capture_stdout=True, capture_stderr=True)
2636 if log.Wait() == 0:
2637 return log.stdout
2638 except GitError:
2639 # worktree may not exist if groups changed for example. In that case,
2640 # try in gitdir instead.
2641 if not os.path.exists(self.worktree):
2642 return self.bare_git.log(*cmd[1:])
2643 else:
2644 raise
2645 return None
2646
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002647 def getAddedAndRemovedLogs(self, toProject, oneline=False, color=True,
2648 pretty_format=None):
Julien Camperguedd654222014-01-09 16:21:37 +01002649 """Get the list of logs from this revision to given revisionId"""
2650 logs = {}
2651 selfId = self.GetRevisionId(self._allrefs)
2652 toId = toProject.GetRevisionId(toProject._allrefs)
2653
Sebastian Schuberth7ecccf62016-03-29 14:11:20 +02002654 logs['added'] = self._getLogs(selfId, toId, oneline=oneline, color=color,
2655 pretty_format=pretty_format)
2656 logs['removed'] = self._getLogs(toId, selfId, oneline=oneline, color=color,
2657 pretty_format=pretty_format)
Julien Camperguedd654222014-01-09 16:21:37 +01002658 return logs
2659
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002660 class _GitGetByExec(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002661
David James8d201162013-10-11 17:03:19 -07002662 def __init__(self, project, bare, gitdir):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002663 self._project = project
2664 self._bare = bare
David James8d201162013-10-11 17:03:19 -07002665 self._gitdir = gitdir
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002666
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002667 def LsOthers(self):
2668 p = GitCommand(self._project,
2669 ['ls-files',
2670 '-z',
2671 '--others',
2672 '--exclude-standard'],
Anthony King7bdac712014-07-16 12:56:40 +01002673 bare=False,
David James8d201162013-10-11 17:03:19 -07002674 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002675 capture_stdout=True,
2676 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002677 if p.Wait() == 0:
2678 out = p.stdout
2679 if out:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002680 # Backslash is not anomalous
David Pursehouse65b0ba52018-06-24 16:21:51 +09002681 return out[:-1].split('\0')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002682 return []
2683
2684 def DiffZ(self, name, *args):
2685 cmd = [name]
2686 cmd.append('-z')
2687 cmd.extend(args)
2688 p = GitCommand(self._project,
2689 cmd,
David James8d201162013-10-11 17:03:19 -07002690 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002691 bare=False,
2692 capture_stdout=True,
2693 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002694 try:
2695 out = p.process.stdout.read()
2696 r = {}
2697 if out:
David Pursehouse65b0ba52018-06-24 16:21:51 +09002698 out = iter(out[:-1].split('\0'))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002699 while out:
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002700 try:
Anthony King2cd1f042014-05-05 21:24:05 +01002701 info = next(out)
2702 path = next(out)
Shawn O. Pearce02dbb6d2008-10-21 13:59:08 -07002703 except StopIteration:
2704 break
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002705
2706 class _Info(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002707
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002708 def __init__(self, path, omode, nmode, oid, nid, state):
2709 self.path = path
2710 self.src_path = None
2711 self.old_mode = omode
2712 self.new_mode = nmode
2713 self.old_id = oid
2714 self.new_id = nid
2715
2716 if len(state) == 1:
2717 self.status = state
2718 self.level = None
2719 else:
2720 self.status = state[:1]
2721 self.level = state[1:]
2722 while self.level.startswith('0'):
2723 self.level = self.level[1:]
2724
2725 info = info[1:].split(' ')
David Pursehouse8f62fb72012-11-14 12:09:38 +09002726 info = _Info(path, *info)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002727 if info.status in ('R', 'C'):
2728 info.src_path = info.path
Anthony King2cd1f042014-05-05 21:24:05 +01002729 info.path = next(out)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002730 r[info.path] = info
2731 return r
2732 finally:
2733 p.Wait()
2734
2735 def GetHead(self):
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002736 if self._bare:
2737 path = os.path.join(self._project.gitdir, HEAD)
2738 else:
2739 path = os.path.join(self._project.worktree, '.git', HEAD)
Conley Owens75ee0572012-11-15 17:33:11 -08002740 try:
Renaud Paquay2a4be942016-11-01 13:48:15 -07002741 fd = open(path)
Dan Sandler53e902a2014-03-09 13:20:02 -04002742 except IOError as e:
2743 raise NoManifestException(path, str(e))
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -07002744 try:
Renaud Paquay2a4be942016-11-01 13:48:15 -07002745 line = fd.readline()
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -07002746 finally:
2747 fd.close()
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302748 try:
2749 line = line.decode()
2750 except AttributeError:
2751 pass
Shawn O. Pearce5b23f242009-04-17 18:43:33 -07002752 if line.startswith('ref: '):
2753 return line[5:-1]
2754 return line[:-1]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002755
2756 def SetHead(self, ref, message=None):
2757 cmdv = []
2758 if message is not None:
2759 cmdv.extend(['-m', message])
2760 cmdv.append(HEAD)
2761 cmdv.append(ref)
2762 self.symbolic_ref(*cmdv)
2763
2764 def DetachHead(self, new, message=None):
2765 cmdv = ['--no-deref']
2766 if message is not None:
2767 cmdv.extend(['-m', message])
2768 cmdv.append(HEAD)
2769 cmdv.append(new)
2770 self.update_ref(*cmdv)
2771
2772 def UpdateRef(self, name, new, old=None,
2773 message=None,
2774 detach=False):
2775 cmdv = []
2776 if message is not None:
2777 cmdv.extend(['-m', message])
2778 if detach:
2779 cmdv.append('--no-deref')
2780 cmdv.append(name)
2781 cmdv.append(new)
2782 if old is not None:
2783 cmdv.append(old)
2784 self.update_ref(*cmdv)
2785
2786 def DeleteRef(self, name, old=None):
2787 if not old:
2788 old = self.rev_parse(name)
2789 self.update_ref('-d', name, old)
Shawn O. Pearcefbcde472009-04-17 20:58:02 -07002790 self._project.bare_ref.deleted(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002791
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002792 def rev_list(self, *args, **kw):
2793 if 'format' in kw:
2794 cmdv = ['log', '--pretty=format:%s' % kw['format']]
2795 else:
2796 cmdv = ['rev-list']
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002797 cmdv.extend(args)
2798 p = GitCommand(self._project,
2799 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002800 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002801 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002802 capture_stdout=True,
2803 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002804 r = []
2805 for line in p.process.stdout:
Shawn O. Pearce8ad8a0e2009-05-29 18:28:25 -07002806 if line[-1] == '\n':
2807 line = line[:-1]
2808 r.append(line)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002809 if p.Wait() != 0:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002810 raise GitError('%s rev-list %s: %s' %
2811 (self._project.name, str(args), p.stderr))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002812 return r
2813
2814 def __getattr__(self, name):
Doug Anderson37282b42011-03-04 11:54:18 -08002815 """Allow arbitrary git commands using pythonic syntax.
2816
2817 This allows you to do things like:
2818 git_obj.rev_parse('HEAD')
2819
2820 Since we don't have a 'rev_parse' method defined, the __getattr__ will
2821 run. We'll replace the '_' with a '-' and try to run a git command.
Dave Borowitz091f8932012-10-23 17:01:04 -07002822 Any other positional arguments will be passed to the git command, and the
2823 following keyword arguments are supported:
2824 config: An optional dict of git config options to be passed with '-c'.
Doug Anderson37282b42011-03-04 11:54:18 -08002825
2826 Args:
2827 name: The name of the git command to call. Any '_' characters will
2828 be replaced with '-'.
2829
2830 Returns:
2831 A callable object that will try to call git with the named command.
2832 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002833 name = name.replace('_', '-')
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002834
Dave Borowitz091f8932012-10-23 17:01:04 -07002835 def runner(*args, **kwargs):
2836 cmdv = []
2837 config = kwargs.pop('config', None)
2838 for k in kwargs:
2839 raise TypeError('%s() got an unexpected keyword argument %r'
2840 % (name, k))
2841 if config is not None:
Dave Borowitzb42b4742012-10-31 12:27:27 -07002842 if not git_require((1, 7, 2)):
2843 raise ValueError('cannot set config on command line for %s()'
2844 % name)
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302845 for k, v in config.items():
Dave Borowitz091f8932012-10-23 17:01:04 -07002846 cmdv.append('-c')
2847 cmdv.append('%s=%s' % (k, v))
2848 cmdv.append(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002849 cmdv.extend(args)
2850 p = GitCommand(self._project,
2851 cmdv,
Anthony King7bdac712014-07-16 12:56:40 +01002852 bare=self._bare,
David James8d201162013-10-11 17:03:19 -07002853 gitdir=self._gitdir,
Anthony King7bdac712014-07-16 12:56:40 +01002854 capture_stdout=True,
2855 capture_stderr=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002856 if p.Wait() != 0:
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002857 raise GitError('%s %s: %s' %
2858 (self._project.name, name, p.stderr))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002859 r = p.stdout
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302860 try:
Conley Owensedd01512013-09-26 12:59:58 -07002861 r = r.decode('utf-8')
Chirayu Desai217ea7d2013-03-01 19:14:38 +05302862 except AttributeError:
2863 pass
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002864 if r.endswith('\n') and r.index('\n') == len(r) - 1:
2865 return r[:-1]
2866 return r
2867 return runner
2868
2869
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002870class _PriorSyncFailedError(Exception):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002871
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002872 def __str__(self):
2873 return 'prior sync failed; rebase still in progress'
2874
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002875
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002876class _DirtyError(Exception):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002877
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002878 def __str__(self):
2879 return 'contains uncommitted changes'
2880
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002881
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002882class _InfoMessage(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002883
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002884 def __init__(self, project, text):
2885 self.project = project
2886 self.text = text
2887
2888 def Print(self, syncbuf):
2889 syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
2890 syncbuf.out.nl()
2891
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002892
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002893class _Failure(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002894
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002895 def __init__(self, project, why):
2896 self.project = project
2897 self.why = why
2898
2899 def Print(self, syncbuf):
2900 syncbuf.out.fail('error: %s/: %s',
2901 self.project.relpath,
2902 str(self.why))
2903 syncbuf.out.nl()
2904
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002905
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002906class _Later(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002907
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002908 def __init__(self, project, action):
2909 self.project = project
2910 self.action = action
2911
2912 def Run(self, syncbuf):
2913 out = syncbuf.out
2914 out.project('project %s/', self.project.relpath)
2915 out.nl()
2916 try:
2917 self.action()
2918 out.nl()
2919 return True
David Pursehouse8a68ff92012-09-24 12:15:13 +09002920 except GitError:
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002921 out.nl()
2922 return False
2923
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002924
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002925class _SyncColoring(Coloring):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002926
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002927 def __init__(self, config):
2928 Coloring.__init__(self, config, 'reposync')
Anthony King7bdac712014-07-16 12:56:40 +01002929 self.project = self.printer('header', attr='bold')
2930 self.info = self.printer('info')
2931 self.fail = self.printer('fail', fg='red')
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002932
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002933
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002934class SyncBuffer(object):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07002935
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002936 def __init__(self, config, detach_head=False):
2937 self._messages = []
2938 self._failures = []
2939 self._later_queue1 = []
2940 self._later_queue2 = []
2941
2942 self.out = _SyncColoring(config)
2943 self.out.redirect(sys.stderr)
2944
2945 self.detach_head = detach_head
2946 self.clean = True
David Rileye0684ad2017-04-05 00:02:59 -07002947 self.recent_clean = True
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002948
2949 def info(self, project, fmt, *args):
2950 self._messages.append(_InfoMessage(project, fmt % args))
2951
2952 def fail(self, project, err=None):
2953 self._failures.append(_Failure(project, err))
David Rileye0684ad2017-04-05 00:02:59 -07002954 self._MarkUnclean()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002955
2956 def later1(self, project, what):
2957 self._later_queue1.append(_Later(project, what))
2958
2959 def later2(self, project, what):
2960 self._later_queue2.append(_Later(project, what))
2961
2962 def Finish(self):
2963 self._PrintMessages()
2964 self._RunLater()
2965 self._PrintMessages()
2966 return self.clean
2967
David Rileye0684ad2017-04-05 00:02:59 -07002968 def Recently(self):
2969 recent_clean = self.recent_clean
2970 self.recent_clean = True
2971 return recent_clean
2972
2973 def _MarkUnclean(self):
2974 self.clean = False
2975 self.recent_clean = False
2976
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002977 def _RunLater(self):
2978 for q in ['_later_queue1', '_later_queue2']:
2979 if not self._RunQueue(q):
2980 return
2981
2982 def _RunQueue(self, queue):
2983 for m in getattr(self, queue):
2984 if not m.Run(self):
David Rileye0684ad2017-04-05 00:02:59 -07002985 self._MarkUnclean()
Shawn O. Pearce350cde42009-04-16 11:21:18 -07002986 return False
2987 setattr(self, queue, [])
2988 return True
2989
2990 def _PrintMessages(self):
2991 for m in self._messages:
2992 m.Print(self)
2993 for m in self._failures:
2994 m.Print(self)
2995
2996 self._messages = []
2997 self._failures = []
2998
2999
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003000class MetaProject(Project):
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003001
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003002 """A special project housed under .repo.
3003 """
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003004
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003005 def __init__(self, manifest, name, gitdir, worktree):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003006 Project.__init__(self,
Anthony King7bdac712014-07-16 12:56:40 +01003007 manifest=manifest,
3008 name=name,
3009 gitdir=gitdir,
3010 objdir=gitdir,
3011 worktree=worktree,
3012 remote=RemoteSpec('origin'),
3013 relpath='.repo/%s' % name,
3014 revisionExpr='refs/heads/master',
3015 revisionId=None,
3016 groups=None)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003017
3018 def PreSync(self):
3019 if self.Exists:
3020 cb = self.CurrentBranch
3021 if cb:
3022 base = self.GetBranch(cb).merge
3023 if base:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07003024 self.revisionExpr = base
3025 self.revisionId = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003026
Martin Kelly224a31a2017-07-10 14:46:25 -07003027 def MetaBranchSwitch(self, submodules=False):
Florian Vallee5d016502012-06-07 17:19:26 +02003028 """ Prepare MetaProject for manifest branch switch
3029 """
3030
3031 # detach and delete manifest branch, allowing a new
3032 # branch to take over
Anthony King7bdac712014-07-16 12:56:40 +01003033 syncbuf = SyncBuffer(self.config, detach_head=True)
Martin Kelly224a31a2017-07-10 14:46:25 -07003034 self.Sync_LocalHalf(syncbuf, submodules=submodules)
Florian Vallee5d016502012-06-07 17:19:26 +02003035 syncbuf.Finish()
3036
3037 return GitCommand(self,
Mark E. Hamilton30b0f4e2016-02-10 10:44:30 -07003038 ['update-ref', '-d', 'refs/heads/default'],
3039 capture_stdout=True,
3040 capture_stderr=True).Wait() == 0
Florian Vallee5d016502012-06-07 17:19:26 +02003041
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003042 @property
Shawn O. Pearcef6906872009-04-18 10:49:00 -07003043 def LastFetch(self):
3044 try:
3045 fh = os.path.join(self.gitdir, 'FETCH_HEAD')
3046 return os.path.getmtime(fh)
3047 except OSError:
3048 return 0
3049
3050 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003051 def HasChanges(self):
3052 """Has the remote received new commits not yet checked out?
3053 """
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07003054 if not self.remote or not self.revisionExpr:
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07003055 return False
3056
David Pursehouse8a68ff92012-09-24 12:15:13 +09003057 all_refs = self.bare_ref.all
3058 revid = self.GetRevisionId(all_refs)
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07003059 head = self.work_git.GetHead()
3060 if head.startswith(R_HEADS):
3061 try:
David Pursehouse8a68ff92012-09-24 12:15:13 +09003062 head = all_refs[head]
Shawn O. Pearce336f7bd2009-04-18 10:39:28 -07003063 except KeyError:
3064 head = None
3065
3066 if revid == head:
3067 return False
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07003068 elif self._revlist(not_rev(HEAD), revid):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07003069 return True
3070 return False