blob: f573b980244aab915b312062402b8a726b0d1dd0 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
David Pursehouse86d973d2012-08-24 10:21:02 +090016import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070017from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import os
19import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070020import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070021import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070022import subprocess
23import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070024import time
David Pursehouse86d973d2012-08-24 10:21:02 +090025import urlparse
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070026import xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070027
Roy Lee18afd7f2010-05-09 04:32:08 +080028try:
29 import threading as _threading
30except ImportError:
31 import dummy_threading as _threading
32
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070033try:
34 import resource
35 def _rlimit_nofile():
36 return resource.getrlimit(resource.RLIMIT_NOFILE)
37except ImportError:
38 def _rlimit_nofile():
39 return (256, 256)
40
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070041from git_command import GIT
David Pursehoused94aaef2012-09-07 09:52:04 +090042from git_refs import R_HEADS, HEAD
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070043from project import Project
44from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080045from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070046from error import RepoChangedException, GitError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070047from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070048from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070049
Doug Andersonfc06ced2011-03-16 15:49:18 -070050class _FetchError(Exception):
51 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
52 pass
53
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080054class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080055 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070056 common = True
57 helpSummary = "Update working tree to the latest revision"
58 helpUsage = """
59%prog [<project>...]
60"""
61 helpDescription = """
62The '%prog' command synchronizes local project directories
63with the remote repositories specified in the manifest. If a local
64project does not yet exist, it will clone a new local directory from
65the remote repository and set up tracking branches as specified in
66the manifest. If the local project already exists, '%prog'
67will update the remote branches and rebase any new local changes
68on top of the new remote changes.
69
70'%prog' will synchronize all projects listed at the command
71line. Projects can be specified either by name, or by a relative
72or absolute path to the project's local directory. If no projects
73are specified, '%prog' will synchronize all projects listed in
74the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070075
76The -d/--detach option can be used to switch specified projects
77back to the manifest revision. This option is especially helpful
78if the project is currently on a topic branch, but the manifest
79revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070080
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070081The -s/--smart-sync option can be used to sync to a known good
82build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +020083manifest. The -t/--smart-tag option is similar and allows you to
84specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070085
David Pursehousecf76b1b2012-09-14 10:31:42 +090086The -u/--manifest-server-username and -p/--manifest-server-password
87options can be used to specify a username and password to authenticate
88with the manifest server when using the -s or -t option.
89
90If -u and -p are not specified when using the -s or -t option, '%prog'
91will attempt to read authentication credentials for the manifest server
92from the user's .netrc file.
93
94'%prog' will not use authentication credentials from -u/-p or .netrc
95if the manifest server specified in the manifest file already includes
96credentials.
97
Andrei Warkentin5df6de02010-07-02 17:58:31 -050098The -f/--force-broken option can be used to proceed with syncing
99other projects if a project sync fails.
100
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700101The --no-clone-bundle option disables any attempt to use
102$URL/clone.bundle to bootstrap a new Git repository from a
103resumeable bundle file on a content delivery network. This
104may be necessary if there are problems with the local Python
105HTTP client or proxy configuration, but the Git binary works.
106
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700107SSH Connections
108---------------
109
110If at least one project remote URL uses an SSH connection (ssh://,
111git+ssh://, or user@host:path syntax) repo will automatically
112enable the SSH ControlMaster option when connecting to that host.
113This feature permits other projects in the same '%prog' session to
114reuse the same SSH tunnel, saving connection setup overheads.
115
116To disable this behavior on UNIX platforms, set the GIT_SSH
117environment variable to 'ssh'. For example:
118
119 export GIT_SSH=ssh
120 %prog
121
122Compatibility
123~~~~~~~~~~~~~
124
125This feature is automatically disabled on Windows, due to the lack
126of UNIX domain socket support.
127
128This feature is not compatible with url.insteadof rewrites in the
129user's ~/.gitconfig. '%prog' is currently not able to perform the
130rewrite early enough to establish the ControlMaster tunnel.
131
132If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
133later is required to fix a server side protocol bug.
134
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700135"""
136
Nico Sallembien6623b212010-05-11 12:57:01 -0700137 def _Options(self, p, show_smart=True):
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700138 self.jobs = self.manifest.default.sync_j
139
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500140 p.add_option('-f', '--force-broken',
141 dest='force_broken', action='store_true',
142 help="continue sync even if a project fails to sync")
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700143 p.add_option('-l','--local-only',
144 dest='local_only', action='store_true',
145 help="only update working tree, don't fetch")
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700146 p.add_option('-n','--network-only',
147 dest='network_only', action='store_true',
148 help="fetch only, don't update working tree")
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700149 p.add_option('-d','--detach',
150 dest='detach_head', action='store_true',
151 help='detach projects back to manifest revision')
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700152 p.add_option('-c','--current-branch',
153 dest='current_branch_only', action='store_true',
154 help='fetch only current branch from server')
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700155 p.add_option('-q','--quiet',
156 dest='quiet', action='store_true',
157 help='be more quiet')
Roy Lee18afd7f2010-05-09 04:32:08 +0800158 p.add_option('-j','--jobs',
159 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700160 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500161 p.add_option('-m', '--manifest-name',
162 dest='manifest_name',
163 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700164 p.add_option('--no-clone-bundle',
165 dest='no_clone_bundle', action='store_true',
166 help='disable use of /clone.bundle on HTTP/HTTPS')
Nico Sallembien6623b212010-05-11 12:57:01 -0700167 if show_smart:
168 p.add_option('-s', '--smart-sync',
169 dest='smart_sync', action='store_true',
170 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200171 p.add_option('-t', '--smart-tag',
172 dest='smart_tag', action='store',
173 help='smart sync using manifest from a known tag')
David Pursehousecf76b1b2012-09-14 10:31:42 +0900174 p.add_option('-u', '--manifest-server-username', action='store',
175 dest='manifest_server_username',
176 help='username to authenticate with the manifest server')
177 p.add_option('-p', '--manifest-server-password', action='store',
178 dest='manifest_server_password',
179 help='password to authenticate with the manifest server')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700180
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700181 g = p.add_option_group('repo Version options')
182 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700183 dest='no_repo_verify', action='store_true',
184 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700185 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800186 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700187 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700188
Doug Andersonfc06ced2011-03-16 15:49:18 -0700189 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
190 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800191
Doug Andersonfc06ced2011-03-16 15:49:18 -0700192 Args:
193 opt: Program options returned from optparse. See _Options().
194 project: Project object for the project to fetch.
195 lock: Lock for accessing objects that are shared amongst multiple
196 _FetchHelper() threads.
197 fetched: set object that we will add project.gitdir to when we're done
198 (with our lock held).
199 pm: Instance of a Project object. We will call pm.update() (with our
200 lock held).
201 sem: We'll release() this semaphore when we exit so that another thread
202 can be started up.
203 err_event: We'll set this event in the case of an error (after printing
204 out info about the error).
205 """
206 # We'll set to true once we've locked the lock.
207 did_lock = False
208
209 # Encapsulate everything in a try/except/finally so that:
210 # - We always set err_event in the case of an exception.
211 # - We always make sure we call sem.release().
212 # - We always make sure we unlock the lock if we locked it.
213 try:
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700214 try:
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700215 success = project.Sync_NetworkHalf(
216 quiet=opt.quiet,
217 current_branch_only=opt.current_branch_only,
218 clone_bundle=not opt.no_clone_bundle)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700219
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700220 # Lock around all the rest of the code, since printing, updating a set
221 # and Progress.update() are not thread safe.
222 lock.acquire()
223 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700224
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700225 if not success:
226 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
227 if opt.force_broken:
228 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
229 else:
230 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700231
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700232 fetched.add(project.gitdir)
233 pm.update()
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -0700234 except _FetchError:
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700235 err_event.set()
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -0700236 except:
237 err_event.set()
238 raise
Doug Andersonfc06ced2011-03-16 15:49:18 -0700239 finally:
240 if did_lock:
241 lock.release()
242 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800243
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700244 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700245 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700246 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800247
248 if self.jobs == 1:
249 for project in projects:
250 pm.update()
Shawn O. Pearce5d0efdb2012-08-02 12:13:01 -0700251 if project.Sync_NetworkHalf(
252 quiet=opt.quiet,
253 current_branch_only=opt.current_branch_only,
254 clone_bundle=not opt.no_clone_bundle):
Roy Lee18afd7f2010-05-09 04:32:08 +0800255 fetched.add(project.gitdir)
256 else:
257 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500258 if opt.force_broken:
259 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
260 else:
261 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800262 else:
263 threads = set()
264 lock = _threading.Lock()
265 sem = _threading.Semaphore(self.jobs)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700266 err_event = _threading.Event()
Roy Lee18afd7f2010-05-09 04:32:08 +0800267 for project in projects:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700268 # Check for any errors before starting any new threads.
269 # ...we'll let existing threads finish, though.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400270 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700271 break
272
Roy Lee18afd7f2010-05-09 04:32:08 +0800273 sem.acquire()
274 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700275 args = (opt,
276 project,
277 lock,
278 fetched,
279 pm,
Doug Andersonfc06ced2011-03-16 15:49:18 -0700280 sem,
281 err_event))
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200282 # Ensure that Ctrl-C will not freeze the repo process.
283 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800284 threads.add(t)
285 t.start()
286
287 for t in threads:
288 t.join()
289
Doug Andersonfc06ced2011-03-16 15:49:18 -0700290 # If we saw an error, exit with code 1 so that other scripts can check.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400291 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700292 print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
293 sys.exit(1)
294
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700295 pm.end()
Shawn O. Pearce0d2b61f2009-07-03 15:22:49 -0700296 for project in projects:
297 project.bare_git.gc('--auto')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700298 return fetched
299
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700300 def UpdateProjectList(self):
301 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700302 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700303 if project.relpath:
304 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700305 file_name = 'project.list'
306 file_path = os.path.join(self.manifest.repodir, file_name)
307 old_project_paths = []
308
309 if os.path.exists(file_path):
310 fd = open(file_path, 'r')
311 try:
312 old_project_paths = fd.read().split('\n')
313 finally:
314 fd.close()
315 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700316 if not path:
317 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700318 if path not in new_project_paths:
Anthonyf3fdf822009-09-26 13:38:52 -0400319 """If the path has already been deleted, we don't need to do it
320 """
321 if os.path.exists(self.manifest.topdir + '/' + path):
322 project = Project(
323 manifest = self.manifest,
324 name = path,
325 remote = RemoteSpec('origin'),
326 gitdir = os.path.join(self.manifest.topdir,
327 path, '.git'),
328 worktree = os.path.join(self.manifest.topdir, path),
329 relpath = path,
330 revisionExpr = 'HEAD',
Colin Cross5acde752012-03-28 20:15:45 -0700331 revisionId = None,
332 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400333
334 if project.IsDirty():
335 print >>sys.stderr, 'error: Cannot remove project "%s": \
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700336uncommitted changes are present' % project.relpath
Anthonyf3fdf822009-09-26 13:38:52 -0400337 print >>sys.stderr, ' commit changes, then run sync again'
338 return -1
339 else:
340 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
341 shutil.rmtree(project.worktree)
342 # Try deleting parent subdirs if they are empty
343 dir = os.path.dirname(project.worktree)
344 while dir != self.manifest.topdir:
345 try:
346 os.rmdir(dir)
347 except OSError:
348 break
349 dir = os.path.dirname(dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700350
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700351 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700352 fd = open(file_path, 'w')
353 try:
354 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700355 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700356 finally:
357 fd.close()
358 return 0
359
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700360 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800361 if opt.jobs:
362 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700363 if self.jobs > 1:
364 soft_limit, _ = _rlimit_nofile()
365 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
366
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700367 if opt.network_only and opt.detach_head:
368 print >>sys.stderr, 'error: cannot combine -n and -d'
369 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700370 if opt.network_only and opt.local_only:
371 print >>sys.stderr, 'error: cannot combine -n and -l'
372 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500373 if opt.manifest_name and opt.smart_sync:
374 print >>sys.stderr, 'error: cannot combine -m and -s'
375 sys.exit(1)
376 if opt.manifest_name and opt.smart_tag:
377 print >>sys.stderr, 'error: cannot combine -m and -t'
378 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900379 if opt.manifest_server_username or opt.manifest_server_password:
380 if not (opt.smart_sync or opt.smart_tag):
381 print >>sys.stderr, 'error: -u and -p may only be combined with ' \
382 '-s or -t'
383 sys.exit(1)
384 if None in [opt.manifest_server_username, opt.manifest_server_password]:
385 print >>sys.stderr, 'error: both -u and -p must be given'
386 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500387
388 if opt.manifest_name:
389 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700390
Victor Boivie08c880d2011-04-19 10:32:52 +0200391 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700392 if not self.manifest.manifest_server:
393 print >>sys.stderr, \
394 'error: cannot smart sync: no manifest server defined in manifest'
395 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900396
397 manifest_server = self.manifest.manifest_server
David Pursehousecf76b1b2012-09-14 10:31:42 +0900398
David Pursehouse86d973d2012-08-24 10:21:02 +0900399 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900400 username = None
401 password = None
402 if opt.manifest_server_username and opt.manifest_server_password:
403 username = opt.manifest_server_username
404 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900405 else:
406 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900407 info = netrc.netrc()
408 except IOError:
409 print >>sys.stderr, '.netrc file does not exist or could not be opened'
David Pursehouse86d973d2012-08-24 10:21:02 +0900410 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900411 try:
412 parse_result = urlparse.urlparse(manifest_server)
413 if parse_result.hostname:
414 username, _account, password = \
415 info.authenticators(parse_result.hostname)
416 except TypeError:
417 # TypeError is raised when the given hostname is not present
418 # in the .netrc file.
419 print >>sys.stderr, 'No credentials found for %s in .netrc' % \
420 parse_result.hostname
421 except netrc.NetrcParseError, e:
422 print >>sys.stderr, 'Error parsing .netrc file: %s' % e
423
424 if (username and password):
425 manifest_server = manifest_server.replace('://', '://%s:%s@' %
426 (username, password),
427 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900428
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700429 try:
David Pursehouse86d973d2012-08-24 10:21:02 +0900430 server = xmlrpclib.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200431 if opt.smart_sync:
432 p = self.manifest.manifestProject
433 b = p.GetBranch(p.CurrentBranch)
434 branch = b.merge
435 if branch.startswith(R_HEADS):
436 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700437
Victor Boivie08c880d2011-04-19 10:32:52 +0200438 env = os.environ.copy()
439 if (env.has_key('TARGET_PRODUCT') and
440 env.has_key('TARGET_BUILD_VARIANT')):
441 target = '%s-%s' % (env['TARGET_PRODUCT'],
442 env['TARGET_BUILD_VARIANT'])
443 [success, manifest_str] = server.GetApprovedManifest(branch, target)
444 else:
445 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700446 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200447 assert(opt.smart_tag)
448 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700449
450 if success:
451 manifest_name = "smart_sync_override.xml"
452 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
453 manifest_name)
454 try:
455 f = open(manifest_path, 'w')
456 try:
457 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700458 finally:
459 f.close()
460 except IOError:
461 print >>sys.stderr, 'error: cannot write manifest to %s' % \
462 manifest_path
463 sys.exit(1)
Nico Sallembien719965a2010-04-20 15:28:19 -0700464 self.manifest.Override(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700465 else:
466 print >>sys.stderr, 'error: %s' % manifest_str
467 sys.exit(1)
David Pursehousebd489c42012-08-23 10:21:26 +0900468 except (socket.error, IOError, xmlrpclib.Fault), e:
469 print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
470 self.manifest.manifest_server, e)
471 sys.exit(1)
472 except xmlrpclib.ProtocolError, e:
473 print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
474 self.manifest.manifest_server, e.errcode, e.errmsg)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700475 sys.exit(1)
476
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700477 rp = self.manifest.repoProject
478 rp.PreSync()
479
480 mp = self.manifest.manifestProject
481 mp.PreSync()
482
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800483 if opt.repo_upgraded:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700484 _PostRepoUpgrade(self.manifest)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800485
Nico Sallembien9bb18162009-12-07 15:38:01 -0800486 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700487 mp.Sync_NetworkHalf(quiet=opt.quiet,
488 current_branch_only=opt.current_branch_only)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800489
490 if mp.HasChanges:
491 syncbuf = SyncBuffer(mp.config)
492 mp.Sync_LocalHalf(syncbuf)
493 if not syncbuf.Finish():
494 sys.exit(1)
495 self.manifest._Unload()
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700496 if opt.jobs is None:
497 self.jobs = self.manifest.default.sync_j
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700498 all = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700499
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700500 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700501 to_fetch = []
502 now = time.time()
503 if (24 * 60 * 60) <= (now - rp.LastFetch):
504 to_fetch.append(rp)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700505 to_fetch.extend(all)
506
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700507 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700508 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700509 if opt.network_only:
510 # bail out now; the rest touches the working tree
511 return
512
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700513 self.manifest._Unload()
514 all = self.GetProjects(args, missing_ok=True)
515 missing = []
516 for project in all:
517 if project.gitdir not in fetched:
518 missing.append(project)
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700519 self._Fetch(missing, opt)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700520
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700521 if self.manifest.IsMirror:
522 # bail out now, we have no working tree
523 return
524
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700525 if self.UpdateProjectList():
526 sys.exit(1)
527
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700528 syncbuf = SyncBuffer(mp.config,
529 detach_head = opt.detach_head)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700530 pm = Progress('Syncing work tree', len(all))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700531 for project in all:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700532 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800533 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700534 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700535 pm.end()
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700536 print >>sys.stderr
537 if not syncbuf.Finish():
538 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700539
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700540 # If there's a notice that's supposed to print at the end of the sync, print
541 # it now...
542 if self.manifest.notice:
543 print self.manifest.notice
544
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700545def _PostRepoUpgrade(manifest):
546 for project in manifest.projects.values():
547 if project.Exists:
548 project.PostRepoUpgrade()
549
550def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
551 if rp.HasChanges:
552 print >>sys.stderr, 'info: A new version of repo is available'
553 print >>sys.stderr, ''
554 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700555 syncbuf = SyncBuffer(rp.config)
556 rp.Sync_LocalHalf(syncbuf)
557 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700558 sys.exit(1)
559 print >>sys.stderr, 'info: Restarting repo with latest version'
560 raise RepoChangedException(['--repo-upgraded'])
561 else:
562 print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
563 else:
564 if verbose:
565 print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
566
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700567def _VerifyTag(project):
568 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
569 if not os.path.exists(gpg_dir):
570 print >>sys.stderr,\
571"""warning: GnuPG was not available during last "repo init"
572warning: Cannot automatically authenticate repo."""
573 return True
574
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700575 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700576 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700577 except GitError:
578 cur = None
579
580 if not cur \
581 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700582 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700583 if rev.startswith(R_HEADS):
584 rev = rev[len(R_HEADS):]
585
586 print >>sys.stderr
587 print >>sys.stderr,\
588 "warning: project '%s' branch '%s' is not signed" \
589 % (project.name, rev)
590 return False
591
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800592 env = os.environ.copy()
593 env['GIT_DIR'] = project.gitdir.encode()
594 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700595
596 cmd = [GIT, 'tag', '-v', cur]
597 proc = subprocess.Popen(cmd,
598 stdout = subprocess.PIPE,
599 stderr = subprocess.PIPE,
600 env = env)
601 out = proc.stdout.read()
602 proc.stdout.close()
603
604 err = proc.stderr.read()
605 proc.stderr.close()
606
607 if proc.wait() != 0:
608 print >>sys.stderr
609 print >>sys.stderr, out
610 print >>sys.stderr, err
611 print >>sys.stderr
612 return False
613 return True