blob: 42c5f91570450aa509190d8939fc6895cffc1f72 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
David Pursehouse86d973d2012-08-24 10:21:02 +090017import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070018from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Dave Borowitz67700e92012-10-23 15:00:54 -070020import pickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070026import time
David Pursehouse86d973d2012-08-24 10:21:02 +090027import urlparse
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070028import xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070029
Roy Lee18afd7f2010-05-09 04:32:08 +080030try:
31 import threading as _threading
32except ImportError:
33 import dummy_threading as _threading
34
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070035try:
36 import resource
37 def _rlimit_nofile():
38 return resource.getrlimit(resource.RLIMIT_NOFILE)
39except ImportError:
40 def _rlimit_nofile():
41 return (256, 256)
42
Dave Borowitz18857212012-10-23 17:02:59 -070043try:
44 import multiprocessing
45except ImportError:
46 multiprocessing = None
47
Dave Borowitze2152672012-10-31 12:24:38 -070048from git_command import GIT, git_require
David Pursehoused94aaef2012-09-07 09:52:04 +090049from git_refs import R_HEADS, HEAD
Conley Owensc9129d92012-10-01 16:12:28 -070050from main import WrapperModule
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070051from project import Project
52from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080053from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000054from error import RepoChangedException, GitError, ManifestParseError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070055from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070056from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070057
Dave Borowitz67700e92012-10-23 15:00:54 -070058_ONE_DAY_S = 24 * 60 * 60
59
Doug Andersonfc06ced2011-03-16 15:49:18 -070060class _FetchError(Exception):
61 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
62 pass
63
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080064class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080065 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070066 common = True
67 helpSummary = "Update working tree to the latest revision"
68 helpUsage = """
69%prog [<project>...]
70"""
71 helpDescription = """
72The '%prog' command synchronizes local project directories
73with the remote repositories specified in the manifest. If a local
74project does not yet exist, it will clone a new local directory from
75the remote repository and set up tracking branches as specified in
76the manifest. If the local project already exists, '%prog'
77will update the remote branches and rebase any new local changes
78on top of the new remote changes.
79
80'%prog' will synchronize all projects listed at the command
81line. Projects can be specified either by name, or by a relative
82or absolute path to the project's local directory. If no projects
83are specified, '%prog' will synchronize all projects listed in
84the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070085
86The -d/--detach option can be used to switch specified projects
87back to the manifest revision. This option is especially helpful
88if the project is currently on a topic branch, but the manifest
89revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070090
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070091The -s/--smart-sync option can be used to sync to a known good
92build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +020093manifest. The -t/--smart-tag option is similar and allows you to
94specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070095
David Pursehousecf76b1b2012-09-14 10:31:42 +090096The -u/--manifest-server-username and -p/--manifest-server-password
97options can be used to specify a username and password to authenticate
98with the manifest server when using the -s or -t option.
99
100If -u and -p are not specified when using the -s or -t option, '%prog'
101will attempt to read authentication credentials for the manifest server
102from the user's .netrc file.
103
104'%prog' will not use authentication credentials from -u/-p or .netrc
105if the manifest server specified in the manifest file already includes
106credentials.
107
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500108The -f/--force-broken option can be used to proceed with syncing
109other projects if a project sync fails.
110
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700111The --no-clone-bundle option disables any attempt to use
112$URL/clone.bundle to bootstrap a new Git repository from a
113resumeable bundle file on a content delivery network. This
114may be necessary if there are problems with the local Python
115HTTP client or proxy configuration, but the Git binary works.
116
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800117The --fetch-submodules option enables fetching Git submodules
118of a project from server.
119
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700120SSH Connections
121---------------
122
123If at least one project remote URL uses an SSH connection (ssh://,
124git+ssh://, or user@host:path syntax) repo will automatically
125enable the SSH ControlMaster option when connecting to that host.
126This feature permits other projects in the same '%prog' session to
127reuse the same SSH tunnel, saving connection setup overheads.
128
129To disable this behavior on UNIX platforms, set the GIT_SSH
130environment variable to 'ssh'. For example:
131
132 export GIT_SSH=ssh
133 %prog
134
135Compatibility
136~~~~~~~~~~~~~
137
138This feature is automatically disabled on Windows, due to the lack
139of UNIX domain socket support.
140
141This feature is not compatible with url.insteadof rewrites in the
142user's ~/.gitconfig. '%prog' is currently not able to perform the
143rewrite early enough to establish the ControlMaster tunnel.
144
145If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
146later is required to fix a server side protocol bug.
147
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700148"""
149
Nico Sallembien6623b212010-05-11 12:57:01 -0700150 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000151 try:
152 self.jobs = self.manifest.default.sync_j
153 except ManifestParseError:
154 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700155
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500156 p.add_option('-f', '--force-broken',
157 dest='force_broken', action='store_true',
158 help="continue sync even if a project fails to sync")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900159 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700160 dest='local_only', action='store_true',
161 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900162 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700163 dest='network_only', action='store_true',
164 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900165 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700166 dest='detach_head', action='store_true',
167 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900168 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700169 dest='current_branch_only', action='store_true',
170 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900171 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700172 dest='quiet', action='store_true',
173 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900174 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800175 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700176 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500177 p.add_option('-m', '--manifest-name',
178 dest='manifest_name',
179 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700180 p.add_option('--no-clone-bundle',
181 dest='no_clone_bundle', action='store_true',
182 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800183 p.add_option('-u', '--manifest-server-username', action='store',
184 dest='manifest_server_username',
185 help='username to authenticate with the manifest server')
186 p.add_option('-p', '--manifest-server-password', action='store',
187 dest='manifest_server_password',
188 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800189 p.add_option('--fetch-submodules',
190 dest='fetch_submodules', action='store_true',
191 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700192 p.add_option('--no-tags',
193 dest='no_tags', action='store_true',
194 help="don't fetch tags")
Nico Sallembien6623b212010-05-11 12:57:01 -0700195 if show_smart:
196 p.add_option('-s', '--smart-sync',
197 dest='smart_sync', action='store_true',
198 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200199 p.add_option('-t', '--smart-tag',
200 dest='smart_tag', action='store',
201 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700202
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700203 g = p.add_option_group('repo Version options')
204 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700205 dest='no_repo_verify', action='store_true',
206 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700207 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800208 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700209 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700210
Doug Andersonfc06ced2011-03-16 15:49:18 -0700211 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
David Pursehousec1b86a22012-11-14 11:36:51 +0900212 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800213
David Pursehousec1b86a22012-11-14 11:36:51 +0900214 Args:
215 opt: Program options returned from optparse. See _Options().
216 project: Project object for the project to fetch.
217 lock: Lock for accessing objects that are shared amongst multiple
218 _FetchHelper() threads.
219 fetched: set object that we will add project.gitdir to when we're done
220 (with our lock held).
221 pm: Instance of a Project object. We will call pm.update() (with our
222 lock held).
223 sem: We'll release() this semaphore when we exit so that another thread
224 can be started up.
225 err_event: We'll set this event in the case of an error (after printing
226 out info about the error).
227 """
228 # We'll set to true once we've locked the lock.
229 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700230
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530231 if not opt.quiet:
232 print('Fetching project %s' % project.name)
233
David Pursehousec1b86a22012-11-14 11:36:51 +0900234 # Encapsulate everything in a try/except/finally so that:
235 # - We always set err_event in the case of an exception.
236 # - We always make sure we call sem.release().
237 # - We always make sure we unlock the lock if we locked it.
238 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700239 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900240 start = time.time()
241 success = project.Sync_NetworkHalf(
242 quiet=opt.quiet,
243 current_branch_only=opt.current_branch_only,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700244 clone_bundle=not opt.no_clone_bundle,
245 no_tags=opt.no_tags)
David Pursehousec1b86a22012-11-14 11:36:51 +0900246 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700247
David Pursehousec1b86a22012-11-14 11:36:51 +0900248 # Lock around all the rest of the code, since printing, updating a set
249 # and Progress.update() are not thread safe.
250 lock.acquire()
251 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700252
David Pursehousec1b86a22012-11-14 11:36:51 +0900253 if not success:
254 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
255 if opt.force_broken:
256 print('warn: --force-broken, continuing to sync',
257 file=sys.stderr)
258 else:
259 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700260
David Pursehousec1b86a22012-11-14 11:36:51 +0900261 fetched.add(project.gitdir)
262 pm.update()
263 except _FetchError:
264 err_event.set()
265 except:
266 err_event.set()
267 raise
268 finally:
269 if did_lock:
270 lock.release()
271 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800272
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700273 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700274 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700275 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800276
277 if self.jobs == 1:
278 for project in projects:
279 pm.update()
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530280 if not opt.quiet:
281 print('Fetching project %s' % project.name)
Shawn O. Pearce5d0efdb2012-08-02 12:13:01 -0700282 if project.Sync_NetworkHalf(
283 quiet=opt.quiet,
284 current_branch_only=opt.current_branch_only,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700285 clone_bundle=not opt.no_clone_bundle,
286 no_tags=opt.no_tags):
Roy Lee18afd7f2010-05-09 04:32:08 +0800287 fetched.add(project.gitdir)
288 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700289 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500290 if opt.force_broken:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700291 print('warn: --force-broken, continuing to sync', file=sys.stderr)
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500292 else:
293 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800294 else:
295 threads = set()
296 lock = _threading.Lock()
297 sem = _threading.Semaphore(self.jobs)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700298 err_event = _threading.Event()
Roy Lee18afd7f2010-05-09 04:32:08 +0800299 for project in projects:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700300 # Check for any errors before starting any new threads.
301 # ...we'll let existing threads finish, though.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400302 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700303 break
304
Roy Lee18afd7f2010-05-09 04:32:08 +0800305 sem.acquire()
306 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700307 args = (opt,
308 project,
309 lock,
310 fetched,
311 pm,
Doug Andersonfc06ced2011-03-16 15:49:18 -0700312 sem,
313 err_event))
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200314 # Ensure that Ctrl-C will not freeze the repo process.
315 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800316 threads.add(t)
317 t.start()
318
319 for t in threads:
320 t.join()
321
Doug Andersonfc06ced2011-03-16 15:49:18 -0700322 # If we saw an error, exit with code 1 so that other scripts can check.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400323 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700324 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700325 sys.exit(1)
326
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700327 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700328 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700329
330 self._GCProjects(projects)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700331 return fetched
332
Dave Borowitz18857212012-10-23 17:02:59 -0700333 def _GCProjects(self, projects):
Dave Borowitze2152672012-10-31 12:24:38 -0700334 has_dash_c = git_require((1, 7, 2))
335 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700336 cpu_count = multiprocessing.cpu_count()
337 else:
338 cpu_count = 1
339 jobs = min(self.jobs, cpu_count)
340
341 if jobs < 2:
342 for project in projects:
343 project.bare_git.gc('--auto')
344 return
345
346 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
347
348 threads = set()
349 sem = _threading.Semaphore(jobs)
350 err_event = _threading.Event()
351
352 def GC(project):
353 try:
354 try:
355 project.bare_git.gc('--auto', config=config)
356 except GitError:
357 err_event.set()
358 except:
359 err_event.set()
360 raise
361 finally:
362 sem.release()
363
364 for project in projects:
365 if err_event.isSet():
366 break
367 sem.acquire()
368 t = _threading.Thread(target=GC, args=(project,))
369 t.daemon = True
370 threads.add(t)
371 t.start()
372
373 for t in threads:
374 t.join()
375
376 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700377 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700378 sys.exit(1)
379
Tim Kilbourn07669002013-03-08 15:02:49 -0800380 def _ReloadManifest(self, manifest_name=None):
381 if manifest_name:
382 # Override calls _Unload already
383 self.manifest.Override(manifest_name)
384 else:
385 self.manifest._Unload()
386
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700387 def UpdateProjectList(self):
388 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700389 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700390 if project.relpath:
391 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700392 file_name = 'project.list'
393 file_path = os.path.join(self.manifest.repodir, file_name)
394 old_project_paths = []
395
396 if os.path.exists(file_path):
397 fd = open(file_path, 'r')
398 try:
399 old_project_paths = fd.read().split('\n')
400 finally:
401 fd.close()
402 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700403 if not path:
404 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700405 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900406 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400407 if os.path.exists(self.manifest.topdir + '/' + path):
David Pursehousec1b86a22012-11-14 11:36:51 +0900408 project = Project(
409 manifest = self.manifest,
410 name = path,
411 remote = RemoteSpec('origin'),
412 gitdir = os.path.join(self.manifest.topdir,
413 path, '.git'),
414 worktree = os.path.join(self.manifest.topdir, path),
415 relpath = path,
416 revisionExpr = 'HEAD',
417 revisionId = None,
418 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400419
David Pursehousec1b86a22012-11-14 11:36:51 +0900420 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900421 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900422 'are present' % project.relpath, file=sys.stderr)
423 print(' commit changes, then run sync again',
424 file=sys.stderr)
425 return -1
426 else:
427 print('Deleting obsolete path %s' % project.worktree,
428 file=sys.stderr)
429 shutil.rmtree(project.worktree)
430 # Try deleting parent subdirs if they are empty
431 project_dir = os.path.dirname(project.worktree)
432 while project_dir != self.manifest.topdir:
433 try:
434 os.rmdir(project_dir)
435 except OSError:
436 break
437 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700438
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700439 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700440 fd = open(file_path, 'w')
441 try:
442 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700443 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700444 finally:
445 fd.close()
446 return 0
447
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700448 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800449 if opt.jobs:
450 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700451 if self.jobs > 1:
452 soft_limit, _ = _rlimit_nofile()
453 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
454
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700455 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700456 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700457 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700458 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700459 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700460 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500461 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700462 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500463 sys.exit(1)
464 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700465 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500466 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900467 if opt.manifest_server_username or opt.manifest_server_password:
468 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700469 print('error: -u and -p may only be combined with -s or -t',
470 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900471 sys.exit(1)
472 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700473 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900474 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500475
476 if opt.manifest_name:
477 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700478
Victor Boivie08c880d2011-04-19 10:32:52 +0200479 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700480 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900481 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700482 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700483 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900484
485 manifest_server = self.manifest.manifest_server
David Pursehousecf76b1b2012-09-14 10:31:42 +0900486
David Pursehouse86d973d2012-08-24 10:21:02 +0900487 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900488 username = None
489 password = None
490 if opt.manifest_server_username and opt.manifest_server_password:
491 username = opt.manifest_server_username
492 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900493 else:
494 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900495 info = netrc.netrc()
496 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700497 print('.netrc file does not exist or could not be opened',
498 file=sys.stderr)
David Pursehouse86d973d2012-08-24 10:21:02 +0900499 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900500 try:
501 parse_result = urlparse.urlparse(manifest_server)
502 if parse_result.hostname:
503 username, _account, password = \
504 info.authenticators(parse_result.hostname)
505 except TypeError:
506 # TypeError is raised when the given hostname is not present
507 # in the .netrc file.
Sarah Owenscecd1d82012-11-01 22:59:27 -0700508 print('No credentials found for %s in .netrc'
509 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700510 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700511 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900512
513 if (username and password):
514 manifest_server = manifest_server.replace('://', '://%s:%s@' %
515 (username, password),
516 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900517
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700518 try:
David Pursehouse86d973d2012-08-24 10:21:02 +0900519 server = xmlrpclib.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200520 if opt.smart_sync:
521 p = self.manifest.manifestProject
522 b = p.GetBranch(p.CurrentBranch)
523 branch = b.merge
524 if branch.startswith(R_HEADS):
525 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700526
Victor Boivie08c880d2011-04-19 10:32:52 +0200527 env = os.environ.copy()
528 if (env.has_key('TARGET_PRODUCT') and
529 env.has_key('TARGET_BUILD_VARIANT')):
530 target = '%s-%s' % (env['TARGET_PRODUCT'],
531 env['TARGET_BUILD_VARIANT'])
532 [success, manifest_str] = server.GetApprovedManifest(branch, target)
533 else:
534 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700535 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200536 assert(opt.smart_tag)
537 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700538
539 if success:
540 manifest_name = "smart_sync_override.xml"
541 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
542 manifest_name)
543 try:
544 f = open(manifest_path, 'w')
545 try:
546 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700547 finally:
548 f.close()
549 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700550 print('error: cannot write manifest to %s' % manifest_path,
551 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700552 sys.exit(1)
Nico Sallembien719965a2010-04-20 15:28:19 -0700553 self.manifest.Override(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700554 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700555 print('error: %s' % manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700556 sys.exit(1)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700557 except (socket.error, IOError, xmlrpclib.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700558 print('error: cannot connect to manifest server %s:\n%s'
559 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900560 sys.exit(1)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700561 except xmlrpclib.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700562 print('error: cannot connect to manifest server %s:\n%d %s'
563 % (self.manifest.manifest_server, e.errcode, e.errmsg),
564 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700565 sys.exit(1)
566
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700567 rp = self.manifest.repoProject
568 rp.PreSync()
569
570 mp = self.manifest.manifestProject
571 mp.PreSync()
572
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800573 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700574 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800575
Nico Sallembien9bb18162009-12-07 15:38:01 -0800576 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700577 mp.Sync_NetworkHalf(quiet=opt.quiet,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700578 current_branch_only=opt.current_branch_only,
579 no_tags=opt.no_tags)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800580
581 if mp.HasChanges:
582 syncbuf = SyncBuffer(mp.config)
583 mp.Sync_LocalHalf(syncbuf)
584 if not syncbuf.Finish():
585 sys.exit(1)
Tim Kilbourn07669002013-03-08 15:02:49 -0800586 self._ReloadManifest(opt.manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700587 if opt.jobs is None:
588 self.jobs = self.manifest.default.sync_j
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800589 all_projects = self.GetProjects(args,
590 missing_ok=True,
591 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700592
Dave Borowitz67700e92012-10-23 15:00:54 -0700593 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700594 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700595 to_fetch = []
596 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700597 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700598 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900599 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700600 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700601
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800602 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700603 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700604 if opt.network_only:
605 # bail out now; the rest touches the working tree
606 return
607
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800608 # Iteratively fetch missing and/or nested unregistered submodules
609 previously_missing_set = set()
610 while True:
Tim Kilbourn07669002013-03-08 15:02:49 -0800611 self._ReloadManifest(opt.manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800612 all_projects = self.GetProjects(args,
613 missing_ok=True,
614 submodules_ok=opt.fetch_submodules)
615 missing = []
616 for project in all_projects:
617 if project.gitdir not in fetched:
618 missing.append(project)
619 if not missing:
620 break
621 # Stop us from non-stopped fetching actually-missing repos: If set of
622 # missing repos has not been changed from last fetch, we break.
623 missing_set = set(p.name for p in missing)
624 if previously_missing_set == missing_set:
625 break
626 previously_missing_set = missing_set
627 fetched.update(self._Fetch(missing, opt))
628
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700629 if self.manifest.IsMirror:
630 # bail out now, we have no working tree
631 return
632
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700633 if self.UpdateProjectList():
634 sys.exit(1)
635
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700636 syncbuf = SyncBuffer(mp.config,
637 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900638 pm = Progress('Syncing work tree', len(all_projects))
639 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700640 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800641 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700642 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700643 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700644 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700645 if not syncbuf.Finish():
646 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700647
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700648 # If there's a notice that's supposed to print at the end of the sync, print
649 # it now...
650 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700651 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700652
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700653def _PostRepoUpgrade(manifest, quiet=False):
Conley Owensc9129d92012-10-01 16:12:28 -0700654 wrapper = WrapperModule()
655 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700656 wrapper.SetupGnuPG(quiet)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700657 for project in manifest.projects.values():
658 if project.Exists:
659 project.PostRepoUpgrade()
660
661def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
662 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700663 print('info: A new version of repo is available', file=sys.stderr)
664 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700665 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700666 syncbuf = SyncBuffer(rp.config)
667 rp.Sync_LocalHalf(syncbuf)
668 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700669 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700670 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700671 raise RepoChangedException(['--repo-upgraded'])
672 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700673 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700674 else:
675 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700676 print('repo version %s is current' % rp.work_git.describe(HEAD),
677 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700678
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700679def _VerifyTag(project):
680 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
681 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700682 print('warning: GnuPG was not available during last "repo init"\n'
683 'warning: Cannot automatically authenticate repo."""',
684 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700685 return True
686
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700687 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700688 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700689 except GitError:
690 cur = None
691
692 if not cur \
693 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700694 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700695 if rev.startswith(R_HEADS):
696 rev = rev[len(R_HEADS):]
697
Sarah Owenscecd1d82012-11-01 22:59:27 -0700698 print(file=sys.stderr)
699 print("warning: project '%s' branch '%s' is not signed"
700 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700701 return False
702
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800703 env = os.environ.copy()
704 env['GIT_DIR'] = project.gitdir.encode()
705 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700706
707 cmd = [GIT, 'tag', '-v', cur]
708 proc = subprocess.Popen(cmd,
709 stdout = subprocess.PIPE,
710 stderr = subprocess.PIPE,
711 env = env)
712 out = proc.stdout.read()
713 proc.stdout.close()
714
715 err = proc.stderr.read()
716 proc.stderr.close()
717
718 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700719 print(file=sys.stderr)
720 print(out, file=sys.stderr)
721 print(err, file=sys.stderr)
722 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700723 return False
724 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700725
726class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700727 _ALPHA = 0.5
728
Dave Borowitz67700e92012-10-23 15:00:54 -0700729 def __init__(self, manifest):
730 self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
731 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700732 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700733
734 def Get(self, project):
735 self._Load()
736 return self._times.get(project.name, _ONE_DAY_S)
737
738 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700739 self._Load()
740 name = project.name
741 old = self._times.get(name, t)
742 self._seen.add(name)
743 a = self._ALPHA
744 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700745
746 def _Load(self):
747 if self._times is None:
748 try:
749 f = open(self._path)
750 except IOError:
751 self._times = {}
752 return self._times
753 try:
754 try:
755 self._times = pickle.load(f)
David Pursehouse1d947b32012-10-25 12:23:11 +0900756 except IOError:
Dave Borowitz67700e92012-10-23 15:00:54 -0700757 try:
758 os.remove(self._path)
759 except OSError:
760 pass
761 self._times = {}
762 finally:
763 f.close()
764 return self._times
765
766 def Save(self):
767 if self._times is None:
768 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700769
770 to_delete = []
771 for name in self._times:
772 if name not in self._seen:
773 to_delete.append(name)
774 for name in to_delete:
775 del self._times[name]
776
Dave Borowitz67700e92012-10-23 15:00:54 -0700777 try:
778 f = open(self._path, 'wb')
779 try:
780 pickle.dump(self._times, f)
781 except (IOError, OSError, pickle.PickleError):
782 try:
783 os.remove(self._path)
784 except OSError:
785 pass
786 finally:
787 f.close()