blob: d4637d0cb3fccb764c570780c03356c3bc74268a [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
David Pursehouse86d973d2012-08-24 10:21:02 +090016import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070017from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import os
Dave Borowitz67700e92012-10-23 15:00:54 -070019import pickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070021import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070022import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023import subprocess
24import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070025import time
David Pursehouse86d973d2012-08-24 10:21:02 +090026import urlparse
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070027import xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070028
Roy Lee18afd7f2010-05-09 04:32:08 +080029try:
30 import threading as _threading
31except ImportError:
32 import dummy_threading as _threading
33
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070034try:
35 import resource
36 def _rlimit_nofile():
37 return resource.getrlimit(resource.RLIMIT_NOFILE)
38except ImportError:
39 def _rlimit_nofile():
40 return (256, 256)
41
Dave Borowitz18857212012-10-23 17:02:59 -070042try:
43 import multiprocessing
44except ImportError:
45 multiprocessing = None
46
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070047from git_command import GIT
David Pursehoused94aaef2012-09-07 09:52:04 +090048from git_refs import R_HEADS, HEAD
Conley Owensc9129d92012-10-01 16:12:28 -070049from main import WrapperModule
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070050from project import Project
51from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080052from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070053from error import RepoChangedException, GitError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070054from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070055from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070056
Dave Borowitz67700e92012-10-23 15:00:54 -070057_ONE_DAY_S = 24 * 60 * 60
58
Doug Andersonfc06ced2011-03-16 15:49:18 -070059class _FetchError(Exception):
60 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
61 pass
62
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080063class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080064 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070065 common = True
66 helpSummary = "Update working tree to the latest revision"
67 helpUsage = """
68%prog [<project>...]
69"""
70 helpDescription = """
71The '%prog' command synchronizes local project directories
72with the remote repositories specified in the manifest. If a local
73project does not yet exist, it will clone a new local directory from
74the remote repository and set up tracking branches as specified in
75the manifest. If the local project already exists, '%prog'
76will update the remote branches and rebase any new local changes
77on top of the new remote changes.
78
79'%prog' will synchronize all projects listed at the command
80line. Projects can be specified either by name, or by a relative
81or absolute path to the project's local directory. If no projects
82are specified, '%prog' will synchronize all projects listed in
83the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070084
85The -d/--detach option can be used to switch specified projects
86back to the manifest revision. This option is especially helpful
87if the project is currently on a topic branch, but the manifest
88revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070089
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070090The -s/--smart-sync option can be used to sync to a known good
91build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +020092manifest. The -t/--smart-tag option is similar and allows you to
93specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070094
David Pursehousecf76b1b2012-09-14 10:31:42 +090095The -u/--manifest-server-username and -p/--manifest-server-password
96options can be used to specify a username and password to authenticate
97with the manifest server when using the -s or -t option.
98
99If -u and -p are not specified when using the -s or -t option, '%prog'
100will attempt to read authentication credentials for the manifest server
101from the user's .netrc file.
102
103'%prog' will not use authentication credentials from -u/-p or .netrc
104if the manifest server specified in the manifest file already includes
105credentials.
106
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500107The -f/--force-broken option can be used to proceed with syncing
108other projects if a project sync fails.
109
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700110The --no-clone-bundle option disables any attempt to use
111$URL/clone.bundle to bootstrap a new Git repository from a
112resumeable bundle file on a content delivery network. This
113may be necessary if there are problems with the local Python
114HTTP client or proxy configuration, but the Git binary works.
115
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700116SSH Connections
117---------------
118
119If at least one project remote URL uses an SSH connection (ssh://,
120git+ssh://, or user@host:path syntax) repo will automatically
121enable the SSH ControlMaster option when connecting to that host.
122This feature permits other projects in the same '%prog' session to
123reuse the same SSH tunnel, saving connection setup overheads.
124
125To disable this behavior on UNIX platforms, set the GIT_SSH
126environment variable to 'ssh'. For example:
127
128 export GIT_SSH=ssh
129 %prog
130
131Compatibility
132~~~~~~~~~~~~~
133
134This feature is automatically disabled on Windows, due to the lack
135of UNIX domain socket support.
136
137This feature is not compatible with url.insteadof rewrites in the
138user's ~/.gitconfig. '%prog' is currently not able to perform the
139rewrite early enough to establish the ControlMaster tunnel.
140
141If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
142later is required to fix a server side protocol bug.
143
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700144"""
145
Nico Sallembien6623b212010-05-11 12:57:01 -0700146 def _Options(self, p, show_smart=True):
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700147 self.jobs = self.manifest.default.sync_j
148
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500149 p.add_option('-f', '--force-broken',
150 dest='force_broken', action='store_true',
151 help="continue sync even if a project fails to sync")
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700152 p.add_option('-l','--local-only',
153 dest='local_only', action='store_true',
154 help="only update working tree, don't fetch")
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700155 p.add_option('-n','--network-only',
156 dest='network_only', action='store_true',
157 help="fetch only, don't update working tree")
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700158 p.add_option('-d','--detach',
159 dest='detach_head', action='store_true',
160 help='detach projects back to manifest revision')
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700161 p.add_option('-c','--current-branch',
162 dest='current_branch_only', action='store_true',
163 help='fetch only current branch from server')
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700164 p.add_option('-q','--quiet',
165 dest='quiet', action='store_true',
166 help='be more quiet')
Roy Lee18afd7f2010-05-09 04:32:08 +0800167 p.add_option('-j','--jobs',
168 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700169 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500170 p.add_option('-m', '--manifest-name',
171 dest='manifest_name',
172 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700173 p.add_option('--no-clone-bundle',
174 dest='no_clone_bundle', action='store_true',
175 help='disable use of /clone.bundle on HTTP/HTTPS')
Nico Sallembien6623b212010-05-11 12:57:01 -0700176 if show_smart:
177 p.add_option('-s', '--smart-sync',
178 dest='smart_sync', action='store_true',
179 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200180 p.add_option('-t', '--smart-tag',
181 dest='smart_tag', action='store',
182 help='smart sync using manifest from a known tag')
David Pursehousecf76b1b2012-09-14 10:31:42 +0900183 p.add_option('-u', '--manifest-server-username', action='store',
184 dest='manifest_server_username',
185 help='username to authenticate with the manifest server')
186 p.add_option('-p', '--manifest-server-password', action='store',
187 dest='manifest_server_password',
188 help='password to authenticate with the manifest server')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700189
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700190 g = p.add_option_group('repo Version options')
191 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700192 dest='no_repo_verify', action='store_true',
193 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700194 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800195 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700196 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700197
Doug Andersonfc06ced2011-03-16 15:49:18 -0700198 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
199 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800200
Doug Andersonfc06ced2011-03-16 15:49:18 -0700201 Args:
202 opt: Program options returned from optparse. See _Options().
203 project: Project object for the project to fetch.
204 lock: Lock for accessing objects that are shared amongst multiple
205 _FetchHelper() threads.
206 fetched: set object that we will add project.gitdir to when we're done
207 (with our lock held).
208 pm: Instance of a Project object. We will call pm.update() (with our
209 lock held).
210 sem: We'll release() this semaphore when we exit so that another thread
211 can be started up.
212 err_event: We'll set this event in the case of an error (after printing
213 out info about the error).
214 """
215 # We'll set to true once we've locked the lock.
216 did_lock = False
217
218 # Encapsulate everything in a try/except/finally so that:
219 # - We always set err_event in the case of an exception.
220 # - We always make sure we call sem.release().
221 # - We always make sure we unlock the lock if we locked it.
222 try:
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700223 try:
Dave Borowitz67700e92012-10-23 15:00:54 -0700224 start = time.time()
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700225 success = project.Sync_NetworkHalf(
226 quiet=opt.quiet,
227 current_branch_only=opt.current_branch_only,
228 clone_bundle=not opt.no_clone_bundle)
Dave Borowitz67700e92012-10-23 15:00:54 -0700229 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700230
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700231 # Lock around all the rest of the code, since printing, updating a set
232 # and Progress.update() are not thread safe.
233 lock.acquire()
234 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700235
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700236 if not success:
237 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
238 if opt.force_broken:
239 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
240 else:
241 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700242
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700243 fetched.add(project.gitdir)
244 pm.update()
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -0700245 except _FetchError:
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700246 err_event.set()
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -0700247 except:
248 err_event.set()
249 raise
Doug Andersonfc06ced2011-03-16 15:49:18 -0700250 finally:
251 if did_lock:
252 lock.release()
253 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800254
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700255 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700256 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700257 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800258
259 if self.jobs == 1:
260 for project in projects:
261 pm.update()
Shawn O. Pearce5d0efdb2012-08-02 12:13:01 -0700262 if project.Sync_NetworkHalf(
263 quiet=opt.quiet,
264 current_branch_only=opt.current_branch_only,
265 clone_bundle=not opt.no_clone_bundle):
Roy Lee18afd7f2010-05-09 04:32:08 +0800266 fetched.add(project.gitdir)
267 else:
268 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500269 if opt.force_broken:
270 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
271 else:
272 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800273 else:
274 threads = set()
275 lock = _threading.Lock()
276 sem = _threading.Semaphore(self.jobs)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700277 err_event = _threading.Event()
Roy Lee18afd7f2010-05-09 04:32:08 +0800278 for project in projects:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700279 # Check for any errors before starting any new threads.
280 # ...we'll let existing threads finish, though.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400281 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700282 break
283
Roy Lee18afd7f2010-05-09 04:32:08 +0800284 sem.acquire()
285 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700286 args = (opt,
287 project,
288 lock,
289 fetched,
290 pm,
Doug Andersonfc06ced2011-03-16 15:49:18 -0700291 sem,
292 err_event))
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200293 # Ensure that Ctrl-C will not freeze the repo process.
294 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800295 threads.add(t)
296 t.start()
297
298 for t in threads:
299 t.join()
300
Doug Andersonfc06ced2011-03-16 15:49:18 -0700301 # If we saw an error, exit with code 1 so that other scripts can check.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400302 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700303 print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
304 sys.exit(1)
305
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700306 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700307 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700308
309 self._GCProjects(projects)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700310 return fetched
311
Dave Borowitz18857212012-10-23 17:02:59 -0700312 def _GCProjects(self, projects):
313 if multiprocessing:
314 cpu_count = multiprocessing.cpu_count()
315 else:
316 cpu_count = 1
317 jobs = min(self.jobs, cpu_count)
318
319 if jobs < 2:
320 for project in projects:
321 project.bare_git.gc('--auto')
322 return
323
324 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
325
326 threads = set()
327 sem = _threading.Semaphore(jobs)
328 err_event = _threading.Event()
329
330 def GC(project):
331 try:
332 try:
333 project.bare_git.gc('--auto', config=config)
334 except GitError:
335 err_event.set()
336 except:
337 err_event.set()
338 raise
339 finally:
340 sem.release()
341
342 for project in projects:
343 if err_event.isSet():
344 break
345 sem.acquire()
346 t = _threading.Thread(target=GC, args=(project,))
347 t.daemon = True
348 threads.add(t)
349 t.start()
350
351 for t in threads:
352 t.join()
353
354 if err_event.isSet():
355 print >>sys.stderr, '\nerror: Exited sync due to gc errors'
356 sys.exit(1)
357
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700358 def UpdateProjectList(self):
359 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700360 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700361 if project.relpath:
362 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700363 file_name = 'project.list'
364 file_path = os.path.join(self.manifest.repodir, file_name)
365 old_project_paths = []
366
367 if os.path.exists(file_path):
368 fd = open(file_path, 'r')
369 try:
370 old_project_paths = fd.read().split('\n')
371 finally:
372 fd.close()
373 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700374 if not path:
375 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700376 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900377 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400378 if os.path.exists(self.manifest.topdir + '/' + path):
379 project = Project(
380 manifest = self.manifest,
381 name = path,
382 remote = RemoteSpec('origin'),
383 gitdir = os.path.join(self.manifest.topdir,
384 path, '.git'),
385 worktree = os.path.join(self.manifest.topdir, path),
386 relpath = path,
387 revisionExpr = 'HEAD',
Colin Cross5acde752012-03-28 20:15:45 -0700388 revisionId = None,
389 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400390
391 if project.IsDirty():
392 print >>sys.stderr, 'error: Cannot remove project "%s": \
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700393uncommitted changes are present' % project.relpath
Anthonyf3fdf822009-09-26 13:38:52 -0400394 print >>sys.stderr, ' commit changes, then run sync again'
395 return -1
396 else:
397 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
398 shutil.rmtree(project.worktree)
399 # Try deleting parent subdirs if they are empty
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200400 project_dir = os.path.dirname(project.worktree)
401 while project_dir != self.manifest.topdir:
Anthonyf3fdf822009-09-26 13:38:52 -0400402 try:
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200403 os.rmdir(project_dir)
Anthonyf3fdf822009-09-26 13:38:52 -0400404 except OSError:
405 break
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200406 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700407
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700408 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700409 fd = open(file_path, 'w')
410 try:
411 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700412 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700413 finally:
414 fd.close()
415 return 0
416
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700417 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800418 if opt.jobs:
419 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700420 if self.jobs > 1:
421 soft_limit, _ = _rlimit_nofile()
422 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
423
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700424 if opt.network_only and opt.detach_head:
425 print >>sys.stderr, 'error: cannot combine -n and -d'
426 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700427 if opt.network_only and opt.local_only:
428 print >>sys.stderr, 'error: cannot combine -n and -l'
429 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500430 if opt.manifest_name and opt.smart_sync:
431 print >>sys.stderr, 'error: cannot combine -m and -s'
432 sys.exit(1)
433 if opt.manifest_name and opt.smart_tag:
434 print >>sys.stderr, 'error: cannot combine -m and -t'
435 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900436 if opt.manifest_server_username or opt.manifest_server_password:
437 if not (opt.smart_sync or opt.smart_tag):
438 print >>sys.stderr, 'error: -u and -p may only be combined with ' \
439 '-s or -t'
440 sys.exit(1)
441 if None in [opt.manifest_server_username, opt.manifest_server_password]:
442 print >>sys.stderr, 'error: both -u and -p must be given'
443 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500444
445 if opt.manifest_name:
446 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700447
Victor Boivie08c880d2011-04-19 10:32:52 +0200448 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700449 if not self.manifest.manifest_server:
450 print >>sys.stderr, \
451 'error: cannot smart sync: no manifest server defined in manifest'
452 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900453
454 manifest_server = self.manifest.manifest_server
David Pursehousecf76b1b2012-09-14 10:31:42 +0900455
David Pursehouse86d973d2012-08-24 10:21:02 +0900456 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900457 username = None
458 password = None
459 if opt.manifest_server_username and opt.manifest_server_password:
460 username = opt.manifest_server_username
461 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900462 else:
463 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900464 info = netrc.netrc()
465 except IOError:
466 print >>sys.stderr, '.netrc file does not exist or could not be opened'
David Pursehouse86d973d2012-08-24 10:21:02 +0900467 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900468 try:
469 parse_result = urlparse.urlparse(manifest_server)
470 if parse_result.hostname:
471 username, _account, password = \
472 info.authenticators(parse_result.hostname)
473 except TypeError:
474 # TypeError is raised when the given hostname is not present
475 # in the .netrc file.
476 print >>sys.stderr, 'No credentials found for %s in .netrc' % \
477 parse_result.hostname
Sarah Owensa5be53f2012-09-09 15:37:57 -0700478 except netrc.NetrcParseError as e:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900479 print >>sys.stderr, 'Error parsing .netrc file: %s' % e
480
481 if (username and password):
482 manifest_server = manifest_server.replace('://', '://%s:%s@' %
483 (username, password),
484 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900485
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700486 try:
David Pursehouse86d973d2012-08-24 10:21:02 +0900487 server = xmlrpclib.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200488 if opt.smart_sync:
489 p = self.manifest.manifestProject
490 b = p.GetBranch(p.CurrentBranch)
491 branch = b.merge
492 if branch.startswith(R_HEADS):
493 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700494
Victor Boivie08c880d2011-04-19 10:32:52 +0200495 env = os.environ.copy()
496 if (env.has_key('TARGET_PRODUCT') and
497 env.has_key('TARGET_BUILD_VARIANT')):
498 target = '%s-%s' % (env['TARGET_PRODUCT'],
499 env['TARGET_BUILD_VARIANT'])
500 [success, manifest_str] = server.GetApprovedManifest(branch, target)
501 else:
502 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700503 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200504 assert(opt.smart_tag)
505 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700506
507 if success:
508 manifest_name = "smart_sync_override.xml"
509 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
510 manifest_name)
511 try:
512 f = open(manifest_path, 'w')
513 try:
514 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700515 finally:
516 f.close()
517 except IOError:
518 print >>sys.stderr, 'error: cannot write manifest to %s' % \
519 manifest_path
520 sys.exit(1)
Nico Sallembien719965a2010-04-20 15:28:19 -0700521 self.manifest.Override(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700522 else:
523 print >>sys.stderr, 'error: %s' % manifest_str
524 sys.exit(1)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700525 except (socket.error, IOError, xmlrpclib.Fault) as e:
David Pursehousebd489c42012-08-23 10:21:26 +0900526 print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
527 self.manifest.manifest_server, e)
528 sys.exit(1)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700529 except xmlrpclib.ProtocolError as e:
David Pursehousebd489c42012-08-23 10:21:26 +0900530 print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
531 self.manifest.manifest_server, e.errcode, e.errmsg)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700532 sys.exit(1)
533
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700534 rp = self.manifest.repoProject
535 rp.PreSync()
536
537 mp = self.manifest.manifestProject
538 mp.PreSync()
539
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800540 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700541 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800542
Nico Sallembien9bb18162009-12-07 15:38:01 -0800543 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700544 mp.Sync_NetworkHalf(quiet=opt.quiet,
545 current_branch_only=opt.current_branch_only)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800546
547 if mp.HasChanges:
548 syncbuf = SyncBuffer(mp.config)
549 mp.Sync_LocalHalf(syncbuf)
550 if not syncbuf.Finish():
551 sys.exit(1)
552 self.manifest._Unload()
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700553 if opt.jobs is None:
554 self.jobs = self.manifest.default.sync_j
David Pursehouse8a68ff92012-09-24 12:15:13 +0900555 all_projects = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700556
Dave Borowitz67700e92012-10-23 15:00:54 -0700557 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700558 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700559 to_fetch = []
560 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700561 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700562 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900563 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700564 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700565
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -0700566 self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700567 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700568 if opt.network_only:
569 # bail out now; the rest touches the working tree
570 return
571
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700572 if self.manifest.IsMirror:
573 # bail out now, we have no working tree
574 return
575
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700576 if self.UpdateProjectList():
577 sys.exit(1)
578
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700579 syncbuf = SyncBuffer(mp.config,
580 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900581 pm = Progress('Syncing work tree', len(all_projects))
582 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700583 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800584 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700585 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700586 pm.end()
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700587 print >>sys.stderr
588 if not syncbuf.Finish():
589 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700590
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700591 # If there's a notice that's supposed to print at the end of the sync, print
592 # it now...
593 if self.manifest.notice:
594 print self.manifest.notice
595
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700596def _PostRepoUpgrade(manifest, quiet=False):
Conley Owensc9129d92012-10-01 16:12:28 -0700597 wrapper = WrapperModule()
598 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700599 wrapper.SetupGnuPG(quiet)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700600 for project in manifest.projects.values():
601 if project.Exists:
602 project.PostRepoUpgrade()
603
604def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
605 if rp.HasChanges:
606 print >>sys.stderr, 'info: A new version of repo is available'
607 print >>sys.stderr, ''
608 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700609 syncbuf = SyncBuffer(rp.config)
610 rp.Sync_LocalHalf(syncbuf)
611 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700612 sys.exit(1)
613 print >>sys.stderr, 'info: Restarting repo with latest version'
614 raise RepoChangedException(['--repo-upgraded'])
615 else:
616 print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
617 else:
618 if verbose:
619 print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
620
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700621def _VerifyTag(project):
622 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
623 if not os.path.exists(gpg_dir):
624 print >>sys.stderr,\
625"""warning: GnuPG was not available during last "repo init"
626warning: Cannot automatically authenticate repo."""
627 return True
628
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700629 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700630 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700631 except GitError:
632 cur = None
633
634 if not cur \
635 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700636 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700637 if rev.startswith(R_HEADS):
638 rev = rev[len(R_HEADS):]
639
640 print >>sys.stderr
641 print >>sys.stderr,\
642 "warning: project '%s' branch '%s' is not signed" \
643 % (project.name, rev)
644 return False
645
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800646 env = os.environ.copy()
647 env['GIT_DIR'] = project.gitdir.encode()
648 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700649
650 cmd = [GIT, 'tag', '-v', cur]
651 proc = subprocess.Popen(cmd,
652 stdout = subprocess.PIPE,
653 stderr = subprocess.PIPE,
654 env = env)
655 out = proc.stdout.read()
656 proc.stdout.close()
657
658 err = proc.stderr.read()
659 proc.stderr.close()
660
661 if proc.wait() != 0:
662 print >>sys.stderr
663 print >>sys.stderr, out
664 print >>sys.stderr, err
665 print >>sys.stderr
666 return False
667 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700668
669class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700670 _ALPHA = 0.5
671
Dave Borowitz67700e92012-10-23 15:00:54 -0700672 def __init__(self, manifest):
673 self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
674 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700675 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700676
677 def Get(self, project):
678 self._Load()
679 return self._times.get(project.name, _ONE_DAY_S)
680
681 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700682 self._Load()
683 name = project.name
684 old = self._times.get(name, t)
685 self._seen.add(name)
686 a = self._ALPHA
687 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700688
689 def _Load(self):
690 if self._times is None:
691 try:
692 f = open(self._path)
693 except IOError:
694 self._times = {}
695 return self._times
696 try:
697 try:
698 self._times = pickle.load(f)
699 except:
700 try:
701 os.remove(self._path)
702 except OSError:
703 pass
704 self._times = {}
705 finally:
706 f.close()
707 return self._times
708
709 def Save(self):
710 if self._times is None:
711 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700712
713 to_delete = []
714 for name in self._times:
715 if name not in self._seen:
716 to_delete.append(name)
717 for name in to_delete:
718 del self._times[name]
719
Dave Borowitz67700e92012-10-23 15:00:54 -0700720 try:
721 f = open(self._path, 'wb')
722 try:
723 pickle.dump(self._times, f)
724 except (IOError, OSError, pickle.PickleError):
725 try:
726 os.remove(self._path)
727 except OSError:
728 pass
729 finally:
730 f.close()