blob: d6389118b6b5a321e5c87fd94e8b8e04941d140e [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
David Pursehouse86d973d2012-08-24 10:21:02 +090016import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070017from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import os
Dave Borowitz67700e92012-10-23 15:00:54 -070019import pickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070021import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070022import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023import subprocess
24import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070025import time
David Pursehouse86d973d2012-08-24 10:21:02 +090026import urlparse
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070027import xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070028
Roy Lee18afd7f2010-05-09 04:32:08 +080029try:
30 import threading as _threading
31except ImportError:
32 import dummy_threading as _threading
33
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070034try:
35 import resource
36 def _rlimit_nofile():
37 return resource.getrlimit(resource.RLIMIT_NOFILE)
38except ImportError:
39 def _rlimit_nofile():
40 return (256, 256)
41
Dave Borowitz18857212012-10-23 17:02:59 -070042try:
43 import multiprocessing
44except ImportError:
45 multiprocessing = None
46
Dave Borowitze2152672012-10-31 12:24:38 -070047from git_command import GIT, git_require
David Pursehoused94aaef2012-09-07 09:52:04 +090048from git_refs import R_HEADS, HEAD
Conley Owensc9129d92012-10-01 16:12:28 -070049from main import WrapperModule
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070050from project import Project
51from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080052from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070053from error import RepoChangedException, GitError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070054from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070055from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070056
Dave Borowitz67700e92012-10-23 15:00:54 -070057_ONE_DAY_S = 24 * 60 * 60
58
Doug Andersonfc06ced2011-03-16 15:49:18 -070059class _FetchError(Exception):
60 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
61 pass
62
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080063class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080064 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070065 common = True
66 helpSummary = "Update working tree to the latest revision"
67 helpUsage = """
68%prog [<project>...]
69"""
70 helpDescription = """
71The '%prog' command synchronizes local project directories
72with the remote repositories specified in the manifest. If a local
73project does not yet exist, it will clone a new local directory from
74the remote repository and set up tracking branches as specified in
75the manifest. If the local project already exists, '%prog'
76will update the remote branches and rebase any new local changes
77on top of the new remote changes.
78
79'%prog' will synchronize all projects listed at the command
80line. Projects can be specified either by name, or by a relative
81or absolute path to the project's local directory. If no projects
82are specified, '%prog' will synchronize all projects listed in
83the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070084
85The -d/--detach option can be used to switch specified projects
86back to the manifest revision. This option is especially helpful
87if the project is currently on a topic branch, but the manifest
88revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070089
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070090The -s/--smart-sync option can be used to sync to a known good
91build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +020092manifest. The -t/--smart-tag option is similar and allows you to
93specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070094
David Pursehousecf76b1b2012-09-14 10:31:42 +090095The -u/--manifest-server-username and -p/--manifest-server-password
96options can be used to specify a username and password to authenticate
97with the manifest server when using the -s or -t option.
98
99If -u and -p are not specified when using the -s or -t option, '%prog'
100will attempt to read authentication credentials for the manifest server
101from the user's .netrc file.
102
103'%prog' will not use authentication credentials from -u/-p or .netrc
104if the manifest server specified in the manifest file already includes
105credentials.
106
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500107The -f/--force-broken option can be used to proceed with syncing
108other projects if a project sync fails.
109
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700110The --no-clone-bundle option disables any attempt to use
111$URL/clone.bundle to bootstrap a new Git repository from a
112resumeable bundle file on a content delivery network. This
113may be necessary if there are problems with the local Python
114HTTP client or proxy configuration, but the Git binary works.
115
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700116SSH Connections
117---------------
118
119If at least one project remote URL uses an SSH connection (ssh://,
120git+ssh://, or user@host:path syntax) repo will automatically
121enable the SSH ControlMaster option when connecting to that host.
122This feature permits other projects in the same '%prog' session to
123reuse the same SSH tunnel, saving connection setup overheads.
124
125To disable this behavior on UNIX platforms, set the GIT_SSH
126environment variable to 'ssh'. For example:
127
128 export GIT_SSH=ssh
129 %prog
130
131Compatibility
132~~~~~~~~~~~~~
133
134This feature is automatically disabled on Windows, due to the lack
135of UNIX domain socket support.
136
137This feature is not compatible with url.insteadof rewrites in the
138user's ~/.gitconfig. '%prog' is currently not able to perform the
139rewrite early enough to establish the ControlMaster tunnel.
140
141If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
142later is required to fix a server side protocol bug.
143
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700144"""
145
Nico Sallembien6623b212010-05-11 12:57:01 -0700146 def _Options(self, p, show_smart=True):
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700147 self.jobs = self.manifest.default.sync_j
148
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500149 p.add_option('-f', '--force-broken',
150 dest='force_broken', action='store_true',
151 help="continue sync even if a project fails to sync")
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700152 p.add_option('-l','--local-only',
153 dest='local_only', action='store_true',
154 help="only update working tree, don't fetch")
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700155 p.add_option('-n','--network-only',
156 dest='network_only', action='store_true',
157 help="fetch only, don't update working tree")
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700158 p.add_option('-d','--detach',
159 dest='detach_head', action='store_true',
160 help='detach projects back to manifest revision')
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700161 p.add_option('-c','--current-branch',
162 dest='current_branch_only', action='store_true',
163 help='fetch only current branch from server')
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700164 p.add_option('-q','--quiet',
165 dest='quiet', action='store_true',
166 help='be more quiet')
Roy Lee18afd7f2010-05-09 04:32:08 +0800167 p.add_option('-j','--jobs',
168 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700169 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500170 p.add_option('-m', '--manifest-name',
171 dest='manifest_name',
172 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700173 p.add_option('--no-clone-bundle',
174 dest='no_clone_bundle', action='store_true',
175 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800176 p.add_option('-u', '--manifest-server-username', action='store',
177 dest='manifest_server_username',
178 help='username to authenticate with the manifest server')
179 p.add_option('-p', '--manifest-server-password', action='store',
180 dest='manifest_server_password',
181 help='password to authenticate with the manifest server')
Nico Sallembien6623b212010-05-11 12:57:01 -0700182 if show_smart:
183 p.add_option('-s', '--smart-sync',
184 dest='smart_sync', action='store_true',
185 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200186 p.add_option('-t', '--smart-tag',
187 dest='smart_tag', action='store',
188 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700189
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700190 g = p.add_option_group('repo Version options')
191 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700192 dest='no_repo_verify', action='store_true',
193 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700194 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800195 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700196 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700197
Doug Andersonfc06ced2011-03-16 15:49:18 -0700198 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
199 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800200
Doug Andersonfc06ced2011-03-16 15:49:18 -0700201 Args:
202 opt: Program options returned from optparse. See _Options().
203 project: Project object for the project to fetch.
204 lock: Lock for accessing objects that are shared amongst multiple
205 _FetchHelper() threads.
206 fetched: set object that we will add project.gitdir to when we're done
207 (with our lock held).
208 pm: Instance of a Project object. We will call pm.update() (with our
209 lock held).
210 sem: We'll release() this semaphore when we exit so that another thread
211 can be started up.
212 err_event: We'll set this event in the case of an error (after printing
213 out info about the error).
214 """
215 # We'll set to true once we've locked the lock.
216 did_lock = False
217
218 # Encapsulate everything in a try/except/finally so that:
219 # - We always set err_event in the case of an exception.
220 # - We always make sure we call sem.release().
221 # - We always make sure we unlock the lock if we locked it.
222 try:
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700223 try:
Dave Borowitz67700e92012-10-23 15:00:54 -0700224 start = time.time()
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700225 success = project.Sync_NetworkHalf(
226 quiet=opt.quiet,
227 current_branch_only=opt.current_branch_only,
228 clone_bundle=not opt.no_clone_bundle)
Dave Borowitz67700e92012-10-23 15:00:54 -0700229 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700230
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700231 # Lock around all the rest of the code, since printing, updating a set
232 # and Progress.update() are not thread safe.
233 lock.acquire()
234 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700235
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700236 if not success:
237 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
238 if opt.force_broken:
239 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
240 else:
241 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700242
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700243 fetched.add(project.gitdir)
244 pm.update()
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -0700245 except _FetchError:
Shawn O. Pearcee6a0eeb2011-03-22 19:04:47 -0700246 err_event.set()
Shawn O. Pearcedf5ee522011-10-11 14:05:21 -0700247 except:
248 err_event.set()
249 raise
Doug Andersonfc06ced2011-03-16 15:49:18 -0700250 finally:
251 if did_lock:
252 lock.release()
253 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800254
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700255 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700256 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700257 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800258
259 if self.jobs == 1:
260 for project in projects:
261 pm.update()
Shawn O. Pearce5d0efdb2012-08-02 12:13:01 -0700262 if project.Sync_NetworkHalf(
263 quiet=opt.quiet,
264 current_branch_only=opt.current_branch_only,
265 clone_bundle=not opt.no_clone_bundle):
Roy Lee18afd7f2010-05-09 04:32:08 +0800266 fetched.add(project.gitdir)
267 else:
268 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500269 if opt.force_broken:
270 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
271 else:
272 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800273 else:
274 threads = set()
275 lock = _threading.Lock()
276 sem = _threading.Semaphore(self.jobs)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700277 err_event = _threading.Event()
Roy Lee18afd7f2010-05-09 04:32:08 +0800278 for project in projects:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700279 # Check for any errors before starting any new threads.
280 # ...we'll let existing threads finish, though.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400281 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700282 break
283
Roy Lee18afd7f2010-05-09 04:32:08 +0800284 sem.acquire()
285 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700286 args = (opt,
287 project,
288 lock,
289 fetched,
290 pm,
Doug Andersonfc06ced2011-03-16 15:49:18 -0700291 sem,
292 err_event))
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200293 # Ensure that Ctrl-C will not freeze the repo process.
294 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800295 threads.add(t)
296 t.start()
297
298 for t in threads:
299 t.join()
300
Doug Andersonfc06ced2011-03-16 15:49:18 -0700301 # If we saw an error, exit with code 1 so that other scripts can check.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400302 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700303 print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
304 sys.exit(1)
305
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700306 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700307 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700308
309 self._GCProjects(projects)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700310 return fetched
311
Dave Borowitz18857212012-10-23 17:02:59 -0700312 def _GCProjects(self, projects):
Dave Borowitze2152672012-10-31 12:24:38 -0700313 has_dash_c = git_require((1, 7, 2))
314 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700315 cpu_count = multiprocessing.cpu_count()
316 else:
317 cpu_count = 1
318 jobs = min(self.jobs, cpu_count)
319
320 if jobs < 2:
321 for project in projects:
322 project.bare_git.gc('--auto')
323 return
324
325 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
326
327 threads = set()
328 sem = _threading.Semaphore(jobs)
329 err_event = _threading.Event()
330
331 def GC(project):
332 try:
333 try:
334 project.bare_git.gc('--auto', config=config)
335 except GitError:
336 err_event.set()
337 except:
338 err_event.set()
339 raise
340 finally:
341 sem.release()
342
343 for project in projects:
344 if err_event.isSet():
345 break
346 sem.acquire()
347 t = _threading.Thread(target=GC, args=(project,))
348 t.daemon = True
349 threads.add(t)
350 t.start()
351
352 for t in threads:
353 t.join()
354
355 if err_event.isSet():
356 print >>sys.stderr, '\nerror: Exited sync due to gc errors'
357 sys.exit(1)
358
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700359 def UpdateProjectList(self):
360 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700361 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700362 if project.relpath:
363 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700364 file_name = 'project.list'
365 file_path = os.path.join(self.manifest.repodir, file_name)
366 old_project_paths = []
367
368 if os.path.exists(file_path):
369 fd = open(file_path, 'r')
370 try:
371 old_project_paths = fd.read().split('\n')
372 finally:
373 fd.close()
374 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700375 if not path:
376 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700377 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900378 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400379 if os.path.exists(self.manifest.topdir + '/' + path):
380 project = Project(
381 manifest = self.manifest,
382 name = path,
383 remote = RemoteSpec('origin'),
384 gitdir = os.path.join(self.manifest.topdir,
385 path, '.git'),
386 worktree = os.path.join(self.manifest.topdir, path),
387 relpath = path,
388 revisionExpr = 'HEAD',
Colin Cross5acde752012-03-28 20:15:45 -0700389 revisionId = None,
390 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400391
392 if project.IsDirty():
393 print >>sys.stderr, 'error: Cannot remove project "%s": \
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700394uncommitted changes are present' % project.relpath
Anthonyf3fdf822009-09-26 13:38:52 -0400395 print >>sys.stderr, ' commit changes, then run sync again'
396 return -1
397 else:
398 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
399 shutil.rmtree(project.worktree)
400 # Try deleting parent subdirs if they are empty
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200401 project_dir = os.path.dirname(project.worktree)
402 while project_dir != self.manifest.topdir:
Anthonyf3fdf822009-09-26 13:38:52 -0400403 try:
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200404 os.rmdir(project_dir)
Anthonyf3fdf822009-09-26 13:38:52 -0400405 except OSError:
406 break
Mickaël Salaün2f6ab7f2012-09-30 00:37:55 +0200407 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700408
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700409 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700410 fd = open(file_path, 'w')
411 try:
412 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700413 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700414 finally:
415 fd.close()
416 return 0
417
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700418 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800419 if opt.jobs:
420 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700421 if self.jobs > 1:
422 soft_limit, _ = _rlimit_nofile()
423 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
424
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700425 if opt.network_only and opt.detach_head:
426 print >>sys.stderr, 'error: cannot combine -n and -d'
427 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700428 if opt.network_only and opt.local_only:
429 print >>sys.stderr, 'error: cannot combine -n and -l'
430 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500431 if opt.manifest_name and opt.smart_sync:
432 print >>sys.stderr, 'error: cannot combine -m and -s'
433 sys.exit(1)
434 if opt.manifest_name and opt.smart_tag:
435 print >>sys.stderr, 'error: cannot combine -m and -t'
436 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900437 if opt.manifest_server_username or opt.manifest_server_password:
438 if not (opt.smart_sync or opt.smart_tag):
439 print >>sys.stderr, 'error: -u and -p may only be combined with ' \
440 '-s or -t'
441 sys.exit(1)
442 if None in [opt.manifest_server_username, opt.manifest_server_password]:
443 print >>sys.stderr, 'error: both -u and -p must be given'
444 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500445
446 if opt.manifest_name:
447 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700448
Victor Boivie08c880d2011-04-19 10:32:52 +0200449 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700450 if not self.manifest.manifest_server:
451 print >>sys.stderr, \
452 'error: cannot smart sync: no manifest server defined in manifest'
453 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900454
455 manifest_server = self.manifest.manifest_server
David Pursehousecf76b1b2012-09-14 10:31:42 +0900456
David Pursehouse86d973d2012-08-24 10:21:02 +0900457 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900458 username = None
459 password = None
460 if opt.manifest_server_username and opt.manifest_server_password:
461 username = opt.manifest_server_username
462 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900463 else:
464 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900465 info = netrc.netrc()
466 except IOError:
467 print >>sys.stderr, '.netrc file does not exist or could not be opened'
David Pursehouse86d973d2012-08-24 10:21:02 +0900468 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900469 try:
470 parse_result = urlparse.urlparse(manifest_server)
471 if parse_result.hostname:
472 username, _account, password = \
473 info.authenticators(parse_result.hostname)
474 except TypeError:
475 # TypeError is raised when the given hostname is not present
476 # in the .netrc file.
477 print >>sys.stderr, 'No credentials found for %s in .netrc' % \
478 parse_result.hostname
Sarah Owensa5be53f2012-09-09 15:37:57 -0700479 except netrc.NetrcParseError as e:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900480 print >>sys.stderr, 'Error parsing .netrc file: %s' % e
481
482 if (username and password):
483 manifest_server = manifest_server.replace('://', '://%s:%s@' %
484 (username, password),
485 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900486
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700487 try:
David Pursehouse86d973d2012-08-24 10:21:02 +0900488 server = xmlrpclib.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200489 if opt.smart_sync:
490 p = self.manifest.manifestProject
491 b = p.GetBranch(p.CurrentBranch)
492 branch = b.merge
493 if branch.startswith(R_HEADS):
494 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700495
Victor Boivie08c880d2011-04-19 10:32:52 +0200496 env = os.environ.copy()
497 if (env.has_key('TARGET_PRODUCT') and
498 env.has_key('TARGET_BUILD_VARIANT')):
499 target = '%s-%s' % (env['TARGET_PRODUCT'],
500 env['TARGET_BUILD_VARIANT'])
501 [success, manifest_str] = server.GetApprovedManifest(branch, target)
502 else:
503 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700504 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200505 assert(opt.smart_tag)
506 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700507
508 if success:
509 manifest_name = "smart_sync_override.xml"
510 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
511 manifest_name)
512 try:
513 f = open(manifest_path, 'w')
514 try:
515 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700516 finally:
517 f.close()
518 except IOError:
519 print >>sys.stderr, 'error: cannot write manifest to %s' % \
520 manifest_path
521 sys.exit(1)
Nico Sallembien719965a2010-04-20 15:28:19 -0700522 self.manifest.Override(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700523 else:
524 print >>sys.stderr, 'error: %s' % manifest_str
525 sys.exit(1)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700526 except (socket.error, IOError, xmlrpclib.Fault) as e:
David Pursehousebd489c42012-08-23 10:21:26 +0900527 print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
528 self.manifest.manifest_server, e)
529 sys.exit(1)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700530 except xmlrpclib.ProtocolError as e:
David Pursehousebd489c42012-08-23 10:21:26 +0900531 print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
532 self.manifest.manifest_server, e.errcode, e.errmsg)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700533 sys.exit(1)
534
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700535 rp = self.manifest.repoProject
536 rp.PreSync()
537
538 mp = self.manifest.manifestProject
539 mp.PreSync()
540
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800541 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700542 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800543
Nico Sallembien9bb18162009-12-07 15:38:01 -0800544 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700545 mp.Sync_NetworkHalf(quiet=opt.quiet,
546 current_branch_only=opt.current_branch_only)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800547
548 if mp.HasChanges:
549 syncbuf = SyncBuffer(mp.config)
550 mp.Sync_LocalHalf(syncbuf)
551 if not syncbuf.Finish():
552 sys.exit(1)
553 self.manifest._Unload()
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700554 if opt.jobs is None:
555 self.jobs = self.manifest.default.sync_j
David Pursehouse8a68ff92012-09-24 12:15:13 +0900556 all_projects = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700557
Dave Borowitz67700e92012-10-23 15:00:54 -0700558 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700559 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700560 to_fetch = []
561 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700562 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700563 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900564 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700565 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700566
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -0700567 self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700568 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700569 if opt.network_only:
570 # bail out now; the rest touches the working tree
571 return
572
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700573 if self.manifest.IsMirror:
574 # bail out now, we have no working tree
575 return
576
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700577 if self.UpdateProjectList():
578 sys.exit(1)
579
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700580 syncbuf = SyncBuffer(mp.config,
581 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900582 pm = Progress('Syncing work tree', len(all_projects))
583 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700584 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800585 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700586 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700587 pm.end()
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700588 print >>sys.stderr
589 if not syncbuf.Finish():
590 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700591
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700592 # If there's a notice that's supposed to print at the end of the sync, print
593 # it now...
594 if self.manifest.notice:
595 print self.manifest.notice
596
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700597def _PostRepoUpgrade(manifest, quiet=False):
Conley Owensc9129d92012-10-01 16:12:28 -0700598 wrapper = WrapperModule()
599 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700600 wrapper.SetupGnuPG(quiet)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700601 for project in manifest.projects.values():
602 if project.Exists:
603 project.PostRepoUpgrade()
604
605def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
606 if rp.HasChanges:
607 print >>sys.stderr, 'info: A new version of repo is available'
608 print >>sys.stderr, ''
609 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700610 syncbuf = SyncBuffer(rp.config)
611 rp.Sync_LocalHalf(syncbuf)
612 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700613 sys.exit(1)
614 print >>sys.stderr, 'info: Restarting repo with latest version'
615 raise RepoChangedException(['--repo-upgraded'])
616 else:
617 print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
618 else:
619 if verbose:
620 print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
621
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700622def _VerifyTag(project):
623 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
624 if not os.path.exists(gpg_dir):
625 print >>sys.stderr,\
626"""warning: GnuPG was not available during last "repo init"
627warning: Cannot automatically authenticate repo."""
628 return True
629
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700630 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700631 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700632 except GitError:
633 cur = None
634
635 if not cur \
636 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700637 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700638 if rev.startswith(R_HEADS):
639 rev = rev[len(R_HEADS):]
640
641 print >>sys.stderr
642 print >>sys.stderr,\
643 "warning: project '%s' branch '%s' is not signed" \
644 % (project.name, rev)
645 return False
646
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800647 env = os.environ.copy()
648 env['GIT_DIR'] = project.gitdir.encode()
649 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700650
651 cmd = [GIT, 'tag', '-v', cur]
652 proc = subprocess.Popen(cmd,
653 stdout = subprocess.PIPE,
654 stderr = subprocess.PIPE,
655 env = env)
656 out = proc.stdout.read()
657 proc.stdout.close()
658
659 err = proc.stderr.read()
660 proc.stderr.close()
661
662 if proc.wait() != 0:
663 print >>sys.stderr
664 print >>sys.stderr, out
665 print >>sys.stderr, err
666 print >>sys.stderr
667 return False
668 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700669
670class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700671 _ALPHA = 0.5
672
Dave Borowitz67700e92012-10-23 15:00:54 -0700673 def __init__(self, manifest):
674 self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
675 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700676 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700677
678 def Get(self, project):
679 self._Load()
680 return self._times.get(project.name, _ONE_DAY_S)
681
682 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700683 self._Load()
684 name = project.name
685 old = self._times.get(name, t)
686 self._seen.add(name)
687 a = self._ALPHA
688 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700689
690 def _Load(self):
691 if self._times is None:
692 try:
693 f = open(self._path)
694 except IOError:
695 self._times = {}
696 return self._times
697 try:
698 try:
699 self._times = pickle.load(f)
David Pursehouse1d947b32012-10-25 12:23:11 +0900700 except IOError:
Dave Borowitz67700e92012-10-23 15:00:54 -0700701 try:
702 os.remove(self._path)
703 except OSError:
704 pass
705 self._times = {}
706 finally:
707 f.close()
708 return self._times
709
710 def Save(self):
711 if self._times is None:
712 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700713
714 to_delete = []
715 for name in self._times:
716 if name not in self._seen:
717 to_delete.append(name)
718 for name in to_delete:
719 del self._times[name]
720
Dave Borowitz67700e92012-10-23 15:00:54 -0700721 try:
722 f = open(self._path, 'wb')
723 try:
724 pickle.dump(self._times, f)
725 except (IOError, OSError, pickle.PickleError):
726 try:
727 os.remove(self._path)
728 except OSError:
729 pass
730 finally:
731 f.close()