blob: 5b3dca78b3f275eac630fe7472c8df69d531e009 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
David Pursehouse86d973d2012-08-24 10:21:02 +090017import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070018from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Dave Borowitz67700e92012-10-23 15:00:54 -070020import pickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070026import time
David Pursehouse86d973d2012-08-24 10:21:02 +090027import urlparse
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070028import xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070029
Roy Lee18afd7f2010-05-09 04:32:08 +080030try:
31 import threading as _threading
32except ImportError:
33 import dummy_threading as _threading
34
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070035try:
36 import resource
37 def _rlimit_nofile():
38 return resource.getrlimit(resource.RLIMIT_NOFILE)
39except ImportError:
40 def _rlimit_nofile():
41 return (256, 256)
42
Dave Borowitz18857212012-10-23 17:02:59 -070043try:
44 import multiprocessing
45except ImportError:
46 multiprocessing = None
47
Dave Borowitze2152672012-10-31 12:24:38 -070048from git_command import GIT, git_require
David Pursehoused94aaef2012-09-07 09:52:04 +090049from git_refs import R_HEADS, HEAD
Conley Owensc9129d92012-10-01 16:12:28 -070050from main import WrapperModule
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070051from project import Project
52from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080053from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070054from error import RepoChangedException, GitError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070055from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070056from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070057
Dave Borowitz67700e92012-10-23 15:00:54 -070058_ONE_DAY_S = 24 * 60 * 60
59
Doug Andersonfc06ced2011-03-16 15:49:18 -070060class _FetchError(Exception):
61 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
62 pass
63
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080064class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080065 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070066 common = True
67 helpSummary = "Update working tree to the latest revision"
68 helpUsage = """
69%prog [<project>...]
70"""
71 helpDescription = """
72The '%prog' command synchronizes local project directories
73with the remote repositories specified in the manifest. If a local
74project does not yet exist, it will clone a new local directory from
75the remote repository and set up tracking branches as specified in
76the manifest. If the local project already exists, '%prog'
77will update the remote branches and rebase any new local changes
78on top of the new remote changes.
79
80'%prog' will synchronize all projects listed at the command
81line. Projects can be specified either by name, or by a relative
82or absolute path to the project's local directory. If no projects
83are specified, '%prog' will synchronize all projects listed in
84the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070085
86The -d/--detach option can be used to switch specified projects
87back to the manifest revision. This option is especially helpful
88if the project is currently on a topic branch, but the manifest
89revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070090
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070091The -s/--smart-sync option can be used to sync to a known good
92build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +020093manifest. The -t/--smart-tag option is similar and allows you to
94specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070095
David Pursehousecf76b1b2012-09-14 10:31:42 +090096The -u/--manifest-server-username and -p/--manifest-server-password
97options can be used to specify a username and password to authenticate
98with the manifest server when using the -s or -t option.
99
100If -u and -p are not specified when using the -s or -t option, '%prog'
101will attempt to read authentication credentials for the manifest server
102from the user's .netrc file.
103
104'%prog' will not use authentication credentials from -u/-p or .netrc
105if the manifest server specified in the manifest file already includes
106credentials.
107
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500108The -f/--force-broken option can be used to proceed with syncing
109other projects if a project sync fails.
110
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700111The --no-clone-bundle option disables any attempt to use
112$URL/clone.bundle to bootstrap a new Git repository from a
113resumeable bundle file on a content delivery network. This
114may be necessary if there are problems with the local Python
115HTTP client or proxy configuration, but the Git binary works.
116
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700117SSH Connections
118---------------
119
120If at least one project remote URL uses an SSH connection (ssh://,
121git+ssh://, or user@host:path syntax) repo will automatically
122enable the SSH ControlMaster option when connecting to that host.
123This feature permits other projects in the same '%prog' session to
124reuse the same SSH tunnel, saving connection setup overheads.
125
126To disable this behavior on UNIX platforms, set the GIT_SSH
127environment variable to 'ssh'. For example:
128
129 export GIT_SSH=ssh
130 %prog
131
132Compatibility
133~~~~~~~~~~~~~
134
135This feature is automatically disabled on Windows, due to the lack
136of UNIX domain socket support.
137
138This feature is not compatible with url.insteadof rewrites in the
139user's ~/.gitconfig. '%prog' is currently not able to perform the
140rewrite early enough to establish the ControlMaster tunnel.
141
142If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
143later is required to fix a server side protocol bug.
144
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700145"""
146
Nico Sallembien6623b212010-05-11 12:57:01 -0700147 def _Options(self, p, show_smart=True):
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700148 self.jobs = self.manifest.default.sync_j
149
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500150 p.add_option('-f', '--force-broken',
151 dest='force_broken', action='store_true',
152 help="continue sync even if a project fails to sync")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900153 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700154 dest='local_only', action='store_true',
155 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900156 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700157 dest='network_only', action='store_true',
158 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900159 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700160 dest='detach_head', action='store_true',
161 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900162 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700163 dest='current_branch_only', action='store_true',
164 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900165 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700166 dest='quiet', action='store_true',
167 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900168 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800169 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700170 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500171 p.add_option('-m', '--manifest-name',
172 dest='manifest_name',
173 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700174 p.add_option('--no-clone-bundle',
175 dest='no_clone_bundle', action='store_true',
176 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800177 p.add_option('-u', '--manifest-server-username', action='store',
178 dest='manifest_server_username',
179 help='username to authenticate with the manifest server')
180 p.add_option('-p', '--manifest-server-password', action='store',
181 dest='manifest_server_password',
182 help='password to authenticate with the manifest server')
Nico Sallembien6623b212010-05-11 12:57:01 -0700183 if show_smart:
184 p.add_option('-s', '--smart-sync',
185 dest='smart_sync', action='store_true',
186 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200187 p.add_option('-t', '--smart-tag',
188 dest='smart_tag', action='store',
189 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700190
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700191 g = p.add_option_group('repo Version options')
192 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700193 dest='no_repo_verify', action='store_true',
194 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700195 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800196 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700197 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700198
Doug Andersonfc06ced2011-03-16 15:49:18 -0700199 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
David Pursehousec1b86a22012-11-14 11:36:51 +0900200 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800201
David Pursehousec1b86a22012-11-14 11:36:51 +0900202 Args:
203 opt: Program options returned from optparse. See _Options().
204 project: Project object for the project to fetch.
205 lock: Lock for accessing objects that are shared amongst multiple
206 _FetchHelper() threads.
207 fetched: set object that we will add project.gitdir to when we're done
208 (with our lock held).
209 pm: Instance of a Project object. We will call pm.update() (with our
210 lock held).
211 sem: We'll release() this semaphore when we exit so that another thread
212 can be started up.
213 err_event: We'll set this event in the case of an error (after printing
214 out info about the error).
215 """
216 # We'll set to true once we've locked the lock.
217 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700218
David Pursehousec1b86a22012-11-14 11:36:51 +0900219 # Encapsulate everything in a try/except/finally so that:
220 # - We always set err_event in the case of an exception.
221 # - We always make sure we call sem.release().
222 # - We always make sure we unlock the lock if we locked it.
223 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700224 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900225 start = time.time()
226 success = project.Sync_NetworkHalf(
227 quiet=opt.quiet,
228 current_branch_only=opt.current_branch_only,
229 clone_bundle=not opt.no_clone_bundle)
230 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700231
David Pursehousec1b86a22012-11-14 11:36:51 +0900232 # Lock around all the rest of the code, since printing, updating a set
233 # and Progress.update() are not thread safe.
234 lock.acquire()
235 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700236
David Pursehousec1b86a22012-11-14 11:36:51 +0900237 if not success:
238 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
239 if opt.force_broken:
240 print('warn: --force-broken, continuing to sync',
241 file=sys.stderr)
242 else:
243 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700244
David Pursehousec1b86a22012-11-14 11:36:51 +0900245 fetched.add(project.gitdir)
246 pm.update()
247 except _FetchError:
248 err_event.set()
249 except:
250 err_event.set()
251 raise
252 finally:
253 if did_lock:
254 lock.release()
255 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800256
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700257 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700258 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700259 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800260
261 if self.jobs == 1:
262 for project in projects:
263 pm.update()
Shawn O. Pearce5d0efdb2012-08-02 12:13:01 -0700264 if project.Sync_NetworkHalf(
265 quiet=opt.quiet,
266 current_branch_only=opt.current_branch_only,
267 clone_bundle=not opt.no_clone_bundle):
Roy Lee18afd7f2010-05-09 04:32:08 +0800268 fetched.add(project.gitdir)
269 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700270 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500271 if opt.force_broken:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700272 print('warn: --force-broken, continuing to sync', file=sys.stderr)
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500273 else:
274 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800275 else:
276 threads = set()
277 lock = _threading.Lock()
278 sem = _threading.Semaphore(self.jobs)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700279 err_event = _threading.Event()
Roy Lee18afd7f2010-05-09 04:32:08 +0800280 for project in projects:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700281 # Check for any errors before starting any new threads.
282 # ...we'll let existing threads finish, though.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400283 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700284 break
285
Roy Lee18afd7f2010-05-09 04:32:08 +0800286 sem.acquire()
287 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700288 args = (opt,
289 project,
290 lock,
291 fetched,
292 pm,
Doug Andersonfc06ced2011-03-16 15:49:18 -0700293 sem,
294 err_event))
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200295 # Ensure that Ctrl-C will not freeze the repo process.
296 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800297 threads.add(t)
298 t.start()
299
300 for t in threads:
301 t.join()
302
Doug Andersonfc06ced2011-03-16 15:49:18 -0700303 # If we saw an error, exit with code 1 so that other scripts can check.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400304 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700305 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700306 sys.exit(1)
307
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700308 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700309 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700310
311 self._GCProjects(projects)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700312 return fetched
313
Dave Borowitz18857212012-10-23 17:02:59 -0700314 def _GCProjects(self, projects):
Dave Borowitze2152672012-10-31 12:24:38 -0700315 has_dash_c = git_require((1, 7, 2))
316 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700317 cpu_count = multiprocessing.cpu_count()
318 else:
319 cpu_count = 1
320 jobs = min(self.jobs, cpu_count)
321
322 if jobs < 2:
323 for project in projects:
324 project.bare_git.gc('--auto')
325 return
326
327 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
328
329 threads = set()
330 sem = _threading.Semaphore(jobs)
331 err_event = _threading.Event()
332
333 def GC(project):
334 try:
335 try:
336 project.bare_git.gc('--auto', config=config)
337 except GitError:
338 err_event.set()
339 except:
340 err_event.set()
341 raise
342 finally:
343 sem.release()
344
345 for project in projects:
346 if err_event.isSet():
347 break
348 sem.acquire()
349 t = _threading.Thread(target=GC, args=(project,))
350 t.daemon = True
351 threads.add(t)
352 t.start()
353
354 for t in threads:
355 t.join()
356
357 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700358 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700359 sys.exit(1)
360
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700361 def UpdateProjectList(self):
362 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700363 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700364 if project.relpath:
365 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700366 file_name = 'project.list'
367 file_path = os.path.join(self.manifest.repodir, file_name)
368 old_project_paths = []
369
370 if os.path.exists(file_path):
371 fd = open(file_path, 'r')
372 try:
373 old_project_paths = fd.read().split('\n')
374 finally:
375 fd.close()
376 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700377 if not path:
378 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700379 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900380 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400381 if os.path.exists(self.manifest.topdir + '/' + path):
David Pursehousec1b86a22012-11-14 11:36:51 +0900382 project = Project(
383 manifest = self.manifest,
384 name = path,
385 remote = RemoteSpec('origin'),
386 gitdir = os.path.join(self.manifest.topdir,
387 path, '.git'),
388 worktree = os.path.join(self.manifest.topdir, path),
389 relpath = path,
390 revisionExpr = 'HEAD',
391 revisionId = None,
392 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400393
David Pursehousec1b86a22012-11-14 11:36:51 +0900394 if project.IsDirty():
395 print('error: Cannot remove project "%s": uncommitted changes'
396 'are present' % project.relpath, file=sys.stderr)
397 print(' commit changes, then run sync again',
398 file=sys.stderr)
399 return -1
400 else:
401 print('Deleting obsolete path %s' % project.worktree,
402 file=sys.stderr)
403 shutil.rmtree(project.worktree)
404 # Try deleting parent subdirs if they are empty
405 project_dir = os.path.dirname(project.worktree)
406 while project_dir != self.manifest.topdir:
407 try:
408 os.rmdir(project_dir)
409 except OSError:
410 break
411 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700412
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700413 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700414 fd = open(file_path, 'w')
415 try:
416 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700417 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700418 finally:
419 fd.close()
420 return 0
421
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700422 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800423 if opt.jobs:
424 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700425 if self.jobs > 1:
426 soft_limit, _ = _rlimit_nofile()
427 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
428
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700429 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700430 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700431 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700432 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700433 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700434 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500435 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700436 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500437 sys.exit(1)
438 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700439 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500440 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900441 if opt.manifest_server_username or opt.manifest_server_password:
442 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700443 print('error: -u and -p may only be combined with -s or -t',
444 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900445 sys.exit(1)
446 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700447 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900448 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500449
450 if opt.manifest_name:
451 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700452
Victor Boivie08c880d2011-04-19 10:32:52 +0200453 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700454 if not self.manifest.manifest_server:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700455 print('error: cannot smart sync: no manifest server defined in'
456 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700457 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900458
459 manifest_server = self.manifest.manifest_server
David Pursehousecf76b1b2012-09-14 10:31:42 +0900460
David Pursehouse86d973d2012-08-24 10:21:02 +0900461 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900462 username = None
463 password = None
464 if opt.manifest_server_username and opt.manifest_server_password:
465 username = opt.manifest_server_username
466 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900467 else:
468 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900469 info = netrc.netrc()
470 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700471 print('.netrc file does not exist or could not be opened',
472 file=sys.stderr)
David Pursehouse86d973d2012-08-24 10:21:02 +0900473 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900474 try:
475 parse_result = urlparse.urlparse(manifest_server)
476 if parse_result.hostname:
477 username, _account, password = \
478 info.authenticators(parse_result.hostname)
479 except TypeError:
480 # TypeError is raised when the given hostname is not present
481 # in the .netrc file.
Sarah Owenscecd1d82012-11-01 22:59:27 -0700482 print('No credentials found for %s in .netrc'
483 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700484 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700485 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900486
487 if (username and password):
488 manifest_server = manifest_server.replace('://', '://%s:%s@' %
489 (username, password),
490 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900491
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700492 try:
David Pursehouse86d973d2012-08-24 10:21:02 +0900493 server = xmlrpclib.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200494 if opt.smart_sync:
495 p = self.manifest.manifestProject
496 b = p.GetBranch(p.CurrentBranch)
497 branch = b.merge
498 if branch.startswith(R_HEADS):
499 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700500
Victor Boivie08c880d2011-04-19 10:32:52 +0200501 env = os.environ.copy()
502 if (env.has_key('TARGET_PRODUCT') and
503 env.has_key('TARGET_BUILD_VARIANT')):
504 target = '%s-%s' % (env['TARGET_PRODUCT'],
505 env['TARGET_BUILD_VARIANT'])
506 [success, manifest_str] = server.GetApprovedManifest(branch, target)
507 else:
508 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700509 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200510 assert(opt.smart_tag)
511 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700512
513 if success:
514 manifest_name = "smart_sync_override.xml"
515 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
516 manifest_name)
517 try:
518 f = open(manifest_path, 'w')
519 try:
520 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700521 finally:
522 f.close()
523 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700524 print('error: cannot write manifest to %s' % manifest_path,
525 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700526 sys.exit(1)
Nico Sallembien719965a2010-04-20 15:28:19 -0700527 self.manifest.Override(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700528 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700529 print('error: %s' % manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700530 sys.exit(1)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700531 except (socket.error, IOError, xmlrpclib.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700532 print('error: cannot connect to manifest server %s:\n%s'
533 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900534 sys.exit(1)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700535 except xmlrpclib.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700536 print('error: cannot connect to manifest server %s:\n%d %s'
537 % (self.manifest.manifest_server, e.errcode, e.errmsg),
538 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700539 sys.exit(1)
540
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700541 rp = self.manifest.repoProject
542 rp.PreSync()
543
544 mp = self.manifest.manifestProject
545 mp.PreSync()
546
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800547 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700548 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800549
Nico Sallembien9bb18162009-12-07 15:38:01 -0800550 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700551 mp.Sync_NetworkHalf(quiet=opt.quiet,
552 current_branch_only=opt.current_branch_only)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800553
554 if mp.HasChanges:
555 syncbuf = SyncBuffer(mp.config)
556 mp.Sync_LocalHalf(syncbuf)
557 if not syncbuf.Finish():
558 sys.exit(1)
559 self.manifest._Unload()
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700560 if opt.jobs is None:
561 self.jobs = self.manifest.default.sync_j
David Pursehouse8a68ff92012-09-24 12:15:13 +0900562 all_projects = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700563
Dave Borowitz67700e92012-10-23 15:00:54 -0700564 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700565 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700566 to_fetch = []
567 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700568 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700569 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900570 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700571 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700572
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -0700573 self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700574 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700575 if opt.network_only:
576 # bail out now; the rest touches the working tree
577 return
578
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700579 if self.manifest.IsMirror:
580 # bail out now, we have no working tree
581 return
582
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700583 if self.UpdateProjectList():
584 sys.exit(1)
585
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700586 syncbuf = SyncBuffer(mp.config,
587 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900588 pm = Progress('Syncing work tree', len(all_projects))
589 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700590 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800591 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700592 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700593 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700594 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700595 if not syncbuf.Finish():
596 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700597
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700598 # If there's a notice that's supposed to print at the end of the sync, print
599 # it now...
600 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700601 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700602
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700603def _PostRepoUpgrade(manifest, quiet=False):
Conley Owensc9129d92012-10-01 16:12:28 -0700604 wrapper = WrapperModule()
605 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700606 wrapper.SetupGnuPG(quiet)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700607 for project in manifest.projects.values():
608 if project.Exists:
609 project.PostRepoUpgrade()
610
611def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
612 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700613 print('info: A new version of repo is available', file=sys.stderr)
614 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700615 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700616 syncbuf = SyncBuffer(rp.config)
617 rp.Sync_LocalHalf(syncbuf)
618 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700619 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700620 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700621 raise RepoChangedException(['--repo-upgraded'])
622 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700623 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700624 else:
625 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700626 print('repo version %s is current' % rp.work_git.describe(HEAD),
627 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700628
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700629def _VerifyTag(project):
630 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
631 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700632 print('warning: GnuPG was not available during last "repo init"\n'
633 'warning: Cannot automatically authenticate repo."""',
634 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700635 return True
636
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700637 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700638 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700639 except GitError:
640 cur = None
641
642 if not cur \
643 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700644 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700645 if rev.startswith(R_HEADS):
646 rev = rev[len(R_HEADS):]
647
Sarah Owenscecd1d82012-11-01 22:59:27 -0700648 print(file=sys.stderr)
649 print("warning: project '%s' branch '%s' is not signed"
650 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700651 return False
652
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800653 env = os.environ.copy()
654 env['GIT_DIR'] = project.gitdir.encode()
655 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700656
657 cmd = [GIT, 'tag', '-v', cur]
658 proc = subprocess.Popen(cmd,
659 stdout = subprocess.PIPE,
660 stderr = subprocess.PIPE,
661 env = env)
662 out = proc.stdout.read()
663 proc.stdout.close()
664
665 err = proc.stderr.read()
666 proc.stderr.close()
667
668 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700669 print(file=sys.stderr)
670 print(out, file=sys.stderr)
671 print(err, file=sys.stderr)
672 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700673 return False
674 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700675
676class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700677 _ALPHA = 0.5
678
Dave Borowitz67700e92012-10-23 15:00:54 -0700679 def __init__(self, manifest):
680 self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
681 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700682 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700683
684 def Get(self, project):
685 self._Load()
686 return self._times.get(project.name, _ONE_DAY_S)
687
688 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700689 self._Load()
690 name = project.name
691 old = self._times.get(name, t)
692 self._seen.add(name)
693 a = self._ALPHA
694 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700695
696 def _Load(self):
697 if self._times is None:
698 try:
699 f = open(self._path)
700 except IOError:
701 self._times = {}
702 return self._times
703 try:
704 try:
705 self._times = pickle.load(f)
David Pursehouse1d947b32012-10-25 12:23:11 +0900706 except IOError:
Dave Borowitz67700e92012-10-23 15:00:54 -0700707 try:
708 os.remove(self._path)
709 except OSError:
710 pass
711 self._times = {}
712 finally:
713 f.close()
714 return self._times
715
716 def Save(self):
717 if self._times is None:
718 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700719
720 to_delete = []
721 for name in self._times:
722 if name not in self._seen:
723 to_delete.append(name)
724 for name in to_delete:
725 del self._times[name]
726
Dave Borowitz67700e92012-10-23 15:00:54 -0700727 try:
728 f = open(self._path, 'wb')
729 try:
730 pickle.dump(self._times, f)
731 except (IOError, OSError, pickle.PickleError):
732 try:
733 os.remove(self._path)
734 except OSError:
735 pass
736 finally:
737 f.close()