blob: a0a689607d5b744a5f7713de7023f6d0a3c12b4b [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
David Pursehouse86d973d2012-08-24 10:21:02 +090017import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070018from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Dave Borowitz67700e92012-10-23 15:00:54 -070020import pickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070026import time
David Pursehouse59bbb582013-05-17 10:49:33 +090027
28from pyversion import is_python3
29if is_python3():
Chirayu Desai217ea7d2013-03-01 19:14:38 +053030 import urllib.parse
David Pursehouse59bbb582013-05-17 10:49:33 +090031 import xmlrpc.client
32else:
Chirayu Desai217ea7d2013-03-01 19:14:38 +053033 import imp
34 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090035 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053036 urllib = imp.new_module('urllib')
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053037 urllib.parse = urlparse
Chirayu Desai217ea7d2013-03-01 19:14:38 +053038 xmlrpc = imp.new_module('xmlrpc')
39 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
Roy Lee18afd7f2010-05-09 04:32:08 +080041try:
42 import threading as _threading
43except ImportError:
44 import dummy_threading as _threading
45
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070046try:
47 import resource
48 def _rlimit_nofile():
49 return resource.getrlimit(resource.RLIMIT_NOFILE)
50except ImportError:
51 def _rlimit_nofile():
52 return (256, 256)
53
Dave Borowitz18857212012-10-23 17:02:59 -070054try:
55 import multiprocessing
56except ImportError:
57 multiprocessing = None
58
Dave Borowitze2152672012-10-31 12:24:38 -070059from git_command import GIT, git_require
David Pursehoused94aaef2012-09-07 09:52:04 +090060from git_refs import R_HEADS, HEAD
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070061from project import Project
62from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080063from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000064from error import RepoChangedException, GitError, ManifestParseError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070065from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070066from progress import Progress
Conley Owens094cdbe2014-01-30 15:09:59 -080067from wrapper import Wrapper
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070068
Dave Borowitz67700e92012-10-23 15:00:54 -070069_ONE_DAY_S = 24 * 60 * 60
70
Doug Andersonfc06ced2011-03-16 15:49:18 -070071class _FetchError(Exception):
72 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
73 pass
74
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080075class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080076 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070077 common = True
78 helpSummary = "Update working tree to the latest revision"
79 helpUsage = """
80%prog [<project>...]
81"""
82 helpDescription = """
83The '%prog' command synchronizes local project directories
84with the remote repositories specified in the manifest. If a local
85project does not yet exist, it will clone a new local directory from
86the remote repository and set up tracking branches as specified in
87the manifest. If the local project already exists, '%prog'
88will update the remote branches and rebase any new local changes
89on top of the new remote changes.
90
91'%prog' will synchronize all projects listed at the command
92line. Projects can be specified either by name, or by a relative
93or absolute path to the project's local directory. If no projects
94are specified, '%prog' will synchronize all projects listed in
95the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070096
97The -d/--detach option can be used to switch specified projects
98back to the manifest revision. This option is especially helpful
99if the project is currently on a topic branch, but the manifest
100revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700101
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700102The -s/--smart-sync option can be used to sync to a known good
103build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200104manifest. The -t/--smart-tag option is similar and allows you to
105specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700106
David Pursehousecf76b1b2012-09-14 10:31:42 +0900107The -u/--manifest-server-username and -p/--manifest-server-password
108options can be used to specify a username and password to authenticate
109with the manifest server when using the -s or -t option.
110
111If -u and -p are not specified when using the -s or -t option, '%prog'
112will attempt to read authentication credentials for the manifest server
113from the user's .netrc file.
114
115'%prog' will not use authentication credentials from -u/-p or .netrc
116if the manifest server specified in the manifest file already includes
117credentials.
118
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500119The -f/--force-broken option can be used to proceed with syncing
120other projects if a project sync fails.
121
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700122The --no-clone-bundle option disables any attempt to use
123$URL/clone.bundle to bootstrap a new Git repository from a
124resumeable bundle file on a content delivery network. This
125may be necessary if there are problems with the local Python
126HTTP client or proxy configuration, but the Git binary works.
127
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800128The --fetch-submodules option enables fetching Git submodules
129of a project from server.
130
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700131SSH Connections
132---------------
133
134If at least one project remote URL uses an SSH connection (ssh://,
135git+ssh://, or user@host:path syntax) repo will automatically
136enable the SSH ControlMaster option when connecting to that host.
137This feature permits other projects in the same '%prog' session to
138reuse the same SSH tunnel, saving connection setup overheads.
139
140To disable this behavior on UNIX platforms, set the GIT_SSH
141environment variable to 'ssh'. For example:
142
143 export GIT_SSH=ssh
144 %prog
145
146Compatibility
147~~~~~~~~~~~~~
148
149This feature is automatically disabled on Windows, due to the lack
150of UNIX domain socket support.
151
152This feature is not compatible with url.insteadof rewrites in the
153user's ~/.gitconfig. '%prog' is currently not able to perform the
154rewrite early enough to establish the ControlMaster tunnel.
155
156If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
157later is required to fix a server side protocol bug.
158
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700159"""
160
Nico Sallembien6623b212010-05-11 12:57:01 -0700161 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000162 try:
163 self.jobs = self.manifest.default.sync_j
164 except ManifestParseError:
165 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700166
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500167 p.add_option('-f', '--force-broken',
168 dest='force_broken', action='store_true',
169 help="continue sync even if a project fails to sync")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900170 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700171 dest='local_only', action='store_true',
172 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900173 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700174 dest='network_only', action='store_true',
175 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900176 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700177 dest='detach_head', action='store_true',
178 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900179 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700180 dest='current_branch_only', action='store_true',
181 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900182 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700183 dest='quiet', action='store_true',
184 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900185 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800186 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700187 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500188 p.add_option('-m', '--manifest-name',
189 dest='manifest_name',
190 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700191 p.add_option('--no-clone-bundle',
192 dest='no_clone_bundle', action='store_true',
193 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800194 p.add_option('-u', '--manifest-server-username', action='store',
195 dest='manifest_server_username',
196 help='username to authenticate with the manifest server')
197 p.add_option('-p', '--manifest-server-password', action='store',
198 dest='manifest_server_password',
199 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800200 p.add_option('--fetch-submodules',
201 dest='fetch_submodules', action='store_true',
202 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700203 p.add_option('--no-tags',
204 dest='no_tags', action='store_true',
205 help="don't fetch tags")
Nico Sallembien6623b212010-05-11 12:57:01 -0700206 if show_smart:
207 p.add_option('-s', '--smart-sync',
208 dest='smart_sync', action='store_true',
209 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200210 p.add_option('-t', '--smart-tag',
211 dest='smart_tag', action='store',
212 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700213
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700214 g = p.add_option_group('repo Version options')
215 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700216 dest='no_repo_verify', action='store_true',
217 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700218 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800219 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700220 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700221
David James89ece422014-01-09 18:51:58 -0800222 def _FetchProjectList(self, opt, projects, *args, **kwargs):
David Pursehousec1b86a22012-11-14 11:36:51 +0900223 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800224
David James8d201162013-10-11 17:03:19 -0700225 Delegates most of the work to _FetchHelper.
226
227 Args:
228 opt: Program options returned from optparse. See _Options().
229 projects: Projects to fetch.
David James89ece422014-01-09 18:51:58 -0800230 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
David James8d201162013-10-11 17:03:19 -0700231 _FetchHelper docstring for details.
232 """
233 for project in projects:
David James89ece422014-01-09 18:51:58 -0800234 success = self._FetchHelper(opt, project, *args, **kwargs)
David James8d201162013-10-11 17:03:19 -0700235 if not success and not opt.force_broken:
236 break
237
238 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
239 """Fetch git objects for a single project.
240
David Pursehousec1b86a22012-11-14 11:36:51 +0900241 Args:
242 opt: Program options returned from optparse. See _Options().
243 project: Project object for the project to fetch.
244 lock: Lock for accessing objects that are shared amongst multiple
245 _FetchHelper() threads.
246 fetched: set object that we will add project.gitdir to when we're done
247 (with our lock held).
248 pm: Instance of a Project object. We will call pm.update() (with our
249 lock held).
250 sem: We'll release() this semaphore when we exit so that another thread
251 can be started up.
252 err_event: We'll set this event in the case of an error (after printing
253 out info about the error).
David James8d201162013-10-11 17:03:19 -0700254
255 Returns:
256 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900257 """
258 # We'll set to true once we've locked the lock.
259 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700260
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530261 if not opt.quiet:
262 print('Fetching project %s' % project.name)
263
David Pursehousec1b86a22012-11-14 11:36:51 +0900264 # Encapsulate everything in a try/except/finally so that:
265 # - We always set err_event in the case of an exception.
266 # - We always make sure we call sem.release().
267 # - We always make sure we unlock the lock if we locked it.
268 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700269 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900270 start = time.time()
271 success = project.Sync_NetworkHalf(
272 quiet=opt.quiet,
273 current_branch_only=opt.current_branch_only,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700274 clone_bundle=not opt.no_clone_bundle,
Julien Campergue335f5ef2013-10-16 11:02:35 +0200275 no_tags=opt.no_tags, archive=self.manifest.IsArchive)
David Pursehousec1b86a22012-11-14 11:36:51 +0900276 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700277
David Pursehousec1b86a22012-11-14 11:36:51 +0900278 # Lock around all the rest of the code, since printing, updating a set
279 # and Progress.update() are not thread safe.
280 lock.acquire()
281 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700282
David Pursehousec1b86a22012-11-14 11:36:51 +0900283 if not success:
284 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
285 if opt.force_broken:
286 print('warn: --force-broken, continuing to sync',
287 file=sys.stderr)
288 else:
289 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700290
David Pursehousec1b86a22012-11-14 11:36:51 +0900291 fetched.add(project.gitdir)
292 pm.update()
293 except _FetchError:
294 err_event.set()
295 except:
296 err_event.set()
297 raise
298 finally:
299 if did_lock:
300 lock.release()
301 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800302
David James8d201162013-10-11 17:03:19 -0700303 return success
304
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700305 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700306 fetched = set()
David James89ece422014-01-09 18:51:58 -0800307 lock = _threading.Lock()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700308 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800309
David James89ece422014-01-09 18:51:58 -0800310 objdir_project_map = dict()
311 for project in projects:
312 objdir_project_map.setdefault(project.objdir, []).append(project)
David James8d201162013-10-11 17:03:19 -0700313
David James89ece422014-01-09 18:51:58 -0800314 threads = set()
315 sem = _threading.Semaphore(self.jobs)
316 err_event = _threading.Event()
317 for project_list in objdir_project_map.values():
318 # Check for any errors before running any more tasks.
319 # ...we'll let existing threads finish, though.
320 if err_event.isSet() and not opt.force_broken:
321 break
Doug Andersonfc06ced2011-03-16 15:49:18 -0700322
David James89ece422014-01-09 18:51:58 -0800323 sem.acquire()
324 kwargs = dict(opt=opt,
325 projects=project_list,
326 lock=lock,
327 fetched=fetched,
328 pm=pm,
329 sem=sem,
330 err_event=err_event)
331 if self.jobs > 1:
David James8d201162013-10-11 17:03:19 -0700332 t = _threading.Thread(target = self._FetchProjectList,
David James89ece422014-01-09 18:51:58 -0800333 kwargs = kwargs)
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200334 # Ensure that Ctrl-C will not freeze the repo process.
335 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800336 threads.add(t)
337 t.start()
David James89ece422014-01-09 18:51:58 -0800338 else:
339 self._FetchProjectList(**kwargs)
Roy Lee18afd7f2010-05-09 04:32:08 +0800340
David James89ece422014-01-09 18:51:58 -0800341 for t in threads:
342 t.join()
Roy Lee18afd7f2010-05-09 04:32:08 +0800343
David James89ece422014-01-09 18:51:58 -0800344 # If we saw an error, exit with code 1 so that other scripts can check.
345 if err_event.isSet():
346 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
347 sys.exit(1)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700348
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700349 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700350 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700351
Julien Campergue335f5ef2013-10-16 11:02:35 +0200352 if not self.manifest.IsArchive:
353 self._GCProjects(projects)
354
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700355 return fetched
356
Dave Borowitz18857212012-10-23 17:02:59 -0700357 def _GCProjects(self, projects):
David James8d201162013-10-11 17:03:19 -0700358 gitdirs = {}
359 for project in projects:
360 gitdirs[project.gitdir] = project.bare_git
361
Dave Borowitze2152672012-10-31 12:24:38 -0700362 has_dash_c = git_require((1, 7, 2))
363 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700364 cpu_count = multiprocessing.cpu_count()
365 else:
366 cpu_count = 1
367 jobs = min(self.jobs, cpu_count)
368
369 if jobs < 2:
David James8d201162013-10-11 17:03:19 -0700370 for bare_git in gitdirs.values():
371 bare_git.gc('--auto')
Dave Borowitz18857212012-10-23 17:02:59 -0700372 return
373
374 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
375
376 threads = set()
377 sem = _threading.Semaphore(jobs)
378 err_event = _threading.Event()
379
David James8d201162013-10-11 17:03:19 -0700380 def GC(bare_git):
Dave Borowitz18857212012-10-23 17:02:59 -0700381 try:
382 try:
David James8d201162013-10-11 17:03:19 -0700383 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700384 except GitError:
385 err_event.set()
386 except:
387 err_event.set()
388 raise
389 finally:
390 sem.release()
391
David James8d201162013-10-11 17:03:19 -0700392 for bare_git in gitdirs.values():
Dave Borowitz18857212012-10-23 17:02:59 -0700393 if err_event.isSet():
394 break
395 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700396 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700397 t.daemon = True
398 threads.add(t)
399 t.start()
400
401 for t in threads:
402 t.join()
403
404 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700405 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700406 sys.exit(1)
407
Tim Kilbourn07669002013-03-08 15:02:49 -0800408 def _ReloadManifest(self, manifest_name=None):
409 if manifest_name:
410 # Override calls _Unload already
411 self.manifest.Override(manifest_name)
412 else:
413 self.manifest._Unload()
414
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700415 def UpdateProjectList(self):
416 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700417 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700418 if project.relpath:
419 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700420 file_name = 'project.list'
421 file_path = os.path.join(self.manifest.repodir, file_name)
422 old_project_paths = []
423
424 if os.path.exists(file_path):
425 fd = open(file_path, 'r')
426 try:
427 old_project_paths = fd.read().split('\n')
428 finally:
429 fd.close()
430 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700431 if not path:
432 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700433 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900434 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400435 if os.path.exists(self.manifest.topdir + '/' + path):
David James8d201162013-10-11 17:03:19 -0700436 gitdir = os.path.join(self.manifest.topdir, path, '.git')
David Pursehousec1b86a22012-11-14 11:36:51 +0900437 project = Project(
438 manifest = self.manifest,
439 name = path,
440 remote = RemoteSpec('origin'),
David James8d201162013-10-11 17:03:19 -0700441 gitdir = gitdir,
442 objdir = gitdir,
David Pursehousec1b86a22012-11-14 11:36:51 +0900443 worktree = os.path.join(self.manifest.topdir, path),
444 relpath = path,
445 revisionExpr = 'HEAD',
446 revisionId = None,
447 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400448
David Pursehousec1b86a22012-11-14 11:36:51 +0900449 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900450 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900451 'are present' % project.relpath, file=sys.stderr)
452 print(' commit changes, then run sync again',
453 file=sys.stderr)
454 return -1
455 else:
456 print('Deleting obsolete path %s' % project.worktree,
457 file=sys.stderr)
458 shutil.rmtree(project.worktree)
459 # Try deleting parent subdirs if they are empty
460 project_dir = os.path.dirname(project.worktree)
461 while project_dir != self.manifest.topdir:
462 try:
463 os.rmdir(project_dir)
464 except OSError:
465 break
466 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700467
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700468 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700469 fd = open(file_path, 'w')
470 try:
471 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700472 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700473 finally:
474 fd.close()
475 return 0
476
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700477 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800478 if opt.jobs:
479 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700480 if self.jobs > 1:
481 soft_limit, _ = _rlimit_nofile()
482 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
483
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700484 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700485 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700486 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700487 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700488 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700489 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500490 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700491 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500492 sys.exit(1)
493 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700494 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500495 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900496 if opt.manifest_server_username or opt.manifest_server_password:
497 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700498 print('error: -u and -p may only be combined with -s or -t',
499 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900500 sys.exit(1)
501 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700502 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900503 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500504
505 if opt.manifest_name:
506 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700507
Chirayu Desaia892b102013-06-11 14:18:46 +0530508 manifest_name = opt.manifest_name
509
Victor Boivie08c880d2011-04-19 10:32:52 +0200510 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700511 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900512 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700513 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700514 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900515
516 manifest_server = self.manifest.manifest_server
David Pursehousefb99c712013-09-25 11:09:34 +0900517 if not opt.quiet:
518 print('Using manifest server %s' % manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900519
David Pursehouse86d973d2012-08-24 10:21:02 +0900520 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900521 username = None
522 password = None
523 if opt.manifest_server_username and opt.manifest_server_password:
524 username = opt.manifest_server_username
525 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900526 else:
527 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900528 info = netrc.netrc()
529 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700530 print('.netrc file does not exist or could not be opened',
531 file=sys.stderr)
David Pursehouse86d973d2012-08-24 10:21:02 +0900532 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900533 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530534 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900535 if parse_result.hostname:
536 username, _account, password = \
537 info.authenticators(parse_result.hostname)
538 except TypeError:
539 # TypeError is raised when the given hostname is not present
540 # in the .netrc file.
Sarah Owenscecd1d82012-11-01 22:59:27 -0700541 print('No credentials found for %s in .netrc'
542 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700543 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700544 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900545
546 if (username and password):
547 manifest_server = manifest_server.replace('://', '://%s:%s@' %
548 (username, password),
549 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900550
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700551 try:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530552 server = xmlrpc.client.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200553 if opt.smart_sync:
554 p = self.manifest.manifestProject
555 b = p.GetBranch(p.CurrentBranch)
556 branch = b.merge
557 if branch.startswith(R_HEADS):
558 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700559
Victor Boivie08c880d2011-04-19 10:32:52 +0200560 env = os.environ.copy()
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530561 if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200562 target = '%s-%s' % (env['TARGET_PRODUCT'],
563 env['TARGET_BUILD_VARIANT'])
564 [success, manifest_str] = server.GetApprovedManifest(branch, target)
565 else:
566 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700567 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200568 assert(opt.smart_tag)
569 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700570
571 if success:
572 manifest_name = "smart_sync_override.xml"
573 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
574 manifest_name)
575 try:
576 f = open(manifest_path, 'w')
577 try:
578 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700579 finally:
580 f.close()
581 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700582 print('error: cannot write manifest to %s' % manifest_path,
583 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700584 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100585 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700586 else:
David Pursehouse351fe2c2013-09-25 17:54:26 +0900587 print('error: manifest server RPC call failed: %s' %
588 manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700589 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530590 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700591 print('error: cannot connect to manifest server %s:\n%s'
592 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900593 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530594 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700595 print('error: cannot connect to manifest server %s:\n%d %s'
596 % (self.manifest.manifest_server, e.errcode, e.errmsg),
597 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700598 sys.exit(1)
599
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700600 rp = self.manifest.repoProject
601 rp.PreSync()
602
603 mp = self.manifest.manifestProject
604 mp.PreSync()
605
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800606 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700607 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800608
Nico Sallembien9bb18162009-12-07 15:38:01 -0800609 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700610 mp.Sync_NetworkHalf(quiet=opt.quiet,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700611 current_branch_only=opt.current_branch_only,
612 no_tags=opt.no_tags)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800613
614 if mp.HasChanges:
615 syncbuf = SyncBuffer(mp.config)
616 mp.Sync_LocalHalf(syncbuf)
617 if not syncbuf.Finish():
618 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100619 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700620 if opt.jobs is None:
621 self.jobs = self.manifest.default.sync_j
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800622 all_projects = self.GetProjects(args,
623 missing_ok=True,
624 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700625
Dave Borowitz67700e92012-10-23 15:00:54 -0700626 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700627 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700628 to_fetch = []
629 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700630 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700631 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900632 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700633 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700634
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800635 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700636 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700637 if opt.network_only:
638 # bail out now; the rest touches the working tree
639 return
640
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800641 # Iteratively fetch missing and/or nested unregistered submodules
642 previously_missing_set = set()
643 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100644 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800645 all_projects = self.GetProjects(args,
646 missing_ok=True,
647 submodules_ok=opt.fetch_submodules)
648 missing = []
649 for project in all_projects:
650 if project.gitdir not in fetched:
651 missing.append(project)
652 if not missing:
653 break
654 # Stop us from non-stopped fetching actually-missing repos: If set of
655 # missing repos has not been changed from last fetch, we break.
656 missing_set = set(p.name for p in missing)
657 if previously_missing_set == missing_set:
658 break
659 previously_missing_set = missing_set
660 fetched.update(self._Fetch(missing, opt))
661
Julien Campergue335f5ef2013-10-16 11:02:35 +0200662 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700663 # bail out now, we have no working tree
664 return
665
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700666 if self.UpdateProjectList():
667 sys.exit(1)
668
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700669 syncbuf = SyncBuffer(mp.config,
670 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900671 pm = Progress('Syncing work tree', len(all_projects))
672 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700673 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800674 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700675 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700676 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700677 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700678 if not syncbuf.Finish():
679 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700680
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700681 # If there's a notice that's supposed to print at the end of the sync, print
682 # it now...
683 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700684 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700685
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700686def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -0800687 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -0700688 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700689 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -0800690 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700691 if project.Exists:
692 project.PostRepoUpgrade()
693
694def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
695 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700696 print('info: A new version of repo is available', file=sys.stderr)
697 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700698 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700699 syncbuf = SyncBuffer(rp.config)
700 rp.Sync_LocalHalf(syncbuf)
701 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700702 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700703 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700704 raise RepoChangedException(['--repo-upgraded'])
705 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700706 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700707 else:
708 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700709 print('repo version %s is current' % rp.work_git.describe(HEAD),
710 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700711
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700712def _VerifyTag(project):
713 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
714 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700715 print('warning: GnuPG was not available during last "repo init"\n'
716 'warning: Cannot automatically authenticate repo."""',
717 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700718 return True
719
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700720 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700721 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700722 except GitError:
723 cur = None
724
725 if not cur \
726 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700727 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700728 if rev.startswith(R_HEADS):
729 rev = rev[len(R_HEADS):]
730
Sarah Owenscecd1d82012-11-01 22:59:27 -0700731 print(file=sys.stderr)
732 print("warning: project '%s' branch '%s' is not signed"
733 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700734 return False
735
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800736 env = os.environ.copy()
737 env['GIT_DIR'] = project.gitdir.encode()
738 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700739
740 cmd = [GIT, 'tag', '-v', cur]
741 proc = subprocess.Popen(cmd,
742 stdout = subprocess.PIPE,
743 stderr = subprocess.PIPE,
744 env = env)
745 out = proc.stdout.read()
746 proc.stdout.close()
747
748 err = proc.stderr.read()
749 proc.stderr.close()
750
751 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700752 print(file=sys.stderr)
753 print(out, file=sys.stderr)
754 print(err, file=sys.stderr)
755 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700756 return False
757 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700758
759class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700760 _ALPHA = 0.5
761
Dave Borowitz67700e92012-10-23 15:00:54 -0700762 def __init__(self, manifest):
763 self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
764 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700765 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700766
767 def Get(self, project):
768 self._Load()
769 return self._times.get(project.name, _ONE_DAY_S)
770
771 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700772 self._Load()
773 name = project.name
774 old = self._times.get(name, t)
775 self._seen.add(name)
776 a = self._ALPHA
777 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700778
779 def _Load(self):
780 if self._times is None:
781 try:
Chirayu Desai0eb35cb2013-11-19 18:46:29 +0530782 f = open(self._path, 'rb')
Dave Borowitz67700e92012-10-23 15:00:54 -0700783 except IOError:
784 self._times = {}
785 return self._times
786 try:
787 try:
788 self._times = pickle.load(f)
David Pursehouse1d947b32012-10-25 12:23:11 +0900789 except IOError:
Dave Borowitz67700e92012-10-23 15:00:54 -0700790 try:
791 os.remove(self._path)
792 except OSError:
793 pass
794 self._times = {}
795 finally:
796 f.close()
797 return self._times
798
799 def Save(self):
800 if self._times is None:
801 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700802
803 to_delete = []
804 for name in self._times:
805 if name not in self._seen:
806 to_delete.append(name)
807 for name in to_delete:
808 del self._times[name]
809
Dave Borowitz67700e92012-10-23 15:00:54 -0700810 try:
811 f = open(self._path, 'wb')
812 try:
813 pickle.dump(self._times, f)
814 except (IOError, OSError, pickle.PickleError):
815 try:
816 os.remove(self._path)
817 except OSError:
818 pass
819 finally:
820 f.close()