blob: b1945d5eeeadaf794d8cd5d19fdc17260a845431 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Anthony King85b24ac2014-05-06 15:57:48 +010017import json
David Pursehouse86d973d2012-08-24 10:21:02 +090018import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070019from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import os
21import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070026import time
David Pursehouse59bbb582013-05-17 10:49:33 +090027
28from pyversion import is_python3
29if is_python3():
Chirayu Desai217ea7d2013-03-01 19:14:38 +053030 import urllib.parse
David Pursehouse59bbb582013-05-17 10:49:33 +090031 import xmlrpc.client
32else:
Chirayu Desai217ea7d2013-03-01 19:14:38 +053033 import imp
34 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090035 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053036 urllib = imp.new_module('urllib')
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053037 urllib.parse = urlparse
Chirayu Desai217ea7d2013-03-01 19:14:38 +053038 xmlrpc = imp.new_module('xmlrpc')
39 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
Roy Lee18afd7f2010-05-09 04:32:08 +080041try:
42 import threading as _threading
43except ImportError:
44 import dummy_threading as _threading
45
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070046try:
47 import resource
48 def _rlimit_nofile():
49 return resource.getrlimit(resource.RLIMIT_NOFILE)
50except ImportError:
51 def _rlimit_nofile():
52 return (256, 256)
53
Dave Borowitz18857212012-10-23 17:02:59 -070054try:
55 import multiprocessing
56except ImportError:
57 multiprocessing = None
58
Dave Borowitze2152672012-10-31 12:24:38 -070059from git_command import GIT, git_require
David Pursehoused94aaef2012-09-07 09:52:04 +090060from git_refs import R_HEADS, HEAD
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070061from project import Project
62from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080063from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000064from error import RepoChangedException, GitError, ManifestParseError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070065from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070066from progress import Progress
Conley Owens094cdbe2014-01-30 15:09:59 -080067from wrapper import Wrapper
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070068
Dave Borowitz67700e92012-10-23 15:00:54 -070069_ONE_DAY_S = 24 * 60 * 60
70
Doug Andersonfc06ced2011-03-16 15:49:18 -070071class _FetchError(Exception):
72 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
73 pass
74
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080075class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080076 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070077 common = True
78 helpSummary = "Update working tree to the latest revision"
79 helpUsage = """
80%prog [<project>...]
81"""
82 helpDescription = """
83The '%prog' command synchronizes local project directories
84with the remote repositories specified in the manifest. If a local
85project does not yet exist, it will clone a new local directory from
86the remote repository and set up tracking branches as specified in
87the manifest. If the local project already exists, '%prog'
88will update the remote branches and rebase any new local changes
89on top of the new remote changes.
90
91'%prog' will synchronize all projects listed at the command
92line. Projects can be specified either by name, or by a relative
93or absolute path to the project's local directory. If no projects
94are specified, '%prog' will synchronize all projects listed in
95the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070096
97The -d/--detach option can be used to switch specified projects
98back to the manifest revision. This option is especially helpful
99if the project is currently on a topic branch, but the manifest
100revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700101
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700102The -s/--smart-sync option can be used to sync to a known good
103build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200104manifest. The -t/--smart-tag option is similar and allows you to
105specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700106
David Pursehousecf76b1b2012-09-14 10:31:42 +0900107The -u/--manifest-server-username and -p/--manifest-server-password
108options can be used to specify a username and password to authenticate
109with the manifest server when using the -s or -t option.
110
111If -u and -p are not specified when using the -s or -t option, '%prog'
112will attempt to read authentication credentials for the manifest server
113from the user's .netrc file.
114
115'%prog' will not use authentication credentials from -u/-p or .netrc
116if the manifest server specified in the manifest file already includes
117credentials.
118
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500119The -f/--force-broken option can be used to proceed with syncing
120other projects if a project sync fails.
121
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700122The --no-clone-bundle option disables any attempt to use
123$URL/clone.bundle to bootstrap a new Git repository from a
124resumeable bundle file on a content delivery network. This
125may be necessary if there are problems with the local Python
126HTTP client or proxy configuration, but the Git binary works.
127
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800128The --fetch-submodules option enables fetching Git submodules
129of a project from server.
130
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700131SSH Connections
132---------------
133
134If at least one project remote URL uses an SSH connection (ssh://,
135git+ssh://, or user@host:path syntax) repo will automatically
136enable the SSH ControlMaster option when connecting to that host.
137This feature permits other projects in the same '%prog' session to
138reuse the same SSH tunnel, saving connection setup overheads.
139
140To disable this behavior on UNIX platforms, set the GIT_SSH
141environment variable to 'ssh'. For example:
142
143 export GIT_SSH=ssh
144 %prog
145
146Compatibility
147~~~~~~~~~~~~~
148
149This feature is automatically disabled on Windows, due to the lack
150of UNIX domain socket support.
151
152This feature is not compatible with url.insteadof rewrites in the
153user's ~/.gitconfig. '%prog' is currently not able to perform the
154rewrite early enough to establish the ControlMaster tunnel.
155
156If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
157later is required to fix a server side protocol bug.
158
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700159"""
160
Nico Sallembien6623b212010-05-11 12:57:01 -0700161 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000162 try:
163 self.jobs = self.manifest.default.sync_j
164 except ManifestParseError:
165 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700166
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500167 p.add_option('-f', '--force-broken',
168 dest='force_broken', action='store_true',
169 help="continue sync even if a project fails to sync")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900170 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700171 dest='local_only', action='store_true',
172 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900173 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700174 dest='network_only', action='store_true',
175 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900176 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700177 dest='detach_head', action='store_true',
178 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900179 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700180 dest='current_branch_only', action='store_true',
181 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900182 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700183 dest='quiet', action='store_true',
184 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900185 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800186 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700187 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500188 p.add_option('-m', '--manifest-name',
189 dest='manifest_name',
190 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700191 p.add_option('--no-clone-bundle',
192 dest='no_clone_bundle', action='store_true',
193 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800194 p.add_option('-u', '--manifest-server-username', action='store',
195 dest='manifest_server_username',
196 help='username to authenticate with the manifest server')
197 p.add_option('-p', '--manifest-server-password', action='store',
198 dest='manifest_server_password',
199 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800200 p.add_option('--fetch-submodules',
201 dest='fetch_submodules', action='store_true',
202 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700203 p.add_option('--no-tags',
204 dest='no_tags', action='store_true',
205 help="don't fetch tags")
Nico Sallembien6623b212010-05-11 12:57:01 -0700206 if show_smart:
207 p.add_option('-s', '--smart-sync',
208 dest='smart_sync', action='store_true',
209 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200210 p.add_option('-t', '--smart-tag',
211 dest='smart_tag', action='store',
212 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700213
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700214 g = p.add_option_group('repo Version options')
215 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700216 dest='no_repo_verify', action='store_true',
217 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700218 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800219 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700220 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700221
David James89ece422014-01-09 18:51:58 -0800222 def _FetchProjectList(self, opt, projects, *args, **kwargs):
David Pursehousec1b86a22012-11-14 11:36:51 +0900223 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800224
David James8d201162013-10-11 17:03:19 -0700225 Delegates most of the work to _FetchHelper.
226
227 Args:
228 opt: Program options returned from optparse. See _Options().
229 projects: Projects to fetch.
David James89ece422014-01-09 18:51:58 -0800230 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
David James8d201162013-10-11 17:03:19 -0700231 _FetchHelper docstring for details.
232 """
233 for project in projects:
David James89ece422014-01-09 18:51:58 -0800234 success = self._FetchHelper(opt, project, *args, **kwargs)
David James8d201162013-10-11 17:03:19 -0700235 if not success and not opt.force_broken:
236 break
237
238 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
239 """Fetch git objects for a single project.
240
David Pursehousec1b86a22012-11-14 11:36:51 +0900241 Args:
242 opt: Program options returned from optparse. See _Options().
243 project: Project object for the project to fetch.
244 lock: Lock for accessing objects that are shared amongst multiple
245 _FetchHelper() threads.
246 fetched: set object that we will add project.gitdir to when we're done
247 (with our lock held).
248 pm: Instance of a Project object. We will call pm.update() (with our
249 lock held).
250 sem: We'll release() this semaphore when we exit so that another thread
251 can be started up.
252 err_event: We'll set this event in the case of an error (after printing
253 out info about the error).
David James8d201162013-10-11 17:03:19 -0700254
255 Returns:
256 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900257 """
258 # We'll set to true once we've locked the lock.
259 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700260
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530261 if not opt.quiet:
262 print('Fetching project %s' % project.name)
263
David Pursehousec1b86a22012-11-14 11:36:51 +0900264 # Encapsulate everything in a try/except/finally so that:
265 # - We always set err_event in the case of an exception.
266 # - We always make sure we call sem.release().
267 # - We always make sure we unlock the lock if we locked it.
268 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700269 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900270 start = time.time()
271 success = project.Sync_NetworkHalf(
272 quiet=opt.quiet,
273 current_branch_only=opt.current_branch_only,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700274 clone_bundle=not opt.no_clone_bundle,
Julien Campergue335f5ef2013-10-16 11:02:35 +0200275 no_tags=opt.no_tags, archive=self.manifest.IsArchive)
David Pursehousec1b86a22012-11-14 11:36:51 +0900276 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700277
David Pursehousec1b86a22012-11-14 11:36:51 +0900278 # Lock around all the rest of the code, since printing, updating a set
279 # and Progress.update() are not thread safe.
280 lock.acquire()
281 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700282
David Pursehousec1b86a22012-11-14 11:36:51 +0900283 if not success:
284 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
285 if opt.force_broken:
286 print('warn: --force-broken, continuing to sync',
287 file=sys.stderr)
288 else:
289 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700290
David Pursehousec1b86a22012-11-14 11:36:51 +0900291 fetched.add(project.gitdir)
292 pm.update()
293 except _FetchError:
294 err_event.set()
295 except:
296 err_event.set()
297 raise
298 finally:
299 if did_lock:
300 lock.release()
301 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800302
David James8d201162013-10-11 17:03:19 -0700303 return success
304
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700305 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700306 fetched = set()
David James89ece422014-01-09 18:51:58 -0800307 lock = _threading.Lock()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700308 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800309
David James89ece422014-01-09 18:51:58 -0800310 objdir_project_map = dict()
311 for project in projects:
312 objdir_project_map.setdefault(project.objdir, []).append(project)
David James8d201162013-10-11 17:03:19 -0700313
David James89ece422014-01-09 18:51:58 -0800314 threads = set()
315 sem = _threading.Semaphore(self.jobs)
316 err_event = _threading.Event()
317 for project_list in objdir_project_map.values():
318 # Check for any errors before running any more tasks.
319 # ...we'll let existing threads finish, though.
320 if err_event.isSet() and not opt.force_broken:
321 break
Doug Andersonfc06ced2011-03-16 15:49:18 -0700322
David James89ece422014-01-09 18:51:58 -0800323 sem.acquire()
324 kwargs = dict(opt=opt,
325 projects=project_list,
326 lock=lock,
327 fetched=fetched,
328 pm=pm,
329 sem=sem,
330 err_event=err_event)
331 if self.jobs > 1:
David James8d201162013-10-11 17:03:19 -0700332 t = _threading.Thread(target = self._FetchProjectList,
David James89ece422014-01-09 18:51:58 -0800333 kwargs = kwargs)
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200334 # Ensure that Ctrl-C will not freeze the repo process.
335 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800336 threads.add(t)
337 t.start()
David James89ece422014-01-09 18:51:58 -0800338 else:
339 self._FetchProjectList(**kwargs)
Roy Lee18afd7f2010-05-09 04:32:08 +0800340
David James89ece422014-01-09 18:51:58 -0800341 for t in threads:
342 t.join()
Roy Lee18afd7f2010-05-09 04:32:08 +0800343
David James89ece422014-01-09 18:51:58 -0800344 # If we saw an error, exit with code 1 so that other scripts can check.
345 if err_event.isSet():
346 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
347 sys.exit(1)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700348
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700349 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700350 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700351
Julien Campergue335f5ef2013-10-16 11:02:35 +0200352 if not self.manifest.IsArchive:
353 self._GCProjects(projects)
354
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700355 return fetched
356
Dave Borowitz18857212012-10-23 17:02:59 -0700357 def _GCProjects(self, projects):
David James8d201162013-10-11 17:03:19 -0700358 gitdirs = {}
359 for project in projects:
360 gitdirs[project.gitdir] = project.bare_git
361
Dave Borowitze2152672012-10-31 12:24:38 -0700362 has_dash_c = git_require((1, 7, 2))
363 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700364 cpu_count = multiprocessing.cpu_count()
365 else:
366 cpu_count = 1
367 jobs = min(self.jobs, cpu_count)
368
369 if jobs < 2:
David James8d201162013-10-11 17:03:19 -0700370 for bare_git in gitdirs.values():
371 bare_git.gc('--auto')
Dave Borowitz18857212012-10-23 17:02:59 -0700372 return
373
374 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
375
376 threads = set()
377 sem = _threading.Semaphore(jobs)
378 err_event = _threading.Event()
379
David James8d201162013-10-11 17:03:19 -0700380 def GC(bare_git):
Dave Borowitz18857212012-10-23 17:02:59 -0700381 try:
382 try:
David James8d201162013-10-11 17:03:19 -0700383 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700384 except GitError:
385 err_event.set()
386 except:
387 err_event.set()
388 raise
389 finally:
390 sem.release()
391
David James8d201162013-10-11 17:03:19 -0700392 for bare_git in gitdirs.values():
Dave Borowitz18857212012-10-23 17:02:59 -0700393 if err_event.isSet():
394 break
395 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700396 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700397 t.daemon = True
398 threads.add(t)
399 t.start()
400
401 for t in threads:
402 t.join()
403
404 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700405 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700406 sys.exit(1)
407
Tim Kilbourn07669002013-03-08 15:02:49 -0800408 def _ReloadManifest(self, manifest_name=None):
409 if manifest_name:
410 # Override calls _Unload already
411 self.manifest.Override(manifest_name)
412 else:
413 self.manifest._Unload()
414
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700415 def UpdateProjectList(self):
416 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700417 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700418 if project.relpath:
419 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700420 file_name = 'project.list'
421 file_path = os.path.join(self.manifest.repodir, file_name)
422 old_project_paths = []
423
424 if os.path.exists(file_path):
425 fd = open(file_path, 'r')
426 try:
427 old_project_paths = fd.read().split('\n')
428 finally:
429 fd.close()
430 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700431 if not path:
432 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700433 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900434 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400435 if os.path.exists(self.manifest.topdir + '/' + path):
David James8d201162013-10-11 17:03:19 -0700436 gitdir = os.path.join(self.manifest.topdir, path, '.git')
David Pursehousec1b86a22012-11-14 11:36:51 +0900437 project = Project(
438 manifest = self.manifest,
439 name = path,
440 remote = RemoteSpec('origin'),
David James8d201162013-10-11 17:03:19 -0700441 gitdir = gitdir,
442 objdir = gitdir,
David Pursehousec1b86a22012-11-14 11:36:51 +0900443 worktree = os.path.join(self.manifest.topdir, path),
444 relpath = path,
445 revisionExpr = 'HEAD',
446 revisionId = None,
447 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400448
David Pursehousec1b86a22012-11-14 11:36:51 +0900449 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900450 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900451 'are present' % project.relpath, file=sys.stderr)
452 print(' commit changes, then run sync again',
453 file=sys.stderr)
454 return -1
455 else:
456 print('Deleting obsolete path %s' % project.worktree,
457 file=sys.stderr)
458 shutil.rmtree(project.worktree)
459 # Try deleting parent subdirs if they are empty
460 project_dir = os.path.dirname(project.worktree)
461 while project_dir != self.manifest.topdir:
462 try:
463 os.rmdir(project_dir)
464 except OSError:
465 break
466 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700467
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700468 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700469 fd = open(file_path, 'w')
470 try:
471 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700472 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700473 finally:
474 fd.close()
475 return 0
476
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700477 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800478 if opt.jobs:
479 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700480 if self.jobs > 1:
481 soft_limit, _ = _rlimit_nofile()
482 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
483
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700484 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700485 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700486 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700487 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700488 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700489 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500490 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700491 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500492 sys.exit(1)
493 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700494 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500495 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900496 if opt.manifest_server_username or opt.manifest_server_password:
497 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700498 print('error: -u and -p may only be combined with -s or -t',
499 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900500 sys.exit(1)
501 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700502 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900503 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500504
505 if opt.manifest_name:
506 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700507
Chirayu Desaia892b102013-06-11 14:18:46 +0530508 manifest_name = opt.manifest_name
509
Victor Boivie08c880d2011-04-19 10:32:52 +0200510 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700511 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900512 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700513 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700514 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900515
516 manifest_server = self.manifest.manifest_server
David Pursehousefb99c712013-09-25 11:09:34 +0900517 if not opt.quiet:
518 print('Using manifest server %s' % manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900519
David Pursehouse86d973d2012-08-24 10:21:02 +0900520 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900521 username = None
522 password = None
523 if opt.manifest_server_username and opt.manifest_server_password:
524 username = opt.manifest_server_username
525 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900526 else:
527 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900528 info = netrc.netrc()
529 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700530 print('.netrc file does not exist or could not be opened',
531 file=sys.stderr)
David Pursehouse86d973d2012-08-24 10:21:02 +0900532 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900533 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530534 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900535 if parse_result.hostname:
536 username, _account, password = \
537 info.authenticators(parse_result.hostname)
538 except TypeError:
539 # TypeError is raised when the given hostname is not present
540 # in the .netrc file.
Sarah Owenscecd1d82012-11-01 22:59:27 -0700541 print('No credentials found for %s in .netrc'
542 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700543 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700544 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900545
546 if (username and password):
547 manifest_server = manifest_server.replace('://', '://%s:%s@' %
548 (username, password),
549 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900550
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700551 try:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530552 server = xmlrpc.client.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200553 if opt.smart_sync:
554 p = self.manifest.manifestProject
555 b = p.GetBranch(p.CurrentBranch)
556 branch = b.merge
557 if branch.startswith(R_HEADS):
558 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700559
Victor Boivie08c880d2011-04-19 10:32:52 +0200560 env = os.environ.copy()
Jeff Davidson5cf16602014-10-02 10:13:38 -0700561 if 'SYNC_TARGET' in env:
562 target = env['SYNC_TARGET']
563 [success, manifest_str] = server.GetApprovedManifest(branch, target)
564 elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200565 target = '%s-%s' % (env['TARGET_PRODUCT'],
566 env['TARGET_BUILD_VARIANT'])
567 [success, manifest_str] = server.GetApprovedManifest(branch, target)
568 else:
569 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700570 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200571 assert(opt.smart_tag)
572 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700573
574 if success:
575 manifest_name = "smart_sync_override.xml"
576 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
577 manifest_name)
578 try:
579 f = open(manifest_path, 'w')
580 try:
581 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700582 finally:
583 f.close()
584 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700585 print('error: cannot write manifest to %s' % manifest_path,
586 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700587 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100588 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700589 else:
David Pursehouse351fe2c2013-09-25 17:54:26 +0900590 print('error: manifest server RPC call failed: %s' %
591 manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700592 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530593 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700594 print('error: cannot connect to manifest server %s:\n%s'
595 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900596 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530597 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700598 print('error: cannot connect to manifest server %s:\n%d %s'
599 % (self.manifest.manifest_server, e.errcode, e.errmsg),
600 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700601 sys.exit(1)
602
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700603 rp = self.manifest.repoProject
604 rp.PreSync()
605
606 mp = self.manifest.manifestProject
607 mp.PreSync()
608
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800609 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700610 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800611
Nico Sallembien9bb18162009-12-07 15:38:01 -0800612 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700613 mp.Sync_NetworkHalf(quiet=opt.quiet,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700614 current_branch_only=opt.current_branch_only,
615 no_tags=opt.no_tags)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800616
617 if mp.HasChanges:
618 syncbuf = SyncBuffer(mp.config)
619 mp.Sync_LocalHalf(syncbuf)
620 if not syncbuf.Finish():
621 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100622 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700623 if opt.jobs is None:
624 self.jobs = self.manifest.default.sync_j
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800625 all_projects = self.GetProjects(args,
626 missing_ok=True,
627 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700628
Dave Borowitz67700e92012-10-23 15:00:54 -0700629 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700630 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700631 to_fetch = []
632 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700633 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700634 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900635 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700636 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700637
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800638 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700639 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700640 if opt.network_only:
641 # bail out now; the rest touches the working tree
642 return
643
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800644 # Iteratively fetch missing and/or nested unregistered submodules
645 previously_missing_set = set()
646 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100647 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800648 all_projects = self.GetProjects(args,
649 missing_ok=True,
650 submodules_ok=opt.fetch_submodules)
651 missing = []
652 for project in all_projects:
653 if project.gitdir not in fetched:
654 missing.append(project)
655 if not missing:
656 break
657 # Stop us from non-stopped fetching actually-missing repos: If set of
658 # missing repos has not been changed from last fetch, we break.
659 missing_set = set(p.name for p in missing)
660 if previously_missing_set == missing_set:
661 break
662 previously_missing_set = missing_set
663 fetched.update(self._Fetch(missing, opt))
664
Julien Campergue335f5ef2013-10-16 11:02:35 +0200665 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700666 # bail out now, we have no working tree
667 return
668
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700669 if self.UpdateProjectList():
670 sys.exit(1)
671
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700672 syncbuf = SyncBuffer(mp.config,
673 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900674 pm = Progress('Syncing work tree', len(all_projects))
675 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700676 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800677 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700678 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700679 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700680 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700681 if not syncbuf.Finish():
682 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700683
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700684 # If there's a notice that's supposed to print at the end of the sync, print
685 # it now...
686 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700687 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700688
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700689def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -0800690 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -0700691 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700692 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -0800693 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700694 if project.Exists:
695 project.PostRepoUpgrade()
696
697def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
698 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700699 print('info: A new version of repo is available', file=sys.stderr)
700 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700701 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700702 syncbuf = SyncBuffer(rp.config)
703 rp.Sync_LocalHalf(syncbuf)
704 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700705 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700706 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700707 raise RepoChangedException(['--repo-upgraded'])
708 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700709 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700710 else:
711 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700712 print('repo version %s is current' % rp.work_git.describe(HEAD),
713 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700714
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700715def _VerifyTag(project):
716 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
717 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700718 print('warning: GnuPG was not available during last "repo init"\n'
719 'warning: Cannot automatically authenticate repo."""',
720 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700721 return True
722
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700723 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700724 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700725 except GitError:
726 cur = None
727
728 if not cur \
729 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700730 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700731 if rev.startswith(R_HEADS):
732 rev = rev[len(R_HEADS):]
733
Sarah Owenscecd1d82012-11-01 22:59:27 -0700734 print(file=sys.stderr)
735 print("warning: project '%s' branch '%s' is not signed"
736 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700737 return False
738
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800739 env = os.environ.copy()
740 env['GIT_DIR'] = project.gitdir.encode()
741 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700742
743 cmd = [GIT, 'tag', '-v', cur]
744 proc = subprocess.Popen(cmd,
745 stdout = subprocess.PIPE,
746 stderr = subprocess.PIPE,
747 env = env)
748 out = proc.stdout.read()
749 proc.stdout.close()
750
751 err = proc.stderr.read()
752 proc.stderr.close()
753
754 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700755 print(file=sys.stderr)
756 print(out, file=sys.stderr)
757 print(err, file=sys.stderr)
758 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700759 return False
760 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700761
762class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700763 _ALPHA = 0.5
764
Dave Borowitz67700e92012-10-23 15:00:54 -0700765 def __init__(self, manifest):
Anthony King85b24ac2014-05-06 15:57:48 +0100766 self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
Dave Borowitz67700e92012-10-23 15:00:54 -0700767 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700768 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700769
770 def Get(self, project):
771 self._Load()
772 return self._times.get(project.name, _ONE_DAY_S)
773
774 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700775 self._Load()
776 name = project.name
777 old = self._times.get(name, t)
778 self._seen.add(name)
779 a = self._ALPHA
780 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700781
782 def _Load(self):
783 if self._times is None:
784 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100785 f = open(self._path)
Dave Borowitz67700e92012-10-23 15:00:54 -0700786 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100787 self._times = json.load(f)
788 finally:
789 f.close()
790 except (IOError, ValueError):
791 try:
792 os.remove(self._path)
793 except OSError:
794 pass
795 self._times = {}
Dave Borowitz67700e92012-10-23 15:00:54 -0700796
797 def Save(self):
798 if self._times is None:
799 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700800
801 to_delete = []
802 for name in self._times:
803 if name not in self._seen:
804 to_delete.append(name)
805 for name in to_delete:
806 del self._times[name]
807
Dave Borowitz67700e92012-10-23 15:00:54 -0700808 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100809 f = open(self._path, 'w')
Dave Borowitz67700e92012-10-23 15:00:54 -0700810 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100811 json.dump(self._times, f, indent=2)
812 finally:
813 f.close()
814 except (IOError, TypeError):
815 try:
816 os.remove(self._path)
817 except OSError:
818 pass