blob: e9d52b7b6562e34fdcdebba496be829a0ff0fe8c [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
David Pursehouse86d973d2012-08-24 10:21:02 +090017import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070018from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Dave Borowitz67700e92012-10-23 15:00:54 -070020import pickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070026import time
David Pursehouse59bbb582013-05-17 10:49:33 +090027
28from pyversion import is_python3
29if is_python3():
Chirayu Desai217ea7d2013-03-01 19:14:38 +053030 import urllib.parse
David Pursehouse59bbb582013-05-17 10:49:33 +090031 import xmlrpc.client
32else:
Chirayu Desai217ea7d2013-03-01 19:14:38 +053033 import imp
34 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090035 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053036 urllib = imp.new_module('urllib')
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053037 urllib.parse = urlparse
Chirayu Desai217ea7d2013-03-01 19:14:38 +053038 xmlrpc = imp.new_module('xmlrpc')
39 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
Roy Lee18afd7f2010-05-09 04:32:08 +080041try:
42 import threading as _threading
43except ImportError:
44 import dummy_threading as _threading
45
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070046try:
47 import resource
48 def _rlimit_nofile():
49 return resource.getrlimit(resource.RLIMIT_NOFILE)
50except ImportError:
51 def _rlimit_nofile():
52 return (256, 256)
53
Dave Borowitz18857212012-10-23 17:02:59 -070054try:
55 import multiprocessing
56except ImportError:
57 multiprocessing = None
58
Dave Borowitze2152672012-10-31 12:24:38 -070059from git_command import GIT, git_require
David Pursehoused94aaef2012-09-07 09:52:04 +090060from git_refs import R_HEADS, HEAD
Conley Owensc9129d92012-10-01 16:12:28 -070061from main import WrapperModule
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070062from project import Project
63from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080064from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000065from error import RepoChangedException, GitError, ManifestParseError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070066from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070067from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070068
Dave Borowitz67700e92012-10-23 15:00:54 -070069_ONE_DAY_S = 24 * 60 * 60
70
Doug Andersonfc06ced2011-03-16 15:49:18 -070071class _FetchError(Exception):
72 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
73 pass
74
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080075class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080076 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070077 common = True
78 helpSummary = "Update working tree to the latest revision"
79 helpUsage = """
80%prog [<project>...]
81"""
82 helpDescription = """
83The '%prog' command synchronizes local project directories
84with the remote repositories specified in the manifest. If a local
85project does not yet exist, it will clone a new local directory from
86the remote repository and set up tracking branches as specified in
87the manifest. If the local project already exists, '%prog'
88will update the remote branches and rebase any new local changes
89on top of the new remote changes.
90
91'%prog' will synchronize all projects listed at the command
92line. Projects can be specified either by name, or by a relative
93or absolute path to the project's local directory. If no projects
94are specified, '%prog' will synchronize all projects listed in
95the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070096
97The -d/--detach option can be used to switch specified projects
98back to the manifest revision. This option is especially helpful
99if the project is currently on a topic branch, but the manifest
100revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700101
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700102The -s/--smart-sync option can be used to sync to a known good
103build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200104manifest. The -t/--smart-tag option is similar and allows you to
105specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700106
David Pursehousecf76b1b2012-09-14 10:31:42 +0900107The -u/--manifest-server-username and -p/--manifest-server-password
108options can be used to specify a username and password to authenticate
109with the manifest server when using the -s or -t option.
110
111If -u and -p are not specified when using the -s or -t option, '%prog'
112will attempt to read authentication credentials for the manifest server
113from the user's .netrc file.
114
115'%prog' will not use authentication credentials from -u/-p or .netrc
116if the manifest server specified in the manifest file already includes
117credentials.
118
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500119The -f/--force-broken option can be used to proceed with syncing
120other projects if a project sync fails.
121
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700122The --no-clone-bundle option disables any attempt to use
123$URL/clone.bundle to bootstrap a new Git repository from a
124resumeable bundle file on a content delivery network. This
125may be necessary if there are problems with the local Python
126HTTP client or proxy configuration, but the Git binary works.
127
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800128The --fetch-submodules option enables fetching Git submodules
129of a project from server.
130
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700131SSH Connections
132---------------
133
134If at least one project remote URL uses an SSH connection (ssh://,
135git+ssh://, or user@host:path syntax) repo will automatically
136enable the SSH ControlMaster option when connecting to that host.
137This feature permits other projects in the same '%prog' session to
138reuse the same SSH tunnel, saving connection setup overheads.
139
140To disable this behavior on UNIX platforms, set the GIT_SSH
141environment variable to 'ssh'. For example:
142
143 export GIT_SSH=ssh
144 %prog
145
146Compatibility
147~~~~~~~~~~~~~
148
149This feature is automatically disabled on Windows, due to the lack
150of UNIX domain socket support.
151
152This feature is not compatible with url.insteadof rewrites in the
153user's ~/.gitconfig. '%prog' is currently not able to perform the
154rewrite early enough to establish the ControlMaster tunnel.
155
156If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
157later is required to fix a server side protocol bug.
158
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700159"""
160
Nico Sallembien6623b212010-05-11 12:57:01 -0700161 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000162 try:
163 self.jobs = self.manifest.default.sync_j
164 except ManifestParseError:
165 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700166
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500167 p.add_option('-f', '--force-broken',
168 dest='force_broken', action='store_true',
169 help="continue sync even if a project fails to sync")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900170 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700171 dest='local_only', action='store_true',
172 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900173 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700174 dest='network_only', action='store_true',
175 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900176 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700177 dest='detach_head', action='store_true',
178 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900179 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700180 dest='current_branch_only', action='store_true',
181 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900182 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700183 dest='quiet', action='store_true',
184 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900185 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800186 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700187 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500188 p.add_option('-m', '--manifest-name',
189 dest='manifest_name',
190 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700191 p.add_option('--no-clone-bundle',
192 dest='no_clone_bundle', action='store_true',
193 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800194 p.add_option('-u', '--manifest-server-username', action='store',
195 dest='manifest_server_username',
196 help='username to authenticate with the manifest server')
197 p.add_option('-p', '--manifest-server-password', action='store',
198 dest='manifest_server_password',
199 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800200 p.add_option('--fetch-submodules',
201 dest='fetch_submodules', action='store_true',
202 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700203 p.add_option('--no-tags',
204 dest='no_tags', action='store_true',
205 help="don't fetch tags")
Nico Sallembien6623b212010-05-11 12:57:01 -0700206 if show_smart:
207 p.add_option('-s', '--smart-sync',
208 dest='smart_sync', action='store_true',
209 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200210 p.add_option('-t', '--smart-tag',
211 dest='smart_tag', action='store',
212 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700213
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700214 g = p.add_option_group('repo Version options')
215 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700216 dest='no_repo_verify', action='store_true',
217 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700218 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800219 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700220 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700221
Doug Andersonfc06ced2011-03-16 15:49:18 -0700222 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
David Pursehousec1b86a22012-11-14 11:36:51 +0900223 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800224
David Pursehousec1b86a22012-11-14 11:36:51 +0900225 Args:
226 opt: Program options returned from optparse. See _Options().
227 project: Project object for the project to fetch.
228 lock: Lock for accessing objects that are shared amongst multiple
229 _FetchHelper() threads.
230 fetched: set object that we will add project.gitdir to when we're done
231 (with our lock held).
232 pm: Instance of a Project object. We will call pm.update() (with our
233 lock held).
234 sem: We'll release() this semaphore when we exit so that another thread
235 can be started up.
236 err_event: We'll set this event in the case of an error (after printing
237 out info about the error).
238 """
239 # We'll set to true once we've locked the lock.
240 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700241
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530242 if not opt.quiet:
243 print('Fetching project %s' % project.name)
244
David Pursehousec1b86a22012-11-14 11:36:51 +0900245 # Encapsulate everything in a try/except/finally so that:
246 # - We always set err_event in the case of an exception.
247 # - We always make sure we call sem.release().
248 # - We always make sure we unlock the lock if we locked it.
249 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700250 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900251 start = time.time()
252 success = project.Sync_NetworkHalf(
253 quiet=opt.quiet,
254 current_branch_only=opt.current_branch_only,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700255 clone_bundle=not opt.no_clone_bundle,
256 no_tags=opt.no_tags)
David Pursehousec1b86a22012-11-14 11:36:51 +0900257 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700258
David Pursehousec1b86a22012-11-14 11:36:51 +0900259 # Lock around all the rest of the code, since printing, updating a set
260 # and Progress.update() are not thread safe.
261 lock.acquire()
262 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700263
David Pursehousec1b86a22012-11-14 11:36:51 +0900264 if not success:
265 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
266 if opt.force_broken:
267 print('warn: --force-broken, continuing to sync',
268 file=sys.stderr)
269 else:
270 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700271
David Pursehousec1b86a22012-11-14 11:36:51 +0900272 fetched.add(project.gitdir)
273 pm.update()
274 except _FetchError:
275 err_event.set()
276 except:
277 err_event.set()
278 raise
279 finally:
280 if did_lock:
281 lock.release()
282 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800283
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700284 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700285 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700286 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800287
288 if self.jobs == 1:
289 for project in projects:
290 pm.update()
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530291 if not opt.quiet:
292 print('Fetching project %s' % project.name)
Shawn O. Pearce5d0efdb2012-08-02 12:13:01 -0700293 if project.Sync_NetworkHalf(
294 quiet=opt.quiet,
295 current_branch_only=opt.current_branch_only,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700296 clone_bundle=not opt.no_clone_bundle,
297 no_tags=opt.no_tags):
Roy Lee18afd7f2010-05-09 04:32:08 +0800298 fetched.add(project.gitdir)
299 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700300 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500301 if opt.force_broken:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700302 print('warn: --force-broken, continuing to sync', file=sys.stderr)
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500303 else:
304 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800305 else:
306 threads = set()
307 lock = _threading.Lock()
308 sem = _threading.Semaphore(self.jobs)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700309 err_event = _threading.Event()
Roy Lee18afd7f2010-05-09 04:32:08 +0800310 for project in projects:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700311 # Check for any errors before starting any new threads.
312 # ...we'll let existing threads finish, though.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400313 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700314 break
315
Roy Lee18afd7f2010-05-09 04:32:08 +0800316 sem.acquire()
317 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700318 args = (opt,
319 project,
320 lock,
321 fetched,
322 pm,
Doug Andersonfc06ced2011-03-16 15:49:18 -0700323 sem,
324 err_event))
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200325 # Ensure that Ctrl-C will not freeze the repo process.
326 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800327 threads.add(t)
328 t.start()
329
330 for t in threads:
331 t.join()
332
Doug Andersonfc06ced2011-03-16 15:49:18 -0700333 # If we saw an error, exit with code 1 so that other scripts can check.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400334 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700335 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700336 sys.exit(1)
337
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700338 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700339 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700340
341 self._GCProjects(projects)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700342 return fetched
343
Dave Borowitz18857212012-10-23 17:02:59 -0700344 def _GCProjects(self, projects):
Dave Borowitze2152672012-10-31 12:24:38 -0700345 has_dash_c = git_require((1, 7, 2))
346 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700347 cpu_count = multiprocessing.cpu_count()
348 else:
349 cpu_count = 1
350 jobs = min(self.jobs, cpu_count)
351
352 if jobs < 2:
353 for project in projects:
354 project.bare_git.gc('--auto')
355 return
356
357 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
358
359 threads = set()
360 sem = _threading.Semaphore(jobs)
361 err_event = _threading.Event()
362
363 def GC(project):
364 try:
365 try:
366 project.bare_git.gc('--auto', config=config)
367 except GitError:
368 err_event.set()
369 except:
370 err_event.set()
371 raise
372 finally:
373 sem.release()
374
375 for project in projects:
376 if err_event.isSet():
377 break
378 sem.acquire()
379 t = _threading.Thread(target=GC, args=(project,))
380 t.daemon = True
381 threads.add(t)
382 t.start()
383
384 for t in threads:
385 t.join()
386
387 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700388 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700389 sys.exit(1)
390
Tim Kilbourn07669002013-03-08 15:02:49 -0800391 def _ReloadManifest(self, manifest_name=None):
392 if manifest_name:
393 # Override calls _Unload already
394 self.manifest.Override(manifest_name)
395 else:
396 self.manifest._Unload()
397
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700398 def UpdateProjectList(self):
399 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700400 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700401 if project.relpath:
402 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700403 file_name = 'project.list'
404 file_path = os.path.join(self.manifest.repodir, file_name)
405 old_project_paths = []
406
407 if os.path.exists(file_path):
408 fd = open(file_path, 'r')
409 try:
410 old_project_paths = fd.read().split('\n')
411 finally:
412 fd.close()
413 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700414 if not path:
415 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700416 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900417 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400418 if os.path.exists(self.manifest.topdir + '/' + path):
David Pursehousec1b86a22012-11-14 11:36:51 +0900419 project = Project(
420 manifest = self.manifest,
421 name = path,
422 remote = RemoteSpec('origin'),
423 gitdir = os.path.join(self.manifest.topdir,
424 path, '.git'),
425 worktree = os.path.join(self.manifest.topdir, path),
426 relpath = path,
427 revisionExpr = 'HEAD',
428 revisionId = None,
429 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400430
David Pursehousec1b86a22012-11-14 11:36:51 +0900431 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900432 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900433 'are present' % project.relpath, file=sys.stderr)
434 print(' commit changes, then run sync again',
435 file=sys.stderr)
436 return -1
437 else:
438 print('Deleting obsolete path %s' % project.worktree,
439 file=sys.stderr)
440 shutil.rmtree(project.worktree)
441 # Try deleting parent subdirs if they are empty
442 project_dir = os.path.dirname(project.worktree)
443 while project_dir != self.manifest.topdir:
444 try:
445 os.rmdir(project_dir)
446 except OSError:
447 break
448 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700449
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700450 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700451 fd = open(file_path, 'w')
452 try:
453 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700454 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700455 finally:
456 fd.close()
457 return 0
458
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700459 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800460 if opt.jobs:
461 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700462 if self.jobs > 1:
463 soft_limit, _ = _rlimit_nofile()
464 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
465
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700466 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700467 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700468 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700469 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700470 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700471 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500472 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700473 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500474 sys.exit(1)
475 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700476 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500477 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900478 if opt.manifest_server_username or opt.manifest_server_password:
479 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700480 print('error: -u and -p may only be combined with -s or -t',
481 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900482 sys.exit(1)
483 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700484 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900485 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500486
487 if opt.manifest_name:
488 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700489
Chirayu Desaia892b102013-06-11 14:18:46 +0530490 manifest_name = opt.manifest_name
491
Victor Boivie08c880d2011-04-19 10:32:52 +0200492 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700493 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900494 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700495 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700496 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900497
498 manifest_server = self.manifest.manifest_server
David Pursehousefb99c712013-09-25 11:09:34 +0900499 if not opt.quiet:
500 print('Using manifest server %s' % manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900501
David Pursehouse86d973d2012-08-24 10:21:02 +0900502 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900503 username = None
504 password = None
505 if opt.manifest_server_username and opt.manifest_server_password:
506 username = opt.manifest_server_username
507 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900508 else:
509 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900510 info = netrc.netrc()
511 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700512 print('.netrc file does not exist or could not be opened',
513 file=sys.stderr)
David Pursehouse86d973d2012-08-24 10:21:02 +0900514 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900515 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530516 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900517 if parse_result.hostname:
518 username, _account, password = \
519 info.authenticators(parse_result.hostname)
520 except TypeError:
521 # TypeError is raised when the given hostname is not present
522 # in the .netrc file.
Sarah Owenscecd1d82012-11-01 22:59:27 -0700523 print('No credentials found for %s in .netrc'
524 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700525 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700526 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900527
528 if (username and password):
529 manifest_server = manifest_server.replace('://', '://%s:%s@' %
530 (username, password),
531 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900532
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700533 try:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530534 server = xmlrpc.client.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200535 if opt.smart_sync:
536 p = self.manifest.manifestProject
537 b = p.GetBranch(p.CurrentBranch)
538 branch = b.merge
539 if branch.startswith(R_HEADS):
540 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700541
Victor Boivie08c880d2011-04-19 10:32:52 +0200542 env = os.environ.copy()
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530543 if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200544 target = '%s-%s' % (env['TARGET_PRODUCT'],
545 env['TARGET_BUILD_VARIANT'])
546 [success, manifest_str] = server.GetApprovedManifest(branch, target)
547 else:
548 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700549 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200550 assert(opt.smart_tag)
551 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700552
553 if success:
554 manifest_name = "smart_sync_override.xml"
555 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
556 manifest_name)
557 try:
558 f = open(manifest_path, 'w')
559 try:
560 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700561 finally:
562 f.close()
563 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700564 print('error: cannot write manifest to %s' % manifest_path,
565 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700566 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100567 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700568 else:
David Pursehouse351fe2c2013-09-25 17:54:26 +0900569 print('error: manifest server RPC call failed: %s' %
570 manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700571 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530572 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700573 print('error: cannot connect to manifest server %s:\n%s'
574 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900575 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530576 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700577 print('error: cannot connect to manifest server %s:\n%d %s'
578 % (self.manifest.manifest_server, e.errcode, e.errmsg),
579 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700580 sys.exit(1)
581
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700582 rp = self.manifest.repoProject
583 rp.PreSync()
584
585 mp = self.manifest.manifestProject
586 mp.PreSync()
587
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800588 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700589 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800590
Nico Sallembien9bb18162009-12-07 15:38:01 -0800591 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700592 mp.Sync_NetworkHalf(quiet=opt.quiet,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700593 current_branch_only=opt.current_branch_only,
594 no_tags=opt.no_tags)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800595
596 if mp.HasChanges:
597 syncbuf = SyncBuffer(mp.config)
598 mp.Sync_LocalHalf(syncbuf)
599 if not syncbuf.Finish():
600 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100601 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700602 if opt.jobs is None:
603 self.jobs = self.manifest.default.sync_j
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800604 all_projects = self.GetProjects(args,
605 missing_ok=True,
606 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700607
Dave Borowitz67700e92012-10-23 15:00:54 -0700608 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700609 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700610 to_fetch = []
611 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700612 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700613 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900614 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700615 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700616
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800617 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700618 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700619 if opt.network_only:
620 # bail out now; the rest touches the working tree
621 return
622
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800623 # Iteratively fetch missing and/or nested unregistered submodules
624 previously_missing_set = set()
625 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100626 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800627 all_projects = self.GetProjects(args,
628 missing_ok=True,
629 submodules_ok=opt.fetch_submodules)
630 missing = []
631 for project in all_projects:
632 if project.gitdir not in fetched:
633 missing.append(project)
634 if not missing:
635 break
636 # Stop us from non-stopped fetching actually-missing repos: If set of
637 # missing repos has not been changed from last fetch, we break.
638 missing_set = set(p.name for p in missing)
639 if previously_missing_set == missing_set:
640 break
641 previously_missing_set = missing_set
642 fetched.update(self._Fetch(missing, opt))
643
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700644 if self.manifest.IsMirror:
645 # bail out now, we have no working tree
646 return
647
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700648 if self.UpdateProjectList():
649 sys.exit(1)
650
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700651 syncbuf = SyncBuffer(mp.config,
652 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900653 pm = Progress('Syncing work tree', len(all_projects))
654 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700655 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800656 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700657 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700658 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700659 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700660 if not syncbuf.Finish():
661 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700662
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700663 # If there's a notice that's supposed to print at the end of the sync, print
664 # it now...
665 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700666 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700667
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700668def _PostRepoUpgrade(manifest, quiet=False):
Conley Owensc9129d92012-10-01 16:12:28 -0700669 wrapper = WrapperModule()
670 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700671 wrapper.SetupGnuPG(quiet)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700672 for project in manifest.projects.values():
673 if project.Exists:
674 project.PostRepoUpgrade()
675
676def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
677 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700678 print('info: A new version of repo is available', file=sys.stderr)
679 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700680 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700681 syncbuf = SyncBuffer(rp.config)
682 rp.Sync_LocalHalf(syncbuf)
683 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700684 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700685 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700686 raise RepoChangedException(['--repo-upgraded'])
687 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700688 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700689 else:
690 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700691 print('repo version %s is current' % rp.work_git.describe(HEAD),
692 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700693
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700694def _VerifyTag(project):
695 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
696 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700697 print('warning: GnuPG was not available during last "repo init"\n'
698 'warning: Cannot automatically authenticate repo."""',
699 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700700 return True
701
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700702 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700703 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700704 except GitError:
705 cur = None
706
707 if not cur \
708 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700709 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700710 if rev.startswith(R_HEADS):
711 rev = rev[len(R_HEADS):]
712
Sarah Owenscecd1d82012-11-01 22:59:27 -0700713 print(file=sys.stderr)
714 print("warning: project '%s' branch '%s' is not signed"
715 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700716 return False
717
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800718 env = os.environ.copy()
719 env['GIT_DIR'] = project.gitdir.encode()
720 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700721
722 cmd = [GIT, 'tag', '-v', cur]
723 proc = subprocess.Popen(cmd,
724 stdout = subprocess.PIPE,
725 stderr = subprocess.PIPE,
726 env = env)
727 out = proc.stdout.read()
728 proc.stdout.close()
729
730 err = proc.stderr.read()
731 proc.stderr.close()
732
733 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700734 print(file=sys.stderr)
735 print(out, file=sys.stderr)
736 print(err, file=sys.stderr)
737 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700738 return False
739 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700740
741class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700742 _ALPHA = 0.5
743
Dave Borowitz67700e92012-10-23 15:00:54 -0700744 def __init__(self, manifest):
745 self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
746 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700747 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700748
749 def Get(self, project):
750 self._Load()
751 return self._times.get(project.name, _ONE_DAY_S)
752
753 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700754 self._Load()
755 name = project.name
756 old = self._times.get(name, t)
757 self._seen.add(name)
758 a = self._ALPHA
759 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700760
761 def _Load(self):
762 if self._times is None:
763 try:
764 f = open(self._path)
765 except IOError:
766 self._times = {}
767 return self._times
768 try:
769 try:
770 self._times = pickle.load(f)
David Pursehouse1d947b32012-10-25 12:23:11 +0900771 except IOError:
Dave Borowitz67700e92012-10-23 15:00:54 -0700772 try:
773 os.remove(self._path)
774 except OSError:
775 pass
776 self._times = {}
777 finally:
778 f.close()
779 return self._times
780
781 def Save(self):
782 if self._times is None:
783 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700784
785 to_delete = []
786 for name in self._times:
787 if name not in self._seen:
788 to_delete.append(name)
789 for name in to_delete:
790 del self._times[name]
791
Dave Borowitz67700e92012-10-23 15:00:54 -0700792 try:
793 f = open(self._path, 'wb')
794 try:
795 pickle.dump(self._times, f)
796 except (IOError, OSError, pickle.PickleError):
797 try:
798 os.remove(self._path)
799 except OSError:
800 pass
801 finally:
802 f.close()