blob: 002d05f0cc92770569a00a4d8d135df40ed953a5 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
David Pursehouse86d973d2012-08-24 10:21:02 +090017import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070018from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Dave Borowitz67700e92012-10-23 15:00:54 -070020import pickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070026import time
David Pursehouse59bbb582013-05-17 10:49:33 +090027
28from pyversion import is_python3
29if is_python3():
Chirayu Desai217ea7d2013-03-01 19:14:38 +053030 import urllib.parse
David Pursehouse59bbb582013-05-17 10:49:33 +090031 import xmlrpc.client
32else:
Chirayu Desai217ea7d2013-03-01 19:14:38 +053033 import imp
34 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090035 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053036 urllib = imp.new_module('urllib')
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053037 urllib.parse = urlparse
Chirayu Desai217ea7d2013-03-01 19:14:38 +053038 xmlrpc = imp.new_module('xmlrpc')
39 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
Roy Lee18afd7f2010-05-09 04:32:08 +080041try:
42 import threading as _threading
43except ImportError:
44 import dummy_threading as _threading
45
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070046try:
47 import resource
48 def _rlimit_nofile():
49 return resource.getrlimit(resource.RLIMIT_NOFILE)
50except ImportError:
51 def _rlimit_nofile():
52 return (256, 256)
53
Dave Borowitz18857212012-10-23 17:02:59 -070054try:
55 import multiprocessing
56except ImportError:
57 multiprocessing = None
58
Dave Borowitze2152672012-10-31 12:24:38 -070059from git_command import GIT, git_require
David Pursehoused94aaef2012-09-07 09:52:04 +090060from git_refs import R_HEADS, HEAD
Conley Owensc9129d92012-10-01 16:12:28 -070061from main import WrapperModule
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070062from project import Project
63from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080064from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000065from error import RepoChangedException, GitError, ManifestParseError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070066from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070067from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070068
Dave Borowitz67700e92012-10-23 15:00:54 -070069_ONE_DAY_S = 24 * 60 * 60
70
Doug Andersonfc06ced2011-03-16 15:49:18 -070071class _FetchError(Exception):
72 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
73 pass
74
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080075class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080076 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070077 common = True
78 helpSummary = "Update working tree to the latest revision"
79 helpUsage = """
80%prog [<project>...]
81"""
82 helpDescription = """
83The '%prog' command synchronizes local project directories
84with the remote repositories specified in the manifest. If a local
85project does not yet exist, it will clone a new local directory from
86the remote repository and set up tracking branches as specified in
87the manifest. If the local project already exists, '%prog'
88will update the remote branches and rebase any new local changes
89on top of the new remote changes.
90
91'%prog' will synchronize all projects listed at the command
92line. Projects can be specified either by name, or by a relative
93or absolute path to the project's local directory. If no projects
94are specified, '%prog' will synchronize all projects listed in
95the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070096
97The -d/--detach option can be used to switch specified projects
98back to the manifest revision. This option is especially helpful
99if the project is currently on a topic branch, but the manifest
100revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700101
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700102The -s/--smart-sync option can be used to sync to a known good
103build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200104manifest. The -t/--smart-tag option is similar and allows you to
105specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700106
David Pursehousecf76b1b2012-09-14 10:31:42 +0900107The -u/--manifest-server-username and -p/--manifest-server-password
108options can be used to specify a username and password to authenticate
109with the manifest server when using the -s or -t option.
110
111If -u and -p are not specified when using the -s or -t option, '%prog'
112will attempt to read authentication credentials for the manifest server
113from the user's .netrc file.
114
115'%prog' will not use authentication credentials from -u/-p or .netrc
116if the manifest server specified in the manifest file already includes
117credentials.
118
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500119The -f/--force-broken option can be used to proceed with syncing
120other projects if a project sync fails.
121
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700122The --no-clone-bundle option disables any attempt to use
123$URL/clone.bundle to bootstrap a new Git repository from a
124resumeable bundle file on a content delivery network. This
125may be necessary if there are problems with the local Python
126HTTP client or proxy configuration, but the Git binary works.
127
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800128The --fetch-submodules option enables fetching Git submodules
129of a project from server.
130
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700131SSH Connections
132---------------
133
134If at least one project remote URL uses an SSH connection (ssh://,
135git+ssh://, or user@host:path syntax) repo will automatically
136enable the SSH ControlMaster option when connecting to that host.
137This feature permits other projects in the same '%prog' session to
138reuse the same SSH tunnel, saving connection setup overheads.
139
140To disable this behavior on UNIX platforms, set the GIT_SSH
141environment variable to 'ssh'. For example:
142
143 export GIT_SSH=ssh
144 %prog
145
146Compatibility
147~~~~~~~~~~~~~
148
149This feature is automatically disabled on Windows, due to the lack
150of UNIX domain socket support.
151
152This feature is not compatible with url.insteadof rewrites in the
153user's ~/.gitconfig. '%prog' is currently not able to perform the
154rewrite early enough to establish the ControlMaster tunnel.
155
156If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
157later is required to fix a server side protocol bug.
158
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700159"""
160
Nico Sallembien6623b212010-05-11 12:57:01 -0700161 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000162 try:
163 self.jobs = self.manifest.default.sync_j
164 except ManifestParseError:
165 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700166
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500167 p.add_option('-f', '--force-broken',
168 dest='force_broken', action='store_true',
169 help="continue sync even if a project fails to sync")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900170 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700171 dest='local_only', action='store_true',
172 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900173 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700174 dest='network_only', action='store_true',
175 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900176 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700177 dest='detach_head', action='store_true',
178 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900179 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700180 dest='current_branch_only', action='store_true',
181 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900182 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700183 dest='quiet', action='store_true',
184 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900185 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800186 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700187 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500188 p.add_option('-m', '--manifest-name',
189 dest='manifest_name',
190 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700191 p.add_option('--no-clone-bundle',
192 dest='no_clone_bundle', action='store_true',
193 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800194 p.add_option('-u', '--manifest-server-username', action='store',
195 dest='manifest_server_username',
196 help='username to authenticate with the manifest server')
197 p.add_option('-p', '--manifest-server-password', action='store',
198 dest='manifest_server_password',
199 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800200 p.add_option('--fetch-submodules',
201 dest='fetch_submodules', action='store_true',
202 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700203 p.add_option('--no-tags',
204 dest='no_tags', action='store_true',
205 help="don't fetch tags")
Nico Sallembien6623b212010-05-11 12:57:01 -0700206 if show_smart:
207 p.add_option('-s', '--smart-sync',
208 dest='smart_sync', action='store_true',
209 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200210 p.add_option('-t', '--smart-tag',
211 dest='smart_tag', action='store',
212 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700213
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700214 g = p.add_option_group('repo Version options')
215 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700216 dest='no_repo_verify', action='store_true',
217 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700218 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800219 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700220 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700221
Doug Andersonfc06ced2011-03-16 15:49:18 -0700222 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
David Pursehousec1b86a22012-11-14 11:36:51 +0900223 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800224
David Pursehousec1b86a22012-11-14 11:36:51 +0900225 Args:
226 opt: Program options returned from optparse. See _Options().
227 project: Project object for the project to fetch.
228 lock: Lock for accessing objects that are shared amongst multiple
229 _FetchHelper() threads.
230 fetched: set object that we will add project.gitdir to when we're done
231 (with our lock held).
232 pm: Instance of a Project object. We will call pm.update() (with our
233 lock held).
234 sem: We'll release() this semaphore when we exit so that another thread
235 can be started up.
236 err_event: We'll set this event in the case of an error (after printing
237 out info about the error).
238 """
239 # We'll set to true once we've locked the lock.
240 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700241
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530242 if not opt.quiet:
243 print('Fetching project %s' % project.name)
244
David Pursehousec1b86a22012-11-14 11:36:51 +0900245 # Encapsulate everything in a try/except/finally so that:
246 # - We always set err_event in the case of an exception.
247 # - We always make sure we call sem.release().
248 # - We always make sure we unlock the lock if we locked it.
249 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700250 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900251 start = time.time()
252 success = project.Sync_NetworkHalf(
253 quiet=opt.quiet,
254 current_branch_only=opt.current_branch_only,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700255 clone_bundle=not opt.no_clone_bundle,
256 no_tags=opt.no_tags)
David Pursehousec1b86a22012-11-14 11:36:51 +0900257 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700258
David Pursehousec1b86a22012-11-14 11:36:51 +0900259 # Lock around all the rest of the code, since printing, updating a set
260 # and Progress.update() are not thread safe.
261 lock.acquire()
262 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700263
David Pursehousec1b86a22012-11-14 11:36:51 +0900264 if not success:
265 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
266 if opt.force_broken:
267 print('warn: --force-broken, continuing to sync',
268 file=sys.stderr)
269 else:
270 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700271
David Pursehousec1b86a22012-11-14 11:36:51 +0900272 fetched.add(project.gitdir)
273 pm.update()
274 except _FetchError:
275 err_event.set()
276 except:
277 err_event.set()
278 raise
279 finally:
280 if did_lock:
281 lock.release()
282 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800283
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700284 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700285 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700286 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800287
288 if self.jobs == 1:
289 for project in projects:
290 pm.update()
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530291 if not opt.quiet:
292 print('Fetching project %s' % project.name)
Shawn O. Pearce5d0efdb2012-08-02 12:13:01 -0700293 if project.Sync_NetworkHalf(
294 quiet=opt.quiet,
295 current_branch_only=opt.current_branch_only,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700296 clone_bundle=not opt.no_clone_bundle,
297 no_tags=opt.no_tags):
Roy Lee18afd7f2010-05-09 04:32:08 +0800298 fetched.add(project.gitdir)
299 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700300 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500301 if opt.force_broken:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700302 print('warn: --force-broken, continuing to sync', file=sys.stderr)
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500303 else:
304 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800305 else:
306 threads = set()
307 lock = _threading.Lock()
308 sem = _threading.Semaphore(self.jobs)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700309 err_event = _threading.Event()
Roy Lee18afd7f2010-05-09 04:32:08 +0800310 for project in projects:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700311 # Check for any errors before starting any new threads.
312 # ...we'll let existing threads finish, though.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400313 if err_event.isSet():
Doug Andersonfc06ced2011-03-16 15:49:18 -0700314 break
315
Roy Lee18afd7f2010-05-09 04:32:08 +0800316 sem.acquire()
317 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700318 args = (opt,
319 project,
320 lock,
321 fetched,
322 pm,
Doug Andersonfc06ced2011-03-16 15:49:18 -0700323 sem,
324 err_event))
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200325 # Ensure that Ctrl-C will not freeze the repo process.
326 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800327 threads.add(t)
328 t.start()
329
330 for t in threads:
331 t.join()
332
Doug Andersonfc06ced2011-03-16 15:49:18 -0700333 # If we saw an error, exit with code 1 so that other scripts can check.
Daniel Sandler723c5dc2011-04-04 11:15:17 -0400334 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700335 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700336 sys.exit(1)
337
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700338 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700339 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700340
341 self._GCProjects(projects)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700342 return fetched
343
Dave Borowitz18857212012-10-23 17:02:59 -0700344 def _GCProjects(self, projects):
Dave Borowitze2152672012-10-31 12:24:38 -0700345 has_dash_c = git_require((1, 7, 2))
346 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700347 cpu_count = multiprocessing.cpu_count()
348 else:
349 cpu_count = 1
350 jobs = min(self.jobs, cpu_count)
351
352 if jobs < 2:
353 for project in projects:
354 project.bare_git.gc('--auto')
355 return
356
357 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
358
359 threads = set()
360 sem = _threading.Semaphore(jobs)
361 err_event = _threading.Event()
362
363 def GC(project):
364 try:
365 try:
366 project.bare_git.gc('--auto', config=config)
367 except GitError:
368 err_event.set()
369 except:
370 err_event.set()
371 raise
372 finally:
373 sem.release()
374
375 for project in projects:
376 if err_event.isSet():
377 break
378 sem.acquire()
379 t = _threading.Thread(target=GC, args=(project,))
380 t.daemon = True
381 threads.add(t)
382 t.start()
383
384 for t in threads:
385 t.join()
386
387 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700388 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700389 sys.exit(1)
390
Tim Kilbourn07669002013-03-08 15:02:49 -0800391 def _ReloadManifest(self, manifest_name=None):
392 if manifest_name:
393 # Override calls _Unload already
394 self.manifest.Override(manifest_name)
395 else:
396 self.manifest._Unload()
397
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700398 def UpdateProjectList(self):
399 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700400 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700401 if project.relpath:
402 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700403 file_name = 'project.list'
404 file_path = os.path.join(self.manifest.repodir, file_name)
405 old_project_paths = []
406
407 if os.path.exists(file_path):
408 fd = open(file_path, 'r')
409 try:
410 old_project_paths = fd.read().split('\n')
411 finally:
412 fd.close()
413 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700414 if not path:
415 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700416 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900417 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400418 if os.path.exists(self.manifest.topdir + '/' + path):
David Pursehousec1b86a22012-11-14 11:36:51 +0900419 project = Project(
420 manifest = self.manifest,
421 name = path,
422 remote = RemoteSpec('origin'),
423 gitdir = os.path.join(self.manifest.topdir,
424 path, '.git'),
425 worktree = os.path.join(self.manifest.topdir, path),
426 relpath = path,
427 revisionExpr = 'HEAD',
428 revisionId = None,
429 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400430
David Pursehousec1b86a22012-11-14 11:36:51 +0900431 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900432 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900433 'are present' % project.relpath, file=sys.stderr)
434 print(' commit changes, then run sync again',
435 file=sys.stderr)
436 return -1
437 else:
438 print('Deleting obsolete path %s' % project.worktree,
439 file=sys.stderr)
440 shutil.rmtree(project.worktree)
441 # Try deleting parent subdirs if they are empty
442 project_dir = os.path.dirname(project.worktree)
443 while project_dir != self.manifest.topdir:
444 try:
445 os.rmdir(project_dir)
446 except OSError:
447 break
448 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700449
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700450 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700451 fd = open(file_path, 'w')
452 try:
453 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700454 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700455 finally:
456 fd.close()
457 return 0
458
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700459 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800460 if opt.jobs:
461 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700462 if self.jobs > 1:
463 soft_limit, _ = _rlimit_nofile()
464 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
465
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700466 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700467 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700468 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700469 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700470 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700471 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500472 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700473 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500474 sys.exit(1)
475 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700476 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500477 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900478 if opt.manifest_server_username or opt.manifest_server_password:
479 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700480 print('error: -u and -p may only be combined with -s or -t',
481 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900482 sys.exit(1)
483 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700484 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900485 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500486
487 if opt.manifest_name:
488 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700489
Chirayu Desaia892b102013-06-11 14:18:46 +0530490 manifest_name = opt.manifest_name
491
Victor Boivie08c880d2011-04-19 10:32:52 +0200492 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700493 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900494 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700495 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700496 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900497
498 manifest_server = self.manifest.manifest_server
David Pursehousecf76b1b2012-09-14 10:31:42 +0900499
David Pursehouse86d973d2012-08-24 10:21:02 +0900500 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900501 username = None
502 password = None
503 if opt.manifest_server_username and opt.manifest_server_password:
504 username = opt.manifest_server_username
505 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900506 else:
507 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900508 info = netrc.netrc()
509 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700510 print('.netrc file does not exist or could not be opened',
511 file=sys.stderr)
David Pursehouse86d973d2012-08-24 10:21:02 +0900512 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900513 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530514 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900515 if parse_result.hostname:
516 username, _account, password = \
517 info.authenticators(parse_result.hostname)
518 except TypeError:
519 # TypeError is raised when the given hostname is not present
520 # in the .netrc file.
Sarah Owenscecd1d82012-11-01 22:59:27 -0700521 print('No credentials found for %s in .netrc'
522 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700523 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700524 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900525
526 if (username and password):
527 manifest_server = manifest_server.replace('://', '://%s:%s@' %
528 (username, password),
529 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900530
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700531 try:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530532 server = xmlrpc.client.Server(manifest_server)
Victor Boivie08c880d2011-04-19 10:32:52 +0200533 if opt.smart_sync:
534 p = self.manifest.manifestProject
535 b = p.GetBranch(p.CurrentBranch)
536 branch = b.merge
537 if branch.startswith(R_HEADS):
538 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700539
Victor Boivie08c880d2011-04-19 10:32:52 +0200540 env = os.environ.copy()
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530541 if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200542 target = '%s-%s' % (env['TARGET_PRODUCT'],
543 env['TARGET_BUILD_VARIANT'])
544 [success, manifest_str] = server.GetApprovedManifest(branch, target)
545 else:
546 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700547 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200548 assert(opt.smart_tag)
549 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700550
551 if success:
552 manifest_name = "smart_sync_override.xml"
553 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
554 manifest_name)
555 try:
556 f = open(manifest_path, 'w')
557 try:
558 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700559 finally:
560 f.close()
561 except IOError:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700562 print('error: cannot write manifest to %s' % manifest_path,
563 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700564 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100565 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700566 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700567 print('error: %s' % manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700568 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530569 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700570 print('error: cannot connect to manifest server %s:\n%s'
571 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900572 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530573 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700574 print('error: cannot connect to manifest server %s:\n%d %s'
575 % (self.manifest.manifest_server, e.errcode, e.errmsg),
576 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700577 sys.exit(1)
578
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700579 rp = self.manifest.repoProject
580 rp.PreSync()
581
582 mp = self.manifest.manifestProject
583 mp.PreSync()
584
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800585 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700586 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800587
Nico Sallembien9bb18162009-12-07 15:38:01 -0800588 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700589 mp.Sync_NetworkHalf(quiet=opt.quiet,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700590 current_branch_only=opt.current_branch_only,
591 no_tags=opt.no_tags)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800592
593 if mp.HasChanges:
594 syncbuf = SyncBuffer(mp.config)
595 mp.Sync_LocalHalf(syncbuf)
596 if not syncbuf.Finish():
597 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100598 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700599 if opt.jobs is None:
600 self.jobs = self.manifest.default.sync_j
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800601 all_projects = self.GetProjects(args,
602 missing_ok=True,
603 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700604
Dave Borowitz67700e92012-10-23 15:00:54 -0700605 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700606 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700607 to_fetch = []
608 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700609 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700610 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900611 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700612 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700613
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800614 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700615 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700616 if opt.network_only:
617 # bail out now; the rest touches the working tree
618 return
619
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800620 # Iteratively fetch missing and/or nested unregistered submodules
621 previously_missing_set = set()
622 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100623 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800624 all_projects = self.GetProjects(args,
625 missing_ok=True,
626 submodules_ok=opt.fetch_submodules)
627 missing = []
628 for project in all_projects:
629 if project.gitdir not in fetched:
630 missing.append(project)
631 if not missing:
632 break
633 # Stop us from non-stopped fetching actually-missing repos: If set of
634 # missing repos has not been changed from last fetch, we break.
635 missing_set = set(p.name for p in missing)
636 if previously_missing_set == missing_set:
637 break
638 previously_missing_set = missing_set
639 fetched.update(self._Fetch(missing, opt))
640
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700641 if self.manifest.IsMirror:
642 # bail out now, we have no working tree
643 return
644
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700645 if self.UpdateProjectList():
646 sys.exit(1)
647
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700648 syncbuf = SyncBuffer(mp.config,
649 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900650 pm = Progress('Syncing work tree', len(all_projects))
651 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700652 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800653 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700654 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700655 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700656 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700657 if not syncbuf.Finish():
658 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700659
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700660 # If there's a notice that's supposed to print at the end of the sync, print
661 # it now...
662 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700663 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700664
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700665def _PostRepoUpgrade(manifest, quiet=False):
Conley Owensc9129d92012-10-01 16:12:28 -0700666 wrapper = WrapperModule()
667 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700668 wrapper.SetupGnuPG(quiet)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700669 for project in manifest.projects.values():
670 if project.Exists:
671 project.PostRepoUpgrade()
672
673def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
674 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700675 print('info: A new version of repo is available', file=sys.stderr)
676 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700677 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700678 syncbuf = SyncBuffer(rp.config)
679 rp.Sync_LocalHalf(syncbuf)
680 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700681 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700682 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700683 raise RepoChangedException(['--repo-upgraded'])
684 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700685 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700686 else:
687 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700688 print('repo version %s is current' % rp.work_git.describe(HEAD),
689 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700690
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700691def _VerifyTag(project):
692 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
693 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700694 print('warning: GnuPG was not available during last "repo init"\n'
695 'warning: Cannot automatically authenticate repo."""',
696 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700697 return True
698
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700699 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700700 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700701 except GitError:
702 cur = None
703
704 if not cur \
705 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700706 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700707 if rev.startswith(R_HEADS):
708 rev = rev[len(R_HEADS):]
709
Sarah Owenscecd1d82012-11-01 22:59:27 -0700710 print(file=sys.stderr)
711 print("warning: project '%s' branch '%s' is not signed"
712 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700713 return False
714
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800715 env = os.environ.copy()
716 env['GIT_DIR'] = project.gitdir.encode()
717 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700718
719 cmd = [GIT, 'tag', '-v', cur]
720 proc = subprocess.Popen(cmd,
721 stdout = subprocess.PIPE,
722 stderr = subprocess.PIPE,
723 env = env)
724 out = proc.stdout.read()
725 proc.stdout.close()
726
727 err = proc.stderr.read()
728 proc.stderr.close()
729
730 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700731 print(file=sys.stderr)
732 print(out, file=sys.stderr)
733 print(err, file=sys.stderr)
734 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700735 return False
736 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700737
738class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700739 _ALPHA = 0.5
740
Dave Borowitz67700e92012-10-23 15:00:54 -0700741 def __init__(self, manifest):
742 self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
743 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700744 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700745
746 def Get(self, project):
747 self._Load()
748 return self._times.get(project.name, _ONE_DAY_S)
749
750 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700751 self._Load()
752 name = project.name
753 old = self._times.get(name, t)
754 self._seen.add(name)
755 a = self._ALPHA
756 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700757
758 def _Load(self):
759 if self._times is None:
760 try:
761 f = open(self._path)
762 except IOError:
763 self._times = {}
764 return self._times
765 try:
766 try:
767 self._times = pickle.load(f)
David Pursehouse1d947b32012-10-25 12:23:11 +0900768 except IOError:
Dave Borowitz67700e92012-10-23 15:00:54 -0700769 try:
770 os.remove(self._path)
771 except OSError:
772 pass
773 self._times = {}
774 finally:
775 f.close()
776 return self._times
777
778 def Save(self):
779 if self._times is None:
780 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700781
782 to_delete = []
783 for name in self._times:
784 if name not in self._seen:
785 to_delete.append(name)
786 for name in to_delete:
787 del self._times[name]
788
Dave Borowitz67700e92012-10-23 15:00:54 -0700789 try:
790 f = open(self._path, 'wb')
791 try:
792 pickle.dump(self._times, f)
793 except (IOError, OSError, pickle.PickleError):
794 try:
795 os.remove(self._path)
796 except OSError:
797 pass
798 finally:
799 f.close()