blob: 943a0264e17c9a7b4830a5a57ca62c85d6bfa97d [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Anthony King85b24ac2014-05-06 15:57:48 +010017import json
David Pursehouse86d973d2012-08-24 10:21:02 +090018import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070019from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import os
21import re
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070022import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023import subprocess
24import sys
Dan Willemsen0745bb22015-08-17 13:41:45 -070025import tempfile
Shawn O. Pearcef6906872009-04-18 10:49:00 -070026import time
David Pursehouse59bbb582013-05-17 10:49:33 +090027
28from pyversion import is_python3
29if is_python3():
Dan Willemsen0745bb22015-08-17 13:41:45 -070030 import http.cookiejar as cookielib
31 import urllib.error
Chirayu Desai217ea7d2013-03-01 19:14:38 +053032 import urllib.parse
Dan Willemsen0745bb22015-08-17 13:41:45 -070033 import urllib.request
David Pursehouse59bbb582013-05-17 10:49:33 +090034 import xmlrpc.client
35else:
Dan Willemsen0745bb22015-08-17 13:41:45 -070036 import cookielib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053037 import imp
Dan Willemsen0745bb22015-08-17 13:41:45 -070038 import urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053039 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090040 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053041 urllib = imp.new_module('urllib')
Dan Willemsen0745bb22015-08-17 13:41:45 -070042 urllib.error = urllib2
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053043 urllib.parse = urlparse
Dan Willemsen0745bb22015-08-17 13:41:45 -070044 urllib.request = urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053045 xmlrpc = imp.new_module('xmlrpc')
46 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070047
Roy Lee18afd7f2010-05-09 04:32:08 +080048try:
49 import threading as _threading
50except ImportError:
51 import dummy_threading as _threading
52
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070053try:
54 import resource
55 def _rlimit_nofile():
56 return resource.getrlimit(resource.RLIMIT_NOFILE)
57except ImportError:
58 def _rlimit_nofile():
59 return (256, 256)
60
Dave Borowitz18857212012-10-23 17:02:59 -070061try:
62 import multiprocessing
63except ImportError:
64 multiprocessing = None
65
David Rileye0684ad2017-04-05 00:02:59 -070066import event_log
Dave Borowitze2152672012-10-31 12:24:38 -070067from git_command import GIT, git_require
David Pursehouseba7bc732015-08-20 16:55:42 +090068from git_config import GetUrlCookieFile
David Pursehoused94aaef2012-09-07 09:52:04 +090069from git_refs import R_HEADS, HEAD
Simran Basibdb52712015-08-10 13:23:23 -070070import gitc_utils
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070071from project import Project
72from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080073from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000074from error import RepoChangedException, GitError, ManifestParseError
Renaud Paquaya65adf72016-11-03 10:37:53 -070075import platform_utils
Shawn O. Pearce350cde42009-04-16 11:21:18 -070076from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070077from progress import Progress
Conley Owens094cdbe2014-01-30 15:09:59 -080078from wrapper import Wrapper
Dan Willemsen5ea32d12015-09-08 13:27:20 -070079from manifest_xml import GitcManifest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070080
Dave Borowitz67700e92012-10-23 15:00:54 -070081_ONE_DAY_S = 24 * 60 * 60
82
Doug Andersonfc06ced2011-03-16 15:49:18 -070083class _FetchError(Exception):
84 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
85 pass
86
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080087class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080088 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070089 common = True
90 helpSummary = "Update working tree to the latest revision"
91 helpUsage = """
92%prog [<project>...]
93"""
94 helpDescription = """
95The '%prog' command synchronizes local project directories
96with the remote repositories specified in the manifest. If a local
97project does not yet exist, it will clone a new local directory from
98the remote repository and set up tracking branches as specified in
99the manifest. If the local project already exists, '%prog'
100will update the remote branches and rebase any new local changes
101on top of the new remote changes.
102
103'%prog' will synchronize all projects listed at the command
104line. Projects can be specified either by name, or by a relative
105or absolute path to the project's local directory. If no projects
106are specified, '%prog' will synchronize all projects listed in
107the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700108
109The -d/--detach option can be used to switch specified projects
110back to the manifest revision. This option is especially helpful
111if the project is currently on a topic branch, but the manifest
112revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700113
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700114The -s/--smart-sync option can be used to sync to a known good
115build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200116manifest. The -t/--smart-tag option is similar and allows you to
117specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700118
David Pursehousecf76b1b2012-09-14 10:31:42 +0900119The -u/--manifest-server-username and -p/--manifest-server-password
120options can be used to specify a username and password to authenticate
121with the manifest server when using the -s or -t option.
122
123If -u and -p are not specified when using the -s or -t option, '%prog'
124will attempt to read authentication credentials for the manifest server
125from the user's .netrc file.
126
127'%prog' will not use authentication credentials from -u/-p or .netrc
128if the manifest server specified in the manifest file already includes
129credentials.
130
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500131The -f/--force-broken option can be used to proceed with syncing
132other projects if a project sync fails.
133
Kevin Degiabaa7f32014-11-12 11:27:45 -0700134The --force-sync option can be used to overwrite existing git
135directories if they have previously been linked to a different
136object direcotry. WARNING: This may cause data to be lost since
137refs may be removed when overwriting.
138
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700139The --no-clone-bundle option disables any attempt to use
140$URL/clone.bundle to bootstrap a new Git repository from a
141resumeable bundle file on a content delivery network. This
142may be necessary if there are problems with the local Python
143HTTP client or proxy configuration, but the Git binary works.
144
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800145The --fetch-submodules option enables fetching Git submodules
146of a project from server.
147
David Pursehousef2fad612015-01-29 14:36:28 +0900148The -c/--current-branch option can be used to only fetch objects that
149are on the branch specified by a project's revision.
150
David Pursehouseb1553542014-09-04 21:28:09 +0900151The --optimized-fetch option can be used to only fetch projects that
152are fixed to a sha1 revision if the sha1 revision does not already
153exist locally.
154
David Pursehouse74cfd272015-10-14 10:50:15 +0900155The --prune option can be used to remove any refs that no longer
156exist on the remote.
157
Mike Frysingerb8f7bb02018-10-10 01:05:11 -0400158# SSH Connections
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700159
160If at least one project remote URL uses an SSH connection (ssh://,
161git+ssh://, or user@host:path syntax) repo will automatically
162enable the SSH ControlMaster option when connecting to that host.
163This feature permits other projects in the same '%prog' session to
164reuse the same SSH tunnel, saving connection setup overheads.
165
166To disable this behavior on UNIX platforms, set the GIT_SSH
167environment variable to 'ssh'. For example:
168
169 export GIT_SSH=ssh
170 %prog
171
Mike Frysingerb8f7bb02018-10-10 01:05:11 -0400172# Compatibility
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700173
174This feature is automatically disabled on Windows, due to the lack
175of UNIX domain socket support.
176
177This feature is not compatible with url.insteadof rewrites in the
178user's ~/.gitconfig. '%prog' is currently not able to perform the
179rewrite early enough to establish the ControlMaster tunnel.
180
181If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
182later is required to fix a server side protocol bug.
183
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700184"""
185
Nico Sallembien6623b212010-05-11 12:57:01 -0700186 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000187 try:
188 self.jobs = self.manifest.default.sync_j
189 except ManifestParseError:
190 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700191
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500192 p.add_option('-f', '--force-broken',
193 dest='force_broken', action='store_true',
194 help="continue sync even if a project fails to sync")
Kevin Degiabaa7f32014-11-12 11:27:45 -0700195 p.add_option('--force-sync',
196 dest='force_sync', action='store_true',
197 help="overwrite an existing git directory if it needs to "
198 "point to a different object directory. WARNING: this "
199 "may cause loss of data")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900200 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700201 dest='local_only', action='store_true',
202 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900203 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700204 dest='network_only', action='store_true',
205 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900206 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700207 dest='detach_head', action='store_true',
208 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900209 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700210 dest='current_branch_only', action='store_true',
211 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900212 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700213 dest='quiet', action='store_true',
214 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900215 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800216 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700217 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500218 p.add_option('-m', '--manifest-name',
219 dest='manifest_name',
220 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700221 p.add_option('--no-clone-bundle',
222 dest='no_clone_bundle', action='store_true',
223 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800224 p.add_option('-u', '--manifest-server-username', action='store',
225 dest='manifest_server_username',
226 help='username to authenticate with the manifest server')
227 p.add_option('-p', '--manifest-server-password', action='store',
228 dest='manifest_server_password',
229 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800230 p.add_option('--fetch-submodules',
231 dest='fetch_submodules', action='store_true',
232 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700233 p.add_option('--no-tags',
234 dest='no_tags', action='store_true',
235 help="don't fetch tags")
David Pursehouseb1553542014-09-04 21:28:09 +0900236 p.add_option('--optimized-fetch',
237 dest='optimized_fetch', action='store_true',
238 help='only fetch projects fixed to sha1 if revision does not exist locally')
David Pursehouse74cfd272015-10-14 10:50:15 +0900239 p.add_option('--prune', dest='prune', action='store_true',
240 help='delete refs that no longer exist on the remote')
Nico Sallembien6623b212010-05-11 12:57:01 -0700241 if show_smart:
242 p.add_option('-s', '--smart-sync',
243 dest='smart_sync', action='store_true',
David Pursehouse79fba682016-04-13 18:03:00 +0900244 help='smart sync using manifest from the latest known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200245 p.add_option('-t', '--smart-tag',
246 dest='smart_tag', action='store',
247 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700248
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700249 g = p.add_option_group('repo Version options')
250 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700251 dest='no_repo_verify', action='store_true',
252 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700253 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800254 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700255 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700256
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500257 def _FetchProjectList(self, opt, projects, sem, *args, **kwargs):
David Pursehousec1b86a22012-11-14 11:36:51 +0900258 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800259
David James8d201162013-10-11 17:03:19 -0700260 Delegates most of the work to _FetchHelper.
261
262 Args:
263 opt: Program options returned from optparse. See _Options().
264 projects: Projects to fetch.
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500265 sem: We'll release() this semaphore when we exit so that another thread
266 can be started up.
David James89ece422014-01-09 18:51:58 -0800267 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
David James8d201162013-10-11 17:03:19 -0700268 _FetchHelper docstring for details.
269 """
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500270 try:
271 for project in projects:
272 success = self._FetchHelper(opt, project, *args, **kwargs)
273 if not success and not opt.force_broken:
274 break
275 finally:
276 sem.release()
David James8d201162013-10-11 17:03:19 -0700277
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500278 def _FetchHelper(self, opt, project, lock, fetched, pm, err_event):
David James8d201162013-10-11 17:03:19 -0700279 """Fetch git objects for a single project.
280
David Pursehousec1b86a22012-11-14 11:36:51 +0900281 Args:
282 opt: Program options returned from optparse. See _Options().
283 project: Project object for the project to fetch.
284 lock: Lock for accessing objects that are shared amongst multiple
285 _FetchHelper() threads.
286 fetched: set object that we will add project.gitdir to when we're done
287 (with our lock held).
288 pm: Instance of a Project object. We will call pm.update() (with our
289 lock held).
David Pursehousec1b86a22012-11-14 11:36:51 +0900290 err_event: We'll set this event in the case of an error (after printing
291 out info about the error).
David James8d201162013-10-11 17:03:19 -0700292
293 Returns:
294 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900295 """
296 # We'll set to true once we've locked the lock.
297 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700298
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530299 if not opt.quiet:
300 print('Fetching project %s' % project.name)
301
David Pursehousec1b86a22012-11-14 11:36:51 +0900302 # Encapsulate everything in a try/except/finally so that:
303 # - We always set err_event in the case of an exception.
304 # - We always make sure we call sem.release().
305 # - We always make sure we unlock the lock if we locked it.
David Rileye0684ad2017-04-05 00:02:59 -0700306 start = time.time()
307 success = False
David Pursehousec1b86a22012-11-14 11:36:51 +0900308 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700309 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900310 success = project.Sync_NetworkHalf(
311 quiet=opt.quiet,
312 current_branch_only=opt.current_branch_only,
Kevin Degiabaa7f32014-11-12 11:27:45 -0700313 force_sync=opt.force_sync,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700314 clone_bundle=not opt.no_clone_bundle,
David Pursehouseb1553542014-09-04 21:28:09 +0900315 no_tags=opt.no_tags, archive=self.manifest.IsArchive,
David Pursehouse74cfd272015-10-14 10:50:15 +0900316 optimized_fetch=opt.optimized_fetch,
317 prune=opt.prune)
David Pursehousec1b86a22012-11-14 11:36:51 +0900318 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700319
David Pursehousec1b86a22012-11-14 11:36:51 +0900320 # Lock around all the rest of the code, since printing, updating a set
321 # and Progress.update() are not thread safe.
322 lock.acquire()
323 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700324
David Pursehousec1b86a22012-11-14 11:36:51 +0900325 if not success:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800326 err_event.set()
Marc Herbertffb4b892017-04-04 22:03:53 -0700327 print('error: Cannot fetch %s from %s'
328 % (project.name, project.remote.url),
329 file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900330 if opt.force_broken:
331 print('warn: --force-broken, continuing to sync',
332 file=sys.stderr)
333 else:
334 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700335
David Pursehousec1b86a22012-11-14 11:36:51 +0900336 fetched.add(project.gitdir)
337 pm.update()
338 except _FetchError:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800339 pass
Dan Sandlerc5cd4332015-07-31 09:37:53 -0400340 except Exception as e:
341 print('error: Cannot fetch %s (%s: %s)' \
342 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900343 err_event.set()
344 raise
345 finally:
346 if did_lock:
347 lock.release()
David Rileye0684ad2017-04-05 00:02:59 -0700348 finish = time.time()
349 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
350 start, finish, success)
Roy Lee18afd7f2010-05-09 04:32:08 +0800351
David James8d201162013-10-11 17:03:19 -0700352 return success
353
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700354 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700355 fetched = set()
David James89ece422014-01-09 18:51:58 -0800356 lock = _threading.Lock()
Tim Schumacher913327f2017-06-05 15:01:41 +0200357 pm = Progress('Fetching projects', len(projects),
Tim Schumacher7be072e2017-06-28 18:29:23 +0200358 print_newline=not(opt.quiet),
359 always_print_percentage=opt.quiet)
Roy Lee18afd7f2010-05-09 04:32:08 +0800360
David James89ece422014-01-09 18:51:58 -0800361 objdir_project_map = dict()
362 for project in projects:
363 objdir_project_map.setdefault(project.objdir, []).append(project)
David James8d201162013-10-11 17:03:19 -0700364
David James89ece422014-01-09 18:51:58 -0800365 threads = set()
366 sem = _threading.Semaphore(self.jobs)
367 err_event = _threading.Event()
368 for project_list in objdir_project_map.values():
369 # Check for any errors before running any more tasks.
370 # ...we'll let existing threads finish, though.
371 if err_event.isSet() and not opt.force_broken:
372 break
Doug Andersonfc06ced2011-03-16 15:49:18 -0700373
David James89ece422014-01-09 18:51:58 -0800374 sem.acquire()
375 kwargs = dict(opt=opt,
376 projects=project_list,
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500377 sem=sem,
David James89ece422014-01-09 18:51:58 -0800378 lock=lock,
379 fetched=fetched,
380 pm=pm,
David James89ece422014-01-09 18:51:58 -0800381 err_event=err_event)
382 if self.jobs > 1:
David James8d201162013-10-11 17:03:19 -0700383 t = _threading.Thread(target = self._FetchProjectList,
David James89ece422014-01-09 18:51:58 -0800384 kwargs = kwargs)
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200385 # Ensure that Ctrl-C will not freeze the repo process.
386 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800387 threads.add(t)
388 t.start()
David James89ece422014-01-09 18:51:58 -0800389 else:
390 self._FetchProjectList(**kwargs)
Roy Lee18afd7f2010-05-09 04:32:08 +0800391
David James89ece422014-01-09 18:51:58 -0800392 for t in threads:
393 t.join()
Roy Lee18afd7f2010-05-09 04:32:08 +0800394
David James89ece422014-01-09 18:51:58 -0800395 # If we saw an error, exit with code 1 so that other scripts can check.
Nicolas Cornu8419ab22017-06-16 12:09:06 +0200396 if err_event.isSet() and not opt.force_broken:
David James89ece422014-01-09 18:51:58 -0800397 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
398 sys.exit(1)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700399
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700400 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700401 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700402
Julien Campergue335f5ef2013-10-16 11:02:35 +0200403 if not self.manifest.IsArchive:
404 self._GCProjects(projects)
405
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700406 return fetched
407
Dave Borowitz18857212012-10-23 17:02:59 -0700408 def _GCProjects(self, projects):
Gabe Black2ff30292014-10-09 17:54:35 -0700409 gc_gitdirs = {}
David James8d201162013-10-11 17:03:19 -0700410 for project in projects:
Gabe Black2ff30292014-10-09 17:54:35 -0700411 if len(project.manifest.GetProjectsWithName(project.name)) > 1:
412 print('Shared project %s found, disabling pruning.' % project.name)
413 project.bare_git.config('--replace-all', 'gc.pruneExpire', 'never')
414 gc_gitdirs[project.gitdir] = project.bare_git
David James8d201162013-10-11 17:03:19 -0700415
Dave Borowitze2152672012-10-31 12:24:38 -0700416 has_dash_c = git_require((1, 7, 2))
417 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700418 cpu_count = multiprocessing.cpu_count()
419 else:
420 cpu_count = 1
421 jobs = min(self.jobs, cpu_count)
422
423 if jobs < 2:
Gabe Black2ff30292014-10-09 17:54:35 -0700424 for bare_git in gc_gitdirs.values():
David James8d201162013-10-11 17:03:19 -0700425 bare_git.gc('--auto')
Dave Borowitz18857212012-10-23 17:02:59 -0700426 return
427
428 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
429
430 threads = set()
431 sem = _threading.Semaphore(jobs)
432 err_event = _threading.Event()
433
David James8d201162013-10-11 17:03:19 -0700434 def GC(bare_git):
Dave Borowitz18857212012-10-23 17:02:59 -0700435 try:
436 try:
David James8d201162013-10-11 17:03:19 -0700437 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700438 except GitError:
439 err_event.set()
440 except:
441 err_event.set()
442 raise
443 finally:
444 sem.release()
445
Gabe Black2ff30292014-10-09 17:54:35 -0700446 for bare_git in gc_gitdirs.values():
Dave Borowitz18857212012-10-23 17:02:59 -0700447 if err_event.isSet():
448 break
449 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700450 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700451 t.daemon = True
452 threads.add(t)
453 t.start()
454
455 for t in threads:
456 t.join()
457
458 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700459 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700460 sys.exit(1)
461
Tim Kilbourn07669002013-03-08 15:02:49 -0800462 def _ReloadManifest(self, manifest_name=None):
463 if manifest_name:
464 # Override calls _Unload already
465 self.manifest.Override(manifest_name)
466 else:
467 self.manifest._Unload()
468
Dan Willemsen43507912016-09-01 16:26:02 -0700469 def _DeleteProject(self, path):
470 print('Deleting obsolete path %s' % path, file=sys.stderr)
471
472 # Delete the .git directory first, so we're less likely to have a partially
473 # working git repository around. There shouldn't be any git projects here,
474 # so rmtree works.
475 try:
Renaud Paquaya65adf72016-11-03 10:37:53 -0700476 platform_utils.rmtree(os.path.join(path, '.git'))
Dan Willemsen43507912016-09-01 16:26:02 -0700477 except OSError:
478 print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr)
479 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
480 print(' remove manually, then run sync again', file=sys.stderr)
481 return -1
482
483 # Delete everything under the worktree, except for directories that contain
484 # another git project
485 dirs_to_remove = []
486 failed = False
487 for root, dirs, files in os.walk(path):
488 for f in files:
489 try:
Renaud Paquay010fed72016-11-11 14:25:29 -0800490 platform_utils.remove(os.path.join(root, f))
Dan Willemsen43507912016-09-01 16:26:02 -0700491 except OSError:
492 print('Failed to remove %s' % os.path.join(root, f), file=sys.stderr)
493 failed = True
494 dirs[:] = [d for d in dirs
495 if not os.path.lexists(os.path.join(root, d, '.git'))]
496 dirs_to_remove += [os.path.join(root, d) for d in dirs
497 if os.path.join(root, d) not in dirs_to_remove]
498 for d in reversed(dirs_to_remove):
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700499 if platform_utils.islink(d):
Dan Willemseneceeb1b2016-09-25 18:24:27 -0700500 try:
Renaud Paquay010fed72016-11-11 14:25:29 -0800501 platform_utils.remove(d)
Dan Willemseneceeb1b2016-09-25 18:24:27 -0700502 except OSError:
503 print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
504 failed = True
505 elif len(os.listdir(d)) == 0:
Dan Willemsen43507912016-09-01 16:26:02 -0700506 try:
507 os.rmdir(d)
508 except OSError:
509 print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
510 failed = True
511 continue
512 if failed:
513 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
514 print(' remove manually, then run sync again', file=sys.stderr)
515 return -1
516
517 # Try deleting parent dirs if they are empty
518 project_dir = path
519 while project_dir != self.manifest.topdir:
520 if len(os.listdir(project_dir)) == 0:
521 os.rmdir(project_dir)
522 else:
523 break
524 project_dir = os.path.dirname(project_dir)
525
526 return 0
527
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700528 def UpdateProjectList(self):
529 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700530 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700531 if project.relpath:
532 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700533 file_name = 'project.list'
534 file_path = os.path.join(self.manifest.repodir, file_name)
535 old_project_paths = []
536
537 if os.path.exists(file_path):
538 fd = open(file_path, 'r')
539 try:
540 old_project_paths = fd.read().split('\n')
541 finally:
542 fd.close()
543 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700544 if not path:
545 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700546 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900547 # If the path has already been deleted, we don't need to do it
Dan Willemsen43507912016-09-01 16:26:02 -0700548 gitdir = os.path.join(self.manifest.topdir, path, '.git')
549 if os.path.exists(gitdir):
David Pursehousec1b86a22012-11-14 11:36:51 +0900550 project = Project(
551 manifest = self.manifest,
552 name = path,
553 remote = RemoteSpec('origin'),
David James8d201162013-10-11 17:03:19 -0700554 gitdir = gitdir,
555 objdir = gitdir,
David Pursehousec1b86a22012-11-14 11:36:51 +0900556 worktree = os.path.join(self.manifest.topdir, path),
557 relpath = path,
558 revisionExpr = 'HEAD',
559 revisionId = None,
560 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400561
David Pursehousec1b86a22012-11-14 11:36:51 +0900562 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900563 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900564 'are present' % project.relpath, file=sys.stderr)
565 print(' commit changes, then run sync again',
566 file=sys.stderr)
567 return -1
Dan Willemsen43507912016-09-01 16:26:02 -0700568 elif self._DeleteProject(project.worktree):
569 return -1
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700570
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700571 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700572 fd = open(file_path, 'w')
573 try:
574 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700575 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700576 finally:
577 fd.close()
578 return 0
579
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700580 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800581 if opt.jobs:
582 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700583 if self.jobs > 1:
584 soft_limit, _ = _rlimit_nofile()
585 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
586
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700587 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700588 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700589 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700590 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700591 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700592 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500593 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700594 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500595 sys.exit(1)
596 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700597 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500598 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900599 if opt.manifest_server_username or opt.manifest_server_password:
600 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700601 print('error: -u and -p may only be combined with -s or -t',
602 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900603 sys.exit(1)
604 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700605 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900606 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500607
608 if opt.manifest_name:
609 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700610
Chirayu Desaia892b102013-06-11 14:18:46 +0530611 manifest_name = opt.manifest_name
David Pursehouse59b41742015-05-07 14:36:09 +0900612 smart_sync_manifest_name = "smart_sync_override.xml"
613 smart_sync_manifest_path = os.path.join(
614 self.manifest.manifestProject.worktree, smart_sync_manifest_name)
Chirayu Desaia892b102013-06-11 14:18:46 +0530615
Victor Boivie08c880d2011-04-19 10:32:52 +0200616 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700617 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900618 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700619 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700620 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900621
622 manifest_server = self.manifest.manifest_server
David Pursehousefb99c712013-09-25 11:09:34 +0900623 if not opt.quiet:
624 print('Using manifest server %s' % manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900625
David Pursehouse86d973d2012-08-24 10:21:02 +0900626 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900627 username = None
628 password = None
629 if opt.manifest_server_username and opt.manifest_server_password:
630 username = opt.manifest_server_username
631 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900632 else:
633 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900634 info = netrc.netrc()
635 except IOError:
David Pursehouseba7bc732015-08-20 16:55:42 +0900636 # .netrc file does not exist or could not be opened
637 pass
David Pursehouse86d973d2012-08-24 10:21:02 +0900638 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900639 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530640 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900641 if parse_result.hostname:
David Pursehouseba7bc732015-08-20 16:55:42 +0900642 auth = info.authenticators(parse_result.hostname)
643 if auth:
644 username, _account, password = auth
645 else:
646 print('No credentials found for %s in .netrc'
647 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700648 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700649 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900650
651 if (username and password):
652 manifest_server = manifest_server.replace('://', '://%s:%s@' %
653 (username, password),
654 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900655
Dan Willemsen0745bb22015-08-17 13:41:45 -0700656 transport = PersistentTransport(manifest_server)
657 if manifest_server.startswith('persistent-'):
658 manifest_server = manifest_server[len('persistent-'):]
659
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700660 try:
Dan Willemsen0745bb22015-08-17 13:41:45 -0700661 server = xmlrpc.client.Server(manifest_server, transport=transport)
Victor Boivie08c880d2011-04-19 10:32:52 +0200662 if opt.smart_sync:
663 p = self.manifest.manifestProject
664 b = p.GetBranch(p.CurrentBranch)
665 branch = b.merge
666 if branch.startswith(R_HEADS):
667 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700668
Victor Boivie08c880d2011-04-19 10:32:52 +0200669 env = os.environ.copy()
Jeff Davidson5cf16602014-10-02 10:13:38 -0700670 if 'SYNC_TARGET' in env:
671 target = env['SYNC_TARGET']
672 [success, manifest_str] = server.GetApprovedManifest(branch, target)
673 elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200674 target = '%s-%s' % (env['TARGET_PRODUCT'],
675 env['TARGET_BUILD_VARIANT'])
676 [success, manifest_str] = server.GetApprovedManifest(branch, target)
677 else:
678 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700679 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200680 assert(opt.smart_tag)
681 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700682
683 if success:
David Pursehouse59b41742015-05-07 14:36:09 +0900684 manifest_name = smart_sync_manifest_name
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700685 try:
David Pursehouse59b41742015-05-07 14:36:09 +0900686 f = open(smart_sync_manifest_path, 'w')
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700687 try:
688 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700689 finally:
690 f.close()
David Pursehouse727cc3e2015-05-07 14:16:49 +0900691 except IOError as e:
692 print('error: cannot write manifest to %s:\n%s'
David Pursehouse59b41742015-05-07 14:36:09 +0900693 % (smart_sync_manifest_path, e),
Sarah Owenscecd1d82012-11-01 22:59:27 -0700694 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700695 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100696 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700697 else:
David Pursehouse351fe2c2013-09-25 17:54:26 +0900698 print('error: manifest server RPC call failed: %s' %
699 manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700700 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530701 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700702 print('error: cannot connect to manifest server %s:\n%s'
703 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900704 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530705 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700706 print('error: cannot connect to manifest server %s:\n%d %s'
707 % (self.manifest.manifest_server, e.errcode, e.errmsg),
708 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700709 sys.exit(1)
David Pursehouse59b41742015-05-07 14:36:09 +0900710 else: # Not smart sync or smart tag mode
711 if os.path.isfile(smart_sync_manifest_path):
712 try:
Renaud Paquay010fed72016-11-11 14:25:29 -0800713 platform_utils.remove(smart_sync_manifest_path)
David Pursehouse59b41742015-05-07 14:36:09 +0900714 except OSError as e:
715 print('error: failed to remove existing smart sync override manifest: %s' %
716 e, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700717
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700718 rp = self.manifest.repoProject
719 rp.PreSync()
720
721 mp = self.manifest.manifestProject
722 mp.PreSync()
723
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800724 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700725 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800726
Nico Sallembien9bb18162009-12-07 15:38:01 -0800727 if not opt.local_only:
David Rileye0684ad2017-04-05 00:02:59 -0700728 start = time.time()
729 success = mp.Sync_NetworkHalf(quiet=opt.quiet,
730 current_branch_only=opt.current_branch_only,
731 no_tags=opt.no_tags,
732 optimized_fetch=opt.optimized_fetch,
733 submodules=self.manifest.HasSubmodules)
734 finish = time.time()
735 self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
736 start, finish, success)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800737
738 if mp.HasChanges:
739 syncbuf = SyncBuffer(mp.config)
David Rileye0684ad2017-04-05 00:02:59 -0700740 start = time.time()
Martin Kellye4e94d22017-03-21 16:05:12 -0700741 mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
David Rileye0684ad2017-04-05 00:02:59 -0700742 clean = syncbuf.Finish()
743 self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
744 start, time.time(), clean)
745 if not clean:
Nico Sallembien9bb18162009-12-07 15:38:01 -0800746 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100747 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700748 if opt.jobs is None:
749 self.jobs = self.manifest.default.sync_j
Simran Basib9a1b732015-08-20 12:19:28 -0700750
Simran Basib9a1b732015-08-20 12:19:28 -0700751 if self.gitc_manifest:
752 gitc_manifest_projects = self.GetProjects(args,
Simran Basib9a1b732015-08-20 12:19:28 -0700753 missing_ok=True)
754 gitc_projects = []
755 opened_projects = []
756 for project in gitc_manifest_projects:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700757 if project.relpath in self.gitc_manifest.paths and \
758 self.gitc_manifest.paths[project.relpath].old_revision:
759 opened_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700760 else:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700761 gitc_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700762
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700763 if not args:
764 gitc_projects = None
765
766 if gitc_projects != [] and not opt.local_only:
Simran Basib9a1b732015-08-20 12:19:28 -0700767 print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700768 manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
769 if manifest_name:
770 manifest.Override(manifest_name)
771 else:
772 manifest.Override(self.manifest.manifestFile)
773 gitc_utils.generate_gitc_manifest(self.gitc_manifest,
774 manifest,
Simran Basib9a1b732015-08-20 12:19:28 -0700775 gitc_projects)
776 print('GITC client successfully synced.')
777
778 # The opened projects need to be synced as normal, therefore we
779 # generate a new args list to represent the opened projects.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700780 # TODO: make this more reliable -- if there's a project name/path overlap,
781 # this may choose the wrong project.
David Pursehouse3bcd3052017-07-10 22:42:22 +0900782 args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
783 for path in opened_projects]
Simran Basib9a1b732015-08-20 12:19:28 -0700784 if not args:
785 return
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800786 all_projects = self.GetProjects(args,
787 missing_ok=True,
788 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700789
Dave Borowitz67700e92012-10-23 15:00:54 -0700790 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700791 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700792 to_fetch = []
793 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700794 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700795 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900796 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700797 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700798
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800799 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700800 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700801 if opt.network_only:
802 # bail out now; the rest touches the working tree
803 return
804
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800805 # Iteratively fetch missing and/or nested unregistered submodules
806 previously_missing_set = set()
807 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100808 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800809 all_projects = self.GetProjects(args,
810 missing_ok=True,
811 submodules_ok=opt.fetch_submodules)
812 missing = []
813 for project in all_projects:
814 if project.gitdir not in fetched:
815 missing.append(project)
816 if not missing:
817 break
818 # Stop us from non-stopped fetching actually-missing repos: If set of
819 # missing repos has not been changed from last fetch, we break.
820 missing_set = set(p.name for p in missing)
821 if previously_missing_set == missing_set:
822 break
823 previously_missing_set = missing_set
824 fetched.update(self._Fetch(missing, opt))
825
Julien Campergue335f5ef2013-10-16 11:02:35 +0200826 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700827 # bail out now, we have no working tree
828 return
829
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700830 if self.UpdateProjectList():
831 sys.exit(1)
832
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700833 syncbuf = SyncBuffer(mp.config,
834 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900835 pm = Progress('Syncing work tree', len(all_projects))
836 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700837 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800838 if project.worktree:
David Rileye0684ad2017-04-05 00:02:59 -0700839 start = time.time()
Kevin Degiabaa7f32014-11-12 11:27:45 -0700840 project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
David Rileye0684ad2017-04-05 00:02:59 -0700841 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
842 start, time.time(), syncbuf.Recently())
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700843 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700844 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700845 if not syncbuf.Finish():
846 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700847
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700848 # If there's a notice that's supposed to print at the end of the sync, print
849 # it now...
850 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700851 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700852
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700853def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -0800854 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -0700855 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700856 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -0800857 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700858 if project.Exists:
859 project.PostRepoUpgrade()
860
861def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
862 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700863 print('info: A new version of repo is available', file=sys.stderr)
864 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700865 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700866 syncbuf = SyncBuffer(rp.config)
867 rp.Sync_LocalHalf(syncbuf)
868 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700869 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700870 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700871 raise RepoChangedException(['--repo-upgraded'])
872 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700873 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700874 else:
875 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700876 print('repo version %s is current' % rp.work_git.describe(HEAD),
877 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700878
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700879def _VerifyTag(project):
880 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
881 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700882 print('warning: GnuPG was not available during last "repo init"\n'
883 'warning: Cannot automatically authenticate repo."""',
884 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700885 return True
886
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700887 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700888 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700889 except GitError:
890 cur = None
891
892 if not cur \
893 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700894 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700895 if rev.startswith(R_HEADS):
896 rev = rev[len(R_HEADS):]
897
Sarah Owenscecd1d82012-11-01 22:59:27 -0700898 print(file=sys.stderr)
899 print("warning: project '%s' branch '%s' is not signed"
900 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700901 return False
902
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800903 env = os.environ.copy()
904 env['GIT_DIR'] = project.gitdir.encode()
905 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700906
907 cmd = [GIT, 'tag', '-v', cur]
908 proc = subprocess.Popen(cmd,
909 stdout = subprocess.PIPE,
910 stderr = subprocess.PIPE,
911 env = env)
912 out = proc.stdout.read()
913 proc.stdout.close()
914
915 err = proc.stderr.read()
916 proc.stderr.close()
917
918 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700919 print(file=sys.stderr)
920 print(out, file=sys.stderr)
921 print(err, file=sys.stderr)
922 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700923 return False
924 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700925
David Rileye0684ad2017-04-05 00:02:59 -0700926
Dave Borowitz67700e92012-10-23 15:00:54 -0700927class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700928 _ALPHA = 0.5
929
Dave Borowitz67700e92012-10-23 15:00:54 -0700930 def __init__(self, manifest):
Anthony King85b24ac2014-05-06 15:57:48 +0100931 self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
Dave Borowitz67700e92012-10-23 15:00:54 -0700932 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700933 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700934
935 def Get(self, project):
936 self._Load()
937 return self._times.get(project.name, _ONE_DAY_S)
938
939 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700940 self._Load()
941 name = project.name
942 old = self._times.get(name, t)
943 self._seen.add(name)
944 a = self._ALPHA
945 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700946
947 def _Load(self):
948 if self._times is None:
949 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100950 f = open(self._path)
Dave Borowitz67700e92012-10-23 15:00:54 -0700951 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100952 self._times = json.load(f)
953 finally:
954 f.close()
955 except (IOError, ValueError):
956 try:
Renaud Paquay010fed72016-11-11 14:25:29 -0800957 platform_utils.remove(self._path)
Anthony King85b24ac2014-05-06 15:57:48 +0100958 except OSError:
959 pass
960 self._times = {}
Dave Borowitz67700e92012-10-23 15:00:54 -0700961
962 def Save(self):
963 if self._times is None:
964 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700965
966 to_delete = []
967 for name in self._times:
968 if name not in self._seen:
969 to_delete.append(name)
970 for name in to_delete:
971 del self._times[name]
972
Dave Borowitz67700e92012-10-23 15:00:54 -0700973 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100974 f = open(self._path, 'w')
Dave Borowitz67700e92012-10-23 15:00:54 -0700975 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100976 json.dump(self._times, f, indent=2)
977 finally:
978 f.close()
979 except (IOError, TypeError):
980 try:
Renaud Paquay010fed72016-11-11 14:25:29 -0800981 platform_utils.remove(self._path)
Anthony King85b24ac2014-05-06 15:57:48 +0100982 except OSError:
983 pass
Dan Willemsen0745bb22015-08-17 13:41:45 -0700984
985# This is a replacement for xmlrpc.client.Transport using urllib2
986# and supporting persistent-http[s]. It cannot change hosts from
987# request to request like the normal transport, the real url
988# is passed during initialization.
989class PersistentTransport(xmlrpc.client.Transport):
990 def __init__(self, orig_host):
991 self.orig_host = orig_host
992
993 def request(self, host, handler, request_body, verbose=False):
994 with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
995 # Python doesn't understand cookies with the #HttpOnly_ prefix
996 # Since we're only using them for HTTP, copy the file temporarily,
997 # stripping those prefixes away.
Dan Willemsen3010e5b2015-08-20 10:09:20 -0700998 if cookiefile:
999 tmpcookiefile = tempfile.NamedTemporaryFile()
David Pursehouse4c5f74e2015-10-02 11:10:10 +09001000 tmpcookiefile.write("# HTTP Cookie File")
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001001 try:
1002 with open(cookiefile) as f:
1003 for line in f:
1004 if line.startswith("#HttpOnly_"):
1005 line = line[len("#HttpOnly_"):]
1006 tmpcookiefile.write(line)
1007 tmpcookiefile.flush()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001008
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001009 cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
David Pursehouseb1ad2192015-09-30 10:35:43 +09001010 try:
1011 cookiejar.load()
1012 except cookielib.LoadError:
1013 cookiejar = cookielib.CookieJar()
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001014 finally:
1015 tmpcookiefile.close()
1016 else:
1017 cookiejar = cookielib.CookieJar()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001018
1019 proxyhandler = urllib.request.ProxyHandler
1020 if proxy:
1021 proxyhandler = urllib.request.ProxyHandler({
1022 "http": proxy,
1023 "https": proxy })
1024
1025 opener = urllib.request.build_opener(
1026 urllib.request.HTTPCookieProcessor(cookiejar),
1027 proxyhandler)
1028
1029 url = urllib.parse.urljoin(self.orig_host, handler)
1030 parse_results = urllib.parse.urlparse(url)
1031
1032 scheme = parse_results.scheme
1033 if scheme == 'persistent-http':
1034 scheme = 'http'
1035 if scheme == 'persistent-https':
1036 # If we're proxying through persistent-https, use http. The
1037 # proxy itself will do the https.
1038 if proxy:
1039 scheme = 'http'
1040 else:
1041 scheme = 'https'
1042
1043 # Parse out any authentication information using the base class
1044 host, extra_headers, _ = self.get_host_info(parse_results.netloc)
1045
1046 url = urllib.parse.urlunparse((
1047 scheme,
1048 host,
1049 parse_results.path,
1050 parse_results.params,
1051 parse_results.query,
1052 parse_results.fragment))
1053
1054 request = urllib.request.Request(url, request_body)
1055 if extra_headers is not None:
1056 for (name, header) in extra_headers:
1057 request.add_header(name, header)
1058 request.add_header('Content-Type', 'text/xml')
1059 try:
1060 response = opener.open(request)
1061 except urllib.error.HTTPError as e:
1062 if e.code == 501:
1063 # We may have been redirected through a login process
1064 # but our POST turned into a GET. Retry.
1065 response = opener.open(request)
1066 else:
1067 raise
1068
1069 p, u = xmlrpc.client.getparser()
1070 while 1:
1071 data = response.read(1024)
1072 if not data:
1073 break
1074 p.feed(data)
1075 p.close()
1076 return u.close()
1077
1078 def close(self):
1079 pass
1080