blob: b88c596d4548b842687b52161a5cc80a1874ef8b [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Anthony King85b24ac2014-05-06 15:57:48 +010017import json
David Pursehouse86d973d2012-08-24 10:21:02 +090018import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070019from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import os
21import re
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070022import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023import subprocess
24import sys
Dan Willemsen0745bb22015-08-17 13:41:45 -070025import tempfile
Shawn O. Pearcef6906872009-04-18 10:49:00 -070026import time
David Pursehouse59bbb582013-05-17 10:49:33 +090027
28from pyversion import is_python3
29if is_python3():
Dan Willemsen0745bb22015-08-17 13:41:45 -070030 import http.cookiejar as cookielib
31 import urllib.error
Chirayu Desai217ea7d2013-03-01 19:14:38 +053032 import urllib.parse
Dan Willemsen0745bb22015-08-17 13:41:45 -070033 import urllib.request
David Pursehouse59bbb582013-05-17 10:49:33 +090034 import xmlrpc.client
35else:
Dan Willemsen0745bb22015-08-17 13:41:45 -070036 import cookielib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053037 import imp
Dan Willemsen0745bb22015-08-17 13:41:45 -070038 import urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053039 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090040 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053041 urllib = imp.new_module('urllib')
Dan Willemsen0745bb22015-08-17 13:41:45 -070042 urllib.error = urllib2
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053043 urllib.parse = urlparse
Dan Willemsen0745bb22015-08-17 13:41:45 -070044 urllib.request = urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053045 xmlrpc = imp.new_module('xmlrpc')
46 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070047
Roy Lee18afd7f2010-05-09 04:32:08 +080048try:
49 import threading as _threading
50except ImportError:
51 import dummy_threading as _threading
52
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070053try:
54 import resource
55 def _rlimit_nofile():
56 return resource.getrlimit(resource.RLIMIT_NOFILE)
57except ImportError:
58 def _rlimit_nofile():
59 return (256, 256)
60
Dave Borowitz18857212012-10-23 17:02:59 -070061try:
62 import multiprocessing
63except ImportError:
64 multiprocessing = None
65
David Rileye0684ad2017-04-05 00:02:59 -070066import event_log
Dave Borowitze2152672012-10-31 12:24:38 -070067from git_command import GIT, git_require
David Pursehouseba7bc732015-08-20 16:55:42 +090068from git_config import GetUrlCookieFile
David Pursehoused94aaef2012-09-07 09:52:04 +090069from git_refs import R_HEADS, HEAD
Simran Basibdb52712015-08-10 13:23:23 -070070import gitc_utils
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070071from project import Project
72from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080073from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000074from error import RepoChangedException, GitError, ManifestParseError
Renaud Paquaya65adf72016-11-03 10:37:53 -070075import platform_utils
Shawn O. Pearce350cde42009-04-16 11:21:18 -070076from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070077from progress import Progress
Conley Owens094cdbe2014-01-30 15:09:59 -080078from wrapper import Wrapper
Dan Willemsen5ea32d12015-09-08 13:27:20 -070079from manifest_xml import GitcManifest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070080
Dave Borowitz67700e92012-10-23 15:00:54 -070081_ONE_DAY_S = 24 * 60 * 60
82
Doug Andersonfc06ced2011-03-16 15:49:18 -070083class _FetchError(Exception):
84 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
85 pass
86
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080087class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080088 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070089 common = True
90 helpSummary = "Update working tree to the latest revision"
91 helpUsage = """
92%prog [<project>...]
93"""
94 helpDescription = """
95The '%prog' command synchronizes local project directories
96with the remote repositories specified in the manifest. If a local
97project does not yet exist, it will clone a new local directory from
98the remote repository and set up tracking branches as specified in
99the manifest. If the local project already exists, '%prog'
100will update the remote branches and rebase any new local changes
101on top of the new remote changes.
102
103'%prog' will synchronize all projects listed at the command
104line. Projects can be specified either by name, or by a relative
105or absolute path to the project's local directory. If no projects
106are specified, '%prog' will synchronize all projects listed in
107the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700108
109The -d/--detach option can be used to switch specified projects
110back to the manifest revision. This option is especially helpful
111if the project is currently on a topic branch, but the manifest
112revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700113
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700114The -s/--smart-sync option can be used to sync to a known good
115build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200116manifest. The -t/--smart-tag option is similar and allows you to
117specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700118
David Pursehousecf76b1b2012-09-14 10:31:42 +0900119The -u/--manifest-server-username and -p/--manifest-server-password
120options can be used to specify a username and password to authenticate
121with the manifest server when using the -s or -t option.
122
123If -u and -p are not specified when using the -s or -t option, '%prog'
124will attempt to read authentication credentials for the manifest server
125from the user's .netrc file.
126
127'%prog' will not use authentication credentials from -u/-p or .netrc
128if the manifest server specified in the manifest file already includes
129credentials.
130
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500131The -f/--force-broken option can be used to proceed with syncing
132other projects if a project sync fails.
133
Kevin Degiabaa7f32014-11-12 11:27:45 -0700134The --force-sync option can be used to overwrite existing git
135directories if they have previously been linked to a different
136object direcotry. WARNING: This may cause data to be lost since
137refs may be removed when overwriting.
138
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700139The --no-clone-bundle option disables any attempt to use
140$URL/clone.bundle to bootstrap a new Git repository from a
141resumeable bundle file on a content delivery network. This
142may be necessary if there are problems with the local Python
143HTTP client or proxy configuration, but the Git binary works.
144
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800145The --fetch-submodules option enables fetching Git submodules
146of a project from server.
147
David Pursehousef2fad612015-01-29 14:36:28 +0900148The -c/--current-branch option can be used to only fetch objects that
149are on the branch specified by a project's revision.
150
David Pursehouseb1553542014-09-04 21:28:09 +0900151The --optimized-fetch option can be used to only fetch projects that
152are fixed to a sha1 revision if the sha1 revision does not already
153exist locally.
154
David Pursehouse74cfd272015-10-14 10:50:15 +0900155The --prune option can be used to remove any refs that no longer
156exist on the remote.
157
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700158SSH Connections
159---------------
160
161If at least one project remote URL uses an SSH connection (ssh://,
162git+ssh://, or user@host:path syntax) repo will automatically
163enable the SSH ControlMaster option when connecting to that host.
164This feature permits other projects in the same '%prog' session to
165reuse the same SSH tunnel, saving connection setup overheads.
166
167To disable this behavior on UNIX platforms, set the GIT_SSH
168environment variable to 'ssh'. For example:
169
170 export GIT_SSH=ssh
171 %prog
172
173Compatibility
174~~~~~~~~~~~~~
175
176This feature is automatically disabled on Windows, due to the lack
177of UNIX domain socket support.
178
179This feature is not compatible with url.insteadof rewrites in the
180user's ~/.gitconfig. '%prog' is currently not able to perform the
181rewrite early enough to establish the ControlMaster tunnel.
182
183If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
184later is required to fix a server side protocol bug.
185
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700186"""
187
Nico Sallembien6623b212010-05-11 12:57:01 -0700188 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000189 try:
190 self.jobs = self.manifest.default.sync_j
191 except ManifestParseError:
192 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700193
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500194 p.add_option('-f', '--force-broken',
195 dest='force_broken', action='store_true',
196 help="continue sync even if a project fails to sync")
Kevin Degiabaa7f32014-11-12 11:27:45 -0700197 p.add_option('--force-sync',
198 dest='force_sync', action='store_true',
199 help="overwrite an existing git directory if it needs to "
200 "point to a different object directory. WARNING: this "
201 "may cause loss of data")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900202 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700203 dest='local_only', action='store_true',
204 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900205 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700206 dest='network_only', action='store_true',
207 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900208 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700209 dest='detach_head', action='store_true',
210 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900211 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700212 dest='current_branch_only', action='store_true',
213 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900214 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700215 dest='quiet', action='store_true',
216 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900217 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800218 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700219 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500220 p.add_option('-m', '--manifest-name',
221 dest='manifest_name',
222 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700223 p.add_option('--no-clone-bundle',
224 dest='no_clone_bundle', action='store_true',
225 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800226 p.add_option('-u', '--manifest-server-username', action='store',
227 dest='manifest_server_username',
228 help='username to authenticate with the manifest server')
229 p.add_option('-p', '--manifest-server-password', action='store',
230 dest='manifest_server_password',
231 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800232 p.add_option('--fetch-submodules',
233 dest='fetch_submodules', action='store_true',
234 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700235 p.add_option('--no-tags',
236 dest='no_tags', action='store_true',
237 help="don't fetch tags")
David Pursehouseb1553542014-09-04 21:28:09 +0900238 p.add_option('--optimized-fetch',
239 dest='optimized_fetch', action='store_true',
240 help='only fetch projects fixed to sha1 if revision does not exist locally')
David Pursehouse74cfd272015-10-14 10:50:15 +0900241 p.add_option('--prune', dest='prune', action='store_true',
242 help='delete refs that no longer exist on the remote')
Nico Sallembien6623b212010-05-11 12:57:01 -0700243 if show_smart:
244 p.add_option('-s', '--smart-sync',
245 dest='smart_sync', action='store_true',
David Pursehouse79fba682016-04-13 18:03:00 +0900246 help='smart sync using manifest from the latest known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200247 p.add_option('-t', '--smart-tag',
248 dest='smart_tag', action='store',
249 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700250
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700251 g = p.add_option_group('repo Version options')
252 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700253 dest='no_repo_verify', action='store_true',
254 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700255 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800256 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700257 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700258
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500259 def _FetchProjectList(self, opt, projects, sem, *args, **kwargs):
David Pursehousec1b86a22012-11-14 11:36:51 +0900260 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800261
David James8d201162013-10-11 17:03:19 -0700262 Delegates most of the work to _FetchHelper.
263
264 Args:
265 opt: Program options returned from optparse. See _Options().
266 projects: Projects to fetch.
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500267 sem: We'll release() this semaphore when we exit so that another thread
268 can be started up.
David James89ece422014-01-09 18:51:58 -0800269 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
David James8d201162013-10-11 17:03:19 -0700270 _FetchHelper docstring for details.
271 """
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500272 try:
273 for project in projects:
274 success = self._FetchHelper(opt, project, *args, **kwargs)
275 if not success and not opt.force_broken:
276 break
277 finally:
278 sem.release()
David James8d201162013-10-11 17:03:19 -0700279
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500280 def _FetchHelper(self, opt, project, lock, fetched, pm, err_event):
David James8d201162013-10-11 17:03:19 -0700281 """Fetch git objects for a single project.
282
David Pursehousec1b86a22012-11-14 11:36:51 +0900283 Args:
284 opt: Program options returned from optparse. See _Options().
285 project: Project object for the project to fetch.
286 lock: Lock for accessing objects that are shared amongst multiple
287 _FetchHelper() threads.
288 fetched: set object that we will add project.gitdir to when we're done
289 (with our lock held).
290 pm: Instance of a Project object. We will call pm.update() (with our
291 lock held).
David Pursehousec1b86a22012-11-14 11:36:51 +0900292 err_event: We'll set this event in the case of an error (after printing
293 out info about the error).
David James8d201162013-10-11 17:03:19 -0700294
295 Returns:
296 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900297 """
298 # We'll set to true once we've locked the lock.
299 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700300
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530301 if not opt.quiet:
302 print('Fetching project %s' % project.name)
303
David Pursehousec1b86a22012-11-14 11:36:51 +0900304 # Encapsulate everything in a try/except/finally so that:
305 # - We always set err_event in the case of an exception.
306 # - We always make sure we call sem.release().
307 # - We always make sure we unlock the lock if we locked it.
David Rileye0684ad2017-04-05 00:02:59 -0700308 start = time.time()
309 success = False
David Pursehousec1b86a22012-11-14 11:36:51 +0900310 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700311 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900312 success = project.Sync_NetworkHalf(
313 quiet=opt.quiet,
314 current_branch_only=opt.current_branch_only,
Kevin Degiabaa7f32014-11-12 11:27:45 -0700315 force_sync=opt.force_sync,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700316 clone_bundle=not opt.no_clone_bundle,
David Pursehouseb1553542014-09-04 21:28:09 +0900317 no_tags=opt.no_tags, archive=self.manifest.IsArchive,
David Pursehouse74cfd272015-10-14 10:50:15 +0900318 optimized_fetch=opt.optimized_fetch,
319 prune=opt.prune)
David Pursehousec1b86a22012-11-14 11:36:51 +0900320 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700321
David Pursehousec1b86a22012-11-14 11:36:51 +0900322 # Lock around all the rest of the code, since printing, updating a set
323 # and Progress.update() are not thread safe.
324 lock.acquire()
325 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700326
David Pursehousec1b86a22012-11-14 11:36:51 +0900327 if not success:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800328 err_event.set()
Marc Herbertffb4b892017-04-04 22:03:53 -0700329 print('error: Cannot fetch %s from %s'
330 % (project.name, project.remote.url),
331 file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900332 if opt.force_broken:
333 print('warn: --force-broken, continuing to sync',
334 file=sys.stderr)
335 else:
336 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700337
David Pursehousec1b86a22012-11-14 11:36:51 +0900338 fetched.add(project.gitdir)
339 pm.update()
340 except _FetchError:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800341 pass
Dan Sandlerc5cd4332015-07-31 09:37:53 -0400342 except Exception as e:
343 print('error: Cannot fetch %s (%s: %s)' \
344 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900345 err_event.set()
346 raise
347 finally:
348 if did_lock:
349 lock.release()
David Rileye0684ad2017-04-05 00:02:59 -0700350 finish = time.time()
351 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
352 start, finish, success)
Roy Lee18afd7f2010-05-09 04:32:08 +0800353
David James8d201162013-10-11 17:03:19 -0700354 return success
355
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700356 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700357 fetched = set()
David James89ece422014-01-09 18:51:58 -0800358 lock = _threading.Lock()
Tim Schumacher913327f2017-06-05 15:01:41 +0200359 pm = Progress('Fetching projects', len(projects),
Tim Schumacher7be072e2017-06-28 18:29:23 +0200360 print_newline=not(opt.quiet),
361 always_print_percentage=opt.quiet)
Roy Lee18afd7f2010-05-09 04:32:08 +0800362
David James89ece422014-01-09 18:51:58 -0800363 objdir_project_map = dict()
364 for project in projects:
365 objdir_project_map.setdefault(project.objdir, []).append(project)
David James8d201162013-10-11 17:03:19 -0700366
David James89ece422014-01-09 18:51:58 -0800367 threads = set()
368 sem = _threading.Semaphore(self.jobs)
369 err_event = _threading.Event()
370 for project_list in objdir_project_map.values():
371 # Check for any errors before running any more tasks.
372 # ...we'll let existing threads finish, though.
373 if err_event.isSet() and not opt.force_broken:
374 break
Doug Andersonfc06ced2011-03-16 15:49:18 -0700375
David James89ece422014-01-09 18:51:58 -0800376 sem.acquire()
377 kwargs = dict(opt=opt,
378 projects=project_list,
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500379 sem=sem,
David James89ece422014-01-09 18:51:58 -0800380 lock=lock,
381 fetched=fetched,
382 pm=pm,
David James89ece422014-01-09 18:51:58 -0800383 err_event=err_event)
384 if self.jobs > 1:
David James8d201162013-10-11 17:03:19 -0700385 t = _threading.Thread(target = self._FetchProjectList,
David James89ece422014-01-09 18:51:58 -0800386 kwargs = kwargs)
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200387 # Ensure that Ctrl-C will not freeze the repo process.
388 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800389 threads.add(t)
390 t.start()
David James89ece422014-01-09 18:51:58 -0800391 else:
392 self._FetchProjectList(**kwargs)
Roy Lee18afd7f2010-05-09 04:32:08 +0800393
David James89ece422014-01-09 18:51:58 -0800394 for t in threads:
395 t.join()
Roy Lee18afd7f2010-05-09 04:32:08 +0800396
David James89ece422014-01-09 18:51:58 -0800397 # If we saw an error, exit with code 1 so that other scripts can check.
Nicolas Cornu8419ab22017-06-16 12:09:06 +0200398 if err_event.isSet() and not opt.force_broken:
David James89ece422014-01-09 18:51:58 -0800399 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
400 sys.exit(1)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700401
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700402 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700403 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700404
Julien Campergue335f5ef2013-10-16 11:02:35 +0200405 if not self.manifest.IsArchive:
406 self._GCProjects(projects)
407
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700408 return fetched
409
Dave Borowitz18857212012-10-23 17:02:59 -0700410 def _GCProjects(self, projects):
Gabe Black2ff30292014-10-09 17:54:35 -0700411 gc_gitdirs = {}
David James8d201162013-10-11 17:03:19 -0700412 for project in projects:
Gabe Black2ff30292014-10-09 17:54:35 -0700413 if len(project.manifest.GetProjectsWithName(project.name)) > 1:
414 print('Shared project %s found, disabling pruning.' % project.name)
415 project.bare_git.config('--replace-all', 'gc.pruneExpire', 'never')
416 gc_gitdirs[project.gitdir] = project.bare_git
David James8d201162013-10-11 17:03:19 -0700417
Dave Borowitze2152672012-10-31 12:24:38 -0700418 has_dash_c = git_require((1, 7, 2))
419 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700420 cpu_count = multiprocessing.cpu_count()
421 else:
422 cpu_count = 1
423 jobs = min(self.jobs, cpu_count)
424
425 if jobs < 2:
Gabe Black2ff30292014-10-09 17:54:35 -0700426 for bare_git in gc_gitdirs.values():
David James8d201162013-10-11 17:03:19 -0700427 bare_git.gc('--auto')
Dave Borowitz18857212012-10-23 17:02:59 -0700428 return
429
430 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
431
432 threads = set()
433 sem = _threading.Semaphore(jobs)
434 err_event = _threading.Event()
435
David James8d201162013-10-11 17:03:19 -0700436 def GC(bare_git):
Dave Borowitz18857212012-10-23 17:02:59 -0700437 try:
438 try:
David James8d201162013-10-11 17:03:19 -0700439 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700440 except GitError:
441 err_event.set()
442 except:
443 err_event.set()
444 raise
445 finally:
446 sem.release()
447
Gabe Black2ff30292014-10-09 17:54:35 -0700448 for bare_git in gc_gitdirs.values():
Dave Borowitz18857212012-10-23 17:02:59 -0700449 if err_event.isSet():
450 break
451 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700452 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700453 t.daemon = True
454 threads.add(t)
455 t.start()
456
457 for t in threads:
458 t.join()
459
460 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700461 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700462 sys.exit(1)
463
Tim Kilbourn07669002013-03-08 15:02:49 -0800464 def _ReloadManifest(self, manifest_name=None):
465 if manifest_name:
466 # Override calls _Unload already
467 self.manifest.Override(manifest_name)
468 else:
469 self.manifest._Unload()
470
Dan Willemsen43507912016-09-01 16:26:02 -0700471 def _DeleteProject(self, path):
472 print('Deleting obsolete path %s' % path, file=sys.stderr)
473
474 # Delete the .git directory first, so we're less likely to have a partially
475 # working git repository around. There shouldn't be any git projects here,
476 # so rmtree works.
477 try:
Renaud Paquaya65adf72016-11-03 10:37:53 -0700478 platform_utils.rmtree(os.path.join(path, '.git'))
Dan Willemsen43507912016-09-01 16:26:02 -0700479 except OSError:
480 print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr)
481 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
482 print(' remove manually, then run sync again', file=sys.stderr)
483 return -1
484
485 # Delete everything under the worktree, except for directories that contain
486 # another git project
487 dirs_to_remove = []
488 failed = False
489 for root, dirs, files in os.walk(path):
490 for f in files:
491 try:
492 os.remove(os.path.join(root, f))
493 except OSError:
494 print('Failed to remove %s' % os.path.join(root, f), file=sys.stderr)
495 failed = True
496 dirs[:] = [d for d in dirs
497 if not os.path.lexists(os.path.join(root, d, '.git'))]
498 dirs_to_remove += [os.path.join(root, d) for d in dirs
499 if os.path.join(root, d) not in dirs_to_remove]
500 for d in reversed(dirs_to_remove):
Dan Willemseneceeb1b2016-09-25 18:24:27 -0700501 if os.path.islink(d):
502 try:
503 os.remove(d)
504 except OSError:
505 print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
506 failed = True
507 elif len(os.listdir(d)) == 0:
Dan Willemsen43507912016-09-01 16:26:02 -0700508 try:
509 os.rmdir(d)
510 except OSError:
511 print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
512 failed = True
513 continue
514 if failed:
515 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
516 print(' remove manually, then run sync again', file=sys.stderr)
517 return -1
518
519 # Try deleting parent dirs if they are empty
520 project_dir = path
521 while project_dir != self.manifest.topdir:
522 if len(os.listdir(project_dir)) == 0:
523 os.rmdir(project_dir)
524 else:
525 break
526 project_dir = os.path.dirname(project_dir)
527
528 return 0
529
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700530 def UpdateProjectList(self):
531 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700532 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700533 if project.relpath:
534 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700535 file_name = 'project.list'
536 file_path = os.path.join(self.manifest.repodir, file_name)
537 old_project_paths = []
538
539 if os.path.exists(file_path):
540 fd = open(file_path, 'r')
541 try:
542 old_project_paths = fd.read().split('\n')
543 finally:
544 fd.close()
545 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700546 if not path:
547 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700548 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900549 # If the path has already been deleted, we don't need to do it
Dan Willemsen43507912016-09-01 16:26:02 -0700550 gitdir = os.path.join(self.manifest.topdir, path, '.git')
551 if os.path.exists(gitdir):
David Pursehousec1b86a22012-11-14 11:36:51 +0900552 project = Project(
553 manifest = self.manifest,
554 name = path,
555 remote = RemoteSpec('origin'),
David James8d201162013-10-11 17:03:19 -0700556 gitdir = gitdir,
557 objdir = gitdir,
David Pursehousec1b86a22012-11-14 11:36:51 +0900558 worktree = os.path.join(self.manifest.topdir, path),
559 relpath = path,
560 revisionExpr = 'HEAD',
561 revisionId = None,
562 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400563
David Pursehousec1b86a22012-11-14 11:36:51 +0900564 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900565 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900566 'are present' % project.relpath, file=sys.stderr)
567 print(' commit changes, then run sync again',
568 file=sys.stderr)
569 return -1
Dan Willemsen43507912016-09-01 16:26:02 -0700570 elif self._DeleteProject(project.worktree):
571 return -1
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700572
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700573 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700574 fd = open(file_path, 'w')
575 try:
576 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700577 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700578 finally:
579 fd.close()
580 return 0
581
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700582 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800583 if opt.jobs:
584 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700585 if self.jobs > 1:
586 soft_limit, _ = _rlimit_nofile()
587 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
588
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700589 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700590 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700591 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700592 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700593 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700594 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500595 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700596 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500597 sys.exit(1)
598 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700599 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500600 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900601 if opt.manifest_server_username or opt.manifest_server_password:
602 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700603 print('error: -u and -p may only be combined with -s or -t',
604 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900605 sys.exit(1)
606 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700607 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900608 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500609
610 if opt.manifest_name:
611 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700612
Chirayu Desaia892b102013-06-11 14:18:46 +0530613 manifest_name = opt.manifest_name
David Pursehouse59b41742015-05-07 14:36:09 +0900614 smart_sync_manifest_name = "smart_sync_override.xml"
615 smart_sync_manifest_path = os.path.join(
616 self.manifest.manifestProject.worktree, smart_sync_manifest_name)
Chirayu Desaia892b102013-06-11 14:18:46 +0530617
Victor Boivie08c880d2011-04-19 10:32:52 +0200618 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700619 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900620 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700621 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700622 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900623
624 manifest_server = self.manifest.manifest_server
David Pursehousefb99c712013-09-25 11:09:34 +0900625 if not opt.quiet:
626 print('Using manifest server %s' % manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900627
David Pursehouse86d973d2012-08-24 10:21:02 +0900628 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900629 username = None
630 password = None
631 if opt.manifest_server_username and opt.manifest_server_password:
632 username = opt.manifest_server_username
633 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900634 else:
635 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900636 info = netrc.netrc()
637 except IOError:
David Pursehouseba7bc732015-08-20 16:55:42 +0900638 # .netrc file does not exist or could not be opened
639 pass
David Pursehouse86d973d2012-08-24 10:21:02 +0900640 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900641 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530642 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900643 if parse_result.hostname:
David Pursehouseba7bc732015-08-20 16:55:42 +0900644 auth = info.authenticators(parse_result.hostname)
645 if auth:
646 username, _account, password = auth
647 else:
648 print('No credentials found for %s in .netrc'
649 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700650 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700651 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900652
653 if (username and password):
654 manifest_server = manifest_server.replace('://', '://%s:%s@' %
655 (username, password),
656 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900657
Dan Willemsen0745bb22015-08-17 13:41:45 -0700658 transport = PersistentTransport(manifest_server)
659 if manifest_server.startswith('persistent-'):
660 manifest_server = manifest_server[len('persistent-'):]
661
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700662 try:
Dan Willemsen0745bb22015-08-17 13:41:45 -0700663 server = xmlrpc.client.Server(manifest_server, transport=transport)
Victor Boivie08c880d2011-04-19 10:32:52 +0200664 if opt.smart_sync:
665 p = self.manifest.manifestProject
666 b = p.GetBranch(p.CurrentBranch)
667 branch = b.merge
668 if branch.startswith(R_HEADS):
669 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700670
Victor Boivie08c880d2011-04-19 10:32:52 +0200671 env = os.environ.copy()
Jeff Davidson5cf16602014-10-02 10:13:38 -0700672 if 'SYNC_TARGET' in env:
673 target = env['SYNC_TARGET']
674 [success, manifest_str] = server.GetApprovedManifest(branch, target)
675 elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200676 target = '%s-%s' % (env['TARGET_PRODUCT'],
677 env['TARGET_BUILD_VARIANT'])
678 [success, manifest_str] = server.GetApprovedManifest(branch, target)
679 else:
680 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700681 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200682 assert(opt.smart_tag)
683 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700684
685 if success:
David Pursehouse59b41742015-05-07 14:36:09 +0900686 manifest_name = smart_sync_manifest_name
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700687 try:
David Pursehouse59b41742015-05-07 14:36:09 +0900688 f = open(smart_sync_manifest_path, 'w')
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700689 try:
690 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700691 finally:
692 f.close()
David Pursehouse727cc3e2015-05-07 14:16:49 +0900693 except IOError as e:
694 print('error: cannot write manifest to %s:\n%s'
David Pursehouse59b41742015-05-07 14:36:09 +0900695 % (smart_sync_manifest_path, e),
Sarah Owenscecd1d82012-11-01 22:59:27 -0700696 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700697 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100698 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700699 else:
David Pursehouse351fe2c2013-09-25 17:54:26 +0900700 print('error: manifest server RPC call failed: %s' %
701 manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700702 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530703 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700704 print('error: cannot connect to manifest server %s:\n%s'
705 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900706 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530707 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700708 print('error: cannot connect to manifest server %s:\n%d %s'
709 % (self.manifest.manifest_server, e.errcode, e.errmsg),
710 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700711 sys.exit(1)
David Pursehouse59b41742015-05-07 14:36:09 +0900712 else: # Not smart sync or smart tag mode
713 if os.path.isfile(smart_sync_manifest_path):
714 try:
715 os.remove(smart_sync_manifest_path)
716 except OSError as e:
717 print('error: failed to remove existing smart sync override manifest: %s' %
718 e, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700719
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700720 rp = self.manifest.repoProject
721 rp.PreSync()
722
723 mp = self.manifest.manifestProject
724 mp.PreSync()
725
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800726 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700727 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800728
Nico Sallembien9bb18162009-12-07 15:38:01 -0800729 if not opt.local_only:
David Rileye0684ad2017-04-05 00:02:59 -0700730 start = time.time()
731 success = mp.Sync_NetworkHalf(quiet=opt.quiet,
732 current_branch_only=opt.current_branch_only,
733 no_tags=opt.no_tags,
734 optimized_fetch=opt.optimized_fetch,
735 submodules=self.manifest.HasSubmodules)
736 finish = time.time()
737 self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
738 start, finish, success)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800739
740 if mp.HasChanges:
741 syncbuf = SyncBuffer(mp.config)
David Rileye0684ad2017-04-05 00:02:59 -0700742 start = time.time()
Martin Kellye4e94d22017-03-21 16:05:12 -0700743 mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
David Rileye0684ad2017-04-05 00:02:59 -0700744 clean = syncbuf.Finish()
745 self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
746 start, time.time(), clean)
747 if not clean:
Nico Sallembien9bb18162009-12-07 15:38:01 -0800748 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100749 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700750 if opt.jobs is None:
751 self.jobs = self.manifest.default.sync_j
Simran Basib9a1b732015-08-20 12:19:28 -0700752
Simran Basib9a1b732015-08-20 12:19:28 -0700753 if self.gitc_manifest:
754 gitc_manifest_projects = self.GetProjects(args,
Simran Basib9a1b732015-08-20 12:19:28 -0700755 missing_ok=True)
756 gitc_projects = []
757 opened_projects = []
758 for project in gitc_manifest_projects:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700759 if project.relpath in self.gitc_manifest.paths and \
760 self.gitc_manifest.paths[project.relpath].old_revision:
761 opened_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700762 else:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700763 gitc_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700764
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700765 if not args:
766 gitc_projects = None
767
768 if gitc_projects != [] and not opt.local_only:
Simran Basib9a1b732015-08-20 12:19:28 -0700769 print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700770 manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
771 if manifest_name:
772 manifest.Override(manifest_name)
773 else:
774 manifest.Override(self.manifest.manifestFile)
775 gitc_utils.generate_gitc_manifest(self.gitc_manifest,
776 manifest,
Simran Basib9a1b732015-08-20 12:19:28 -0700777 gitc_projects)
778 print('GITC client successfully synced.')
779
780 # The opened projects need to be synced as normal, therefore we
781 # generate a new args list to represent the opened projects.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700782 # TODO: make this more reliable -- if there's a project name/path overlap,
783 # this may choose the wrong project.
David Pursehouse3bcd3052017-07-10 22:42:22 +0900784 args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
785 for path in opened_projects]
Simran Basib9a1b732015-08-20 12:19:28 -0700786 if not args:
787 return
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800788 all_projects = self.GetProjects(args,
789 missing_ok=True,
790 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700791
Dave Borowitz67700e92012-10-23 15:00:54 -0700792 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700793 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700794 to_fetch = []
795 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700796 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700797 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900798 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700799 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700800
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800801 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700802 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700803 if opt.network_only:
804 # bail out now; the rest touches the working tree
805 return
806
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800807 # Iteratively fetch missing and/or nested unregistered submodules
808 previously_missing_set = set()
809 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100810 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800811 all_projects = self.GetProjects(args,
812 missing_ok=True,
813 submodules_ok=opt.fetch_submodules)
814 missing = []
815 for project in all_projects:
816 if project.gitdir not in fetched:
817 missing.append(project)
818 if not missing:
819 break
820 # Stop us from non-stopped fetching actually-missing repos: If set of
821 # missing repos has not been changed from last fetch, we break.
822 missing_set = set(p.name for p in missing)
823 if previously_missing_set == missing_set:
824 break
825 previously_missing_set = missing_set
826 fetched.update(self._Fetch(missing, opt))
827
Julien Campergue335f5ef2013-10-16 11:02:35 +0200828 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700829 # bail out now, we have no working tree
830 return
831
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700832 if self.UpdateProjectList():
833 sys.exit(1)
834
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700835 syncbuf = SyncBuffer(mp.config,
836 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900837 pm = Progress('Syncing work tree', len(all_projects))
838 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700839 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800840 if project.worktree:
David Rileye0684ad2017-04-05 00:02:59 -0700841 start = time.time()
Kevin Degiabaa7f32014-11-12 11:27:45 -0700842 project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
David Rileye0684ad2017-04-05 00:02:59 -0700843 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
844 start, time.time(), syncbuf.Recently())
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700845 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700846 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700847 if not syncbuf.Finish():
848 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700849
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700850 # If there's a notice that's supposed to print at the end of the sync, print
851 # it now...
852 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700853 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700854
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700855def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -0800856 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -0700857 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700858 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -0800859 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700860 if project.Exists:
861 project.PostRepoUpgrade()
862
863def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
864 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700865 print('info: A new version of repo is available', file=sys.stderr)
866 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700867 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700868 syncbuf = SyncBuffer(rp.config)
869 rp.Sync_LocalHalf(syncbuf)
870 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700871 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700872 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700873 raise RepoChangedException(['--repo-upgraded'])
874 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700875 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700876 else:
877 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700878 print('repo version %s is current' % rp.work_git.describe(HEAD),
879 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700880
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700881def _VerifyTag(project):
882 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
883 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700884 print('warning: GnuPG was not available during last "repo init"\n'
885 'warning: Cannot automatically authenticate repo."""',
886 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700887 return True
888
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700889 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700890 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700891 except GitError:
892 cur = None
893
894 if not cur \
895 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700896 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700897 if rev.startswith(R_HEADS):
898 rev = rev[len(R_HEADS):]
899
Sarah Owenscecd1d82012-11-01 22:59:27 -0700900 print(file=sys.stderr)
901 print("warning: project '%s' branch '%s' is not signed"
902 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700903 return False
904
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800905 env = os.environ.copy()
906 env['GIT_DIR'] = project.gitdir.encode()
907 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700908
909 cmd = [GIT, 'tag', '-v', cur]
910 proc = subprocess.Popen(cmd,
911 stdout = subprocess.PIPE,
912 stderr = subprocess.PIPE,
913 env = env)
914 out = proc.stdout.read()
915 proc.stdout.close()
916
917 err = proc.stderr.read()
918 proc.stderr.close()
919
920 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700921 print(file=sys.stderr)
922 print(out, file=sys.stderr)
923 print(err, file=sys.stderr)
924 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700925 return False
926 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700927
David Rileye0684ad2017-04-05 00:02:59 -0700928
Dave Borowitz67700e92012-10-23 15:00:54 -0700929class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700930 _ALPHA = 0.5
931
Dave Borowitz67700e92012-10-23 15:00:54 -0700932 def __init__(self, manifest):
Anthony King85b24ac2014-05-06 15:57:48 +0100933 self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
Dave Borowitz67700e92012-10-23 15:00:54 -0700934 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700935 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700936
937 def Get(self, project):
938 self._Load()
939 return self._times.get(project.name, _ONE_DAY_S)
940
941 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700942 self._Load()
943 name = project.name
944 old = self._times.get(name, t)
945 self._seen.add(name)
946 a = self._ALPHA
947 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700948
949 def _Load(self):
950 if self._times is None:
951 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100952 f = open(self._path)
Dave Borowitz67700e92012-10-23 15:00:54 -0700953 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100954 self._times = json.load(f)
955 finally:
956 f.close()
957 except (IOError, ValueError):
958 try:
959 os.remove(self._path)
960 except OSError:
961 pass
962 self._times = {}
Dave Borowitz67700e92012-10-23 15:00:54 -0700963
964 def Save(self):
965 if self._times is None:
966 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700967
968 to_delete = []
969 for name in self._times:
970 if name not in self._seen:
971 to_delete.append(name)
972 for name in to_delete:
973 del self._times[name]
974
Dave Borowitz67700e92012-10-23 15:00:54 -0700975 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100976 f = open(self._path, 'w')
Dave Borowitz67700e92012-10-23 15:00:54 -0700977 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100978 json.dump(self._times, f, indent=2)
979 finally:
980 f.close()
981 except (IOError, TypeError):
982 try:
983 os.remove(self._path)
984 except OSError:
985 pass
Dan Willemsen0745bb22015-08-17 13:41:45 -0700986
987# This is a replacement for xmlrpc.client.Transport using urllib2
988# and supporting persistent-http[s]. It cannot change hosts from
989# request to request like the normal transport, the real url
990# is passed during initialization.
991class PersistentTransport(xmlrpc.client.Transport):
992 def __init__(self, orig_host):
993 self.orig_host = orig_host
994
995 def request(self, host, handler, request_body, verbose=False):
996 with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
997 # Python doesn't understand cookies with the #HttpOnly_ prefix
998 # Since we're only using them for HTTP, copy the file temporarily,
999 # stripping those prefixes away.
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001000 if cookiefile:
1001 tmpcookiefile = tempfile.NamedTemporaryFile()
David Pursehouse4c5f74e2015-10-02 11:10:10 +09001002 tmpcookiefile.write("# HTTP Cookie File")
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001003 try:
1004 with open(cookiefile) as f:
1005 for line in f:
1006 if line.startswith("#HttpOnly_"):
1007 line = line[len("#HttpOnly_"):]
1008 tmpcookiefile.write(line)
1009 tmpcookiefile.flush()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001010
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001011 cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
David Pursehouseb1ad2192015-09-30 10:35:43 +09001012 try:
1013 cookiejar.load()
1014 except cookielib.LoadError:
1015 cookiejar = cookielib.CookieJar()
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001016 finally:
1017 tmpcookiefile.close()
1018 else:
1019 cookiejar = cookielib.CookieJar()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001020
1021 proxyhandler = urllib.request.ProxyHandler
1022 if proxy:
1023 proxyhandler = urllib.request.ProxyHandler({
1024 "http": proxy,
1025 "https": proxy })
1026
1027 opener = urllib.request.build_opener(
1028 urllib.request.HTTPCookieProcessor(cookiejar),
1029 proxyhandler)
1030
1031 url = urllib.parse.urljoin(self.orig_host, handler)
1032 parse_results = urllib.parse.urlparse(url)
1033
1034 scheme = parse_results.scheme
1035 if scheme == 'persistent-http':
1036 scheme = 'http'
1037 if scheme == 'persistent-https':
1038 # If we're proxying through persistent-https, use http. The
1039 # proxy itself will do the https.
1040 if proxy:
1041 scheme = 'http'
1042 else:
1043 scheme = 'https'
1044
1045 # Parse out any authentication information using the base class
1046 host, extra_headers, _ = self.get_host_info(parse_results.netloc)
1047
1048 url = urllib.parse.urlunparse((
1049 scheme,
1050 host,
1051 parse_results.path,
1052 parse_results.params,
1053 parse_results.query,
1054 parse_results.fragment))
1055
1056 request = urllib.request.Request(url, request_body)
1057 if extra_headers is not None:
1058 for (name, header) in extra_headers:
1059 request.add_header(name, header)
1060 request.add_header('Content-Type', 'text/xml')
1061 try:
1062 response = opener.open(request)
1063 except urllib.error.HTTPError as e:
1064 if e.code == 501:
1065 # We may have been redirected through a login process
1066 # but our POST turned into a GET. Retry.
1067 response = opener.open(request)
1068 else:
1069 raise
1070
1071 p, u = xmlrpc.client.getparser()
1072 while 1:
1073 data = response.read(1024)
1074 if not data:
1075 break
1076 p.feed(data)
1077 p.close()
1078 return u.close()
1079
1080 def close(self):
1081 pass
1082