blob: ef023274460ce23b0d7b80aa981830592c87aa37 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Anthony King85b24ac2014-05-06 15:57:48 +010017import json
David Pursehouse86d973d2012-08-24 10:21:02 +090018import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070019from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import os
21import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Dan Willemsen0745bb22015-08-17 13:41:45 -070026import tempfile
Shawn O. Pearcef6906872009-04-18 10:49:00 -070027import time
David Pursehouse59bbb582013-05-17 10:49:33 +090028
29from pyversion import is_python3
30if is_python3():
Dan Willemsen0745bb22015-08-17 13:41:45 -070031 import http.cookiejar as cookielib
32 import urllib.error
Chirayu Desai217ea7d2013-03-01 19:14:38 +053033 import urllib.parse
Dan Willemsen0745bb22015-08-17 13:41:45 -070034 import urllib.request
David Pursehouse59bbb582013-05-17 10:49:33 +090035 import xmlrpc.client
36else:
Dan Willemsen0745bb22015-08-17 13:41:45 -070037 import cookielib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053038 import imp
Dan Willemsen0745bb22015-08-17 13:41:45 -070039 import urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053040 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090041 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053042 urllib = imp.new_module('urllib')
Dan Willemsen0745bb22015-08-17 13:41:45 -070043 urllib.error = urllib2
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053044 urllib.parse = urlparse
Dan Willemsen0745bb22015-08-17 13:41:45 -070045 urllib.request = urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053046 xmlrpc = imp.new_module('xmlrpc')
47 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070048
Roy Lee18afd7f2010-05-09 04:32:08 +080049try:
50 import threading as _threading
51except ImportError:
52 import dummy_threading as _threading
53
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070054try:
55 import resource
56 def _rlimit_nofile():
57 return resource.getrlimit(resource.RLIMIT_NOFILE)
58except ImportError:
59 def _rlimit_nofile():
60 return (256, 256)
61
Dave Borowitz18857212012-10-23 17:02:59 -070062try:
63 import multiprocessing
64except ImportError:
65 multiprocessing = None
66
David Rileye0684ad2017-04-05 00:02:59 -070067import event_log
Dave Borowitze2152672012-10-31 12:24:38 -070068from git_command import GIT, git_require
David Pursehouseba7bc732015-08-20 16:55:42 +090069from git_config import GetUrlCookieFile
David Pursehoused94aaef2012-09-07 09:52:04 +090070from git_refs import R_HEADS, HEAD
Simran Basibdb52712015-08-10 13:23:23 -070071import gitc_utils
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070072from project import Project
73from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080074from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000075from error import RepoChangedException, GitError, ManifestParseError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070076from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070077from progress import Progress
Conley Owens094cdbe2014-01-30 15:09:59 -080078from wrapper import Wrapper
Dan Willemsen5ea32d12015-09-08 13:27:20 -070079from manifest_xml import GitcManifest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070080
Dave Borowitz67700e92012-10-23 15:00:54 -070081_ONE_DAY_S = 24 * 60 * 60
82
Doug Andersonfc06ced2011-03-16 15:49:18 -070083class _FetchError(Exception):
84 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
85 pass
86
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080087class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080088 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070089 common = True
90 helpSummary = "Update working tree to the latest revision"
91 helpUsage = """
92%prog [<project>...]
93"""
94 helpDescription = """
95The '%prog' command synchronizes local project directories
96with the remote repositories specified in the manifest. If a local
97project does not yet exist, it will clone a new local directory from
98the remote repository and set up tracking branches as specified in
99the manifest. If the local project already exists, '%prog'
100will update the remote branches and rebase any new local changes
101on top of the new remote changes.
102
103'%prog' will synchronize all projects listed at the command
104line. Projects can be specified either by name, or by a relative
105or absolute path to the project's local directory. If no projects
106are specified, '%prog' will synchronize all projects listed in
107the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700108
109The -d/--detach option can be used to switch specified projects
110back to the manifest revision. This option is especially helpful
111if the project is currently on a topic branch, but the manifest
112revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700113
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700114The -s/--smart-sync option can be used to sync to a known good
115build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200116manifest. The -t/--smart-tag option is similar and allows you to
117specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700118
David Pursehousecf76b1b2012-09-14 10:31:42 +0900119The -u/--manifest-server-username and -p/--manifest-server-password
120options can be used to specify a username and password to authenticate
121with the manifest server when using the -s or -t option.
122
123If -u and -p are not specified when using the -s or -t option, '%prog'
124will attempt to read authentication credentials for the manifest server
125from the user's .netrc file.
126
127'%prog' will not use authentication credentials from -u/-p or .netrc
128if the manifest server specified in the manifest file already includes
129credentials.
130
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500131The -f/--force-broken option can be used to proceed with syncing
132other projects if a project sync fails.
133
Kevin Degiabaa7f32014-11-12 11:27:45 -0700134The --force-sync option can be used to overwrite existing git
135directories if they have previously been linked to a different
136object direcotry. WARNING: This may cause data to be lost since
137refs may be removed when overwriting.
138
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700139The --no-clone-bundle option disables any attempt to use
140$URL/clone.bundle to bootstrap a new Git repository from a
141resumeable bundle file on a content delivery network. This
142may be necessary if there are problems with the local Python
143HTTP client or proxy configuration, but the Git binary works.
144
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800145The --fetch-submodules option enables fetching Git submodules
146of a project from server.
147
David Pursehousef2fad612015-01-29 14:36:28 +0900148The -c/--current-branch option can be used to only fetch objects that
149are on the branch specified by a project's revision.
150
David Pursehouseb1553542014-09-04 21:28:09 +0900151The --optimized-fetch option can be used to only fetch projects that
152are fixed to a sha1 revision if the sha1 revision does not already
153exist locally.
154
David Pursehouse74cfd272015-10-14 10:50:15 +0900155The --prune option can be used to remove any refs that no longer
156exist on the remote.
157
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700158SSH Connections
159---------------
160
161If at least one project remote URL uses an SSH connection (ssh://,
162git+ssh://, or user@host:path syntax) repo will automatically
163enable the SSH ControlMaster option when connecting to that host.
164This feature permits other projects in the same '%prog' session to
165reuse the same SSH tunnel, saving connection setup overheads.
166
167To disable this behavior on UNIX platforms, set the GIT_SSH
168environment variable to 'ssh'. For example:
169
170 export GIT_SSH=ssh
171 %prog
172
173Compatibility
174~~~~~~~~~~~~~
175
176This feature is automatically disabled on Windows, due to the lack
177of UNIX domain socket support.
178
179This feature is not compatible with url.insteadof rewrites in the
180user's ~/.gitconfig. '%prog' is currently not able to perform the
181rewrite early enough to establish the ControlMaster tunnel.
182
183If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
184later is required to fix a server side protocol bug.
185
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700186"""
187
Nico Sallembien6623b212010-05-11 12:57:01 -0700188 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000189 try:
190 self.jobs = self.manifest.default.sync_j
191 except ManifestParseError:
192 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700193
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500194 p.add_option('-f', '--force-broken',
195 dest='force_broken', action='store_true',
196 help="continue sync even if a project fails to sync")
Kevin Degiabaa7f32014-11-12 11:27:45 -0700197 p.add_option('--force-sync',
198 dest='force_sync', action='store_true',
199 help="overwrite an existing git directory if it needs to "
200 "point to a different object directory. WARNING: this "
201 "may cause loss of data")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900202 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700203 dest='local_only', action='store_true',
204 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900205 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700206 dest='network_only', action='store_true',
207 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900208 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700209 dest='detach_head', action='store_true',
210 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900211 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700212 dest='current_branch_only', action='store_true',
213 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900214 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700215 dest='quiet', action='store_true',
216 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900217 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800218 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700219 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500220 p.add_option('-m', '--manifest-name',
221 dest='manifest_name',
222 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700223 p.add_option('--no-clone-bundle',
224 dest='no_clone_bundle', action='store_true',
225 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800226 p.add_option('-u', '--manifest-server-username', action='store',
227 dest='manifest_server_username',
228 help='username to authenticate with the manifest server')
229 p.add_option('-p', '--manifest-server-password', action='store',
230 dest='manifest_server_password',
231 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800232 p.add_option('--fetch-submodules',
233 dest='fetch_submodules', action='store_true',
234 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700235 p.add_option('--no-tags',
236 dest='no_tags', action='store_true',
237 help="don't fetch tags")
David Pursehouseb1553542014-09-04 21:28:09 +0900238 p.add_option('--optimized-fetch',
239 dest='optimized_fetch', action='store_true',
240 help='only fetch projects fixed to sha1 if revision does not exist locally')
David Pursehouse74cfd272015-10-14 10:50:15 +0900241 p.add_option('--prune', dest='prune', action='store_true',
242 help='delete refs that no longer exist on the remote')
Nico Sallembien6623b212010-05-11 12:57:01 -0700243 if show_smart:
244 p.add_option('-s', '--smart-sync',
245 dest='smart_sync', action='store_true',
David Pursehouse79fba682016-04-13 18:03:00 +0900246 help='smart sync using manifest from the latest known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200247 p.add_option('-t', '--smart-tag',
248 dest='smart_tag', action='store',
249 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700250
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700251 g = p.add_option_group('repo Version options')
252 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700253 dest='no_repo_verify', action='store_true',
254 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700255 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800256 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700257 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700258
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500259 def _FetchProjectList(self, opt, projects, sem, *args, **kwargs):
David Pursehousec1b86a22012-11-14 11:36:51 +0900260 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800261
David James8d201162013-10-11 17:03:19 -0700262 Delegates most of the work to _FetchHelper.
263
264 Args:
265 opt: Program options returned from optparse. See _Options().
266 projects: Projects to fetch.
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500267 sem: We'll release() this semaphore when we exit so that another thread
268 can be started up.
David James89ece422014-01-09 18:51:58 -0800269 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
David James8d201162013-10-11 17:03:19 -0700270 _FetchHelper docstring for details.
271 """
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500272 try:
273 for project in projects:
274 success = self._FetchHelper(opt, project, *args, **kwargs)
275 if not success and not opt.force_broken:
276 break
277 finally:
278 sem.release()
David James8d201162013-10-11 17:03:19 -0700279
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500280 def _FetchHelper(self, opt, project, lock, fetched, pm, err_event):
David James8d201162013-10-11 17:03:19 -0700281 """Fetch git objects for a single project.
282
David Pursehousec1b86a22012-11-14 11:36:51 +0900283 Args:
284 opt: Program options returned from optparse. See _Options().
285 project: Project object for the project to fetch.
286 lock: Lock for accessing objects that are shared amongst multiple
287 _FetchHelper() threads.
288 fetched: set object that we will add project.gitdir to when we're done
289 (with our lock held).
290 pm: Instance of a Project object. We will call pm.update() (with our
291 lock held).
David Pursehousec1b86a22012-11-14 11:36:51 +0900292 err_event: We'll set this event in the case of an error (after printing
293 out info about the error).
David James8d201162013-10-11 17:03:19 -0700294
295 Returns:
296 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900297 """
298 # We'll set to true once we've locked the lock.
299 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700300
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530301 if not opt.quiet:
302 print('Fetching project %s' % project.name)
303
David Pursehousec1b86a22012-11-14 11:36:51 +0900304 # Encapsulate everything in a try/except/finally so that:
305 # - We always set err_event in the case of an exception.
306 # - We always make sure we call sem.release().
307 # - We always make sure we unlock the lock if we locked it.
David Rileye0684ad2017-04-05 00:02:59 -0700308 start = time.time()
309 success = False
David Pursehousec1b86a22012-11-14 11:36:51 +0900310 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700311 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900312 success = project.Sync_NetworkHalf(
313 quiet=opt.quiet,
314 current_branch_only=opt.current_branch_only,
Kevin Degiabaa7f32014-11-12 11:27:45 -0700315 force_sync=opt.force_sync,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700316 clone_bundle=not opt.no_clone_bundle,
David Pursehouseb1553542014-09-04 21:28:09 +0900317 no_tags=opt.no_tags, archive=self.manifest.IsArchive,
David Pursehouse74cfd272015-10-14 10:50:15 +0900318 optimized_fetch=opt.optimized_fetch,
319 prune=opt.prune)
David Pursehousec1b86a22012-11-14 11:36:51 +0900320 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700321
David Pursehousec1b86a22012-11-14 11:36:51 +0900322 # Lock around all the rest of the code, since printing, updating a set
323 # and Progress.update() are not thread safe.
324 lock.acquire()
325 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700326
David Pursehousec1b86a22012-11-14 11:36:51 +0900327 if not success:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800328 err_event.set()
Marc Herbertffb4b892017-04-04 22:03:53 -0700329 print('error: Cannot fetch %s from %s'
330 % (project.name, project.remote.url),
331 file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900332 if opt.force_broken:
333 print('warn: --force-broken, continuing to sync',
334 file=sys.stderr)
335 else:
336 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700337
David Pursehousec1b86a22012-11-14 11:36:51 +0900338 fetched.add(project.gitdir)
339 pm.update()
340 except _FetchError:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800341 pass
Dan Sandlerc5cd4332015-07-31 09:37:53 -0400342 except Exception as e:
343 print('error: Cannot fetch %s (%s: %s)' \
344 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900345 err_event.set()
346 raise
347 finally:
348 if did_lock:
349 lock.release()
David Rileye0684ad2017-04-05 00:02:59 -0700350 finish = time.time()
351 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
352 start, finish, success)
Roy Lee18afd7f2010-05-09 04:32:08 +0800353
David James8d201162013-10-11 17:03:19 -0700354 return success
355
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700356 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700357 fetched = set()
David James89ece422014-01-09 18:51:58 -0800358 lock = _threading.Lock()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700359 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800360
David James89ece422014-01-09 18:51:58 -0800361 objdir_project_map = dict()
362 for project in projects:
363 objdir_project_map.setdefault(project.objdir, []).append(project)
David James8d201162013-10-11 17:03:19 -0700364
David James89ece422014-01-09 18:51:58 -0800365 threads = set()
366 sem = _threading.Semaphore(self.jobs)
367 err_event = _threading.Event()
368 for project_list in objdir_project_map.values():
369 # Check for any errors before running any more tasks.
370 # ...we'll let existing threads finish, though.
371 if err_event.isSet() and not opt.force_broken:
372 break
Doug Andersonfc06ced2011-03-16 15:49:18 -0700373
David James89ece422014-01-09 18:51:58 -0800374 sem.acquire()
375 kwargs = dict(opt=opt,
376 projects=project_list,
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500377 sem=sem,
David James89ece422014-01-09 18:51:58 -0800378 lock=lock,
379 fetched=fetched,
380 pm=pm,
David James89ece422014-01-09 18:51:58 -0800381 err_event=err_event)
382 if self.jobs > 1:
David James8d201162013-10-11 17:03:19 -0700383 t = _threading.Thread(target = self._FetchProjectList,
David James89ece422014-01-09 18:51:58 -0800384 kwargs = kwargs)
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200385 # Ensure that Ctrl-C will not freeze the repo process.
386 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800387 threads.add(t)
388 t.start()
David James89ece422014-01-09 18:51:58 -0800389 else:
390 self._FetchProjectList(**kwargs)
Roy Lee18afd7f2010-05-09 04:32:08 +0800391
David James89ece422014-01-09 18:51:58 -0800392 for t in threads:
393 t.join()
Roy Lee18afd7f2010-05-09 04:32:08 +0800394
David James89ece422014-01-09 18:51:58 -0800395 # If we saw an error, exit with code 1 so that other scripts can check.
396 if err_event.isSet():
397 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
398 sys.exit(1)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700399
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700400 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700401 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700402
Julien Campergue335f5ef2013-10-16 11:02:35 +0200403 if not self.manifest.IsArchive:
404 self._GCProjects(projects)
405
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700406 return fetched
407
Dave Borowitz18857212012-10-23 17:02:59 -0700408 def _GCProjects(self, projects):
Gabe Black2ff30292014-10-09 17:54:35 -0700409 gc_gitdirs = {}
David James8d201162013-10-11 17:03:19 -0700410 for project in projects:
Gabe Black2ff30292014-10-09 17:54:35 -0700411 if len(project.manifest.GetProjectsWithName(project.name)) > 1:
412 print('Shared project %s found, disabling pruning.' % project.name)
413 project.bare_git.config('--replace-all', 'gc.pruneExpire', 'never')
414 gc_gitdirs[project.gitdir] = project.bare_git
David James8d201162013-10-11 17:03:19 -0700415
Dave Borowitze2152672012-10-31 12:24:38 -0700416 has_dash_c = git_require((1, 7, 2))
417 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700418 cpu_count = multiprocessing.cpu_count()
419 else:
420 cpu_count = 1
421 jobs = min(self.jobs, cpu_count)
422
423 if jobs < 2:
Gabe Black2ff30292014-10-09 17:54:35 -0700424 for bare_git in gc_gitdirs.values():
David James8d201162013-10-11 17:03:19 -0700425 bare_git.gc('--auto')
Dave Borowitz18857212012-10-23 17:02:59 -0700426 return
427
428 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
429
430 threads = set()
431 sem = _threading.Semaphore(jobs)
432 err_event = _threading.Event()
433
David James8d201162013-10-11 17:03:19 -0700434 def GC(bare_git):
Dave Borowitz18857212012-10-23 17:02:59 -0700435 try:
436 try:
David James8d201162013-10-11 17:03:19 -0700437 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700438 except GitError:
439 err_event.set()
440 except:
441 err_event.set()
442 raise
443 finally:
444 sem.release()
445
Gabe Black2ff30292014-10-09 17:54:35 -0700446 for bare_git in gc_gitdirs.values():
Dave Borowitz18857212012-10-23 17:02:59 -0700447 if err_event.isSet():
448 break
449 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700450 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700451 t.daemon = True
452 threads.add(t)
453 t.start()
454
455 for t in threads:
456 t.join()
457
458 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700459 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700460 sys.exit(1)
461
Tim Kilbourn07669002013-03-08 15:02:49 -0800462 def _ReloadManifest(self, manifest_name=None):
463 if manifest_name:
464 # Override calls _Unload already
465 self.manifest.Override(manifest_name)
466 else:
467 self.manifest._Unload()
468
Dan Willemsen43507912016-09-01 16:26:02 -0700469 def _DeleteProject(self, path):
470 print('Deleting obsolete path %s' % path, file=sys.stderr)
471
472 # Delete the .git directory first, so we're less likely to have a partially
473 # working git repository around. There shouldn't be any git projects here,
474 # so rmtree works.
475 try:
476 shutil.rmtree(os.path.join(path, '.git'))
477 except OSError:
478 print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr)
479 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
480 print(' remove manually, then run sync again', file=sys.stderr)
481 return -1
482
483 # Delete everything under the worktree, except for directories that contain
484 # another git project
485 dirs_to_remove = []
486 failed = False
487 for root, dirs, files in os.walk(path):
488 for f in files:
489 try:
490 os.remove(os.path.join(root, f))
491 except OSError:
492 print('Failed to remove %s' % os.path.join(root, f), file=sys.stderr)
493 failed = True
494 dirs[:] = [d for d in dirs
495 if not os.path.lexists(os.path.join(root, d, '.git'))]
496 dirs_to_remove += [os.path.join(root, d) for d in dirs
497 if os.path.join(root, d) not in dirs_to_remove]
498 for d in reversed(dirs_to_remove):
Dan Willemseneceeb1b2016-09-25 18:24:27 -0700499 if os.path.islink(d):
500 try:
501 os.remove(d)
502 except OSError:
503 print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
504 failed = True
505 elif len(os.listdir(d)) == 0:
Dan Willemsen43507912016-09-01 16:26:02 -0700506 try:
507 os.rmdir(d)
508 except OSError:
509 print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
510 failed = True
511 continue
512 if failed:
513 print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
514 print(' remove manually, then run sync again', file=sys.stderr)
515 return -1
516
517 # Try deleting parent dirs if they are empty
518 project_dir = path
519 while project_dir != self.manifest.topdir:
520 if len(os.listdir(project_dir)) == 0:
521 os.rmdir(project_dir)
522 else:
523 break
524 project_dir = os.path.dirname(project_dir)
525
526 return 0
527
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700528 def UpdateProjectList(self):
529 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700530 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700531 if project.relpath:
532 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700533 file_name = 'project.list'
534 file_path = os.path.join(self.manifest.repodir, file_name)
535 old_project_paths = []
536
537 if os.path.exists(file_path):
538 fd = open(file_path, 'r')
539 try:
540 old_project_paths = fd.read().split('\n')
541 finally:
542 fd.close()
543 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700544 if not path:
545 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700546 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900547 # If the path has already been deleted, we don't need to do it
Dan Willemsen43507912016-09-01 16:26:02 -0700548 gitdir = os.path.join(self.manifest.topdir, path, '.git')
549 if os.path.exists(gitdir):
David Pursehousec1b86a22012-11-14 11:36:51 +0900550 project = Project(
551 manifest = self.manifest,
552 name = path,
553 remote = RemoteSpec('origin'),
David James8d201162013-10-11 17:03:19 -0700554 gitdir = gitdir,
555 objdir = gitdir,
David Pursehousec1b86a22012-11-14 11:36:51 +0900556 worktree = os.path.join(self.manifest.topdir, path),
557 relpath = path,
558 revisionExpr = 'HEAD',
559 revisionId = None,
560 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400561
David Pursehousec1b86a22012-11-14 11:36:51 +0900562 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900563 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900564 'are present' % project.relpath, file=sys.stderr)
565 print(' commit changes, then run sync again',
566 file=sys.stderr)
567 return -1
Dan Willemsen43507912016-09-01 16:26:02 -0700568 elif self._DeleteProject(project.worktree):
569 return -1
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700570
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700571 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700572 fd = open(file_path, 'w')
573 try:
574 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700575 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700576 finally:
577 fd.close()
578 return 0
579
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700580 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800581 if opt.jobs:
582 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700583 if self.jobs > 1:
584 soft_limit, _ = _rlimit_nofile()
585 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
586
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700587 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700588 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700589 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700590 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700591 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700592 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500593 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700594 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500595 sys.exit(1)
596 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700597 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500598 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900599 if opt.manifest_server_username or opt.manifest_server_password:
600 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700601 print('error: -u and -p may only be combined with -s or -t',
602 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900603 sys.exit(1)
604 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700605 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900606 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500607
608 if opt.manifest_name:
609 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700610
Chirayu Desaia892b102013-06-11 14:18:46 +0530611 manifest_name = opt.manifest_name
David Pursehouse59b41742015-05-07 14:36:09 +0900612 smart_sync_manifest_name = "smart_sync_override.xml"
613 smart_sync_manifest_path = os.path.join(
614 self.manifest.manifestProject.worktree, smart_sync_manifest_name)
Chirayu Desaia892b102013-06-11 14:18:46 +0530615
Victor Boivie08c880d2011-04-19 10:32:52 +0200616 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700617 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900618 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700619 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700620 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900621
622 manifest_server = self.manifest.manifest_server
David Pursehousefb99c712013-09-25 11:09:34 +0900623 if not opt.quiet:
624 print('Using manifest server %s' % manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900625
David Pursehouse86d973d2012-08-24 10:21:02 +0900626 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900627 username = None
628 password = None
629 if opt.manifest_server_username and opt.manifest_server_password:
630 username = opt.manifest_server_username
631 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900632 else:
633 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900634 info = netrc.netrc()
635 except IOError:
David Pursehouseba7bc732015-08-20 16:55:42 +0900636 # .netrc file does not exist or could not be opened
637 pass
David Pursehouse86d973d2012-08-24 10:21:02 +0900638 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900639 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530640 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900641 if parse_result.hostname:
David Pursehouseba7bc732015-08-20 16:55:42 +0900642 auth = info.authenticators(parse_result.hostname)
643 if auth:
644 username, _account, password = auth
645 else:
646 print('No credentials found for %s in .netrc'
647 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700648 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700649 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900650
651 if (username and password):
652 manifest_server = manifest_server.replace('://', '://%s:%s@' %
653 (username, password),
654 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900655
Dan Willemsen0745bb22015-08-17 13:41:45 -0700656 transport = PersistentTransport(manifest_server)
657 if manifest_server.startswith('persistent-'):
658 manifest_server = manifest_server[len('persistent-'):]
659
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700660 try:
Dan Willemsen0745bb22015-08-17 13:41:45 -0700661 server = xmlrpc.client.Server(manifest_server, transport=transport)
Victor Boivie08c880d2011-04-19 10:32:52 +0200662 if opt.smart_sync:
663 p = self.manifest.manifestProject
664 b = p.GetBranch(p.CurrentBranch)
665 branch = b.merge
666 if branch.startswith(R_HEADS):
667 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700668
Victor Boivie08c880d2011-04-19 10:32:52 +0200669 env = os.environ.copy()
Jeff Davidson5cf16602014-10-02 10:13:38 -0700670 if 'SYNC_TARGET' in env:
671 target = env['SYNC_TARGET']
672 [success, manifest_str] = server.GetApprovedManifest(branch, target)
673 elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200674 target = '%s-%s' % (env['TARGET_PRODUCT'],
675 env['TARGET_BUILD_VARIANT'])
676 [success, manifest_str] = server.GetApprovedManifest(branch, target)
677 else:
678 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700679 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200680 assert(opt.smart_tag)
681 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700682
683 if success:
David Pursehouse59b41742015-05-07 14:36:09 +0900684 manifest_name = smart_sync_manifest_name
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700685 try:
David Pursehouse59b41742015-05-07 14:36:09 +0900686 f = open(smart_sync_manifest_path, 'w')
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700687 try:
688 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700689 finally:
690 f.close()
David Pursehouse727cc3e2015-05-07 14:16:49 +0900691 except IOError as e:
692 print('error: cannot write manifest to %s:\n%s'
David Pursehouse59b41742015-05-07 14:36:09 +0900693 % (smart_sync_manifest_path, e),
Sarah Owenscecd1d82012-11-01 22:59:27 -0700694 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700695 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100696 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700697 else:
David Pursehouse351fe2c2013-09-25 17:54:26 +0900698 print('error: manifest server RPC call failed: %s' %
699 manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700700 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530701 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700702 print('error: cannot connect to manifest server %s:\n%s'
703 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900704 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530705 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700706 print('error: cannot connect to manifest server %s:\n%d %s'
707 % (self.manifest.manifest_server, e.errcode, e.errmsg),
708 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700709 sys.exit(1)
David Pursehouse59b41742015-05-07 14:36:09 +0900710 else: # Not smart sync or smart tag mode
711 if os.path.isfile(smart_sync_manifest_path):
712 try:
713 os.remove(smart_sync_manifest_path)
714 except OSError as e:
715 print('error: failed to remove existing smart sync override manifest: %s' %
716 e, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700717
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700718 rp = self.manifest.repoProject
719 rp.PreSync()
720
721 mp = self.manifest.manifestProject
722 mp.PreSync()
723
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800724 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700725 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800726
Nico Sallembien9bb18162009-12-07 15:38:01 -0800727 if not opt.local_only:
David Rileye0684ad2017-04-05 00:02:59 -0700728 start = time.time()
729 success = mp.Sync_NetworkHalf(quiet=opt.quiet,
730 current_branch_only=opt.current_branch_only,
731 no_tags=opt.no_tags,
732 optimized_fetch=opt.optimized_fetch,
733 submodules=self.manifest.HasSubmodules)
734 finish = time.time()
735 self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
736 start, finish, success)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800737
738 if mp.HasChanges:
739 syncbuf = SyncBuffer(mp.config)
David Rileye0684ad2017-04-05 00:02:59 -0700740 start = time.time()
Martin Kellye4e94d22017-03-21 16:05:12 -0700741 mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
David Rileye0684ad2017-04-05 00:02:59 -0700742 clean = syncbuf.Finish()
743 self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
744 start, time.time(), clean)
745 if not clean:
Nico Sallembien9bb18162009-12-07 15:38:01 -0800746 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100747 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700748 if opt.jobs is None:
749 self.jobs = self.manifest.default.sync_j
Simran Basib9a1b732015-08-20 12:19:28 -0700750
Simran Basib9a1b732015-08-20 12:19:28 -0700751 if self.gitc_manifest:
752 gitc_manifest_projects = self.GetProjects(args,
Simran Basib9a1b732015-08-20 12:19:28 -0700753 missing_ok=True)
754 gitc_projects = []
755 opened_projects = []
756 for project in gitc_manifest_projects:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700757 if project.relpath in self.gitc_manifest.paths and \
758 self.gitc_manifest.paths[project.relpath].old_revision:
759 opened_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700760 else:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700761 gitc_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700762
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700763 if not args:
764 gitc_projects = None
765
766 if gitc_projects != [] and not opt.local_only:
Simran Basib9a1b732015-08-20 12:19:28 -0700767 print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700768 manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
769 if manifest_name:
770 manifest.Override(manifest_name)
771 else:
772 manifest.Override(self.manifest.manifestFile)
773 gitc_utils.generate_gitc_manifest(self.gitc_manifest,
774 manifest,
Simran Basib9a1b732015-08-20 12:19:28 -0700775 gitc_projects)
776 print('GITC client successfully synced.')
777
778 # The opened projects need to be synced as normal, therefore we
779 # generate a new args list to represent the opened projects.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700780 # TODO: make this more reliable -- if there's a project name/path overlap,
781 # this may choose the wrong project.
782 args = [os.path.relpath(self.manifest.paths[p].worktree, os.getcwd())
783 for p in opened_projects]
Simran Basib9a1b732015-08-20 12:19:28 -0700784 if not args:
785 return
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800786 all_projects = self.GetProjects(args,
787 missing_ok=True,
788 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700789
Dave Borowitz67700e92012-10-23 15:00:54 -0700790 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700791 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700792 to_fetch = []
793 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700794 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700795 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900796 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700797 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700798
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800799 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700800 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700801 if opt.network_only:
802 # bail out now; the rest touches the working tree
803 return
804
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800805 # Iteratively fetch missing and/or nested unregistered submodules
806 previously_missing_set = set()
807 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100808 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800809 all_projects = self.GetProjects(args,
810 missing_ok=True,
811 submodules_ok=opt.fetch_submodules)
812 missing = []
813 for project in all_projects:
814 if project.gitdir not in fetched:
815 missing.append(project)
816 if not missing:
817 break
818 # Stop us from non-stopped fetching actually-missing repos: If set of
819 # missing repos has not been changed from last fetch, we break.
820 missing_set = set(p.name for p in missing)
821 if previously_missing_set == missing_set:
822 break
823 previously_missing_set = missing_set
824 fetched.update(self._Fetch(missing, opt))
825
Julien Campergue335f5ef2013-10-16 11:02:35 +0200826 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700827 # bail out now, we have no working tree
828 return
829
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700830 if self.UpdateProjectList():
831 sys.exit(1)
832
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700833 syncbuf = SyncBuffer(mp.config,
834 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900835 pm = Progress('Syncing work tree', len(all_projects))
836 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700837 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800838 if project.worktree:
David Rileye0684ad2017-04-05 00:02:59 -0700839 start = time.time()
Kevin Degiabaa7f32014-11-12 11:27:45 -0700840 project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
David Rileye0684ad2017-04-05 00:02:59 -0700841 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
842 start, time.time(), syncbuf.Recently())
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700843 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700844 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700845 if not syncbuf.Finish():
846 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700847
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700848 # If there's a notice that's supposed to print at the end of the sync, print
849 # it now...
850 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700851 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700852
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700853def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -0800854 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -0700855 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700856 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -0800857 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700858 if project.Exists:
859 project.PostRepoUpgrade()
860
861def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
862 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700863 print('info: A new version of repo is available', file=sys.stderr)
864 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700865 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700866 syncbuf = SyncBuffer(rp.config)
867 rp.Sync_LocalHalf(syncbuf)
868 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700869 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700870 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700871 raise RepoChangedException(['--repo-upgraded'])
872 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700873 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700874 else:
875 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700876 print('repo version %s is current' % rp.work_git.describe(HEAD),
877 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700878
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700879def _VerifyTag(project):
880 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
881 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700882 print('warning: GnuPG was not available during last "repo init"\n'
883 'warning: Cannot automatically authenticate repo."""',
884 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700885 return True
886
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700887 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700888 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700889 except GitError:
890 cur = None
891
892 if not cur \
893 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700894 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700895 if rev.startswith(R_HEADS):
896 rev = rev[len(R_HEADS):]
897
Sarah Owenscecd1d82012-11-01 22:59:27 -0700898 print(file=sys.stderr)
899 print("warning: project '%s' branch '%s' is not signed"
900 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700901 return False
902
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800903 env = os.environ.copy()
904 env['GIT_DIR'] = project.gitdir.encode()
905 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700906
907 cmd = [GIT, 'tag', '-v', cur]
908 proc = subprocess.Popen(cmd,
909 stdout = subprocess.PIPE,
910 stderr = subprocess.PIPE,
911 env = env)
912 out = proc.stdout.read()
913 proc.stdout.close()
914
915 err = proc.stderr.read()
916 proc.stderr.close()
917
918 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700919 print(file=sys.stderr)
920 print(out, file=sys.stderr)
921 print(err, file=sys.stderr)
922 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700923 return False
924 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700925
David Rileye0684ad2017-04-05 00:02:59 -0700926
Dave Borowitz67700e92012-10-23 15:00:54 -0700927class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700928 _ALPHA = 0.5
929
Dave Borowitz67700e92012-10-23 15:00:54 -0700930 def __init__(self, manifest):
Anthony King85b24ac2014-05-06 15:57:48 +0100931 self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
Dave Borowitz67700e92012-10-23 15:00:54 -0700932 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700933 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700934
935 def Get(self, project):
936 self._Load()
937 return self._times.get(project.name, _ONE_DAY_S)
938
939 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700940 self._Load()
941 name = project.name
942 old = self._times.get(name, t)
943 self._seen.add(name)
944 a = self._ALPHA
945 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700946
947 def _Load(self):
948 if self._times is None:
949 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100950 f = open(self._path)
Dave Borowitz67700e92012-10-23 15:00:54 -0700951 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100952 self._times = json.load(f)
953 finally:
954 f.close()
955 except (IOError, ValueError):
956 try:
957 os.remove(self._path)
958 except OSError:
959 pass
960 self._times = {}
Dave Borowitz67700e92012-10-23 15:00:54 -0700961
962 def Save(self):
963 if self._times is None:
964 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700965
966 to_delete = []
967 for name in self._times:
968 if name not in self._seen:
969 to_delete.append(name)
970 for name in to_delete:
971 del self._times[name]
972
Dave Borowitz67700e92012-10-23 15:00:54 -0700973 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100974 f = open(self._path, 'w')
Dave Borowitz67700e92012-10-23 15:00:54 -0700975 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100976 json.dump(self._times, f, indent=2)
977 finally:
978 f.close()
979 except (IOError, TypeError):
980 try:
981 os.remove(self._path)
982 except OSError:
983 pass
Dan Willemsen0745bb22015-08-17 13:41:45 -0700984
985# This is a replacement for xmlrpc.client.Transport using urllib2
986# and supporting persistent-http[s]. It cannot change hosts from
987# request to request like the normal transport, the real url
988# is passed during initialization.
989class PersistentTransport(xmlrpc.client.Transport):
990 def __init__(self, orig_host):
991 self.orig_host = orig_host
992
993 def request(self, host, handler, request_body, verbose=False):
994 with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
995 # Python doesn't understand cookies with the #HttpOnly_ prefix
996 # Since we're only using them for HTTP, copy the file temporarily,
997 # stripping those prefixes away.
Dan Willemsen3010e5b2015-08-20 10:09:20 -0700998 if cookiefile:
999 tmpcookiefile = tempfile.NamedTemporaryFile()
David Pursehouse4c5f74e2015-10-02 11:10:10 +09001000 tmpcookiefile.write("# HTTP Cookie File")
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001001 try:
1002 with open(cookiefile) as f:
1003 for line in f:
1004 if line.startswith("#HttpOnly_"):
1005 line = line[len("#HttpOnly_"):]
1006 tmpcookiefile.write(line)
1007 tmpcookiefile.flush()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001008
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001009 cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
David Pursehouseb1ad2192015-09-30 10:35:43 +09001010 try:
1011 cookiejar.load()
1012 except cookielib.LoadError:
1013 cookiejar = cookielib.CookieJar()
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001014 finally:
1015 tmpcookiefile.close()
1016 else:
1017 cookiejar = cookielib.CookieJar()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001018
1019 proxyhandler = urllib.request.ProxyHandler
1020 if proxy:
1021 proxyhandler = urllib.request.ProxyHandler({
1022 "http": proxy,
1023 "https": proxy })
1024
1025 opener = urllib.request.build_opener(
1026 urllib.request.HTTPCookieProcessor(cookiejar),
1027 proxyhandler)
1028
1029 url = urllib.parse.urljoin(self.orig_host, handler)
1030 parse_results = urllib.parse.urlparse(url)
1031
1032 scheme = parse_results.scheme
1033 if scheme == 'persistent-http':
1034 scheme = 'http'
1035 if scheme == 'persistent-https':
1036 # If we're proxying through persistent-https, use http. The
1037 # proxy itself will do the https.
1038 if proxy:
1039 scheme = 'http'
1040 else:
1041 scheme = 'https'
1042
1043 # Parse out any authentication information using the base class
1044 host, extra_headers, _ = self.get_host_info(parse_results.netloc)
1045
1046 url = urllib.parse.urlunparse((
1047 scheme,
1048 host,
1049 parse_results.path,
1050 parse_results.params,
1051 parse_results.query,
1052 parse_results.fragment))
1053
1054 request = urllib.request.Request(url, request_body)
1055 if extra_headers is not None:
1056 for (name, header) in extra_headers:
1057 request.add_header(name, header)
1058 request.add_header('Content-Type', 'text/xml')
1059 try:
1060 response = opener.open(request)
1061 except urllib.error.HTTPError as e:
1062 if e.code == 501:
1063 # We may have been redirected through a login process
1064 # but our POST turned into a GET. Retry.
1065 response = opener.open(request)
1066 else:
1067 raise
1068
1069 p, u = xmlrpc.client.getparser()
1070 while 1:
1071 data = response.read(1024)
1072 if not data:
1073 break
1074 p.feed(data)
1075 p.close()
1076 return u.close()
1077
1078 def close(self):
1079 pass
1080