blob: 4af411c977fd6aa7c37ed1c40e83183ab36c1daa [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Anthony King85b24ac2014-05-06 15:57:48 +010017import json
David Pursehouse86d973d2012-08-24 10:21:02 +090018import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070019from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import os
21import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Dan Willemsen0745bb22015-08-17 13:41:45 -070026import tempfile
Shawn O. Pearcef6906872009-04-18 10:49:00 -070027import time
David Pursehouse59bbb582013-05-17 10:49:33 +090028
29from pyversion import is_python3
30if is_python3():
Dan Willemsen0745bb22015-08-17 13:41:45 -070031 import http.cookiejar as cookielib
32 import urllib.error
Chirayu Desai217ea7d2013-03-01 19:14:38 +053033 import urllib.parse
Dan Willemsen0745bb22015-08-17 13:41:45 -070034 import urllib.request
David Pursehouse59bbb582013-05-17 10:49:33 +090035 import xmlrpc.client
36else:
Dan Willemsen0745bb22015-08-17 13:41:45 -070037 import cookielib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053038 import imp
Dan Willemsen0745bb22015-08-17 13:41:45 -070039 import urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053040 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090041 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053042 urllib = imp.new_module('urllib')
Dan Willemsen0745bb22015-08-17 13:41:45 -070043 urllib.error = urllib2
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053044 urllib.parse = urlparse
Dan Willemsen0745bb22015-08-17 13:41:45 -070045 urllib.request = urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053046 xmlrpc = imp.new_module('xmlrpc')
47 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070048
Roy Lee18afd7f2010-05-09 04:32:08 +080049try:
50 import threading as _threading
51except ImportError:
52 import dummy_threading as _threading
53
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070054try:
55 import resource
56 def _rlimit_nofile():
57 return resource.getrlimit(resource.RLIMIT_NOFILE)
58except ImportError:
59 def _rlimit_nofile():
60 return (256, 256)
61
Dave Borowitz18857212012-10-23 17:02:59 -070062try:
63 import multiprocessing
64except ImportError:
65 multiprocessing = None
66
Dave Borowitze2152672012-10-31 12:24:38 -070067from git_command import GIT, git_require
David Pursehouseba7bc732015-08-20 16:55:42 +090068from git_config import GetUrlCookieFile
David Pursehoused94aaef2012-09-07 09:52:04 +090069from git_refs import R_HEADS, HEAD
Simran Basibdb52712015-08-10 13:23:23 -070070import gitc_utils
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070071from project import Project
72from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080073from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000074from error import RepoChangedException, GitError, ManifestParseError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070075from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070076from progress import Progress
Conley Owens094cdbe2014-01-30 15:09:59 -080077from wrapper import Wrapper
Dan Willemsen5ea32d12015-09-08 13:27:20 -070078from manifest_xml import GitcManifest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070079
Dave Borowitz67700e92012-10-23 15:00:54 -070080_ONE_DAY_S = 24 * 60 * 60
81
Doug Andersonfc06ced2011-03-16 15:49:18 -070082class _FetchError(Exception):
83 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
84 pass
85
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080086class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080087 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070088 common = True
89 helpSummary = "Update working tree to the latest revision"
90 helpUsage = """
91%prog [<project>...]
92"""
93 helpDescription = """
94The '%prog' command synchronizes local project directories
95with the remote repositories specified in the manifest. If a local
96project does not yet exist, it will clone a new local directory from
97the remote repository and set up tracking branches as specified in
98the manifest. If the local project already exists, '%prog'
99will update the remote branches and rebase any new local changes
100on top of the new remote changes.
101
102'%prog' will synchronize all projects listed at the command
103line. Projects can be specified either by name, or by a relative
104or absolute path to the project's local directory. If no projects
105are specified, '%prog' will synchronize all projects listed in
106the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700107
108The -d/--detach option can be used to switch specified projects
109back to the manifest revision. This option is especially helpful
110if the project is currently on a topic branch, but the manifest
111revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700112
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700113The -s/--smart-sync option can be used to sync to a known good
114build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200115manifest. The -t/--smart-tag option is similar and allows you to
116specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700117
David Pursehousecf76b1b2012-09-14 10:31:42 +0900118The -u/--manifest-server-username and -p/--manifest-server-password
119options can be used to specify a username and password to authenticate
120with the manifest server when using the -s or -t option.
121
122If -u and -p are not specified when using the -s or -t option, '%prog'
123will attempt to read authentication credentials for the manifest server
124from the user's .netrc file.
125
126'%prog' will not use authentication credentials from -u/-p or .netrc
127if the manifest server specified in the manifest file already includes
128credentials.
129
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500130The -f/--force-broken option can be used to proceed with syncing
131other projects if a project sync fails.
132
Kevin Degiabaa7f32014-11-12 11:27:45 -0700133The --force-sync option can be used to overwrite existing git
134directories if they have previously been linked to a different
135object direcotry. WARNING: This may cause data to be lost since
136refs may be removed when overwriting.
137
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700138The --no-clone-bundle option disables any attempt to use
139$URL/clone.bundle to bootstrap a new Git repository from a
140resumeable bundle file on a content delivery network. This
141may be necessary if there are problems with the local Python
142HTTP client or proxy configuration, but the Git binary works.
143
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800144The --fetch-submodules option enables fetching Git submodules
145of a project from server.
146
David Pursehousef2fad612015-01-29 14:36:28 +0900147The -c/--current-branch option can be used to only fetch objects that
148are on the branch specified by a project's revision.
149
David Pursehouseb1553542014-09-04 21:28:09 +0900150The --optimized-fetch option can be used to only fetch projects that
151are fixed to a sha1 revision if the sha1 revision does not already
152exist locally.
153
David Pursehouse74cfd272015-10-14 10:50:15 +0900154The --prune option can be used to remove any refs that no longer
155exist on the remote.
156
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700157SSH Connections
158---------------
159
160If at least one project remote URL uses an SSH connection (ssh://,
161git+ssh://, or user@host:path syntax) repo will automatically
162enable the SSH ControlMaster option when connecting to that host.
163This feature permits other projects in the same '%prog' session to
164reuse the same SSH tunnel, saving connection setup overheads.
165
166To disable this behavior on UNIX platforms, set the GIT_SSH
167environment variable to 'ssh'. For example:
168
169 export GIT_SSH=ssh
170 %prog
171
172Compatibility
173~~~~~~~~~~~~~
174
175This feature is automatically disabled on Windows, due to the lack
176of UNIX domain socket support.
177
178This feature is not compatible with url.insteadof rewrites in the
179user's ~/.gitconfig. '%prog' is currently not able to perform the
180rewrite early enough to establish the ControlMaster tunnel.
181
182If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
183later is required to fix a server side protocol bug.
184
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700185"""
186
Nico Sallembien6623b212010-05-11 12:57:01 -0700187 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000188 try:
189 self.jobs = self.manifest.default.sync_j
190 except ManifestParseError:
191 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700192
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500193 p.add_option('-f', '--force-broken',
194 dest='force_broken', action='store_true',
195 help="continue sync even if a project fails to sync")
Kevin Degiabaa7f32014-11-12 11:27:45 -0700196 p.add_option('--force-sync',
197 dest='force_sync', action='store_true',
198 help="overwrite an existing git directory if it needs to "
199 "point to a different object directory. WARNING: this "
200 "may cause loss of data")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900201 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700202 dest='local_only', action='store_true',
203 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900204 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700205 dest='network_only', action='store_true',
206 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900207 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700208 dest='detach_head', action='store_true',
209 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900210 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700211 dest='current_branch_only', action='store_true',
212 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900213 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700214 dest='quiet', action='store_true',
215 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900216 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800217 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700218 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500219 p.add_option('-m', '--manifest-name',
220 dest='manifest_name',
221 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700222 p.add_option('--no-clone-bundle',
223 dest='no_clone_bundle', action='store_true',
224 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800225 p.add_option('-u', '--manifest-server-username', action='store',
226 dest='manifest_server_username',
227 help='username to authenticate with the manifest server')
228 p.add_option('-p', '--manifest-server-password', action='store',
229 dest='manifest_server_password',
230 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800231 p.add_option('--fetch-submodules',
232 dest='fetch_submodules', action='store_true',
233 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700234 p.add_option('--no-tags',
235 dest='no_tags', action='store_true',
236 help="don't fetch tags")
David Pursehouseb1553542014-09-04 21:28:09 +0900237 p.add_option('--optimized-fetch',
238 dest='optimized_fetch', action='store_true',
239 help='only fetch projects fixed to sha1 if revision does not exist locally')
David Pursehouse74cfd272015-10-14 10:50:15 +0900240 p.add_option('--prune', dest='prune', action='store_true',
241 help='delete refs that no longer exist on the remote')
Nico Sallembien6623b212010-05-11 12:57:01 -0700242 if show_smart:
243 p.add_option('-s', '--smart-sync',
244 dest='smart_sync', action='store_true',
245 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200246 p.add_option('-t', '--smart-tag',
247 dest='smart_tag', action='store',
248 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700249
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700250 g = p.add_option_group('repo Version options')
251 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700252 dest='no_repo_verify', action='store_true',
253 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700254 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800255 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700256 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700257
David James89ece422014-01-09 18:51:58 -0800258 def _FetchProjectList(self, opt, projects, *args, **kwargs):
David Pursehousec1b86a22012-11-14 11:36:51 +0900259 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800260
David James8d201162013-10-11 17:03:19 -0700261 Delegates most of the work to _FetchHelper.
262
263 Args:
264 opt: Program options returned from optparse. See _Options().
265 projects: Projects to fetch.
David James89ece422014-01-09 18:51:58 -0800266 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
David James8d201162013-10-11 17:03:19 -0700267 _FetchHelper docstring for details.
268 """
269 for project in projects:
David James89ece422014-01-09 18:51:58 -0800270 success = self._FetchHelper(opt, project, *args, **kwargs)
David James8d201162013-10-11 17:03:19 -0700271 if not success and not opt.force_broken:
272 break
273
274 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
275 """Fetch git objects for a single project.
276
David Pursehousec1b86a22012-11-14 11:36:51 +0900277 Args:
278 opt: Program options returned from optparse. See _Options().
279 project: Project object for the project to fetch.
280 lock: Lock for accessing objects that are shared amongst multiple
281 _FetchHelper() threads.
282 fetched: set object that we will add project.gitdir to when we're done
283 (with our lock held).
284 pm: Instance of a Project object. We will call pm.update() (with our
285 lock held).
286 sem: We'll release() this semaphore when we exit so that another thread
287 can be started up.
288 err_event: We'll set this event in the case of an error (after printing
289 out info about the error).
David James8d201162013-10-11 17:03:19 -0700290
291 Returns:
292 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900293 """
294 # We'll set to true once we've locked the lock.
295 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700296
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530297 if not opt.quiet:
298 print('Fetching project %s' % project.name)
299
David Pursehousec1b86a22012-11-14 11:36:51 +0900300 # Encapsulate everything in a try/except/finally so that:
301 # - We always set err_event in the case of an exception.
302 # - We always make sure we call sem.release().
303 # - We always make sure we unlock the lock if we locked it.
304 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700305 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900306 start = time.time()
307 success = project.Sync_NetworkHalf(
308 quiet=opt.quiet,
309 current_branch_only=opt.current_branch_only,
Kevin Degiabaa7f32014-11-12 11:27:45 -0700310 force_sync=opt.force_sync,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700311 clone_bundle=not opt.no_clone_bundle,
David Pursehouseb1553542014-09-04 21:28:09 +0900312 no_tags=opt.no_tags, archive=self.manifest.IsArchive,
David Pursehouse74cfd272015-10-14 10:50:15 +0900313 optimized_fetch=opt.optimized_fetch,
314 prune=opt.prune)
David Pursehousec1b86a22012-11-14 11:36:51 +0900315 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700316
David Pursehousec1b86a22012-11-14 11:36:51 +0900317 # Lock around all the rest of the code, since printing, updating a set
318 # and Progress.update() are not thread safe.
319 lock.acquire()
320 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700321
David Pursehousec1b86a22012-11-14 11:36:51 +0900322 if not success:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800323 err_event.set()
David Pursehousec1b86a22012-11-14 11:36:51 +0900324 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
325 if opt.force_broken:
326 print('warn: --force-broken, continuing to sync',
327 file=sys.stderr)
328 else:
329 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700330
David Pursehousec1b86a22012-11-14 11:36:51 +0900331 fetched.add(project.gitdir)
332 pm.update()
333 except _FetchError:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800334 pass
Dan Sandlerc5cd4332015-07-31 09:37:53 -0400335 except Exception as e:
336 print('error: Cannot fetch %s (%s: %s)' \
337 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900338 err_event.set()
339 raise
340 finally:
341 if did_lock:
342 lock.release()
343 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800344
David James8d201162013-10-11 17:03:19 -0700345 return success
346
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700347 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700348 fetched = set()
David James89ece422014-01-09 18:51:58 -0800349 lock = _threading.Lock()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700350 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800351
David James89ece422014-01-09 18:51:58 -0800352 objdir_project_map = dict()
353 for project in projects:
354 objdir_project_map.setdefault(project.objdir, []).append(project)
David James8d201162013-10-11 17:03:19 -0700355
David James89ece422014-01-09 18:51:58 -0800356 threads = set()
357 sem = _threading.Semaphore(self.jobs)
358 err_event = _threading.Event()
359 for project_list in objdir_project_map.values():
360 # Check for any errors before running any more tasks.
361 # ...we'll let existing threads finish, though.
362 if err_event.isSet() and not opt.force_broken:
363 break
Doug Andersonfc06ced2011-03-16 15:49:18 -0700364
David James89ece422014-01-09 18:51:58 -0800365 sem.acquire()
366 kwargs = dict(opt=opt,
367 projects=project_list,
368 lock=lock,
369 fetched=fetched,
370 pm=pm,
371 sem=sem,
372 err_event=err_event)
373 if self.jobs > 1:
David James8d201162013-10-11 17:03:19 -0700374 t = _threading.Thread(target = self._FetchProjectList,
David James89ece422014-01-09 18:51:58 -0800375 kwargs = kwargs)
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200376 # Ensure that Ctrl-C will not freeze the repo process.
377 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800378 threads.add(t)
379 t.start()
David James89ece422014-01-09 18:51:58 -0800380 else:
381 self._FetchProjectList(**kwargs)
Roy Lee18afd7f2010-05-09 04:32:08 +0800382
David James89ece422014-01-09 18:51:58 -0800383 for t in threads:
384 t.join()
Roy Lee18afd7f2010-05-09 04:32:08 +0800385
David James89ece422014-01-09 18:51:58 -0800386 # If we saw an error, exit with code 1 so that other scripts can check.
387 if err_event.isSet():
388 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
389 sys.exit(1)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700390
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700391 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700392 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700393
Julien Campergue335f5ef2013-10-16 11:02:35 +0200394 if not self.manifest.IsArchive:
395 self._GCProjects(projects)
396
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700397 return fetched
398
Dave Borowitz18857212012-10-23 17:02:59 -0700399 def _GCProjects(self, projects):
David James8d201162013-10-11 17:03:19 -0700400 gitdirs = {}
401 for project in projects:
402 gitdirs[project.gitdir] = project.bare_git
403
Dave Borowitze2152672012-10-31 12:24:38 -0700404 has_dash_c = git_require((1, 7, 2))
405 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700406 cpu_count = multiprocessing.cpu_count()
407 else:
408 cpu_count = 1
409 jobs = min(self.jobs, cpu_count)
410
411 if jobs < 2:
David James8d201162013-10-11 17:03:19 -0700412 for bare_git in gitdirs.values():
413 bare_git.gc('--auto')
Dave Borowitz18857212012-10-23 17:02:59 -0700414 return
415
416 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
417
418 threads = set()
419 sem = _threading.Semaphore(jobs)
420 err_event = _threading.Event()
421
David James8d201162013-10-11 17:03:19 -0700422 def GC(bare_git):
Dave Borowitz18857212012-10-23 17:02:59 -0700423 try:
424 try:
David James8d201162013-10-11 17:03:19 -0700425 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700426 except GitError:
427 err_event.set()
428 except:
429 err_event.set()
430 raise
431 finally:
432 sem.release()
433
David James8d201162013-10-11 17:03:19 -0700434 for bare_git in gitdirs.values():
Dave Borowitz18857212012-10-23 17:02:59 -0700435 if err_event.isSet():
436 break
437 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700438 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700439 t.daemon = True
440 threads.add(t)
441 t.start()
442
443 for t in threads:
444 t.join()
445
446 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700447 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700448 sys.exit(1)
449
Tim Kilbourn07669002013-03-08 15:02:49 -0800450 def _ReloadManifest(self, manifest_name=None):
451 if manifest_name:
452 # Override calls _Unload already
453 self.manifest.Override(manifest_name)
454 else:
455 self.manifest._Unload()
456
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700457 def UpdateProjectList(self):
458 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700459 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700460 if project.relpath:
461 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700462 file_name = 'project.list'
463 file_path = os.path.join(self.manifest.repodir, file_name)
464 old_project_paths = []
465
466 if os.path.exists(file_path):
467 fd = open(file_path, 'r')
468 try:
469 old_project_paths = fd.read().split('\n')
470 finally:
471 fd.close()
472 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700473 if not path:
474 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700475 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900476 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400477 if os.path.exists(self.manifest.topdir + '/' + path):
David James8d201162013-10-11 17:03:19 -0700478 gitdir = os.path.join(self.manifest.topdir, path, '.git')
David Pursehousec1b86a22012-11-14 11:36:51 +0900479 project = Project(
480 manifest = self.manifest,
481 name = path,
482 remote = RemoteSpec('origin'),
David James8d201162013-10-11 17:03:19 -0700483 gitdir = gitdir,
484 objdir = gitdir,
David Pursehousec1b86a22012-11-14 11:36:51 +0900485 worktree = os.path.join(self.manifest.topdir, path),
486 relpath = path,
487 revisionExpr = 'HEAD',
488 revisionId = None,
489 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400490
David Pursehousec1b86a22012-11-14 11:36:51 +0900491 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900492 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900493 'are present' % project.relpath, file=sys.stderr)
494 print(' commit changes, then run sync again',
495 file=sys.stderr)
496 return -1
497 else:
498 print('Deleting obsolete path %s' % project.worktree,
499 file=sys.stderr)
500 shutil.rmtree(project.worktree)
501 # Try deleting parent subdirs if they are empty
502 project_dir = os.path.dirname(project.worktree)
503 while project_dir != self.manifest.topdir:
504 try:
505 os.rmdir(project_dir)
506 except OSError:
507 break
508 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700509
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700510 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700511 fd = open(file_path, 'w')
512 try:
513 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700514 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700515 finally:
516 fd.close()
517 return 0
518
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700519 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800520 if opt.jobs:
521 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700522 if self.jobs > 1:
523 soft_limit, _ = _rlimit_nofile()
524 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
525
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700526 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700527 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700528 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700529 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700530 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700531 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500532 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700533 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500534 sys.exit(1)
535 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700536 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500537 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900538 if opt.manifest_server_username or opt.manifest_server_password:
539 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700540 print('error: -u and -p may only be combined with -s or -t',
541 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900542 sys.exit(1)
543 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700544 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900545 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500546
547 if opt.manifest_name:
548 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700549
Chirayu Desaia892b102013-06-11 14:18:46 +0530550 manifest_name = opt.manifest_name
David Pursehouse59b41742015-05-07 14:36:09 +0900551 smart_sync_manifest_name = "smart_sync_override.xml"
552 smart_sync_manifest_path = os.path.join(
553 self.manifest.manifestProject.worktree, smart_sync_manifest_name)
Chirayu Desaia892b102013-06-11 14:18:46 +0530554
Victor Boivie08c880d2011-04-19 10:32:52 +0200555 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700556 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900557 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700558 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700559 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900560
561 manifest_server = self.manifest.manifest_server
David Pursehousefb99c712013-09-25 11:09:34 +0900562 if not opt.quiet:
563 print('Using manifest server %s' % manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900564
David Pursehouse86d973d2012-08-24 10:21:02 +0900565 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900566 username = None
567 password = None
568 if opt.manifest_server_username and opt.manifest_server_password:
569 username = opt.manifest_server_username
570 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900571 else:
572 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900573 info = netrc.netrc()
574 except IOError:
David Pursehouseba7bc732015-08-20 16:55:42 +0900575 # .netrc file does not exist or could not be opened
576 pass
David Pursehouse86d973d2012-08-24 10:21:02 +0900577 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900578 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530579 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900580 if parse_result.hostname:
David Pursehouseba7bc732015-08-20 16:55:42 +0900581 auth = info.authenticators(parse_result.hostname)
582 if auth:
583 username, _account, password = auth
584 else:
585 print('No credentials found for %s in .netrc'
586 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700587 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700588 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900589
590 if (username and password):
591 manifest_server = manifest_server.replace('://', '://%s:%s@' %
592 (username, password),
593 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900594
Dan Willemsen0745bb22015-08-17 13:41:45 -0700595 transport = PersistentTransport(manifest_server)
596 if manifest_server.startswith('persistent-'):
597 manifest_server = manifest_server[len('persistent-'):]
598
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700599 try:
Dan Willemsen0745bb22015-08-17 13:41:45 -0700600 server = xmlrpc.client.Server(manifest_server, transport=transport)
Victor Boivie08c880d2011-04-19 10:32:52 +0200601 if opt.smart_sync:
602 p = self.manifest.manifestProject
603 b = p.GetBranch(p.CurrentBranch)
604 branch = b.merge
605 if branch.startswith(R_HEADS):
606 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700607
Victor Boivie08c880d2011-04-19 10:32:52 +0200608 env = os.environ.copy()
Jeff Davidson5cf16602014-10-02 10:13:38 -0700609 if 'SYNC_TARGET' in env:
610 target = env['SYNC_TARGET']
611 [success, manifest_str] = server.GetApprovedManifest(branch, target)
612 elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200613 target = '%s-%s' % (env['TARGET_PRODUCT'],
614 env['TARGET_BUILD_VARIANT'])
615 [success, manifest_str] = server.GetApprovedManifest(branch, target)
616 else:
617 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700618 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200619 assert(opt.smart_tag)
620 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700621
622 if success:
David Pursehouse59b41742015-05-07 14:36:09 +0900623 manifest_name = smart_sync_manifest_name
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700624 try:
David Pursehouse59b41742015-05-07 14:36:09 +0900625 f = open(smart_sync_manifest_path, 'w')
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700626 try:
627 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700628 finally:
629 f.close()
David Pursehouse727cc3e2015-05-07 14:16:49 +0900630 except IOError as e:
631 print('error: cannot write manifest to %s:\n%s'
David Pursehouse59b41742015-05-07 14:36:09 +0900632 % (smart_sync_manifest_path, e),
Sarah Owenscecd1d82012-11-01 22:59:27 -0700633 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700634 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100635 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700636 else:
David Pursehouse351fe2c2013-09-25 17:54:26 +0900637 print('error: manifest server RPC call failed: %s' %
638 manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700639 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530640 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700641 print('error: cannot connect to manifest server %s:\n%s'
642 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900643 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530644 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700645 print('error: cannot connect to manifest server %s:\n%d %s'
646 % (self.manifest.manifest_server, e.errcode, e.errmsg),
647 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700648 sys.exit(1)
David Pursehouse59b41742015-05-07 14:36:09 +0900649 else: # Not smart sync or smart tag mode
650 if os.path.isfile(smart_sync_manifest_path):
651 try:
652 os.remove(smart_sync_manifest_path)
653 except OSError as e:
654 print('error: failed to remove existing smart sync override manifest: %s' %
655 e, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700656
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700657 rp = self.manifest.repoProject
658 rp.PreSync()
659
660 mp = self.manifest.manifestProject
661 mp.PreSync()
662
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800663 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700664 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800665
Nico Sallembien9bb18162009-12-07 15:38:01 -0800666 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700667 mp.Sync_NetworkHalf(quiet=opt.quiet,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700668 current_branch_only=opt.current_branch_only,
David Pursehouseb1553542014-09-04 21:28:09 +0900669 no_tags=opt.no_tags,
670 optimized_fetch=opt.optimized_fetch)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800671
672 if mp.HasChanges:
673 syncbuf = SyncBuffer(mp.config)
674 mp.Sync_LocalHalf(syncbuf)
675 if not syncbuf.Finish():
676 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100677 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700678 if opt.jobs is None:
679 self.jobs = self.manifest.default.sync_j
Simran Basib9a1b732015-08-20 12:19:28 -0700680
Simran Basib9a1b732015-08-20 12:19:28 -0700681 if self.gitc_manifest:
682 gitc_manifest_projects = self.GetProjects(args,
Simran Basib9a1b732015-08-20 12:19:28 -0700683 missing_ok=True)
684 gitc_projects = []
685 opened_projects = []
686 for project in gitc_manifest_projects:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700687 if project.relpath in self.gitc_manifest.paths and \
688 self.gitc_manifest.paths[project.relpath].old_revision:
689 opened_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700690 else:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700691 gitc_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700692
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700693 if not args:
694 gitc_projects = None
695
696 if gitc_projects != [] and not opt.local_only:
Simran Basib9a1b732015-08-20 12:19:28 -0700697 print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700698 manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
699 if manifest_name:
700 manifest.Override(manifest_name)
701 else:
702 manifest.Override(self.manifest.manifestFile)
703 gitc_utils.generate_gitc_manifest(self.gitc_manifest,
704 manifest,
Simran Basib9a1b732015-08-20 12:19:28 -0700705 gitc_projects)
706 print('GITC client successfully synced.')
707
708 # The opened projects need to be synced as normal, therefore we
709 # generate a new args list to represent the opened projects.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700710 # TODO: make this more reliable -- if there's a project name/path overlap,
711 # this may choose the wrong project.
712 args = [os.path.relpath(self.manifest.paths[p].worktree, os.getcwd())
713 for p in opened_projects]
Simran Basib9a1b732015-08-20 12:19:28 -0700714 if not args:
715 return
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800716 all_projects = self.GetProjects(args,
717 missing_ok=True,
718 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700719
Dave Borowitz67700e92012-10-23 15:00:54 -0700720 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700721 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700722 to_fetch = []
723 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700724 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700725 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900726 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700727 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700728
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800729 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700730 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700731 if opt.network_only:
732 # bail out now; the rest touches the working tree
733 return
734
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800735 # Iteratively fetch missing and/or nested unregistered submodules
736 previously_missing_set = set()
737 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100738 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800739 all_projects = self.GetProjects(args,
740 missing_ok=True,
741 submodules_ok=opt.fetch_submodules)
742 missing = []
743 for project in all_projects:
744 if project.gitdir not in fetched:
745 missing.append(project)
746 if not missing:
747 break
748 # Stop us from non-stopped fetching actually-missing repos: If set of
749 # missing repos has not been changed from last fetch, we break.
750 missing_set = set(p.name for p in missing)
751 if previously_missing_set == missing_set:
752 break
753 previously_missing_set = missing_set
754 fetched.update(self._Fetch(missing, opt))
755
Julien Campergue335f5ef2013-10-16 11:02:35 +0200756 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700757 # bail out now, we have no working tree
758 return
759
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700760 if self.UpdateProjectList():
761 sys.exit(1)
762
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700763 syncbuf = SyncBuffer(mp.config,
764 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900765 pm = Progress('Syncing work tree', len(all_projects))
766 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700767 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800768 if project.worktree:
Kevin Degiabaa7f32014-11-12 11:27:45 -0700769 project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700770 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700771 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700772 if not syncbuf.Finish():
773 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700774
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700775 # If there's a notice that's supposed to print at the end of the sync, print
776 # it now...
777 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700778 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700779
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700780def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -0800781 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -0700782 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700783 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -0800784 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700785 if project.Exists:
786 project.PostRepoUpgrade()
787
788def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
789 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700790 print('info: A new version of repo is available', file=sys.stderr)
791 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700792 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700793 syncbuf = SyncBuffer(rp.config)
794 rp.Sync_LocalHalf(syncbuf)
795 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700796 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700797 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700798 raise RepoChangedException(['--repo-upgraded'])
799 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700800 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700801 else:
802 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700803 print('repo version %s is current' % rp.work_git.describe(HEAD),
804 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700805
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700806def _VerifyTag(project):
807 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
808 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700809 print('warning: GnuPG was not available during last "repo init"\n'
810 'warning: Cannot automatically authenticate repo."""',
811 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700812 return True
813
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700814 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700815 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700816 except GitError:
817 cur = None
818
819 if not cur \
820 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700821 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700822 if rev.startswith(R_HEADS):
823 rev = rev[len(R_HEADS):]
824
Sarah Owenscecd1d82012-11-01 22:59:27 -0700825 print(file=sys.stderr)
826 print("warning: project '%s' branch '%s' is not signed"
827 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700828 return False
829
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800830 env = os.environ.copy()
831 env['GIT_DIR'] = project.gitdir.encode()
832 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700833
834 cmd = [GIT, 'tag', '-v', cur]
835 proc = subprocess.Popen(cmd,
836 stdout = subprocess.PIPE,
837 stderr = subprocess.PIPE,
838 env = env)
839 out = proc.stdout.read()
840 proc.stdout.close()
841
842 err = proc.stderr.read()
843 proc.stderr.close()
844
845 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700846 print(file=sys.stderr)
847 print(out, file=sys.stderr)
848 print(err, file=sys.stderr)
849 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700850 return False
851 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700852
853class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700854 _ALPHA = 0.5
855
Dave Borowitz67700e92012-10-23 15:00:54 -0700856 def __init__(self, manifest):
Anthony King85b24ac2014-05-06 15:57:48 +0100857 self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
Dave Borowitz67700e92012-10-23 15:00:54 -0700858 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700859 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700860
861 def Get(self, project):
862 self._Load()
863 return self._times.get(project.name, _ONE_DAY_S)
864
865 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700866 self._Load()
867 name = project.name
868 old = self._times.get(name, t)
869 self._seen.add(name)
870 a = self._ALPHA
871 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700872
873 def _Load(self):
874 if self._times is None:
875 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100876 f = open(self._path)
Dave Borowitz67700e92012-10-23 15:00:54 -0700877 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100878 self._times = json.load(f)
879 finally:
880 f.close()
881 except (IOError, ValueError):
882 try:
883 os.remove(self._path)
884 except OSError:
885 pass
886 self._times = {}
Dave Borowitz67700e92012-10-23 15:00:54 -0700887
888 def Save(self):
889 if self._times is None:
890 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700891
892 to_delete = []
893 for name in self._times:
894 if name not in self._seen:
895 to_delete.append(name)
896 for name in to_delete:
897 del self._times[name]
898
Dave Borowitz67700e92012-10-23 15:00:54 -0700899 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100900 f = open(self._path, 'w')
Dave Borowitz67700e92012-10-23 15:00:54 -0700901 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100902 json.dump(self._times, f, indent=2)
903 finally:
904 f.close()
905 except (IOError, TypeError):
906 try:
907 os.remove(self._path)
908 except OSError:
909 pass
Dan Willemsen0745bb22015-08-17 13:41:45 -0700910
911# This is a replacement for xmlrpc.client.Transport using urllib2
912# and supporting persistent-http[s]. It cannot change hosts from
913# request to request like the normal transport, the real url
914# is passed during initialization.
915class PersistentTransport(xmlrpc.client.Transport):
916 def __init__(self, orig_host):
917 self.orig_host = orig_host
918
919 def request(self, host, handler, request_body, verbose=False):
920 with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
921 # Python doesn't understand cookies with the #HttpOnly_ prefix
922 # Since we're only using them for HTTP, copy the file temporarily,
923 # stripping those prefixes away.
Dan Willemsen3010e5b2015-08-20 10:09:20 -0700924 if cookiefile:
925 tmpcookiefile = tempfile.NamedTemporaryFile()
David Pursehouse4c5f74e2015-10-02 11:10:10 +0900926 tmpcookiefile.write("# HTTP Cookie File")
Dan Willemsen3010e5b2015-08-20 10:09:20 -0700927 try:
928 with open(cookiefile) as f:
929 for line in f:
930 if line.startswith("#HttpOnly_"):
931 line = line[len("#HttpOnly_"):]
932 tmpcookiefile.write(line)
933 tmpcookiefile.flush()
Dan Willemsen0745bb22015-08-17 13:41:45 -0700934
Dan Willemsen3010e5b2015-08-20 10:09:20 -0700935 cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
David Pursehouseb1ad2192015-09-30 10:35:43 +0900936 try:
937 cookiejar.load()
938 except cookielib.LoadError:
939 cookiejar = cookielib.CookieJar()
Dan Willemsen3010e5b2015-08-20 10:09:20 -0700940 finally:
941 tmpcookiefile.close()
942 else:
943 cookiejar = cookielib.CookieJar()
Dan Willemsen0745bb22015-08-17 13:41:45 -0700944
945 proxyhandler = urllib.request.ProxyHandler
946 if proxy:
947 proxyhandler = urllib.request.ProxyHandler({
948 "http": proxy,
949 "https": proxy })
950
951 opener = urllib.request.build_opener(
952 urllib.request.HTTPCookieProcessor(cookiejar),
953 proxyhandler)
954
955 url = urllib.parse.urljoin(self.orig_host, handler)
956 parse_results = urllib.parse.urlparse(url)
957
958 scheme = parse_results.scheme
959 if scheme == 'persistent-http':
960 scheme = 'http'
961 if scheme == 'persistent-https':
962 # If we're proxying through persistent-https, use http. The
963 # proxy itself will do the https.
964 if proxy:
965 scheme = 'http'
966 else:
967 scheme = 'https'
968
969 # Parse out any authentication information using the base class
970 host, extra_headers, _ = self.get_host_info(parse_results.netloc)
971
972 url = urllib.parse.urlunparse((
973 scheme,
974 host,
975 parse_results.path,
976 parse_results.params,
977 parse_results.query,
978 parse_results.fragment))
979
980 request = urllib.request.Request(url, request_body)
981 if extra_headers is not None:
982 for (name, header) in extra_headers:
983 request.add_header(name, header)
984 request.add_header('Content-Type', 'text/xml')
985 try:
986 response = opener.open(request)
987 except urllib.error.HTTPError as e:
988 if e.code == 501:
989 # We may have been redirected through a login process
990 # but our POST turned into a GET. Retry.
991 response = opener.open(request)
992 else:
993 raise
994
995 p, u = xmlrpc.client.getparser()
996 while 1:
997 data = response.read(1024)
998 if not data:
999 break
1000 p.feed(data)
1001 p.close()
1002 return u.close()
1003
1004 def close(self):
1005 pass
1006