blob: 2e9bbe7058b0bdf9f09442aa6141db6c4d01df66 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Anthony King85b24ac2014-05-06 15:57:48 +010017import json
David Pursehouse86d973d2012-08-24 10:21:02 +090018import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070019from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import os
21import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070022import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070023import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import subprocess
25import sys
Dan Willemsen0745bb22015-08-17 13:41:45 -070026import tempfile
Shawn O. Pearcef6906872009-04-18 10:49:00 -070027import time
David Pursehouse59bbb582013-05-17 10:49:33 +090028
29from pyversion import is_python3
30if is_python3():
Dan Willemsen0745bb22015-08-17 13:41:45 -070031 import http.cookiejar as cookielib
32 import urllib.error
Chirayu Desai217ea7d2013-03-01 19:14:38 +053033 import urllib.parse
Dan Willemsen0745bb22015-08-17 13:41:45 -070034 import urllib.request
David Pursehouse59bbb582013-05-17 10:49:33 +090035 import xmlrpc.client
36else:
Dan Willemsen0745bb22015-08-17 13:41:45 -070037 import cookielib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053038 import imp
Dan Willemsen0745bb22015-08-17 13:41:45 -070039 import urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053040 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090041 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053042 urllib = imp.new_module('urllib')
Dan Willemsen0745bb22015-08-17 13:41:45 -070043 urllib.error = urllib2
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053044 urllib.parse = urlparse
Dan Willemsen0745bb22015-08-17 13:41:45 -070045 urllib.request = urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053046 xmlrpc = imp.new_module('xmlrpc')
47 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070048
Roy Lee18afd7f2010-05-09 04:32:08 +080049try:
50 import threading as _threading
51except ImportError:
52 import dummy_threading as _threading
53
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070054try:
55 import resource
56 def _rlimit_nofile():
57 return resource.getrlimit(resource.RLIMIT_NOFILE)
58except ImportError:
59 def _rlimit_nofile():
60 return (256, 256)
61
Dave Borowitz18857212012-10-23 17:02:59 -070062try:
63 import multiprocessing
64except ImportError:
65 multiprocessing = None
66
Dave Borowitze2152672012-10-31 12:24:38 -070067from git_command import GIT, git_require
David Pursehouseba7bc732015-08-20 16:55:42 +090068from git_config import GetUrlCookieFile
David Pursehoused94aaef2012-09-07 09:52:04 +090069from git_refs import R_HEADS, HEAD
Simran Basibdb52712015-08-10 13:23:23 -070070import gitc_utils
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070071from project import Project
72from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080073from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000074from error import RepoChangedException, GitError, ManifestParseError
Shawn O. Pearce350cde42009-04-16 11:21:18 -070075from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070076from progress import Progress
Conley Owens094cdbe2014-01-30 15:09:59 -080077from wrapper import Wrapper
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070078
Dave Borowitz67700e92012-10-23 15:00:54 -070079_ONE_DAY_S = 24 * 60 * 60
80
Doug Andersonfc06ced2011-03-16 15:49:18 -070081class _FetchError(Exception):
82 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
83 pass
84
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080085class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080086 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070087 common = True
88 helpSummary = "Update working tree to the latest revision"
89 helpUsage = """
90%prog [<project>...]
91"""
92 helpDescription = """
93The '%prog' command synchronizes local project directories
94with the remote repositories specified in the manifest. If a local
95project does not yet exist, it will clone a new local directory from
96the remote repository and set up tracking branches as specified in
97the manifest. If the local project already exists, '%prog'
98will update the remote branches and rebase any new local changes
99on top of the new remote changes.
100
101'%prog' will synchronize all projects listed at the command
102line. Projects can be specified either by name, or by a relative
103or absolute path to the project's local directory. If no projects
104are specified, '%prog' will synchronize all projects listed in
105the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700106
107The -d/--detach option can be used to switch specified projects
108back to the manifest revision. This option is especially helpful
109if the project is currently on a topic branch, but the manifest
110revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700111
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700112The -s/--smart-sync option can be used to sync to a known good
113build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200114manifest. The -t/--smart-tag option is similar and allows you to
115specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700116
David Pursehousecf76b1b2012-09-14 10:31:42 +0900117The -u/--manifest-server-username and -p/--manifest-server-password
118options can be used to specify a username and password to authenticate
119with the manifest server when using the -s or -t option.
120
121If -u and -p are not specified when using the -s or -t option, '%prog'
122will attempt to read authentication credentials for the manifest server
123from the user's .netrc file.
124
125'%prog' will not use authentication credentials from -u/-p or .netrc
126if the manifest server specified in the manifest file already includes
127credentials.
128
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500129The -f/--force-broken option can be used to proceed with syncing
130other projects if a project sync fails.
131
Kevin Degiabaa7f32014-11-12 11:27:45 -0700132The --force-sync option can be used to overwrite existing git
133directories if they have previously been linked to a different
134object direcotry. WARNING: This may cause data to be lost since
135refs may be removed when overwriting.
136
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700137The --no-clone-bundle option disables any attempt to use
138$URL/clone.bundle to bootstrap a new Git repository from a
139resumeable bundle file on a content delivery network. This
140may be necessary if there are problems with the local Python
141HTTP client or proxy configuration, but the Git binary works.
142
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800143The --fetch-submodules option enables fetching Git submodules
144of a project from server.
145
David Pursehousef2fad612015-01-29 14:36:28 +0900146The -c/--current-branch option can be used to only fetch objects that
147are on the branch specified by a project's revision.
148
David Pursehouseb1553542014-09-04 21:28:09 +0900149The --optimized-fetch option can be used to only fetch projects that
150are fixed to a sha1 revision if the sha1 revision does not already
151exist locally.
152
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700153SSH Connections
154---------------
155
156If at least one project remote URL uses an SSH connection (ssh://,
157git+ssh://, or user@host:path syntax) repo will automatically
158enable the SSH ControlMaster option when connecting to that host.
159This feature permits other projects in the same '%prog' session to
160reuse the same SSH tunnel, saving connection setup overheads.
161
162To disable this behavior on UNIX platforms, set the GIT_SSH
163environment variable to 'ssh'. For example:
164
165 export GIT_SSH=ssh
166 %prog
167
168Compatibility
169~~~~~~~~~~~~~
170
171This feature is automatically disabled on Windows, due to the lack
172of UNIX domain socket support.
173
174This feature is not compatible with url.insteadof rewrites in the
175user's ~/.gitconfig. '%prog' is currently not able to perform the
176rewrite early enough to establish the ControlMaster tunnel.
177
178If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
179later is required to fix a server side protocol bug.
180
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700181"""
182
Nico Sallembien6623b212010-05-11 12:57:01 -0700183 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000184 try:
185 self.jobs = self.manifest.default.sync_j
186 except ManifestParseError:
187 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700188
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500189 p.add_option('-f', '--force-broken',
190 dest='force_broken', action='store_true',
191 help="continue sync even if a project fails to sync")
Kevin Degiabaa7f32014-11-12 11:27:45 -0700192 p.add_option('--force-sync',
193 dest='force_sync', action='store_true',
194 help="overwrite an existing git directory if it needs to "
195 "point to a different object directory. WARNING: this "
196 "may cause loss of data")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900197 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700198 dest='local_only', action='store_true',
199 help="only update working tree, don't fetch")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900200 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700201 dest='network_only', action='store_true',
202 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900203 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700204 dest='detach_head', action='store_true',
205 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900206 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700207 dest='current_branch_only', action='store_true',
208 help='fetch only current branch from server')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900209 p.add_option('-q', '--quiet',
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700210 dest='quiet', action='store_true',
211 help='be more quiet')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900212 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800213 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700214 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500215 p.add_option('-m', '--manifest-name',
216 dest='manifest_name',
217 help='temporary manifest to use for this sync', metavar='NAME.xml')
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700218 p.add_option('--no-clone-bundle',
219 dest='no_clone_bundle', action='store_true',
220 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800221 p.add_option('-u', '--manifest-server-username', action='store',
222 dest='manifest_server_username',
223 help='username to authenticate with the manifest server')
224 p.add_option('-p', '--manifest-server-password', action='store',
225 dest='manifest_server_password',
226 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800227 p.add_option('--fetch-submodules',
228 dest='fetch_submodules', action='store_true',
229 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700230 p.add_option('--no-tags',
231 dest='no_tags', action='store_true',
232 help="don't fetch tags")
David Pursehouseb1553542014-09-04 21:28:09 +0900233 p.add_option('--optimized-fetch',
234 dest='optimized_fetch', action='store_true',
235 help='only fetch projects fixed to sha1 if revision does not exist locally')
Nico Sallembien6623b212010-05-11 12:57:01 -0700236 if show_smart:
237 p.add_option('-s', '--smart-sync',
238 dest='smart_sync', action='store_true',
239 help='smart sync using manifest from a known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200240 p.add_option('-t', '--smart-tag',
241 dest='smart_tag', action='store',
242 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700243
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700244 g = p.add_option_group('repo Version options')
245 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700246 dest='no_repo_verify', action='store_true',
247 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700248 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800249 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700250 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700251
David James89ece422014-01-09 18:51:58 -0800252 def _FetchProjectList(self, opt, projects, *args, **kwargs):
David Pursehousec1b86a22012-11-14 11:36:51 +0900253 """Main function of the fetch threads when jobs are > 1.
Roy Lee18afd7f2010-05-09 04:32:08 +0800254
David James8d201162013-10-11 17:03:19 -0700255 Delegates most of the work to _FetchHelper.
256
257 Args:
258 opt: Program options returned from optparse. See _Options().
259 projects: Projects to fetch.
David James89ece422014-01-09 18:51:58 -0800260 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
David James8d201162013-10-11 17:03:19 -0700261 _FetchHelper docstring for details.
262 """
263 for project in projects:
David James89ece422014-01-09 18:51:58 -0800264 success = self._FetchHelper(opt, project, *args, **kwargs)
David James8d201162013-10-11 17:03:19 -0700265 if not success and not opt.force_broken:
266 break
267
268 def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
269 """Fetch git objects for a single project.
270
David Pursehousec1b86a22012-11-14 11:36:51 +0900271 Args:
272 opt: Program options returned from optparse. See _Options().
273 project: Project object for the project to fetch.
274 lock: Lock for accessing objects that are shared amongst multiple
275 _FetchHelper() threads.
276 fetched: set object that we will add project.gitdir to when we're done
277 (with our lock held).
278 pm: Instance of a Project object. We will call pm.update() (with our
279 lock held).
280 sem: We'll release() this semaphore when we exit so that another thread
281 can be started up.
282 err_event: We'll set this event in the case of an error (after printing
283 out info about the error).
David James8d201162013-10-11 17:03:19 -0700284
285 Returns:
286 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900287 """
288 # We'll set to true once we've locked the lock.
289 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700290
Chirayu Desaifef4ae72013-04-12 14:54:32 +0530291 if not opt.quiet:
292 print('Fetching project %s' % project.name)
293
David Pursehousec1b86a22012-11-14 11:36:51 +0900294 # Encapsulate everything in a try/except/finally so that:
295 # - We always set err_event in the case of an exception.
296 # - We always make sure we call sem.release().
297 # - We always make sure we unlock the lock if we locked it.
298 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700299 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900300 start = time.time()
301 success = project.Sync_NetworkHalf(
302 quiet=opt.quiet,
303 current_branch_only=opt.current_branch_only,
Kevin Degiabaa7f32014-11-12 11:27:45 -0700304 force_sync=opt.force_sync,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700305 clone_bundle=not opt.no_clone_bundle,
David Pursehouseb1553542014-09-04 21:28:09 +0900306 no_tags=opt.no_tags, archive=self.manifest.IsArchive,
307 optimized_fetch=opt.optimized_fetch)
David Pursehousec1b86a22012-11-14 11:36:51 +0900308 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700309
David Pursehousec1b86a22012-11-14 11:36:51 +0900310 # Lock around all the rest of the code, since printing, updating a set
311 # and Progress.update() are not thread safe.
312 lock.acquire()
313 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700314
David Pursehousec1b86a22012-11-14 11:36:51 +0900315 if not success:
316 print('error: Cannot fetch %s' % project.name, file=sys.stderr)
317 if opt.force_broken:
318 print('warn: --force-broken, continuing to sync',
319 file=sys.stderr)
320 else:
321 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700322
David Pursehousec1b86a22012-11-14 11:36:51 +0900323 fetched.add(project.gitdir)
324 pm.update()
325 except _FetchError:
326 err_event.set()
Dan Sandlerc5cd4332015-07-31 09:37:53 -0400327 except Exception as e:
328 print('error: Cannot fetch %s (%s: %s)' \
329 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900330 err_event.set()
331 raise
332 finally:
333 if did_lock:
334 lock.release()
335 sem.release()
Roy Lee18afd7f2010-05-09 04:32:08 +0800336
David James8d201162013-10-11 17:03:19 -0700337 return success
338
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700339 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700340 fetched = set()
David James89ece422014-01-09 18:51:58 -0800341 lock = _threading.Lock()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700342 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800343
David James89ece422014-01-09 18:51:58 -0800344 objdir_project_map = dict()
345 for project in projects:
346 objdir_project_map.setdefault(project.objdir, []).append(project)
David James8d201162013-10-11 17:03:19 -0700347
David James89ece422014-01-09 18:51:58 -0800348 threads = set()
349 sem = _threading.Semaphore(self.jobs)
350 err_event = _threading.Event()
351 for project_list in objdir_project_map.values():
352 # Check for any errors before running any more tasks.
353 # ...we'll let existing threads finish, though.
354 if err_event.isSet() and not opt.force_broken:
355 break
Doug Andersonfc06ced2011-03-16 15:49:18 -0700356
David James89ece422014-01-09 18:51:58 -0800357 sem.acquire()
358 kwargs = dict(opt=opt,
359 projects=project_list,
360 lock=lock,
361 fetched=fetched,
362 pm=pm,
363 sem=sem,
364 err_event=err_event)
365 if self.jobs > 1:
David James8d201162013-10-11 17:03:19 -0700366 t = _threading.Thread(target = self._FetchProjectList,
David James89ece422014-01-09 18:51:58 -0800367 kwargs = kwargs)
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200368 # Ensure that Ctrl-C will not freeze the repo process.
369 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800370 threads.add(t)
371 t.start()
David James89ece422014-01-09 18:51:58 -0800372 else:
373 self._FetchProjectList(**kwargs)
Roy Lee18afd7f2010-05-09 04:32:08 +0800374
David James89ece422014-01-09 18:51:58 -0800375 for t in threads:
376 t.join()
Roy Lee18afd7f2010-05-09 04:32:08 +0800377
David James89ece422014-01-09 18:51:58 -0800378 # If we saw an error, exit with code 1 so that other scripts can check.
379 if err_event.isSet():
380 print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
381 sys.exit(1)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700382
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700383 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700384 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700385
Julien Campergue335f5ef2013-10-16 11:02:35 +0200386 if not self.manifest.IsArchive:
387 self._GCProjects(projects)
388
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700389 return fetched
390
Dave Borowitz18857212012-10-23 17:02:59 -0700391 def _GCProjects(self, projects):
David James8d201162013-10-11 17:03:19 -0700392 gitdirs = {}
393 for project in projects:
394 gitdirs[project.gitdir] = project.bare_git
395
Dave Borowitze2152672012-10-31 12:24:38 -0700396 has_dash_c = git_require((1, 7, 2))
397 if multiprocessing and has_dash_c:
Dave Borowitz18857212012-10-23 17:02:59 -0700398 cpu_count = multiprocessing.cpu_count()
399 else:
400 cpu_count = 1
401 jobs = min(self.jobs, cpu_count)
402
403 if jobs < 2:
David James8d201162013-10-11 17:03:19 -0700404 for bare_git in gitdirs.values():
405 bare_git.gc('--auto')
Dave Borowitz18857212012-10-23 17:02:59 -0700406 return
407
408 config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1}
409
410 threads = set()
411 sem = _threading.Semaphore(jobs)
412 err_event = _threading.Event()
413
David James8d201162013-10-11 17:03:19 -0700414 def GC(bare_git):
Dave Borowitz18857212012-10-23 17:02:59 -0700415 try:
416 try:
David James8d201162013-10-11 17:03:19 -0700417 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700418 except GitError:
419 err_event.set()
420 except:
421 err_event.set()
422 raise
423 finally:
424 sem.release()
425
David James8d201162013-10-11 17:03:19 -0700426 for bare_git in gitdirs.values():
Dave Borowitz18857212012-10-23 17:02:59 -0700427 if err_event.isSet():
428 break
429 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700430 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700431 t.daemon = True
432 threads.add(t)
433 t.start()
434
435 for t in threads:
436 t.join()
437
438 if err_event.isSet():
Sarah Owenscecd1d82012-11-01 22:59:27 -0700439 print('\nerror: Exited sync due to gc errors', file=sys.stderr)
Dave Borowitz18857212012-10-23 17:02:59 -0700440 sys.exit(1)
441
Tim Kilbourn07669002013-03-08 15:02:49 -0800442 def _ReloadManifest(self, manifest_name=None):
443 if manifest_name:
444 # Override calls _Unload already
445 self.manifest.Override(manifest_name)
446 else:
447 self.manifest._Unload()
448
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700449 def UpdateProjectList(self):
450 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700451 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700452 if project.relpath:
453 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700454 file_name = 'project.list'
455 file_path = os.path.join(self.manifest.repodir, file_name)
456 old_project_paths = []
457
458 if os.path.exists(file_path):
459 fd = open(file_path, 'r')
460 try:
461 old_project_paths = fd.read().split('\n')
462 finally:
463 fd.close()
464 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700465 if not path:
466 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700467 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900468 # If the path has already been deleted, we don't need to do it
Anthonyf3fdf822009-09-26 13:38:52 -0400469 if os.path.exists(self.manifest.topdir + '/' + path):
David James8d201162013-10-11 17:03:19 -0700470 gitdir = os.path.join(self.manifest.topdir, path, '.git')
David Pursehousec1b86a22012-11-14 11:36:51 +0900471 project = Project(
472 manifest = self.manifest,
473 name = path,
474 remote = RemoteSpec('origin'),
David James8d201162013-10-11 17:03:19 -0700475 gitdir = gitdir,
476 objdir = gitdir,
David Pursehousec1b86a22012-11-14 11:36:51 +0900477 worktree = os.path.join(self.manifest.topdir, path),
478 relpath = path,
479 revisionExpr = 'HEAD',
480 revisionId = None,
481 groups = None)
Anthonyf3fdf822009-09-26 13:38:52 -0400482
David Pursehousec1b86a22012-11-14 11:36:51 +0900483 if project.IsDirty():
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900484 print('error: Cannot remove project "%s": uncommitted changes '
David Pursehousec1b86a22012-11-14 11:36:51 +0900485 'are present' % project.relpath, file=sys.stderr)
486 print(' commit changes, then run sync again',
487 file=sys.stderr)
488 return -1
489 else:
490 print('Deleting obsolete path %s' % project.worktree,
491 file=sys.stderr)
492 shutil.rmtree(project.worktree)
493 # Try deleting parent subdirs if they are empty
494 project_dir = os.path.dirname(project.worktree)
495 while project_dir != self.manifest.topdir:
496 try:
497 os.rmdir(project_dir)
498 except OSError:
499 break
500 project_dir = os.path.dirname(project_dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700501
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700502 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700503 fd = open(file_path, 'w')
504 try:
505 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700506 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700507 finally:
508 fd.close()
509 return 0
510
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700511 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800512 if opt.jobs:
513 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700514 if self.jobs > 1:
515 soft_limit, _ = _rlimit_nofile()
516 self.jobs = min(self.jobs, (soft_limit - 5) / 3)
517
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700518 if opt.network_only and opt.detach_head:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700519 print('error: cannot combine -n and -d', file=sys.stderr)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700520 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700521 if opt.network_only and opt.local_only:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700522 print('error: cannot combine -n and -l', file=sys.stderr)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700523 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500524 if opt.manifest_name and opt.smart_sync:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700525 print('error: cannot combine -m and -s', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500526 sys.exit(1)
527 if opt.manifest_name and opt.smart_tag:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700528 print('error: cannot combine -m and -t', file=sys.stderr)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500529 sys.exit(1)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900530 if opt.manifest_server_username or opt.manifest_server_password:
531 if not (opt.smart_sync or opt.smart_tag):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700532 print('error: -u and -p may only be combined with -s or -t',
533 file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900534 sys.exit(1)
535 if None in [opt.manifest_server_username, opt.manifest_server_password]:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700536 print('error: both -u and -p must be given', file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900537 sys.exit(1)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500538
539 if opt.manifest_name:
540 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700541
Chirayu Desaia892b102013-06-11 14:18:46 +0530542 manifest_name = opt.manifest_name
David Pursehouse59b41742015-05-07 14:36:09 +0900543 smart_sync_manifest_name = "smart_sync_override.xml"
544 smart_sync_manifest_path = os.path.join(
545 self.manifest.manifestProject.worktree, smart_sync_manifest_name)
Chirayu Desaia892b102013-06-11 14:18:46 +0530546
Victor Boivie08c880d2011-04-19 10:32:52 +0200547 if opt.smart_sync or opt.smart_tag:
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700548 if not self.manifest.manifest_server:
David Pursehouse2f9e7e42013-03-05 17:26:46 +0900549 print('error: cannot smart sync: no manifest server defined in '
Sarah Owenscecd1d82012-11-01 22:59:27 -0700550 'manifest', file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700551 sys.exit(1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900552
553 manifest_server = self.manifest.manifest_server
David Pursehousefb99c712013-09-25 11:09:34 +0900554 if not opt.quiet:
555 print('Using manifest server %s' % manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900556
David Pursehouse86d973d2012-08-24 10:21:02 +0900557 if not '@' in manifest_server:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900558 username = None
559 password = None
560 if opt.manifest_server_username and opt.manifest_server_password:
561 username = opt.manifest_server_username
562 password = opt.manifest_server_password
David Pursehouse86d973d2012-08-24 10:21:02 +0900563 else:
564 try:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900565 info = netrc.netrc()
566 except IOError:
David Pursehouseba7bc732015-08-20 16:55:42 +0900567 # .netrc file does not exist or could not be opened
568 pass
David Pursehouse86d973d2012-08-24 10:21:02 +0900569 else:
David Pursehousecf76b1b2012-09-14 10:31:42 +0900570 try:
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +0530571 parse_result = urllib.parse.urlparse(manifest_server)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900572 if parse_result.hostname:
David Pursehouseba7bc732015-08-20 16:55:42 +0900573 auth = info.authenticators(parse_result.hostname)
574 if auth:
575 username, _account, password = auth
576 else:
577 print('No credentials found for %s in .netrc'
578 % parse_result.hostname, file=sys.stderr)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700579 except netrc.NetrcParseError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700580 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
David Pursehousecf76b1b2012-09-14 10:31:42 +0900581
582 if (username and password):
583 manifest_server = manifest_server.replace('://', '://%s:%s@' %
584 (username, password),
585 1)
David Pursehouse86d973d2012-08-24 10:21:02 +0900586
Dan Willemsen0745bb22015-08-17 13:41:45 -0700587 transport = PersistentTransport(manifest_server)
588 if manifest_server.startswith('persistent-'):
589 manifest_server = manifest_server[len('persistent-'):]
590
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700591 try:
Dan Willemsen0745bb22015-08-17 13:41:45 -0700592 server = xmlrpc.client.Server(manifest_server, transport=transport)
Victor Boivie08c880d2011-04-19 10:32:52 +0200593 if opt.smart_sync:
594 p = self.manifest.manifestProject
595 b = p.GetBranch(p.CurrentBranch)
596 branch = b.merge
597 if branch.startswith(R_HEADS):
598 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700599
Victor Boivie08c880d2011-04-19 10:32:52 +0200600 env = os.environ.copy()
Jeff Davidson5cf16602014-10-02 10:13:38 -0700601 if 'SYNC_TARGET' in env:
602 target = env['SYNC_TARGET']
603 [success, manifest_str] = server.GetApprovedManifest(branch, target)
604 elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
Victor Boivie08c880d2011-04-19 10:32:52 +0200605 target = '%s-%s' % (env['TARGET_PRODUCT'],
606 env['TARGET_BUILD_VARIANT'])
607 [success, manifest_str] = server.GetApprovedManifest(branch, target)
608 else:
609 [success, manifest_str] = server.GetApprovedManifest(branch)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700610 else:
Victor Boivie08c880d2011-04-19 10:32:52 +0200611 assert(opt.smart_tag)
612 [success, manifest_str] = server.GetManifest(opt.smart_tag)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700613
614 if success:
David Pursehouse59b41742015-05-07 14:36:09 +0900615 manifest_name = smart_sync_manifest_name
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700616 try:
David Pursehouse59b41742015-05-07 14:36:09 +0900617 f = open(smart_sync_manifest_path, 'w')
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700618 try:
619 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700620 finally:
621 f.close()
David Pursehouse727cc3e2015-05-07 14:16:49 +0900622 except IOError as e:
623 print('error: cannot write manifest to %s:\n%s'
David Pursehouse59b41742015-05-07 14:36:09 +0900624 % (smart_sync_manifest_path, e),
Sarah Owenscecd1d82012-11-01 22:59:27 -0700625 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700626 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100627 self._ReloadManifest(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700628 else:
David Pursehouse351fe2c2013-09-25 17:54:26 +0900629 print('error: manifest server RPC call failed: %s' %
630 manifest_str, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700631 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530632 except (socket.error, IOError, xmlrpc.client.Fault) as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700633 print('error: cannot connect to manifest server %s:\n%s'
634 % (self.manifest.manifest_server, e), file=sys.stderr)
David Pursehousebd489c42012-08-23 10:21:26 +0900635 sys.exit(1)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530636 except xmlrpc.client.ProtocolError as e:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700637 print('error: cannot connect to manifest server %s:\n%d %s'
638 % (self.manifest.manifest_server, e.errcode, e.errmsg),
639 file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700640 sys.exit(1)
David Pursehouse59b41742015-05-07 14:36:09 +0900641 else: # Not smart sync or smart tag mode
642 if os.path.isfile(smart_sync_manifest_path):
643 try:
644 os.remove(smart_sync_manifest_path)
645 except OSError as e:
646 print('error: failed to remove existing smart sync override manifest: %s' %
647 e, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700648
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700649 rp = self.manifest.repoProject
650 rp.PreSync()
651
652 mp = self.manifest.manifestProject
653 mp.PreSync()
654
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800655 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700656 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800657
Nico Sallembien9bb18162009-12-07 15:38:01 -0800658 if not opt.local_only:
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700659 mp.Sync_NetworkHalf(quiet=opt.quiet,
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700660 current_branch_only=opt.current_branch_only,
David Pursehouseb1553542014-09-04 21:28:09 +0900661 no_tags=opt.no_tags,
662 optimized_fetch=opt.optimized_fetch)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800663
664 if mp.HasChanges:
665 syncbuf = SyncBuffer(mp.config)
666 mp.Sync_LocalHalf(syncbuf)
667 if not syncbuf.Finish():
668 sys.exit(1)
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100669 self._ReloadManifest(manifest_name)
Shawn O. Pearcec4657962011-09-26 09:08:01 -0700670 if opt.jobs is None:
671 self.jobs = self.manifest.default.sync_j
Simran Basib9a1b732015-08-20 12:19:28 -0700672
673 # TODO (sbasi) - Add support for manifest changes, aka projects
674 # have been added or deleted from the manifest.
675 if self.gitc_manifest:
676 gitc_manifest_projects = self.GetProjects(args,
677 manifest=self.gitc_manifest,
678 missing_ok=True)
679 gitc_projects = []
680 opened_projects = []
681 for project in gitc_manifest_projects:
682 if not project.old_revision:
683 gitc_projects.append(project)
684 else:
685 opened_projects.append(project)
686
687 if gitc_projects and not opt.local_only:
688 print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
689 gitc_utils.generate_gitc_manifest(self.gitc_manifest.gitc_client_dir,
690 self.gitc_manifest,
691 gitc_projects)
692 print('GITC client successfully synced.')
693
694 # The opened projects need to be synced as normal, therefore we
695 # generate a new args list to represent the opened projects.
696 args = []
697 for proj in opened_projects:
698 args.append(os.path.relpath(proj.worktree, cwd))
699 if not args:
700 return
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800701 all_projects = self.GetProjects(args,
702 missing_ok=True,
703 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700704
Dave Borowitz67700e92012-10-23 15:00:54 -0700705 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700706 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700707 to_fetch = []
708 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700709 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700710 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900711 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700712 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700713
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800714 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700715 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700716 if opt.network_only:
717 # bail out now; the rest touches the working tree
718 return
719
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800720 # Iteratively fetch missing and/or nested unregistered submodules
721 previously_missing_set = set()
722 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100723 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800724 all_projects = self.GetProjects(args,
725 missing_ok=True,
726 submodules_ok=opt.fetch_submodules)
727 missing = []
728 for project in all_projects:
729 if project.gitdir not in fetched:
730 missing.append(project)
731 if not missing:
732 break
733 # Stop us from non-stopped fetching actually-missing repos: If set of
734 # missing repos has not been changed from last fetch, we break.
735 missing_set = set(p.name for p in missing)
736 if previously_missing_set == missing_set:
737 break
738 previously_missing_set = missing_set
739 fetched.update(self._Fetch(missing, opt))
740
Julien Campergue335f5ef2013-10-16 11:02:35 +0200741 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700742 # bail out now, we have no working tree
743 return
744
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700745 if self.UpdateProjectList():
746 sys.exit(1)
747
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700748 syncbuf = SyncBuffer(mp.config,
749 detach_head = opt.detach_head)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900750 pm = Progress('Syncing work tree', len(all_projects))
751 for project in all_projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700752 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800753 if project.worktree:
Kevin Degiabaa7f32014-11-12 11:27:45 -0700754 project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700755 pm.end()
Sarah Owenscecd1d82012-11-01 22:59:27 -0700756 print(file=sys.stderr)
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700757 if not syncbuf.Finish():
758 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700759
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700760 # If there's a notice that's supposed to print at the end of the sync, print
761 # it now...
762 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700763 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700764
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700765def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -0800766 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -0700767 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700768 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -0800769 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700770 if project.Exists:
771 project.PostRepoUpgrade()
772
773def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
774 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700775 print('info: A new version of repo is available', file=sys.stderr)
776 print(file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700777 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700778 syncbuf = SyncBuffer(rp.config)
779 rp.Sync_LocalHalf(syncbuf)
780 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700781 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -0700782 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700783 raise RepoChangedException(['--repo-upgraded'])
784 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700785 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700786 else:
787 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700788 print('repo version %s is current' % rp.work_git.describe(HEAD),
789 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700790
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700791def _VerifyTag(project):
792 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
793 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -0700794 print('warning: GnuPG was not available during last "repo init"\n'
795 'warning: Cannot automatically authenticate repo."""',
796 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700797 return True
798
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700799 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700800 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700801 except GitError:
802 cur = None
803
804 if not cur \
805 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700806 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700807 if rev.startswith(R_HEADS):
808 rev = rev[len(R_HEADS):]
809
Sarah Owenscecd1d82012-11-01 22:59:27 -0700810 print(file=sys.stderr)
811 print("warning: project '%s' branch '%s' is not signed"
812 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700813 return False
814
Shawn O. Pearcef18cb762010-12-07 11:41:05 -0800815 env = os.environ.copy()
816 env['GIT_DIR'] = project.gitdir.encode()
817 env['GNUPGHOME'] = gpg_dir.encode()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700818
819 cmd = [GIT, 'tag', '-v', cur]
820 proc = subprocess.Popen(cmd,
821 stdout = subprocess.PIPE,
822 stderr = subprocess.PIPE,
823 env = env)
824 out = proc.stdout.read()
825 proc.stdout.close()
826
827 err = proc.stderr.read()
828 proc.stderr.close()
829
830 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700831 print(file=sys.stderr)
832 print(out, file=sys.stderr)
833 print(err, file=sys.stderr)
834 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700835 return False
836 return True
Dave Borowitz67700e92012-10-23 15:00:54 -0700837
838class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -0700839 _ALPHA = 0.5
840
Dave Borowitz67700e92012-10-23 15:00:54 -0700841 def __init__(self, manifest):
Anthony King85b24ac2014-05-06 15:57:48 +0100842 self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
Dave Borowitz67700e92012-10-23 15:00:54 -0700843 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -0700844 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -0700845
846 def Get(self, project):
847 self._Load()
848 return self._times.get(project.name, _ONE_DAY_S)
849
850 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -0700851 self._Load()
852 name = project.name
853 old = self._times.get(name, t)
854 self._seen.add(name)
855 a = self._ALPHA
856 self._times[name] = (a*t) + ((1-a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -0700857
858 def _Load(self):
859 if self._times is None:
860 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100861 f = open(self._path)
Dave Borowitz67700e92012-10-23 15:00:54 -0700862 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100863 self._times = json.load(f)
864 finally:
865 f.close()
866 except (IOError, ValueError):
867 try:
868 os.remove(self._path)
869 except OSError:
870 pass
871 self._times = {}
Dave Borowitz67700e92012-10-23 15:00:54 -0700872
873 def Save(self):
874 if self._times is None:
875 return
Dave Borowitzd9478582012-10-23 16:35:39 -0700876
877 to_delete = []
878 for name in self._times:
879 if name not in self._seen:
880 to_delete.append(name)
881 for name in to_delete:
882 del self._times[name]
883
Dave Borowitz67700e92012-10-23 15:00:54 -0700884 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100885 f = open(self._path, 'w')
Dave Borowitz67700e92012-10-23 15:00:54 -0700886 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100887 json.dump(self._times, f, indent=2)
888 finally:
889 f.close()
890 except (IOError, TypeError):
891 try:
892 os.remove(self._path)
893 except OSError:
894 pass
Dan Willemsen0745bb22015-08-17 13:41:45 -0700895
896# This is a replacement for xmlrpc.client.Transport using urllib2
897# and supporting persistent-http[s]. It cannot change hosts from
898# request to request like the normal transport, the real url
899# is passed during initialization.
900class PersistentTransport(xmlrpc.client.Transport):
901 def __init__(self, orig_host):
902 self.orig_host = orig_host
903
904 def request(self, host, handler, request_body, verbose=False):
905 with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
906 # Python doesn't understand cookies with the #HttpOnly_ prefix
907 # Since we're only using them for HTTP, copy the file temporarily,
908 # stripping those prefixes away.
Dan Willemsen3010e5b2015-08-20 10:09:20 -0700909 if cookiefile:
910 tmpcookiefile = tempfile.NamedTemporaryFile()
911 try:
912 with open(cookiefile) as f:
913 for line in f:
914 if line.startswith("#HttpOnly_"):
915 line = line[len("#HttpOnly_"):]
916 tmpcookiefile.write(line)
917 tmpcookiefile.flush()
Dan Willemsen0745bb22015-08-17 13:41:45 -0700918
Dan Willemsen3010e5b2015-08-20 10:09:20 -0700919 cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
920 cookiejar.load()
921 finally:
922 tmpcookiefile.close()
923 else:
924 cookiejar = cookielib.CookieJar()
Dan Willemsen0745bb22015-08-17 13:41:45 -0700925
926 proxyhandler = urllib.request.ProxyHandler
927 if proxy:
928 proxyhandler = urllib.request.ProxyHandler({
929 "http": proxy,
930 "https": proxy })
931
932 opener = urllib.request.build_opener(
933 urllib.request.HTTPCookieProcessor(cookiejar),
934 proxyhandler)
935
936 url = urllib.parse.urljoin(self.orig_host, handler)
937 parse_results = urllib.parse.urlparse(url)
938
939 scheme = parse_results.scheme
940 if scheme == 'persistent-http':
941 scheme = 'http'
942 if scheme == 'persistent-https':
943 # If we're proxying through persistent-https, use http. The
944 # proxy itself will do the https.
945 if proxy:
946 scheme = 'http'
947 else:
948 scheme = 'https'
949
950 # Parse out any authentication information using the base class
951 host, extra_headers, _ = self.get_host_info(parse_results.netloc)
952
953 url = urllib.parse.urlunparse((
954 scheme,
955 host,
956 parse_results.path,
957 parse_results.params,
958 parse_results.query,
959 parse_results.fragment))
960
961 request = urllib.request.Request(url, request_body)
962 if extra_headers is not None:
963 for (name, header) in extra_headers:
964 request.add_header(name, header)
965 request.add_header('Content-Type', 'text/xml')
966 try:
967 response = opener.open(request)
968 except urllib.error.HTTPError as e:
969 if e.code == 501:
970 # We may have been redirected through a login process
971 # but our POST turned into a GET. Retry.
972 response = opener.open(request)
973 else:
974 raise
975
976 p, u = xmlrpc.client.getparser()
977 while 1:
978 data = response.read(1024)
979 if not data:
980 break
981 p.feed(data)
982 p.close()
983 return u.close()
984
985 def close(self):
986 pass
987