blob: ca78467b4946495cbfb942aecd4119bc2b307f80 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070016from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
18import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070019import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070020import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import subprocess
22import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070023import time
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070024import xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025
Roy Lee18afd7f2010-05-09 04:32:08 +080026try:
27 import threading as _threading
28except ImportError:
29 import dummy_threading as _threading
30
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070031from git_command import GIT
Nico Sallembien5732e472010-04-26 11:17:29 -070032from git_refs import R_HEADS
Shawn O. Pearcee756c412009-04-13 11:51:15 -070033from project import HEAD
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070034from project import Project
35from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080036from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070037from error import RepoChangedException, GitError
38from project import R_HEADS
Shawn O. Pearce350cde42009-04-16 11:21:18 -070039from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070040from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070041
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080042class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080043 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070044 common = True
45 helpSummary = "Update working tree to the latest revision"
46 helpUsage = """
47%prog [<project>...]
48"""
49 helpDescription = """
50The '%prog' command synchronizes local project directories
51with the remote repositories specified in the manifest. If a local
52project does not yet exist, it will clone a new local directory from
53the remote repository and set up tracking branches as specified in
54the manifest. If the local project already exists, '%prog'
55will update the remote branches and rebase any new local changes
56on top of the new remote changes.
57
58'%prog' will synchronize all projects listed at the command
59line. Projects can be specified either by name, or by a relative
60or absolute path to the project's local directory. If no projects
61are specified, '%prog' will synchronize all projects listed in
62the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070063
64The -d/--detach option can be used to switch specified projects
65back to the manifest revision. This option is especially helpful
66if the project is currently on a topic branch, but the manifest
67revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070068
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070069The -s/--smart-sync option can be used to sync to a known good
70build as specified by the manifest-server element in the current
71manifest.
72
Andrei Warkentin5df6de02010-07-02 17:58:31 -050073The -f/--force-broken option can be used to proceed with syncing
74other projects if a project sync fails.
75
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070076SSH Connections
77---------------
78
79If at least one project remote URL uses an SSH connection (ssh://,
80git+ssh://, or user@host:path syntax) repo will automatically
81enable the SSH ControlMaster option when connecting to that host.
82This feature permits other projects in the same '%prog' session to
83reuse the same SSH tunnel, saving connection setup overheads.
84
85To disable this behavior on UNIX platforms, set the GIT_SSH
86environment variable to 'ssh'. For example:
87
88 export GIT_SSH=ssh
89 %prog
90
91Compatibility
92~~~~~~~~~~~~~
93
94This feature is automatically disabled on Windows, due to the lack
95of UNIX domain socket support.
96
97This feature is not compatible with url.insteadof rewrites in the
98user's ~/.gitconfig. '%prog' is currently not able to perform the
99rewrite early enough to establish the ControlMaster tunnel.
100
101If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
102later is required to fix a server side protocol bug.
103
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700104"""
105
Nico Sallembien6623b212010-05-11 12:57:01 -0700106 def _Options(self, p, show_smart=True):
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500107 p.add_option('-f', '--force-broken',
108 dest='force_broken', action='store_true',
109 help="continue sync even if a project fails to sync")
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700110 p.add_option('-l','--local-only',
111 dest='local_only', action='store_true',
112 help="only update working tree, don't fetch")
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700113 p.add_option('-n','--network-only',
114 dest='network_only', action='store_true',
115 help="fetch only, don't update working tree")
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700116 p.add_option('-d','--detach',
117 dest='detach_head', action='store_true',
118 help='detach projects back to manifest revision')
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700119 p.add_option('-q','--quiet',
120 dest='quiet', action='store_true',
121 help='be more quiet')
Roy Lee18afd7f2010-05-09 04:32:08 +0800122 p.add_option('-j','--jobs',
123 dest='jobs', action='store', type='int',
124 help="number of projects to fetch simultaneously")
Nico Sallembien6623b212010-05-11 12:57:01 -0700125 if show_smart:
126 p.add_option('-s', '--smart-sync',
127 dest='smart_sync', action='store_true',
128 help='smart sync using manifest from a known good build')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700129
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700130 g = p.add_option_group('repo Version options')
131 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700132 dest='no_repo_verify', action='store_true',
133 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700134 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800135 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700136 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700137
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700138 def _FetchHelper(self, opt, project, lock, fetched, pm, sem):
139 if not project.Sync_NetworkHalf(quiet=opt.quiet):
Roy Lee18afd7f2010-05-09 04:32:08 +0800140 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500141 if opt.force_broken:
142 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
143 else:
144 sem.release()
145 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800146
147 lock.acquire()
148 fetched.add(project.gitdir)
149 pm.update()
150 lock.release()
151 sem.release()
152
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700153 def _Fetch(self, projects, opt):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700154 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700155 pm = Progress('Fetching projects', len(projects))
Roy Lee18afd7f2010-05-09 04:32:08 +0800156
157 if self.jobs == 1:
158 for project in projects:
159 pm.update()
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700160 if project.Sync_NetworkHalf(quiet=opt.quiet):
Roy Lee18afd7f2010-05-09 04:32:08 +0800161 fetched.add(project.gitdir)
162 else:
163 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500164 if opt.force_broken:
165 print >>sys.stderr, 'warn: --force-broken, continuing to sync'
166 else:
167 sys.exit(1)
Roy Lee18afd7f2010-05-09 04:32:08 +0800168 else:
169 threads = set()
170 lock = _threading.Lock()
171 sem = _threading.Semaphore(self.jobs)
172 for project in projects:
173 sem.acquire()
174 t = _threading.Thread(target = self._FetchHelper,
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700175 args = (opt,
176 project,
177 lock,
178 fetched,
179 pm,
180 sem))
Roy Lee18afd7f2010-05-09 04:32:08 +0800181 threads.add(t)
182 t.start()
183
184 for t in threads:
185 t.join()
186
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700187 pm.end()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700188 return fetched
189
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700190 def UpdateProjectList(self):
191 new_project_paths = []
192 for project in self.manifest.projects.values():
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700193 if project.relpath:
194 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700195 file_name = 'project.list'
196 file_path = os.path.join(self.manifest.repodir, file_name)
197 old_project_paths = []
198
199 if os.path.exists(file_path):
200 fd = open(file_path, 'r')
201 try:
202 old_project_paths = fd.read().split('\n')
203 finally:
204 fd.close()
205 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700206 if not path:
207 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700208 if path not in new_project_paths:
Anthonyf3fdf822009-09-26 13:38:52 -0400209 """If the path has already been deleted, we don't need to do it
210 """
211 if os.path.exists(self.manifest.topdir + '/' + path):
212 project = Project(
213 manifest = self.manifest,
214 name = path,
215 remote = RemoteSpec('origin'),
216 gitdir = os.path.join(self.manifest.topdir,
217 path, '.git'),
218 worktree = os.path.join(self.manifest.topdir, path),
219 relpath = path,
220 revisionExpr = 'HEAD',
221 revisionId = None)
222
223 if project.IsDirty():
224 print >>sys.stderr, 'error: Cannot remove project "%s": \
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700225uncommitted changes are present' % project.relpath
Anthonyf3fdf822009-09-26 13:38:52 -0400226 print >>sys.stderr, ' commit changes, then run sync again'
227 return -1
228 else:
229 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
230 shutil.rmtree(project.worktree)
231 # Try deleting parent subdirs if they are empty
232 dir = os.path.dirname(project.worktree)
233 while dir != self.manifest.topdir:
234 try:
235 os.rmdir(dir)
236 except OSError:
237 break
238 dir = os.path.dirname(dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700239
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700240 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700241 fd = open(file_path, 'w')
242 try:
243 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700244 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700245 finally:
246 fd.close()
247 return 0
248
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700249 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800250 if opt.jobs:
251 self.jobs = opt.jobs
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700252 if opt.network_only and opt.detach_head:
253 print >>sys.stderr, 'error: cannot combine -n and -d'
254 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700255 if opt.network_only and opt.local_only:
256 print >>sys.stderr, 'error: cannot combine -n and -l'
257 sys.exit(1)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700258
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700259 if opt.smart_sync:
260 if not self.manifest.manifest_server:
261 print >>sys.stderr, \
262 'error: cannot smart sync: no manifest server defined in manifest'
263 sys.exit(1)
264 try:
265 server = xmlrpclib.Server(self.manifest.manifest_server)
266 p = self.manifest.manifestProject
267 b = p.GetBranch(p.CurrentBranch)
268 branch = b.merge
Nico Sallembien5732e472010-04-26 11:17:29 -0700269 if branch.startswith(R_HEADS):
270 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700271
272 env = dict(os.environ)
273 if (env.has_key('TARGET_PRODUCT') and
274 env.has_key('TARGET_BUILD_VARIANT')):
275 target = '%s-%s' % (env['TARGET_PRODUCT'],
276 env['TARGET_BUILD_VARIANT'])
277 [success, manifest_str] = server.GetApprovedManifest(branch, target)
278 else:
279 [success, manifest_str] = server.GetApprovedManifest(branch)
280
281 if success:
282 manifest_name = "smart_sync_override.xml"
283 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
284 manifest_name)
285 try:
286 f = open(manifest_path, 'w')
287 try:
288 f.write(manifest_str)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700289 finally:
290 f.close()
291 except IOError:
292 print >>sys.stderr, 'error: cannot write manifest to %s' % \
293 manifest_path
294 sys.exit(1)
Nico Sallembien719965a2010-04-20 15:28:19 -0700295 self.manifest.Override(manifest_name)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700296 else:
297 print >>sys.stderr, 'error: %s' % manifest_str
298 sys.exit(1)
299 except socket.error:
300 print >>sys.stderr, 'error: cannot connect to manifest server %s' % (
301 self.manifest.manifest_server)
302 sys.exit(1)
303
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700304 rp = self.manifest.repoProject
305 rp.PreSync()
306
307 mp = self.manifest.manifestProject
308 mp.PreSync()
309
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800310 if opt.repo_upgraded:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700311 _PostRepoUpgrade(self.manifest)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800312
Nico Sallembien9bb18162009-12-07 15:38:01 -0800313 if not opt.local_only:
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700314 mp.Sync_NetworkHalf(quiet=opt.quiet)
Nico Sallembien9bb18162009-12-07 15:38:01 -0800315
316 if mp.HasChanges:
317 syncbuf = SyncBuffer(mp.config)
318 mp.Sync_LocalHalf(syncbuf)
319 if not syncbuf.Finish():
320 sys.exit(1)
321 self.manifest._Unload()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700322 all = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700323
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700324 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700325 to_fetch = []
326 now = time.time()
327 if (24 * 60 * 60) <= (now - rp.LastFetch):
328 to_fetch.append(rp)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700329 to_fetch.extend(all)
330
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700331 fetched = self._Fetch(to_fetch, opt)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700332 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700333 if opt.network_only:
334 # bail out now; the rest touches the working tree
335 return
336
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700337 self.manifest._Unload()
338 all = self.GetProjects(args, missing_ok=True)
339 missing = []
340 for project in all:
341 if project.gitdir not in fetched:
342 missing.append(project)
Shawn O. Pearce16614f82010-10-29 12:05:43 -0700343 self._Fetch(missing, opt)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700344
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700345 if self.manifest.IsMirror:
346 # bail out now, we have no working tree
347 return
348
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700349 if self.UpdateProjectList():
350 sys.exit(1)
351
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700352 syncbuf = SyncBuffer(mp.config,
353 detach_head = opt.detach_head)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700354 pm = Progress('Syncing work tree', len(all))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700355 for project in all:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700356 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800357 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700358 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700359 pm.end()
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700360 print >>sys.stderr
361 if not syncbuf.Finish():
362 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700363
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700364def _PostRepoUpgrade(manifest):
365 for project in manifest.projects.values():
366 if project.Exists:
367 project.PostRepoUpgrade()
368
369def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
370 if rp.HasChanges:
371 print >>sys.stderr, 'info: A new version of repo is available'
372 print >>sys.stderr, ''
373 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700374 syncbuf = SyncBuffer(rp.config)
375 rp.Sync_LocalHalf(syncbuf)
376 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700377 sys.exit(1)
378 print >>sys.stderr, 'info: Restarting repo with latest version'
379 raise RepoChangedException(['--repo-upgraded'])
380 else:
381 print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
382 else:
383 if verbose:
384 print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
385
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700386def _VerifyTag(project):
387 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
388 if not os.path.exists(gpg_dir):
389 print >>sys.stderr,\
390"""warning: GnuPG was not available during last "repo init"
391warning: Cannot automatically authenticate repo."""
392 return True
393
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700394 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700395 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700396 except GitError:
397 cur = None
398
399 if not cur \
400 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700401 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700402 if rev.startswith(R_HEADS):
403 rev = rev[len(R_HEADS):]
404
405 print >>sys.stderr
406 print >>sys.stderr,\
407 "warning: project '%s' branch '%s' is not signed" \
408 % (project.name, rev)
409 return False
410
411 env = dict(os.environ)
412 env['GIT_DIR'] = project.gitdir
413 env['GNUPGHOME'] = gpg_dir
414
415 cmd = [GIT, 'tag', '-v', cur]
416 proc = subprocess.Popen(cmd,
417 stdout = subprocess.PIPE,
418 stderr = subprocess.PIPE,
419 env = env)
420 out = proc.stdout.read()
421 proc.stdout.close()
422
423 err = proc.stderr.read()
424 proc.stderr.close()
425
426 if proc.wait() != 0:
427 print >>sys.stderr
428 print >>sys.stderr, out
429 print >>sys.stderr, err
430 print >>sys.stderr
431 return False
432 return True