blob: 67213d3ab778dcd578602709ff5800ec27fc5728 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070016from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
18import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070019import shutil
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070020import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import subprocess
22import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070023import time
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070024import xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025
26from git_command import GIT
Nico Sallembien5732e472010-04-26 11:17:29 -070027from git_refs import R_HEADS
Shawn O. Pearcee756c412009-04-13 11:51:15 -070028from project import HEAD
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070029from project import Project
30from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080031from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070032from error import RepoChangedException, GitError
33from project import R_HEADS
Shawn O. Pearce350cde42009-04-16 11:21:18 -070034from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070035from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070036
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080037class Sync(Command, MirrorSafeCommand):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070038 common = True
39 helpSummary = "Update working tree to the latest revision"
40 helpUsage = """
41%prog [<project>...]
42"""
43 helpDescription = """
44The '%prog' command synchronizes local project directories
45with the remote repositories specified in the manifest. If a local
46project does not yet exist, it will clone a new local directory from
47the remote repository and set up tracking branches as specified in
48the manifest. If the local project already exists, '%prog'
49will update the remote branches and rebase any new local changes
50on top of the new remote changes.
51
52'%prog' will synchronize all projects listed at the command
53line. Projects can be specified either by name, or by a relative
54or absolute path to the project's local directory. If no projects
55are specified, '%prog' will synchronize all projects listed in
56the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070057
58The -d/--detach option can be used to switch specified projects
59back to the manifest revision. This option is especially helpful
60if the project is currently on a topic branch, but the manifest
61revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070062
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070063The -s/--smart-sync option can be used to sync to a known good
64build as specified by the manifest-server element in the current
65manifest.
66
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070067SSH Connections
68---------------
69
70If at least one project remote URL uses an SSH connection (ssh://,
71git+ssh://, or user@host:path syntax) repo will automatically
72enable the SSH ControlMaster option when connecting to that host.
73This feature permits other projects in the same '%prog' session to
74reuse the same SSH tunnel, saving connection setup overheads.
75
76To disable this behavior on UNIX platforms, set the GIT_SSH
77environment variable to 'ssh'. For example:
78
79 export GIT_SSH=ssh
80 %prog
81
82Compatibility
83~~~~~~~~~~~~~
84
85This feature is automatically disabled on Windows, due to the lack
86of UNIX domain socket support.
87
88This feature is not compatible with url.insteadof rewrites in the
89user's ~/.gitconfig. '%prog' is currently not able to perform the
90rewrite early enough to establish the ControlMaster tunnel.
91
92If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
93later is required to fix a server side protocol bug.
94
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070095"""
96
97 def _Options(self, p):
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -070098 p.add_option('-l','--local-only',
99 dest='local_only', action='store_true',
100 help="only update working tree, don't fetch")
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700101 p.add_option('-n','--network-only',
102 dest='network_only', action='store_true',
103 help="fetch only, don't update working tree")
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700104 p.add_option('-d','--detach',
105 dest='detach_head', action='store_true',
106 help='detach projects back to manifest revision')
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700107 p.add_option('-s', '--smart-sync',
108 dest='smart_sync', action='store_true',
109 help='smart sync using manifest from a known good build')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700110
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700111 g = p.add_option_group('repo Version options')
112 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700113 dest='no_repo_verify', action='store_true',
114 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700115 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800116 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700117 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700118
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700119 def _Fetch(self, projects):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700120 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700121 pm = Progress('Fetching projects', len(projects))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700122 for project in projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700123 pm.update()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700124 if project.Sync_NetworkHalf():
125 fetched.add(project.gitdir)
126 else:
127 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
128 sys.exit(1)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700129 pm.end()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700130 return fetched
131
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700132 def UpdateProjectList(self):
133 new_project_paths = []
134 for project in self.manifest.projects.values():
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700135 if project.relpath:
136 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700137 file_name = 'project.list'
138 file_path = os.path.join(self.manifest.repodir, file_name)
139 old_project_paths = []
140
141 if os.path.exists(file_path):
142 fd = open(file_path, 'r')
143 try:
144 old_project_paths = fd.read().split('\n')
145 finally:
146 fd.close()
147 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700148 if not path:
149 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700150 if path not in new_project_paths:
Anthonyf3fdf822009-09-26 13:38:52 -0400151 """If the path has already been deleted, we don't need to do it
152 """
153 if os.path.exists(self.manifest.topdir + '/' + path):
154 project = Project(
155 manifest = self.manifest,
156 name = path,
157 remote = RemoteSpec('origin'),
158 gitdir = os.path.join(self.manifest.topdir,
159 path, '.git'),
160 worktree = os.path.join(self.manifest.topdir, path),
161 relpath = path,
162 revisionExpr = 'HEAD',
163 revisionId = None)
164
165 if project.IsDirty():
166 print >>sys.stderr, 'error: Cannot remove project "%s": \
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700167uncommitted changes are present' % project.relpath
Anthonyf3fdf822009-09-26 13:38:52 -0400168 print >>sys.stderr, ' commit changes, then run sync again'
169 return -1
170 else:
171 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
172 shutil.rmtree(project.worktree)
173 # Try deleting parent subdirs if they are empty
174 dir = os.path.dirname(project.worktree)
175 while dir != self.manifest.topdir:
176 try:
177 os.rmdir(dir)
178 except OSError:
179 break
180 dir = os.path.dirname(dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700181
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700182 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700183 fd = open(file_path, 'w')
184 try:
185 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700186 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700187 finally:
188 fd.close()
189 return 0
190
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700191 def Execute(self, opt, args):
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700192 if opt.network_only and opt.detach_head:
193 print >>sys.stderr, 'error: cannot combine -n and -d'
194 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700195 if opt.network_only and opt.local_only:
196 print >>sys.stderr, 'error: cannot combine -n and -l'
197 sys.exit(1)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700198
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700199 if opt.smart_sync:
200 if not self.manifest.manifest_server:
201 print >>sys.stderr, \
202 'error: cannot smart sync: no manifest server defined in manifest'
203 sys.exit(1)
204 try:
205 server = xmlrpclib.Server(self.manifest.manifest_server)
206 p = self.manifest.manifestProject
207 b = p.GetBranch(p.CurrentBranch)
208 branch = b.merge
Nico Sallembien5732e472010-04-26 11:17:29 -0700209 if branch.startswith(R_HEADS):
210 branch = branch[len(R_HEADS):]
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700211
212 env = dict(os.environ)
213 if (env.has_key('TARGET_PRODUCT') and
214 env.has_key('TARGET_BUILD_VARIANT')):
215 target = '%s-%s' % (env['TARGET_PRODUCT'],
216 env['TARGET_BUILD_VARIANT'])
217 [success, manifest_str] = server.GetApprovedManifest(branch, target)
218 else:
219 [success, manifest_str] = server.GetApprovedManifest(branch)
220
221 if success:
222 manifest_name = "smart_sync_override.xml"
223 manifest_path = os.path.join(self.manifest.manifestProject.worktree,
224 manifest_name)
225 try:
226 f = open(manifest_path, 'w')
227 try:
228 f.write(manifest_str)
229 self.manifest.Override(manifest_name)
230 finally:
231 f.close()
232 except IOError:
233 print >>sys.stderr, 'error: cannot write manifest to %s' % \
234 manifest_path
235 sys.exit(1)
236 else:
237 print >>sys.stderr, 'error: %s' % manifest_str
238 sys.exit(1)
239 except socket.error:
240 print >>sys.stderr, 'error: cannot connect to manifest server %s' % (
241 self.manifest.manifest_server)
242 sys.exit(1)
243
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700244 rp = self.manifest.repoProject
245 rp.PreSync()
246
247 mp = self.manifest.manifestProject
248 mp.PreSync()
249
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800250 if opt.repo_upgraded:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700251 _PostRepoUpgrade(self.manifest)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800252
Nico Sallembien9bb18162009-12-07 15:38:01 -0800253 if not opt.local_only:
254 mp.Sync_NetworkHalf()
255
256 if mp.HasChanges:
257 syncbuf = SyncBuffer(mp.config)
258 mp.Sync_LocalHalf(syncbuf)
259 if not syncbuf.Finish():
260 sys.exit(1)
261 self.manifest._Unload()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700262 all = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700263
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700264 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700265 to_fetch = []
266 now = time.time()
267 if (24 * 60 * 60) <= (now - rp.LastFetch):
268 to_fetch.append(rp)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700269 to_fetch.extend(all)
270
271 fetched = self._Fetch(to_fetch)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700272 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700273 if opt.network_only:
274 # bail out now; the rest touches the working tree
275 return
276
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700277 self.manifest._Unload()
278 all = self.GetProjects(args, missing_ok=True)
279 missing = []
280 for project in all:
281 if project.gitdir not in fetched:
282 missing.append(project)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700283 self._Fetch(missing)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700284
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700285 if self.manifest.IsMirror:
286 # bail out now, we have no working tree
287 return
288
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700289 if self.UpdateProjectList():
290 sys.exit(1)
291
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700292 syncbuf = SyncBuffer(mp.config,
293 detach_head = opt.detach_head)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700294 pm = Progress('Syncing work tree', len(all))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700295 for project in all:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700296 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800297 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700298 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700299 pm.end()
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700300 print >>sys.stderr
301 if not syncbuf.Finish():
302 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700303
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700304def _PostRepoUpgrade(manifest):
305 for project in manifest.projects.values():
306 if project.Exists:
307 project.PostRepoUpgrade()
308
309def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
310 if rp.HasChanges:
311 print >>sys.stderr, 'info: A new version of repo is available'
312 print >>sys.stderr, ''
313 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700314 syncbuf = SyncBuffer(rp.config)
315 rp.Sync_LocalHalf(syncbuf)
316 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700317 sys.exit(1)
318 print >>sys.stderr, 'info: Restarting repo with latest version'
319 raise RepoChangedException(['--repo-upgraded'])
320 else:
321 print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
322 else:
323 if verbose:
324 print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
325
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700326def _VerifyTag(project):
327 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
328 if not os.path.exists(gpg_dir):
329 print >>sys.stderr,\
330"""warning: GnuPG was not available during last "repo init"
331warning: Cannot automatically authenticate repo."""
332 return True
333
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700334 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700335 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700336 except GitError:
337 cur = None
338
339 if not cur \
340 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700341 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700342 if rev.startswith(R_HEADS):
343 rev = rev[len(R_HEADS):]
344
345 print >>sys.stderr
346 print >>sys.stderr,\
347 "warning: project '%s' branch '%s' is not signed" \
348 % (project.name, rev)
349 return False
350
351 env = dict(os.environ)
352 env['GIT_DIR'] = project.gitdir
353 env['GNUPGHOME'] = gpg_dir
354
355 cmd = [GIT, 'tag', '-v', cur]
356 proc = subprocess.Popen(cmd,
357 stdout = subprocess.PIPE,
358 stderr = subprocess.PIPE,
359 env = env)
360 out = proc.stdout.read()
361 proc.stdout.close()
362
363 err = proc.stderr.read()
364 proc.stderr.close()
365
366 if proc.wait() != 0:
367 print >>sys.stderr
368 print >>sys.stderr, out
369 print >>sys.stderr, err
370 print >>sys.stderr
371 return False
372 return True