blob: afd44dab3791925579d729ffc74bc9342cc2a791 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070016from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
18import re
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070019import shutil
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import subprocess
21import sys
Shawn O. Pearcef6906872009-04-18 10:49:00 -070022import time
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023
24from git_command import GIT
Shawn O. Pearcee756c412009-04-13 11:51:15 -070025from project import HEAD
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070026from project import Project
27from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080028from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070029from error import RepoChangedException, GitError
30from project import R_HEADS
Shawn O. Pearce350cde42009-04-16 11:21:18 -070031from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070032from progress import Progress
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070033
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080034class Sync(Command, MirrorSafeCommand):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070035 common = True
36 helpSummary = "Update working tree to the latest revision"
37 helpUsage = """
38%prog [<project>...]
39"""
40 helpDescription = """
41The '%prog' command synchronizes local project directories
42with the remote repositories specified in the manifest. If a local
43project does not yet exist, it will clone a new local directory from
44the remote repository and set up tracking branches as specified in
45the manifest. If the local project already exists, '%prog'
46will update the remote branches and rebase any new local changes
47on top of the new remote changes.
48
49'%prog' will synchronize all projects listed at the command
50line. Projects can be specified either by name, or by a relative
51or absolute path to the project's local directory. If no projects
52are specified, '%prog' will synchronize all projects listed in
53the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070054
55The -d/--detach option can be used to switch specified projects
56back to the manifest revision. This option is especially helpful
57if the project is currently on a topic branch, but the manifest
58revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070059
60SSH Connections
61---------------
62
63If at least one project remote URL uses an SSH connection (ssh://,
64git+ssh://, or user@host:path syntax) repo will automatically
65enable the SSH ControlMaster option when connecting to that host.
66This feature permits other projects in the same '%prog' session to
67reuse the same SSH tunnel, saving connection setup overheads.
68
69To disable this behavior on UNIX platforms, set the GIT_SSH
70environment variable to 'ssh'. For example:
71
72 export GIT_SSH=ssh
73 %prog
74
75Compatibility
76~~~~~~~~~~~~~
77
78This feature is automatically disabled on Windows, due to the lack
79of UNIX domain socket support.
80
81This feature is not compatible with url.insteadof rewrites in the
82user's ~/.gitconfig. '%prog' is currently not able to perform the
83rewrite early enough to establish the ControlMaster tunnel.
84
85If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
86later is required to fix a server side protocol bug.
87
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070088"""
89
90 def _Options(self, p):
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -070091 p.add_option('-l','--local-only',
92 dest='local_only', action='store_true',
93 help="only update working tree, don't fetch")
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -070094 p.add_option('-n','--network-only',
95 dest='network_only', action='store_true',
96 help="fetch only, don't update working tree")
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070097 p.add_option('-d','--detach',
98 dest='detach_head', action='store_true',
99 help='detach projects back to manifest revision')
100
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700101 g = p.add_option_group('repo Version options')
102 g.add_option('--no-repo-verify',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700103 dest='no_repo_verify', action='store_true',
104 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700105 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800106 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700107 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700108
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700109 def _Fetch(self, projects):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700110 fetched = set()
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700111 pm = Progress('Fetching projects', len(projects))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700112 for project in projects:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700113 pm.update()
114
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700115 if project.Sync_NetworkHalf():
116 fetched.add(project.gitdir)
117 else:
118 print >>sys.stderr, 'error: Cannot fetch %s' % project.name
119 sys.exit(1)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700120 pm.end()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700121 return fetched
122
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700123 def UpdateProjectList(self):
124 new_project_paths = []
125 for project in self.manifest.projects.values():
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700126 if project.relpath:
127 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700128 file_name = 'project.list'
129 file_path = os.path.join(self.manifest.repodir, file_name)
130 old_project_paths = []
131
132 if os.path.exists(file_path):
133 fd = open(file_path, 'r')
134 try:
135 old_project_paths = fd.read().split('\n')
136 finally:
137 fd.close()
138 for path in old_project_paths:
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700139 if not path:
140 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700141 if path not in new_project_paths:
142 project = Project(
143 manifest = self.manifest,
144 name = path,
145 remote = RemoteSpec('origin'),
146 gitdir = os.path.join(self.manifest.topdir,
147 path, '.git'),
148 worktree = os.path.join(self.manifest.topdir, path),
149 relpath = path,
150 revisionExpr = 'HEAD',
151 revisionId = None)
152 if project.IsDirty():
153 print >>sys.stderr, 'error: Cannot remove project "%s": \
154uncommitted changes are present' % project.relpath
155 print >>sys.stderr, ' commit changes, then run sync again'
156 return -1
157 else:
158 print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
159 shutil.rmtree(project.worktree)
Jaikumar Ganesh8135cdc2009-06-02 15:07:44 -0700160 # Try deleting parent subdirs if they are empty
161 dir = os.path.dirname(project.worktree)
162 while dir != self.manifest.topdir:
163 try:
164 os.rmdir(dir)
165 except OSError:
166 break
167 dir = os.path.dirname(dir)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700168
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700169 new_project_paths.sort()
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700170 fd = open(file_path, 'w')
171 try:
172 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700173 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700174 finally:
175 fd.close()
176 return 0
177
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700178 def Execute(self, opt, args):
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700179 if opt.network_only and opt.detach_head:
180 print >>sys.stderr, 'error: cannot combine -n and -d'
181 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700182 if opt.network_only and opt.local_only:
183 print >>sys.stderr, 'error: cannot combine -n and -l'
184 sys.exit(1)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700185
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700186 rp = self.manifest.repoProject
187 rp.PreSync()
188
189 mp = self.manifest.manifestProject
190 mp.PreSync()
191
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800192 if opt.repo_upgraded:
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700193 _PostRepoUpgrade(self.manifest)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800194
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700195 all = self.GetProjects(args, missing_ok=True)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700196
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700197 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700198 to_fetch = []
199 now = time.time()
200 if (24 * 60 * 60) <= (now - rp.LastFetch):
201 to_fetch.append(rp)
202 to_fetch.append(mp)
203 to_fetch.extend(all)
204
205 fetched = self._Fetch(to_fetch)
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700206 _PostRepoFetch(rp, opt.no_repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700207 if opt.network_only:
208 # bail out now; the rest touches the working tree
209 return
210
211 if mp.HasChanges:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700212 syncbuf = SyncBuffer(mp.config)
213 mp.Sync_LocalHalf(syncbuf)
214 if not syncbuf.Finish():
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700215 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700216
Shawn O. Pearcef1a6b142009-06-03 16:01:11 -0700217 self.GetManifest(reparse=True)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700218 all = self.GetProjects(args, missing_ok=True)
219 missing = []
220 for project in all:
221 if project.gitdir not in fetched:
222 missing.append(project)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700223 self._Fetch(missing)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700224
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700225 if self.manifest.IsMirror:
226 # bail out now, we have no working tree
227 return
228
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700229 if self.UpdateProjectList():
230 sys.exit(1)
231
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700232 syncbuf = SyncBuffer(mp.config,
233 detach_head = opt.detach_head)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700234 pm = Progress('Syncing work tree', len(all))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700235 for project in all:
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700236 pm.update()
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800237 if project.worktree:
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700238 project.Sync_LocalHalf(syncbuf)
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700239 pm.end()
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700240 print >>sys.stderr
241 if not syncbuf.Finish():
242 sys.exit(1)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700243
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700244
245def _PostRepoUpgrade(manifest):
246 for project in manifest.projects.values():
247 if project.Exists:
248 project.PostRepoUpgrade()
249
250def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
251 if rp.HasChanges:
252 print >>sys.stderr, 'info: A new version of repo is available'
253 print >>sys.stderr, ''
254 if no_repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -0700255 syncbuf = SyncBuffer(rp.config)
256 rp.Sync_LocalHalf(syncbuf)
257 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -0700258 sys.exit(1)
259 print >>sys.stderr, 'info: Restarting repo with latest version'
260 raise RepoChangedException(['--repo-upgraded'])
261 else:
262 print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
263 else:
264 if verbose:
265 print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
266
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700267def _VerifyTag(project):
268 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
269 if not os.path.exists(gpg_dir):
270 print >>sys.stderr,\
271"""warning: GnuPG was not available during last "repo init"
272warning: Cannot automatically authenticate repo."""
273 return True
274
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700275 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700276 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700277 except GitError:
278 cur = None
279
280 if not cur \
281 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700282 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700283 if rev.startswith(R_HEADS):
284 rev = rev[len(R_HEADS):]
285
286 print >>sys.stderr
287 print >>sys.stderr,\
288 "warning: project '%s' branch '%s' is not signed" \
289 % (project.name, rev)
290 return False
291
292 env = dict(os.environ)
293 env['GIT_DIR'] = project.gitdir
294 env['GNUPGHOME'] = gpg_dir
295
296 cmd = [GIT, 'tag', '-v', cur]
297 proc = subprocess.Popen(cmd,
298 stdout = subprocess.PIPE,
299 stderr = subprocess.PIPE,
300 env = env)
301 out = proc.stdout.read()
302 proc.stdout.close()
303
304 err = proc.stderr.read()
305 proc.stderr.close()
306
307 if proc.wait() != 0:
308 print >>sys.stderr
309 print >>sys.stderr, out
310 print >>sys.stderr, err
311 print >>sys.stderr
312 return False
313 return True