blob: 6a6d30c9a84baadeb6f5f53b7f6de32c38bc145c [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Takeshi Kanemotoa7694982014-04-14 17:36:57 +090017import errno
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070018import fcntl
Takeshi Kanemotoa7694982014-04-14 17:36:57 +090019import multiprocessing
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import re
21import os
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070022import select
Colin Cross31a7be52015-05-13 00:04:36 -070023import signal
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070024import sys
25import subprocess
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070026
27from color import Coloring
Shawn O. Pearce44469462009-03-03 17:51:01 -080028from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070029
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070030_CAN_COLOR = [
31 'branch',
32 'diff',
33 'grep',
34 'log',
35]
36
Takeshi Kanemotoa7694982014-04-14 17:36:57 +090037
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070038class ForallColoring(Coloring):
39 def __init__(self, config):
40 Coloring.__init__(self, config, 'forall')
41 self.project = self.printer('project', attr='bold')
42
43
Shawn O. Pearce44469462009-03-03 17:51:01 -080044class Forall(Command, MirrorSafeCommand):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070045 common = False
46 helpSummary = "Run a shell command in each project"
47 helpUsage = """
48%prog [<project>...] -c <command> [<arg>...]
Zhiguang Lia8864fb2013-03-15 10:32:10 +080049%prog -r str1 [str2] ... -c <command> [<arg>...]"
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070050"""
51 helpDescription = """
52Executes the same shell command in each project.
53
Zhiguang Lia8864fb2013-03-15 10:32:10 +080054The -r option allows running the command only on projects matching
55regex or wildcard expression.
56
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070057Output Formatting
58-----------------
59
60The -p option causes '%prog' to bind pipes to the command's stdin,
61stdout and stderr streams, and pipe all output into a continuous
62stream that is displayed in a single pager session. Project headings
63are inserted before the output of each command is displayed. If the
64command produces no output in a project, no heading is displayed.
65
66The formatting convention used by -p is very suitable for some
67types of searching, e.g. `repo forall -p -c git log -SFoo` will
68print all commits that add or remove references to Foo.
69
70The -v option causes '%prog' to display stderr messages if a
71command produces output only on stderr. Normally the -p option
72causes command output to be suppressed until the command produces
73at least one byte of output on stdout.
74
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070075Environment
76-----------
Shawn O. Pearceff84fea2009-04-13 12:11:59 -070077
Shawn O. Pearce44469462009-03-03 17:51:01 -080078pwd is the project's working directory. If the current client is
79a mirror client, then pwd is the Git repository.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070080
81REPO_PROJECT is set to the unique name of the project.
82
Jeff Baileybe0e8ac2009-01-21 19:05:15 -050083REPO_PATH is the path relative the the root of the client.
84
85REPO_REMOTE is the name of the remote system from the manifest.
86
87REPO_LREV is the name of the revision from the manifest, translated
88to a local tracking branch. If you need to pass the manifest
89revision to a locally executed git command, use REPO_LREV.
90
91REPO_RREV is the name of the revision from the manifest, exactly
92as written in the manifest.
93
Mitchel Humpheryse81bc032014-03-31 11:36:56 -070094REPO_COUNT is the total number of projects being iterated.
95
96REPO_I is the current (1-based) iteration count. Can be used in
97conjunction with REPO_COUNT to add a simple progress indicator to your
98command.
99
James W. Mills24c13082012-04-12 15:04:13 -0500100REPO__* are any extra environment variables, specified by the
101"annotation" element under any project element. This can be useful
102for differentiating trees based on user-specific criteria, or simply
103annotating tree details.
104
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700105shell positional arguments ($1, $2, .., $#) are set to any arguments
106following <command>.
107
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700108Unless -p is used, stdin, stdout, stderr are inherited from the
109terminal and are not redirected.
Victor Boivie88b86722011-09-07 09:43:28 +0200110
111If -e is used, when a command exits unsuccessfully, '%prog' will abort
112without iterating through the remaining projects.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700113"""
114
115 def _Options(self, p):
116 def cmd(option, opt_str, value, parser):
117 setattr(parser.values, option.dest, list(parser.rargs))
118 while parser.rargs:
119 del parser.rargs[0]
Zhiguang Lia8864fb2013-03-15 10:32:10 +0800120 p.add_option('-r', '--regex',
121 dest='regex', action='store_true',
122 help="Execute the command only on projects matching regex or wildcard expression")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700123 p.add_option('-c', '--command',
124 help='Command (and arguments) to execute',
125 dest='command',
126 action='callback',
127 callback=cmd)
Victor Boivie88b86722011-09-07 09:43:28 +0200128 p.add_option('-e', '--abort-on-errors',
129 dest='abort_on_errors', action='store_true',
130 help='Abort if a command exits unsuccessfully')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700131
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700132 g = p.add_option_group('Output')
133 g.add_option('-p',
134 dest='project_header', action='store_true',
135 help='Show project headers before output')
136 g.add_option('-v', '--verbose',
137 dest='verbose', action='store_true',
138 help='Show command error messages')
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900139 g.add_option('-j', '--jobs',
140 dest='jobs', action='store', type='int', default=1,
141 help='number of commands to execute simultaneously')
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700142
143 def WantPager(self, opt):
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900144 return opt.project_header and opt.jobs == 1
145
146 def _SerializeProject(self, project):
147 """ Serialize a project._GitGetByExec instance.
148
149 project._GitGetByExec is not pickle-able. Instead of trying to pass it
150 around between processes, make a dict ourselves containing only the
151 attributes that we need.
152
153 """
154 return {
155 'name': project.name,
156 'relpath': project.relpath,
157 'remote_name': project.remote.name,
158 'lrev': project.GetRevisionId(),
159 'rrev': project.revisionExpr,
160 'annotations': dict((a.name, a.value) for a in project.annotations),
161 'gitdir': project.gitdir,
162 'worktree': project.worktree,
163 }
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700164
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700165 def Execute(self, opt, args):
166 if not opt.command:
167 self.Usage()
168
169 cmd = [opt.command[0]]
170
171 shell = True
172 if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]):
173 shell = False
174
175 if shell:
176 cmd.append(cmd[0])
177 cmd.extend(opt.command[1:])
178
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700179 if opt.project_header \
180 and not shell \
181 and cmd[0] == 'git':
182 # If this is a direct git command that can enable colorized
183 # output and the user prefers coloring, add --color into the
184 # command line because we are going to wrap the command into
185 # a pipe and git won't know coloring should activate.
186 #
187 for cn in cmd[1:]:
188 if not cn.startswith('-'):
189 break
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900190 else:
191 cn = None
David Pursehouse4f7bdea2012-10-22 12:50:15 +0900192 # pylint: disable=W0631
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900193 if cn and cn in _CAN_COLOR:
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700194 class ColorCmd(Coloring):
195 def __init__(self, config, cmd):
196 Coloring.__init__(self, config, cmd)
197 if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
198 cmd.insert(cmd.index(cn) + 1, '--color')
David Pursehouse4f7bdea2012-10-22 12:50:15 +0900199 # pylint: enable=W0631
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700200
Shawn O. Pearce44469462009-03-03 17:51:01 -0800201 mirror = self.manifest.IsMirror
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700202 rc = 0
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700203
Zhiguang Lia8864fb2013-03-15 10:32:10 +0800204 if not opt.regex:
205 projects = self.GetProjects(args)
206 else:
207 projects = self.FindProjects(args)
208
Mitchel Humpheryse81bc032014-03-31 11:36:56 -0700209 os.environ['REPO_COUNT'] = str(len(projects))
210
Colin Cross31a7be52015-05-13 00:04:36 -0700211 pool = multiprocessing.Pool(opt.jobs, InitWorker)
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900212 try:
213 config = self.manifest.manifestProject.config
214 results_it = pool.imap(
215 DoWorkWrapper,
Colin Cross31a7be52015-05-13 00:04:36 -0700216 self.ProjectArgs(projects, mirror, opt, cmd, shell, config))
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900217 pool.close()
218 for r in results_it:
219 rc = rc or r
220 if r != 0 and opt.abort_on_errors:
221 raise Exception('Aborting due to previous error')
222 except (KeyboardInterrupt, WorkerKeyboardInterrupt):
223 # Catch KeyboardInterrupt raised inside and outside of workers
224 print('Interrupted - terminating the pool')
225 pool.terminate()
226 rc = rc or errno.EINTR
227 except Exception as e:
228 # Catch any other exceptions raised
229 print('Got an error, terminating the pool: %r' % e,
230 file=sys.stderr)
231 pool.terminate()
232 rc = rc or getattr(e, 'errno', 1)
233 finally:
234 pool.join()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700235 if rc != 0:
236 sys.exit(rc)
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900237
Colin Cross31a7be52015-05-13 00:04:36 -0700238 def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
239 for cnt, p in enumerate(projects):
240 try:
241 project = self._SerializeProject(p)
242 except Exception as e:
243 print('Project list error: %r' % e,
244 file=sys.stderr)
245 return
246 except KeyboardInterrupt:
247 print('Project list interrupted',
248 file=sys.stderr)
249 return
250 yield [mirror, opt, cmd, shell, cnt, config, project]
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900251
252class WorkerKeyboardInterrupt(Exception):
253 """ Keyboard interrupt exception for worker processes. """
254 pass
255
256
Colin Cross31a7be52015-05-13 00:04:36 -0700257def InitWorker():
258 signal.signal(signal.SIGINT, signal.SIG_IGN)
259
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900260def DoWorkWrapper(args):
261 """ A wrapper around the DoWork() method.
262
263 Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
264 ``Exception``-based exception to stop it flooding the console with stacktraces
265 and making the parent hang indefinitely.
266
267 """
268 project = args.pop()
269 try:
270 return DoWork(project, *args)
271 except KeyboardInterrupt:
272 print('%s: Worker interrupted' % project['name'])
273 raise WorkerKeyboardInterrupt()
274
275
276def DoWork(project, mirror, opt, cmd, shell, cnt, config):
277 env = os.environ.copy()
278 def setenv(name, val):
279 if val is None:
280 val = ''
281 env[name] = val.encode()
282
283 setenv('REPO_PROJECT', project['name'])
284 setenv('REPO_PATH', project['relpath'])
285 setenv('REPO_REMOTE', project['remote_name'])
286 setenv('REPO_LREV', project['lrev'])
287 setenv('REPO_RREV', project['rrev'])
288 setenv('REPO_I', str(cnt + 1))
289 for name in project['annotations']:
290 setenv("REPO__%s" % (name), project['annotations'][name])
291
292 if mirror:
293 setenv('GIT_DIR', project['gitdir'])
294 cwd = project['gitdir']
295 else:
296 cwd = project['worktree']
297
298 if not os.path.exists(cwd):
299 if (opt.project_header and opt.verbose) \
300 or not opt.project_header:
301 print('skipping %s/' % project['relpath'], file=sys.stderr)
302 return
303
304 if opt.project_header:
305 stdin = subprocess.PIPE
306 stdout = subprocess.PIPE
307 stderr = subprocess.PIPE
308 else:
309 stdin = None
310 stdout = None
311 stderr = None
312
313 p = subprocess.Popen(cmd,
314 cwd=cwd,
315 shell=shell,
316 env=env,
317 stdin=stdin,
318 stdout=stdout,
319 stderr=stderr)
320
321 if opt.project_header:
322 out = ForallColoring(config)
323 out.redirect(sys.stdout)
324 class sfd(object):
325 def __init__(self, fd, dest):
326 self.fd = fd
327 self.dest = dest
328 def fileno(self):
329 return self.fd.fileno()
330
331 empty = True
332 errbuf = ''
333
334 p.stdin.close()
335 s_in = [sfd(p.stdout, sys.stdout),
336 sfd(p.stderr, sys.stderr)]
337
338 for s in s_in:
339 flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
340 fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
341
342 while s_in:
343 in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
344 for s in in_ready:
345 buf = s.fd.read(4096)
346 if not buf:
347 s.fd.close()
348 s_in.remove(s)
349 continue
350
351 if not opt.verbose:
352 if s.fd != p.stdout:
353 errbuf += buf
354 continue
355
356 if empty and out:
357 if not cnt == 0:
358 out.nl()
359
360 if mirror:
361 project_header_path = project['name']
362 else:
363 project_header_path = project['relpath']
364 out.project('project %s/', project_header_path)
365 out.nl()
366 out.flush()
367 if errbuf:
368 sys.stderr.write(errbuf)
369 sys.stderr.flush()
370 errbuf = ''
371 empty = False
372
373 s.dest.write(buf)
374 s.dest.flush()
375
376 r = p.wait()
377 return r