blob: 88b23fbdde6fa757bb34c32d2ad2618b5f01f177 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Takeshi Kanemotoa7694982014-04-14 17:36:57 +090017import errno
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070018import fcntl
Takeshi Kanemotoa7694982014-04-14 17:36:57 +090019import multiprocessing
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import re
21import os
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070022import select
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023import sys
24import subprocess
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070025
26from color import Coloring
Shawn O. Pearce44469462009-03-03 17:51:01 -080027from command import Command, MirrorSafeCommand
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070028
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070029_CAN_COLOR = [
30 'branch',
31 'diff',
32 'grep',
33 'log',
34]
35
Takeshi Kanemotoa7694982014-04-14 17:36:57 +090036
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070037class ForallColoring(Coloring):
38 def __init__(self, config):
39 Coloring.__init__(self, config, 'forall')
40 self.project = self.printer('project', attr='bold')
41
42
Shawn O. Pearce44469462009-03-03 17:51:01 -080043class Forall(Command, MirrorSafeCommand):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070044 common = False
45 helpSummary = "Run a shell command in each project"
46 helpUsage = """
47%prog [<project>...] -c <command> [<arg>...]
Zhiguang Lia8864fb2013-03-15 10:32:10 +080048%prog -r str1 [str2] ... -c <command> [<arg>...]"
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070049"""
50 helpDescription = """
51Executes the same shell command in each project.
52
Zhiguang Lia8864fb2013-03-15 10:32:10 +080053The -r option allows running the command only on projects matching
54regex or wildcard expression.
55
Shawn O. Pearcedb45da12009-04-18 13:49:13 -070056Output Formatting
57-----------------
58
59The -p option causes '%prog' to bind pipes to the command's stdin,
60stdout and stderr streams, and pipe all output into a continuous
61stream that is displayed in a single pager session. Project headings
62are inserted before the output of each command is displayed. If the
63command produces no output in a project, no heading is displayed.
64
65The formatting convention used by -p is very suitable for some
66types of searching, e.g. `repo forall -p -c git log -SFoo` will
67print all commits that add or remove references to Foo.
68
69The -v option causes '%prog' to display stderr messages if a
70command produces output only on stderr. Normally the -p option
71causes command output to be suppressed until the command produces
72at least one byte of output on stdout.
73
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070074Environment
75-----------
Shawn O. Pearceff84fea2009-04-13 12:11:59 -070076
Shawn O. Pearce44469462009-03-03 17:51:01 -080077pwd is the project's working directory. If the current client is
78a mirror client, then pwd is the Git repository.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070079
80REPO_PROJECT is set to the unique name of the project.
81
Jeff Baileybe0e8ac2009-01-21 19:05:15 -050082REPO_PATH is the path relative the the root of the client.
83
84REPO_REMOTE is the name of the remote system from the manifest.
85
86REPO_LREV is the name of the revision from the manifest, translated
87to a local tracking branch. If you need to pass the manifest
88revision to a locally executed git command, use REPO_LREV.
89
90REPO_RREV is the name of the revision from the manifest, exactly
91as written in the manifest.
92
Mitchel Humpheryse81bc032014-03-31 11:36:56 -070093REPO_COUNT is the total number of projects being iterated.
94
95REPO_I is the current (1-based) iteration count. Can be used in
96conjunction with REPO_COUNT to add a simple progress indicator to your
97command.
98
James W. Mills24c13082012-04-12 15:04:13 -050099REPO__* are any extra environment variables, specified by the
100"annotation" element under any project element. This can be useful
101for differentiating trees based on user-specific criteria, or simply
102annotating tree details.
103
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700104shell positional arguments ($1, $2, .., $#) are set to any arguments
105following <command>.
106
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700107Unless -p is used, stdin, stdout, stderr are inherited from the
108terminal and are not redirected.
Victor Boivie88b86722011-09-07 09:43:28 +0200109
110If -e is used, when a command exits unsuccessfully, '%prog' will abort
111without iterating through the remaining projects.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700112"""
113
114 def _Options(self, p):
115 def cmd(option, opt_str, value, parser):
116 setattr(parser.values, option.dest, list(parser.rargs))
117 while parser.rargs:
118 del parser.rargs[0]
Zhiguang Lia8864fb2013-03-15 10:32:10 +0800119 p.add_option('-r', '--regex',
120 dest='regex', action='store_true',
121 help="Execute the command only on projects matching regex or wildcard expression")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700122 p.add_option('-c', '--command',
123 help='Command (and arguments) to execute',
124 dest='command',
125 action='callback',
126 callback=cmd)
Victor Boivie88b86722011-09-07 09:43:28 +0200127 p.add_option('-e', '--abort-on-errors',
128 dest='abort_on_errors', action='store_true',
129 help='Abort if a command exits unsuccessfully')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700130
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700131 g = p.add_option_group('Output')
132 g.add_option('-p',
133 dest='project_header', action='store_true',
134 help='Show project headers before output')
135 g.add_option('-v', '--verbose',
136 dest='verbose', action='store_true',
137 help='Show command error messages')
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900138 g.add_option('-j', '--jobs',
139 dest='jobs', action='store', type='int', default=1,
140 help='number of commands to execute simultaneously')
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700141
142 def WantPager(self, opt):
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900143 return opt.project_header and opt.jobs == 1
144
145 def _SerializeProject(self, project):
146 """ Serialize a project._GitGetByExec instance.
147
148 project._GitGetByExec is not pickle-able. Instead of trying to pass it
149 around between processes, make a dict ourselves containing only the
150 attributes that we need.
151
152 """
153 return {
154 'name': project.name,
155 'relpath': project.relpath,
156 'remote_name': project.remote.name,
157 'lrev': project.GetRevisionId(),
158 'rrev': project.revisionExpr,
159 'annotations': dict((a.name, a.value) for a in project.annotations),
160 'gitdir': project.gitdir,
161 'worktree': project.worktree,
162 }
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700163
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700164 def Execute(self, opt, args):
165 if not opt.command:
166 self.Usage()
167
168 cmd = [opt.command[0]]
169
170 shell = True
171 if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]):
172 shell = False
173
174 if shell:
175 cmd.append(cmd[0])
176 cmd.extend(opt.command[1:])
177
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700178 if opt.project_header \
179 and not shell \
180 and cmd[0] == 'git':
181 # If this is a direct git command that can enable colorized
182 # output and the user prefers coloring, add --color into the
183 # command line because we are going to wrap the command into
184 # a pipe and git won't know coloring should activate.
185 #
186 for cn in cmd[1:]:
187 if not cn.startswith('-'):
188 break
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900189 else:
190 cn = None
David Pursehouse4f7bdea2012-10-22 12:50:15 +0900191 # pylint: disable=W0631
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900192 if cn and cn in _CAN_COLOR:
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700193 class ColorCmd(Coloring):
194 def __init__(self, config, cmd):
195 Coloring.__init__(self, config, cmd)
196 if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
197 cmd.insert(cmd.index(cn) + 1, '--color')
David Pursehouse4f7bdea2012-10-22 12:50:15 +0900198 # pylint: enable=W0631
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700199
Shawn O. Pearce44469462009-03-03 17:51:01 -0800200 mirror = self.manifest.IsMirror
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700201 rc = 0
Shawn O. Pearcedb45da12009-04-18 13:49:13 -0700202
Zhiguang Lia8864fb2013-03-15 10:32:10 +0800203 if not opt.regex:
204 projects = self.GetProjects(args)
205 else:
206 projects = self.FindProjects(args)
207
Mitchel Humpheryse81bc032014-03-31 11:36:56 -0700208 os.environ['REPO_COUNT'] = str(len(projects))
209
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900210 pool = multiprocessing.Pool(opt.jobs)
211 try:
212 config = self.manifest.manifestProject.config
213 results_it = pool.imap(
214 DoWorkWrapper,
Anthony King6ad6dbe2015-04-08 13:22:30 +0100215 ([mirror, opt, cmd, shell, cnt, config, self._SerializeProject(p)]
216 for cnt, p in enumerate(projects))
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900217 )
218 pool.close()
219 for r in results_it:
220 rc = rc or r
221 if r != 0 and opt.abort_on_errors:
222 raise Exception('Aborting due to previous error')
223 except (KeyboardInterrupt, WorkerKeyboardInterrupt):
224 # Catch KeyboardInterrupt raised inside and outside of workers
225 print('Interrupted - terminating the pool')
226 pool.terminate()
227 rc = rc or errno.EINTR
228 except Exception as e:
229 # Catch any other exceptions raised
230 print('Got an error, terminating the pool: %r' % e,
231 file=sys.stderr)
232 pool.terminate()
233 rc = rc or getattr(e, 'errno', 1)
234 finally:
235 pool.join()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700236 if rc != 0:
237 sys.exit(rc)
Takeshi Kanemotoa7694982014-04-14 17:36:57 +0900238
239
240class WorkerKeyboardInterrupt(Exception):
241 """ Keyboard interrupt exception for worker processes. """
242 pass
243
244
245def DoWorkWrapper(args):
246 """ A wrapper around the DoWork() method.
247
248 Catch the KeyboardInterrupt exceptions here and re-raise them as a different,
249 ``Exception``-based exception to stop it flooding the console with stacktraces
250 and making the parent hang indefinitely.
251
252 """
253 project = args.pop()
254 try:
255 return DoWork(project, *args)
256 except KeyboardInterrupt:
257 print('%s: Worker interrupted' % project['name'])
258 raise WorkerKeyboardInterrupt()
259
260
261def DoWork(project, mirror, opt, cmd, shell, cnt, config):
262 env = os.environ.copy()
263 def setenv(name, val):
264 if val is None:
265 val = ''
266 env[name] = val.encode()
267
268 setenv('REPO_PROJECT', project['name'])
269 setenv('REPO_PATH', project['relpath'])
270 setenv('REPO_REMOTE', project['remote_name'])
271 setenv('REPO_LREV', project['lrev'])
272 setenv('REPO_RREV', project['rrev'])
273 setenv('REPO_I', str(cnt + 1))
274 for name in project['annotations']:
275 setenv("REPO__%s" % (name), project['annotations'][name])
276
277 if mirror:
278 setenv('GIT_DIR', project['gitdir'])
279 cwd = project['gitdir']
280 else:
281 cwd = project['worktree']
282
283 if not os.path.exists(cwd):
284 if (opt.project_header and opt.verbose) \
285 or not opt.project_header:
286 print('skipping %s/' % project['relpath'], file=sys.stderr)
287 return
288
289 if opt.project_header:
290 stdin = subprocess.PIPE
291 stdout = subprocess.PIPE
292 stderr = subprocess.PIPE
293 else:
294 stdin = None
295 stdout = None
296 stderr = None
297
298 p = subprocess.Popen(cmd,
299 cwd=cwd,
300 shell=shell,
301 env=env,
302 stdin=stdin,
303 stdout=stdout,
304 stderr=stderr)
305
306 if opt.project_header:
307 out = ForallColoring(config)
308 out.redirect(sys.stdout)
309 class sfd(object):
310 def __init__(self, fd, dest):
311 self.fd = fd
312 self.dest = dest
313 def fileno(self):
314 return self.fd.fileno()
315
316 empty = True
317 errbuf = ''
318
319 p.stdin.close()
320 s_in = [sfd(p.stdout, sys.stdout),
321 sfd(p.stderr, sys.stderr)]
322
323 for s in s_in:
324 flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
325 fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
326
327 while s_in:
328 in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
329 for s in in_ready:
330 buf = s.fd.read(4096)
331 if not buf:
332 s.fd.close()
333 s_in.remove(s)
334 continue
335
336 if not opt.verbose:
337 if s.fd != p.stdout:
338 errbuf += buf
339 continue
340
341 if empty and out:
342 if not cnt == 0:
343 out.nl()
344
345 if mirror:
346 project_header_path = project['name']
347 else:
348 project_header_path = project['relpath']
349 out.project('project %s/', project_header_path)
350 out.nl()
351 out.flush()
352 if errbuf:
353 sys.stderr.write(errbuf)
354 sys.stderr.flush()
355 errbuf = ''
356 empty = False
357
358 s.dest.write(buf)
359 s.dest.flush()
360
361 r = p.wait()
362 return r