blob: a2a739a49fac4badfbec186a8cc38d6d07e39e99 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Shawn O. Pearcec12c3602009-04-17 21:03:32 -070016import cPickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
18import re
Shawn O. Pearcefb231612009-04-10 18:53:46 -070019import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import sys
Shawn O. Pearcefb231612009-04-10 18:53:46 -070021import time
22from signal import SIGTERM
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080023from urllib2 import urlopen, HTTPError
24from error import GitError, UploadError
Shawn O. Pearcead3193a2009-04-18 09:54:51 -070025from trace import Trace
Shawn O. Pearcefb231612009-04-10 18:53:46 -070026from git_command import GitCommand, _ssh_sock
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070027
28R_HEADS = 'refs/heads/'
29R_TAGS = 'refs/tags/'
30ID_RE = re.compile('^[0-9a-f]{40}$')
31
Shawn O. Pearce146fe902009-03-25 14:06:43 -070032REVIEW_CACHE = dict()
33
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070034def IsId(rev):
35 return ID_RE.match(rev)
36
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070037def _key(name):
38 parts = name.split('.')
39 if len(parts) < 2:
40 return name.lower()
41 parts[ 0] = parts[ 0].lower()
42 parts[-1] = parts[-1].lower()
43 return '.'.join(parts)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070044
45class GitConfig(object):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070046 _ForUser = None
47
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070048 @classmethod
49 def ForUser(cls):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070050 if cls._ForUser is None:
51 cls._ForUser = cls(file = os.path.expanduser('~/.gitconfig'))
52 return cls._ForUser
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070053
54 @classmethod
55 def ForRepository(cls, gitdir, defaults=None):
56 return cls(file = os.path.join(gitdir, 'config'),
57 defaults = defaults)
58
Shawn O. Pearce1b34c912009-05-21 18:52:49 -070059 def __init__(self, file, defaults=None, pickleFile=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070060 self.file = file
61 self.defaults = defaults
62 self._cache_dict = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070063 self._section_dict = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070064 self._remotes = {}
65 self._branches = {}
Shawn O. Pearce1b34c912009-05-21 18:52:49 -070066
67 if pickleFile is None:
68 self._pickle = os.path.join(
69 os.path.dirname(self.file),
70 '.repopickle_' + os.path.basename(self.file))
71 else:
72 self._pickle = pickleFile
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070073
74 def Has(self, name, include_defaults = True):
75 """Return true if this configuration file has the key.
76 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070077 if _key(name) in self._cache:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070078 return True
79 if include_defaults and self.defaults:
80 return self.defaults.Has(name, include_defaults = True)
81 return False
82
83 def GetBoolean(self, name):
84 """Returns a boolean from the configuration file.
85 None : The value was not defined, or is not a boolean.
86 True : The value was set to true or yes.
87 False: The value was set to false or no.
88 """
89 v = self.GetString(name)
90 if v is None:
91 return None
92 v = v.lower()
93 if v in ('true', 'yes'):
94 return True
95 if v in ('false', 'no'):
96 return False
97 return None
98
99 def GetString(self, name, all=False):
100 """Get the first value for a key, or None if it is not defined.
101
102 This configuration file is used first, if the key is not
103 defined or all = True then the defaults are also searched.
104 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700105 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700106 v = self._cache[_key(name)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700107 except KeyError:
108 if self.defaults:
109 return self.defaults.GetString(name, all = all)
110 v = []
111
112 if not all:
113 if v:
114 return v[0]
115 return None
116
117 r = []
118 r.extend(v)
119 if self.defaults:
120 r.extend(self.defaults.GetString(name, all = True))
121 return r
122
123 def SetString(self, name, value):
124 """Set the value(s) for a key.
125 Only this configuration file is modified.
126
127 The supplied value should be either a string,
128 or a list of strings (to store multiple values).
129 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700130 key = _key(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700131
132 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700133 old = self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700134 except KeyError:
135 old = []
136
137 if value is None:
138 if old:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700139 del self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700140 self._do('--unset-all', name)
141
142 elif isinstance(value, list):
143 if len(value) == 0:
144 self.SetString(name, None)
145
146 elif len(value) == 1:
147 self.SetString(name, value[0])
148
149 elif old != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700150 self._cache[key] = list(value)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700151 self._do('--replace-all', name, value[0])
152 for i in xrange(1, len(value)):
153 self._do('--add', name, value[i])
154
155 elif len(old) != 1 or old[0] != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700156 self._cache[key] = [value]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700157 self._do('--replace-all', name, value)
158
159 def GetRemote(self, name):
160 """Get the remote.$name.* configuration values as an object.
161 """
162 try:
163 r = self._remotes[name]
164 except KeyError:
165 r = Remote(self, name)
166 self._remotes[r.name] = r
167 return r
168
169 def GetBranch(self, name):
170 """Get the branch.$name.* configuration values as an object.
171 """
172 try:
173 b = self._branches[name]
174 except KeyError:
175 b = Branch(self, name)
176 self._branches[b.name] = b
177 return b
178
Shawn O. Pearce366ad212009-05-19 12:47:37 -0700179 def GetSubSections(self, section):
180 """List all subsection names matching $section.*.*
181 """
182 return self._sections.get(section, set())
183
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700184 def HasSection(self, section, subsection = ''):
185 """Does at least one key in section.subsection exist?
186 """
187 try:
188 return subsection in self._sections[section]
189 except KeyError:
190 return False
191
192 @property
193 def _sections(self):
194 d = self._section_dict
195 if d is None:
196 d = {}
197 for name in self._cache.keys():
198 p = name.split('.')
199 if 2 == len(p):
200 section = p[0]
201 subsect = ''
202 else:
203 section = p[0]
204 subsect = '.'.join(p[1:-1])
205 if section not in d:
206 d[section] = set()
207 d[section].add(subsect)
208 self._section_dict = d
209 return d
210
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700211 @property
212 def _cache(self):
213 if self._cache_dict is None:
214 self._cache_dict = self._Read()
215 return self._cache_dict
216
217 def _Read(self):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700218 d = self._ReadPickle()
219 if d is None:
220 d = self._ReadGit()
221 self._SavePickle(d)
222 return d
223
224 def _ReadPickle(self):
225 try:
226 if os.path.getmtime(self._pickle) \
227 <= os.path.getmtime(self.file):
228 os.remove(self._pickle)
229 return None
230 except OSError:
231 return None
232 try:
Shawn O. Pearcead3193a2009-04-18 09:54:51 -0700233 Trace(': unpickle %s', self.file)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700234 fd = open(self._pickle, 'rb')
235 try:
236 return cPickle.load(fd)
237 finally:
238 fd.close()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700239 except IOError:
240 os.remove(self._pickle)
241 return None
242 except cPickle.PickleError:
243 os.remove(self._pickle)
244 return None
245
246 def _SavePickle(self, cache):
247 try:
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700248 fd = open(self._pickle, 'wb')
249 try:
250 cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
251 finally:
252 fd.close()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700253 except IOError:
254 os.remove(self._pickle)
255 except cPickle.PickleError:
256 os.remove(self._pickle)
257
258 def _ReadGit(self):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700259 d = self._do('--null', '--list')
260 c = {}
261 while d:
262 lf = d.index('\n')
263 nul = d.index('\0', lf + 1)
264
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700265 key = _key(d[0:lf])
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700266 val = d[lf + 1:nul]
267
268 if key in c:
269 c[key].append(val)
270 else:
271 c[key] = [val]
272
273 d = d[nul + 1:]
274 return c
275
276 def _do(self, *args):
277 command = ['config', '--file', self.file]
278 command.extend(args)
279
280 p = GitCommand(None,
281 command,
282 capture_stdout = True,
283 capture_stderr = True)
284 if p.Wait() == 0:
285 return p.stdout
286 else:
287 GitError('git config %s: %s' % (str(args), p.stderr))
288
289
290class RefSpec(object):
291 """A Git refspec line, split into its components:
292
293 forced: True if the line starts with '+'
294 src: Left side of the line
295 dst: Right side of the line
296 """
297
298 @classmethod
299 def FromString(cls, rs):
300 lhs, rhs = rs.split(':', 2)
301 if lhs.startswith('+'):
302 lhs = lhs[1:]
303 forced = True
304 else:
305 forced = False
306 return cls(forced, lhs, rhs)
307
308 def __init__(self, forced, lhs, rhs):
309 self.forced = forced
310 self.src = lhs
311 self.dst = rhs
312
313 def SourceMatches(self, rev):
314 if self.src:
315 if rev == self.src:
316 return True
317 if self.src.endswith('/*') and rev.startswith(self.src[:-1]):
318 return True
319 return False
320
321 def DestMatches(self, ref):
322 if self.dst:
323 if ref == self.dst:
324 return True
325 if self.dst.endswith('/*') and ref.startswith(self.dst[:-1]):
326 return True
327 return False
328
329 def MapSource(self, rev):
330 if self.src.endswith('/*'):
331 return self.dst[:-1] + rev[len(self.src) - 1:]
332 return self.dst
333
334 def __str__(self):
335 s = ''
336 if self.forced:
337 s += '+'
338 if self.src:
339 s += self.src
340 if self.dst:
341 s += ':'
342 s += self.dst
343 return s
344
345
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700346_ssh_cache = {}
347_ssh_master = True
348
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700349def _open_ssh(host, port):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700350 global _ssh_master
351
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700352 key = '%s:%s' % (host, port)
353 if key in _ssh_cache:
354 return True
355
356 if not _ssh_master \
357 or 'GIT_SSH' in os.environ \
Shawn O. Pearce2b5b4ac2009-04-23 17:22:18 -0700358 or sys.platform in ('win32', 'cygwin'):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700359 # failed earlier, or cygwin ssh can't do this
360 #
361 return False
362
363 command = ['ssh',
364 '-o','ControlPath %s' % _ssh_sock(),
365 '-p',str(port),
366 '-M',
367 '-N',
368 host]
369 try:
370 Trace(': %s', ' '.join(command))
371 p = subprocess.Popen(command)
372 except Exception, e:
373 _ssh_master = False
374 print >>sys.stderr, \
375 '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
376 % (host,port, str(e))
377 return False
378
379 _ssh_cache[key] = p
380 time.sleep(1)
381 return True
382
383def close_ssh():
384 for key,p in _ssh_cache.iteritems():
385 os.kill(p.pid, SIGTERM)
386 p.wait()
387 _ssh_cache.clear()
388
389 d = _ssh_sock(create=False)
390 if d:
391 try:
392 os.rmdir(os.path.dirname(d))
393 except OSError:
394 pass
395
396URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
Shawn O. Pearce2f968c92009-04-30 14:30:28 -0700397URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700398
399def _preconnect(url):
400 m = URI_ALL.match(url)
401 if m:
402 scheme = m.group(1)
403 host = m.group(2)
404 if ':' in host:
405 host, port = host.split(':')
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700406 else:
407 port = 22
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700408 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
409 return _open_ssh(host, port)
410 return False
411
412 m = URI_SCP.match(url)
413 if m:
414 host = m.group(1)
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700415 return _open_ssh(host, 22)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700416
Shawn O. Pearce7b4f4352009-06-12 09:06:35 -0700417 return False
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700418
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700419class Remote(object):
420 """Configuration options related to a remote.
421 """
422 def __init__(self, config, name):
423 self._config = config
424 self.name = name
425 self.url = self._Get('url')
426 self.review = self._Get('review')
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800427 self.projectname = self._Get('projectname')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700428 self.fetch = map(lambda x: RefSpec.FromString(x),
429 self._Get('fetch', all=True))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800430 self._review_protocol = None
431
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700432 def PreConnectFetch(self):
433 return _preconnect(self.url)
434
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800435 @property
436 def ReviewProtocol(self):
437 if self._review_protocol is None:
438 if self.review is None:
439 return None
440
441 u = self.review
442 if not u.startswith('http:') and not u.startswith('https:'):
443 u = 'http://%s' % u
Shawn O. Pearce13cc3842009-03-25 13:54:54 -0700444 if u.endswith('/Gerrit'):
445 u = u[:len(u) - len('/Gerrit')]
446 if not u.endswith('/ssh_info'):
447 if not u.endswith('/'):
448 u += '/'
449 u += 'ssh_info'
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800450
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700451 if u in REVIEW_CACHE:
452 info = REVIEW_CACHE[u]
453 self._review_protocol = info[0]
454 self._review_host = info[1]
455 self._review_port = info[2]
456 else:
457 try:
458 info = urlopen(u).read()
459 if info == 'NOT_AVAILABLE':
460 raise UploadError('Upload over ssh unavailable')
461 if '<' in info:
462 # Assume the server gave us some sort of HTML
463 # response back, like maybe a login page.
464 #
465 raise UploadError('Cannot read %s:\n%s' % (u, info))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800466
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700467 self._review_protocol = 'ssh'
468 self._review_host = info.split(" ")[0]
469 self._review_port = info.split(" ")[1]
470 except HTTPError, e:
471 if e.code == 404:
472 self._review_protocol = 'http-post'
473 self._review_host = None
474 self._review_port = None
475 else:
476 raise UploadError('Cannot guess Gerrit version')
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800477
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700478 REVIEW_CACHE[u] = (
479 self._review_protocol,
480 self._review_host,
481 self._review_port)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800482 return self._review_protocol
483
484 def SshReviewUrl(self, userEmail):
485 if self.ReviewProtocol != 'ssh':
486 return None
487 return 'ssh://%s@%s:%s/%s' % (
488 userEmail.split("@")[0],
489 self._review_host,
490 self._review_port,
491 self.projectname)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700492
493 def ToLocal(self, rev):
494 """Convert a remote revision string to something we have locally.
495 """
496 if IsId(rev):
497 return rev
498 if rev.startswith(R_TAGS):
499 return rev
500
501 if not rev.startswith('refs/'):
502 rev = R_HEADS + rev
503
504 for spec in self.fetch:
505 if spec.SourceMatches(rev):
506 return spec.MapSource(rev)
507 raise GitError('remote %s does not have %s' % (self.name, rev))
508
509 def WritesTo(self, ref):
510 """True if the remote stores to the tracking ref.
511 """
512 for spec in self.fetch:
513 if spec.DestMatches(ref):
514 return True
515 return False
516
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800517 def ResetFetch(self, mirror=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700518 """Set the fetch refspec to its default value.
519 """
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800520 if mirror:
521 dst = 'refs/heads/*'
522 else:
523 dst = 'refs/remotes/%s/*' % self.name
524 self.fetch = [RefSpec(True, 'refs/heads/*', dst)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700525
526 def Save(self):
527 """Save this remote to the configuration.
528 """
529 self._Set('url', self.url)
530 self._Set('review', self.review)
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800531 self._Set('projectname', self.projectname)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700532 self._Set('fetch', map(lambda x: str(x), self.fetch))
533
534 def _Set(self, key, value):
535 key = 'remote.%s.%s' % (self.name, key)
536 return self._config.SetString(key, value)
537
538 def _Get(self, key, all=False):
539 key = 'remote.%s.%s' % (self.name, key)
540 return self._config.GetString(key, all = all)
541
542
543class Branch(object):
544 """Configuration options related to a single branch.
545 """
546 def __init__(self, config, name):
547 self._config = config
548 self.name = name
549 self.merge = self._Get('merge')
550
551 r = self._Get('remote')
552 if r:
553 self.remote = self._config.GetRemote(r)
554 else:
555 self.remote = None
556
557 @property
558 def LocalMerge(self):
559 """Convert the merge spec to a local name.
560 """
561 if self.remote and self.merge:
562 return self.remote.ToLocal(self.merge)
563 return None
564
565 def Save(self):
566 """Save this branch back into the configuration.
567 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700568 if self._config.HasSection('branch', self.name):
569 if self.remote:
570 self._Set('remote', self.remote.name)
571 else:
572 self._Set('remote', None)
573 self._Set('merge', self.merge)
574
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700575 else:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700576 fd = open(self._config.file, 'ab')
577 try:
578 fd.write('[branch "%s"]\n' % self.name)
579 if self.remote:
580 fd.write('\tremote = %s\n' % self.remote.name)
581 if self.merge:
582 fd.write('\tmerge = %s\n' % self.merge)
583 finally:
584 fd.close()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700585
586 def _Set(self, key, value):
587 key = 'branch.%s.%s' % (self.name, key)
588 return self._config.SetString(key, value)
589
590 def _Get(self, key, all=False):
591 key = 'branch.%s.%s' % (self.name, key)
592 return self._config.GetString(key, all = all)