blob: 36e8d939e91887714a3868dc11e45701b43a091a [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Shawn O. Pearcec12c3602009-04-17 21:03:32 -070016import cPickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
18import re
Shawn O. Pearcefb231612009-04-10 18:53:46 -070019import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import sys
Shawn O. Pearcefb231612009-04-10 18:53:46 -070021import time
22from signal import SIGTERM
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080023from urllib2 import urlopen, HTTPError
24from error import GitError, UploadError
Shawn O. Pearcead3193a2009-04-18 09:54:51 -070025from trace import Trace
Shawn O. Pearcefb231612009-04-10 18:53:46 -070026from git_command import GitCommand, _ssh_sock
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070027
28R_HEADS = 'refs/heads/'
29R_TAGS = 'refs/tags/'
30ID_RE = re.compile('^[0-9a-f]{40}$')
31
Shawn O. Pearce146fe902009-03-25 14:06:43 -070032REVIEW_CACHE = dict()
33
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070034def IsId(rev):
35 return ID_RE.match(rev)
36
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070037def _key(name):
38 parts = name.split('.')
39 if len(parts) < 2:
40 return name.lower()
41 parts[ 0] = parts[ 0].lower()
42 parts[-1] = parts[-1].lower()
43 return '.'.join(parts)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070044
45class GitConfig(object):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070046 _ForUser = None
47
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070048 @classmethod
49 def ForUser(cls):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070050 if cls._ForUser is None:
51 cls._ForUser = cls(file = os.path.expanduser('~/.gitconfig'))
52 return cls._ForUser
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070053
54 @classmethod
55 def ForRepository(cls, gitdir, defaults=None):
56 return cls(file = os.path.join(gitdir, 'config'),
57 defaults = defaults)
58
Shawn O. Pearce1b34c912009-05-21 18:52:49 -070059 def __init__(self, file, defaults=None, pickleFile=None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070060 self.file = file
61 self.defaults = defaults
62 self._cache_dict = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070063 self._section_dict = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070064 self._remotes = {}
65 self._branches = {}
Shawn O. Pearce1b34c912009-05-21 18:52:49 -070066
67 if pickleFile is None:
68 self._pickle = os.path.join(
69 os.path.dirname(self.file),
70 '.repopickle_' + os.path.basename(self.file))
71 else:
72 self._pickle = pickleFile
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070073
74 def Has(self, name, include_defaults = True):
75 """Return true if this configuration file has the key.
76 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070077 if _key(name) in self._cache:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070078 return True
79 if include_defaults and self.defaults:
80 return self.defaults.Has(name, include_defaults = True)
81 return False
82
83 def GetBoolean(self, name):
84 """Returns a boolean from the configuration file.
85 None : The value was not defined, or is not a boolean.
86 True : The value was set to true or yes.
87 False: The value was set to false or no.
88 """
89 v = self.GetString(name)
90 if v is None:
91 return None
92 v = v.lower()
93 if v in ('true', 'yes'):
94 return True
95 if v in ('false', 'no'):
96 return False
97 return None
98
99 def GetString(self, name, all=False):
100 """Get the first value for a key, or None if it is not defined.
101
102 This configuration file is used first, if the key is not
103 defined or all = True then the defaults are also searched.
104 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700105 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700106 v = self._cache[_key(name)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700107 except KeyError:
108 if self.defaults:
109 return self.defaults.GetString(name, all = all)
110 v = []
111
112 if not all:
113 if v:
114 return v[0]
115 return None
116
117 r = []
118 r.extend(v)
119 if self.defaults:
120 r.extend(self.defaults.GetString(name, all = True))
121 return r
122
123 def SetString(self, name, value):
124 """Set the value(s) for a key.
125 Only this configuration file is modified.
126
127 The supplied value should be either a string,
128 or a list of strings (to store multiple values).
129 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700130 key = _key(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700131
132 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700133 old = self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700134 except KeyError:
135 old = []
136
137 if value is None:
138 if old:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700139 del self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700140 self._do('--unset-all', name)
141
142 elif isinstance(value, list):
143 if len(value) == 0:
144 self.SetString(name, None)
145
146 elif len(value) == 1:
147 self.SetString(name, value[0])
148
149 elif old != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700150 self._cache[key] = list(value)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700151 self._do('--replace-all', name, value[0])
152 for i in xrange(1, len(value)):
153 self._do('--add', name, value[i])
154
155 elif len(old) != 1 or old[0] != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700156 self._cache[key] = [value]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700157 self._do('--replace-all', name, value)
158
159 def GetRemote(self, name):
160 """Get the remote.$name.* configuration values as an object.
161 """
162 try:
163 r = self._remotes[name]
164 except KeyError:
165 r = Remote(self, name)
166 self._remotes[r.name] = r
167 return r
168
169 def GetBranch(self, name):
170 """Get the branch.$name.* configuration values as an object.
171 """
172 try:
173 b = self._branches[name]
174 except KeyError:
175 b = Branch(self, name)
176 self._branches[b.name] = b
177 return b
178
Shawn O. Pearce366ad212009-05-19 12:47:37 -0700179 def GetSubSections(self, section):
180 """List all subsection names matching $section.*.*
181 """
182 return self._sections.get(section, set())
183
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700184 def HasSection(self, section, subsection = ''):
185 """Does at least one key in section.subsection exist?
186 """
187 try:
188 return subsection in self._sections[section]
189 except KeyError:
190 return False
191
192 @property
193 def _sections(self):
194 d = self._section_dict
195 if d is None:
196 d = {}
197 for name in self._cache.keys():
198 p = name.split('.')
199 if 2 == len(p):
200 section = p[0]
201 subsect = ''
202 else:
203 section = p[0]
204 subsect = '.'.join(p[1:-1])
205 if section not in d:
206 d[section] = set()
207 d[section].add(subsect)
208 self._section_dict = d
209 return d
210
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700211 @property
212 def _cache(self):
213 if self._cache_dict is None:
214 self._cache_dict = self._Read()
215 return self._cache_dict
216
217 def _Read(self):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700218 d = self._ReadPickle()
219 if d is None:
220 d = self._ReadGit()
221 self._SavePickle(d)
222 return d
223
224 def _ReadPickle(self):
225 try:
226 if os.path.getmtime(self._pickle) \
227 <= os.path.getmtime(self.file):
228 os.remove(self._pickle)
229 return None
230 except OSError:
231 return None
232 try:
Shawn O. Pearcead3193a2009-04-18 09:54:51 -0700233 Trace(': unpickle %s', self.file)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700234 fd = open(self._pickle, 'rb')
235 try:
236 return cPickle.load(fd)
237 finally:
238 fd.close()
Shawn O. Pearce2a3a81b2009-06-12 09:10:07 -0700239 except EOFError:
240 os.remove(self._pickle)
241 return None
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700242 except IOError:
243 os.remove(self._pickle)
244 return None
245 except cPickle.PickleError:
246 os.remove(self._pickle)
247 return None
248
249 def _SavePickle(self, cache):
250 try:
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700251 fd = open(self._pickle, 'wb')
252 try:
253 cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
254 finally:
255 fd.close()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700256 except IOError:
257 os.remove(self._pickle)
258 except cPickle.PickleError:
259 os.remove(self._pickle)
260
261 def _ReadGit(self):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700262 d = self._do('--null', '--list')
263 c = {}
264 while d:
265 lf = d.index('\n')
266 nul = d.index('\0', lf + 1)
267
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700268 key = _key(d[0:lf])
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700269 val = d[lf + 1:nul]
270
271 if key in c:
272 c[key].append(val)
273 else:
274 c[key] = [val]
275
276 d = d[nul + 1:]
277 return c
278
279 def _do(self, *args):
280 command = ['config', '--file', self.file]
281 command.extend(args)
282
283 p = GitCommand(None,
284 command,
285 capture_stdout = True,
286 capture_stderr = True)
287 if p.Wait() == 0:
288 return p.stdout
289 else:
290 GitError('git config %s: %s' % (str(args), p.stderr))
291
292
293class RefSpec(object):
294 """A Git refspec line, split into its components:
295
296 forced: True if the line starts with '+'
297 src: Left side of the line
298 dst: Right side of the line
299 """
300
301 @classmethod
302 def FromString(cls, rs):
303 lhs, rhs = rs.split(':', 2)
304 if lhs.startswith('+'):
305 lhs = lhs[1:]
306 forced = True
307 else:
308 forced = False
309 return cls(forced, lhs, rhs)
310
311 def __init__(self, forced, lhs, rhs):
312 self.forced = forced
313 self.src = lhs
314 self.dst = rhs
315
316 def SourceMatches(self, rev):
317 if self.src:
318 if rev == self.src:
319 return True
320 if self.src.endswith('/*') and rev.startswith(self.src[:-1]):
321 return True
322 return False
323
324 def DestMatches(self, ref):
325 if self.dst:
326 if ref == self.dst:
327 return True
328 if self.dst.endswith('/*') and ref.startswith(self.dst[:-1]):
329 return True
330 return False
331
332 def MapSource(self, rev):
333 if self.src.endswith('/*'):
334 return self.dst[:-1] + rev[len(self.src) - 1:]
335 return self.dst
336
337 def __str__(self):
338 s = ''
339 if self.forced:
340 s += '+'
341 if self.src:
342 s += self.src
343 if self.dst:
344 s += ':'
345 s += self.dst
346 return s
347
348
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700349_ssh_cache = {}
350_ssh_master = True
351
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700352def _open_ssh(host, port):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700353 global _ssh_master
354
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700355 key = '%s:%s' % (host, port)
356 if key in _ssh_cache:
357 return True
358
359 if not _ssh_master \
360 or 'GIT_SSH' in os.environ \
Shawn O. Pearce2b5b4ac2009-04-23 17:22:18 -0700361 or sys.platform in ('win32', 'cygwin'):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700362 # failed earlier, or cygwin ssh can't do this
363 #
364 return False
365
366 command = ['ssh',
367 '-o','ControlPath %s' % _ssh_sock(),
368 '-p',str(port),
369 '-M',
370 '-N',
371 host]
372 try:
373 Trace(': %s', ' '.join(command))
374 p = subprocess.Popen(command)
375 except Exception, e:
376 _ssh_master = False
377 print >>sys.stderr, \
378 '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
379 % (host,port, str(e))
380 return False
381
382 _ssh_cache[key] = p
383 time.sleep(1)
384 return True
385
386def close_ssh():
387 for key,p in _ssh_cache.iteritems():
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700388 try:
389 os.kill(p.pid, SIGTERM)
390 p.wait()
391 catch OSError:
392 pass
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700393 _ssh_cache.clear()
394
395 d = _ssh_sock(create=False)
396 if d:
397 try:
398 os.rmdir(os.path.dirname(d))
399 except OSError:
400 pass
401
402URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
Shawn O. Pearce2f968c92009-04-30 14:30:28 -0700403URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700404
405def _preconnect(url):
406 m = URI_ALL.match(url)
407 if m:
408 scheme = m.group(1)
409 host = m.group(2)
410 if ':' in host:
411 host, port = host.split(':')
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700412 else:
413 port = 22
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700414 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
415 return _open_ssh(host, port)
416 return False
417
418 m = URI_SCP.match(url)
419 if m:
420 host = m.group(1)
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700421 return _open_ssh(host, 22)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700422
Shawn O. Pearce7b4f4352009-06-12 09:06:35 -0700423 return False
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700424
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700425class Remote(object):
426 """Configuration options related to a remote.
427 """
428 def __init__(self, config, name):
429 self._config = config
430 self.name = name
431 self.url = self._Get('url')
432 self.review = self._Get('review')
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800433 self.projectname = self._Get('projectname')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700434 self.fetch = map(lambda x: RefSpec.FromString(x),
435 self._Get('fetch', all=True))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800436 self._review_protocol = None
437
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700438 def PreConnectFetch(self):
439 return _preconnect(self.url)
440
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800441 @property
442 def ReviewProtocol(self):
443 if self._review_protocol is None:
444 if self.review is None:
445 return None
446
447 u = self.review
448 if not u.startswith('http:') and not u.startswith('https:'):
449 u = 'http://%s' % u
Shawn O. Pearce13cc3842009-03-25 13:54:54 -0700450 if u.endswith('/Gerrit'):
451 u = u[:len(u) - len('/Gerrit')]
452 if not u.endswith('/ssh_info'):
453 if not u.endswith('/'):
454 u += '/'
455 u += 'ssh_info'
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800456
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700457 if u in REVIEW_CACHE:
458 info = REVIEW_CACHE[u]
459 self._review_protocol = info[0]
460 self._review_host = info[1]
461 self._review_port = info[2]
462 else:
463 try:
464 info = urlopen(u).read()
465 if info == 'NOT_AVAILABLE':
466 raise UploadError('Upload over ssh unavailable')
467 if '<' in info:
468 # Assume the server gave us some sort of HTML
469 # response back, like maybe a login page.
470 #
471 raise UploadError('Cannot read %s:\n%s' % (u, info))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800472
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700473 self._review_protocol = 'ssh'
474 self._review_host = info.split(" ")[0]
475 self._review_port = info.split(" ")[1]
476 except HTTPError, e:
477 if e.code == 404:
478 self._review_protocol = 'http-post'
479 self._review_host = None
480 self._review_port = None
481 else:
482 raise UploadError('Cannot guess Gerrit version')
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800483
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700484 REVIEW_CACHE[u] = (
485 self._review_protocol,
486 self._review_host,
487 self._review_port)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800488 return self._review_protocol
489
490 def SshReviewUrl(self, userEmail):
491 if self.ReviewProtocol != 'ssh':
492 return None
493 return 'ssh://%s@%s:%s/%s' % (
494 userEmail.split("@")[0],
495 self._review_host,
496 self._review_port,
497 self.projectname)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700498
499 def ToLocal(self, rev):
500 """Convert a remote revision string to something we have locally.
501 """
502 if IsId(rev):
503 return rev
504 if rev.startswith(R_TAGS):
505 return rev
506
507 if not rev.startswith('refs/'):
508 rev = R_HEADS + rev
509
510 for spec in self.fetch:
511 if spec.SourceMatches(rev):
512 return spec.MapSource(rev)
513 raise GitError('remote %s does not have %s' % (self.name, rev))
514
515 def WritesTo(self, ref):
516 """True if the remote stores to the tracking ref.
517 """
518 for spec in self.fetch:
519 if spec.DestMatches(ref):
520 return True
521 return False
522
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800523 def ResetFetch(self, mirror=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700524 """Set the fetch refspec to its default value.
525 """
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800526 if mirror:
527 dst = 'refs/heads/*'
528 else:
529 dst = 'refs/remotes/%s/*' % self.name
530 self.fetch = [RefSpec(True, 'refs/heads/*', dst)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700531
532 def Save(self):
533 """Save this remote to the configuration.
534 """
535 self._Set('url', self.url)
536 self._Set('review', self.review)
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800537 self._Set('projectname', self.projectname)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700538 self._Set('fetch', map(lambda x: str(x), self.fetch))
539
540 def _Set(self, key, value):
541 key = 'remote.%s.%s' % (self.name, key)
542 return self._config.SetString(key, value)
543
544 def _Get(self, key, all=False):
545 key = 'remote.%s.%s' % (self.name, key)
546 return self._config.GetString(key, all = all)
547
548
549class Branch(object):
550 """Configuration options related to a single branch.
551 """
552 def __init__(self, config, name):
553 self._config = config
554 self.name = name
555 self.merge = self._Get('merge')
556
557 r = self._Get('remote')
558 if r:
559 self.remote = self._config.GetRemote(r)
560 else:
561 self.remote = None
562
563 @property
564 def LocalMerge(self):
565 """Convert the merge spec to a local name.
566 """
567 if self.remote and self.merge:
568 return self.remote.ToLocal(self.merge)
569 return None
570
571 def Save(self):
572 """Save this branch back into the configuration.
573 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700574 if self._config.HasSection('branch', self.name):
575 if self.remote:
576 self._Set('remote', self.remote.name)
577 else:
578 self._Set('remote', None)
579 self._Set('merge', self.merge)
580
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700581 else:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700582 fd = open(self._config.file, 'ab')
583 try:
584 fd.write('[branch "%s"]\n' % self.name)
585 if self.remote:
586 fd.write('\tremote = %s\n' % self.remote.name)
587 if self.merge:
588 fd.write('\tmerge = %s\n' % self.merge)
589 finally:
590 fd.close()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700591
592 def _Set(self, key, value):
593 key = 'branch.%s.%s' % (self.name, key)
594 return self._config.SetString(key, value)
595
596 def _Get(self, key, all=False):
597 key = 'branch.%s.%s' % (self.name, key)
598 return self._config.GetString(key, all = all)