blob: 6ea505f96bbc231521cf9f23bcb435739f3141fb [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Shawn O. Pearcec12c3602009-04-17 21:03:32 -070016import cPickle
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070017import os
18import re
Shawn O. Pearcefb231612009-04-10 18:53:46 -070019import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import sys
Shawn O. Pearcefb231612009-04-10 18:53:46 -070021import time
22from signal import SIGTERM
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080023from urllib2 import urlopen, HTTPError
24from error import GitError, UploadError
Shawn O. Pearcead3193a2009-04-18 09:54:51 -070025from trace import Trace
Shawn O. Pearcefb231612009-04-10 18:53:46 -070026from git_command import GitCommand, _ssh_sock
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070027
28R_HEADS = 'refs/heads/'
29R_TAGS = 'refs/tags/'
30ID_RE = re.compile('^[0-9a-f]{40}$')
31
Shawn O. Pearce146fe902009-03-25 14:06:43 -070032REVIEW_CACHE = dict()
33
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070034def IsId(rev):
35 return ID_RE.match(rev)
36
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070037def _key(name):
38 parts = name.split('.')
39 if len(parts) < 2:
40 return name.lower()
41 parts[ 0] = parts[ 0].lower()
42 parts[-1] = parts[-1].lower()
43 return '.'.join(parts)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070044
45class GitConfig(object):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070046 _ForUser = None
47
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070048 @classmethod
49 def ForUser(cls):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070050 if cls._ForUser is None:
51 cls._ForUser = cls(file = os.path.expanduser('~/.gitconfig'))
52 return cls._ForUser
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070053
54 @classmethod
55 def ForRepository(cls, gitdir, defaults=None):
56 return cls(file = os.path.join(gitdir, 'config'),
57 defaults = defaults)
58
59 def __init__(self, file, defaults=None):
60 self.file = file
61 self.defaults = defaults
62 self._cache_dict = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070063 self._section_dict = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070064 self._remotes = {}
65 self._branches = {}
Shawn O. Pearcec12c3602009-04-17 21:03:32 -070066 self._pickle = os.path.join(
67 os.path.dirname(self.file),
68 '.repopickle_' + os.path.basename(self.file))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070069
70 def Has(self, name, include_defaults = True):
71 """Return true if this configuration file has the key.
72 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070073 if _key(name) in self._cache:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070074 return True
75 if include_defaults and self.defaults:
76 return self.defaults.Has(name, include_defaults = True)
77 return False
78
79 def GetBoolean(self, name):
80 """Returns a boolean from the configuration file.
81 None : The value was not defined, or is not a boolean.
82 True : The value was set to true or yes.
83 False: The value was set to false or no.
84 """
85 v = self.GetString(name)
86 if v is None:
87 return None
88 v = v.lower()
89 if v in ('true', 'yes'):
90 return True
91 if v in ('false', 'no'):
92 return False
93 return None
94
95 def GetString(self, name, all=False):
96 """Get the first value for a key, or None if it is not defined.
97
98 This configuration file is used first, if the key is not
99 defined or all = True then the defaults are also searched.
100 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700101 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700102 v = self._cache[_key(name)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700103 except KeyError:
104 if self.defaults:
105 return self.defaults.GetString(name, all = all)
106 v = []
107
108 if not all:
109 if v:
110 return v[0]
111 return None
112
113 r = []
114 r.extend(v)
115 if self.defaults:
116 r.extend(self.defaults.GetString(name, all = True))
117 return r
118
119 def SetString(self, name, value):
120 """Set the value(s) for a key.
121 Only this configuration file is modified.
122
123 The supplied value should be either a string,
124 or a list of strings (to store multiple values).
125 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700126 key = _key(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700127
128 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700129 old = self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700130 except KeyError:
131 old = []
132
133 if value is None:
134 if old:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700135 del self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700136 self._do('--unset-all', name)
137
138 elif isinstance(value, list):
139 if len(value) == 0:
140 self.SetString(name, None)
141
142 elif len(value) == 1:
143 self.SetString(name, value[0])
144
145 elif old != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700146 self._cache[key] = list(value)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700147 self._do('--replace-all', name, value[0])
148 for i in xrange(1, len(value)):
149 self._do('--add', name, value[i])
150
151 elif len(old) != 1 or old[0] != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700152 self._cache[key] = [value]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700153 self._do('--replace-all', name, value)
154
155 def GetRemote(self, name):
156 """Get the remote.$name.* configuration values as an object.
157 """
158 try:
159 r = self._remotes[name]
160 except KeyError:
161 r = Remote(self, name)
162 self._remotes[r.name] = r
163 return r
164
165 def GetBranch(self, name):
166 """Get the branch.$name.* configuration values as an object.
167 """
168 try:
169 b = self._branches[name]
170 except KeyError:
171 b = Branch(self, name)
172 self._branches[b.name] = b
173 return b
174
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700175 def HasSection(self, section, subsection = ''):
176 """Does at least one key in section.subsection exist?
177 """
178 try:
179 return subsection in self._sections[section]
180 except KeyError:
181 return False
182
183 @property
184 def _sections(self):
185 d = self._section_dict
186 if d is None:
187 d = {}
188 for name in self._cache.keys():
189 p = name.split('.')
190 if 2 == len(p):
191 section = p[0]
192 subsect = ''
193 else:
194 section = p[0]
195 subsect = '.'.join(p[1:-1])
196 if section not in d:
197 d[section] = set()
198 d[section].add(subsect)
199 self._section_dict = d
200 return d
201
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700202 @property
203 def _cache(self):
204 if self._cache_dict is None:
205 self._cache_dict = self._Read()
206 return self._cache_dict
207
208 def _Read(self):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700209 d = self._ReadPickle()
210 if d is None:
211 d = self._ReadGit()
212 self._SavePickle(d)
213 return d
214
215 def _ReadPickle(self):
216 try:
217 if os.path.getmtime(self._pickle) \
218 <= os.path.getmtime(self.file):
219 os.remove(self._pickle)
220 return None
221 except OSError:
222 return None
223 try:
Shawn O. Pearcead3193a2009-04-18 09:54:51 -0700224 Trace(': unpickle %s', self.file)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700225 fd = open(self._pickle, 'rb')
226 try:
227 return cPickle.load(fd)
228 finally:
229 fd.close()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700230 except IOError:
231 os.remove(self._pickle)
232 return None
233 except cPickle.PickleError:
234 os.remove(self._pickle)
235 return None
236
237 def _SavePickle(self, cache):
238 try:
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700239 fd = open(self._pickle, 'wb')
240 try:
241 cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
242 finally:
243 fd.close()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700244 except IOError:
245 os.remove(self._pickle)
246 except cPickle.PickleError:
247 os.remove(self._pickle)
248
249 def _ReadGit(self):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700250 d = self._do('--null', '--list')
251 c = {}
252 while d:
253 lf = d.index('\n')
254 nul = d.index('\0', lf + 1)
255
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700256 key = _key(d[0:lf])
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700257 val = d[lf + 1:nul]
258
259 if key in c:
260 c[key].append(val)
261 else:
262 c[key] = [val]
263
264 d = d[nul + 1:]
265 return c
266
267 def _do(self, *args):
268 command = ['config', '--file', self.file]
269 command.extend(args)
270
271 p = GitCommand(None,
272 command,
273 capture_stdout = True,
274 capture_stderr = True)
275 if p.Wait() == 0:
276 return p.stdout
277 else:
278 GitError('git config %s: %s' % (str(args), p.stderr))
279
280
281class RefSpec(object):
282 """A Git refspec line, split into its components:
283
284 forced: True if the line starts with '+'
285 src: Left side of the line
286 dst: Right side of the line
287 """
288
289 @classmethod
290 def FromString(cls, rs):
291 lhs, rhs = rs.split(':', 2)
292 if lhs.startswith('+'):
293 lhs = lhs[1:]
294 forced = True
295 else:
296 forced = False
297 return cls(forced, lhs, rhs)
298
299 def __init__(self, forced, lhs, rhs):
300 self.forced = forced
301 self.src = lhs
302 self.dst = rhs
303
304 def SourceMatches(self, rev):
305 if self.src:
306 if rev == self.src:
307 return True
308 if self.src.endswith('/*') and rev.startswith(self.src[:-1]):
309 return True
310 return False
311
312 def DestMatches(self, ref):
313 if self.dst:
314 if ref == self.dst:
315 return True
316 if self.dst.endswith('/*') and ref.startswith(self.dst[:-1]):
317 return True
318 return False
319
320 def MapSource(self, rev):
321 if self.src.endswith('/*'):
322 return self.dst[:-1] + rev[len(self.src) - 1:]
323 return self.dst
324
325 def __str__(self):
326 s = ''
327 if self.forced:
328 s += '+'
329 if self.src:
330 s += self.src
331 if self.dst:
332 s += ':'
333 s += self.dst
334 return s
335
336
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700337_ssh_cache = {}
338_ssh_master = True
339
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700340def _open_ssh(host, port):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700341 global _ssh_master
342
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700343 key = '%s:%s' % (host, port)
344 if key in _ssh_cache:
345 return True
346
347 if not _ssh_master \
348 or 'GIT_SSH' in os.environ \
Shawn O. Pearce2b5b4ac2009-04-23 17:22:18 -0700349 or sys.platform in ('win32', 'cygwin'):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700350 # failed earlier, or cygwin ssh can't do this
351 #
352 return False
353
354 command = ['ssh',
355 '-o','ControlPath %s' % _ssh_sock(),
356 '-p',str(port),
357 '-M',
358 '-N',
359 host]
360 try:
361 Trace(': %s', ' '.join(command))
362 p = subprocess.Popen(command)
363 except Exception, e:
364 _ssh_master = False
365 print >>sys.stderr, \
366 '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
367 % (host,port, str(e))
368 return False
369
370 _ssh_cache[key] = p
371 time.sleep(1)
372 return True
373
374def close_ssh():
375 for key,p in _ssh_cache.iteritems():
376 os.kill(p.pid, SIGTERM)
377 p.wait()
378 _ssh_cache.clear()
379
380 d = _ssh_sock(create=False)
381 if d:
382 try:
383 os.rmdir(os.path.dirname(d))
384 except OSError:
385 pass
386
387URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
Shawn O. Pearce2f968c92009-04-30 14:30:28 -0700388URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700389
390def _preconnect(url):
391 m = URI_ALL.match(url)
392 if m:
393 scheme = m.group(1)
394 host = m.group(2)
395 if ':' in host:
396 host, port = host.split(':')
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700397 else:
398 port = 22
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700399 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
400 return _open_ssh(host, port)
401 return False
402
403 m = URI_SCP.match(url)
404 if m:
405 host = m.group(1)
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700406 return _open_ssh(host, 22)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700407
408
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700409class Remote(object):
410 """Configuration options related to a remote.
411 """
412 def __init__(self, config, name):
413 self._config = config
414 self.name = name
415 self.url = self._Get('url')
416 self.review = self._Get('review')
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800417 self.projectname = self._Get('projectname')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700418 self.fetch = map(lambda x: RefSpec.FromString(x),
419 self._Get('fetch', all=True))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800420 self._review_protocol = None
421
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700422 def PreConnectFetch(self):
423 return _preconnect(self.url)
424
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800425 @property
426 def ReviewProtocol(self):
427 if self._review_protocol is None:
428 if self.review is None:
429 return None
430
431 u = self.review
432 if not u.startswith('http:') and not u.startswith('https:'):
433 u = 'http://%s' % u
Shawn O. Pearce13cc3842009-03-25 13:54:54 -0700434 if u.endswith('/Gerrit'):
435 u = u[:len(u) - len('/Gerrit')]
436 if not u.endswith('/ssh_info'):
437 if not u.endswith('/'):
438 u += '/'
439 u += 'ssh_info'
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800440
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700441 if u in REVIEW_CACHE:
442 info = REVIEW_CACHE[u]
443 self._review_protocol = info[0]
444 self._review_host = info[1]
445 self._review_port = info[2]
446 else:
447 try:
448 info = urlopen(u).read()
449 if info == 'NOT_AVAILABLE':
450 raise UploadError('Upload over ssh unavailable')
451 if '<' in info:
452 # Assume the server gave us some sort of HTML
453 # response back, like maybe a login page.
454 #
455 raise UploadError('Cannot read %s:\n%s' % (u, info))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800456
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700457 self._review_protocol = 'ssh'
458 self._review_host = info.split(" ")[0]
459 self._review_port = info.split(" ")[1]
460 except HTTPError, e:
461 if e.code == 404:
462 self._review_protocol = 'http-post'
463 self._review_host = None
464 self._review_port = None
465 else:
466 raise UploadError('Cannot guess Gerrit version')
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800467
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700468 REVIEW_CACHE[u] = (
469 self._review_protocol,
470 self._review_host,
471 self._review_port)
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800472 return self._review_protocol
473
474 def SshReviewUrl(self, userEmail):
475 if self.ReviewProtocol != 'ssh':
476 return None
477 return 'ssh://%s@%s:%s/%s' % (
478 userEmail.split("@")[0],
479 self._review_host,
480 self._review_port,
481 self.projectname)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700482
483 def ToLocal(self, rev):
484 """Convert a remote revision string to something we have locally.
485 """
486 if IsId(rev):
487 return rev
488 if rev.startswith(R_TAGS):
489 return rev
490
491 if not rev.startswith('refs/'):
492 rev = R_HEADS + rev
493
494 for spec in self.fetch:
495 if spec.SourceMatches(rev):
496 return spec.MapSource(rev)
497 raise GitError('remote %s does not have %s' % (self.name, rev))
498
499 def WritesTo(self, ref):
500 """True if the remote stores to the tracking ref.
501 """
502 for spec in self.fetch:
503 if spec.DestMatches(ref):
504 return True
505 return False
506
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800507 def ResetFetch(self, mirror=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700508 """Set the fetch refspec to its default value.
509 """
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800510 if mirror:
511 dst = 'refs/heads/*'
512 else:
513 dst = 'refs/remotes/%s/*' % self.name
514 self.fetch = [RefSpec(True, 'refs/heads/*', dst)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700515
516 def Save(self):
517 """Save this remote to the configuration.
518 """
519 self._Set('url', self.url)
520 self._Set('review', self.review)
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800521 self._Set('projectname', self.projectname)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700522 self._Set('fetch', map(lambda x: str(x), self.fetch))
523
524 def _Set(self, key, value):
525 key = 'remote.%s.%s' % (self.name, key)
526 return self._config.SetString(key, value)
527
528 def _Get(self, key, all=False):
529 key = 'remote.%s.%s' % (self.name, key)
530 return self._config.GetString(key, all = all)
531
532
533class Branch(object):
534 """Configuration options related to a single branch.
535 """
536 def __init__(self, config, name):
537 self._config = config
538 self.name = name
539 self.merge = self._Get('merge')
540
541 r = self._Get('remote')
542 if r:
543 self.remote = self._config.GetRemote(r)
544 else:
545 self.remote = None
546
547 @property
548 def LocalMerge(self):
549 """Convert the merge spec to a local name.
550 """
551 if self.remote and self.merge:
552 return self.remote.ToLocal(self.merge)
553 return None
554
555 def Save(self):
556 """Save this branch back into the configuration.
557 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700558 if self._config.HasSection('branch', self.name):
559 if self.remote:
560 self._Set('remote', self.remote.name)
561 else:
562 self._Set('remote', None)
563 self._Set('merge', self.merge)
564
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700565 else:
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700566 fd = open(self._config.file, 'ab')
567 try:
568 fd.write('[branch "%s"]\n' % self.name)
569 if self.remote:
570 fd.write('\tremote = %s\n' % self.remote.name)
571 if self.merge:
572 fd.write('\tmerge = %s\n' % self.merge)
573 finally:
574 fd.close()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700575
576 def _Set(self, key, value):
577 key = 'branch.%s.%s' % (self.name, key)
578 return self._config.SetString(key, value)
579
580 def _Get(self, key, all=False):
581 key = 'branch.%s.%s' % (self.name, key)
582 return self._config.GetString(key, all = all)