blob: 70b22ce1ffe803aba8f255e43d5d6f0badc72630 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Chirayu Desai217ea7d2013-03-01 19:14:38 +053017
Dan Willemsen0745bb22015-08-17 13:41:45 -070018import contextlib
19import errno
Anthony King85b24ac2014-05-06 15:57:48 +010020import json
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import os
22import re
Łukasz Gardońbed59ce2017-08-08 10:18:11 +020023import ssl
Shawn O. Pearcefb231612009-04-10 18:53:46 -070024import subprocess
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Doug Anderson0048b692010-12-21 13:39:23 -080026try:
27 import threading as _threading
28except ImportError:
29 import dummy_threading as _threading
Shawn O. Pearcefb231612009-04-10 18:53:46 -070030import time
David Pursehouse59bbb582013-05-17 10:49:33 +090031
32from pyversion import is_python3
33if is_python3():
Sarah Owens1f7627f2012-10-31 09:21:55 -070034 import urllib.request
35 import urllib.error
36else:
David Pursehouse59bbb582013-05-17 10:49:33 +090037 import urllib2
Sarah Owens1f7627f2012-10-31 09:21:55 -070038 import imp
39 urllib = imp.new_module('urllib')
40 urllib.request = urllib2
41 urllib.error = urllib2
Shawn O. Pearcef00e0ce2009-08-22 18:39:49 -070042
Shawn O. Pearcefb231612009-04-10 18:53:46 -070043from signal import SIGTERM
Shawn O. Pearceb54a3922009-01-05 16:18:58 -080044from error import GitError, UploadError
Renaud Paquay010fed72016-11-11 14:25:29 -080045import platform_utils
Shawn O. Pearcead3193a2009-04-18 09:54:51 -070046from trace import Trace
David Pursehouseecf8f2b2013-05-24 12:12:23 +090047if is_python3():
48 from http.client import HTTPException
49else:
50 from httplib import HTTPException
Shawn O. Pearceca8c32c2010-05-11 18:21:33 -070051
52from git_command import GitCommand
53from git_command import ssh_sock
54from git_command import terminate_ssh_clients
Zac Livingston9ead97b2017-06-13 08:29:04 -060055from git_refs import R_CHANGES, R_HEADS, R_TAGS
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070056
David Pursehouse1d947b32012-10-25 12:23:11 +090057ID_RE = re.compile(r'^[0-9a-f]{40}$')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070058
Shawn O. Pearce146fe902009-03-25 14:06:43 -070059REVIEW_CACHE = dict()
60
Zac Livingston9ead97b2017-06-13 08:29:04 -060061def IsChange(rev):
62 return rev.startswith(R_CHANGES)
63
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070064def IsId(rev):
65 return ID_RE.match(rev)
66
Zac Livingston9ead97b2017-06-13 08:29:04 -060067def IsTag(rev):
68 return rev.startswith(R_TAGS)
69
70def IsImmutable(rev):
71 return IsChange(rev) or IsId(rev) or IsTag(rev)
72
Shawn O. Pearcef8e32732009-04-17 11:00:31 -070073def _key(name):
74 parts = name.split('.')
75 if len(parts) < 2:
76 return name.lower()
77 parts[ 0] = parts[ 0].lower()
78 parts[-1] = parts[-1].lower()
79 return '.'.join(parts)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070080
81class GitConfig(object):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070082 _ForUser = None
83
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070084 @classmethod
85 def ForUser(cls):
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070086 if cls._ForUser is None:
David Pursehouse8a68ff92012-09-24 12:15:13 +090087 cls._ForUser = cls(configfile = os.path.expanduser('~/.gitconfig'))
Shawn O. Pearce90be5c02008-10-29 15:21:24 -070088 return cls._ForUser
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070089
90 @classmethod
91 def ForRepository(cls, gitdir, defaults=None):
David Pursehouse8a68ff92012-09-24 12:15:13 +090092 return cls(configfile = os.path.join(gitdir, 'config'),
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070093 defaults = defaults)
94
Anthony King85b24ac2014-05-06 15:57:48 +010095 def __init__(self, configfile, defaults=None, jsonFile=None):
David Pursehouse8a68ff92012-09-24 12:15:13 +090096 self.file = configfile
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070097 self.defaults = defaults
98 self._cache_dict = None
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -070099 self._section_dict = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700100 self._remotes = {}
101 self._branches = {}
Shawn O. Pearce1b34c912009-05-21 18:52:49 -0700102
Anthony King85b24ac2014-05-06 15:57:48 +0100103 self._json = jsonFile
104 if self._json is None:
105 self._json = os.path.join(
Shawn O. Pearce1b34c912009-05-21 18:52:49 -0700106 os.path.dirname(self.file),
Anthony King85b24ac2014-05-06 15:57:48 +0100107 '.repo_' + os.path.basename(self.file) + '.json')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700108
109 def Has(self, name, include_defaults = True):
110 """Return true if this configuration file has the key.
111 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700112 if _key(name) in self._cache:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700113 return True
114 if include_defaults and self.defaults:
115 return self.defaults.Has(name, include_defaults = True)
116 return False
117
118 def GetBoolean(self, name):
119 """Returns a boolean from the configuration file.
120 None : The value was not defined, or is not a boolean.
121 True : The value was set to true or yes.
122 False: The value was set to false or no.
123 """
124 v = self.GetString(name)
125 if v is None:
126 return None
127 v = v.lower()
128 if v in ('true', 'yes'):
129 return True
130 if v in ('false', 'no'):
131 return False
132 return None
133
David Pursehouse8a68ff92012-09-24 12:15:13 +0900134 def GetString(self, name, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700135 """Get the first value for a key, or None if it is not defined.
136
137 This configuration file is used first, if the key is not
David Pursehouse8a68ff92012-09-24 12:15:13 +0900138 defined or all_keys = True then the defaults are also searched.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700139 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700140 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700141 v = self._cache[_key(name)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700142 except KeyError:
143 if self.defaults:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900144 return self.defaults.GetString(name, all_keys = all_keys)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700145 v = []
146
David Pursehouse8a68ff92012-09-24 12:15:13 +0900147 if not all_keys:
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700148 if v:
149 return v[0]
150 return None
151
152 r = []
153 r.extend(v)
154 if self.defaults:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900155 r.extend(self.defaults.GetString(name, all_keys = True))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700156 return r
157
158 def SetString(self, name, value):
159 """Set the value(s) for a key.
160 Only this configuration file is modified.
161
162 The supplied value should be either a string,
163 or a list of strings (to store multiple values).
164 """
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700165 key = _key(name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700166
167 try:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700168 old = self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700169 except KeyError:
170 old = []
171
172 if value is None:
173 if old:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700174 del self._cache[key]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700175 self._do('--unset-all', name)
176
177 elif isinstance(value, list):
178 if len(value) == 0:
179 self.SetString(name, None)
180
181 elif len(value) == 1:
182 self.SetString(name, value[0])
183
184 elif old != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700185 self._cache[key] = list(value)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700186 self._do('--replace-all', name, value[0])
Sarah Owensa6053d52012-11-01 13:36:50 -0700187 for i in range(1, len(value)):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700188 self._do('--add', name, value[i])
189
190 elif len(old) != 1 or old[0] != value:
Shawn O. Pearcef8e32732009-04-17 11:00:31 -0700191 self._cache[key] = [value]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700192 self._do('--replace-all', name, value)
193
194 def GetRemote(self, name):
195 """Get the remote.$name.* configuration values as an object.
196 """
197 try:
198 r = self._remotes[name]
199 except KeyError:
200 r = Remote(self, name)
201 self._remotes[r.name] = r
202 return r
203
204 def GetBranch(self, name):
205 """Get the branch.$name.* configuration values as an object.
206 """
207 try:
208 b = self._branches[name]
209 except KeyError:
210 b = Branch(self, name)
211 self._branches[b.name] = b
212 return b
213
Shawn O. Pearce366ad212009-05-19 12:47:37 -0700214 def GetSubSections(self, section):
215 """List all subsection names matching $section.*.*
216 """
217 return self._sections.get(section, set())
218
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700219 def HasSection(self, section, subsection = ''):
220 """Does at least one key in section.subsection exist?
221 """
222 try:
223 return subsection in self._sections[section]
224 except KeyError:
225 return False
226
Shawn O. Pearce13111b42011-09-19 11:00:31 -0700227 def UrlInsteadOf(self, url):
228 """Resolve any url.*.insteadof references.
229 """
230 for new_url in self.GetSubSections('url'):
Dan Willemsen4e4d40f2013-10-28 22:28:42 -0700231 for old_url in self.GetString('url.%s.insteadof' % new_url, True):
232 if old_url is not None and url.startswith(old_url):
233 return new_url + url[len(old_url):]
Shawn O. Pearce13111b42011-09-19 11:00:31 -0700234 return url
235
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700236 @property
237 def _sections(self):
238 d = self._section_dict
239 if d is None:
240 d = {}
241 for name in self._cache.keys():
242 p = name.split('.')
243 if 2 == len(p):
244 section = p[0]
245 subsect = ''
246 else:
247 section = p[0]
248 subsect = '.'.join(p[1:-1])
249 if section not in d:
250 d[section] = set()
251 d[section].add(subsect)
252 self._section_dict = d
253 return d
254
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700255 @property
256 def _cache(self):
257 if self._cache_dict is None:
258 self._cache_dict = self._Read()
259 return self._cache_dict
260
261 def _Read(self):
Anthony King85b24ac2014-05-06 15:57:48 +0100262 d = self._ReadJson()
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700263 if d is None:
264 d = self._ReadGit()
Anthony King85b24ac2014-05-06 15:57:48 +0100265 self._SaveJson(d)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700266 return d
267
Anthony King85b24ac2014-05-06 15:57:48 +0100268 def _ReadJson(self):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700269 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100270 if os.path.getmtime(self._json) \
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700271 <= os.path.getmtime(self.file):
Renaud Paquay010fed72016-11-11 14:25:29 -0800272 platform_utils.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700273 return None
274 except OSError:
275 return None
276 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100277 Trace(': parsing %s', self.file)
278 fd = open(self._json)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700279 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100280 return json.load(fd)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700281 finally:
282 fd.close()
Anthony King85b24ac2014-05-06 15:57:48 +0100283 except (IOError, ValueError):
Renaud Paquay010fed72016-11-11 14:25:29 -0800284 platform_utils.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700285 return None
286
Anthony King85b24ac2014-05-06 15:57:48 +0100287 def _SaveJson(self, cache):
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700288 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100289 fd = open(self._json, 'w')
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700290 try:
Anthony King85b24ac2014-05-06 15:57:48 +0100291 json.dump(cache, fd, indent=2)
Shawn O. Pearce76ca9f82009-04-18 14:48:03 -0700292 finally:
293 fd.close()
Anthony King85b24ac2014-05-06 15:57:48 +0100294 except (IOError, TypeError):
Anthony Kingb1d1fd72015-06-03 17:02:26 +0100295 if os.path.exists(self._json):
Renaud Paquay010fed72016-11-11 14:25:29 -0800296 platform_utils.remove(self._json)
Shawn O. Pearcec12c3602009-04-17 21:03:32 -0700297
298 def _ReadGit(self):
David Aguilar438c5472009-06-28 15:09:16 -0700299 """
300 Read configuration data from git.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700301
David Aguilar438c5472009-06-28 15:09:16 -0700302 This internal method populates the GitConfig cache.
303
304 """
David Aguilar438c5472009-06-28 15:09:16 -0700305 c = {}
Shawn O. Pearcec24c7202009-07-02 16:12:57 -0700306 d = self._do('--null', '--list')
307 if d is None:
308 return c
Dylan Denge469a0c2018-06-23 15:02:26 +0800309 if not is_python3():
310 d = d.decode('utf-8')
311 for line in d.rstrip('\0').split('\0'):
David Aguilar438c5472009-06-28 15:09:16 -0700312 if '\n' in line:
David Pursehousec1b86a22012-11-14 11:36:51 +0900313 key, val = line.split('\n', 1)
David Aguilar438c5472009-06-28 15:09:16 -0700314 else:
David Pursehousec1b86a22012-11-14 11:36:51 +0900315 key = line
316 val = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700317
318 if key in c:
319 c[key].append(val)
320 else:
321 c[key] = [val]
322
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700323 return c
324
325 def _do(self, *args):
326 command = ['config', '--file', self.file]
327 command.extend(args)
328
329 p = GitCommand(None,
330 command,
331 capture_stdout = True,
332 capture_stderr = True)
333 if p.Wait() == 0:
334 return p.stdout
335 else:
336 GitError('git config %s: %s' % (str(args), p.stderr))
337
338
339class RefSpec(object):
340 """A Git refspec line, split into its components:
341
342 forced: True if the line starts with '+'
343 src: Left side of the line
344 dst: Right side of the line
345 """
346
347 @classmethod
348 def FromString(cls, rs):
349 lhs, rhs = rs.split(':', 2)
350 if lhs.startswith('+'):
351 lhs = lhs[1:]
352 forced = True
353 else:
354 forced = False
355 return cls(forced, lhs, rhs)
356
357 def __init__(self, forced, lhs, rhs):
358 self.forced = forced
359 self.src = lhs
360 self.dst = rhs
361
362 def SourceMatches(self, rev):
363 if self.src:
364 if rev == self.src:
365 return True
366 if self.src.endswith('/*') and rev.startswith(self.src[:-1]):
367 return True
368 return False
369
370 def DestMatches(self, ref):
371 if self.dst:
372 if ref == self.dst:
373 return True
374 if self.dst.endswith('/*') and ref.startswith(self.dst[:-1]):
375 return True
376 return False
377
378 def MapSource(self, rev):
379 if self.src.endswith('/*'):
380 return self.dst[:-1] + rev[len(self.src) - 1:]
381 return self.dst
382
383 def __str__(self):
384 s = ''
385 if self.forced:
386 s += '+'
387 if self.src:
388 s += self.src
389 if self.dst:
390 s += ':'
391 s += self.dst
392 return s
393
394
Doug Anderson06d029c2010-10-27 17:06:01 -0700395_master_processes = []
396_master_keys = set()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700397_ssh_master = True
Doug Anderson0048b692010-12-21 13:39:23 -0800398_master_keys_lock = None
399
400def init_ssh():
401 """Should be called once at the start of repo to init ssh master handling.
402
403 At the moment, all we do is to create our lock.
404 """
405 global _master_keys_lock
406 assert _master_keys_lock is None, "Should only call init_ssh once"
407 _master_keys_lock = _threading.Lock()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700408
Josh Guilfoyle71985722009-08-16 09:44:40 -0700409def _open_ssh(host, port=None):
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700410 global _ssh_master
411
Doug Anderson0048b692010-12-21 13:39:23 -0800412 # Acquire the lock. This is needed to prevent opening multiple masters for
413 # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
414 # manifest <remote fetch="ssh://xyz"> specifies a different host from the
415 # one that was passed to repo init.
416 _master_keys_lock.acquire()
Doug Anderson06d029c2010-10-27 17:06:01 -0700417 try:
Doug Anderson06d029c2010-10-27 17:06:01 -0700418
Doug Anderson0048b692010-12-21 13:39:23 -0800419 # Check to see whether we already think that the master is running; if we
420 # think it's already running, return right away.
421 if port is not None:
422 key = '%s:%s' % (host, port)
423 else:
424 key = host
425
426 if key in _master_keys:
Doug Anderson06d029c2010-10-27 17:06:01 -0700427 return True
Doug Anderson06d029c2010-10-27 17:06:01 -0700428
Doug Anderson0048b692010-12-21 13:39:23 -0800429 if not _ssh_master \
430 or 'GIT_SSH' in os.environ \
431 or sys.platform in ('win32', 'cygwin'):
432 # failed earlier, or cygwin ssh can't do this
433 #
434 return False
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700435
Doug Anderson0048b692010-12-21 13:39:23 -0800436 # We will make two calls to ssh; this is the common part of both calls.
437 command_base = ['ssh',
438 '-o','ControlPath %s' % ssh_sock(),
439 host]
440 if port is not None:
David Pursehouse8f62fb72012-11-14 12:09:38 +0900441 command_base[1:1] = ['-p', str(port)]
Doug Anderson0048b692010-12-21 13:39:23 -0800442
443 # Since the key wasn't in _master_keys, we think that master isn't running.
444 # ...but before actually starting a master, we'll double-check. This can
445 # be important because we can't tell that that 'git@myhost.com' is the same
446 # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
447 check_command = command_base + ['-O','check']
448 try:
449 Trace(': %s', ' '.join(check_command))
450 check_process = subprocess.Popen(check_command,
451 stdout=subprocess.PIPE,
452 stderr=subprocess.PIPE)
453 check_process.communicate() # read output, but ignore it...
454 isnt_running = check_process.wait()
455
456 if not isnt_running:
457 # Our double-check found that the master _was_ infact running. Add to
458 # the list of keys.
459 _master_keys.add(key)
460 return True
461 except Exception:
462 # Ignore excpetions. We we will fall back to the normal command and print
463 # to the log there.
464 pass
465
466 command = command_base[:1] + \
467 ['-M', '-N'] + \
468 command_base[1:]
469 try:
470 Trace(': %s', ' '.join(command))
471 p = subprocess.Popen(command)
Sarah Owensa5be53f2012-09-09 15:37:57 -0700472 except Exception as e:
Doug Anderson0048b692010-12-21 13:39:23 -0800473 _ssh_master = False
Sarah Owenscecd1d82012-11-01 22:59:27 -0700474 print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
475 % (host,port, str(e)), file=sys.stderr)
Doug Anderson0048b692010-12-21 13:39:23 -0800476 return False
477
Timo Lotterbach05dc46b2016-07-15 16:48:42 +0200478 time.sleep(1)
479 ssh_died = (p.poll() is not None)
480 if ssh_died:
481 return False
482
Doug Anderson0048b692010-12-21 13:39:23 -0800483 _master_processes.append(p)
484 _master_keys.add(key)
Doug Anderson0048b692010-12-21 13:39:23 -0800485 return True
486 finally:
487 _master_keys_lock.release()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700488
489def close_ssh():
Doug Anderson0048b692010-12-21 13:39:23 -0800490 global _master_keys_lock
491
Shawn O. Pearceca8c32c2010-05-11 18:21:33 -0700492 terminate_ssh_clients()
493
Doug Anderson06d029c2010-10-27 17:06:01 -0700494 for p in _master_processes:
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700495 try:
496 os.kill(p.pid, SIGTERM)
497 p.wait()
Shawn O. Pearcefb5c8fd2009-06-16 14:57:46 -0700498 except OSError:
Shawn O. Pearce26120ca2009-06-16 11:49:10 -0700499 pass
Doug Anderson06d029c2010-10-27 17:06:01 -0700500 del _master_processes[:]
501 _master_keys.clear()
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700502
Nico Sallembien1c85f4e2010-04-27 14:35:27 -0700503 d = ssh_sock(create=False)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700504 if d:
505 try:
506 os.rmdir(os.path.dirname(d))
507 except OSError:
508 pass
509
Doug Anderson0048b692010-12-21 13:39:23 -0800510 # We're done with the lock, so we can delete it.
511 _master_keys_lock = None
512
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700513URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
Shawn O. Pearce898e12a2012-03-14 15:22:28 -0700514URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700515
Shawn O. Pearcef322b9a2011-09-19 14:50:58 -0700516def GetSchemeFromUrl(url):
517 m = URI_ALL.match(url)
518 if m:
519 return m.group(1)
520 return None
521
Dan Willemsen0745bb22015-08-17 13:41:45 -0700522@contextlib.contextmanager
523def GetUrlCookieFile(url, quiet):
524 if url.startswith('persistent-'):
525 try:
526 p = subprocess.Popen(
527 ['git-remote-persistent-https', '-print_config', url],
528 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
529 stderr=subprocess.PIPE)
530 try:
531 cookieprefix = 'http.cookiefile='
532 proxyprefix = 'http.proxy='
533 cookiefile = None
534 proxy = None
535 for line in p.stdout:
536 line = line.strip()
537 if line.startswith(cookieprefix):
Daichi Ueurace7e0262018-02-26 08:49:36 +0900538 cookiefile = os.path.expanduser(line[len(cookieprefix):])
Dan Willemsen0745bb22015-08-17 13:41:45 -0700539 if line.startswith(proxyprefix):
540 proxy = line[len(proxyprefix):]
541 # Leave subprocess open, as cookie file may be transient.
542 if cookiefile or proxy:
543 yield cookiefile, proxy
544 return
545 finally:
546 p.stdin.close()
547 if p.wait():
548 err_msg = p.stderr.read()
549 if ' -print_config' in err_msg:
550 pass # Persistent proxy doesn't support -print_config.
551 elif not quiet:
552 print(err_msg, file=sys.stderr)
553 except OSError as e:
554 if e.errno == errno.ENOENT:
555 pass # No persistent proxy.
556 raise
Daichi Ueurace7e0262018-02-26 08:49:36 +0900557 cookiefile = GitConfig.ForUser().GetString('http.cookiefile')
558 if cookiefile:
559 cookiefile = os.path.expanduser(cookiefile)
560 yield cookiefile, None
Dan Willemsen0745bb22015-08-17 13:41:45 -0700561
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700562def _preconnect(url):
563 m = URI_ALL.match(url)
564 if m:
565 scheme = m.group(1)
566 host = m.group(2)
567 if ':' in host:
568 host, port = host.split(':')
Shawn O. Pearce896d5df2009-04-21 14:51:04 -0700569 else:
Josh Guilfoyle71985722009-08-16 09:44:40 -0700570 port = None
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700571 if scheme in ('ssh', 'git+ssh', 'ssh+git'):
572 return _open_ssh(host, port)
573 return False
574
575 m = URI_SCP.match(url)
576 if m:
577 host = m.group(1)
Josh Guilfoyle71985722009-08-16 09:44:40 -0700578 return _open_ssh(host)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700579
Shawn O. Pearce7b4f4352009-06-12 09:06:35 -0700580 return False
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700581
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700582class Remote(object):
583 """Configuration options related to a remote.
584 """
585 def __init__(self, config, name):
586 self._config = config
587 self.name = name
588 self.url = self._Get('url')
Steve Raed6480452016-08-10 15:00:00 -0700589 self.pushUrl = self._Get('pushurl')
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700590 self.review = self._Get('review')
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800591 self.projectname = self._Get('projectname')
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530592 self.fetch = list(map(RefSpec.FromString,
593 self._Get('fetch', all_keys=True)))
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800594 self._review_url = None
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800595
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100596 def _InsteadOf(self):
597 globCfg = GitConfig.ForUser()
598 urlList = globCfg.GetSubSections('url')
599 longest = ""
600 longestUrl = ""
601
602 for url in urlList:
603 key = "url." + url + ".insteadOf"
David Pursehouse8a68ff92012-09-24 12:15:13 +0900604 insteadOfList = globCfg.GetString(key, all_keys=True)
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100605
606 for insteadOf in insteadOfList:
607 if self.url.startswith(insteadOf) \
608 and len(insteadOf) > len(longest):
609 longest = insteadOf
610 longestUrl = url
611
612 if len(longest) == 0:
613 return self.url
614
615 return self.url.replace(longest, longestUrl, 1)
616
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700617 def PreConnectFetch(self):
Ulrik Sjolinb6ea3bf2010-01-03 18:20:17 +0100618 connectionUrl = self._InsteadOf()
619 return _preconnect(connectionUrl)
Shawn O. Pearcefb231612009-04-10 18:53:46 -0700620
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200621 def ReviewUrl(self, userEmail, validate_certs):
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800622 if self._review_url is None:
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800623 if self.review is None:
624 return None
625
626 u = self.review
Conley Owens7e12e0a2014-10-23 15:40:00 -0700627 if u.startswith('persistent-'):
628 u = u[len('persistent-'):]
Christian Koestlin2ec2a5d2016-12-05 20:32:45 +0100629 if u.split(':')[0] not in ('http', 'https', 'sso', 'ssh'):
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800630 u = 'http://%s' % u
Shawn O. Pearce13cc3842009-03-25 13:54:54 -0700631 if u.endswith('/Gerrit'):
632 u = u[:len(u) - len('/Gerrit')]
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800633 if u.endswith('/ssh_info'):
634 u = u[:len(u) - len('/ssh_info')]
635 if not u.endswith('/'):
David Pursehouse8a68ff92012-09-24 12:15:13 +0900636 u += '/'
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800637 http_url = u
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800638
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700639 if u in REVIEW_CACHE:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800640 self._review_url = REVIEW_CACHE[u]
Shawn O. Pearce1a68dc52011-10-11 14:12:46 -0700641 elif 'REPO_HOST_PORT_INFO' in os.environ:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800642 host, port = os.environ['REPO_HOST_PORT_INFO'].split()
643 self._review_url = self._SshReviewUrl(userEmail, host, port)
644 REVIEW_CACHE[u] = self._review_url
Christian Koestlin2ec2a5d2016-12-05 20:32:45 +0100645 elif u.startswith('sso:') or u.startswith('ssh:'):
Steve Pucci143d8a72014-01-30 09:45:53 -0800646 self._review_url = u # Assume it's right
647 REVIEW_CACHE[u] = self._review_url
Timo Lotterbacheec726c2016-10-07 10:52:08 +0200648 elif 'REPO_IGNORE_SSH_INFO' in os.environ:
649 self._review_url = http_url
650 REVIEW_CACHE[u] = self._review_url
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700651 else:
652 try:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800653 info_url = u + 'ssh_info'
Łukasz Gardońbed59ce2017-08-08 10:18:11 +0200654 if not validate_certs:
655 context = ssl._create_unverified_context()
656 info = urllib.request.urlopen(info_url, context=context).read()
657 else:
658 info = urllib.request.urlopen(info_url).read()
Conley Owens745a39b2013-06-05 13:16:18 -0700659 if info == 'NOT_AVAILABLE' or '<' in info:
660 # If `info` contains '<', we assume the server gave us some sort
661 # of HTML response back, like maybe a login page.
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700662 #
Conley Owens745a39b2013-06-05 13:16:18 -0700663 # Assume HTTP if SSH is not enabled or ssh_info doesn't look right.
Conley Owens2cd38a02014-02-04 15:32:29 -0800664 self._review_url = http_url
Shawn O. Pearce146fe902009-03-25 14:06:43 -0700665 else:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800666 host, port = info.split()
Dan Willemsen16889ba2016-09-22 16:39:06 +0000667 self._review_url = self._SshReviewUrl(userEmail, host, port)
Sarah Owens1f7627f2012-10-31 09:21:55 -0700668 except urllib.error.HTTPError as e:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800669 raise UploadError('%s: %s' % (self.review, str(e)))
Sarah Owens1f7627f2012-10-31 09:21:55 -0700670 except urllib.error.URLError as e:
Shawn O. Pearcebf1fbb22011-10-11 09:31:58 -0700671 raise UploadError('%s: %s' % (self.review, str(e)))
David Pursehouseecf8f2b2013-05-24 12:12:23 +0900672 except HTTPException as e:
673 raise UploadError('%s: %s' % (self.review, e.__class__.__name__))
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800674
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800675 REVIEW_CACHE[u] = self._review_url
676 return self._review_url + self.projectname
Shawn O. Pearceb54a3922009-01-05 16:18:58 -0800677
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800678 def _SshReviewUrl(self, userEmail, host, port):
Shawn O. Pearce3575b8f2010-07-15 17:00:14 -0700679 username = self._config.GetString('review.%s.username' % self.review)
680 if username is None:
Shawn O. Pearcec9571422012-01-11 14:58:54 -0800681 username = userEmail.split('@')[0]
682 return 'ssh://%s@%s:%s/' % (username, host, port)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700683
684 def ToLocal(self, rev):
685 """Convert a remote revision string to something we have locally.
686 """
Yann Droneaud936183a2013-09-12 10:51:18 +0200687 if self.name == '.' or IsId(rev):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700688 return rev
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700689
690 if not rev.startswith('refs/'):
691 rev = R_HEADS + rev
692
693 for spec in self.fetch:
694 if spec.SourceMatches(rev):
695 return spec.MapSource(rev)
Nasser Grainawi909d58b2014-09-19 12:13:04 -0600696
697 if not rev.startswith(R_HEADS):
698 return rev
699
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700700 raise GitError('remote %s does not have %s' % (self.name, rev))
701
702 def WritesTo(self, ref):
703 """True if the remote stores to the tracking ref.
704 """
705 for spec in self.fetch:
706 if spec.DestMatches(ref):
707 return True
708 return False
709
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800710 def ResetFetch(self, mirror=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700711 """Set the fetch refspec to its default value.
712 """
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800713 if mirror:
714 dst = 'refs/heads/*'
715 else:
716 dst = 'refs/remotes/%s/*' % self.name
717 self.fetch = [RefSpec(True, 'refs/heads/*', dst)]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700718
719 def Save(self):
720 """Save this remote to the configuration.
721 """
722 self._Set('url', self.url)
Steve Raed6480452016-08-10 15:00:00 -0700723 if self.pushUrl is not None:
724 self._Set('pushurl', self.pushUrl + '/' + self.projectname)
725 else:
726 self._Set('pushurl', self.pushUrl)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700727 self._Set('review', self.review)
Shawn O. Pearce339ba9f2008-11-06 09:52:51 -0800728 self._Set('projectname', self.projectname)
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530729 self._Set('fetch', list(map(str, self.fetch)))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700730
731 def _Set(self, key, value):
732 key = 'remote.%s.%s' % (self.name, key)
733 return self._config.SetString(key, value)
734
David Pursehouse8a68ff92012-09-24 12:15:13 +0900735 def _Get(self, key, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700736 key = 'remote.%s.%s' % (self.name, key)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900737 return self._config.GetString(key, all_keys = all_keys)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700738
739
740class Branch(object):
741 """Configuration options related to a single branch.
742 """
743 def __init__(self, config, name):
744 self._config = config
745 self.name = name
746 self.merge = self._Get('merge')
747
748 r = self._Get('remote')
749 if r:
750 self.remote = self._config.GetRemote(r)
751 else:
752 self.remote = None
753
754 @property
755 def LocalMerge(self):
756 """Convert the merge spec to a local name.
757 """
758 if self.remote and self.merge:
759 return self.remote.ToLocal(self.merge)
760 return None
761
762 def Save(self):
763 """Save this branch back into the configuration.
764 """
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700765 if self._config.HasSection('branch', self.name):
766 if self.remote:
767 self._Set('remote', self.remote.name)
768 else:
769 self._Set('remote', None)
770 self._Set('merge', self.merge)
771
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700772 else:
Chirayu Desai303a82f2014-08-19 22:57:17 +0530773 fd = open(self._config.file, 'a')
Shawn O. Pearceaccc56d2009-04-18 14:45:51 -0700774 try:
775 fd.write('[branch "%s"]\n' % self.name)
776 if self.remote:
777 fd.write('\tremote = %s\n' % self.remote.name)
778 if self.merge:
779 fd.write('\tmerge = %s\n' % self.merge)
780 finally:
781 fd.close()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700782
783 def _Set(self, key, value):
784 key = 'branch.%s.%s' % (self.name, key)
785 return self._config.SetString(key, value)
786
David Pursehouse8a68ff92012-09-24 12:15:13 +0900787 def _Get(self, key, all_keys=False):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700788 key = 'branch.%s.%s' % (self.name, key)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900789 return self._config.GetString(key, all_keys = all_keys)