blob: 457d5ab02d3b603d0087d814de25074373b25df7 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001#
2# Copyright (C) 2008 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Sarah Owenscecd1d82012-11-01 22:59:27 -070016from __future__ import print_function
Colin Cross23acdd32012-04-21 00:33:54 -070017import itertools
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import os
Conley Owensdb728cd2011-09-26 16:34:01 -070019import re
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070020import sys
David Pursehouse59bbb582013-05-17 10:49:33 +090021import xml.dom.minidom
22
23from pyversion import is_python3
24if is_python3():
Chirayu Desai217ea7d2013-03-01 19:14:38 +053025 import urllib.parse
David Pursehouse59bbb582013-05-17 10:49:33 +090026else:
Chirayu Desai217ea7d2013-03-01 19:14:38 +053027 import imp
28 import urlparse
29 urllib = imp.new_module('urllib')
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053030 urllib.parse = urlparse
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070031
David Pursehousee15c65a2012-08-22 10:46:11 +090032from git_config import GitConfig
David Pursehousee00aa6b2012-09-11 14:33:51 +090033from git_refs import R_HEADS, HEAD
34from project import RemoteSpec, Project, MetaProject
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070035from error import ManifestParseError
36
37MANIFEST_FILE_NAME = 'manifest.xml'
Shawn O. Pearce5cc66792008-10-23 16:19:27 -070038LOCAL_MANIFEST_NAME = 'local_manifest.xml'
David Pursehouse2d5a0df2012-11-13 02:50:36 +090039LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
Chirayu Desai217ea7d2013-03-01 19:14:38 +053041urllib.parse.uses_relative.extend(['ssh', 'git'])
42urllib.parse.uses_netloc.extend(['ssh', 'git'])
Conley Owensdb728cd2011-09-26 16:34:01 -070043
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070044class _Default(object):
45 """Project defaults within the manifest."""
46
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -070047 revisionExpr = None
Conley Owensb6a16e62013-09-25 15:06:09 -070048 destBranchExpr = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070049 remote = None
Shawn O. Pearce6392c872011-09-22 17:44:31 -070050 sync_j = 1
Anatol Pomazau79770d22012-04-20 14:41:59 -070051 sync_c = False
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +080052 sync_s = False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070053
Julien Campergue74879922013-10-09 14:38:46 +020054 def __eq__(self, other):
55 return self.__dict__ == other.__dict__
56
57 def __ne__(self, other):
58 return self.__dict__ != other.__dict__
59
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070060class _XmlRemote(object):
61 def __init__(self,
62 name,
Yestin Sunb292b982012-07-02 07:32:50 -070063 alias=None,
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070064 fetch=None,
Conley Owensdb728cd2011-09-26 16:34:01 -070065 manifestUrl=None,
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070066 review=None):
67 self.name = name
68 self.fetchUrl = fetch
Conley Owensdb728cd2011-09-26 16:34:01 -070069 self.manifestUrl = manifestUrl
Yestin Sunb292b982012-07-02 07:32:50 -070070 self.remoteAlias = alias
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070071 self.reviewUrl = review
Conley Owensceea3682011-10-20 10:45:47 -070072 self.resolvedFetchUrl = self._resolveFetchUrl()
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070073
David Pursehouse717ece92012-11-13 08:49:16 +090074 def __eq__(self, other):
75 return self.__dict__ == other.__dict__
76
77 def __ne__(self, other):
78 return self.__dict__ != other.__dict__
79
Conley Owensceea3682011-10-20 10:45:47 -070080 def _resolveFetchUrl(self):
81 url = self.fetchUrl.rstrip('/')
Conley Owensdb728cd2011-09-26 16:34:01 -070082 manifestUrl = self.manifestUrl.rstrip('/')
Shawn Pearcea9f11b32013-01-02 15:40:48 -080083 p = manifestUrl.startswith('persistent-http')
84 if p:
85 manifestUrl = manifestUrl[len('persistent-'):]
86
Conley Owensdb728cd2011-09-26 16:34:01 -070087 # urljoin will get confused if there is no scheme in the base url
88 # ie, if manifestUrl is of the form <hostname:port>
89 if manifestUrl.find(':') != manifestUrl.find('/') - 1:
David Pursehousec1b86a22012-11-14 11:36:51 +090090 manifestUrl = 'gopher://' + manifestUrl
Chirayu Desai217ea7d2013-03-01 19:14:38 +053091 url = urllib.parse.urljoin(manifestUrl, url)
Shawn Pearcea9f11b32013-01-02 15:40:48 -080092 url = re.sub(r'^gopher://', '', url)
93 if p:
94 url = 'persistent-' + url
95 return url
Conley Owensceea3682011-10-20 10:45:47 -070096
97 def ToRemoteSpec(self, projectName):
Conley Owens9d8f9142011-10-20 14:36:35 -070098 url = self.resolvedFetchUrl.rstrip('/') + '/' + projectName
Yestin Sunb292b982012-07-02 07:32:50 -070099 remoteName = self.name
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700100 if self.remoteAlias:
David Pursehouse37128b62013-10-15 10:48:40 +0900101 remoteName = self.remoteAlias
Yestin Sunb292b982012-07-02 07:32:50 -0700102 return RemoteSpec(remoteName, url, self.reviewUrl)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700103
Shawn O. Pearcec8a300f2009-05-18 13:19:57 -0700104class XmlManifest(object):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700105 """manages the repo configuration file"""
106
107 def __init__(self, repodir):
108 self.repodir = os.path.abspath(repodir)
109 self.topdir = os.path.dirname(self.repodir)
110 self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700111 self.globalConfig = GitConfig.ForUser()
David Pursehouse4eb285c2013-02-14 16:28:44 +0900112 self.localManifestWarning = False
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700113
114 self.repoProject = MetaProject(self, 'repo',
115 gitdir = os.path.join(repodir, 'repo/.git'),
116 worktree = os.path.join(repodir, 'repo'))
117
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700118 self.manifestProject = MetaProject(self, 'manifests',
Shawn O. Pearcef5c25a62008-11-04 08:11:53 -0800119 gitdir = os.path.join(repodir, 'manifests.git'),
120 worktree = os.path.join(repodir, 'manifests'))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700121
122 self._Unload()
123
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700124 def Override(self, name):
125 """Use a different manifest, just for the current instantiation.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700126 """
127 path = os.path.join(self.manifestProject.worktree, name)
128 if not os.path.isfile(path):
129 raise ManifestParseError('manifest %s not found' % name)
130
131 old = self.manifestFile
132 try:
133 self.manifestFile = path
134 self._Unload()
135 self._Load()
136 finally:
137 self.manifestFile = old
138
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700139 def Link(self, name):
140 """Update the repo metadata to use a different manifest.
141 """
142 self.Override(name)
143
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700144 try:
Sebastian Frias223bf962012-11-21 19:09:25 +0100145 if os.path.lexists(self.manifestFile):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700146 os.remove(self.manifestFile)
147 os.symlink('manifests/%s' % name, self.manifestFile)
Sebastian Frias223bf962012-11-21 19:09:25 +0100148 except OSError as e:
149 raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700150
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800151 def _RemoteToXml(self, r, doc, root):
152 e = doc.createElement('remote')
153 root.appendChild(e)
154 e.setAttribute('name', r.name)
155 e.setAttribute('fetch', r.fetchUrl)
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700156 if r.remoteAlias is not None:
157 e.setAttribute('alias', r.remoteAlias)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800158 if r.reviewUrl is not None:
159 e.setAttribute('review', r.reviewUrl)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800160
Brian Harring14a66742012-09-28 20:21:57 -0700161 def Save(self, fd, peg_rev=False, peg_rev_upstream=True):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800162 """Write the current manifest out to the given file descriptor.
163 """
Colin Cross5acde752012-03-28 20:15:45 -0700164 mp = self.manifestProject
165
166 groups = mp.config.GetString('manifest.groups')
Matt Gumbel0c635bb2012-12-21 10:14:53 -0800167 if groups:
168 groups = [x for x in re.split(r'[,\s]+', groups) if x]
Colin Cross5acde752012-03-28 20:15:45 -0700169
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800170 doc = xml.dom.minidom.Document()
171 root = doc.createElement('manifest')
172 doc.appendChild(root)
173
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700174 # Save out the notice. There's a little bit of work here to give it the
175 # right whitespace, which assumes that the notice is automatically indented
176 # by 4 by minidom.
177 if self.notice:
178 notice_element = root.appendChild(doc.createElement('notice'))
179 notice_lines = self.notice.splitlines()
180 indented_notice = ('\n'.join(" "*4 + line for line in notice_lines))[4:]
181 notice_element.appendChild(doc.createTextNode(indented_notice))
182
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800183 d = self.default
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800184
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530185 for r in sorted(self.remotes):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800186 self._RemoteToXml(self.remotes[r], doc, root)
187 if self.remotes:
188 root.appendChild(doc.createTextNode(''))
189
190 have_default = False
191 e = doc.createElement('default')
192 if d.remote:
193 have_default = True
194 e.setAttribute('remote', d.remote.name)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700195 if d.revisionExpr:
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800196 have_default = True
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700197 e.setAttribute('revision', d.revisionExpr)
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700198 if d.sync_j > 1:
199 have_default = True
200 e.setAttribute('sync-j', '%d' % d.sync_j)
Anatol Pomazau79770d22012-04-20 14:41:59 -0700201 if d.sync_c:
202 have_default = True
203 e.setAttribute('sync-c', 'true')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800204 if d.sync_s:
205 have_default = True
206 e.setAttribute('sync-s', 'true')
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800207 if have_default:
208 root.appendChild(e)
209 root.appendChild(doc.createTextNode(''))
210
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700211 if self._manifest_server:
212 e = doc.createElement('manifest-server')
213 e.setAttribute('url', self._manifest_server)
214 root.appendChild(e)
215 root.appendChild(doc.createTextNode(''))
216
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800217 def output_projects(parent, parent_node, projects):
218 for p in projects:
219 output_project(parent, parent_node, self.projects[p])
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800220
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800221 def output_project(parent, parent_node, p):
Colin Cross5acde752012-03-28 20:15:45 -0700222 if not p.MatchesGroups(groups):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800223 return
224
225 name = p.name
226 relpath = p.relpath
227 if parent:
228 name = self._UnjoinName(parent.name, name)
229 relpath = self._UnjoinRelpath(parent.relpath, relpath)
Colin Cross5acde752012-03-28 20:15:45 -0700230
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800231 e = doc.createElement('project')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800232 parent_node.appendChild(e)
233 e.setAttribute('name', name)
234 if relpath != name:
235 e.setAttribute('path', relpath)
Conley Owensa17d7af2013-10-16 14:38:09 -0700236 remoteName = None
237 if d.remote:
Conley Owensce201a52013-10-16 14:42:42 -0700238 remoteName = d.remote.remoteAlias or d.remote.name
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700239 if not d.remote or p.remote.name != remoteName:
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800240 e.setAttribute('remote', p.remote.name)
241 if peg_rev:
242 if self.IsMirror:
Brian Harring14a66742012-09-28 20:21:57 -0700243 value = p.bare_git.rev_parse(p.revisionExpr + '^0')
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800244 else:
Brian Harring14a66742012-09-28 20:21:57 -0700245 value = p.work_git.rev_parse(HEAD + '^0')
246 e.setAttribute('revision', value)
247 if peg_rev_upstream and value != p.revisionExpr:
248 # Only save the origin if the origin is not a sha1, and the default
249 # isn't our value, and the if the default doesn't already have that
250 # covered.
251 e.setAttribute('upstream', p.revisionExpr)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700252 elif not d.revisionExpr or p.revisionExpr != d.revisionExpr:
253 e.setAttribute('revision', p.revisionExpr)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800254
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800255 for c in p.copyfiles:
256 ce = doc.createElement('copyfile')
257 ce.setAttribute('src', c.src)
258 ce.setAttribute('dest', c.dest)
259 e.appendChild(ce)
260
Conley Owensbb1b5f52012-08-13 13:11:18 -0700261 default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
Dmitry Fink17f85ea2012-08-06 14:52:29 -0700262 egroups = [g for g in p.groups if g not in default_groups]
Conley Owens971de8e2012-04-16 10:36:08 -0700263 if egroups:
264 e.setAttribute('groups', ','.join(egroups))
Colin Cross5acde752012-03-28 20:15:45 -0700265
James W. Mills24c13082012-04-12 15:04:13 -0500266 for a in p.annotations:
267 if a.keep == "true":
268 ae = doc.createElement('annotation')
269 ae.setAttribute('name', a.name)
270 ae.setAttribute('value', a.value)
271 e.appendChild(ae)
272
Anatol Pomazau79770d22012-04-20 14:41:59 -0700273 if p.sync_c:
274 e.setAttribute('sync-c', 'true')
275
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800276 if p.sync_s:
277 e.setAttribute('sync-s', 'true')
278
279 if p.subprojects:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530280 sort_projects = list(sorted([subp.name for subp in p.subprojects]))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800281 output_projects(p, e, sort_projects)
282
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530283 sort_projects = list(sorted([key for key, value in self.projects.items()
284 if not value.parent]))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800285 sort_projects.sort()
286 output_projects(None, root, sort_projects)
287
Doug Anderson37282b42011-03-04 11:54:18 -0800288 if self._repo_hooks_project:
289 root.appendChild(doc.createTextNode(''))
290 e = doc.createElement('repo-hooks')
291 e.setAttribute('in-project', self._repo_hooks_project.name)
292 e.setAttribute('enabled-list',
293 ' '.join(self._repo_hooks_project.enabled_repo_hooks))
294 root.appendChild(e)
295
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800296 doc.writexml(fd, '', ' ', '\n', 'UTF-8')
297
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700298 @property
299 def projects(self):
300 self._Load()
301 return self._projects
302
303 @property
304 def remotes(self):
305 self._Load()
306 return self._remotes
307
308 @property
309 def default(self):
310 self._Load()
311 return self._default
312
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800313 @property
Doug Anderson37282b42011-03-04 11:54:18 -0800314 def repo_hooks_project(self):
315 self._Load()
316 return self._repo_hooks_project
317
318 @property
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700319 def notice(self):
320 self._Load()
321 return self._notice
322
323 @property
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700324 def manifest_server(self):
325 self._Load()
Shawn O. Pearce34fb20f2011-11-30 13:41:02 -0800326 return self._manifest_server
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700327
328 @property
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800329 def IsMirror(self):
330 return self.manifestProject.config.GetBoolean('repo.mirror')
331
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700332 def _Unload(self):
333 self._loaded = False
334 self._projects = {}
335 self._remotes = {}
336 self._default = None
Doug Anderson37282b42011-03-04 11:54:18 -0800337 self._repo_hooks_project = None
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700338 self._notice = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700339 self.branch = None
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700340 self._manifest_server = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700341
342 def _Load(self):
343 if not self._loaded:
Shawn O. Pearce2450a292008-11-04 08:22:07 -0800344 m = self.manifestProject
345 b = m.GetBranch(m.CurrentBranch).merge
Shawn O. Pearce21c5c342009-06-25 16:47:30 -0700346 if b is not None and b.startswith(R_HEADS):
Shawn O. Pearce2450a292008-11-04 08:22:07 -0800347 b = b[len(R_HEADS):]
348 self.branch = b
349
Colin Cross23acdd32012-04-21 00:33:54 -0700350 nodes = []
Brian Harring475a47d2012-06-07 20:05:35 -0700351 nodes.append(self._ParseManifestXml(self.manifestFile,
352 self.manifestProject.worktree))
Shawn O. Pearce5cc66792008-10-23 16:19:27 -0700353
354 local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
355 if os.path.exists(local):
David Pursehouse4eb285c2013-02-14 16:28:44 +0900356 if not self.localManifestWarning:
357 self.localManifestWarning = True
358 print('warning: %s is deprecated; put local manifests in `%s` instead'
359 % (LOCAL_MANIFEST_NAME, os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
360 file=sys.stderr)
Brian Harring475a47d2012-06-07 20:05:35 -0700361 nodes.append(self._ParseManifestXml(local, self.repodir))
Colin Cross23acdd32012-04-21 00:33:54 -0700362
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900363 local_dir = os.path.abspath(os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME))
364 try:
David Pursehouse52f1e5d2012-11-14 04:53:24 +0900365 for local_file in sorted(os.listdir(local_dir)):
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900366 if local_file.endswith('.xml'):
David Pursehouse5f434ed2012-11-22 13:48:10 +0900367 local = os.path.join(local_dir, local_file)
368 nodes.append(self._ParseManifestXml(local, self.repodir))
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900369 except OSError:
370 pass
371
Joe Onorato26e24752013-01-11 12:35:53 -0800372 try:
373 self._ParseManifest(nodes)
374 except ManifestParseError as e:
375 # There was a problem parsing, unload ourselves in case they catch
376 # this error and try again later, we will show the correct error
377 self._Unload()
378 raise e
Shawn O. Pearce5cc66792008-10-23 16:19:27 -0700379
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800380 if self.IsMirror:
381 self._AddMetaProjectMirror(self.repoProject)
382 self._AddMetaProjectMirror(self.manifestProject)
383
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700384 self._loaded = True
385
Brian Harring475a47d2012-06-07 20:05:35 -0700386 def _ParseManifestXml(self, path, include_root):
David Pursehousef7fc8a92012-11-13 04:00:28 +0900387 try:
388 root = xml.dom.minidom.parse(path)
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900389 except (OSError, xml.parsers.expat.ExpatError) as e:
David Pursehousef7fc8a92012-11-13 04:00:28 +0900390 raise ManifestParseError("error parsing manifest %s: %s" % (path, e))
391
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700392 if not root or not root.childNodes:
Brian Harring26448742011-04-28 05:04:41 -0700393 raise ManifestParseError("no root node in %s" % (path,))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700394
Jooncheol Park34acdd22012-08-27 02:25:59 +0900395 for manifest in root.childNodes:
396 if manifest.nodeName == 'manifest':
397 break
398 else:
Brian Harring26448742011-04-28 05:04:41 -0700399 raise ManifestParseError("no <manifest> in %s" % (path,))
400
Colin Cross23acdd32012-04-21 00:33:54 -0700401 nodes = []
David Pursehouse4f7bdea2012-10-22 12:50:15 +0900402 for node in manifest.childNodes: # pylint:disable=W0631
David Pursehouse5c6eeac2012-10-11 16:44:48 +0900403 # We only get here if manifest is initialised
David Pursehousec1b86a22012-11-14 11:36:51 +0900404 if node.nodeName == 'include':
405 name = self._reqatt(node, 'name')
406 fp = os.path.join(include_root, name)
407 if not os.path.isfile(fp):
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530408 raise ManifestParseError("include %s doesn't exist or isn't a file"
409 % (name,))
David Pursehousec1b86a22012-11-14 11:36:51 +0900410 try:
411 nodes.extend(self._ParseManifestXml(fp, include_root))
412 # should isolate this to the exact exception, but that's
413 # tricky. actual parsing implementation may vary.
414 except (KeyboardInterrupt, RuntimeError, SystemExit):
415 raise
416 except Exception as e:
417 raise ManifestParseError(
418 "failed parsing included manifest %s: %s", (name, e))
419 else:
420 nodes.append(node)
Colin Cross23acdd32012-04-21 00:33:54 -0700421 return nodes
Brian Harring26448742011-04-28 05:04:41 -0700422
Colin Cross23acdd32012-04-21 00:33:54 -0700423 def _ParseManifest(self, node_list):
424 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700425 if node.nodeName == 'remote':
426 remote = self._ParseRemote(node)
David Pursehouse717ece92012-11-13 08:49:16 +0900427 if remote:
428 if remote.name in self._remotes:
429 if remote != self._remotes[remote.name]:
430 raise ManifestParseError(
431 'remote %s already exists with different attributes' %
432 (remote.name))
433 else:
434 self._remotes[remote.name] = remote
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700435
Colin Cross23acdd32012-04-21 00:33:54 -0700436 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700437 if node.nodeName == 'default':
Julien Campergue74879922013-10-09 14:38:46 +0200438 new_default = self._ParseDefault(node)
439 if self._default is None:
440 self._default = new_default
441 elif new_default != self._default:
David Pursehouse37128b62013-10-15 10:48:40 +0900442 raise ManifestParseError('duplicate default in %s' %
443 (self.manifestFile))
Julien Campergue74879922013-10-09 14:38:46 +0200444
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700445 if self._default is None:
446 self._default = _Default()
447
Colin Cross23acdd32012-04-21 00:33:54 -0700448 for node in itertools.chain(*node_list):
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700449 if node.nodeName == 'notice':
450 if self._notice is not None:
Doug Anderson37282b42011-03-04 11:54:18 -0800451 raise ManifestParseError(
452 'duplicate notice in %s' %
453 (self.manifestFile))
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700454 self._notice = self._ParseNotice(node)
455
Colin Cross23acdd32012-04-21 00:33:54 -0700456 for node in itertools.chain(*node_list):
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700457 if node.nodeName == 'manifest-server':
458 url = self._reqatt(node, 'url')
459 if self._manifest_server is not None:
David Pursehousec1b86a22012-11-14 11:36:51 +0900460 raise ManifestParseError(
461 'duplicate manifest-server in %s' %
462 (self.manifestFile))
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700463 self._manifest_server = url
464
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800465 def recursively_add_projects(project):
466 if self._projects.get(project.name):
467 raise ManifestParseError(
468 'duplicate project %s in %s' %
469 (project.name, self.manifestFile))
470 self._projects[project.name] = project
471 for subproject in project.subprojects:
472 recursively_add_projects(subproject)
473
Colin Cross23acdd32012-04-21 00:33:54 -0700474 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700475 if node.nodeName == 'project':
476 project = self._ParseProject(node)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800477 recursively_add_projects(project)
Doug Anderson37282b42011-03-04 11:54:18 -0800478 if node.nodeName == 'repo-hooks':
479 # Get the name of the project and the (space-separated) list of enabled.
480 repo_hooks_project = self._reqatt(node, 'in-project')
481 enabled_repo_hooks = self._reqatt(node, 'enabled-list').split()
482
483 # Only one project can be the hooks project
484 if self._repo_hooks_project is not None:
485 raise ManifestParseError(
486 'duplicate repo-hooks in %s' %
487 (self.manifestFile))
488
489 # Store a reference to the Project.
490 try:
491 self._repo_hooks_project = self._projects[repo_hooks_project]
492 except KeyError:
493 raise ManifestParseError(
494 'project %s not found for repo-hooks' %
495 (repo_hooks_project))
496
497 # Store the enabled hooks in the Project object.
498 self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
Colin Cross23acdd32012-04-21 00:33:54 -0700499 if node.nodeName == 'remove-project':
500 name = self._reqatt(node, 'name')
501 try:
502 del self._projects[name]
503 except KeyError:
David Pursehousef9107482012-11-16 19:12:32 +0900504 raise ManifestParseError('remove-project element specifies non-existent '
505 'project: %s' % name)
Colin Cross23acdd32012-04-21 00:33:54 -0700506
507 # If the manifest removes the hooks project, treat it as if it deleted
508 # the repo-hooks element too.
509 if self._repo_hooks_project and (self._repo_hooks_project.name == name):
510 self._repo_hooks_project = None
511
Doug Anderson37282b42011-03-04 11:54:18 -0800512
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800513 def _AddMetaProjectMirror(self, m):
514 name = None
515 m_url = m.GetRemote(m.remote.name).url
516 if m_url.endswith('/.git'):
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530517 raise ManifestParseError('refusing to mirror %s' % m_url)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800518
519 if self._default and self._default.remote:
Conley Owensceea3682011-10-20 10:45:47 -0700520 url = self._default.remote.resolvedFetchUrl
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800521 if not url.endswith('/'):
522 url += '/'
523 if m_url.startswith(url):
524 remote = self._default.remote
525 name = m_url[len(url):]
526
527 if name is None:
528 s = m_url.rindex('/') + 1
Conley Owensdb728cd2011-09-26 16:34:01 -0700529 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
Shawn O. Pearcef35b2d92012-08-02 11:46:22 -0700530 remote = _XmlRemote('origin', fetch=m_url[:s], manifestUrl=manifestUrl)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800531 name = m_url[s:]
532
533 if name.endswith('.git'):
534 name = name[:-4]
535
536 if name not in self._projects:
537 m.PreSync()
538 gitdir = os.path.join(self.topdir, '%s.git' % name)
539 project = Project(manifest = self,
540 name = name,
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700541 remote = remote.ToRemoteSpec(name),
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800542 gitdir = gitdir,
543 worktree = None,
544 relpath = None,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700545 revisionExpr = m.revisionExpr,
546 revisionId = None)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800547 self._projects[project.name] = project
548
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700549 def _ParseRemote(self, node):
550 """
551 reads a <remote> element from the manifest file
552 """
553 name = self._reqatt(node, 'name')
Yestin Sunb292b982012-07-02 07:32:50 -0700554 alias = node.getAttribute('alias')
555 if alias == '':
556 alias = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700557 fetch = self._reqatt(node, 'fetch')
558 review = node.getAttribute('review')
Shawn O. Pearceae6e0942008-11-06 10:25:35 -0800559 if review == '':
560 review = None
Conley Owensdb728cd2011-09-26 16:34:01 -0700561 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
Yestin Sunb292b982012-07-02 07:32:50 -0700562 return _XmlRemote(name, alias, fetch, manifestUrl, review)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700563
564 def _ParseDefault(self, node):
565 """
566 reads a <default> element from the manifest file
567 """
568 d = _Default()
569 d.remote = self._get_remote(node)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700570 d.revisionExpr = node.getAttribute('revision')
571 if d.revisionExpr == '':
572 d.revisionExpr = None
Anatol Pomazau79770d22012-04-20 14:41:59 -0700573
Bryan Jacobsf609f912013-05-06 13:36:24 -0400574 d.destBranchExpr = node.getAttribute('dest-branch') or None
575
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700576 sync_j = node.getAttribute('sync-j')
577 if sync_j == '' or sync_j is None:
578 d.sync_j = 1
579 else:
580 d.sync_j = int(sync_j)
Anatol Pomazau79770d22012-04-20 14:41:59 -0700581
582 sync_c = node.getAttribute('sync-c')
583 if not sync_c:
584 d.sync_c = False
585 else:
586 d.sync_c = sync_c.lower() in ("yes", "true", "1")
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800587
588 sync_s = node.getAttribute('sync-s')
589 if not sync_s:
590 d.sync_s = False
591 else:
592 d.sync_s = sync_s.lower() in ("yes", "true", "1")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700593 return d
594
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700595 def _ParseNotice(self, node):
596 """
597 reads a <notice> element from the manifest file
598
599 The <notice> element is distinct from other tags in the XML in that the
600 data is conveyed between the start and end tag (it's not an empty-element
601 tag).
602
603 The white space (carriage returns, indentation) for the notice element is
604 relevant and is parsed in a way that is based on how python docstrings work.
605 In fact, the code is remarkably similar to here:
606 http://www.python.org/dev/peps/pep-0257/
607 """
608 # Get the data out of the node...
609 notice = node.childNodes[0].data
610
611 # Figure out minimum indentation, skipping the first line (the same line
612 # as the <notice> tag)...
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530613 minIndent = sys.maxsize
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700614 lines = notice.splitlines()
615 for line in lines[1:]:
616 lstrippedLine = line.lstrip()
617 if lstrippedLine:
618 indent = len(line) - len(lstrippedLine)
619 minIndent = min(indent, minIndent)
620
621 # Strip leading / trailing blank lines and also indentation.
622 cleanLines = [lines[0].strip()]
623 for line in lines[1:]:
624 cleanLines.append(line[minIndent:].rstrip())
625
626 # Clear completely blank lines from front and back...
627 while cleanLines and not cleanLines[0]:
628 del cleanLines[0]
629 while cleanLines and not cleanLines[-1]:
630 del cleanLines[-1]
631
632 return '\n'.join(cleanLines)
633
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800634 def _JoinName(self, parent_name, name):
635 return os.path.join(parent_name, name)
636
637 def _UnjoinName(self, parent_name, name):
638 return os.path.relpath(name, parent_name)
639
640 def _ParseProject(self, node, parent = None):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700641 """
642 reads a <project> element from the manifest file
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700643 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700644 name = self._reqatt(node, 'name')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800645 if parent:
646 name = self._JoinName(parent.name, name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700647
648 remote = self._get_remote(node)
649 if remote is None:
650 remote = self._default.remote
651 if remote is None:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530652 raise ManifestParseError("no remote for project %s within %s" %
653 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700654
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700655 revisionExpr = node.getAttribute('revision')
656 if not revisionExpr:
657 revisionExpr = self._default.revisionExpr
658 if not revisionExpr:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530659 raise ManifestParseError("no revision for project %s within %s" %
660 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700661
662 path = node.getAttribute('path')
663 if not path:
664 path = name
665 if path.startswith('/'):
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530666 raise ManifestParseError("project %s path cannot be absolute in %s" %
667 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700668
Mike Pontillod3153822012-02-28 11:53:24 -0800669 rebase = node.getAttribute('rebase')
670 if not rebase:
671 rebase = True
672 else:
673 rebase = rebase.lower() in ("yes", "true", "1")
674
Anatol Pomazau79770d22012-04-20 14:41:59 -0700675 sync_c = node.getAttribute('sync-c')
676 if not sync_c:
677 sync_c = False
678 else:
679 sync_c = sync_c.lower() in ("yes", "true", "1")
680
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800681 sync_s = node.getAttribute('sync-s')
682 if not sync_s:
683 sync_s = self._default.sync_s
684 else:
685 sync_s = sync_s.lower() in ("yes", "true", "1")
686
David Pursehouseede7f122012-11-27 22:25:30 +0900687 clone_depth = node.getAttribute('clone-depth')
688 if clone_depth:
689 try:
690 clone_depth = int(clone_depth)
691 if clone_depth <= 0:
692 raise ValueError()
693 except ValueError:
694 raise ManifestParseError('invalid clone-depth %s in %s' %
695 (clone_depth, self.manifestFile))
696
Bryan Jacobsf609f912013-05-06 13:36:24 -0400697 dest_branch = node.getAttribute('dest-branch') or self._default.destBranchExpr
698
Brian Harring14a66742012-09-28 20:21:57 -0700699 upstream = node.getAttribute('upstream')
700
Conley Owens971de8e2012-04-16 10:36:08 -0700701 groups = ''
702 if node.hasAttribute('groups'):
703 groups = node.getAttribute('groups')
David Pursehouse1d947b32012-10-25 12:23:11 +0900704 groups = [x for x in re.split(r'[,\s]+', groups) if x]
Brian Harring7da13142012-06-15 02:24:20 -0700705
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800706 if parent is None:
707 relpath, worktree, gitdir = self.GetProjectPaths(name, path)
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -0700708 else:
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800709 relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path)
710
711 default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
712 groups.extend(set(default_groups).difference(groups))
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -0700713
Scott Fandb83b1b2013-02-28 09:34:14 +0800714 if self.IsMirror and node.hasAttribute('force-path'):
715 if node.getAttribute('force-path').lower() in ("yes", "true", "1"):
716 gitdir = os.path.join(self.topdir, '%s.git' % path)
717
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700718 project = Project(manifest = self,
719 name = name,
Shawn O. Pearced1f70d92009-05-19 14:58:02 -0700720 remote = remote.ToRemoteSpec(name),
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700721 gitdir = gitdir,
722 worktree = worktree,
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800723 relpath = relpath,
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700724 revisionExpr = revisionExpr,
Mike Pontillod3153822012-02-28 11:53:24 -0800725 revisionId = None,
Colin Cross5acde752012-03-28 20:15:45 -0700726 rebase = rebase,
Anatol Pomazau79770d22012-04-20 14:41:59 -0700727 groups = groups,
Brian Harring14a66742012-09-28 20:21:57 -0700728 sync_c = sync_c,
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800729 sync_s = sync_s,
David Pursehouseede7f122012-11-27 22:25:30 +0900730 clone_depth = clone_depth,
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800731 upstream = upstream,
Bryan Jacobsf609f912013-05-06 13:36:24 -0400732 parent = parent,
733 dest_branch = dest_branch)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700734
735 for n in node.childNodes:
Shawn O. Pearce242b5262009-05-19 13:00:29 -0700736 if n.nodeName == 'copyfile':
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700737 self._ParseCopyFile(project, n)
James W. Mills24c13082012-04-12 15:04:13 -0500738 if n.nodeName == 'annotation':
739 self._ParseAnnotation(project, n)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800740 if n.nodeName == 'project':
741 project.subprojects.append(self._ParseProject(n, parent = project))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700742
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700743 return project
744
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800745 def GetProjectPaths(self, name, path):
746 relpath = path
747 if self.IsMirror:
748 worktree = None
749 gitdir = os.path.join(self.topdir, '%s.git' % name)
750 else:
751 worktree = os.path.join(self.topdir, path).replace('\\', '/')
752 gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
753 return relpath, worktree, gitdir
754
755 def GetSubprojectName(self, parent, submodule_path):
756 return os.path.join(parent.name, submodule_path)
757
758 def _JoinRelpath(self, parent_relpath, relpath):
759 return os.path.join(parent_relpath, relpath)
760
761 def _UnjoinRelpath(self, parent_relpath, relpath):
762 return os.path.relpath(relpath, parent_relpath)
763
764 def GetSubprojectPaths(self, parent, path):
765 relpath = self._JoinRelpath(parent.relpath, path)
766 gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
767 if self.IsMirror:
768 worktree = None
769 else:
770 worktree = os.path.join(parent.worktree, path).replace('\\', '/')
771 return relpath, worktree, gitdir
772
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700773 def _ParseCopyFile(self, project, node):
774 src = self._reqatt(node, 'src')
775 dest = self._reqatt(node, 'dest')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800776 if not self.IsMirror:
777 # src is project relative;
778 # dest is relative to the top of the tree
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800779 project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700780
James W. Mills24c13082012-04-12 15:04:13 -0500781 def _ParseAnnotation(self, project, node):
782 name = self._reqatt(node, 'name')
783 value = self._reqatt(node, 'value')
784 try:
785 keep = self._reqatt(node, 'keep').lower()
786 except ManifestParseError:
787 keep = "true"
788 if keep != "true" and keep != "false":
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530789 raise ManifestParseError('optional "keep" attribute must be '
790 '"true" or "false"')
James W. Mills24c13082012-04-12 15:04:13 -0500791 project.AddAnnotation(name, value, keep)
792
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700793 def _get_remote(self, node):
794 name = node.getAttribute('remote')
795 if not name:
796 return None
797
798 v = self._remotes.get(name)
799 if not v:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530800 raise ManifestParseError("remote %s not defined in %s" %
801 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700802 return v
803
804 def _reqatt(self, node, attname):
805 """
806 reads a required attribute from the node.
807 """
808 v = node.getAttribute(attname)
809 if not v:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530810 raise ManifestParseError("no %s in <%s> within %s" %
811 (attname, node.nodeName, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700812 return v