blob: 7a35e7a02a73b9301130ab91054b3ea25d5244c0 [file] [log] [blame]
Simran Basibdb52712015-08-10 13:23:23 -07001#
2# Copyright (C) 2015 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from __future__ import print_function
17import os
Dan Willemsen5ea32d12015-09-08 13:27:20 -070018import platform
19import re
David Pursehouse46496d82015-08-20 16:37:09 +090020import sys
Simran Basib9a1b732015-08-20 12:19:28 -070021import time
Simran Basibdb52712015-08-10 13:23:23 -070022
23import git_command
24import git_config
Simran Basi8ce50412015-08-28 14:25:44 -070025import wrapper
Simran Basibdb52712015-08-10 13:23:23 -070026
Xin Lif97e72e2016-06-24 12:17:14 -070027from error import ManifestParseError
28
Simran Basibdb52712015-08-10 13:23:23 -070029NUM_BATCH_RETRIEVE_REVISIONID = 300
30
Simran Basi8ce50412015-08-28 14:25:44 -070031def get_gitc_manifest_dir():
32 return wrapper.Wrapper().get_gitc_manifest_dir()
33
Simran Basib9a1b732015-08-20 12:19:28 -070034def parse_clientdir(gitc_fs_path):
Dan Willemsen745b4ad2015-10-06 15:23:19 -070035 return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
Simran Basib9a1b732015-08-20 12:19:28 -070036
Simran Basibdb52712015-08-10 13:23:23 -070037def _set_project_revisions(projects):
38 """Sets the revisionExpr for a list of projects.
39
40 Because of the limit of open file descriptors allowed, length of projects
41 should not be overly large. Recommend calling this function multiple times
42 with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
43
44 @param projects: List of project objects to set the revionExpr for.
45 """
46 # Retrieve the commit id for each project based off of it's current
47 # revisionExpr and it is not already a commit id.
48 project_gitcmds = [(
49 project, git_command.GitCommand(None,
50 ['ls-remote',
51 project.remote.url,
52 project.revisionExpr],
53 capture_stdout=True, cwd='/tmp'))
54 for project in projects if not git_config.IsId(project.revisionExpr)]
55 for proj, gitcmd in project_gitcmds:
56 if gitcmd.Wait():
David Pursehouse022a1d42015-08-20 16:41:04 +090057 print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
Simran Basibdb52712015-08-10 13:23:23 -070058 sys.exit(1)
Xin Lif97e72e2016-06-24 12:17:14 -070059 revisionExpr = gitcmd.stdout.split('\t')[0]
60 if not revisionExpr:
61 raise(ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
62 (proj.remote.url, proj.revisionExpr)))
63 proj.revisionExpr = revisionExpr
Simran Basibdb52712015-08-10 13:23:23 -070064
Dan Willemsen5ea32d12015-09-08 13:27:20 -070065def _manifest_groups(manifest):
66 """Returns the manifest group string that should be synced
67
68 This is the same logic used by Command.GetProjects(), which is used during
69 repo sync
70
71 @param manifest: The XmlManifest object
72 """
73 mp = manifest.manifestProject
74 groups = mp.config.GetString('manifest.groups')
75 if not groups:
76 groups = 'default,platform-' + platform.system().lower()
77 return groups
78
79def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
Simran Basibdb52712015-08-10 13:23:23 -070080 """Generate a manifest for shafsd to use for this GITC client.
81
Dan Willemsen5ea32d12015-09-08 13:27:20 -070082 @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
83 @param manifest: A GitcManifest object loaded with the current repo manifest.
84 @param paths: List of project paths we want to update.
Simran Basibdb52712015-08-10 13:23:23 -070085 """
Dan Willemsen5ea32d12015-09-08 13:27:20 -070086
Simran Basibdb52712015-08-10 13:23:23 -070087 print('Generating GITC Manifest by fetching revision SHAs for each '
88 'project.')
Dan Willemsen5ea32d12015-09-08 13:27:20 -070089 if paths is None:
90 paths = manifest.paths.keys()
91
92 groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
93
94 # Convert the paths to projects, and filter them to the matched groups.
95 projects = [manifest.paths[p] for p in paths]
96 projects = [p for p in projects if p.MatchesGroups(groups)]
97
98 if gitc_manifest is not None:
99 for path, proj in manifest.paths.iteritems():
100 if not proj.MatchesGroups(groups):
101 continue
102
103 if not proj.upstream and not git_config.IsId(proj.revisionExpr):
104 proj.upstream = proj.revisionExpr
105
106 if not path in gitc_manifest.paths:
107 # Any new projects need their first revision, even if we weren't asked
108 # for them.
109 projects.append(proj)
110 elif not path in paths:
111 # And copy revisions from the previous manifest if we're not updating
112 # them now.
113 gitc_proj = gitc_manifest.paths[path]
114 if gitc_proj.old_revision:
115 proj.revisionExpr = None
116 proj.old_revision = gitc_proj.old_revision
117 else:
118 proj.revisionExpr = gitc_proj.revisionExpr
119
Simran Basibdb52712015-08-10 13:23:23 -0700120 index = 0
Simran Basib9a1b732015-08-20 12:19:28 -0700121 while index < len(projects):
Simran Basibdb52712015-08-10 13:23:23 -0700122 _set_project_revisions(
Simran Basib9a1b732015-08-20 12:19:28 -0700123 projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
Simran Basibdb52712015-08-10 13:23:23 -0700124 index += NUM_BATCH_RETRIEVE_REVISIONID
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700125
126 if gitc_manifest is not None:
127 for path, proj in gitc_manifest.paths.iteritems():
128 if proj.old_revision and path in paths:
129 # If we updated a project that has been started, keep the old-revision
130 # updated.
131 repo_proj = manifest.paths[path]
132 repo_proj.old_revision = repo_proj.revisionExpr
133 repo_proj.revisionExpr = None
134
135 # Convert URLs from relative to absolute.
Stefan Beller9d2b14d2016-06-21 11:48:57 -0700136 for _name, remote in manifest.remotes.iteritems():
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700137 remote.fetchUrl = remote.resolvedFetchUrl
138
Simran Basibdb52712015-08-10 13:23:23 -0700139 # Save the manifest.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700140 save_manifest(manifest)
Simran Basib9a1b732015-08-20 12:19:28 -0700141
142def save_manifest(manifest, client_dir=None):
143 """Save the manifest file in the client_dir.
144
145 @param client_dir: Client directory to save the manifest in.
146 @param manifest: Manifest object to save.
147 """
148 if not client_dir:
149 client_dir = manifest.gitc_client_dir
Simran Basibdb52712015-08-10 13:23:23 -0700150 with open(os.path.join(client_dir, '.manifest'), 'w') as f:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700151 manifest.Save(f, groups=_manifest_groups(manifest))
Simran Basib9a1b732015-08-20 12:19:28 -0700152 # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
153 # Give the GITC filesystem time to register the manifest changes.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700154 time.sleep(3)