blob: 0f3e1818827eeea9051af7815eeb42bbb7ee1aa2 [file] [log] [blame]
Simran Basibdb52712015-08-10 13:23:23 -07001#
2# Copyright (C) 2015 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from __future__ import print_function
17import os
Dan Willemsen5ea32d12015-09-08 13:27:20 -070018import platform
19import re
David Pursehouse46496d82015-08-20 16:37:09 +090020import sys
Simran Basib9a1b732015-08-20 12:19:28 -070021import time
Simran Basibdb52712015-08-10 13:23:23 -070022
23import git_command
24import git_config
Simran Basi8ce50412015-08-28 14:25:44 -070025import wrapper
Simran Basibdb52712015-08-10 13:23:23 -070026
Simran Basibdb52712015-08-10 13:23:23 -070027NUM_BATCH_RETRIEVE_REVISIONID = 300
28
Simran Basi8ce50412015-08-28 14:25:44 -070029def get_gitc_manifest_dir():
30 return wrapper.Wrapper().get_gitc_manifest_dir()
31
Simran Basib9a1b732015-08-20 12:19:28 -070032def parse_clientdir(gitc_fs_path):
Dan Willemsen745b4ad2015-10-06 15:23:19 -070033 return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
Simran Basib9a1b732015-08-20 12:19:28 -070034
Simran Basibdb52712015-08-10 13:23:23 -070035def _set_project_revisions(projects):
36 """Sets the revisionExpr for a list of projects.
37
38 Because of the limit of open file descriptors allowed, length of projects
39 should not be overly large. Recommend calling this function multiple times
40 with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
41
42 @param projects: List of project objects to set the revionExpr for.
43 """
44 # Retrieve the commit id for each project based off of it's current
45 # revisionExpr and it is not already a commit id.
46 project_gitcmds = [(
47 project, git_command.GitCommand(None,
48 ['ls-remote',
49 project.remote.url,
50 project.revisionExpr],
51 capture_stdout=True, cwd='/tmp'))
52 for project in projects if not git_config.IsId(project.revisionExpr)]
53 for proj, gitcmd in project_gitcmds:
54 if gitcmd.Wait():
David Pursehouse022a1d42015-08-20 16:41:04 +090055 print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
Simran Basibdb52712015-08-10 13:23:23 -070056 sys.exit(1)
57 proj.revisionExpr = gitcmd.stdout.split('\t')[0]
58
Dan Willemsen5ea32d12015-09-08 13:27:20 -070059def _manifest_groups(manifest):
60 """Returns the manifest group string that should be synced
61
62 This is the same logic used by Command.GetProjects(), which is used during
63 repo sync
64
65 @param manifest: The XmlManifest object
66 """
67 mp = manifest.manifestProject
68 groups = mp.config.GetString('manifest.groups')
69 if not groups:
70 groups = 'default,platform-' + platform.system().lower()
71 return groups
72
73def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
Simran Basibdb52712015-08-10 13:23:23 -070074 """Generate a manifest for shafsd to use for this GITC client.
75
Dan Willemsen5ea32d12015-09-08 13:27:20 -070076 @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
77 @param manifest: A GitcManifest object loaded with the current repo manifest.
78 @param paths: List of project paths we want to update.
Simran Basibdb52712015-08-10 13:23:23 -070079 """
Dan Willemsen5ea32d12015-09-08 13:27:20 -070080
Simran Basibdb52712015-08-10 13:23:23 -070081 print('Generating GITC Manifest by fetching revision SHAs for each '
82 'project.')
Dan Willemsen5ea32d12015-09-08 13:27:20 -070083 if paths is None:
84 paths = manifest.paths.keys()
85
86 groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
87
88 # Convert the paths to projects, and filter them to the matched groups.
89 projects = [manifest.paths[p] for p in paths]
90 projects = [p for p in projects if p.MatchesGroups(groups)]
91
92 if gitc_manifest is not None:
93 for path, proj in manifest.paths.iteritems():
94 if not proj.MatchesGroups(groups):
95 continue
96
97 if not proj.upstream and not git_config.IsId(proj.revisionExpr):
98 proj.upstream = proj.revisionExpr
99
100 if not path in gitc_manifest.paths:
101 # Any new projects need their first revision, even if we weren't asked
102 # for them.
103 projects.append(proj)
104 elif not path in paths:
105 # And copy revisions from the previous manifest if we're not updating
106 # them now.
107 gitc_proj = gitc_manifest.paths[path]
108 if gitc_proj.old_revision:
109 proj.revisionExpr = None
110 proj.old_revision = gitc_proj.old_revision
111 else:
112 proj.revisionExpr = gitc_proj.revisionExpr
113
Simran Basibdb52712015-08-10 13:23:23 -0700114 index = 0
Simran Basib9a1b732015-08-20 12:19:28 -0700115 while index < len(projects):
Simran Basibdb52712015-08-10 13:23:23 -0700116 _set_project_revisions(
Simran Basib9a1b732015-08-20 12:19:28 -0700117 projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
Simran Basibdb52712015-08-10 13:23:23 -0700118 index += NUM_BATCH_RETRIEVE_REVISIONID
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700119
120 if gitc_manifest is not None:
121 for path, proj in gitc_manifest.paths.iteritems():
122 if proj.old_revision and path in paths:
123 # If we updated a project that has been started, keep the old-revision
124 # updated.
125 repo_proj = manifest.paths[path]
126 repo_proj.old_revision = repo_proj.revisionExpr
127 repo_proj.revisionExpr = None
128
129 # Convert URLs from relative to absolute.
130 for name, remote in manifest.remotes.iteritems():
131 remote.fetchUrl = remote.resolvedFetchUrl
132
Simran Basibdb52712015-08-10 13:23:23 -0700133 # Save the manifest.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700134 save_manifest(manifest)
Simran Basib9a1b732015-08-20 12:19:28 -0700135
136def save_manifest(manifest, client_dir=None):
137 """Save the manifest file in the client_dir.
138
139 @param client_dir: Client directory to save the manifest in.
140 @param manifest: Manifest object to save.
141 """
142 if not client_dir:
143 client_dir = manifest.gitc_client_dir
Simran Basibdb52712015-08-10 13:23:23 -0700144 with open(os.path.join(client_dir, '.manifest'), 'w') as f:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700145 manifest.Save(f, groups=_manifest_groups(manifest))
Simran Basib9a1b732015-08-20 12:19:28 -0700146 # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
147 # Give the GITC filesystem time to register the manifest changes.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700148 time.sleep(3)