Merge "Include project path in --force-sync error message"
diff --git a/command.py b/command.py
index 207ef46..38cacd3 100644
--- a/command.py
+++ b/command.py
@@ -126,7 +126,7 @@
pass
return project
- def GetProjects(self, args, missing_ok=False, submodules_ok=False):
+ def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False):
"""A list of projects that match the arguments.
"""
all_projects_list = self.manifest.projects
@@ -134,7 +134,8 @@
mp = self.manifest.manifestProject
- groups = mp.config.GetString('manifest.groups')
+ if not groups:
+ groups = mp.config.GetString('manifest.groups')
if not groups:
groups = 'default,platform-' + platform.system().lower()
groups = [x for x in re.split(r'[,\s]+', groups) if x]
diff --git a/gitc_utils.py b/gitc_utils.py
new file mode 100644
index 0000000..bf79bd2
--- /dev/null
+++ b/gitc_utils.py
@@ -0,0 +1,69 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import shutil
+
+import git_command
+import git_config
+
+
+# TODO (sbasi) - Remove this constant and fetch manifest dir from /gitc/.config
+GITC_MANIFEST_DIR = '/usr/local/google/gitc/'
+GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
+NUM_BATCH_RETRIEVE_REVISIONID = 300
+
+def _set_project_revisions(projects):
+ """Sets the revisionExpr for a list of projects.
+
+ Because of the limit of open file descriptors allowed, length of projects
+ should not be overly large. Recommend calling this function multiple times
+ with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+ @param projects: List of project objects to set the revionExpr for.
+ """
+ # Retrieve the commit id for each project based off of it's current
+ # revisionExpr and it is not already a commit id.
+ project_gitcmds = [(
+ project, git_command.GitCommand(None,
+ ['ls-remote',
+ project.remote.url,
+ project.revisionExpr],
+ capture_stdout=True, cwd='/tmp'))
+ for project in projects if not git_config.IsId(project.revisionExpr)]
+ for proj, gitcmd in project_gitcmds:
+ if gitcmd.Wait():
+ print('FATAL: Failed to retrieve revisionExpr for %s' % project)
+ sys.exit(1)
+ proj.revisionExpr = gitcmd.stdout.split('\t')[0]
+
+def generate_gitc_manifest(client_dir, manifest):
+ """Generate a manifest for shafsd to use for this GITC client.
+
+ @param client_dir: GITC client directory to install the .manifest file in.
+ @param manifest: XmlManifest object representing the repo manifest.
+ """
+ print('Generating GITC Manifest by fetching revision SHAs for each '
+ 'project.')
+ project_gitcmd_dict = {}
+ index = 0
+ while index < len(manifest.projects):
+ _set_project_revisions(
+ manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+ index += NUM_BATCH_RETRIEVE_REVISIONID
+ # Save the manifest.
+ with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+ manifest.Save(f)
diff --git a/manifest_xml.py b/manifest_xml.py
index 7e71960..6dc01a4 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -303,6 +303,9 @@
if p.sync_s:
e.setAttribute('sync-s', 'true')
+ if p.clone_depth:
+ e.setAttribute('clone-depth', str(p.clone_depth))
+
if p.subprojects:
subprojects = set(subp.name for subp in p.subprojects)
output_projects(p, e, list(sorted(subprojects)))
diff --git a/project.py b/project.py
index fb9df2a..82fa848 100644
--- a/project.py
+++ b/project.py
@@ -1877,6 +1877,13 @@
if depth:
cmd.append('--depth=%s' % depth)
+ else:
+ # If this repo has shallow objects, then we don't know which refs have
+ # shallow objects or not. Tell git to unshallow all fetched refs. Don't
+ # do this with projects that don't have shallow objects, since it is less
+ # efficient.
+ if os.path.exists(os.path.join(self.gitdir, 'shallow')):
+ cmd.append('--depth=2147483647')
if quiet:
cmd.append('--quiet')
@@ -1914,16 +1921,6 @@
spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
cmd.extend(spec)
- shallowfetch = self.config.GetString('repo.shallowfetch')
- if shallowfetch and shallowfetch != ' '.join(spec):
- GitCommand(self, ['fetch', '--depth=2147483647', name]
- + shallowfetch.split(),
- bare=True, ssh_proxy=ssh_proxy).Wait()
- if depth:
- self.config.SetString('repo.shallowfetch', ' '.join(spec))
- else:
- self.config.SetString('repo.shallowfetch', None)
-
ok = False
for _i in range(2):
gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy)
diff --git a/repo b/repo
index f12354a..bf8fa3d 100755
--- a/repo
+++ b/repo
@@ -108,6 +108,7 @@
S_manifests = 'manifests' # special manifest repository
REPO_MAIN = S_repo + '/main.py' # main script
MIN_PYTHON_VERSION = (2, 6) # minimum supported python version
+GITC_MANIFEST_DIR = '/usr/local/google/gitc'
import errno
@@ -212,14 +213,25 @@
dest='config_name', action="store_true", default=False,
help='Always prompt for name/e-mail')
+def _GitcInitOptions(init_optparse):
+ g = init_optparse.add_option_group('GITC options')
+ g.add_option('-f', '--manifest-file',
+ dest='manifest_file',
+ help='Optional manifest file to use for this GITC client.')
+ g.add_option('-c', '--gitc-client',
+ dest='gitc_client',
+ help='The name for the new gitc_client instance.')
+
class CloneFailure(Exception):
"""Indicate the remote clone of repo itself failed.
"""
-def _Init(args):
+def _Init(args, gitc_init=False):
"""Installs repo by cloning it over the network.
"""
+ if gitc_init:
+ _GitcInitOptions(init_optparse)
opt, args = init_optparse.parse_args(args)
if args:
init_optparse.print_usage()
@@ -242,6 +254,15 @@
raise CloneFailure()
try:
+ if gitc_init:
+ client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client)
+ if not os.path.exists(client_dir):
+ os.makedirs(client_dir)
+ os.chdir(client_dir)
+ if os.path.exists(repodir):
+ # This GITC Client has already initialized repo so continue.
+ return
+
os.mkdir(repodir)
except OSError as e:
if e.errno != errno.EEXIST:
@@ -732,11 +753,11 @@
_Help(args)
if not cmd:
_NotInstalled()
- if cmd == 'init':
+ if cmd == 'init' or cmd == 'gitc-init':
if my_git:
_SetDefaultsTo(my_git)
try:
- _Init(args)
+ _Init(args, gitc_init=(cmd == 'gitc-init'))
except CloneFailure:
shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
sys.exit(1)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index b93cd6d..96dc99d 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -120,6 +120,9 @@
p.add_option('-r', '--regex',
dest='regex', action='store_true',
help="Execute the command only on projects matching regex or wildcard expression")
+ p.add_option('-g', '--groups',
+ dest='groups',
+ help="Execute the command only on projects matching the specified groups")
p.add_option('-c', '--command',
help='Command (and arguments) to execute',
dest='command',
@@ -213,7 +216,7 @@
self.manifest.Override(smart_sync_manifest_path)
if not opt.regex:
- projects = self.GetProjects(args)
+ projects = self.GetProjects(args, groups=opt.groups)
else:
projects = self.FindProjects(args)
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
new file mode 100644
index 0000000..03d8cc3
--- /dev/null
+++ b/subcmds/gitc_init.py
@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import shutil
+import sys
+
+import gitc_utils
+from subcmds import init
+
+
+class GitcInit(init.Init):
+ common = True
+ helpSummary = "Initialize a GITC Client."
+ helpUsage = """
+%prog [options] [client name]
+"""
+ helpDescription = """
+The '%prog' command is ran to initialize a new GITC client for use
+with the GITC file system.
+
+This command will setup the client directory, initialize repo, just
+like repo init does, and then downloads the manifest collection
+and installs in in the .repo/directory of the GITC client.
+
+Once this is done, a GITC manifest is generated by pulling the HEAD
+SHA for each project and generates the properly formatted XML file
+and installs it as .manifest in the GITC client directory.
+
+The -c argument is required to specify the GITC client name.
+
+The optional -f argument can be used to specify the manifest file to
+use for this GITC client.
+"""
+
+ def _Options(self, p):
+ super(GitcInit, self)._Options(p)
+ g = p.add_option_group('GITC options')
+ g.add_option('-f', '--manifest-file',
+ dest='manifest_file',
+ help='Optional manifest file to use for this GITC client.')
+ g.add_option('-c', '--gitc-client',
+ dest='gitc_client',
+ help='The name for the new gitc_client instance.')
+
+ def Execute(self, opt, args):
+ if not opt.gitc_client:
+ print('fatal: gitc client (-c) is required', file=sys.stderr)
+ sys.exit(1)
+ self.client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
+ opt.gitc_client)
+ if not os.path.exists(gitc_utils.GITC_MANIFEST_DIR):
+ os.makedirs(gitc_utils.GITC_MANIFEST_DIR)
+ if not os.path.exists(self.client_dir):
+ os.mkdir(self.client_dir)
+ super(GitcInit, self).Execute(opt, args)
+ # Make the destination manifest file a symlink to repo's so both repo and
+ # GITC refer to the same manifest.
+ if opt.manifest_file:
+ if not os.path.exists(opt.manifest_file):
+ print('fatal: Specified manifest file %s does not exist.' %
+ opt.manifest_file)
+ sys.exit(1)
+ self.manifest.Override(opt.manifest_file)
+ gitc_utils.generate_gitc_manifest(self.client_dir, self.manifest)
+ print('Please run `cd %s` to view your GITC client.' %
+ os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client))
\ No newline at end of file
diff --git a/subcmds/list.py b/subcmds/list.py
index 945c28d..ca51c5f 100644
--- a/subcmds/list.py
+++ b/subcmds/list.py
@@ -35,6 +35,9 @@
p.add_option('-r', '--regex',
dest='regex', action='store_true',
help="Filter the project list based on regex or wildcard matching of strings")
+ p.add_option('-g', '--groups',
+ dest='groups',
+ help="Filter the project list based on the groups the project is in")
p.add_option('-f', '--fullpath',
dest='fullpath', action='store_true',
help="Display the full work tree path instead of the relative path")
@@ -62,7 +65,7 @@
sys.exit(1)
if not opt.regex:
- projects = self.GetProjects(args)
+ projects = self.GetProjects(args, groups=opt.groups)
else:
projects = self.FindProjects(args)
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 43d450b..652a0c0 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -58,6 +58,7 @@
from git_command import GIT, git_require
from git_refs import R_HEADS, HEAD
+import gitc_utils
from project import Project
from project import RemoteSpec
from command import Command, MirrorSafeCommand
@@ -184,6 +185,9 @@
help="overwrite an existing git directory if it needs to "
"point to a different object directory. WARNING: this "
"may cause loss of data")
+ p.add_option('--force-gitc',
+ dest='force_gitc', action='store_true',
+ help="actually sync sources in the gitc client directory.")
p.add_option('-l', '--local-only',
dest='local_only', action='store_true',
help="only update working tree, don't fetch")
@@ -526,6 +530,25 @@
print('error: both -u and -p must be given', file=sys.stderr)
sys.exit(1)
+ cwd = os.getcwd()
+ if cwd.startswith(gitc_utils.GITC_MANIFEST_DIR) and not opt.force_gitc:
+ print('WARNING this will pull all the sources like a normal repo sync.\n'
+ '\nIf you want to update your GITC Client View please rerun this '
+ 'command in \n%s%s.\nOr if you actually want to pull the sources, '
+ 'rerun with --force-gitc.' %
+ (gitc_utils.GITC_FS_ROOT_DIR,
+ cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1]))
+ sys.exit(1)
+
+ self._gitc_sync = False
+ if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR):
+ self._gitc_sync = True
+ self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split(
+ '/')[0]
+ self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
+ self._client_name)
+ print('Updating GITC client: %s' % self._client_name)
+
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
@@ -642,6 +665,12 @@
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
+ if self._gitc_sync:
+ gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest)
+ print('GITC client successfully synced.')
+ return
+
+
if not opt.local_only:
mp.Sync_NetworkHalf(quiet=opt.quiet,
current_branch_only=opt.current_branch_only,