diff --git a/.travis.yml b/.travis.yml index b32f81bd28..1990cb9604 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,3 @@ -# NOTE(bja, 2017-11) travis-ci dosen't support python language builds -# on mac os. As a work around, we use built-in python on linux, and -# declare osx a 'generic' language, and create our own python env. - language: python os: linux python: @@ -9,17 +5,8 @@ python: - "3.4" - "3.5" - "3.6" -matrix: - include: - - os: osx - language: generic - before_install: - # NOTE(bja, 2017-11) update is slow, 2.7.12 installed by default, good enough! - # - brew update - # - brew outdated python2 || brew upgrade python2 - - pip install virtualenv - - virtualenv env -p python2 - - source env/bin/activate + - "3.7" + - "3.8" install: - pip install -r test/requirements.txt before_script: diff --git a/README.md b/README.md index 15e45ffb71..c931c8e213 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. description file: $ cd ${SRC_ROOT} - $ ./manage_externals/checkout_externals --excernals my-externals.cfg + $ ./manage_externals/checkout_externals --externals my-externals.cfg * Status summary of the repositories managed by checkout_externals: @@ -202,6 +202,21 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. + * from_submodule (True / False) : used to pull the repo_url, local_path, + and hash properties for this external from the .gitmodules file in + this repository. Note that the section name (the entry in square + brackets) must match the name in the .gitmodules file. + If from_submodule is True, the protocol must be git and no repo_url, + local_path, hash, branch, or tag entries are allowed. + Default: False + + * sparse (string) : used to control a sparse checkout. This optional + entry should point to a filename (path relative to local_path) that + contains instructions on which repository paths to include (or + exclude) from the working tree. + See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree + Default: sparse checkout is disabled + * Lines begining with '#' or ';' are comments and will be ignored. # Obtaining this tool, reporting issues, etc. diff --git a/manic/checkout.py b/manic/checkout.py index afd3a27886..edc5655954 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -227,6 +227,21 @@ def commandline_arguments(args=None): Now, %(prog)s will process Externals.cfg and also process Externals_LIBX.cfg as if it was a sub-external. + * from_submodule (True / False) : used to pull the repo_url, local_path, + and hash properties for this external from the .gitmodules file in + this repository. Note that the section name (the entry in square + brackets) must match the name in the .gitmodules file. + If from_submodule is True, the protocol must be git and no repo_url, + local_path, hash, branch, or tag entries are allowed. + Default: False + + * sparse (string) : used to control a sparse checkout. This optional + entry should point to a filename (path relative to local_path) that + contains instructions on which repository paths to include (or + exclude) from the working tree. + See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree + Default: sparse checkout is disabled + * Lines beginning with '#' or ';' are comments and will be ignored. # Obtaining this tool, reporting issues, etc. diff --git a/manic/externals_description.py b/manic/externals_description.py index 3cebf525b5..b0c4f736a7 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -351,6 +351,7 @@ class ExternalsDescription(dict): REPO_URL = 'repo_url' REQUIRED = 'required' TAG = 'tag' + SPARSE = 'sparse' PROTOCOL_EXTERNALS_ONLY = 'externals_only' PROTOCOL_GIT = 'git' @@ -374,6 +375,7 @@ class ExternalsDescription(dict): TAG: 'string', BRANCH: 'string', HASH: 'string', + SPARSE: 'string', } } @@ -562,6 +564,8 @@ def _check_optional(self): self[field][self.REPO][self.HASH] = EMPTY_STR if self.REPO_URL not in self[field][self.REPO]: self[field][self.REPO][self.REPO_URL] = EMPTY_STR + if self.SPARSE not in self[field][self.REPO]: + self[field][self.REPO][self.SPARSE] = EMPTY_STR # from_submodule has a complex relationship with other fields if self.SUBMODULE in self[field]: diff --git a/manic/repository.py b/manic/repository.py index 4488c6be9e..ea4230fb7b 100644 --- a/manic/repository.py +++ b/manic/repository.py @@ -21,6 +21,7 @@ def __init__(self, component_name, repo): self._branch = repo[ExternalsDescription.BRANCH] self._hash = repo[ExternalsDescription.HASH] self._url = repo[ExternalsDescription.REPO_URL] + self._sparse = repo[ExternalsDescription.SPARSE] if self._url is EMPTY_STR: fatal_error('repo must have a URL') diff --git a/manic/repository_git.py b/manic/repository_git.py index c0e64eb551..f986051001 100644 --- a/manic/repository_git.py +++ b/manic/repository_git.py @@ -316,8 +316,11 @@ def _checkout_ref(self, repo_dir, verbosity, submodules): else: self._checkout_external_ref(verbosity, submodules) + if self._sparse: + self._sparse_checkout(repo_dir, verbosity) os.chdir(cwd) + def _checkout_local_ref(self, verbosity, submodules): """Checkout the reference considering the local repo only. Do not fetch any additional remotes or specify the remote when @@ -362,6 +365,20 @@ def _checkout_external_ref(self, verbosity, submodules): ref = '{0}/{1}'.format(remote_name, ref) self._git_checkout_ref(ref, verbosity, submodules) + def _sparse_checkout(self, repo_dir, verbosity): + """Use git read-tree to thin the working tree.""" + cwd = os.getcwd() + + cmd = ['cp', self._sparse, os.path.join(repo_dir, + '.git/info/sparse-checkout')] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + os.chdir(repo_dir) + self._git_sparse_checkout(verbosity) + + os.chdir(cwd) + def _check_for_valid_ref(self, ref, remote_name=None): """Try some basic sanity checks on the user supplied reference so we can provide a more useful error message than calledprocess @@ -776,6 +793,18 @@ def _git_checkout_ref(ref, verbosity, submodules): if submodules: GitRepository._git_update_submodules(verbosity) + @staticmethod + def _git_sparse_checkout(verbosity): + """Configure repo via read-tree.""" + cmd = ['git', 'config', 'core.sparsecheckout', 'true'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + cmd = ['git', 'read-tree', '-mu', 'HEAD'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + @staticmethod def _git_update_submodules(verbosity): """Run git submodule update for the side effect of updating this diff --git a/manic/repository_svn.py b/manic/repository_svn.py index 2f0d4d848c..408ed84676 100644 --- a/manic/repository_svn.py +++ b/manic/repository_svn.py @@ -220,9 +220,8 @@ def xml_status_is_dirty(svn_output): continue if item == SVN_UNVERSIONED: continue - else: - is_dirty = True - break + is_dirty = True + break return is_dirty # ---------------------------------------------------------------- diff --git a/manic/sourcetree.py b/manic/sourcetree.py index 83676b776b..b9c9c21082 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -45,6 +45,7 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): self._externals = EMPTY_STR self._externals_sourcetree = None self._stat = ExternalStatus() + self._sparse = None # Parse the sub-elements # _path : local path relative to the containing source tree @@ -298,18 +299,20 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR): for comp in load_comps: printlog('{0}, '.format(comp), end='') stat = self._all_components[comp].status() + stat_final = {} for name in stat.keys(): # check if we need to append the relative_path_base to # the path so it will be sorted in the correct order. - if not stat[name].path.startswith(relative_path_base): - stat[name].path = os.path.join(relative_path_base, - stat[name].path) - # store under key = updated path, and delete the - # old key. - comp_stat = stat[name] - del stat[name] - stat[comp_stat.path] = comp_stat - summary.update(stat) + if stat[name].path.startswith(relative_path_base): + # use as is, without any changes to path + stat_final[name] = stat[name] + else: + # append relative_path_base to path and store under key = updated path + modified_path = os.path.join(relative_path_base, + stat[name].path) + stat_final[modified_path] = stat[name] + stat_final[modified_path].path = modified_path + summary.update(stat_final) return summary diff --git a/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d b/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d new file mode 100644 index 0000000000..acaf7889b4 Binary files /dev/null and b/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d differ diff --git a/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 b/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 new file mode 100644 index 0000000000..3f6959cc54 Binary files /dev/null and b/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 differ diff --git a/test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7 b/test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7 new file mode 100644 index 0000000000..1b3b272442 Binary files /dev/null and b/test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7 differ diff --git a/test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b b/test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b new file mode 100644 index 0000000000..04e760363a Binary files /dev/null and b/test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b differ diff --git a/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 b/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 new file mode 100644 index 0000000000..f08ae820c9 Binary files /dev/null and b/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 differ diff --git a/test/repos/simple-ext.git/refs/heads/master b/test/repos/simple-ext.git/refs/heads/master index 5c67504966..adf1ccb002 100644 --- a/test/repos/simple-ext.git/refs/heads/master +++ b/test/repos/simple-ext.git/refs/heads/master @@ -1 +1 @@ -9b75494003deca69527bb64bcaa352e801611dd2 +607ec299c17dd285c029edc41a0109e49d441380 diff --git a/test/repos/simple-ext.git/refs/tags/tag2 b/test/repos/simple-ext.git/refs/tags/tag2 new file mode 100644 index 0000000000..4160b6c494 --- /dev/null +++ b/test/repos/simple-ext.git/refs/tags/tag2 @@ -0,0 +1 @@ +b7692b6d391899680da7b9b6fd8af4c413f06fe7 diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index 63adcacdde..df726f2b70 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -88,6 +88,8 @@ SVN_TEST_REPO = 'https://github.com/escomp/cesm' +# Disable too-many-public-methods error +# pylint: disable=R0904 def setUpModule(): # pylint: disable=C0103 """Setup for all tests in this module. It is called once per module! @@ -183,6 +185,25 @@ def container_simple_svn(self, dest_dir): self.write_config(dest_dir) + def container_sparse(self, dest_dir): + """Create a container with a full external and a sparse external + + """ + # Create a file for a sparse pattern match + sparse_filename = 'sparse_checkout' + with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile: + sfile.write('readme.txt') + + self.create_config() + self.create_section(SIMPLE_REPO_NAME, 'simp_tag', + tag='tag2') + + sparse_relpath = '../../{}'.format(sparse_filename) + self.create_section(SIMPLE_REPO_NAME, 'simp_sparse', + tag='tag2', sparse=sparse_relpath) + + self.write_config(dest_dir) + def mixed_simple_base(self, dest_dir): """Create a mixed-use base externals file with only simple externals. @@ -239,7 +260,8 @@ def create_metadata(self): def create_section(self, repo_type, name, tag='', branch='', ref_hash='', required=True, path=EXTERNALS_NAME, - externals='', repo_path=None, from_submodule=False): + externals='', repo_path=None, from_submodule=False, + sparse=''): # pylint: disable=too-many-branches """Create a config section with autofilling some items and handling optional items. @@ -287,6 +309,9 @@ def create_section(self, repo_type, name, tag='', branch='', if externals: self._config.set(name, ExternalsDescription.EXTERNALS, externals) + if sparse: + self._config.set(name, ExternalsDescription.SPARSE, sparse) + if from_submodule: self._config.set(name, ExternalsDescription.SUBMODULE, "True") @@ -710,6 +735,14 @@ def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME): name = './{0}/mixed_req'.format(directory) self._check_generic_modified_ok_required(tree, name) + def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME): + name = './{0}/simp_sparse'.format(directory) + self._check_generic_empty_default_required(tree, name) + + def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME): + name = './{0}/simp_sparse'.format(directory) + self._check_generic_ok_clean_required(tree, name) + # ---------------------------------------------------------------- # # Check results for groups of externals under specific conditions @@ -870,6 +903,23 @@ def _check_mixed_cont_simple_required_post_checkout(self, overall, tree): self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME) self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH) + def _check_container_sparse_pre_checkout(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_tag_empty(tree) + self._check_simple_sparse_empty(tree) + + def _check_container_sparse_post_checkout(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_tag_ok(tree) + self._check_simple_sparse_ok(tree) + + def _check_file_exists(self, repo_dir, pathname): + "Check that exists in " + self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname))) + + def _check_file_absent(self, repo_dir, pathname): + "Check that does not exist in " + self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname))) class TestSysCheckout(BaseTestSysCheckout): """Run systems level tests of checkout_externals @@ -1234,6 +1284,14 @@ def test_container_full(self): self.status_args) self._check_container_full_post_checkout(overall, tree) + # Check existance of some files + subrepo_path = os.path.join('externals', 'simp_tag') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + # update the mixed-use repo to point to different branch self._generator.update_branch(under_test_dir, 'mixed_req', 'new-feature', MIXED_REPO_NAME) @@ -1314,6 +1372,40 @@ def test_mixed_simple(self): self.status_args) self._check_mixed_cont_simple_required_post_checkout(overall, tree) + def test_container_sparse(self): + """Verify that 'full' container with simple subrepo + can run a sparse checkout and generate the correct initial status. + + """ + # create the test repository + under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + + # create the top level externals file + self._generator.container_sparse(under_test_dir) + + # inital checkout + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.checkout_args) + self._check_container_sparse_pre_checkout(overall, tree) + + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_sparse_post_checkout(overall, tree) + + # Check existance of some files + subrepo_path = os.path.join('externals', 'simp_tag') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_exists(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + subrepo_path = os.path.join('externals', 'simp_sparse') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + class TestSysCheckoutSVN(BaseTestSysCheckout): """Run systems level tests of checkout_externals accessing svn repositories diff --git a/test/test_unit_repository.py b/test/test_unit_repository.py index 2152503c2d..5b9c242fd3 100644 --- a/test/test_unit_repository.py +++ b/test/test_unit_repository.py @@ -36,7 +36,8 @@ def setUp(self): ExternalsDescription.REPO_URL: 'junk_root', ExternalsDescription.TAG: 'junk_tag', ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } def test_create_repo_git(self): """Verify that several possible names for the 'git' protocol @@ -95,7 +96,8 @@ def test_tag(self): ExternalsDescription.REPO_URL: url, ExternalsDescription.TAG: tag, ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.tag(), tag) @@ -112,7 +114,8 @@ def test_branch(self): ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.branch(), branch) @@ -125,11 +128,13 @@ def test_hash(self): protocol = 'test_protocol' url = 'test_url' ref = 'deadc0de' + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: EMPTY_STR, ExternalsDescription.TAG: EMPTY_STR, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.hash(), ref) @@ -146,11 +151,13 @@ def test_tag_branch(self): branch = 'test_branch' tag = 'test_tag' ref = EMPTY_STR + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) @@ -165,11 +172,13 @@ def test_tag_branch_hash(self): branch = 'test_branch' tag = 'test_tag' ref = 'deadc0de' + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) @@ -184,11 +193,13 @@ def test_no_tag_no_branch(self): branch = EMPTY_STR tag = EMPTY_STR ref = EMPTY_STR + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) diff --git a/test/test_unit_repository_git.py b/test/test_unit_repository_git.py index b025fbd429..4a0a334bb1 100644 --- a/test/test_unit_repository_git.py +++ b/test/test_unit_repository_git.py @@ -547,7 +547,8 @@ def setUp(self): ExternalsDescription.TAG: 'very_useful_tag', ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } self._repo = GitRepository('test', self._rdata) def test_remote_git_proto(self):