Squashed 'manage_externals/' changes from a48558d..fde04e4

fde04e4 Merge pull request ESCOMP#138 from billsacks/add_python38_tests
37e4c4a Do not update dictionary in-place in loop
7e8474b Remove testing on mac os
7f41c56 Fix pylint issue
3065b0d Add travis-ci tests with python3.7 and python3.8
34fbf55 Add support for git sparse checkout
6c6ef9f Fix pylint errors
6a659ad Added test for sparse checkout and updated documentation
1443243 Support for git sparsecheckout via read-tree.

git-subtree-dir: manage_externals
git-subtree-split: fde04e4

.travis.yml

@@ -1,25 +1,12 @@
-# NOTE(bja, 2017-11) travis-ci dosen't support python language builds
-#  on mac os. As a work around, we use built-in python on linux, and
-#  declare osx a 'generic' language, and create our own python env.
-
 language: python
 os: linux
 python: 
   - "2.7"
   - "3.4"
   - "3.5"
   - "3.6"
-matrix:
-  include:
-  - os: osx
-    language: generic
-    before_install:
-      # NOTE(bja, 2017-11) update is slow, 2.7.12 installed by default, good enough!
-      # - brew update
-      # - brew outdated python2 || brew upgrade python2
-      - pip install virtualenv
-      - virtualenv env -p python2
-      - source env/bin/activate
+  - "3.7"
+  - "3.8"
 install:
   - pip install -r test/requirements.txt
 before_script:

README.md

@@ -85,7 +85,7 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     description file:
 
         $ cd ${SRC_ROOT}
-        $ ./manage_externals/checkout_externals --excernals my-externals.cfg
+        $ ./manage_externals/checkout_externals --externals my-externals.cfg
 
   * Status summary of the repositories managed by checkout_externals:
 
@@ -202,6 +202,21 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     Then the main 'externals' field in the top level repo should point to
     'sub-externals.cfg'.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines begining with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manic/checkout.py

@@ -227,6 +227,21 @@ def commandline_arguments(args=None):
     Now, %(prog)s will process Externals.cfg and also process
     Externals_LIBX.cfg as if it was a sub-external.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines beginning with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manic/externals_description.py

@@ -351,6 +351,7 @@ class ExternalsDescription(dict):
     REPO_URL = 'repo_url'
     REQUIRED = 'required'
     TAG = 'tag'
+    SPARSE = 'sparse'
 
     PROTOCOL_EXTERNALS_ONLY = 'externals_only'
     PROTOCOL_GIT = 'git'
@@ -374,6 +375,7 @@ class ExternalsDescription(dict):
                              TAG: 'string',
                              BRANCH: 'string',
                              HASH: 'string',
+                             SPARSE: 'string',
                             }
                      }
 
@@ -562,6 +564,8 @@ def _check_optional(self):
                 self[field][self.REPO][self.HASH] = EMPTY_STR
             if self.REPO_URL not in self[field][self.REPO]:
                 self[field][self.REPO][self.REPO_URL] = EMPTY_STR
+            if self.SPARSE not in self[field][self.REPO]:
+                self[field][self.REPO][self.SPARSE] = EMPTY_STR
 
             # from_submodule has a complex relationship with other fields
             if self.SUBMODULE in self[field]:

manic/repository.py

@@ -21,6 +21,7 @@ def __init__(self, component_name, repo):
         self._branch = repo[ExternalsDescription.BRANCH]
         self._hash = repo[ExternalsDescription.HASH]
         self._url = repo[ExternalsDescription.REPO_URL]
+        self._sparse = repo[ExternalsDescription.SPARSE]
 
         if self._url is EMPTY_STR:
             fatal_error('repo must have a URL')

manic/repository_git.py

@@ -316,8 +316,11 @@ def _checkout_ref(self, repo_dir, verbosity, submodules):
         else:
             self._checkout_external_ref(verbosity, submodules)
 
+        if self._sparse:
+            self._sparse_checkout(repo_dir, verbosity)
         os.chdir(cwd)
 
+
     def _checkout_local_ref(self, verbosity, submodules):
         """Checkout the reference considering the local repo only. Do not
         fetch any additional remotes or specify the remote when
@@ -362,6 +365,20 @@ def _checkout_external_ref(self, verbosity, submodules):
             ref = '{0}/{1}'.format(remote_name, ref)
         self._git_checkout_ref(ref, verbosity, submodules)
 
+    def _sparse_checkout(self, repo_dir, verbosity):
+        """Use git read-tree to thin the working tree."""
+        cwd = os.getcwd()
+
+        cmd = ['cp', self._sparse, os.path.join(repo_dir,
+                                                '.git/info/sparse-checkout')]
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+        os.chdir(repo_dir)
+        self._git_sparse_checkout(verbosity)
+
+        os.chdir(cwd)
+
     def _check_for_valid_ref(self, ref, remote_name=None):
         """Try some basic sanity checks on the user supplied reference so we
         can provide a more useful error message than calledprocess
@@ -776,6 +793,18 @@ def _git_checkout_ref(ref, verbosity, submodules):
         if submodules:
             GitRepository._git_update_submodules(verbosity)
 
+    @staticmethod
+    def _git_sparse_checkout(verbosity):
+        """Configure repo via read-tree."""
+        cmd = ['git', 'config', 'core.sparsecheckout', 'true']
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+        cmd = ['git', 'read-tree', '-mu', 'HEAD']
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+
     @staticmethod
     def _git_update_submodules(verbosity):
         """Run git submodule update for the side effect of updating this

manic/repository_svn.py

@@ -220,9 +220,8 @@ def xml_status_is_dirty(svn_output):
                 continue
             if item == SVN_UNVERSIONED:
                 continue
-            else:
-                is_dirty = True
-                break
+            is_dirty = True
+            break
         return is_dirty
 
     # ----------------------------------------------------------------

manic/sourcetree.py

@@ -45,6 +45,7 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry):
         self._externals = EMPTY_STR
         self._externals_sourcetree = None
         self._stat = ExternalStatus()
+        self._sparse = None
         # Parse the sub-elements
 
         # _path : local path relative to the containing source tree
@@ -298,18 +299,20 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR):
         for comp in load_comps:
             printlog('{0}, '.format(comp), end='')
             stat = self._all_components[comp].status()
+            stat_final = {}
             for name in stat.keys():
                 # check if we need to append the relative_path_base to
                 # the path so it will be sorted in the correct order.
-                if not stat[name].path.startswith(relative_path_base):
-                    stat[name].path = os.path.join(relative_path_base,
-                                                   stat[name].path)
-                    # store under key = updated path, and delete the
-                    # old key.
-                    comp_stat = stat[name]
-                    del stat[name]
-                    stat[comp_stat.path] = comp_stat
-            summary.update(stat)
+                if stat[name].path.startswith(relative_path_base):
+                    # use as is, without any changes to path
+                    stat_final[name] = stat[name]
+                else:
+                    # append relative_path_base to path and store under key = updated path
+                    modified_path = os.path.join(relative_path_base,
+                                                 stat[name].path)
+                    stat_final[modified_path] = stat[name]
+                    stat_final[modified_path].path = modified_path
+            summary.update(stat_final)
 
         return summary
 

test/repos/simple-ext.git/refs/heads/master

@@ -1 +1 @@
-9b75494003deca69527bb64bcaa352e801611dd2
+607ec299c17dd285c029edc41a0109e49d441380

test/repos/simple-ext.git/refs/tags/tag2

@@ -0,0 +1 @@
+b7692b6d391899680da7b9b6fd8af4c413f06fe7

test/test_sys_checkout.py

@@ -88,6 +88,8 @@
 
 SVN_TEST_REPO = 'https://github.com/escomp/cesm'
 
+# Disable too-many-public-methods error
+# pylint: disable=R0904
 
 def setUpModule():  # pylint: disable=C0103
     """Setup for all tests in this module. It is called once per module!
@@ -183,6 +185,25 @@ def container_simple_svn(self, dest_dir):
 
         self.write_config(dest_dir)
 
+    def container_sparse(self, dest_dir):
+        """Create a container with a full external and a sparse external
+
+        """
+        # Create a file for a sparse pattern match
+        sparse_filename = 'sparse_checkout'
+        with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile:
+            sfile.write('readme.txt')
+
+        self.create_config()
+        self.create_section(SIMPLE_REPO_NAME, 'simp_tag',
+                            tag='tag2')
+
+        sparse_relpath = '../../{}'.format(sparse_filename)
+        self.create_section(SIMPLE_REPO_NAME, 'simp_sparse',
+                            tag='tag2', sparse=sparse_relpath)
+
+        self.write_config(dest_dir)
+
     def mixed_simple_base(self, dest_dir):
         """Create a mixed-use base externals file with only simple externals.
 
@@ -239,7 +260,8 @@ def create_metadata(self):
 
     def create_section(self, repo_type, name, tag='', branch='',
                        ref_hash='', required=True, path=EXTERNALS_NAME,
-                       externals='', repo_path=None, from_submodule=False):
+                       externals='', repo_path=None, from_submodule=False,
+                       sparse=''):
         # pylint: disable=too-many-branches
         """Create a config section with autofilling some items and handling
         optional items.
@@ -287,6 +309,9 @@ def create_section(self, repo_type, name, tag='', branch='',
         if externals:
             self._config.set(name, ExternalsDescription.EXTERNALS, externals)
 
+        if sparse:
+            self._config.set(name, ExternalsDescription.SPARSE, sparse)
+
         if from_submodule:
             self._config.set(name, ExternalsDescription.SUBMODULE, "True")
 
@@ -710,6 +735,14 @@ def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/mixed_req'.format(directory)
         self._check_generic_modified_ok_required(tree, name)
 
+    def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME):
+        name = './{0}/simp_sparse'.format(directory)
+        self._check_generic_empty_default_required(tree, name)
+
+    def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME):
+        name = './{0}/simp_sparse'.format(directory)
+        self._check_generic_ok_clean_required(tree, name)
+
     # ----------------------------------------------------------------
     #
     # Check results for groups of externals under specific conditions
@@ -870,6 +903,23 @@ def _check_mixed_cont_simple_required_post_checkout(self, overall, tree):
         self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME)
         self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH)
 
+    def _check_container_sparse_pre_checkout(self, overall, tree):
+        self.assertEqual(overall, 0)
+        self._check_simple_tag_empty(tree)
+        self._check_simple_sparse_empty(tree)
+
+    def _check_container_sparse_post_checkout(self, overall, tree):
+        self.assertEqual(overall, 0)
+        self._check_simple_tag_ok(tree)
+        self._check_simple_sparse_ok(tree)
+
+    def _check_file_exists(self, repo_dir, pathname):
+        "Check that <pathname> exists in <repo_dir>"
+        self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname)))
+
+    def _check_file_absent(self, repo_dir, pathname):
+        "Check that <pathname> does not exist in <repo_dir>"
+        self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname)))
 
 class TestSysCheckout(BaseTestSysCheckout):
     """Run systems level tests of checkout_externals
@@ -1234,6 +1284,14 @@ def test_container_full(self):
                                                 self.status_args)
         self._check_container_full_post_checkout(overall, tree)
 
+        # Check existance of some files
+        subrepo_path = os.path.join('externals', 'simp_tag')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_absent(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+
         # update the mixed-use repo to point to different branch
         self._generator.update_branch(under_test_dir, 'mixed_req',
                                       'new-feature', MIXED_REPO_NAME)
@@ -1314,6 +1372,40 @@ def test_mixed_simple(self):
                                                 self.status_args)
         self._check_mixed_cont_simple_required_post_checkout(overall, tree)
 
+    def test_container_sparse(self):
+        """Verify that 'full' container with simple subrepo
+        can run a sparse checkout and generate the correct initial status.
+
+        """
+        # create the test repository
+        under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME)
+
+        # create the top level externals file
+        self._generator.container_sparse(under_test_dir)
+
+        # inital checkout
+        overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                self.checkout_args)
+        self._check_container_sparse_pre_checkout(overall, tree)
+
+        overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                self.status_args)
+        self._check_container_sparse_post_checkout(overall, tree)
+
+        # Check existance of some files
+        subrepo_path = os.path.join('externals', 'simp_tag')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_exists(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+        subrepo_path = os.path.join('externals', 'simp_sparse')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_absent(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+
 
 class TestSysCheckoutSVN(BaseTestSysCheckout):
     """Run systems level tests of checkout_externals accessing svn repositories