DAOS-17117 build: build dependencies as git submodules

* add --deps-as-gitmodules-subdir flag that ignores branch, tag, and
  commit information for dependencies in build.config and expects
  such dependencies to exist as submodules in the supplied
  sub-directory (relative to base of repo)
* build such dependencies in place by inserting relative
  symlinks into the build directory

Signed-off-by: Nicholas Murphy <[email protected]>

SConstruct

@@ -86,6 +86,13 @@ def add_command_line_options():
               default=os.path.join(Dir('#').abspath, 'utils', 'build.config'),
               help='build config file to use. [%default]')
 
+    AddOption('--deps-as-gitmodules-subdir',
+              dest='deps_as_gitmodules_subdir',
+              default=None,
+              help='Ignore the versions/branches/patches specified in build.config and \
+                    use the specified relative sub-directory containing all \
+                    dependencies as git submodules instead')
+
 
 def parse_and_save_conf(env, opts_file):
     """Parse daos.conf
@@ -443,6 +450,18 @@ def scons():
 
     # Define and load the components.  This will add more values to opt.
     prereqs = PreReqComponent(deps_env, opts)
+
+    gitmodules_subdir = GetOption("deps_as_gitmodules_subdir")
+    if gitmodules_subdir:
+        print(f"WARNING: --deps-as-gitmodules-subdir specified; ignoring build.config "
+              f"versions, branches, and patches and using gitmodules installed in "
+              f"{gitmodules_subdir} instead")
+        gitmodules_absdir = os.path.join(Dir('#').abspath, gitmodules_subdir)
+        if not os.path.isdir(gitmodules_absdir):
+            print(f"ERROR: --deps-as-gitmodules-subdir was specified but {gitmodules_absdir} "
+                  f"is not a valid directory.")
+            Exit(-1)
+
     # Now save the daos.conf file before attempting to build anything.  This means that options
     # are sticky even if there's a failed build.
     opts.Save(opts_file, deps_env)

site_scons/prereq_tools/base.py

@@ -452,6 +452,8 @@ def __init__(self, env, opts):
         self.__top_dir = Dir('#').abspath
         install_dir = os.path.join(self.__top_dir, 'install')
 
+        self.deps_as_gitmodules_subdir = GetOption("deps_as_gitmodules_subdir")
+
         RUNNER.initialize(self.__env)
 
         opts.Add(PathVariable('PREFIX', 'Installation path', install_dir,
@@ -489,7 +491,7 @@ def __init__(self, env, opts):
 
         self.system_env = env.Clone()
 
-        self.__build_dir = self._sub_path(build_dir_name)
+        self.__build_dir = self.sub_path(build_dir_name)
 
         opts.Add(PathVariable('GOPATH', 'Location of your GOPATH for the build',
                               f'{self.__build_dir}/go', PathVariable.PathIsDirCreate))
@@ -715,15 +717,15 @@ def __parse_build_deps(self):
             if not skip_download:
                 self.download_deps = True
 
-    def _sub_path(self, path):
+    def sub_path(self, path):
         """Resolve the real path"""
         return os.path.realpath(os.path.join(self.__top_dir, path))
 
     def _setup_path_var(self, var):
         """Create a command line variable for a path"""
         tmp = self.__env.get(var)
         if tmp:
-            value = self._sub_path(tmp)
+            value = self.sub_path(tmp)
             self.__env[var] = value
 
     def define(self, name, **kw):
@@ -1093,8 +1095,33 @@ def get(self):
         commit_sha = self.prereqs.get_config("commit_versions", self.name)
         repo = self.prereqs.get_config("repos", self.name)
 
-        if not self.retriever:
-            print(f'Using installed version of {self.name}')
+        if self.prereqs.deps_as_gitmodules_subdir is None and \
+                not self.retriever:
+            print(f"Using installed version of {self.name}")
+            return
+
+        if self.prereqs.deps_as_gitmodules_subdir:
+            builddir, _ = os.path.split(self.src_path)
+            target = os.path.join(
+                self.prereqs.sub_path(self.prereqs.deps_as_gitmodules_subdir),
+                self.name)
+
+            if not os.path.isdir(target):
+                print(f"Symlink target {target} is not a valid directory")
+                raise BuildFailure(self.name)
+
+            relpath = os.path.relpath(target, builddir)
+            if os.path.exists(self.src_path):
+                if not os.path.islink(self.src_path) or \
+                        os.readlink(self.src_path) != relpath:
+
+                    print(f"{self.src_path} already exists in build directory.")
+                    raise BuildFailure(self.name)
+            else:
+                print(f"Creating symlink {self.src_path} to {relpath}")
+                if not RUNNER.run_commands([['ln', '-s', relpath, self.src_path]]):
+                    print("Error creating symlink")
+                    raise BuildFailure(self.name)
             return
 
         # Source code is retrieved using retriever