JCF: Issue #361: be able to toggle smoothly at the build-release.sh l…

…evel between fddaq/nddaq vs. fddatautilities during this Issue's development process

scripts/spack/build-release.sh

@@ -60,27 +60,36 @@ cd $DAQ_RELEASE_REPO
 if [[ "$TARGET" != "core" ]]; then
     spack_template_dir=spack-repos/${TARGET}-repo-template
 else
-    spack_template_dir=spack-repos/dunedaq-repo-template
+    spack_template_dir=spack-repos/coredaq-repo-template
 fi
 
-echo python3 scripts/spack/make-release-repo.py -u \
+is_fddatautilities=false
+
+if [[ "$FULL_RELEASE_DIR" =~ "FDDU" ]]; then
+    is_fddatautilities=true
+fi
+
+if $is_fddatautilities; then
+    cmd="python3 scripts/spack/make-release-repo.py -u \
   -i ${release_yaml} \
   -t $spack_template_dir \
   -r ${RELEASE_TAG} \
   -o ${SPACK_AREA}/spack-installation \
   ${base_release_arg} \
   ${branch_arg} \
-  || exit 5
-
-
-python3 scripts/spack/make-release-repo.py -u \
+  --fddatautilities"
+else
+    cmd="python3 scripts/spack/make-release-repo.py -u \
   -i ${release_yaml} \
   -t $spack_template_dir \
   -r ${RELEASE_TAG} \
   -o ${SPACK_AREA}/spack-installation \
   ${base_release_arg} \
-  ${branch_arg} \
-  || exit 5
+  ${branch_arg}"
+fi
+
+echo $cmd
+$cmd || exit 5
 
 
 cd $SPACK_AREA
@@ -90,10 +99,15 @@ if [[ "$TARGET" != "core" ]]; then
     retval=$?
     cat $SPACK_AREA/spec_${TARGET}_log.txt 
 else
-    echo "FOR ISSUE #361 TESTING PURPOSES, FORCE subset=datautilities"
-    spack spec -l --reuse dunedaq@${RELEASE_TAG}%[email protected] subset=datautilities build_type=RelWithDebInfo arch=linux-${OS}-x86_64 > $SPACK_AREA/spec_dunedaq_log.txt 2>&1
+
+    if $is_fddatautilities ; then
+	spack spec -l --reuse coredaq@${RELEASE_TAG}%[email protected] subset=datautilities build_type=RelWithDebInfo arch=linux-${OS}-x86_64 > $SPACK_AREA/spec_coredaq_log.txt 2>&1
+    else
+	spack spec -l --reuse coredaq@${RELEASE_TAG}%[email protected] subset=all build_type=RelWithDebInfo arch=linux-${OS}-x86_64 > $SPACK_AREA/spec_coredaq_log.txt 2>&1
+    fi
+
     retval=$?
-    cat $SPACK_AREA/spec_dunedaq_log.txt 
+    cat $SPACK_AREA/spec_coredaq_log.txt 
 fi
 
 if [[ $retval != 0 ]]; then
@@ -102,9 +116,7 @@ fi
 
 build_dbe=false
 
-echo "FOR ISSUE #361 TESTING PURPOSES, IGNORE dbe"
-#if [[ $TARGET == "core" ]]; then
-if false; then
+if [[ $TARGET == "core" ]] && ! $is_fddatautilities ; then
     spack spec -l --reuse dbe%[email protected] build_type=RelWithDebInfo arch=linux-${OS}-x86_64 > $SPACK_AREA/spec_dbe_log.txt 2>&1
     retval=$?    
 
@@ -121,8 +133,11 @@ fi
 if [[ "$TARGET" != "core" ]]; then
     spack install --reuse ${TARGET}@${RELEASE_TAG}%[email protected] build_type=RelWithDebInfo arch=linux-${OS}-x86_64 || exit 7
 else
-    echo "FOR ISSUE #361 TESTING PURPOSES, FORCE subset=datautilities"
-    spack install --reuse dunedaq@${RELEASE_TAG}%[email protected] subset=datautilities build_type=RelWithDebInfo arch=linux-${OS}-x86_64 || exit 7
+    if $is_fddatautilities ; then
+	spack install --reuse coredaq@${RELEASE_TAG}%[email protected] subset=datautilities build_type=RelWithDebInfo arch=linux-${OS}-x86_64 || exit 7
+    else
+	spack install --reuse coredaq@${RELEASE_TAG}%[email protected] subset=all build_type=RelWithDebInfo arch=linux-${OS}-x86_64 || exit 7
+    fi
 fi
 
 if $build_dbe; then
@@ -134,23 +149,23 @@ if [[ "$TARGET" != "core" ]]; then
     spack load ${TARGET}@${RELEASE_TAG} || exit 9
 
     cd $DAQ_RELEASE_REPO
-    echo /usr/bin/python3 scripts/spack/make-release-repo.py \
+    cmd="echo /usr/bin/python3 scripts/spack/make-release-repo.py \
         -o ${SPACK_AREA} \
         --pyvenv-requirements \
-        -i ${release_yaml}
+        -i ${release_yaml}"
 
-    /usr/bin/python3 scripts/spack/make-release-repo.py \
-        -o ${SPACK_AREA} \
-        --pyvenv-requirements \
-        -i ${release_yaml} \
-        || exit 10
+    echo $cmd
+    $cmd || exit 10
 
     python -m venv --prompt dbt ${SPACK_AREA}/.venv
     source ${SPACK_AREA}/.venv/bin/activate
 
-    echo "FOR ISSUE #361 TESTING PURPOSES, DISREGARD ANY PIP INSTALLATION ERRORS"
-    python -m pip install -r ${SPACK_AREA}/pyvenv_requirements.txt
-    #python -m pip install -r ${SPACK_AREA}/pyvenv_requirements.txt || exit 11
+    if $is_fddatautilities ; then
+	echo "FOR ISSUE #361 TESTING PURPOSES, DISREGARD ANY PIP INSTALLATION ERRORS"
+	python -m pip install -r ${SPACK_AREA}/pyvenv_requirements.txt || true
+    else
+	python -m pip install -r ${SPACK_AREA}/pyvenv_requirements.txt || exit 11
+    fi
 
     pushd $FULL_RELEASE_DIR
     cp $DAQ_RELEASE_REPO/$( dirname $release_yaml )/dbt-build-order.cmake .

scripts/spack/make-release-repo.py

@@ -185,29 +185,31 @@ def generate_daq_package(self, repo_path, template_dir):
                 print(f"Info: package.py has been written at {ipkgpy}.")
         return
 
-    def generate_external_umbrella_package(self, repo_path, template_dir):
+    def generate_external_umbrella_package(self, repo_path, template_dir, fddatautilities):
         repo_dir = os.path.join(repo_path, "spack-repo", "packages")
         template_dir = os.path.join(template_dir, "packages")
 
         for ipkg in ['devtools', 'externals', 'systems']:
 
-            # JCF, Apr-9-2024: the following special treatment for the externals
-            # umbrella package is solely to be used during the development
-            # process for Issue #361
-
-            if ipkg == "externals":
-                this_dir=os.path.dirname(os.path.abspath(__file__))
-
-                ipkg_dir = os.path.join(repo_dir, ipkg)
-                os.makedirs(ipkg_dir)
-                ipkgpy = os.path.join(ipkg_dir, "package.py")
-                srcpkgpy = f"{this_dir}/../../issue361_dev_files/package_externals.py"
-                print(f"srcpkgpy is {srcpkgpy}")
-                assert(os.path.exists(srcpkgpy))
-
-                shutil.copyfile(srcpkgpy, ipkgpy)
-                continue
-
+            if fddatautilities:
+
+                # JCF, Apr-9-2024: the following special treatment for the externals
+                # umbrella package is solely to be used during the development
+                # process for Issue #361
+
+                if ipkg == "externals":
+                    this_dir=os.path.dirname(os.path.abspath(__file__))
+
+                    ipkg_dir = os.path.join(repo_dir, ipkg)
+                    os.makedirs(ipkg_dir)
+                    ipkgpy = os.path.join(ipkg_dir, "package.py")
+                    srcpkgpy = f"{this_dir}/../../issue361_dev_files/package_externals.py"
+                    print(f"srcpkgpy is {srcpkgpy}")
+                    assert(os.path.exists(srcpkgpy))
+
+                    shutil.copyfile(srcpkgpy, ipkgpy)
+                    continue
+
             itemp = os.path.join(template_dir, ipkg, 'package.py')
             if not os.path.exists(itemp):
                 print(f"Error: template file {itemp} is not found!")
@@ -236,23 +238,24 @@ def generate_external_umbrella_package(self, repo_path, template_dir):
                 print(f"Info: package.py has been written at {ipkgpy}.")
         return
 
-    def generate_daq_umbrella_package(self, repo_path, template_dir):
+    def generate_daq_umbrella_package(self, repo_path, template_dir, fddatautilities):
         repo_dir = os.path.join(repo_path, "spack-repo", "packages")
         template_dir = os.path.join(template_dir, "packages")
         ipkg = self.rtype
 
-        this_dir=os.path.dirname(os.path.abspath(__file__))
+        if fddatautilities:
+            this_dir=os.path.dirname(os.path.abspath(__file__))
 
-        ipkg_dir = os.path.join(repo_dir, ipkg)
-        os.makedirs(ipkg_dir)
-        ipkgpy = os.path.join(ipkg_dir, "package.py")
+            ipkg_dir = os.path.join(repo_dir, ipkg)
+            os.makedirs(ipkg_dir)
+            ipkgpy = os.path.join(ipkg_dir, "package.py")
 
-        srcpkgpy = f"{this_dir}/../../issue361_dev_files/package_{ipkg}.py"
-        print(f"srcpkgpy is {srcpkgpy}")
-        assert(os.path.exists(srcpkgpy))
+            srcpkgpy = f"{this_dir}/../../issue361_dev_files/package_{ipkg}.py"
+            print(f"srcpkgpy is {srcpkgpy}")
+            assert(os.path.exists(srcpkgpy))
 
-        shutil.copyfile(srcpkgpy, ipkgpy)
-        return
+            shutil.copyfile(srcpkgpy, ipkgpy)
+            return
 
         itemp = os.path.join(template_dir, ipkg, 'package.py')
         if not os.path.exists(itemp):
@@ -289,19 +292,19 @@ def generate_daq_umbrella_package(self, repo_path, template_dir):
             print(f"Info: package.py has been written at {ipkgpy}.")
         return
 
-    def generate_umbrella_package(self, repo_path, template_dir):
+    def generate_umbrella_package(self, repo_path, template_dir, fddatautilities):
         if self.rtype == "coredaq":
-            self.generate_external_umbrella_package(repo_path, template_dir)
-        self.generate_daq_umbrella_package(repo_path, template_dir)
+            self.generate_external_umbrella_package(repo_path, template_dir, fddatautilities)
+        self.generate_daq_umbrella_package(repo_path, template_dir, fddatautilities)
         return
 
-    def generate_repo(self, outdir, tempdir, update_hash, release_name, base_release):
+    def generate_repo(self, outdir, tempdir, update_hash, release_name, base_release, fddatautilities):
         if release_name is not None:
             self.set_release(release_name, base_release)
         self.copy_release_yaml(outdir, update_hash)
         self.generate_repo_file(outdir)
         self.generate_daq_package(outdir, tempdir)
-        self.generate_umbrella_package(outdir, tempdir)
+        self.generate_umbrella_package(outdir, tempdir, fddatautilities)
         return
 
     def generate_pypi_manifest(self, output_file):
@@ -366,6 +369,8 @@ def generate_pyvenv_requirements(self, output_file):
                         help="whether to generate file containing bash array for python modules;")
     parser.add_argument('--pyvenv-requirements', action='store_true',
                         help="whether to generate requirements file for pyvenv;")
+    parser.add_argument('--fddatautilities', action='store_false',
+                        help="whether this is for the fddatautilities package (TEMPORARY)")
     parser.add_argument('--base-release',
                         help="base release name")
 
@@ -387,4 +392,4 @@ def generate_pyvenv_requirements(self, output_file):
     else:
         daq_release.generate_repo(args.output_path, args.template_path,
                                   args.update_hash, args.release_name,
-                                  args.base_release)
+                                  args.base_release, args.fddatautilities)