From 379c0af7d3d496d14c9397b5f596fe15705a594b Mon Sep 17 00:00:00 2001 From: Sudhar Krishnakumar Date: Tue, 10 Dec 2024 11:22:20 -0500 Subject: [PATCH 1/2] Change to replace manual step(used by redhat, where ipu_port1_setup.sh script is used). By creating post_init_app.sh script, through ipu-plugin. post_init_app.sh, gets invoked by IMC software, after ipumgmt daemon, starts running. post_init_app.sh, creates this script->port0-setup.sh which runs in the background, and waits until D4 interface gets created on ACC. Once D4 interface gets created on ACC, port0-setup.sh uses devmem command to add D4 interface to the forwarding VSI group 1. --- ipu-plugin/pkg/ipuplugin/lifecycleservice.go | 103 ++++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/ipu-plugin/pkg/ipuplugin/lifecycleservice.go b/ipu-plugin/pkg/ipuplugin/lifecycleservice.go index c6284c91..1f82ed7b 100644 --- a/ipu-plugin/pkg/ipuplugin/lifecycleservice.go +++ b/ipu-plugin/pkg/ipuplugin/lifecycleservice.go @@ -495,6 +495,13 @@ func setBaseMacAddr() (string, error) { return macAddress, nil } +/* +Updates files below on IMC, and does IMC reboot. +1. Copy P4 package from container to IMC +2. Update load_custom_pkg.sh +3. Create post_init_app.sh +4. Create uuid file. +*/ func (s *SSHHandlerImpl) sshFunc() error { config := &ssh.ClientConfig{ User: "root", @@ -585,6 +592,38 @@ func (s *SSHHandlerImpl) sshFunc() error { return fmt.Errorf("failed to sync load_custom_pkg.sh: %s", err) } + err = sftpClient.Chmod(loadCustomPkgFilePath, 0755) + if err != nil { + log.Errorf("failed to chmod load_custom_pkg.sh file: %s", err) + return fmt.Errorf("failed to chmod load_custom_pkg.sh file: %s", err) + } + + //Create post_init_app.sh + postInitAppFileStr := postInitAppScript() + postInitRemoteFilePath := "/work/scripts/post_init_app.sh" + postInitFile, err := sftpClient.Create(postInitRemoteFilePath) + if err != nil { + log.Errorf("failed to create post_init_app.sh file: %s", err) + return fmt.Errorf("failed to create post_init_app.sh file: %s", err) + } + defer postInitFile.Close() + + _, err = postInitFile.Write([]byte(postInitAppFileStr)) + if err != nil { + return fmt.Errorf("failed to write to post_init_app.sh file: %s", err) + } + + err = postInitFile.Sync() + if err != nil { + return fmt.Errorf("failed to sync post_init_app.sh file: %s", err) + } + + err = sftpClient.Chmod(postInitRemoteFilePath, 0755) + if err != nil { + log.Errorf("failed to chmod post_init_app.sh file: %s", err) + return fmt.Errorf("failed to chmod post_init_app.sh file: %s", err) + } + uuidFilePath := "/work/uuid" uuidFile, err := sftpClient.Create(uuidFilePath) if err != nil { @@ -642,6 +681,53 @@ func countAPFDevices() int { return len(pfList) } +func postInitAppScript() string { + + postInitAppScriptStr := `PORT0_SETUP_SCRIPT=/work/scripts/port0-setup.sh +PORT0_SETUP_LOG=/work/port0-setup.log + +pkill -9 $(basename ${PORT0_SETUP_SCRIPT}) +/bin/rm -f ${PORT0_SETUP_SCRIPT} ${PORT0_SETUP_LOG} + +cat< ${PORT0_SETUP_SCRIPT} +#!/bin/bash +IDPF_VPORT_NAME="enp0s1f0d4" +ACC_VPORT_ID=0x4 +retry=0 + +while true ; do +sleep 2 +cli_entry=(\$(cli_client -qc | grep "fn_id: 0x4 .* vport_id \${ACC_VPORT_ID}" | sed 's/: / /g' | sed 's/addr //g')) +if [ \${#cli_entry[@]} -gt 1 ] ; then + + for (( id=0 ; id<\${#cli_entry[@]} ; id+=2 )) ; do + declare "\${cli_entry[id]}"="\${cli_entry[\$((id+1))]}" + #echo "\${cli_entry[id]}"="\${cli_entry[\$((id+1))]}" + done + + if [ X\${is_created} == X"yes" ] && [ X\${is_enabled} == X"yes" ] ; then + IDPF_VPORT_VSI_HEX=\${vsi_id} + VSI_GROUP_INIT=\$(printf "0x%x" \$((0x8000050000000000 + IDPF_VPORT_VSI_HEX))) + VSI_GROUP_WRITE=\$(printf "0x%x" \$((0xA000050000000000 + IDPF_VPORT_VSI_HEX))) + echo "#Add to VSI Group 1 : \${IDPF_VPORT_NAME} [vsi: \${IDPF_VPORT_VSI_HEX}]" + devmem 0x20292002a0 64 \${VSI_GROUP_INIT} + devmem 0x2029200388 64 0x1 + devmem 0x20292002a0 64 \${VSI_GROUP_WRITE} + exit 0 + fi +else + retry=\$((retry+1)) + echo "RETRY: \${retry} : #Add to VSI Group 1 : \${IDPF_VPORT_NAME} .. " +fi +done +PORT0_CONFIG_EOF + +/usr/bin/chmod a+x ${PORT0_SETUP_SCRIPT} +/usr/bin/nohup ${PORT0_SETUP_SCRIPT} 0>&- &> ${PORT0_SETUP_LOG} &` + + return postInitAppScriptStr +} + func genLoadCustomPkgFile(macAddress string) string { p4PkgName := os.Getenv("P4_NAME") + ".pkg" @@ -679,6 +765,7 @@ fi 1. First time provisioning of IPU system(where MAC gets set in node policy) 2. Upgrade-for any update to P4 package. 3. Upgrade-for node policy. Other changes in node policy thro load_custom_pkg.sh. +4. Check if file-> post_init_app.sh exists. Returns-> bool(returns false, if IMC reboot is required), string->for any error or success string. */ func skipIMCReboot() (bool, string) { @@ -715,7 +802,7 @@ func skipIMCReboot() (bool, string) { p4pkgMatch := false uuidFileExists := false lcpkgFileMatch := false - + piaFileExists := false outputStr := strings.TrimSuffix(string(outputBytes), "\n") if outputStr == "File does not exist" { @@ -805,7 +892,19 @@ func skipIMCReboot() (bool, string) { return false, "lcpkgFileMatch mismatch" } - log.Infof("uuidFileExists->%v, p4pkgMatch->%v, lcpkgFileMatch->%v", uuidFileExists, p4pkgMatch, lcpkgFileMatch) + postInitRemoteFilePath := "/work/scripts/post_init_app.sh" + postInitFile, err := sftpClient.Open(postInitRemoteFilePath) + if err != nil { + log.Errorf("failed to open post_init_app.sh file: %s", err) + return false, fmt.Sprintf("failed to open post_init_app.sh file: %s", err) + } else { + log.Infof("post_init_app.sh file exists") + piaFileExists = true + } + defer postInitFile.Close() + + log.Infof("uuidFileExists->%v, p4pkgMatch->%v, lcpkgFileMatch->%v, piaFileExists->%v", + uuidFileExists, p4pkgMatch, lcpkgFileMatch, piaFileExists) return true, fmt.Sprintf("checks pass, imc reboot not required") } From bc548a9bc710631fc2ab5e47dc7252f5caaf7934 Mon Sep 17 00:00:00 2001 From: Sudhar Krishnakumar Date: Tue, 10 Dec 2024 15:04:55 -0500 Subject: [PATCH 2/2] Updated comment, per review. Also added changes, to handle case, where post_init_app.sh file exits, but ipu-plugin has an different version of it(update scenario). --- ipu-plugin/pkg/ipuplugin/lifecycleservice.go | 38 ++++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/ipu-plugin/pkg/ipuplugin/lifecycleservice.go b/ipu-plugin/pkg/ipuplugin/lifecycleservice.go index 1f82ed7b..984750c9 100644 --- a/ipu-plugin/pkg/ipuplugin/lifecycleservice.go +++ b/ipu-plugin/pkg/ipuplugin/lifecycleservice.go @@ -762,6 +762,8 @@ fi /* IMC reboot needed for following cases: +Note: Changes to load_custom_pkg.sh or post_init_app.sh is managed +thro ipu-plugin->using genLoadCustomPkgFile and postInitAppScript. 1. First time provisioning of IPU system(where MAC gets set in node policy) 2. Upgrade-for any update to P4 package. 3. Upgrade-for node policy. Other changes in node policy thro load_custom_pkg.sh. @@ -802,7 +804,7 @@ func skipIMCReboot() (bool, string) { p4pkgMatch := false uuidFileExists := false lcpkgFileMatch := false - piaFileExists := false + piaFileMatch := false outputStr := strings.TrimSuffix(string(outputBytes), "\n") if outputStr == "File does not exist" { @@ -892,19 +894,41 @@ func skipIMCReboot() (bool, string) { return false, "lcpkgFileMatch mismatch" } + postInitAppFile := postInitAppScript() + postInitAppFileHash := md5.Sum([]byte(postInitAppFile)) + postInitAppFileHashStr := hex.EncodeToString(postInitAppFileHash[:]) + postInitRemoteFilePath := "/work/scripts/post_init_app.sh" - postInitFile, err := sftpClient.Open(postInitRemoteFilePath) + imcPostInitFile, err := sftpClient.Open(postInitRemoteFilePath) if err != nil { log.Errorf("failed to open post_init_app.sh file: %s", err) return false, fmt.Sprintf("failed to open post_init_app.sh file: %s", err) + } + log.Infof("post_init_app.sh file exists") + defer imcPostInitFile.Close() + + imcPostInitFileBytes, err := io.ReadAll(imcPostInitFile) + if err != nil { + log.Errorf("failed to read post_init_app.sh: %s", err) + return false, fmt.Sprintf("failed to read post_init_app.sh: %s", err) + } + + imcPostInitFileHash := md5.Sum(imcPostInitFileBytes) + imcPostInitFileHashStr := hex.EncodeToString(imcPostInitFileHash[:]) + + if postInitAppFileHashStr != imcPostInitFileHashStr { + log.Infof("post_init_app.sh md5 mismatch, generated->%v, on IMC->%v", postInitAppFileHashStr, imcPostInitFileHashStr) } else { - log.Infof("post_init_app.sh file exists") - piaFileExists = true + log.Infof("post_init_app.sh md5 match, generated->%v, on IMC->%v", postInitAppFileHashStr, imcPostInitFileHashStr) + piaFileMatch = true + } + + if !piaFileMatch { + return false, "piaFileMatch mismatch" } - defer postInitFile.Close() - log.Infof("uuidFileExists->%v, p4pkgMatch->%v, lcpkgFileMatch->%v, piaFileExists->%v", - uuidFileExists, p4pkgMatch, lcpkgFileMatch, piaFileExists) + log.Infof("uuidFileExists->%v, p4pkgMatch->%v, lcpkgFileMatch->%v, piaFileMatch->%v", + uuidFileExists, p4pkgMatch, lcpkgFileMatch, piaFileMatch) return true, fmt.Sprintf("checks pass, imc reboot not required") }