From a4cc58ee2e9615a2023d7e0046c4d5498bcb8a4e Mon Sep 17 00:00:00 2001 From: Javier Galan Date: Thu, 28 Dec 2023 08:03:02 +0100 Subject: [PATCH] Adding an option that allows to execute any command in a condor job --- condor/restManagerToCondor.py | 102 ++++++++-------------------------- 1 file changed, 23 insertions(+), 79 deletions(-) diff --git a/condor/restManagerToCondor.py b/condor/restManagerToCondor.py index c59c5c7..6698fde 100755 --- a/condor/restManagerToCondor.py +++ b/condor/restManagerToCondor.py @@ -17,6 +17,7 @@ sectionName = "" jobName = "" fileList = "" +singleCommand = "" onlyScripts=0 @@ -54,7 +55,10 @@ def ProcessFilesInList( file_list ): command = os.environ['REST_PATH'] + "/bin/restManager --c " + os.environ['PWD'] + "/" + cfgFile + " --f " + fileToProcess if sectionName != "": command = command + " --n " + sectionName - f.write( command + "\n" ) + if( singleCommand != "" ): + f.write( singleCommand + "\n" ) + else: + f.write( command + "\n" ) f.close() st = os.stat( scriptName + ".sh" ) @@ -88,81 +92,6 @@ def ProcessFilesInList( file_list ): print "---> Produced condor script : " + str( scriptName ) + "_" + str(cont) + "_" + str( jobName ) + ".condor" print "---> To launch : " + command - if (fileList == "" ): - scriptName = "condor/" + jobName - - f = open( scriptName + ".sh", "w" ) - f.write("#!/bin/bash\n") - #f.write("export REST_DATAPATH="+ os.environ['REST_DATAPATH']+"\n") - - for key in os.environ.keys(): - if key.find( "HOME") == 0: - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - print( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find( "DATA") == 0: - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - print( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("GDML") == 0: - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - print( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("GEOMETRY") >= 0: - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - print( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("REST") == 0: - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - print( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("G4") == 0: - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("PATH") == 0: - print( "export " + key + "=" + os.environ[key] +"\n" ) - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("LD_LIBRARY_PATH") == 0: - print( "export " + key + "=" + os.environ[key] +"\n" ) - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("GARFIELD_") == 0: - print( "export " + key + "=" + os.environ[key] +"\n" ) - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("HEED_") == 0: - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - if key.find("PWD") == 0: - f.write( "export " + key + "=" + os.environ[key] +"\n" ) - - f.write("export USER="+ os.environ['USER']+"\n\n") - - command = "restManager --c " + cfgFile - if sectionName != "": - command = command + " --n " + sectionName - f.write( command + "\n" ) - f.close() - - st = os.stat( scriptName + ".sh" ) - os.chmod( scriptName + ".sh", st.st_mode | stat.S_IEXEC) - - cont = cont + 1 - - g = open( scriptName + ".condor", "w" ) - g.write("Executable = " + scriptName + ".sh\n" ) - g.write("Arguments = \n" ) - g.write("Log = " + scriptName + ".log\n" ) - g.write("Output = " + scriptName + ".out\n" ) - g.write("Error = " + scriptName + ".err\n" ) - g.write("Queue\n" ) - g.close() - - if onlyScripts == 0: - print "---> Launching : " + command - - condorCommand = "condor_submit " + scriptName + ".condor" - print "Condor command : " + condorCommand - - print "Waiting " + str(sleep) + " seconds to launch next job" - time.sleep(sleep) - - print commands.getstatusoutput( condorCommand ) - else: - print "---> Produced condor script : " + str( scriptName ) + "_" + str( jobName ) + ".condor" - print "---> To launch : " + command - def ProcessWithoutFile( ): rpt = repeat cont = 0 @@ -247,6 +176,13 @@ def ProcessWithoutFile( ): print " - FILELIST. A file containing one file per line" print " - GLOBPATTERN. A glob filename pattern using wild cards *?" print "" + print "" + print " An additional option will allow you to launch any command (without " + print " need to call restManager) using:" + print "" + print " Usage : restManagerToCondor.py --command \"cmd + arguments\"" + print "" + print "" print " In all cases full path file location should be given!" print "" print " - Other options : " @@ -303,17 +239,23 @@ def ProcessWithoutFile( ): if ( sys.argv[x+1] == "--fileList" or sys.argv[x+1] == "-f" ): fileList = sys.argv[x+2] + if ( sys.argv[x+1] == "--command" or sys.argv[x+1] == "-C" ): + singleCommand = sys.argv[x+2] + if ( sys.argv[x+1] == "--onlyScripts" or sys.argv[x+1] == "-o" ): onlyScripts = 1 -if jobName == "": +if jobName == "" and cfgFile != "": jobName = cfgFile[cfgFile.rfind("/")+1:cfgFile.rfind(".rml")] +if jobName == "": + jobName = "noName" + if not os.path.exists("condor/" + jobName): os.makedirs("condor/" + jobName) -if cfgFile == "": +if singleCommand == "" and cfgFile == "": print "Please specify a RML config file list using -c flag." sys.exit( 1 ) @@ -332,8 +274,10 @@ def ProcessWithoutFile( ): print ( "Files to process:" ) print ( lines ) + ProcessFilesInList( lines ) +elif fileList == "" and singleCommand != "": + lines.append( singleCommand ) ProcessFilesInList( lines ) else: print( " ======> Launching processing without input file <======" ) ProcessWithoutFile( ) -