Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support nano flavours T0 #5031

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

Support nano flavours T0 #5031

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ada743f
Support nano flavours T0
Jan 15, 2025
3d29969
SUpport nano flavours in database
Jan 16, 2025
a5d4318
Nano flavours from configuration file
Jan 20, 2025
1db87d7
Update RunConfigAPI
Jan 21, 2025
304689b
Fix oracle implementation of nano flavours
Jan 22, 2025
ca79992
Update ProdOfflineConfiguration.py
LinaresToine Jan 27, 2025
05af49f
Update ReplayOfflineConfiguration.py
LinaresToine Jan 27, 2025
27adece
Update ProdOfflineConfiguration.py
LinaresToine Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions etc/ProdOfflineConfiguration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@
siteWhitelist = ["T2_CH_CERN", "T1_UK_RAL", "T1_DE_KIT", "T1_FR_CCIN2P3", "T1_ES_PIC", "T1_IT_CNAF", "T2_CH_CERN_P5"],
raw_to_disk=False,
aod_to_disk=True,
nano_flavours=['@PHYS', '@L1'],
blockCloseDelay=24 * 3600,
timePerEvent=5,
sizePerEvent=1500,
Expand Down
1 change: 1 addition & 0 deletions etc/ReplayOfflineConfiguration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@
tape_node="T0_CH_CERN_Disk",
disk_node="T0_CH_CERN_Disk",
#raw_to_disk=False,
nano_flavours=['@PHYS', '@L1'],
blockCloseDelay=1200,
timePerEvent=5,
sizePerEvent=1500,
Expand Down
9 changes: 7 additions & 2 deletions src/python/T0/RunConfig/RunConfigAPI.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -858,6 +858,10 @@ def releasePromptReco(tier0Config, specDirectory, dqmUploadProxy):
if len(datasetConfig.DqmSequences) > 0:
dqmSeq = ",".join(datasetConfig.DqmSequences)

nanoFlavours = None
if len(datasetConfig.NanoFlavours) > 0:
nanoFlavours = ",".join(datasetConfig.NanoFlavours)

datasetConfig.ScramArch = tier0Config.Global.ScramArches.get(datasetConfig.CMSSWVersion,
tier0Config.Global.DefaultScramArch)

Expand All @@ -874,6 +878,7 @@ def releasePromptReco(tier0Config, specDirectory, dqmUploadProxy):
'ALCA_SKIM' : alcaSkim,
'PHYSICS_SKIM' : physicsSkim,
'DQM_SEQ' : dqmSeq,
'NANO_FLAVOUR' : nanoFlavours,
'CMSSW' : datasetConfig.CMSSWVersion,
'SCRAM_ARCH' : datasetConfig.ScramArch,
'MULTICORE' : datasetConfig.Multicore,
Expand Down Expand Up @@ -1063,7 +1068,7 @@ def releasePromptReco(tier0Config, specDirectory, dqmUploadProxy):
specArguments['AlcaSkims'] = datasetConfig.AlcaSkims
specArguments['PhysicsSkims'] = datasetConfig.PhysicsSkims
specArguments['DQMSequences'] = datasetConfig.DqmSequences

specArguments['NanoFlavours'] = datasetConfig.NanoFlavours
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this be nanoFlavours according to https://github.com/cms-sw/cmssw/blob/master/Configuration/DataProcessing/python/Reco.py#L72 ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you elaborate? The spec arguments are cought in WMCore here (in your PR): https://github.com/dmwm/WMCore/blob/ebbe830106165340ebae9301620acec7672643c4/src/python/WMCore/WMSpec/StdSpecs/PromptReco.py
Or is there an intermediate step that I am missing?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok I guess the translation is done here: https://github.com/dmwm/WMCore/blob/ebbe830106165340ebae9301620acec7672643c4/src/python/WMCore/WMSpec/StdSpecs/PromptReco.py#L183

specArguments['UnmergedLFNBase'] = "/store/unmerged/%s" % runInfo['bulk_data_type']
if runInfo['backfill']:
specArguments['MergedLFNBase'] = "/store/backfill/%s/%s" % (runInfo['backfill'],
Expand Down Expand Up @@ -1192,4 +1197,4 @@ def setWMTaskOverrideCatalog(task, site):
logging.info("RucioCatalog")
for childTask in task.childTaskIterator():
setWMTaskOverrideCatalog(childTask, site)
return
return
3 changes: 2 additions & 1 deletion src/python/T0/RunConfig/Tier0Config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -605,7 +605,8 @@ def addDataset(config, datasetName, **settings):
datasetConfig.AlcaSkims = settings.get("alca_producers", [])
datasetConfig.PhysicsSkims = settings.get("physics_skims", [])
datasetConfig.DqmSequences = settings.get("dqm_sequences", [])

datasetConfig.NanoFlavours = settings.get("nano_flavours", [])

if hasattr(datasetConfig, "MaxMemoryperCore"):
datasetConfig.MaxMemoryperCore = settings.get("maxMemoryperCore", datasetConfig.MaxMemoryperCore)
else:
Expand Down
1 change: 1 addition & 0 deletions src/python/T0/WMBS/Oracle/Create.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,7 @@ def __init__(self, logger = None, dbi = None, params = None):
alca_skim varchar2(700),
physics_skim varchar2(700),
dqm_seq varchar2(700),
nano_flavour varchar2(50),
primary key (run_id, primds_id)
) ORGANIZATION INDEX"""

Expand Down
12 changes: 7 additions & 5 deletions src/python/T0/WMBS/Oracle/RunConfig/GetRecoConfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def execute(self, run, stream, conn = None, transaction = False):
reco_config.proc_version,
reco_config.alca_skim,
reco_config.dqm_seq,
reco_config.nano_flavour,
cmssw_version.name,
reco_config.scram_arch,
reco_config.multicore,
Expand Down Expand Up @@ -69,10 +70,11 @@ def execute(self, run, stream, conn = None, transaction = False):
resultDict[primds]['proc_ver'] = result[8]
resultDict[primds]['alca_skim'] = result[9]
resultDict[primds]['dqm_seq'] = result[10]
resultDict[primds]['cmssw'] = result[11]
resultDict[primds]['scram_arch'] = result[12]
resultDict[primds]['multicore'] = result[13]
resultDict[primds]['global_tag'] = result[14]
resultDict[primds]['scenario'] = result[15]
resultDict[primds]['nano_flavour'] = result[11]
resultDict[primds]['cmssw'] = result[12]
resultDict[primds]['scram_arch'] = result[13]
resultDict[primds]['multicore'] = result[14]
resultDict[primds]['global_tag'] = result[15]
resultDict[primds]['scenario'] = result[16]

return resultDict
3 changes: 2 additions & 1 deletion src/python/T0/WMBS/Oracle/RunConfig/InsertRecoConfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def execute(self, binds, conn = None, transaction = False):
sql = """INSERT INTO reco_config
(RUN_ID, PRIMDS_ID, DO_RECO, RECO_SPLIT, WRITE_RECO, WRITE_DQM,
WRITE_AOD, WRITE_MINIAOD, WRITE_NANOAOD, PROC_VERSION, ALCA_SKIM, PHYSICS_SKIM,
DQM_SEQ, CMSSW_ID, MULTICORE, SCRAM_ARCH, GLOBAL_TAG)
DQM_SEQ, NANO_FLAVOUR, CMSSW_ID, MULTICORE, SCRAM_ARCH, GLOBAL_TAG)
VALUES (:RUN,
(SELECT id FROM primary_dataset WHERE name = :PRIMDS),
:DO_RECO,
Expand All @@ -28,6 +28,7 @@ def execute(self, binds, conn = None, transaction = False):
:ALCA_SKIM,
:PHYSICS_SKIM,
:DQM_SEQ,
:NANO_FLAVOUR,
(SELECT id FROM cmssw_version WHERE name = :CMSSW),
:MULTICORE,
:SCRAM_ARCH,
Expand Down