diff --git a/test/data/ReqMgr/requests/DMWM/DQMHarvest_RunWhitelist.json b/test/data/ReqMgr/requests/DMWM/DQMHarvest_RunWhitelist.json index 19510488be..af20c081cc 100644 --- a/test/data/ReqMgr/requests/DMWM/DQMHarvest_RunWhitelist.json +++ b/test/data/ReqMgr/requests/DMWM/DQMHarvest_RunWhitelist.json @@ -18,32 +18,32 @@ "UnmergedLFNBase": "/store/unmerged" }, "createRequest": { - "AcquisitionEra": "Run2022D", - "CMSSWVersion": "CMSSW_12_4_11_patch3", + "AcquisitionEra": "Run2024D", + "CMSSWVersion": "CMSSW_14_0_19_patch2", "Campaign": "Campaign-OVERRIDE-ME", - "Comments": {"WorkFlowDesc": "Data BTagMu harvesting byRun; 4 runs whitelisted; read input data from DBS Prod", + "Comments": {"WorkFlowDesc": "Run2024D data harvesting byRun; 6 runs whitelisted; read input data from DBS Prod", "CheckList": "DQMHarvest with run whitelist; DQMHarvest real data; DQMHarvest on EL8"}, "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", "CouchDBName": "reqmgr_config_cache", - "DQMConfigCacheID": "d657f5aa83cffd6153ada2747a77b3d4", + "DQMConfigCacheID": "7ab63e3a9a3e83c05ebdd4d4772004f3", "DQMHarvestUnit": "byRun", "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", - "GlobalTag": "124X_dataRun3_v11", - "InputDataset": "/BTagMu/Run2022D-10Dec2022-v1/DQMIO", + "GlobalTag": "140X_dataRun3_v20", + "InputDataset": "/JetMET0/Run2024D-2024CDEReprocessing-v1/DQMIO", "Memory": 2000, "Multicore": 1, - "PrepID": "TEST-ReReco-Run2022D-BTagMu-10Dec2022-00001", - "ProcessingString": "10Dec2022", + "PrepID": "TEST-ReReco-Run2024D-JetMET0-2024CDEReprocessing-00001", + "ProcessingString": "2024CDEReprocessing", "ProcessingVersion": 1, "RequestPriority": 999999, "RequestString": "RequestString-OVERRIDE-ME", "RequestType": "DQMHarvest", - "RunWhitelist": [357538, 357539, 357540, 357550], + "RunWhitelist": [380306, 380307, 380346, 380348, 380385, 380400], "ScramArch": [ - "el8_amd64_gcc10" + "el8_amd64_gcc12" ], - "SizePerEvent": 300, + "SizePerEvent": 2, "TimePerEvent": 1 } } diff --git a/test/data/ReqMgr/requests/DMWM/ReReco_Run2022C_LumiMask.json b/test/data/ReqMgr/requests/DMWM/ReReco_Run2022C_LumiMask.json deleted file mode 100644 index 1173f27aa2..0000000000 --- a/test/data/ReqMgr/requests/DMWM/ReReco_Run2022C_LumiMask.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "assignRequest": { - "AcquisitionEra": "AcquisitionEra-OVERRIDE-ME", - "Dashboard": "Dashboard-OVERRIDE-ME", - "GracePeriod": 300, - "MergedLFNBase": "/store/backfill/1", - "Override": { - "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" - }, - "ProcessingString": "ProcessingString-OVERRIDE-ME", - "ProcessingVersion": 19, - "SiteBlacklist": [], - "SiteWhitelist": [ - "SiteWhitelist-OVERRIDE-ME" - ], - "SoftTimeout": 129600, - "Team": "Team-OVERRIDE-ME", - "UnmergedLFNBase": "/store/unmerged" - }, - "createRequest": { - "AcquisitionEra": "Run2022C", - "CMSSWVersion": "CMSSW_12_4_14_patch1", - "Campaign": "Campaign-OVERRIDE-ME", - "Comments": { - "CheckList": "ReReco workflow; ReReco with lumiMask input; ReReco multicore; ReReco harvesting", - "WorkFlowDesc": "ReReco with lumi list, total of 4 runs, 67 lumis; 4 cores/10GB" - }, - "ConfigCacheID": "a8fc2cf8e27cd8dd50cfde908ca8115f", - "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", - "CouchDBName": "reqmgr_config_cache", - "DQMConfigCacheID": "a8fc2cf8e27cd8dd50cfde908ca913f5", - "DQMHarvestUnit": "byRun", - "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", - "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader/", - "EnableHarvesting": true, - "EventStreams": 0, - "GPUParams": "null", - "GlobalTag": "124X_dataRun3_v15", - "IncludeParents": false, - "InputDataset": "/MuonEG/Run2022C-v1/RAW", - "LumiList": { - "355862": [ - [ - 121, - 133 - ] - ], - "355863": [ - [ - 1, - 14 - ] - ], - "355865": [ - [ - 1, - 287 - ] - ], - "355870": [ - [ - 31, - 67 - ] - ] - }, - "Memory": 10000, - "Multicore": 4, - "PrepID": "TEST-ReReco-Run2022C-MuonEG-testlumi-00001", - "ProcessingString": "testlumi", - "ProcessingVersion": 1, - "RequestPriority": 610000, - "RequestString": "RequestString-OVERRIDE-ME", - "RequestType": "ReReco", - "RequiresGPU": "forbidden", - "Scenario": "pp", - "ScramArch": [ - "el8_amd64_gcc10" - ], - "SizePerEvent": 1, - "SplittingAlgo": "EventAwareLumiBased", - "TimePerEvent": 2 - } -} \ No newline at end of file diff --git a/test/data/ReqMgr/requests/DMWM/ReReco_RunBlockWhite.json b/test/data/ReqMgr/requests/DMWM/ReReco_RunBlockWhite.json deleted file mode 100644 index e296b0a690..0000000000 --- a/test/data/ReqMgr/requests/DMWM/ReReco_RunBlockWhite.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "assignRequest": { - "Dashboard": "Dashboard-OVERRIDE-ME", - "GracePeriod": 300, - "MergedLFNBase": "/store/backfill/1", - "Override": {"eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED"}, - "ProcessingString": "ProcessingString-OVERRIDE-ME", - "ProcessingVersion": 19, - "SiteBlacklist": [], - "SiteWhitelist": [ - "SiteWhitelist-OVERRIDE-ME" - ], - "SoftTimeout": 129600, - "Team": "Team-OVERRIDE-ME", - "UnmergedLFNBase": "/store/unmerged" - }, - "createRequest": { - "AcquisitionEra": "Run2016H", - "BlockWhitelist": ["/HLTPhysicsIsolatedBunch/Run2016H-v1/RAW#92049f6e-92f9-11e6-b150-001e67abf228"], - "CMSSWVersion": "CMSSW_8_0_23", - "Campaign": "Campaign-OVERRIDE-ME", - "Comments": {"WorkFlowDesc": ["ReReco run 283171 from one block (out of 2); harvesting enabled; 3 lumis only", - "Automatic splitting at 389 EpJob (actually 1 lumi and file per job)"], - "CheckList": ["ReReco harvesting"]}, - "ConfigCacheID": "31b62d6a0629158d7246ce34e82c20c3", - "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", - "CouchDBName": "reqmgr_config_cache", - "DQMConfigCacheID": "947d11a5076ed983a3f9aaafd7a2b04c", - "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", - "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader", - "EnableHarvesting": true, - "GlobalTag": "80X_dataRun2_SPECIALHIGHPUFILL_v0", - "Group": "DATAOPS", - "IncludeParents": false, - "InputDataset": "/HLTPhysicsIsolatedBunch/Run2016H-v1/RAW", - "Memory": 12000, - "Multicore": 8, - "PrepID": "TEST-ReReco-Run2016H-v1-09Nov2016-0016", - "ProcessingString": "09Nov2016", - "ProcessingVersion": 2, - "RequestPriority": 190000, - "RequestString": "RequestString-OVERRIDE-ME", - "RequestType": "ReReco", - "RunWhitelist": [ - 283171 - ], - "Scenario": "pp", - "ScramArch": "slc6_amd64_gcc530", - "SizePerEvent": 1500, - "SplittingAlgo": "EventAwareLumiBased", - "TimePerEvent": 73.85 - } -} diff --git a/test/data/ReqMgr/requests/DMWM/ReReco_RunBlocklists_Parents.json b/test/data/ReqMgr/requests/DMWM/ReReco_RunBlocklists_Parents.json new file mode 100644 index 0000000000..e8cbe09415 --- /dev/null +++ b/test/data/ReqMgr/requests/DMWM/ReReco_RunBlocklists_Parents.json @@ -0,0 +1,61 @@ +{ + "assignRequest": { + "AcquisitionEra": "AcquisitionEra-OVERRIDE-ME", + "Dashboard": "Dashboard-OVERRIDE-ME", + "GracePeriod": 300, + "MergedLFNBase": "/store/backfill/1", + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" + }, + "ProcessingString": "ProcessingString-OVERRIDE-ME", + "ProcessingVersion": 19, + "SiteBlacklist": [], + "SiteWhitelist": [ + "SiteWhitelist-OVERRIDE-ME" + ], + "SoftTimeout": 129600, + "Team": "Team-OVERRIDE-ME", + "UnmergedLFNBase": "/store/unmerged" + }, + "createRequest": { + "AcquisitionEra": "Run2023C", + "BlockWhitelist": ["/MinimumBias/Run2023C-22Sep2023_v4-v1/MINIAOD#146eb70b-12ad-47dd-a109-b7feef4927cf", + "/MinimumBias/Run2023C-22Sep2023_v4-v1/MINIAOD#5b047ae4-f01e-44b4-b960-ae443ed46c2e", + "/MinimumBias/Run2023C-22Sep2023_v4-v1/MINIAOD#7597e092-5056-4316-b7b6-0b5aa48d6829"], + "CMSSWVersion": "CMSSW_13_2_6_patch2", + "Campaign": "Campaign-OVERRIDE-ME", + "Comments": { + "CheckList": "ReReco workflow; ReReco with IncludeParents; ReReco with run whitelist; ReReco with block whitelist; ReReco multicore", + "WorkFlowDesc": ["ReReco MINIAOD with 3 blocks whitelisted; but only 2 blocks processed out of 4 runs whitelisted;", + "Including 4 AOD parent blocks; Running with 2 cores and 3GB of memory"] + }, + "ConfigCacheID": "544d8059f5f200f8959e2e4fab11a94e", + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "CouchDBName": "reqmgr_config_cache", + "DQMUploadUrl": "https://cmsweb.cern.ch/dqm/dev", + "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader/", + "EnableHarvesting": false, + "EventStreams": 0, + "GPUParams": "null", + "GlobalTag": "130X_dataRun3_Prompt_v4", + "IncludeParents": true, + "InputDataset": "/MinimumBias/Run2023C-22Sep2023_v4-v1/MINIAOD", + "Memory": 3000, + "Multicore": 2, + "PrepID": "TEST-ReReco-Run2023C-MinimumBias-JMENano12p5_v4-00001", + "ProcessingString": "JMENano12p5_v4", + "ProcessingVersion": 1, + "RequestPriority": 613000, + "RequestString": "RequestString-OVERRIDE-ME", + "RequestType": "ReReco", + "RequiresGPU": "forbidden", + "RunWhitelist": [367786, 367832, 367913, 368851], + "Scenario": "pp", + "ScramArch": [ + "el8_amd64_gcc11" + ], + "SizePerEvent": 100, + "SplittingAlgo": "EventAwareLumiBased", + "TimePerEvent": 0.5 + } +} diff --git a/test/data/ReqMgr/requests/DMWM/ReReco_failCase_Nvidia.json b/test/data/ReqMgr/requests/DMWM/ReReco_failCase_Nvidia.json deleted file mode 100644 index 29ef861c15..0000000000 --- a/test/data/ReqMgr/requests/DMWM/ReReco_failCase_Nvidia.json +++ /dev/null @@ -1,190 +0,0 @@ -{ - "assignRequest": { - "AcquisitionEra": "AcquisitionEra-OVERRIDE-ME", - "Dashboard": "Dashboard-OVERRIDE-ME", - "GracePeriod": 300, - "MergedLFNBase": "/store/backfill/1", - "Override": { - "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" - }, - "ProcessingString": "ProcessingString-OVERRIDE-ME", - "ProcessingVersion": 19, - "SiteBlacklist": [], - "SiteWhitelist": [ - "SiteWhitelist-OVERRIDE-ME" - ], - "SoftTimeout": 129600, - "Team": "Team-OVERRIDE-ME", - "TrustSitelists": true, - "UnmergedLFNBase": "/store/unmerged" - }, - "createRequest": { - "AcquisitionEra": "Run2018A", - "BlockWhitelist": ["/DoubleMuon/Run2018A-v1/RAW#7bb3cd63-f1a9-4a76-bd89-ccdd81af526a"], - "CMSSWVersion": "CMSSW_10_2_4_patch1", - "Campaign": "Campaign-OVERRIDE-ME", - "Comments": {"WorkFlowDesc": ["Large RunWhitelist and 1 block BlockWhitelisted, which has only invalid files; thus, this wf will go to failed"], - "CheckList": ["Record in LogDB reason for workflows going into failed status"]}, - "ConfigCacheID": "573976a90ba04b7a9dbdd98e43ffcc9d", - "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", - "CouchDBName": "reqmgr_config_cache", - "DQMConfigCacheID": "573976a90ba04b7a9dbdd98e43fcfeea", - "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", - "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader", - "EnableHarvesting": false, - "EventsPerJob": 2215, - "FilesPerJob": 1, - "GlobalTag": "102X_dataRun2_Sep2018Rereco_v1", - "RequiresGPU": "optional", - "GPUParams": "{\"GPUMemoryMB\": 123, \"CUDARuntime\": \"11.2\", \"CUDACapabilities\": [\"11.2\", \"11.3\"]}", - "IncludeParents": false, - "InputDataset": "/DoubleMuon/Run2018A-v1/RAW", - "LumisPerJob": 8, - "Memory": 14000, - "Multicore": 8, - "PrepID": "TEST-ReReco-Run2018A-DoubleMuon-17Sep2018-0002", - "ProcessingString": "17Sep2018", - "ProcessingVersion": 1, - "RequestPriority": 210000, - "RequestString": "RequestString-OVERRIDE-ME", - "RequestType": "ReReco", - "RunWhitelist": [ - 315257, - 315258, - 315259, - 315264, - 315265, - 315267, - 315270, - 315322, - 315339, - 315357, - 315361, - 315363, - 315365, - 315366, - 315420, - 315488, - 315489, - 315490, - 315506, - 315509, - 315510, - 315512, - 315543, - 315555, - 315556, - 315557, - 315640, - 315641, - 315642, - 315644, - 315645, - 315646, - 315647, - 315648, - 315689, - 315690, - 315702, - 315703, - 315704, - 315705, - 315713, - 315721, - 315741, - 315764, - 315770, - 315784, - 315785, - 315786, - 315788, - 315789, - 315790, - 315800, - 315801, - 315840, - 315973, - 315974, - 316058, - 316059, - 316060, - 316061, - 316062, - 316082, - 316109, - 316110, - 316111, - 316112, - 316113, - 316114, - 316151, - 316153, - 316186, - 316187, - 316199, - 316200, - 316201, - 316202, - 316216, - 316217, - 316218, - 316219, - 316239, - 316240, - 316241, - 316271, - 316361, - 316362, - 316363, - 316377, - 316378, - 316379, - 316380, - 316455, - 316456, - 316457, - 316469, - 316470, - 316472, - 316505, - 316569, - 316590, - 316613, - 316615, - 316664, - 316665, - 316666, - 316667, - 316700, - 316701, - 316702, - 316715, - 316716, - 316717, - 316718, - 316719, - 316720, - 316721, - 316722, - 316723, - 316758, - 316766, - 316876, - 316877, - 316879, - 316928, - 316944, - 316985, - 316993, - 316994, - 316995 - ], - "Scenario": "pp", - "ScramArch": [ - "slc6_amd64_gcc700" - ], - "SizePerEvent": 800, - "SplittingAlgo": "EventAwareLumiBased", - "TimePerEvent": 13 - } -} diff --git a/test/data/ReqMgr/requests/Integration/DQMHarvesting.json b/test/data/ReqMgr/requests/Integration/DQMHarvesting.json index 9d613842d6..0905e9e5ff 100644 --- a/test/data/ReqMgr/requests/Integration/DQMHarvesting.json +++ b/test/data/ReqMgr/requests/Integration/DQMHarvesting.json @@ -4,7 +4,9 @@ "Dashboard": "Dashboard-OVERRIDE-ME", "GracePeriod": 300, "MergedLFNBase": "/store/backfill/1", - "Override": {"eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED"}, + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" + }, "ProcessingString": "ProcessingString-OVERRIDE-ME", "ProcessingVersion": 19, "SiteBlacklist": [], @@ -16,31 +18,32 @@ "UnmergedLFNBase": "/store/unmerged" }, "createRequest": { - "AcquisitionEra": "Run2022D", - "CMSSWVersion": "CMSSW_12_4_11_patch3", + "AcquisitionEra": "Run2024D", + "CMSSWVersion": "CMSSW_14_0_19_patch2", "Campaign": "Campaign-OVERRIDE-ME", - "Comments": {"WorkFlowDesc": ["Harvest all 69 runs in byRun mode (separate jobs)"], + "Comments": {"WorkFlowDesc": ["Harvest all 47 runs in byRun mode (separate jobs)"], "CheckList": ["DQMHarvest workflow"]}, "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", "CouchDBName": "reqmgr_config_cache", - "DQMConfigCacheID": "d657f5aa83cffd6153ada2747a77b3d4", + "DQMConfigCacheID": "7ab63e3a9a3e83c05ebdd4d4772004f3", "DQMHarvestUnit": "byRun", "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", - "EnableHarvesting": false, - "GlobalTag": "124X_dataRun3_v11", - "IncludeParents": false, - "InputDataset": "/BTagMu/Run2022D-10Dec2022-v1/DQMIO", + "GlobalTag": "140X_dataRun3_v20", + "InputDataset": "/JetMET0/Run2024D-2024CDEReprocessing-v1/DQMIO", "Memory": 2000, "Multicore": 1, - "PrepID": "TEST-ReReco-Run2022D-BTagMu-10Dec2022-00001", - "ProcessingString": "10Dec2022", + "PrepID": "TEST-ReReco-Run2024D-JetMET0-2024CDEReprocessing-00001", + "ProcessingString": "2024CDEReprocessing", "ProcessingVersion": 1, "RequestPriority": 999999, "RequestString": "RequestString-OVERRIDE-ME", - "RequestType": "DQMHarvest", - "ScramArch": "el8_amd64_gcc10", - "SizePerEvent": 1600, + "RequestType": "DQMHarvest", + "RunWhitelist": [], + "ScramArch": [ + "el8_amd64_gcc12" + ], + "SizePerEvent": 2, "TimePerEvent": 1 } } diff --git a/test/data/ReqMgr/requests/Integration/DQMHarvesting_LumiMask.json b/test/data/ReqMgr/requests/Integration/DQMHarvesting_LumiMask.json deleted file mode 100644 index 4b58282fe7..0000000000 --- a/test/data/ReqMgr/requests/Integration/DQMHarvesting_LumiMask.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "assignRequest": { - "AcquisitionEra": "AcquisitionEra-OVERRIDE-ME", - "Dashboard": "Dashboard-OVERRIDE-ME", - "GracePeriod": 300, - "MergedLFNBase": "/store/backfill/1", - "Override": {"eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED"}, - "ProcessingString": "ProcessingString-OVERRIDE-ME", - "ProcessingVersion": 19, - "SiteBlacklist": [], - "SiteWhitelist": [ - "SiteWhitelist-OVERRIDE-ME" - ], - "SoftTimeout": 129600, - "Team": "Team-OVERRIDE-ME", - "UnmergedLFNBase": "/store/unmerged" - }, - "createRequest": { - "AcquisitionEra": "Run2022D", - "CMSSWVersion": "CMSSW_12_4_11_patch3", - "Campaign": "Campaign-OVERRIDE-ME", - "Comments": {"WorkFlowDesc": ["Harvest 2 runs out of 69 (using LumiList). Only 357538 and 357550, total of 19 lumis"], - "CheckList": ["DQMHarvest with lumiMask input"]}, - "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", - "CouchDBName": "reqmgr_config_cache", - "DQMConfigCacheID": "d657f5aa83cffd6153ada2747a77b3d4", - "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", - "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", - "GlobalTag": "124X_dataRun3_v11", - "InputDataset": "/BTagMu/Run2022D-10Dec2022-v1/DQMIO", - "LumiList": {"357538": [[21, 26], [43, 45], [50, 50]], "357550": [[5, 10], [43, 45], []]}, - "Memory": 2000, - "Multicore": 1, - "PrepID": "TEST-ReReco-Run2022D-BTagMu-10Dec2022-00001", - "ProcessingString": "10Dec2022", - "ProcessingVersion": 1, - "RequestPriority": 999999, - "RequestString": "RequestString-OVERRIDE-ME", - "RequestType": "DQMHarvest", - "ScramArch": "el8_amd64_gcc10", - "SizePerEvent": 1600, - "TimePerEvent": 1 - } -} diff --git a/test/data/ReqMgr/requests/Integration/DQMHarvesting_LumiMask_MultiRun.json b/test/data/ReqMgr/requests/Integration/DQMHarvesting_LumiMask_MultiRun.json new file mode 100644 index 0000000000..c74f6e836d --- /dev/null +++ b/test/data/ReqMgr/requests/Integration/DQMHarvesting_LumiMask_MultiRun.json @@ -0,0 +1,49 @@ +{ + "assignRequest": { + "AcquisitionEra": "AcquisitionEra-OVERRIDE-ME", + "Dashboard": "Dashboard-OVERRIDE-ME", + "GracePeriod": 300, + "MergedLFNBase": "/store/backfill/1", + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" + }, + "ProcessingString": "ProcessingString-OVERRIDE-ME", + "ProcessingVersion": 19, + "SiteBlacklist": [], + "SiteWhitelist": [ + "SiteWhitelist-OVERRIDE-ME" + ], + "SoftTimeout": 129600, + "Team": "Team-OVERRIDE-ME", + "UnmergedLFNBase": "/store/unmerged" + }, + "createRequest": { + "AcquisitionEra": "Run2024D", + "CMSSWVersion": "CMSSW_14_0_19_patch2", + "Campaign": "Campaign-OVERRIDE-ME", + "Comments": {"WorkFlowDesc": ["Harvest 3 runs out of 47 (using LumiList), with a total of 36 lumis; Execute it in multiRun mode (single job)."], + "CheckList": ["DQMHarvest with lumiMask input", "DQMHarvest multirun"]}, + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "CouchDBName": "reqmgr_config_cache", + "DQMConfigCacheID": "7ab63e3a9a3e83c05ebdd4d4772004f3", + "DQMHarvestUnit": "multiRun", + "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", + "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "GlobalTag": "140X_dataRun3_v20", + "InputDataset": "/JetMET0/Run2024D-2024CDEReprocessing-v1/DQMIO", + "LumiList": {"380306": [[28, 35], [41, 50], [60, 60]], "380307": [[5, 10], [20, 21]], "380346": [[61, 70]]}, + "Memory": 2000, + "Multicore": 1, + "PrepID": "TEST-ReReco-Run2024D-JetMET0-2024CDEReprocessing-00001", + "ProcessingString": "2024CDEReprocessing", + "ProcessingVersion": 1, + "RequestPriority": 999999, + "RequestString": "RequestString-OVERRIDE-ME", + "RequestType": "DQMHarvest", + "ScramArch": [ + "el8_amd64_gcc12" + ], + "SizePerEvent": 2, + "TimePerEvent": 1 + } +} diff --git a/test/data/ReqMgr/requests/Integration/DQMHarvesting_MultiRun.json b/test/data/ReqMgr/requests/Integration/DQMHarvesting_MultiRun.json deleted file mode 100644 index aeb81eee20..0000000000 --- a/test/data/ReqMgr/requests/Integration/DQMHarvesting_MultiRun.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "assignRequest": { - "AcquisitionEra": "AcquisitionEra-OVERRIDE-ME", - "Dashboard": "Dashboard-OVERRIDE-ME", - "GracePeriod": 300, - "MergedLFNBase": "/store/backfill/1", - "Override": {"eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED"}, - "ProcessingString": "ProcessingString-OVERRIDE-ME", - "ProcessingVersion": 19, - "SiteBlacklist": [], - "SiteWhitelist": [ - "SiteWhitelist-OVERRIDE-ME" - ], - "SoftTimeout": 129600, - "Team": "Team-OVERRIDE-ME", - "UnmergedLFNBase": "/store/unmerged" - }, - "createRequest": { - "AcquisitionEra": "Run2022D", - "CMSSWVersion": "CMSSW_12_4_11_patch3", - "Campaign": "Campaign-OVERRIDE-ME", - "Comments": {"WorkFlowDesc": ["Harvest 4 runs out of 68 in a multiRun mode (single job)"], - "CheckList": ["DQMHarvest multirun"]}, - "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", - "CouchDBName": "reqmgr_config_cache", - "DQMConfigCacheID": "d657f5aa83cffd6153ada2747a77b3d4", - "DQMHarvestUnit": "multiRun", - "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", - "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", - "GlobalTag": "124X_dataRun3_v11", - "InputDataset": "/BTagMu/Run2022D-10Dec2022-v1/DQMIO", - "Memory": 2000, - "Multicore": 1, - "PrepID": "TEST-ReReco-Run2022D-BTagMu-10Dec2022-00001", - "ProcessingString": "10Dec2022", - "ProcessingVersion": 1, - "RequestPriority": 999999, - "RequestString": "RequestString-OVERRIDE-ME", - "RequestType": "DQMHarvest", - "RunWhitelist": [357538, 357539, 357540, 357550], - "ScramArch": "el8_amd64_gcc10", - "SizePerEvent": 1600, - "TimePerEvent": 1 - } -} diff --git a/test/data/ReqMgr/requests/Integration/ReReco_LumiMask.json b/test/data/ReqMgr/requests/Integration/ReReco_LumiMask.json index adbbe67b7a..c864be9a2b 100644 --- a/test/data/ReqMgr/requests/Integration/ReReco_LumiMask.json +++ b/test/data/ReqMgr/requests/Integration/ReReco_LumiMask.json @@ -18,35 +18,36 @@ "UnmergedLFNBase": "/store/unmerged" }, "createRequest": { - "AcquisitionEra": "Run2018D", - "CMSSWVersion": "CMSSW_10_6_12", + "AcquisitionEra": "Run2022C", + "CMSSWVersion": "CMSSW_12_4_14_patch1", "Campaign": "Campaign-OVERRIDE-ME", "Comments": { - "CheckList": "ReReco with LumiList; ReReco with growing dataset", - "WorkFlowDesc": ["ReReco NanoAOD with LumiList, 3 runs and 46 lumis, bringing in 2 blocks; 2cores and 2GB;", - "do not write logs to EOS; keep WorkQueue inbox open for 2h"] + "CheckList": "ReReco workflow; ReReco+skim workflow; ReReco with LumiList; ReReco with growing dataset; ReReco harvesting", + "WorkFlowDesc": ["ReReco reading 1 RAW block based on LumiList; 3 runs and 33 lumis; Creating Skim dataset", + "Open in running-open for 2 hours; harvest 3 runs; 4 cores and 10GB of RAM"] }, - "ConfigCacheID": "6d0b906879c210bde4baecd2e78ded41", + "ConfigCacheID": "3852d43544da0301ed0e04da715182b5", "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", "CouchDBName": "reqmgr_config_cache", "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader", - "GlobalTag": "106X_dataRun2_v28", - "InputDataset": "/NoBPTX/Run2018D-12Nov2019_UL2018-v1/MINIAOD", - "LumiList": {"320836": [[1, 10], [16, 20]], "322605": [[264, 275]], "324841": [[560, 565], [855, 860], [920, 926]]}, - "Memory": 2000, - "Multicore": 2, + "DQMConfigCacheID": "3852d43544da0301ed0e04da7152ca79", + "DQMUploadUrl": "https://cmsweb-testbed.cern.ch/dqm/dev", + "EnableHarvesting": true, + "GlobalTag": "124X_dataRun3_v15", + "InputDataset": "/MuonEG/Run2022C-v1/RAW", + "LumiList": {"356164": [[11, 15], [21, 30], [40, 40]], "356165": [[1, 10], [16, 20]], "356174": [[1, 2]]}, + "Memory": 10000, + "Multicore": 4, "OpenRunningTimeout": 7200, - "PrepID": "TEST-ReReco-Run2018D-NoBPTX-Nano02Dec2019-00001", - "ProcessingString": "Nano02Dec2019_12Nov2019_UL2018", + "PrepID": "TEST-ReReco-Run2022C-MuonEG-27Jun2023-00001", + "ProcessingString": "27Jun2023", "ProcessingVersion": 2, "RequestPriority": 600000, "RequestString": "RequestString-OVERRIDE-ME", "RequestType": "ReReco", - "ScramArch": [ - "slc7_amd64_gcc700" - ], + "ScramArch": "el8_amd64_gcc10", "SizePerEvent": 1, "SplittingAlgo": "EventAwareLumiBased", - "TimePerEvent": 0.2 + "TimePerEvent": 1 } } diff --git a/test/data/ReqMgr/requests/Integration/ReReco_Parents.json b/test/data/ReqMgr/requests/Integration/ReReco_Parents.json deleted file mode 100644 index 0b7667d069..0000000000 --- a/test/data/ReqMgr/requests/Integration/ReReco_Parents.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "assignRequest": { - "AcquisitionEra": "AcquisitionEra-OVERRIDE-ME", - "Dashboard": "Dashboard-OVERRIDE-ME", - "GracePeriod": 300, - "MergedLFNBase": "/store/backfill/1", - "Override": { - "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" - }, - "ProcessingString": "ProcessingString-OVERRIDE-ME", - "ProcessingVersion": 19, - "SiteBlacklist": [], - "SiteWhitelist": [ - "SiteWhitelist-OVERRIDE-ME" - ], - "SoftTimeout": 129600, - "Team": "Team-OVERRIDE-ME", - "UnmergedLFNBase": "/store/unmerged" - }, - "createRequest": { - "AcquisitionEra": "Run2018D", - "CMSSWVersion": "CMSSW_10_6_12", - "Campaign": "Campaign-OVERRIDE-ME", - "Comments": { - "CheckList": "ReReco with IncludeParents; ReReco with Run white list;", - "WorkFlowDesc": "ReReco NanoAOD with 6 runs whitelisted and include parents; bringing 2 parent blocks and 13 small MiniAOD blocks; 2cores and 2GB" - }, - "ConfigCacheID": "6d0b906879c210bde4baecd2e78ded41", - "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", - "CouchDBName": "reqmgr_config_cache", - "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader", - "GlobalTag": "106X_dataRun2_v28", - "IncludeParents": true, - "InputDataset": "/NoBPTX/Run2018D-12Nov2019_UL2018-v1/MINIAOD", - "Memory": 2000, - "Multicore": 2, - "PrepID": "TEST-ReReco-Run2018D-NoBPTX-Nano02Dec2019-00001", - "ProcessingString": "Nano02Dec2019_12Nov2019_UL2018", - "ProcessingVersion": 2, - "RequestPriority": 600000, - "RequestString": "RequestString-OVERRIDE-ME", - "RequestType": "ReReco", - "RunWhitelist": [ - 320809, - 320810, - 320830, - 324900, - 324940, - 324941 - ], - "ScramArch": [ - "slc7_amd64_gcc700" - ], - "SizePerEvent": 1, - "SplittingAlgo": "EventAwareLumiBased", - "TimePerEvent": 0.02 - } -} diff --git a/test/data/ReqMgr/requests/DMWM/ReReco_RunBlockWhite_Nvidia.json b/test/data/ReqMgr/requests/Integration/ReReco_RunBlockWhite_Nvidia.json similarity index 100% rename from test/data/ReqMgr/requests/DMWM/ReReco_RunBlockWhite_Nvidia.json rename to test/data/ReqMgr/requests/Integration/ReReco_RunBlockWhite_Nvidia.json diff --git a/test/data/ReqMgr/requests/DMWM/ReReco_badBlocks.json b/test/data/ReqMgr/requests/Integration/ReReco_badBlocks.json similarity index 100% rename from test/data/ReqMgr/requests/DMWM/ReReco_badBlocks.json rename to test/data/ReqMgr/requests/Integration/ReReco_badBlocks.json diff --git a/test/data/ReqMgr/requests/DMWM/ReReco_failCase.json b/test/data/ReqMgr/requests/Integration/ReReco_failCase.json similarity index 100% rename from test/data/ReqMgr/requests/DMWM/ReReco_failCase.json rename to test/data/ReqMgr/requests/Integration/ReReco_failCase.json diff --git a/test/data/ReqMgr/requests/Integration/SC_PU_5Steps.json b/test/data/ReqMgr/requests/Integration/SC_PU_5Steps.json new file mode 100644 index 0000000000..251aa8b477 --- /dev/null +++ b/test/data/ReqMgr/requests/Integration/SC_PU_5Steps.json @@ -0,0 +1,165 @@ +{ + "assignRequest": { + "AcquisitionEra": { + "EXO_Run3Summer22EEDRPremix_01310_0": "AcquisitionEra-OVERRIDE-ME", + "EXO_Run3Summer22EEDRPremix_01310_1": "AcquisitionEra-OVERRIDE-ME", + "EXO_Run3Summer22EEMiniAODv4_01191_0": "AcquisitionEra-OVERRIDE-ME", + "EXO_Run3Summer22EENanoAODv12_01191_0": "AcquisitionEra-OVERRIDE-ME", + "EXO_Run3Summer22EEwmLHEGS_01479_0": "AcquisitionEra-OVERRIDE-ME" + }, + "Dashboard": "Dashboard-OVERRIDE-ME", + "GracePeriod": 300, + "MergedLFNBase": "/store/backfill/1", + "Override": { + "eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/TESTBED" + }, + "ProcessingString": { + "EXO_Run3Summer22EEDRPremix_01310_0": "ProcessingString-OVERRIDE-ME", + "EXO_Run3Summer22EEDRPremix_01310_1": "ProcessingString-OVERRIDE-ME", + "EXO_Run3Summer22EEMiniAODv4_01191_0": "ProcessingString-OVERRIDE-ME", + "EXO_Run3Summer22EENanoAODv12_01191_0": "ProcessingString-OVERRIDE-ME", + "EXO_Run3Summer22EEwmLHEGS_01479_0": "ProcessingString-OVERRIDE-ME" + }, + "ProcessingVersion": 19, + "SiteBlacklist": [], + "SiteWhitelist": [ + "SiteWhitelist-OVERRIDE-ME" + ], + "SoftTimeout": 129600, + "Team": "Team-OVERRIDE-ME", + "TrustPUSitelists": true, + "UnmergedLFNBase": "/store/unmerged" + }, + "createRequest": { + "AcquisitionEra": "DEFAULT_AcqEra", + "CMSSWVersion": "CMSSW_13_0_13", + "Campaign": "Campaign-OVERRIDE-ME", + "Comments": { + "CheckList": "From cmsunified_task_EXO-Run3Summer22EEwmLHEGS-01479__v1_T_240206_120800_9709; Secondary AAA enabled", + "WorkFlowDesc": "SC with 5 steps from scratch; Reading PREMIX PU; KeepOutput of steps 3/4/5; Multiple campaign, prepid and CMSSW; 8 cores 16GB" + }, + "ConfigCacheUrl": "https://cmsweb.cern.ch/couchdb", + "CouchDBName": "reqmgr_config_cache", + "DQMUploadUrl": "https://cmsweb.cern.ch/dqm/dev", + "DbsUrl": "https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader/", + "DeterministicPileup": false, + "EnableHarvesting": false, + "EventStreams": 0, + "GPUParams": "null", + "GlobalTag": "130X_mcRun3_2022_realistic_postEE_v6", + "IncludeParents": false, + "Memory": 16000, + "Multicore": 8, + "PrepID": "TEST-task_EXO-Run3Summer22EEwmLHEGS-01479", + "ProcessingString": "DEFAULT_ProcStr", + "ProcessingVersion": 2, + "RequestPriority": 190017, + "RequestString": "RequestString-OVERRIDE-ME", + "RequestType": "StepChain", + "ScramArch": [ + "el8_amd64_gcc11" + ], + "SizePerEvent": 2556.8795242428296, + "Step1": { + "AcquisitionEra": "Run3Summer22EEwmLHEGS", + "CMSSWVersion": "CMSSW_12_4_19", + "Campaign": "Run3Summer22EEwmLHEGS", + "ConfigCacheID": "99f7b36120327e13f1d36e86f6891c0e", + "FilterEfficiency": 0.38005390835579517, + "GlobalTag": "124X_mcRun3_2022_realistic_postEE_v1", + "KeepOutput": false, + "LheInputFiles": false, + "PrepID": "EXO-Run3Summer22EEwmLHEGS-01479", + "PrimaryDataset": "RPVStopStopToJets_UDD323_M-2000_TuneCP5_13p6TeV-madgraphMLM-pythia8", + "ProcessingString": "Step1_WMCore_TEST", + "RequestNumEvents": 1500000, + "ScramArch": [ + "el8_amd64_gcc10" + ], + "Seeding": "AutomaticSeeding", + "SplittingAlgo": "EventBased", + "StepName": "EXO_Run3Summer22EEwmLHEGS_01479_0" + }, + "Step2": { + "AcquisitionEra": "Run3Summer22EEDRPremix", + "CMSSWVersion": "CMSSW_12_4_16", + "Campaign": "Run3Summer22EEDRPremix", + "ConfigCacheID": "db3b77054b46fa573d592e93f9d182f8", + "FilterEfficiency": 1, + "GlobalTag": "124X_mcRun3_2022_realistic_postEE_v1", + "InputFromOutputModule": "RAWSIMoutput", + "InputStep": "EXO_Run3Summer22EEwmLHEGS_01479_0", + "KeepOutput": false, + "MCPileup": "/Neutrino_E-10_gun/Run3Summer21PrePremix-Summer22_124X_mcRun3_2022_realistic_v11-v2/PREMIX", + "PrepID": "EXO-Run3Summer22EEDRPremix-01310", + "PrimaryDataset": "RPVStopStopToJets_UDD323_M-2000_TuneCP5_13p6TeV-madgraphMLM-pythia8", + "ProcessingString": "Step2_WMCore_TEST", + "ScramArch": [ + "el8_amd64_gcc10" + ], + "SplittingAlgo": "EventAwareLumiBased", + "StepName": "EXO_Run3Summer22EEDRPremix_01310_0" + }, + "Step3": { + "AcquisitionEra": "Run3Summer22EEDRPremix", + "CMSSWVersion": "CMSSW_12_4_16", + "Campaign": "Run3Summer22EEDRPremix", + "ConfigCacheID": "db3b77054b46fa573d592e93f9d183ef", + "FilterEfficiency": 1, + "GlobalTag": "124X_mcRun3_2022_realistic_postEE_v1", + "InputFromOutputModule": "PREMIXRAWoutput", + "InputStep": "EXO_Run3Summer22EEDRPremix_01310_0", + "KeepOutput": true, + "MCPileup": "/Neutrino_E-10_gun/Run3Summer21PrePremix-Summer22_124X_mcRun3_2022_realistic_v11-v2/PREMIX", + "PrepID": "EXO-Run3Summer22EEDRPremix-01310", + "PrimaryDataset": "RPVStopStopToJets_UDD323_M-2000_TuneCP5_13p6TeV-madgraphMLM-pythia8", + "ProcessingString": "Step3_WMCore_TEST", + "ScramArch": [ + "el8_amd64_gcc10" + ], + "SplittingAlgo": "EventAwareLumiBased", + "StepName": "EXO_Run3Summer22EEDRPremix_01310_1" + }, + "Step4": { + "AcquisitionEra": "Run3Summer22EEMiniAODv4", + "CMSSWVersion": "CMSSW_13_0_13", + "Campaign": "Run3Summer22EEMiniAODv4", + "ConfigCacheID": "2302da479f718297210b244fba1c8529", + "FilterEfficiency": 1, + "GlobalTag": "130X_mcRun3_2022_realistic_postEE_v6", + "InputFromOutputModule": "AODSIMoutput", + "InputStep": "EXO_Run3Summer22EEDRPremix_01310_1", + "KeepOutput": true, + "PrepID": "EXO-Run3Summer22EEMiniAODv4-01191", + "PrimaryDataset": "RPVStopStopToJets_UDD323_M-2000_TuneCP5_13p6TeV-madgraphMLM-pythia8", + "ProcessingString": "Step4_WMCore_TEST", + "ScramArch": [ + "el8_amd64_gcc11" + ], + "SplittingAlgo": "EventAwareLumiBased", + "StepName": "EXO_Run3Summer22EEMiniAODv4_01191_0" + }, + "Step5": { + "AcquisitionEra": "Run3Summer22EENanoAODv12", + "CMSSWVersion": "CMSSW_13_0_13", + "Campaign": "Run3Summer22EENanoAODv12", + "ConfigCacheID": "6f0058ffc581f8f0032aece2361c66b2", + "FilterEfficiency": 1, + "GlobalTag": "130X_mcRun3_2022_realistic_postEE_v6", + "InputFromOutputModule": "MINIAODSIMoutput", + "InputStep": "EXO_Run3Summer22EEMiniAODv4_01191_0", + "KeepOutput": true, + "PrepID": "EXO-Run3Summer22EENanoAODv12-01191", + "PrimaryDataset": "RPVStopStopToJets_UDD323_M-2000_TuneCP5_13p6TeV-madgraphMLM-pythia8", + "ProcessingString": "Step5_WMCore_TEST", + "ScramArch": [ + "el8_amd64_gcc11" + ], + "SplittingAlgo": "EventAwareLumiBased", + "StepName": "EXO_Run3Summer22EENanoAODv12_01191_0" + }, + "StepChain": 5, + "SubRequestType": "ReDigi", + "TimePerEvent": 17.548029956077905 + } +} diff --git a/test/data/WMCore/MicroService/DataStructs/pileups_dev.json b/test/data/WMCore/MicroService/DataStructs/pileups_dev.json index 64449495c7..f279964da1 100644 --- a/test/data/WMCore/MicroService/DataStructs/pileups_dev.json +++ b/test/data/WMCore/MicroService/DataStructs/pileups_dev.json @@ -38,5 +38,13 @@ "expectedRSEs": ["T2_CH_CERN"], "campaigns": ["Apr2023_Val"], "active": true + }, + { + "pileupName": "/Neutrino_E-10_gun/Run3Summer21PrePremix-Summer22_124X_mcRun3_2022_realistic_v11-v2/PREMIX", + "customName": "", + "pileupType": "premix", + "expectedRSEs": ["T1_US_FNAL_Disk", "T2_CH_CERN"], + "campaigns": ["Apr2023_Val"], + "active": true } ]