backporting #12488, #12485, #12490, #12478 to 2.4.5. wmagent

.github/workflows/pypi_build_and_images.yaml

@@ -52,11 +52,13 @@ jobs:
           sudo apt update
           sudo apt install -y curl libcurl4-openssl-dev
 
-      # upgrade pip
+      # upgrade pip and set setuptools constraint for build isolation
       - name: Upgrade pip3 and install build tools
         run: |
           python3 -m pip install --upgrade pip
-          python3 -m pip install "setuptools>=45,<82" wheel
+          echo 'setuptools<80' > /tmp/constraints.txt
+          echo "PIP_CONSTRAINT=/tmp/constraints.txt" >> "$GITHUB_ENV"
+          python3 -m pip install setuptools wheel
 
       # create appropriate setup.py for python builds
       - name: Update the setup script template with package name

src/python/WMComponent/JobSubmitter/JobSubmitterPoller.py

@@ -639,6 +639,11 @@ def _getJobSubmitCondition(self, jobPrio, siteName, jobType):
             totalJobThreshold = totalPendingThreshold + totalRunningSlots
             totalTaskTheshold = taskPendingThreshold + taskRunningSlots
 
+        # Bypass NoPendingSlot limit for job type (Merge)
+        # These jobs should not be blocked by pending slot limits
+        if jobType == 'Merge':
+            totalPendingThreshold = float('inf')
+
         jobStats = [{"Condition": "NoPendingSlot",
                      "Current": totalPendingJobs,
                      "Threshold": totalPendingThreshold},

src/python/WMCore/WMSpec/StdSpecs/StdBase.py

@@ -711,19 +711,21 @@ def addMergeTask(self, parentTask, parentTaskSplitting, parentOutputModuleName,
                                         max_wait_time=self.maxWaitTime,
                                         initial_lfn_counter=lfn_counter)
 
+        primaryDataset = getattr(parentOutputModule, "primaryDataset")
         if dataTier == "DQMIO":
             mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge",
                                                          newDQMIO=True)
         elif dataTier in ("NANOAOD", "NANOAODSIM"):
             mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge",
-                                                         mergeNANO=True)
+                                                     mergeNANO=True,
+                                                     isL1Scouting=(primaryDataset.startswith("L1Scouting")))
         else:
             mergeTaskCmsswHelper.setDataProcessingConfig("do_not_use", "merge")
 
         mergeTaskStageHelper.setMinMergeSize(0, 0)
 
         self.addOutputModule(mergeTask, "Merged",
-                             primaryDataset=getattr(parentOutputModule, "primaryDataset"),
+                             primaryDataset=primaryDataset,
                              dataTier=getattr(parentOutputModule, "dataTier"),
                              filterName=getattr(parentOutputModule, "filterName"),
                              forceMerged=True, taskConf=taskConf)

src/python/WMCore/WMSpec/Steps/Executors/AlcaHarvest.py

@@ -126,7 +126,7 @@ def execute(self, emulator=None):
                     logging.info("==> copy %s", file2copy)
 
                     targetLFN = os.path.join(self.step.condition.lfnbase, file2copy)
-                    targetPFN = "root://eoscms//eos/cms%s" % targetLFN
+                    targetPFN = "root://eoscms.cern.ch//eos/cms%s" % targetLFN
 
                     command = "env XRD_WRITERECOVERY=0 xrdcp -s -f %s %s" % (file2copy, targetPFN)
 

test/python/WMCore_t/Storage_t/StageOutImpl_t.py

@@ -16,28 +16,28 @@ def setUp(self):
     def testSplitPFN_noOpaque(self):
         self.assertEqual(("splitable/test/test1", "test1", "splitable/test/test1", ""),
                          StageOutImpl.splitPFN("splitable/test/test1"))
-        self.assertEqual(('//eoscms//eos/cms/store/', 'eoscms', '//eoscms//eos/cms/store/', ""),
-                         StageOutImpl.splitPFN("//eoscms//eos/cms/store/"))
+        self.assertEqual(('//eoscms.cern.ch//eos/cms/store/', 'eoscms.cern.ch', '//eoscms.cern.ch//eos/cms/store/', ""),
+                         StageOutImpl.splitPFN("//eoscms.cern.ch//eos/cms/store/"))
 
     def testSplitPFN_doubleSlashRoot(self):
-        self.assertEqual(('root', 'eoscms', '/eos/cms/store/', ""),
-                         StageOutImpl.splitPFN("root://eoscms//eos/cms/store/"))
+        self.assertEqual(('root', 'eoscms.cern.ch', '/eos/cms/store/', ""),
+                         StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store/"))
 
     def testSplitPFN_path(self):
-        self.assertEqual(('root', 'eoscms', 'default', "?"),
-                         StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default"))
+        self.assertEqual(('root', 'eoscms.cern.ch', 'default', "?"),
+                         StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store?path=default"))
 
     def testSplitPFN_pathConnector(self):
-        self.assertEqual(('root', 'eoscms', 'default', "?default2"),
-                         StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default&default2"))
+        self.assertEqual(('root', 'eoscms.cern.ch', 'default', "?default2"),
+                         StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store?path=default&default2"))
 
     def testSplitPFN_path2(self):
-        self.assertEqual(('root', 'eoscms', 'cms', "?cms2ss&path=default"),
-                         StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&cms2ss&path=default"))
+        self.assertEqual(('root', 'eoscms.cern.ch', 'cms', "?cms2ss&path=default"),
+                         StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store?path=cms&cms2ss&path=default"))
 
     def testSplitPFN_pathConnector2(self):
-        self.assertEqual(('root', 'eoscms', 'cms', "?path=default&default2"),
-                         StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&path=default&default2"))
+        self.assertEqual(('root', 'eoscms.cern.ch', 'cms', "?path=default&default2"),
+                         StageOutImpl.splitPFN("root://eoscms.cern.ch//eos/cms/store?path=cms&path=default&default2"))
 
     @mock.patch('WMCore.Storage.StageOutImpl.runCommandWithOutput')
     def testExecuteCommand_stageOutError(self, mock_runCommand):