Automatic cleaning of input data Rucio rules

src/python/WMQuality/Emulators/RucioClient/MockRucioApi.py

@@ -51,6 +51,7 @@ def __init__(self, acct, hostUrl=None, authUrl=None, configDict=None):
         self.dataBlocks = DataBlockGenerator()
         self.subRequests = {}
         self.rucioParams = {}
+        self._mockRules = {}  # ruleId -> rule dict, used by getRule/listDataRules/deleteRule
 
     def sitesByBlock(self, block):
         """
@@ -131,13 +132,55 @@ def attachDIDs(self, rse, superDID, portion, scope='cms'):
         logging.info("%s attachDID rse=%s, suportDID=%s, portion=%s, scope=%s", cname, rse, superDID, portion, scope)
         return True
 
-    def createReplicationRule(self, portion, rseExpression, scope='cms'):
+    def createReplicationRule(self, names, rseExpression, scope='cms', copies=1, **kwargs):
         """
-        Emulate createReplicationRule Rucio API
+        Emulate createReplicationRule Rucio API — stores the rule in _mockRules and returns a fake rule id list.
         """
+        import hashlib
         cname = self.__class__.__name__
-        logging.info("%s createReplicationRule portion=%s, rseExpression=%s", cname, portion, rseExpression)
-        return [rseExpression]
+        logging.info("%s createReplicationRule names=%s, rseExpression=%s", cname, names, rseExpression)
+        ruleId = hashlib.md5("{}{}".format(names, rseExpression).encode()).hexdigest()
+        self._mockRules[ruleId] = {
+            'id': ruleId,
+            'name': names,
+            'rse_expression': rseExpression,
+            'scope': scope,
+            'copies': copies,
+            'account': kwargs.get('account', 'mock'),
+            'state': 'OK',
+            'bytes': 0,
+        }
+        return [ruleId]
+
+    def getRule(self, ruleId):
+        """
+        Emulate getRule Rucio API — return a minimal rule dict for an existing mock rule id.
+        """
+        cname = self.__class__.__name__
+        logging.info("%s getRule ruleId=%s", cname, ruleId)
+        # Mock rule storage: non-empty dict means the rule exists
+        if ruleId in self._mockRules:
+            return self._mockRules[ruleId]
+        return {}
+
+    def listDataRules(self, name, **kwargs):
+        """
+        Emulate listDataRules Rucio API — return mock rules for the given DID.
+        """
+        cname = self.__class__.__name__
+        logging.info("%s listDataRules name=%s kwargs=%s", cname, name, kwargs)
+        account = kwargs.get('account', None)
+        return [rule for rule in self._mockRules.values()
+                if rule.get('name') == name and (account is None or rule.get('account') == account)]
+
+    def deleteRule(self, ruleId, purgeReplicas=False):
+        """
+        Emulate deleteRule Rucio API — remove the rule from the mock store.
+        """
+        cname = self.__class__.__name__
+        logging.info("%s deleteRule ruleId=%s", cname, ruleId)
+        self._mockRules.pop(ruleId, None)
+        return True
 
     def updateRule(self, rid, opts):
         """

src/python/WMQuality/TestInitCouchApp.py

@@ -25,6 +25,7 @@
 
 from WMQuality.TestInit import TestInit
 
+from WMCore.WMBase import getWMBASE
 
 class CouchAppTestHarness(object):
     """
@@ -111,8 +112,10 @@ def couchAppRoot(self, couchapp):
 
         :param couchapp: couch application to use
         """
-        wmcoreroot = os.path.normpath(os.path.join(self.init.getWMBASE(), '..', '..', '..'))
-        develPath = os.path.join(self.init.getWMBASE(), "src", "couchapps")
+        #wmcoreroot = os.path.normpath(os.path.join(self.init.getWMBASE(), '..', '..', '..'))
+        #develPath = os.path.join(self.init.getWMBASE(), "src", "couchapps")
+        wmcoreroot = os.path.normpath(os.path.join(getWMBASE(), '..', '..', '..'))
+        develPath = os.path.join(getWMBASE(), "src", "couchapps")
         if os.path.exists(os.path.join(develPath, couchapp)):
             return develPath
         elif os.path.exists(os.path.join(wmcoreroot, 'xdata', 'couchapps', couchapp)):

test/data/Mock/DBSMockData.json

@@ -13793,6 +13793,11 @@
    "dataset": "/MinimumBias/ComissioningHI-v1/RAW"
   }
  ],
+ "listDatasets:[('dataset', '/JetHT/Run2012C-v1/RAW'), ('dataset_access_type', '*')]": [
+    {
+     "dataset": "/JetHT/Run2012C-v1/RAW"
+    }
+ ],
  "listDatasets:[('dataset', '/MinimumBias/FAKE-Filter-v1/RECO'), ('dataset_access_type', '*')]": [],
  "listDatasets:[('dataset', '/NoBPTX/Run2016F-23Sep2016-v1/DQMIO'), ('dataset_access_type', '*')]": [
   {
@@ -25424,6 +25429,14 @@
    "md5": "NOTSET"
   }
  ],
+ "listBlocks:[('dataset', '/JetHT/Run2012C-v1/RAW'), ('detail', False)]": [
+    {
+     "block_name": "/JetHT/Run2012C-v1/RAW#001975f4-e9fa-11e1-9597-842b2b4671d8"
+    },
+    {
+     "block_name": "/JetHT/Run2012C-v1/RAW#004e510c-07fa-11e2-ad47-842b2b4671d8"
+    }
+ ],
  "listFileArray:[('dataset', '/Cosmics/ComissioningHI-v1/RAW'), ('detail', False), ('validFileOnly', 1)]": [
   {
    "logical_file_name": "/store/data/ComissioningHI/Cosmics/RAW/v1/000/180/855/B452861C-1108-E111-B809-BCAEC53296FE.root"
@@ -545728,6 +545741,30 @@
    "num_lumi": 51879
   }
  ],
+ "listFileSummaries:[('block_name', '/JetHT/Run2012C-v1/RAW#001975f4-e9fa-11e1-9597-842b2b4671d8'), ('validFileOnly', 1)]": [
+    {
+     "file_size": 19829448550,
+     "max_ldate": 1321139371,
+     "median_cdate": 1321135779,
+     "median_ldate": 1321139371,
+     "num_block": 1,
+     "num_event": 14558,
+     "num_file": 5,
+     "num_lumi": 22
+    }
+ ],
+ "listFileSummaries:[('block_name', '/JetHT/Run2012C-v1/RAW#004e510c-07fa-11e2-ad47-842b2b4671d8'), ('validFileOnly', 1)]": [
+    {
+     "file_size": 6879134,
+     "max_ldate": 1321139371,
+     "median_cdate": 1321135779,
+     "median_ldate": 1321139371,
+     "num_block": 1,
+     "num_event": 14558,
+     "num_file": 1,
+     "num_lumi": 4
+    }
+ ],
  "listFiles:[('block_name', '/Cosmics/ComissioningHI-PromptReco-v1/RECO#7020873e-0dcd-11e1-9b6c-003048caaace')]": [
   {
    "logical_file_name": "/store/data/ComissioningHI/Cosmics/RECO/PromptReco-v1/000/181/369/F2189593-CC0D-E111-8C3C-BCAEC518FF89.root"

test/data/Mock/RucioMockData.json

@@ -1597,6 +1597,28 @@
   "scope": "cms",
   "type": "CONTAINER"
  },
+ "getDID:[('didName', '/JetHT/Run2012C-v1/RAW#001975f4-e9fa-11e1-9597-842b2b4671d8'), ('dynamic', False)]": {
+  "account": "sync_t0_ch_cern_tape",
+  "bytes": 19829448550,
+  "expired_at": null,
+  "length": 1,
+  "monotonic": false,
+  "name": "/JetHT/Run2012C-v1/RAW#001975f4-e9fa-11e1-9597-842b2b4671d8",
+  "open": true,
+  "scope": "cms",
+  "type": "DATASET"
+ },
+ "getDID:[('didName', '/JetHT/Run2012C-v1/RAW#004e510c-07fa-11e2-ad47-842b2b4671d8'), ('dynamic', False)]": {
+   "account": "sync_t0_ch_cern_tape",
+   "bytes": 6879134,
+   "expired_at": null,
+   "length": 1,
+   "monotonic": false,
+   "name": "/JetHT/Run2012C-v1/RAW#004e510c-07fa-11e2-ad47-842b2b4671d8",
+   "open": true,
+   "scope": "cms",
+   "type": "DATASET"
+ },
  "listDataRules:[('account', 'ms-pileup'), ('scope', 'cms')]": [{
   "state": "OK",
   "id": "123",

test/deploy/env_unittest_py3.sh

@@ -33,11 +33,14 @@ ln -s $TEST_SRC/javascript/ $ORG_SRC_OTHER
 ln -s $TEST_SRC/template/ $ORG_SRC_OTHER
 
 export WMAGENT_SECRETS_LOCATION=$ADMIN_DIR/WMAgent_unittest.secrets
-export X509_HOST_CERT=$CERT_DIR/servicecert.pem
-export X509_HOST_KEY=$CERT_DIR/servicekey.pem
-export X509_USER_CERT=$CERT_DIR/servicecert.pem
-export X509_USER_KEY=$CERT_DIR/servicekey.pem
-
+#export X509_HOST_CERT=$CERT_DIR/servicecert.pem
+#export X509_HOST_KEY=$CERT_DIR/servicekey.pem
+#export X509_USER_CERT=$CERT_DIR/servicecert.pem
+#export X509_USER_KEY=$CERT_DIR/servicekey.pem
+export X509_HOST_CERT=$CERT_DIR/usercert.pem
+export X509_HOST_KEY=$CERT_DIR/userkey_nopwd.pem
+export X509_USER_CERT=$CERT_DIR/usercert.pem
+export X509_USER_KEY=$CERT_DIR/userkey_nopwd.pem
 export install=$INSTALL_DIR/current/install/wmagentpy3
 export config=$INSTALL_DIR/current/config/wmagentpy3
 export manage=$config/manage