Merge branch 'develop' into release-7.1.2

Author: mesmith75

ganga/GangaDirac/Lib/Backends/DiracBase.py

@@ -700,11 +700,29 @@ def removeOutputData(self):
         # Note when the API can accept a list for removeFile I will change
         # this.
         j = self.getJobObject()
+        logger.info("Removing all DiracFile output for job %s" % j.id)
+        lfnsToRemove = []
         if j.subjobs:
             for sj in j.subjobs:
-                outputfiles_foreach(sj, DiracFile, lambda x: x.remove())
+                outputfiles_foreach(sj, DiracFile, lambda x: lfnsToRemove.append(x.lfn))
         else:
-            outputfiles_foreach(j, DiracFile, lambda x: x.remove())
+            outputfiles_foreach(j, DiracFile, lambda x: lfnsToRemove.append(x.lfn))
+        dirac_cmd = "removeFile(%s)" % lfnsToRemove
+        try:
+            result = execute(dirac_cmd, cred_req=self.credential_requirements)
+        except GangaDiracError as err:
+            msg = 'Problem removing files: %s' % str(err)
+            logger.warning(msg)
+            return False
+        def clearFileInfo(f):
+            f.lfn = ""
+            f.locations = []
+            f.guid = ''
+        if j.subjobs:
+            for sj in j.subjobs:
+                outputfiles_foreach(sj, DiracFile, lambda x: clearFileInfo(x))
+        else:
+            outputfiles_foreach(j, DiracFile, lambda x: clearFileInfo(x))
 
     def getOutputData(self, outputDir=None, names=None, force=False):
         """Retrieve data stored on SE to dir (default=job output workspace).

ganga/GangaDirac/test/Unit/Backends/TestDiracBase.py

@@ -300,7 +300,7 @@ def test_removeOutputData(db):
 
     class TestFile(object):
         def __init__(self):
-            pass
+            self.lfn = 27
 
         def remove(self):
             return 27
@@ -316,18 +316,18 @@ def fake_outputfiles_foreach(job, file_type, func):
             assert job.master is None
             assert file_type == DiracFile
             assert isinstance(func, types.FunctionType)
-            assert func(TestFile()) == 27, 'Didn\'t call remove function'
 
     with patch('GangaDirac.Lib.Backends.DiracBase.outputfiles_foreach', fake_outputfiles_foreach):
-        subjob = False
-        assert db.removeOutputData() is None
+        with patch('GangaDirac.Lib.Backends.DiracBase.execute', return_value=True):
+            subjob = False
+            assert db.removeOutputData() is None
 
-        j.subjobs = [Job(), Job(), Job()]
-        for sj in j.subjobs:
-            sj._setParent(j)
+            j.subjobs = [Job(), Job(), Job()]
+            for sj in j.subjobs:
+                sj._setParent(j)
 
-        subjob = True
-        assert db.removeOutputData() is None
+            subjob = True
+            assert db.removeOutputData() is None
 
 
 def test_getOutputData(db, tmpdir):