test(sync): stabilize file deletion in some of the tests (#4197)

See failure in https://github.com/project-zot/zot/actions/runs/28843928068/job/85548725800?pr=4185


(cherry picked from commit 71beab9)

Signed-off-by: Andrei Aaron <andreifdaaron@gmail.com>

Author: andaaron

pkg/extensions/sync/sync_test.go

@@ -1399,6 +1399,22 @@ func TestSyncWithNonDistributableBlob(t *testing.T) {
 	})
 }
 
+// removeAllWithRetry retries os.RemoveAll until it succeeds or the timeout expires.
+// This is primarily to tolerate brief ENOTEMPTY races when background sync/meta
+// tasks momentarily re-create entries under repoDir while RemoveAll is deleting it,
+// making rmdir observe a non-empty directory.
+func removeAllWithRetry(repoDir string, timeout time.Duration) error {
+	var err error
+
+	for deadline := time.Now().Add(timeout); ; {
+		if err = os.RemoveAll(repoDir); err == nil || time.Now().After(deadline) {
+			return err
+		}
+
+		time.Sleep(50 * time.Millisecond)
+	}
+}
+
 func TestDockerImagesAreSkipped(t *testing.T) {
 	testCases := []struct {
 		name           string
@@ -1550,7 +1566,7 @@ func TestDockerImagesAreSkipped(t *testing.T) {
 
 				// trigger config blob upstream error
 				// remove synced image
-				err = os.RemoveAll(path.Join(destDir, testImage))
+				err = removeAllWithRetry(path.Join(destDir, testImage), 10*time.Second)
 				So(err, ShouldBeNil)
 
 				configBlobPath := path.Join(srcDir, testImage, "blobs/sha256", configBlobDigest.Encoded())
@@ -1720,7 +1736,7 @@ func TestDockerImagesAreSkipped(t *testing.T) {
 
 				// trigger config blob upstream error
 				// remove synced image
-				err = os.RemoveAll(path.Join(destDir, indexRepoName))
+				err = removeAllWithRetry(path.Join(destDir, indexRepoName), 10*time.Second)
 				So(err, ShouldBeNil)
 
 				configBlobPath := path.Join(srcDir, indexRepoName, "blobs/sha256", configBlobDigest.Encoded())