HIVE-29522: Prevent cleaner from creating COMPLETED_COMPACTIONS entries for soft-deleted ACID tables

Author: Venu Reddy

ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java

@@ -158,6 +158,7 @@ private void clean(CompactionInfo ci, long minOpenTxn, boolean metricsEnabled) t
           cleanUsingAcidDir(ci, t, path, cleanerWaterMark);
         }
       } else {
+        ci.setSoftDelete(true);
         cleanUsingLocation(ci, location, false);
       }
     } catch (Exception e) {

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfo.java

@@ -69,6 +69,7 @@ public class CompactionInfo implements Comparable<CompactionInfo> {
   public int numberOfBuckets = 0;
   public String orderByClause;
   public long minOpenWriteTxnId = 0;
+  public boolean softDelete;
 
   /**
    * The highest write id that the compaction job will pay attention to.
@@ -190,6 +191,7 @@ public String toString() {
         .append("numberOfBuckets", numberOfBuckets)
         .append("orderByClause", orderByClause)
         .append("minOpenWriteTxnId", minOpenWriteTxnId)
+        .append("softDelete", softDelete)
         .build();
   }
 
@@ -368,4 +370,12 @@ public void setWriteIds(boolean hasUncompactedAborts, Set<Long> writeIds) {
   public boolean isAbortedTxnCleanup() {
     return type == CompactionType.ABORT_TXN_CLEANUP;
   }
+
+  public void setSoftDelete(boolean softDelete) {
+    this.softDelete = softDelete;
+  }
+
+  public boolean isSoftDelete() {
+    return softDelete;
+  }
 }

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/functions/MarkCleanedFunction.java

@@ -50,7 +50,7 @@ public MarkCleanedFunction(CompactionInfo info) {
   public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaException {
     NamedParameterJdbcTemplate jdbcTemplate = jdbcResource.getJdbcTemplate();
     MapSqlParameterSource param;
-    if (!info.isAbortedTxnCleanup()) {
+    if (!info.isAbortedTxnCleanup() && !info.isSoftDelete()) {
       param = new MapSqlParameterSource()
           .addValue("id", info.id)
           .addValue("succeeded", Character.toString(SUCCEEDED_STATE), Types.CHAR);
@@ -85,6 +85,10 @@ public Void execute(MultiDataSourceJdbcResource jdbcResource) throws MetaExcepti
      */
     removeCompactionAndAbortRetryEntries(info, jdbcTemplate);
 
+    if (info.isSoftDelete()) {
+      return null;
+    }
+
     if (!info.isAbortedTxnCleanup()) {
       // Remove entries from completed_txn_components as well, so we don't start looking there
       // again but only up to the highest write ID include in this compaction job.
@@ -175,7 +179,7 @@ private void removeCompactionAndAbortRetryEntries(CompactionInfo info, NamedPara
     String deleteQuery = """
         DELETE FROM "COMPACTION_QUEUE" WHERE "CQ_ID" = :id
         """;
-    if (!info.isAbortedTxnCleanup()) {
+    if (!info.isAbortedTxnCleanup() && !info.isSoftDelete()) {
       deleteQuery += """
           OR ("CQ_DATABASE" = :db AND "CQ_TABLE" = :table
             AND (:partition is NULL OR "CQ_PARTITION" = :partition)