chunk validation tasks

Author: Fallen-Breath

lang/en_us.yml

@@ -310,6 +310,7 @@ prime_backup:
         missing: 'Missing blob amount: {}'
         corrupted: 'Corrupted blob amount: {}'
         mismatched: 'Mismatched blob amount: {}'
+        bad_layout: 'Bad-layout blob amount: {}'
         orphan: 'Orphan blob amount: {}'
         affected: 'Affected range: {} / {} file objects, {} / {} filesets, {} / {} backups'
         see_log: 'See log file {} for details and affected stuffs of these bad blobs'
@@ -537,6 +538,8 @@ prime_backup:
     blob_hash_not_unique.candidates: 'Found at least {} candidates: {}'
     chunk_id_not_found: Chunk with id {} does not exist
     chunk_hash_not_found: Chunk with hash {} does not exist
+    chunk_group_id_not_found: Chunk group with id {} does not exist
+    chunk_group_hash_not_found: Chunk group with hash {} does not exist
     db_locked: Task {} execution error cuz the database is locked. Please retry later
     initializing: Plugin initializing
     disabled: Plugin disabled

lang/zh_cn.yml

@@ -310,6 +310,7 @@ prime_backup:
         missing: '文件缺失的数据对象: {}个'
         corrupted: '文件损坏的数据对象: {}个'
         mismatched: '信息不匹配的数据对象: {}个'
+        bad_layout: '分块布局异常的数据对象: {}个'
         orphan: '孤儿数据对象: {}个'
         affected: '影响范围: {}/{}个文件对象, {}/{}个文件集, {}/{}个备份'
         see_log: '见日志文件 {} 以了解这些数据对象的详细信息及影响范围'
@@ -537,6 +538,8 @@ prime_backup:
     blob_hash_not_unique.candidates: '找到了至少{}个可能的数据对象: {}'
     chunk_id_not_found: ID为{}的数据块不存在
     chunk_hash_not_found: 哈希值为{}的数据块不存在
+    chunk_group_id_not_found: ID为{}的数据块组不存在
+    chunk_group_hash_not_found: 哈希值为{}的数据块组不存在
     db_locked: 任务{}执行失败, 数据库已被锁定。请稍候再试
     initializing: 插件初始化中
     disabled: 插件已禁用

prime_backup/action/scan_unknown_blob_files.py

@@ -57,10 +57,13 @@ def run(self) -> ScanUnknownBlobFilesResult:
 						count += 1
 						size = unknown_blob_file.stat().st_size
 						size_sum += size
-						self.logger.debug('Found unknown blob at {} with size {}, deleting'.format(unknown_blob_file, size))
 
+						self.logger.debug('Found unknown blob file at {} with size {}{}'.format(unknown_blob_file, size, ', deleting' if self.delete else ''))
+						if self.delete:
+							unknown_chunk_file.unlink(missing_ok=True)
 						if self.delete:
 							unknown_blob_file.unlink(missing_ok=True)
+
 						if len(unknown_blob_file_samples) < 5:
 							unknown_blob_file_samples.append(str(unknown_blob_file))
 						if self.result_sample_limit is not None and len(result_files) < self.result_sample_limit:

prime_backup/action/validate_blob_chunk_group_bindings_action.py

@@ -0,0 +1,66 @@
+import dataclasses
+import enum
+from typing import List
+
+from typing_extensions import override
+
+from prime_backup.action import Action
+from prime_backup.db.access import DbAccess
+from prime_backup.db.session import DbSession
+from prime_backup.types.chunk_group_info import BlobChunkGroupBindingInfo
+
+
+class BadBlobChunkGroupBindingItemType(enum.Enum):
+	orphan = enum.auto()
+	bad_storage_method = enum.auto()
+
+
+@dataclasses.dataclass(frozen=True)
+class BadBlobChunkGroupBindingItem:
+	binding: BlobChunkGroupBindingInfo
+	typ: BadBlobChunkGroupBindingItemType
+	desc: str
+
+
+@dataclasses.dataclass
+class ValidateBlobChunkGroupBindingsResult:
+	total: int = 0
+	bad_bindings: List[BadBlobChunkGroupBindingItem] = dataclasses.field(default_factory=list)
+
+	@property
+	def ok(self) -> int:
+		return self.total - self.bad
+
+	@property
+	def bad(self) -> int:
+		return len(self.bad_bindings)
+
+	def add_bad(self, binding: BlobChunkGroupBindingInfo, typ: BadBlobChunkGroupBindingItemType, msg: str):
+		self.bad_bindings.append(BadBlobChunkGroupBindingItem(binding, typ, msg))
+
+
+class ValidateBlobChunkGroupBindingsAction(Action[ValidateBlobChunkGroupBindingsResult]):
+	"""
+	NOTE: BlobChunkGroupBinding's .chunk_group_offset and .chunk_group_id checks are done in ValidateBlobsAction
+	"""
+
+	@override
+	def run(self) -> ValidateBlobChunkGroupBindingsResult:
+		self.logger.info('Scanning all blob chunk group bindings for orphan check')
+		result = ValidateBlobChunkGroupBindingsResult()
+
+		session: DbSession
+		with DbAccess.open_session() as session:
+			self.logger.info('Scanning all blob chunk group bindings for orphan check')
+			result.total = session.get_blob_chunk_group_binding_count()
+
+			for lor in session.list_orphan_blob_chunk_group_bindings(limit=1000):
+				if lor.blob is None:
+					result.add_bad(BlobChunkGroupBindingInfo.of(lor.binding), BadBlobChunkGroupBindingItemType.orphan, f'orphan binding refers to a non-existent blob {lor.binding.blob_id}')
+				else:
+					result.add_bad(BlobChunkGroupBindingInfo.of(lor.binding), BadBlobChunkGroupBindingItemType.bad_storage_method, f'Chunk group binding has invalid storage method {lor.blob.storage_method}')
+
+		self.logger.info('BlobChunkGroupBindingInfo validation done: total {}, ok {}, bad {}'.format(
+			result.total, result.ok, result.bad,
+		))
+		return result