use binary to store hash in db attempt

Author: Fallen-Breath

prime_backup/action/delete_backup_action.py

@@ -12,7 +12,7 @@
 from prime_backup.types.backup_info import BackupInfo
 from prime_backup.types.blob_info import BlobDeltaSummary
 from prime_backup.types.units import ByteCount
-from prime_backup.utils import misc_utils
+from prime_backup.utils import misc_utils, hash_utils
 
 
 @dataclasses.dataclass(frozen=True)
@@ -46,7 +46,7 @@ def run(self) -> DeleteBackupResult:
 						base_fileset_alive = False
 					for file in session.get_fileset_files(fileset.id):
 						if file.blob_hash is not None:
-							deleted_file_hashes.add(file.blob_hash)
+							deleted_file_hashes.add(hash_utils.to_hash_hex(file.blob_hash))
 						session.delete_file(file)
 					session.delete_fileset(fileset)
 

prime_backup/action/delete_backup_file_action.py

@@ -14,6 +14,7 @@
 from prime_backup.types.backup_info import BackupInfo
 from prime_backup.types.blob_info import BlobDeltaSummary
 from prime_backup.types.units import ByteCount
+from prime_backup.utils import hash_utils
 from prime_backup.utils.path_like import PathLike
 
 
@@ -87,8 +88,9 @@ def run(self) -> BlobDeltaSummary:
 					else:
 						raise AssertionError('unexpected delta file role {} for file {!r}'.format(file_existing.role, file_existing))
 					if file_existing.blob_hash is not None:
-						deleted_blob_hashes.add(file_existing.blob_hash)
-						self.logger.debug('Added blob_hash {} to deleted_blob_hashes'.format(file_existing.blob_hash))
+						fbh_hex = hash_utils.to_hash_hex(file_existing.blob_hash)
+						deleted_blob_hashes.add(fbh_hex)
+						self.logger.debug('Added blob_hash {} to deleted_blob_hashes'.format(fbh_hex))
 					session.delete_file(file_existing)
 				elif file_existing.fileset_id != fileset_id_base:
 					raise AssertionError('unexpected fileset id {} for file {!r}'.format(file_existing.fileset_id, file_existing))

prime_backup/action/delete_file_action.py

@@ -11,6 +11,7 @@
 from prime_backup.db.values import FileIdentifier
 from prime_backup.exceptions import FilesetFileNotFound
 from prime_backup.types.file_info import FileListSummary
+from prime_backup.utils import hash_utils
 
 
 class DeleteFilesStep(Action[FileListSummary]):
@@ -31,7 +32,7 @@ def run(self) -> FileListSummary:
 		deleted_blob_hashes: Set[str] = set()
 		for file in self.files:
 			if file.blob_hash is not None:
-				deleted_blob_hashes.add(file.blob_hash)
+				deleted_blob_hashes.add(hash_utils.to_hash_hex(file.blob_hash))
 			self.session.delete_file(file)
 		deleted_file_count = len(self.files)
 

prime_backup/action/export_backup_action_base.py

@@ -11,7 +11,7 @@
 from prime_backup.types.backup_info import BackupInfo
 from prime_backup.types.export_failure import ExportFailures
 from prime_backup.types.file_info import FileInfo
-from prime_backup.utils import misc_utils
+from prime_backup.utils import misc_utils, hash_utils
 
 
 class ExportBackupActionCommonInitKwargs(TypedDict):
@@ -69,7 +69,9 @@ def _on_unsupported_file_mode(cls, file: schema.File):
 
 	@classmethod
 	def _verify_exported_blob(cls, file: schema.File, written_size: int, written_hash: str):
+		if file.blob_hash is None:
+			raise AssertionError('File {!r} has no blob_hash'.format(file))
 		if written_size != file.blob_raw_size:
 			raise VerificationError('raw size mismatched for {}, expected {}, actual written {}'.format(file.path, file.blob_raw_size, written_size))
-		if written_hash != file.blob_hash:
-			raise VerificationError('hash mismatched for {}, expected {}, actual written {}'.format(file.path, file.blob_hash, written_hash))
+		if written_hash != (fbh_hex := hash_utils.to_hash_hex(file.blob_hash)):
+			raise VerificationError('hash mismatched for {}, expected {}, actual written {}'.format(file.path, fbh_hex, written_hash))

prime_backup/action/get_blob_action.py

@@ -9,6 +9,7 @@
 from prime_backup.db.session import DbSession
 from prime_backup.exceptions import BlobHashNotFound, BlobHashNotUnique
 from prime_backup.types.blob_info import BlobInfo
+from prime_backup.utils import hash_utils
 
 
 class _GetBlobActionBase(Action[BlobInfo], ABC):
@@ -24,8 +25,9 @@ def run(self) -> BlobInfo:
 		"""
 		with DbAccess.open_session() as session:
 			blob = self._do_get_blob(session)
-			file_count = session.get_file_count_by_blob_hashes([blob.hash]) if self.count_files else 0
-			file_samples = session.get_file_by_blob_hashes([blob.hash], limit=self.sample_file_num) if self.sample_file_num is not None else None
+			blob_hash_hex = hash_utils.to_hash_hex(blob.hash)
+			file_count = session.get_file_count_by_blob_hashes([blob_hash_hex]) if self.count_files else 0
+			file_samples = session.get_file_by_blob_hashes([blob_hash_hex], limit=self.sample_file_num) if self.sample_file_num is not None else None
 			return BlobInfo.of(blob, file_count=file_count, file_samples=file_samples)
 
 	@abstractmethod

prime_backup/action/helpers/backup_finalizer.py

@@ -4,6 +4,7 @@
 from prime_backup.action.helpers.fileset_allocator import FilesetAllocateArgs, FilesetAllocator
 from prime_backup.db import schema
 from prime_backup.db.session import DbSession
+from prime_backup.utils import hash_utils
 
 
 class BackupFinalizer:
@@ -19,12 +20,12 @@ def finalize_files_and_backup(self, backup: schema.Backup, files: List[schema.Fi
 		self.session.flush()  # ensure all blobs has their blob.id allocated
 
 		file_blobs = self.session.get_blobs_by_hashes_opt([
-			file.blob_hash for file in files
+			hash_utils.to_hash_hex(file.blob_hash) for file in files
 			if file.blob_hash is not None
 		])
 		for file in files:
 			if file.blob_hash is not None:
-				file_blob = file_blobs[file.blob_hash]
+				file_blob = file_blobs[hash_utils.to_hash_hex(file.blob_hash)]
 				if file_blob is None:
 					raise AssertionError('blob of file does not exists: {}'.format(file))
 				file.blob_id = file_blob.id

prime_backup/action/helpers/blob_allocator.py

@@ -517,17 +517,19 @@ def __try_get_or_create_chunked_blob(self, src_path: Path, src_path_md5: str, st
 						compress_method: CompressMethod = self.config.backup.get_compress_method_from_size(chunk.length)
 						compressor = Compressor.create(compress_method)
 
+						chunk_hash_hex = chunk.hash
 						db_chunk = self.session.create_chunk(
-							hash=chunk.hash,
+							hash=chunk_hash_hex,
 							compress=compress_method.name,
 							raw_size=len(chunk_buf),
 							stored_size=-1,
 						)
 						new_db_chunks.append(db_chunk)
-						known_db_chunks[db_chunk.hash] = db_chunk
+						known_db_chunks[chunk_hash_hex] = db_chunk
 
-						def write_task(db_chunk_=db_chunk, compressor_=compressor, chunk_buf_=chunk_buf):
-							chunk_path = chunk_utils.get_chunk_path(db_chunk_.hash)
+						def write_task(db_chunk_=db_chunk, compressor_=compressor, chunk_buf_=chunk_buf, chunk_hash_hex_=chunk_hash_hex):
+							# WARNING: cannot reference any not-captured locals since they might be modified in other thread
+							chunk_path = chunk_utils.get_chunk_path(chunk_hash_hex_)
 							self.__blob_recorder.add_remove_file_rollbacker(chunk_path)
 
 							with compressor_.open_compressed_bypassed(chunk_path) as (writer, f):
@@ -724,7 +726,7 @@ def callback(query_rsp: BqmRsp, g=gen):
 	def __update_blob_cache(self, blob: schema.Blob):
 		if blob is not None:
 			self.__blob_by_size_cache[blob.raw_size] = True
-			self.__blob_by_hash_cache[blob.hash] = blob
+			self.__blob_by_hash_cache[hash_utils.to_hash_hex(blob.hash)] = blob
 
 	def add_existing_sizes(self, existing_sizes: Dict[int, bool]):
 		self.__blob_by_size_cache.update(existing_sizes)

prime_backup/action/helpers/blob_exporter.py

@@ -258,7 +258,8 @@ def __verify_exported_blob(self, written_size: int, written_hash: str):
 		self.__verify_exported_data(lambda: 'blob', self.blob.raw_size, self.blob.hash, written_size, written_hash)
 
 	def __verify_exported_chunk(self, chunk: Union[schema.Chunk, ChunkInfo], written_size: int, written_hash: str):
-		self.__verify_exported_data(lambda: f'chunk {chunk.hash}', chunk.raw_size, chunk.hash, written_size, written_hash)
+		hash_hex = hash_utils.to_hash_hex(chunk.hash)
+		self.__verify_exported_data(lambda: f'chunk {hash_hex}', chunk.raw_size, hash_hex, written_size, written_hash)
 
 	def __verify_exported_data(self, what: Callable[[], str], expected_size: int, expected_hash: str, written_size: int, written_hash: str):
 		if written_size != expected_size:

prime_backup/action/helpers/chunk_grouper.py

@@ -7,7 +7,7 @@
 from prime_backup.constants import chunk_constants
 from prime_backup.db import schema
 from prime_backup.db.session import DbSession
-from prime_backup.utils import chunk_utils, collection_utils
+from prime_backup.utils import chunk_utils, collection_utils, hash_utils
 from prime_backup.utils.time_cost_stats import TimeCostStats
 
 
@@ -57,9 +57,9 @@ def create_chunk_groups(self, blob: schema.Blob, blob_chunks: Dict[int, schema.C
 			needs_cut |= i == len(blob_chunks) - 1  # last chunk group
 			if len(current_group.chunks) >= chunk_constants.CHUNK_GROUP_MIN_SIZE:
 				needs_cut |= len(current_group.chunks) >= chunk_constants.CHUNK_GROUP_MAX_SIZE  # reach max size
-				needs_cut |= chunk.hash.endswith('00')  # 1/256 chance
+				needs_cut |= chunk.hash.endswith(b'\0')  # last byte is 0 -- 1/256 chance
 			if needs_cut:
-				current_group.hash = chunk_utils.create_chunk_group_hash(chunk.hash for chunk in current_group.chunks)
+				current_group.hash = chunk_utils.create_chunk_group_hash(hash_utils.to_hash_hex(chunk.hash) for chunk in current_group.chunks)
 				raw_chunk_groups.append(current_group)
 				chunk_group_hashes_to_chunks[current_group.hash] = current_group.chunks
 				current_group = _RawChunkGroup()
@@ -77,7 +77,7 @@ def create_chunk_groups(self, blob: schema.Blob, blob_chunks: Dict[int, schema.C
 					chunk_stored_size_sum=sum({chunk.hash: chunk.stored_size for chunk in cg_chunks}.values()),
 				)
 				known_chunk_groups[cg_hash] = new_chunk_group
-				new_chunk_group_hashes.append(new_chunk_group.hash)
+				new_chunk_group_hashes.append(hash_utils.to_hash_hex(new_chunk_group.hash))
 		if len(new_chunk_group_hashes) > 0:
 			with self.__time_costs.measure_time_cost(CreateBackupTimeCostKey.kind_db):
 				self.session.flush()  # creates chunk_group.id

prime_backup/action/import_backup_action.py

@@ -24,7 +24,7 @@
 from prime_backup.types.standalone_backup_format import StandaloneBackupFormat
 from prime_backup.types.tar_format import TarFormat
 from prime_backup.types.units import ByteCount
-from prime_backup.utils import blob_utils, misc_utils, chunk_utils, collection_utils, file_utils
+from prime_backup.utils import blob_utils, misc_utils, chunk_utils, collection_utils, file_utils, hash_utils
 from prime_backup.utils.chunker import Chunker
 from prime_backup.utils.hash_utils import SizeAndHash
 
@@ -142,7 +142,7 @@ def __create_blob(self, session: DbSession, file_path: str, file_reader: IO[byte
 		else:
 			blob = self.__create_blob_direct(session, file_reader, SizeAndHash(pre_cal_result.size, pre_cal_result.hash))
 
-		self.__blob_cache[blob.hash] = blob
+		self.__blob_cache[hash_utils.to_hash_hex(blob.hash)] = blob
 		return blob
 
 	@classmethod