_BLOB_ALLOC_PERF_MODE

Author: Fallen-Breath

prime_backup/action/helpers/blob_allocator.py

@@ -11,7 +11,7 @@
 from typing_extensions import override
 
 from prime_backup.action.helpers.blob_creator_chunked import ChunkedBlobCreator
-from prime_backup.action.helpers.blob_creator_common import BlobCreateContext, BlobFileChanged, VolatileBlobFile, LookupBlobBySizeRequest, LookupBlobByHashRequest, BlobLookupRequest, BlobLookupRoutine
+from prime_backup.action.helpers.blob_creator_common import BlobCreateContext, BlobFileChanged, VolatileBlobFile, LookupBlobBySizeRequest, LookupBlobByHashRequest, BlobLookupRequest, BlobLookupRoutine, _BLOB_ALLOC_PERF_MODE
 from prime_backup.action.helpers.blob_creator_direct import DirectBlobCreator
 from prime_backup.action.helpers.blob_pre_calc_result import BlobPrecalculateResult
 from prime_backup.action.helpers.blob_recorder import BlobRecorder
@@ -271,6 +271,9 @@ def init_blob_store(self):
 			self.__ctx.blob_store_in_cow_fs = file_utils.does_fs_support_cow(bs_path)
 
 	def schedule_loop(self, gen_list: List[BlobLookupRoutine[schema.File]]) -> List[schema.File]:
+		if _BLOB_ALLOC_PERF_MODE:
+			return self.__schedule_loop_simple(gen_list)
+
 		files: List[schema.File] = []
 
 		schedule_queue: Deque[BlobLookupRoutine[schema.File]] = collections.deque()
@@ -308,5 +311,19 @@ def callback(g: BlobLookupRoutine[schema.File] = scheduled):
 
 		return files
 
+	def __schedule_loop_simple(self, gen_list: List[BlobLookupRoutine[schema.File]]) -> List[schema.File]:
+		files: List[schema.File] = []
+
+		def gen_wrapper(g: BlobLookupRoutine[schema.File]):
+			f = yield from g
+			files.append(f)
+
+		for gen in gen_list:
+			for query_req in gen_wrapper(gen):
+				self.__batch_lookup_manager.query(query_req, lambda: None)
+				self.__batch_lookup_manager.flush()
+		return files
+
+
 	def add_existing_sizes(self, existing_sizes: Dict[int, bool]):
 		self.__blob_by_size_cache.update(existing_sizes)

prime_backup/action/helpers/blob_creator_chunked.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import contextlib
 import dataclasses
 import enum
@@ -9,7 +7,7 @@
 from pathlib import Path
 from typing import BinaryIO, Dict, List, Optional
 
-from prime_backup.action.helpers.blob_creator_common import BlobLookupRoutine, BlobCreateContext, BlobCreatorBase
+from prime_backup.action.helpers.blob_creator_common import BlobLookupRoutine, BlobCreateContext, BlobCreatorBase, _BLOB_ALLOC_PERF_MODE
 from prime_backup.action.helpers.blob_pre_calc_result import BlobPrecalculateResult
 from prime_backup.action.helpers.chunk_grouper import ChunkGrouper
 from prime_backup.action.helpers.create_backup_utils import CreateBackupTimeCostKey, SourceFileNotFoundWrapper
@@ -251,7 +249,12 @@ def write_task(db_chunk_: schema.Chunk = db_chunk, compressor_: Compressor = com
 				compressed = compressor_.compress_bytes(chunk_buf_)
 				return _CompressedChunk(db_chunk_, compressed)
 
-			write_state.futures.append(pool.submit(write_task))
+			if _BLOB_ALLOC_PERF_MODE:
+				fut: Future[_CompressedChunk] = Future()
+				fut.set_result(write_task())
+				write_state.futures.append(fut)
+			else:
+				write_state.futures.append(pool.submit(write_task))
 			write_state.pending_bytes += db_chunk.raw_size
 
 			self.__flush_compressed_chunk_futures(write_state, force=False)

prime_backup/action/helpers/blob_creator_common.py

@@ -24,6 +24,7 @@
 if TYPE_CHECKING:
 	from prime_backup.action.helpers.pack_writer import PackWriter
 
+_BLOB_ALLOC_PERF_MODE = False
 _T = TypeVar('_T')
 
 

prime_backup/action/helpers/blob_creator_direct.py

@@ -167,7 +167,7 @@ def __create_blob_artifact(self, plan: _DirectBlobPlan) -> BlobLookupRoutine[_Di
 		if plan.policy == _DirectBlobCreatePolicy.hash_once:
 			return self.__create_by_hash_once(compressor, plan)
 		if plan.policy in (_DirectBlobCreatePolicy.read_all, _DirectBlobCreatePolicy.default):
-			return self.__create_by_prehashed_content(compressor, plan)
+			return self.__create_by_hashed_content(compressor, plan)
 		raise AssertionError('bad policy {!r}'.format(plan.policy))
 
 	def __create_by_copy_hash(self, compressor: Compressor) -> BlobLookupRoutine[_DirectBlobCreateResult]:
@@ -212,8 +212,7 @@ def __create_by_hash_once(self, compressor: Compressor, plan: _DirectBlobPlan) -
 					file_utils.copy_file_fast(temp_file_path, blob_path)
 			return _DirectBlobCreateResult.created(blob_hash, cr.read_size, cr.write_size)
 
-	def __create_by_prehashed_content(self, compressor: Compressor, plan: _DirectBlobPlan) -> _DirectBlobCreateResult:
-		misc_utils.assert_true(plan.blob_hash is not None, 'blob_hash is None')
+	def __create_by_hashed_content(self, compressor: Compressor, plan: _DirectBlobPlan) -> _DirectBlobCreateResult:
 		blob_hash = misc_utils.ensure_type(plan.blob_hash, str)
 		blob_path = blob_utils.get_blob_path(blob_hash)