Skip to content

Commit 7d38021

Browse files
Fallen-BreathFallen-Breath
committed
add DbFeatures to centralize all db feature checks
1 parent 934f57e commit 7d38021

6 files changed

Lines changed: 101 additions & 94 deletions

File tree

‎prime_backup/db/db_features.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import functools
2+
import sqlite3
3+
4+
from typing import Callable
5+
6+
from prime_backup.utils import db_utils
7+
8+
9+
class DbFeatures:
10+
@classmethod
11+
def __check_support(cls, check_func: Callable[[], bool], msg: str) -> bool:
12+
if not (is_supported := check_func()):
13+
from prime_backup import logger
14+
logger.get().warning(f'WARN: {msg}. SQLite version: {db_utils.get_sqlite_version()}')
15+
return is_supported
16+
17+
@classmethod
18+
@functools.lru_cache(None)
19+
def supports_json_query(cls) -> bool:
20+
"""
21+
https://sqlite.org/json1.html#compiling_in_json_support
22+
A simple version check might not work, so here's a test
23+
"""
24+
def do_check() -> bool:
25+
try:
26+
with sqlite3.connect(':memory:') as conn:
27+
cursor = conn.cursor()
28+
cursor.execute('SELECT JSON(\'{"a": "b"}\')')
29+
cursor.fetchone()
30+
cursor.close()
31+
except sqlite3.OperationalError:
32+
return False
33+
else:
34+
return True
35+
36+
return cls.__check_support(do_check, 'SQLite backend does not support json query. Inefficient manual query is used as the fallback')
37+
38+
@classmethod
39+
@functools.lru_cache(None)
40+
def supports_vacuum_into(cls) -> bool:
41+
"""
42+
https://sqlite.org/releaselog/3_27_0.html
43+
"""
44+
return cls.__check_support(
45+
lambda: sqlite3.sqlite_version_info >= (3, 27, 0),
46+
'SQLite backend does not support VACUUM INTO statement. Insecure manual file copy is used as the fallback',
47+
)
48+
49+
@classmethod
50+
@functools.lru_cache(None)
51+
def supports_row_number(cls) -> bool:
52+
"""
53+
https://sqlite.org/windowfunctions.html#history
54+
"""
55+
return cls.__check_support(
56+
lambda: sqlite3.sqlite_version_info >= (3, 25, 0),
57+
'SQLite backend does not support ROW_NUMBER() statement, ID reassignment is not available',
58+
)
59+
60+
@classmethod
61+
@functools.lru_cache(None)
62+
def supports_returning(cls) -> bool:
63+
"""
64+
https://sqlite.org/lang_returning.html#overview
65+
"""
66+
return sqlite3.sqlite_version_info >= (3, 35, 0)
67+
68+
@classmethod
69+
@functools.lru_cache(None)
70+
def supports_without_rowid(cls) -> bool:
71+
"""
72+
https://sqlite.org/withoutrowid.html#compatibility
73+
"""
74+
return sqlite3.sqlite_version_info >= (3, 8, 2)
75+
76+
@classmethod
77+
def _debug_print_states(cls):
78+
print('version:', db_utils.get_sqlite_version())
79+
print('json query:', cls.supports_json_query())
80+
print('vacuum into:', cls.supports_vacuum_into())
81+
print('returning:', cls.supports_returning())
82+
print('row number:', cls.supports_row_number())
83+
print('without rowid:', cls.supports_without_rowid())
84+
85+
86+
if __name__ == '__main__':
87+
DbFeatures._debug_print_states()

‎prime_backup/db/migrations/migration_3_4.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@
77
from sqlalchemy.orm import Session, declarative_base
88
from typing_extensions import override
99

10+
from prime_backup.db.db_features import DbFeatures
1011
from prime_backup.db.migrations import MigrationImplBase
1112
from prime_backup.db.values import BlobStorageMethod
12-
from prime_backup.utils import db_utils
1313

1414
_with_rowid_kwargs: Dict[str, Any] = {}
15-
if db_utils.check_sqlite_without_rowid():
15+
if DbFeatures.supports_without_rowid():
1616
_with_rowid_kwargs['sqlite_with_rowid'] = False
1717

1818

‎prime_backup/db/schema.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@
33
from sqlalchemy import String, Integer, ForeignKey, BigInteger, JSON, LargeBinary
44
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
55

6+
from prime_backup.db.db_features import DbFeatures
67
from prime_backup.db.types import HashHex
78
from prime_backup.db.values import BackupTagDict
8-
from prime_backup.utils import db_utils
99

1010

1111
class Base(DeclarativeBase):
@@ -100,7 +100,7 @@ class ChunkGroupChunkBinding(Base):
100100
"""
101101

102102
__tablename__ = 'chunk_group_chunk_binding'
103-
__table_args__ = {'sqlite_with_rowid': False} if db_utils.check_sqlite_without_rowid() else {}
103+
__table_args__ = {'sqlite_with_rowid': False} if DbFeatures.supports_without_rowid() else {}
104104

105105
chunk_group_id: Mapped[int] = mapped_column(ForeignKey('chunk_group.id'), primary_key=True)
106106
chunk_offset: Mapped[int] = mapped_column(BigInteger, primary_key=True) # &chunk[0] - &chunk_group[0]
@@ -117,7 +117,7 @@ class BlobChunkGroupBinding(Base):
117117
"""
118118

119119
__tablename__ = 'blob_chunk_group_binding'
120-
__table_args__ = {'sqlite_with_rowid': False} if db_utils.check_sqlite_without_rowid() else {}
120+
__table_args__ = {'sqlite_with_rowid': False} if DbFeatures.supports_without_rowid() else {}
121121

122122
blob_id: Mapped[int] = mapped_column(ForeignKey('blob.id'), primary_key=True)
123123
chunk_group_offset: Mapped[int] = mapped_column(BigInteger, primary_key=True)

‎prime_backup/db/session.py‎

Lines changed: 7 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,15 @@
44
import sqlite3
55
import string
66
from pathlib import Path
7-
from typing import Optional, Sequence, Dict, Iterator, Callable, Set, Generator, Iterable, Tuple, Any, Type, TYPE_CHECKING
7+
from typing import Optional, Sequence, Dict, Iterator, Set, Generator, Iterable, Tuple, Any, Type, TYPE_CHECKING
88
from typing import TypeVar, List
99

1010
from sqlalchemy import select, delete, desc, func, Select, JSON, text, or_, not_, and_, exists, Row, update, inspect, ColumnElement, insert
1111
from sqlalchemy.orm import Session, Mapper, InstrumentedAttribute
1212
from typing_extensions import overload, Union, TypedDict, Unpack, NotRequired
1313

1414
from prime_backup.db import schema, db_constants
15+
from prime_backup.db.db_features import DbFeatures
1516
from prime_backup.db.values import FileRole, BackupTagDict, OffsetChunk, OffsetChunkGroup, BlobStorageMethod, ChunkGroupChunkBindingIdentifier, BlobChunkGroupBindingIdentifier, FileIdentifier
1617
from prime_backup.exceptions import BackupNotFound, BackupFileNotFound, BlobHashNotFound, PrimeBackupError, FilesetNotFound, FilesetFileNotFound, BlobIdNotFound, ChunkHashNotFound, ChunkIdNotFound, ChunkGroupChunkBindingNotFound, BlobChunkGroupBindingNotFound, ChunkGroupIdNotFound, ChunkGroupHashNotFound, PackIdNotFound
1718
from prime_backup.types.backup_filter import BackupFilter, BackupTagFilter, BackupSortOrder
@@ -67,34 +68,6 @@ def __init__(self, session: Session, db_path: Optional[Path] = None):
6768
# the limit in old sqlite (https://www.sqlite.org/limits.html#max_variable_number)
6869
self.__safe_var_limit = 999 - 20
6970

70-
@classmethod
71-
def __check_support(cls, check_func: Callable[[], bool], msg: str):
72-
if not (is_supported := check_func()):
73-
from prime_backup import logger
74-
import sqlite3
75-
logger.get().warning(f'WARN: {msg}. SQLite version: {sqlite3.sqlite_version}')
76-
return is_supported
77-
78-
@classmethod
79-
@functools.lru_cache
80-
def __supports_json_query(cls) -> bool:
81-
return cls.__check_support(db_utils.check_sqlite_json_query_support, 'SQLite backend does not support json query. Inefficient manual query is used as the fallback')
82-
83-
@classmethod
84-
@functools.lru_cache
85-
def __supports_vacuum_into(cls) -> bool:
86-
return cls.__check_support(db_utils.check_sqlite_vacuum_into_support, 'SQLite backend does not support VACUUM INTO statement. Insecure manual file copy is used as the fallback')
87-
88-
@classmethod
89-
@functools.lru_cache
90-
def __supports_row_number(cls) -> bool:
91-
return cls.__check_support(db_utils.check_sqlite_row_number, 'SQLite backend does not support ROW_NUMBER() statement, ID reassignment is not available')
92-
93-
@classmethod
94-
@functools.lru_cache
95-
def __supports_returning(cls) -> bool:
96-
return db_utils.check_sqlite_returning_support()
97-
9871
@classmethod
9972
@functools.lru_cache(None)
10073
def __get_schema_column_fields(cls, typ: Type[schema.Base]) -> List[Tuple[str, 'TypeEngine']]:
@@ -169,7 +142,7 @@ def no_auto_flush(self) -> Generator[None, None, None]:
169142
def vacuum(self, into_file: Optional[str] = None, allow_vacuum_into_fallback: bool = True):
170143
# https://www.sqlite.org/lang_vacuum.html
171144
if into_file is not None:
172-
if self.__supports_vacuum_into():
145+
if DbFeatures.supports_vacuum_into():
173146
self.session.execute(text('VACUUM INTO :into_file').bindparams(into_file=str(into_file)))
174147
elif allow_vacuum_into_fallback:
175148
if self.db_path is None:
@@ -548,7 +521,7 @@ def insert_chunks(self, rows: List[CreateChunkKwargs]) -> Dict[str, int]:
548521

549522
hashes: List[str] = [row['hash'] for row in rows]
550523
hash_to_id: Dict[str, int] = {}
551-
if self.__supports_returning():
524+
if DbFeatures.supports_returning():
552525
for row_page in collection_utils.slicing_iterate(rows, self.__safe_var_limit // self.__CHUNK_INSERT_FIELD_COUNT):
553526
for chunk_hash, chunk_id in self.session.execute(
554527
insert(schema.Chunk).
@@ -1584,7 +1557,7 @@ def __needs_manual_backup_tag_filter(cls, backup_filter: Optional[BackupFilter])
15841557
"""
15851558
SQLite does not support json query, and the backup filter contains tag filter
15861559
"""
1587-
return backup_filter is not None and len(backup_filter.tag_filters) > 0 and not cls.__supports_json_query()
1560+
return backup_filter is not None and len(backup_filter.tag_filters) > 0 and not DbFeatures.supports_json_query()
15881561

15891562
@classmethod
15901563
def __manual_backup_tag_filter(cls, backup: schema.Backup, backup_filter: BackupFilter) -> bool:
@@ -1669,7 +1642,7 @@ def __apply_backup_filter(cls, s: Select[_TP], backup_filter: BackupFilter) -> S
16691642
schema.Backup.timestamp < ts_sec,
16701643
and_(schema.Backup.timestamp == ts_sec, schema.Backup.timestamp_ns_part <= ts_ns_part)
16711644
))
1672-
if cls.__supports_json_query():
1645+
if DbFeatures.supports_json_query():
16731646
s = cls.__sql_backup_tag_filter(s, backup_filter)
16741647

16751648
sort_order = backup_filter.sort_order or BackupSortOrder.time_r
@@ -1897,7 +1870,7 @@ def delete_backup(self, backup: schema.Backup):
18971870
self.session.delete(backup)
18981871

18991872
def reassign_backup_id(self, order: BackupSortOrder) -> Optional[int]:
1900-
if not self.__supports_row_number():
1873+
if not DbFeatures.supports_row_number():
19011874
raise RuntimeError('Current SQLite version {} does not support ROW_NUMBER() function'.format(db_utils.get_sqlite_version()))
19021875

19031876
order_by: list

‎prime_backup/utils/db_utils.py‎

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -10,51 +10,6 @@ def get_sqlite_version() -> str:
1010
return sqlite3.sqlite_version
1111

1212

13-
def check_sqlite_json_query_support() -> bool:
14-
"""
15-
https://sqlite.org/json1.html#compiling_in_json_support
16-
A simple version check might not work, so here's a test
17-
"""
18-
try:
19-
with sqlite3.connect(':memory:') as conn:
20-
cursor = conn.cursor()
21-
cursor.execute('SELECT JSON(\'{"a": "b"}\')')
22-
cursor.fetchone()
23-
cursor.close()
24-
except sqlite3.OperationalError:
25-
return False
26-
else:
27-
return True
28-
29-
30-
def check_sqlite_vacuum_into_support() -> bool:
31-
"""
32-
https://sqlite.org/releaselog/3_27_0.html
33-
"""
34-
return sqlite3.sqlite_version_info >= (3, 27, 0)
35-
36-
37-
def check_sqlite_returning_support() -> bool:
38-
"""
39-
https://sqlite.org/lang_returning.html#overview
40-
"""
41-
return sqlite3.sqlite_version_info >= (3, 35, 0)
42-
43-
44-
def check_sqlite_row_number() -> bool:
45-
"""
46-
https://sqlite.org/windowfunctions.html#history
47-
"""
48-
return sqlite3.sqlite_version_info >= (3, 25, 0)
49-
50-
51-
def check_sqlite_without_rowid() -> bool:
52-
"""
53-
https://sqlite.org/withoutrowid.html#compatibility
54-
"""
55-
return sqlite3.sqlite_version_info >= (3, 8, 2)
56-
57-
5813
def vacuum_into_via_backup_api(src_db_path: 'PathLike', into_path: 'PathLike'):
5914
"""
6015
Fallback for VACUUM INTO on old SQLite versions that do not support "VACUUM INTO"
@@ -68,12 +23,3 @@ def vacuum_into_via_backup_api(src_db_path: 'PathLike', into_path: 'PathLike'):
6823
finally:
6924
dest_conn.close()
7025
src_conn.close()
71-
72-
73-
if __name__ == '__main__':
74-
print('version:', sqlite3.sqlite_version)
75-
print('json query:', check_sqlite_json_query_support())
76-
print('vacuum into:', check_sqlite_vacuum_into_support())
77-
print('returning:', check_sqlite_returning_support())
78-
print('row number:', check_sqlite_row_number())
79-
print('without rowid:', check_sqlite_without_rowid())

‎tests/test_schema_ddl.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
from ruamel.yaml import YAML
1313

14+
from prime_backup.db.db_features import DbFeatures
1415
from prime_backup.utils import db_utils
1516
from tests import schema_utils
1617

@@ -24,7 +25,7 @@ def __load_expected(filename: str) -> schema_utils.SchemaDDL:
2425

2526

2627
def _assert_schema_matches(tc: unittest.TestCase, actual: schema_utils.SchemaDDL, filename: str, exact_match: bool):
27-
if not db_utils.check_sqlite_without_rowid():
28+
if not DbFeatures.supports_without_rowid():
2829
tc.fail(f'SQLite without rowid is not supported, version: {db_utils.get_sqlite_version()}')
2930

3031
frozen_answer = __load_expected(filename)

0 commit comments

Comments
 (0)