Skip to content

Commit fede402

Browse files
oberstetoberstet
authored
sync (#62)
add database export/import helpers
1 parent 49ec0dc commit fede402

10 files changed

Lines changed: 451 additions & 51 deletions

File tree

‎cfxdb/_version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@
55
#
66
##############################################################################
77

8-
__version__ = u'20.8.2.dev1'
8+
__version__ = u'20.9.1'

‎cfxdb/exporter.py‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
##############################################################################
2+
#
3+
# Crossbar.io FX
4+
# Copyright (C) Crossbar.io Technologies GmbH. All rights reserved.
5+
#
6+
##############################################################################
7+
8+
import os
9+
import uuid
10+
11+
import cbor2
12+
import click
13+
import numpy as np
14+
15+
import zlmdb
16+
import cfxdb
17+
from cfxdb.xbrnetwork import Account, UserKey
18+
19+
from txaio import time_ns
20+
21+
22+
class Exporter(object):
23+
"""
24+
CFXDB database exporter.
25+
"""
26+
def __init__(self, dbpath):
27+
"""
28+
29+
:param dbpath: Database file to open.
30+
"""
31+
self._dbpath = os.path.abspath(dbpath)
32+
self._db = zlmdb.Database(dbpath=self._dbpath, maxsize=2**30, readonly=False)
33+
self._db.__enter__()
34+
35+
# cfxdb.schema.Schema
36+
# cfxdb.meta.Schema
37+
# cfxdb.globalschema.GlobalSchema
38+
# cfxdb.mrealmschema.MrealmSchema
39+
# cfxdb.xbr.Schema
40+
# cfxdb.xbrmm.Schema
41+
# cfxdb.xbrnetwork.Schema
42+
43+
self._meta = cfxdb.meta.Schema.attach(self._db)
44+
self._globalschema = cfxdb.globalschema.GlobalSchema.attach(self._db)
45+
self._mrealmschema = cfxdb.mrealmschema.MrealmSchema.attach(self._db)
46+
self._xbr = cfxdb.xbr.Schema.attach(self._db)
47+
self._xbrmm = cfxdb.xbrmm.Schema.attach(self._db)
48+
self._xbrnetwork = cfxdb.xbrnetwork.Schema.attach(self._db)
49+
50+
self._schemata = {
51+
# 'meta': self._meta,
52+
# 'globalschema': self._globalschema,
53+
# 'mrealmschema': self._mrealmschema,
54+
# 'xbr': self._xbr,
55+
# 'xbrmm': self._xbrmm,
56+
'xbrnetwork': self._xbrnetwork,
57+
}
58+
59+
self._schema_tables = {}
60+
61+
for schema_name, schema in self._schemata.items():
62+
tables = {}
63+
first = None
64+
for k, v in schema.__annotations__.items():
65+
for line in v.__doc__.splitlines():
66+
line = line.strip()
67+
if line != "":
68+
first = line[:80]
69+
break
70+
tables[k] = first
71+
self._schema_tables[schema_name] = tables
72+
73+
def _add_test_data(self):
74+
account = Account()
75+
account.oid = uuid.uuid4()
76+
account.created = np.datetime64(time_ns(), 'ns')
77+
account.username = 'alice'
78+
account.email = 'alice@example.com'
79+
account.wallet_type = 2 # metamask
80+
account.wallet_address = os.urandom(20)
81+
# account.wallet_address = binascii.a2b_hex('f5173a6111B2A6B3C20fceD53B2A8405EC142bF6')
82+
83+
userkey = UserKey()
84+
userkey.pubkey = os.urandom(32)
85+
# userkey.pubkey = binascii.a2b_hex('b7e6462121b9632b2bfcc5a3beef0b49dd865093ad003d011d4abbb68476d5b4')
86+
userkey.created = account.created
87+
userkey.owner = account.oid
88+
89+
with self._db.begin(write=True) as txn:
90+
self._xbrnetwork.accounts[txn, account.oid] = account
91+
self._xbrnetwork.user_keys[txn, userkey.pubkey] = userkey
92+
93+
def print_slots(self, include_description=False):
94+
"""
95+
96+
:param include_description:
97+
:return:
98+
"""
99+
print('\nDatabase slots [dbpath="{dbpath}"]:\n'.format(dbpath=self._dbpath))
100+
slots = self._db._get_slots()
101+
for slot_id in slots:
102+
slot = slots[slot_id]
103+
if include_description:
104+
print(' Slot {} using DB table class {}: {}'.format(
105+
click.style(str(slot_id), fg='white', bold=True), click.style(slot.name, fg='yellow'),
106+
slot.description))
107+
else:
108+
print(' Slot {} using DB table class {}'.format(
109+
click.style(str(slot_id), fg='white', bold=True), click.style(slot.name, fg='yellow')))
110+
print('')
111+
112+
def print_stats(self):
113+
"""
114+
115+
:return:
116+
"""
117+
print('\nDatabase table statistics [dbpath="{dbpath}"]:\n'.format(dbpath=self._dbpath))
118+
stats = {}
119+
with self._db.begin() as txn:
120+
for schema_name in self._schemata:
121+
stats[schema_name] = {}
122+
for table_name in self._schema_tables[schema_name]:
123+
table = self._schemata[schema_name].__dict__[table_name]
124+
cnt = table.count(txn)
125+
stats[schema_name][table_name] = cnt
126+
for schema_name in stats:
127+
for table_name in stats[schema_name]:
128+
cnt = stats[schema_name][table_name]
129+
if cnt:
130+
print('{:.<52}: {}'.format(
131+
click.style('{}.{}'.format(schema_name, table_name), fg='white', bold=True),
132+
click.style(str(cnt) + ' records', fg='yellow')))
133+
134+
def export_database(self, filename, include_indexes=False):
135+
"""
136+
137+
:param filename:
138+
:param include_indexes:
139+
:return:
140+
"""
141+
result = {}
142+
with self._db.begin() as txn:
143+
for schema_name in self._schemata:
144+
for table_name in self._schema_tables[schema_name]:
145+
table = self._schemata[schema_name].__dict__[table_name]
146+
if not table.is_index() or include_indexes:
147+
recs = []
148+
for key, val in table.select(txn, return_keys=True, return_values=True):
149+
if val:
150+
if hasattr(val, 'marshal'):
151+
val = val.marshal()
152+
recs.append((table._serialize_key(key), val))
153+
if recs:
154+
if schema_name not in result:
155+
result[schema_name] = {}
156+
result[schema_name][table_name] = recs
157+
158+
data = cbor2.dumps(result)
159+
with open(filename, 'wb') as f:
160+
f.write(data)
161+
162+
# data_recovered = cbor2.loads(data)
163+
print('\nExported database [dbpath="{dbpath}", filename="{filename}", filesize={filesize}]:\n'.format(
164+
dbpath=self._dbpath, filename=filename, filesize=len(data)))
165+
166+
for schema_name in result:
167+
for table_name in result[schema_name]:
168+
cnt = len(result[schema_name][table_name])
169+
if cnt:
170+
print('{:.<52}: {}'.format(
171+
click.style('{}.{}'.format(schema_name, table_name), fg='white', bold=True),
172+
click.style(str(cnt) + ' records', fg='yellow')))
173+
174+
def import_database(self, filename, include_indexes=True):
175+
"""
176+
177+
:param filename:
178+
:param include_indexes:
179+
:return:
180+
"""
181+
with open(filename, 'rb') as f:
182+
data = f.read()
183+
db_data = cbor2.loads(data)
184+
185+
print(
186+
'\nImporting database [dbpath="{dbpath}", filename="{filename}", filesize={filesize}]:\n'.format(
187+
dbpath=self._dbpath, filename=filename, filesize=len(data)))
188+
189+
with self._db.begin(write=True) as txn:
190+
for schema_name in self._schemata:
191+
for table_name in self._schema_tables[schema_name]:
192+
table = self._schemata[schema_name].__dict__[table_name]
193+
if not table.is_index() or include_indexes:
194+
if schema_name in db_data and table_name in db_data[schema_name]:
195+
cnt = 0
196+
for key, val in db_data[schema_name][table_name]:
197+
key = table._deserialize_key(key)
198+
val = table.parse(val)
199+
table[txn, key] = val
200+
cnt += 1
201+
if cnt:
202+
print('{:.<52}: {}'.format(
203+
click.style('{}.{}'.format(schema_name, table_name),
204+
fg='white',
205+
bold=True), click.style(str(cnt) + ' records', fg='yellow')))
206+
else:
207+
print('No data to import for {}.{}!'.format(schema_name, table_name))

‎cfxdb/meta/schema.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,35 @@
55
#
66
##############################################################################
77

8+
from zlmdb import table
9+
from zlmdb import MapUuidCbor, MapSlotUuidUuid
10+
811
from .attribute import Attributes
912

1013

14+
#
15+
# Docs metadata
16+
#
17+
@table('e11680d5-e20c-40b1-97d9-380b5ace1cb3', marshal=(lambda x: x), parse=(lambda x: x))
18+
class Docs(MapUuidCbor):
19+
"""
20+
* Database table: ``doc_oid -> doc``
21+
* Table type :class:`zlmdb.MapUuidCbor`
22+
* Key type :class:`uuid.UUID`
23+
* Record type :class:`cfxdb.mrealmschema.Doc`
24+
"""
25+
26+
27+
@table('d1de0b8c-3b6d-488b-8778-5bac8528ab4b')
28+
class IndexObjectToDoc(MapSlotUuidUuid):
29+
"""
30+
* Database index table ``(object_slot, object_oid) -> doc_oid``
31+
* Table type :class:`zlmdb.MapSlotUuidUuid`
32+
* Key type :class:`typing.Tuple[int, uuid.UUID]`
33+
* Indexed table :class:`cfxdb.mrealmschema.Docs`
34+
"""
35+
36+
1137
class Schema(object):
1238
"""
1339
Generic metadata, like documentation, tags, comments and reactions that
@@ -19,6 +45,20 @@ class Schema(object):
1945
Generic **meta data attributes** for objects in other tables.
2046
Primary key of this table is ``(table_oid, object_oid, attribute)``.
2147
"""
48+
49+
docs: Docs
50+
"""
51+
Documentation attached to a database object.
52+
53+
* Database table :class:`cfxdb.mrealmschema.Docs`
54+
"""
55+
56+
idx_object_to_doc: IndexObjectToDoc
57+
"""
58+
Index on documentation: by documented object ID
59+
60+
* Database table :class:`cfxdb.mrealmschema.IndexObjectToDoc`
61+
"""
2262
def __init__(self, db):
2363
self.db = db
2464

@@ -31,5 +71,11 @@ def attach(db):
3171
:type db: :class:`zlmdb.Database`
3272
"""
3373
schema = Schema(db)
74+
3475
schema.attributes = db.attach_table(Attributes)
76+
77+
schema.docs = db.attach_table(Docs)
78+
schema.idx_object_to_doc = db.attach_table(IndexObjectToDoc)
79+
schema.docs.attach_index('idx1', schema.idx_object_to_doc, lambda obj: obj.oid)
80+
3581
return schema

‎cfxdb/mrealmschema.py‎

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
##############################################################################
77

88
from zlmdb import table
9-
from zlmdb import MapStringUuid, MapUuidCbor, MapSlotUuidUuid, MapUuidStringUuid, MapUuidUuidUuid
9+
from zlmdb import MapStringUuid, MapUuidCbor, MapUuidStringUuid, MapUuidUuidUuid
1010
from zlmdb import MapUuidUuidCbor, MapUuidUuidUuidStringUuid, MapStringStringStringUuid, MapUuidTimestampUuid
1111

1212
from cfxdb.mrealm import RouterCluster, WebCluster, WebService, WebClusterNodeMembership, RouterClusterNodeMembership, RouterWorkerGroup, RouterWorkerGroupClusterPlacement
@@ -335,29 +335,6 @@ class IndexWebClusterPathToWebService(MapUuidStringUuid):
335335
"""
336336

337337

338-
#
339-
# Docs metadata
340-
#
341-
@table('e11680d5-e20c-40b1-97d9-380b5ace1cb3', marshal=(lambda x: x), parse=(lambda x: x))
342-
class Docs(MapUuidCbor):
343-
"""
344-
* Database table: ``doc_oid -> doc``
345-
* Table type :class:`zlmdb.MapUuidCbor`
346-
* Key type :class:`uuid.UUID`
347-
* Record type :class:`cfxdb.mrealmschema.Doc`
348-
"""
349-
350-
351-
@table('d1de0b8c-3b6d-488b-8778-5bac8528ab4b')
352-
class IndexObjectToDoc(MapSlotUuidUuid):
353-
"""
354-
* Database index table ``(object_slot, object_oid) -> doc_oid``
355-
* Table type :class:`zlmdb.MapSlotUuidUuid`
356-
* Key type :class:`typing.Tuple[int, uuid.UUID]`
357-
* Indexed table :class:`cfxdb.mrealmschema.Docs`
358-
"""
359-
360-
361338
class MrealmSchema(object):
362339
"""
363340
Management realm database schema.
@@ -540,25 +517,11 @@ def __init__(self, db):
540517
* Database table :class:`cfxdb.mrealmschema.IndexWebServiceByPath`
541518
"""
542519

543-
idx_webcluster_path_to_service: IndexWebClusterPathToWebService
520+
idx_webcluster_webservices: IndexWebClusterWebServices
544521
"""
545522
Index on web service: by ...
546523
547-
* Database table :class:`cfxdb.mrealmschema.IndexWebClusterPathToWebService`
548-
"""
549-
550-
docs: Docs
551-
"""
552-
Documentation attached to a database object.
553-
554-
* Database table :class:`cfxdb.mrealmschema.Docs`
555-
"""
556-
557-
idx_object_to_doc: IndexObjectToDoc
558-
"""
559-
Index on documentation: by documented object ID
560-
561-
* Database table :class:`cfxdb.mrealmschema.IndexObjectToDoc`
524+
* Database table :class:`cfxdb.mrealmschema.IndexWebClusterWebServices`
562525
"""
563526

564527
mnode_logs: MNodeLogs

‎cfxdb/xbr/schema.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ def __init__(self, db):
160160
is indexed by).
161161
"""
162162

163-
transaction: Transactions
163+
transactions: Transactions
164164
"""
165165
"""
166166

0 commit comments

Comments
 (0)