fixes

Author: oberstet

cfxdb/exporter.py

@@ -7,6 +7,7 @@
 
 import os
 import uuid
+from typing import List
 
 import cbor2
 import click
@@ -32,14 +33,6 @@ def __init__(self, dbpath):
         self._db = zlmdb.Database(dbpath=self._dbpath, maxsize=2**30, readonly=False)
         self._db.__enter__()
 
-        # cfxdb.schema.Schema
-        # cfxdb.meta.Schema
-        # cfxdb.globalschema.GlobalSchema
-        # cfxdb.mrealmschema.MrealmSchema
-        # cfxdb.xbr.Schema
-        # cfxdb.xbrmm.Schema
-        # cfxdb.xbrnetwork.Schema
-
         self._meta = cfxdb.meta.Schema.attach(self._db)
         self._globalschema = cfxdb.globalschema.GlobalSchema.attach(self._db)
         self._mrealmschema = cfxdb.mrealmschema.MrealmSchema.attach(self._db)
@@ -48,11 +41,11 @@ def __init__(self, dbpath):
         self._xbrnetwork = cfxdb.xbrnetwork.Schema.attach(self._db)
 
         self._schemata = {
-            # 'meta': self._meta,
-            # 'globalschema': self._globalschema,
-            # 'mrealmschema': self._mrealmschema,
-            # 'xbr': self._xbr,
-            # 'xbrmm': self._xbrmm,
+            'meta': self._meta,
+            'globalschema': self._globalschema,
+            'mrealmschema': self._mrealmschema,
+            'xbr': self._xbr,
+            'xbrmm': self._xbrmm,
             'xbrnetwork': self._xbrnetwork,
         }
 
@@ -70,6 +63,44 @@ def __init__(self, dbpath):
                 tables[k] = first
             self._schema_tables[schema_name] = tables
 
+    @property
+    def dbpath(self) -> str:
+        """
+
+        :return:
+        """
+        return self._dbpath
+
+    def schemata(self) -> List[str]:
+        """
+
+        :return:
+        """
+        return sorted(self._schemata.keys())
+
+    def tables(self, schema_name):
+        """
+
+        :param schema_name:
+        :return:
+        """
+        if schema_name in self._schema_tables:
+            return sorted(self._schema_tables[schema_name].keys())
+        else:
+            return None
+
+    def table_docs(self, schema_name, table_name):
+        """
+
+        :param schema_name:
+        :param table_name:
+        :return:
+        """
+        if schema_name in self._schema_tables and table_name in self._schema_tables[schema_name]:
+            return self._schema_tables[schema_name][table_name]
+        else:
+            return None
+
     def _add_test_data(self):
         account = Account()
         account.oid = uuid.uuid4()
@@ -131,29 +162,43 @@ def print_stats(self):
                         click.style('{}.{}'.format(schema_name, table_name), fg='white', bold=True),
                         click.style(str(cnt) + ' records', fg='yellow')))
 
-    def export_database(self, filename, include_indexes=False):
+    def export_database(self, filename, include_indexes=False, include_schemata=None, exclude_tables=None):
         """
 
         :param filename:
         :param include_indexes:
         :return:
         """
+        if include_schemata is None:
+            schemata = sorted(self._schemata.keys())
+        else:
+            assert type(include_schemata) == list
+            schemata = sorted(list(set(include_schemata).intersection(self._schemata.keys())))
+
+        if exclude_tables is None:
+            exclude_tables = set()
+        else:
+            assert type(exclude_tables) == list
+            exclude_tables = set(exclude_tables)
+
         result = {}
         with self._db.begin() as txn:
-            for schema_name in self._schemata:
+            for schema_name in schemata:
                 for table_name in self._schema_tables[schema_name]:
-                    table = self._schemata[schema_name].__dict__[table_name]
-                    if not table.is_index() or include_indexes:
-                        recs = []
-                        for key, val in table.select(txn, return_keys=True, return_values=True):
-                            if val:
-                                if hasattr(val, 'marshal'):
-                                    val = val.marshal()
-                            recs.append((table._serialize_key(key), val))
-                        if recs:
-                            if schema_name not in result:
-                                result[schema_name] = {}
-                            result[schema_name][table_name] = recs
+                    fq_table_name = '{}.{}'.format(schema_name, table_name)
+                    if fq_table_name not in exclude_tables:
+                        table = self._schemata[schema_name].__dict__[table_name]
+                        if not table.is_index() or include_indexes:
+                            recs = []
+                            for key, val in table.select(txn, return_keys=True, return_values=True):
+                                if val:
+                                    if hasattr(val, 'marshal'):
+                                        val = val.marshal()
+                                recs.append((table._serialize_key(key), val))
+                            if recs:
+                                if schema_name not in result:
+                                    result[schema_name] = {}
+                                result[schema_name][table_name] = recs
 
         data = cbor2.dumps(result)
         with open(filename, 'wb') as f:

cfxdb/xbr/actor.py

@@ -111,15 +111,15 @@ def __init__(self, from_fbs=None):
         # [uint8] (ethsig)
         self._signature = None
 
-    def marshal(self) -> dict:
+    def marshal(self):
         obj = {
             'timestamp': int(self.timestamp) if self.timestamp else None,
             'market': self.market.bytes if self.market else None,
             'actor': bytes(self.actor) if self.actor else None,
             'actor_type': self.actor_type,
             'joined': pack_uint256(self.joined) if self.joined else None,
             'security': pack_uint256(self.security) if self.security else None,
-            'meta': self.meta,
+            'meta': bytes(self.meta) if self.meta else None,
             'tid': bytes(self.tid) if self.tid else None,
             'signature': bytes(self.signature) if self.signature else None,
         }

cfxdb/xbr/member.py

@@ -106,9 +106,9 @@ def __init__(self, from_fbs=None):
 
     def marshal(self) -> dict:
         obj = {
-            'address': self.address,
+            'address': bytes(self.address) if self._address else None,
             'account_oid': self._account_oid.bytes if self._account_oid else None,
-            'timestamp': self.timestamp,
+            'timestamp': int(self.timestamp) if self.timestamp else None,
             'registered': self.registered,
             'eula': self.eula,
             'profile': self.profile,

cfxdb/xbr/schema.py

@@ -209,6 +209,8 @@ def attach(db):
         schema.actors = db.attach_table(Actors)
         schema.idx_markets_by_actor = db.attach_table(IndexMarketsByActor)
 
+        # FIXME: we manually maintain the index, and hence must mark the pmap as an index manually
+        # schema.idx_markets_by_actor._index_attached_to = schema.actors
         # schema.actors.attach_index('idx1', schema.idx_markets_by_actor, lambda actor:
         #                            (actor.actor, actor.timestamp))
 

cfxdb/xbr/token.py

@@ -82,10 +82,10 @@ def __init__(self, from_fbs=None):
 
     def marshal(self) -> dict:
         obj = {
-            'tx_hash': self._tx_hash,
-            'block_hash': self._block_hash,
-            'from_address': self._owner_address,
-            'to_address': self._spender_address,
+            'tx_hash': bytes(self._tx_hash) if self._tx_hash else None,
+            'block_hash': bytes(self._block_hash) if self._block_hash else None,
+            'from_address': bytes(self._owner_address) if self._owner_address else None,
+            'to_address': bytes(self._spender_address) if self._spender_address else None,
             'value': self._value,
         }
         return obj