chainsyncer

Blockchain syncer driver
Log | Files | Refs | LICENSE

commit 197560ae16bf26454a35dd29b624ef9615365d99
parent f385b26e1ee9a85ba59bb0da7da89c190793c46a
Author: lash <dev@holbrook.no>
Date:   Wed, 20 Apr 2022 14:27:59 +0000

Implement rocksdb and default test

Diffstat:
Mchainsyncer/store/base.py | 13++++++++++++-
Mchainsyncer/store/fs.py | 17+++--------------
Achainsyncer/store/rocksdb.py | 57+++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Atests/test_rocksdb.py | 37+++++++++++++++++++++++++++++++++++++
4 files changed, 109 insertions(+), 15 deletions(-)

diff --git a/chainsyncer/store/base.py b/chainsyncer/store/base.py @@ -1,4 +1,5 @@ # standard imports +import os import logging # local imports @@ -145,7 +146,7 @@ class SyncItem: class SyncStore: - def __init__(self, session_id=None): + def __init__(self, path, session_id=None): self.session_id = None self.session_path = None self.is_default = False @@ -155,6 +156,16 @@ class SyncStore: self.item_keys = [] self.started = False self.thresholds = [] + self.default_path = os.path.join(path, 'default') + + if session_id == None: + self.session_path = os.path.realpath(self.default_path) + self.is_default = True + else: + if session_id == 'default': + self.is_default = True + given_path = os.path.join(path, session_id) + self.session_path = os.path.realpath(given_path) def setup_sync_state(self, factory, event_callback): diff --git a/chainsyncer/store/fs.py b/chainsyncer/store/fs.py @@ -18,18 +18,7 @@ logg = logging.getLogger(__name__) class SyncFsStore(SyncStore): def __init__(self, base_path, session_id=None, state_event_callback=None, filter_state_event_callback=None): - super(SyncFsStore, self).__init__(session_id=session_id) - - default_path = os.path.join(base_path, 'default') - - if session_id == None: - self.session_path = os.path.realpath(default_path) - self.is_default = True - else: - if session_id == 'default': - self.is_default = True - given_path = os.path.join(base_path, session_id) - self.session_path = os.path.realpath(given_path) + super(SyncFsStore, self).__init__(base_path, session_id=session_id) create_path = False try: @@ -38,9 +27,9 @@ class SyncFsStore(SyncStore): create_path = True if create_path: - self.__create_path(base_path, default_path, session_id=session_id) - self.session_id = os.path.basename(self.session_path) + self.__create_path(base_path, self.default_path, session_id=session_id) + self.session_id = os.path.basename(self.session_path) logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path)) base_sync_path = os.path.join(self.session_path, 'sync') diff --git a/chainsyncer/store/rocksdb.py b/chainsyncer/store/rocksdb.py @@ -0,0 +1,57 @@ +# standard imports +import uuid +import os +import logging + +# external imports +from shep.store.rocksdb import RocksDbStoreFactory + +# local imports +from chainsyncer.store import ( + SyncItem, + SyncStore, + ) + +logg = logging.getLogger(__name__) + + +class RocksDbStoreAdder: + + def __init__(self, factory, prefix): + self.factory = factory + self.prefix = prefix + + + def add(self, k): + path = os.path.join(self.prefix, k) + return self.factory.add(path) + + + def ls(self): + return self.factory.ls() + + +class SyncRocksDbStore(SyncStore): + + def __init__(self, base_path, session_id=None, state_event_callback=None, filter_state_event_callback=None): + super(SyncRocksDbStore, self).__init__(base_path, session_id=session_id) + + factory = RocksDbStoreFactory(self.session_path, binary=True) + prefix_factory = RocksDbStoreAdder(factory, 'sync') + self.setup_sync_state(prefix_factory, state_event_callback) + + prefix_factory = RocksDbStoreAdder(factory, 'filter') + self.setup_filter_state(prefix_factory, filter_state_event_callback) + + self.session_id = os.path.basename(self.session_path) + logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path)) + + self.target_db = RocksDbStoreAdder(factory, '.stat').add('target') + + + def get_target(self): + self.target_db.get('target') + + + def set_target(self, v): + self.target_db.put('target') diff --git a/tests/test_rocksdb.py b/tests/test_rocksdb.py @@ -0,0 +1,37 @@ +# standard imports +import unittest +import tempfile +import shutil +import logging +import stat +import os + +# local imports +from chainsyncer.store.rocksdb import SyncRocksDbStore +from chainsyncer.session import SyncSession +from chainsyncer.error import ( + LockError, + FilterDone, + IncompleteFilterError, + SyncDone, + ) +from chainsyncer.unittest import MockFilter + +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() + + +class TestFs(unittest.TestCase): + + def setUp(self): + self.path = tempfile.mkdtemp() + + + def test_default(self): + store = SyncRocksDbStore(self.path) + store.start(42) + self.assertTrue(store.first) + + +if __name__ == '__main__': + unittest.main()