commit 044e85fb993b0d5b396e40fd77bc3b2813cef533
parent 927913bd02c303f2232a4a08df15888db567a310
Author: lash <dev@holbrook.no>
Date: Tue, 26 Apr 2022 07:56:04 +0000
Allow memory-only syncing
Diffstat:
8 files changed, 147 insertions(+), 30 deletions(-)
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,7 @@
+* 0.3.5
+ - Allow memory-only shep if factory set to None in store constructor
+* 0.3.4
+ - Use explicit bool check in filter interrupt check
* 0.3.3
- Include shep persistent state bootstrap sync
- Add chainsyncer extras
diff --git a/chainsyncer/store/base.py b/chainsyncer/store/base.py
@@ -4,6 +4,7 @@ import logging
# local imports
from shep.persist import PersistedState
+from shep import State
from shep.error import StateInvalid
from chainsyncer.filter import FilterState
from chainsyncer.error import (
@@ -168,15 +169,22 @@ class SyncStore:
self.session_path = os.path.realpath(given_path)
- def setup_sync_state(self, factory, event_callback):
- self.state = PersistedState(factory.add, 2, event_callback=event_callback)
+ def setup_sync_state(self, factory=None, event_callback=None):
+ if factory == None:
+ self.state = State(2, event_callback=event_callback)
+ else:
+ self.state = PersistedState(factory.add, 2, event_callback=event_callback)
self.state.add('SYNC')
self.state.add('DONE')
- def setup_filter_state(self, factory, event_callback):
- filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback)
- self.filter_state = FilterState(filter_state_backend, scan=factory.ls)
+ def setup_filter_state(self, factory=None, event_callback=None):
+ if factory == None:
+ filter_state_backend = State(0, check_alias=False, event_callback=event_callback)
+ self.filter_state = FilterState(filter_state_backend)
+ else:
+ filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback)
+ self.filter_state = FilterState(filter_state_backend, scan=factory.ls)
self.filters = []
@@ -202,7 +210,7 @@ class SyncStore:
if self.first:
state_bytes = sync_state_serialize(offset, 0, target)
block_number_str = str(offset)
- self.state.put(block_number_str, state_bytes)
+ self.state.put(block_number_str, contents=state_bytes)
self.filter_state.put(block_number_str)
o = SyncItem(offset, target, self.state, self.filter_state)
self.items[offset] = o
@@ -226,7 +234,7 @@ class SyncStore:
self.state.move(item.state_key, self.state.DONE)
state_bytes = sync_state_serialize(item.cursor, 0, -1)
- self.state.put(str(item.cursor), state_bytes)
+ self.state.put(str(item.cursor), contents=state_bytes)
def load(self, target):
diff --git a/chainsyncer/store/fs.py b/chainsyncer/store/fs.py
@@ -7,10 +7,7 @@ import logging
from shep.store.file import SimpleFileStoreFactory
# local imports
-from chainsyncer.store import (
- SyncItem,
- SyncStore,
- )
+from chainsyncer.store import SyncStore
logg = logging.getLogger(__name__)
diff --git a/chainsyncer/store/mem.py b/chainsyncer/store/mem.py
@@ -0,0 +1,34 @@
+# standard imports
+import logging
+import os
+
+# external imports
+from shep import State
+
+# local imports
+from chainsyncer.store import SyncStore
+
+logg = logging.getLogger(__name__)
+
+
+class SyncMemStore(SyncStore):
+
+ def __init__(self, session_id=None, state_event_callback=None, filter_state_event_callback=None):
+ super(SyncMemStore, self).__init__('/dev/null', session_id=session_id)
+
+ self.session_id = os.path.basename(self.session_path)
+ logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path))
+
+ factory = None
+ self.setup_sync_state(factory, state_event_callback)
+
+ factory = None
+ self.setup_filter_state(factory, filter_state_event_callback)
+
+
+ def set_target(self, v):
+ self.target = int(v)
+
+
+ def get_target(self):
+ return self.target
diff --git a/chainsyncer/unittest/store.py b/chainsyncer/unittest/store.py
@@ -19,7 +19,17 @@ from chainsyncer.unittest import (
MockItem,
)
+logging.STATETRACE = 5
logg = logging.getLogger(__name__)
+logg.setLevel(logging.STATETRACE)
+
+
+def state_change_callback(k, old_state, new_state):
+ logg.log(logging.STATETRACE, 'state change: {} {} -> {}'.format(k, old_state, new_state))
+
+
+def filter_change_callback(k, old_state, new_state):
+ logg.log(logging.STATETRACE, 'filter change: {} {} -> {}'.format(k, old_state, new_state))
class TestStoreBase(unittest.TestCase):
@@ -27,6 +37,7 @@ class TestStoreBase(unittest.TestCase):
def setUp(self):
self.path = tempfile.mkdtemp()
self.store_factory = None
+ self.persist = True
@classmethod
@@ -58,14 +69,28 @@ class TestStoreBase(unittest.TestCase):
fp = os.path.join(self.path, store.session_id)
session_id = store.session_id
- st = os.stat(fp)
- self.assertTrue(stat.S_ISDIR(st.st_mode))
- self.assertTrue(store.is_default)
-
+ st = None
+ try:
+ st = os.stat(fp)
+ except FileNotFoundError as e:
+ logg.warning('error {} persist {}'.format(e, self.persist))
+ if self.persist:
+ raise e
+
+ if st != None:
+ self.assertTrue(stat.S_ISDIR(st.st_mode))
+ self.assertTrue(store.is_default)
+
fpd = os.path.join(self.path, 'default')
- st = os.stat(fpd)
- self.assertTrue(stat.S_ISDIR(st.st_mode))
- self.assertTrue(store.is_default)
+ try:
+ st = os.stat(fpd)
+ except FileNotFoundError as e:
+ logg.warning('error {} persist {}'.format(e, self.persist))
+ if self.persist:
+ raise e
+ if st != None:
+ self.assertTrue(stat.S_ISDIR(st.st_mode))
+ self.assertTrue(store.is_default)
fpd = os.path.realpath(fpd)
self.assertEqual(fpd, fp)
@@ -85,9 +110,15 @@ class TestStoreBase(unittest.TestCase):
store.stop(bogus_item)
store = self.store_factory('foo')
fpf = os.path.join(self.path, 'foo')
- st = os.stat(fpf)
- self.assertTrue(stat.S_ISDIR(st.st_mode))
- self.assertFalse(store.is_default)
+ try:
+ st = os.stat(fpf)
+ except FileNotFoundError as e:
+ logg.warning('error {} persist {}'.format(e, self.persist))
+ if self.persist:
+ raise e
+ if st != None:
+ self.assertTrue(stat.S_ISDIR(st.st_mode))
+ self.assertFalse(store.is_default)
def t_store_start(self):
@@ -97,9 +128,11 @@ class TestStoreBase(unittest.TestCase):
self.assertTrue(store.first)
store.stop(bogus_item)
- store = self.store_factory()
- store.start()
- self.assertFalse(store.first)
+
+ if self.persist:
+ store = self.store_factory()
+ store.start()
+ self.assertFalse(store.first)
def t_store_resume(self):
@@ -226,12 +259,16 @@ class TestStoreBase(unittest.TestCase):
o.next(advance_block=True)
session.stop(o)
- store = self.store_factory('foo')
- store.start()
- o = store.get(2)
+ if self.persist:
+ store = self.store_factory('foo')
+ store.start()
+ o = store.get(2)
def t_sync_history_interrupted(self):
+ if not self.persist:
+ return
+
bogus_item = MockItem(0, 0, 0, 0)
store = self.store_factory('foo')
session = SyncSession(store)
diff --git a/setup.cfg b/setup.cfg
@@ -1,6 +1,6 @@
[metadata]
name = chainsyncer
-version = 0.3.3
+version = 0.3.5
description = Generic blockchain syncer driver
author = Louis Holbrook
author_email = dev@holbrook.no
diff --git a/tests/store/test_0_mem.py b/tests/store/test_0_mem.py
@@ -0,0 +1,33 @@
+# standard imports
+import unittest
+import logging
+
+# external imports
+from shep import State
+
+# local imports
+from chainsyncer.store.mem import SyncMemStore
+from chainsyncer.unittest.store import TestStoreBase
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class StoreFactory:
+
+ def create(self, session_id=None):
+ return SyncMemStore(session_id=session_id)
+
+
+class TestMem(TestStoreBase):
+
+ def setUp(self):
+ super(TestMem, self).setUp()
+ self.store_factory = StoreFactory().create
+ self.persist = False
+
+
+if __name__ == '__main__':
+ TestStoreBase.link(TestMem)
+ # Remove tests that test persistence of state
+ unittest.main()
diff --git a/tests/store/test_rocksdb.py b/tests/store/test_rocksdb.py
@@ -4,7 +4,11 @@ import logging
# local imports
from chainsyncer.store.rocksdb import SyncRocksDbStore
-from chainsyncer.unittest.store import TestStoreBase
+from chainsyncer.unittest.store import (
+ TestStoreBase,
+ filter_change_callback,
+ state_change_callback,
+ )
logging.basicConfig(level=logging.DEBUG)
logg = logging.getLogger()
@@ -16,7 +20,7 @@ class StoreFactory:
def create(self, session_id=None):
- return SyncRocksDbStore(self.path, session_id=session_id)
+ return SyncRocksDbStore(self.path, session_id=session_id, state_event_callback=state_change_callback, filter_state_event_callback=filter_change_callback)
class TestRocksDb(TestStoreBase):
