chainsyncer

Blockchain syncer driver
Log | Files | Refs | LICENSE

commit dcf095cc863dbbaea905d6b824ee6d0cc26b40a6
parent 5968a190422c07ea4b30e8826eb5b5a5aa601bc8
Author: lash <dev@holbrook.no>
Date:   Thu, 17 Mar 2022 22:07:19 +0000

Complete syncitem filter advance

Diffstat:
Mchainsyncer/error.py | 15+++++++++++++++
Mchainsyncer/session.py | 22+++++++++++++---------
Mchainsyncer/state/base.py | 29++++++++++++++++++++++++-----
Mchainsyncer/store/fs.py | 89+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------
Achainsyncer/unittest/__init__.py | 1+
Mchainsyncer/unittest/base.py | 109+++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------
Mtests/test_basic.py | 39+++++----------------------------------
Mtests/test_fs.py | 96+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
8 files changed, 307 insertions(+), 93 deletions(-)

diff --git a/chainsyncer/error.py b/chainsyncer/error.py @@ -3,6 +3,7 @@ class SyncDone(Exception): """ pass + class NoBlockForYou(Exception): """Exception raised when attempt to retrieve a block from network that does not (yet) exist. """ @@ -27,6 +28,20 @@ class LockError(Exception): pass +class FilterDone(Exception): + """Exception raised when all registered filters have been executed + """ + + +class InterruptError(FilterDone): + """Exception for interrupting or attempting to use an interrupted sync + """ + + +class IncompleteFilterError(Exception): + """Exception raised if filter reset is executed prematurely + """ + #class AbortTx(Exception): # """ # """ diff --git a/chainsyncer/session.py b/chainsyncer/session.py @@ -7,24 +7,28 @@ class SyncSession: def __init__(self, session_store): self.session_store = session_store self.filters = [] - self.started = False + self.start = self.session_store.start + self.get = self.session_store.get + self.started = self.session_store.started - def add_filter(self, fltr): + def register(self, fltr): if self.started: raise RuntimeError('filters cannot be changed after syncer start') self.session_store.register(fltr) self.filters.append(fltr) - def start(self): - self.started = True - - def filter(self, conn, block, tx): self.sync_state.connect() for fltr in filters: - self.sync_start.lock() - self.sync_start.unlock() + try: + self.sync_start.advance() + except FilterDone: + break + interrupt = fltr(conn, block, tx) + try: + self.sync_start.release(interrupt=interrupt) + except FilterDone: + break self.sync_start.disconnect() - diff --git a/chainsyncer/state/base.py b/chainsyncer/state/base.py @@ -1,5 +1,8 @@ # standard imports import hashlib +import logging + +logg = logging.getLogger(__name__) class SyncState: @@ -11,8 +14,20 @@ class SyncState: self.__syncs = {} self.synced = False self.connected = False - self.state_store.add('INTERRUPT') + self.state_store.add('DONE') self.state_store.add('LOCK') + self.state_store.add('INTERRUPT') + self.state_store.add('RESET') + self.state = self.state_store.state + self.put = self.state_store.put + self.set = self.state_store.set + self.next = self.state_store.next + self.move = self.state_store.move + self.unset = self.state_store.unset + self.from_name = self.state_store.from_name + self.state_store.sync() + self.all = self.state_store.all + self.started = False def __verify_sum(self, v): @@ -30,6 +45,9 @@ class SyncState: self.digest += z s = fltr.common_name() self.state_store.add(s) + n = self.state_store.from_name(s) + logg.debug('add {} {} {}'.format(s, n, self)) + def sum(self): @@ -53,9 +71,10 @@ class SyncState: self.connected = False - def lock(self): - pass + def start(self): + self.state_store.start() + self.started = True - def unlock(self): - pass + def get(self, k): + raise NotImplementedError() diff --git a/chainsyncer/store/fs.py b/chainsyncer/store/fs.py @@ -6,31 +6,91 @@ import logging # external imports from shep.store.file import SimpleFileStoreFactory from shep.persist import PersistedState +from shep.error import StateInvalid # local imports from chainsyncer.state import SyncState - +from chainsyncer.error import ( + LockError, + FilterDone, + InterruptError, + IncompleteFilterError, + ) logg = logging.getLogger(__name__) +# NOT thread safe class SyncFsItem: - def __init__(self, offset, target, sync_state, filter_state, started=False): + def __init__(self, offset, target, sync_state, filter_state, started=False, ignore_invalid=False): self.offset = offset self.target = target self.sync_state = sync_state self.filter_state = filter_state - s = str(offset) + self.state_key = str(offset) match_state = self.sync_state.NEW if started: match_state = self.sync_state.SYNC - v = self.sync_state.get(s) + v = self.sync_state.get(self.state_key) self.cursor = int.from_bytes(v, 'big') + if self.filter_state.state(self.state_key) & self.filter_state.from_name('LOCK') and not ignore_invalid: + raise LockError(s) + + self.count = len(self.filter_state.all(pure=True)) - 3 + self.skip_filter = False + if self.count == 0: + self.skip_filter = True + else: + self.filter_state.move(self.state_key, self.filter_state.from_name('RESET')) + + + def __check_done(self): + if self.filter_state.state(self.state_key) & self.filter_state.from_name('INTERRUPT') > 0: + raise InterruptError(self.state_key) + if self.filter_state.state(self.state_key) & self.filter_state.from_name('DONE') > 0: + raise FilterDone(self.state_key) + + + def reset(self): + if self.filter_state.state(self.state_key) & self.filter_state.from_name('LOCK') > 0: + raise LockError('reset attempt on {} when state locked'.format(self.state_key)) + if self.filter_state.state(self.state_key) & self.filter_state.from_name('DONE') == 0: + raise IncompleteFilterError('reset attempt on {} when incomplete'.format(self.state_key)) + self.filter_state.move(self.state_key, self.filter_state.from_name('RESET')) + - def next(self): - pass + def advance(self): + if self.skip_filter: + raise FilterDone() + self.__check_done() + if self.filter_state.state(self.state_key) & self.filter_state.from_name('LOCK') > 0: + raise LockError('advance attempt on {} when state locked'.format(self.state_key)) + done = False + try: + self.filter_state.next(self.state_key) + except StateInvalid: + done = True + if done: + self.filter_state.set(self.state_key, self.filter_state.from_name('DONE')) + raise FilterDone() + self.filter_state.set(self.state_key, self.filter_state.from_name('LOCK')) + + + def release(self, interrupt=False): + if self.skip_filter: + raise FilterDone() + if interrupt: + self.filter_state.set(self.state_key, self.filter_state.from_name('INTERRUPT')) + self.filter_state.set(self.state_key, self.filter_state.from_name('DONE')) + return + + state = self.filter_state.state(self.state_key) + if state & self.filter_state.from_name('LOCK') == 0: + raise LockError('release attempt on {} when state unlocked'.format(self.state_key)) + self.filter_state.unset(self.state_key, self.filter_state.from_name('LOCK')) + def __str__(self): return 'syncitem offset {} target {} cursor {}'.format(self.offset, self.target, self.cursor) @@ -46,6 +106,7 @@ class SyncFsStore: self.first = False self.target = None self.items = {} + self.started = False default_path = os.path.join(base_path, 'default') @@ -76,10 +137,10 @@ class SyncFsStore: base_filter_path = os.path.join(self.session_path, 'filter') factory = SimpleFileStoreFactory(base_filter_path, binary=True) - filter_state_backend = PersistedState(factory, 0) + filter_state_backend = PersistedState(factory.add, 0, check_alias=False) self.filter_state = SyncState(filter_state_backend) self.register = self.filter_state.register - + def __create_path(self, base_path, default_path, session_id=None): logg.debug('fs store path {} does not exist, creating'.format(self.session_path)) @@ -144,12 +205,22 @@ class SyncFsStore: if self.first: block_number = offset block_number_bytes = block_number.to_bytes(4, 'big') - self.state.put(str(block_number), block_number_bytes) + block_number_str = str(block_number) + self.state.put(block_number_str, block_number_bytes) + self.filter_state.put(block_number_str) + o = SyncFsItem(block_number, target, self.state, self.filter_state) + self.items[block_number] = o elif offset > 0: logg.warning('block number argument {} for start ignored for already initiated sync {}'.format(offset, self.session_id)) + self.started = True + def stop(self): if self.target == 0: block_number = self.height + 1 block_number_bytes = block_number.to_bytes(4, 'big') self.state.put(str(block_number), block_number_bytes) + + + def get(self, k): + return self.items[k] diff --git a/chainsyncer/unittest/__init__.py b/chainsyncer/unittest/__init__.py @@ -0,0 +1 @@ +from .base import * diff --git a/chainsyncer/unittest/base.py b/chainsyncer/unittest/base.py @@ -1,12 +1,14 @@ # standard imports import os import logging +import hashlib # external imports from hexathon import add_0x +from shep.state import State # local imports -from chainsyncer.driver.history import HistorySyncer +#from chainsyncer.driver.history import HistorySyncer from chainsyncer.error import NoBlockForYou logg = logging.getLogger().getChild(__name__) @@ -67,42 +69,77 @@ class MockBlock: return MockTx(i, self.txs[i]) -class TestSyncer(HistorySyncer): - """Unittest extension of history syncer driver. +class MockStore(State): - :param backend: Syncer backend - :type backend: chainsyncer.backend.base.Backend implementation - :param chain_interface: Chain interface - :type chain_interface: chainlib.interface.ChainInterface implementation - :param tx_counts: List of integer values defining how many mock transactions to generate per block. Mock blocks will be generated for each element in list. - :type tx_counts: list - """ - - def __init__(self, backend, chain_interface, tx_counts=[]): - self.tx_counts = tx_counts - super(TestSyncer, self).__init__(backend, chain_interface) + def __init__(self, bits=0): + super(MockStore, self).__init__(bits, check_alias=False) + + def start(self): + pass - def get(self, conn): - """Implements the block getter of chainsyncer.driver.base.Syncer. - :param conn: RPC connection - :type conn: chainlib.connection.RPCConnection - :raises NoBlockForYou: End of mocked block array reached - :rtype: chainsyncer.unittest.base.MockBlock - :returns: Mock block. - """ - (pair, fltr) = self.backend.get() - (target_block, fltr) = self.backend.target() - block_height = pair[0] - - if block_height == target_block: - self.running = False - raise NoBlockForYou() - - block_txs = [] - if block_height < len(self.tx_counts): - for i in range(self.tx_counts[block_height]): - block_txs.append(add_0x(os.urandom(32).hex())) - - return MockBlock(block_height, block_txs) +class MockFilter: + + def __init__(self, name, brk=False, z=None): + self.name = name + if z == None: + h = hashlib.sha256() + h.update(self.name.encode('utf-8')) + z = h.digest() + self.z = z + self.brk = brk + + + def sum(self): + return self.z + + + def common_name(self): + return self.name + + + def filter(self, conn, block, tx): + return self.brk + + + +#class TestSyncer(HistorySyncer): +# """Unittest extension of history syncer driver. +# +# :param backend: Syncer backend +# :type backend: chainsyncer.backend.base.Backend implementation +# :param chain_interface: Chain interface +# :type chain_interface: chainlib.interface.ChainInterface implementation +# :param tx_counts: List of integer values defining how many mock transactions to generate per block. Mock blocks will be generated for each element in list. +# :type tx_counts: list +# """ +# +# def __init__(self, backend, chain_interface, tx_counts=[]): +# self.tx_counts = tx_counts +# super(TestSyncer, self).__init__(backend, chain_interface) +# +# +# def get(self, conn): +# """Implements the block getter of chainsyncer.driver.base.Syncer. +# +# :param conn: RPC connection +# :type conn: chainlib.connection.RPCConnection +# :raises NoBlockForYou: End of mocked block array reached +# :rtype: chainsyncer.unittest.base.MockBlock +# :returns: Mock block. +# """ +# (pair, fltr) = self.backend.get() +# (target_block, fltr) = self.backend.target() +# block_height = pair[0] +# +# if block_height == target_block: +# self.running = False +# raise NoBlockForYou() +# +# block_txs = [] +# if block_height < len(self.tx_counts): +# for i in range(self.tx_counts[block_height]): +# block_txs.append(add_0x(os.urandom(32).hex())) +# +# return MockBlock(block_height, block_txs) diff --git a/tests/test_basic.py b/tests/test_basic.py @@ -1,50 +1,21 @@ # standard imports import unittest -import hashlib import tempfile import shutil import logging -# external imports -from shep.state import State - # local imports from chainsyncer.session import SyncSession from chainsyncer.state import SyncState from chainsyncer.store.fs import SyncFsStore +from chainsyncer.unittest import ( + MockStore, + MockFilter, + ) logging.basicConfig(level=logging.DEBUG) logg = logging.getLogger() -class MockStore(State): - - def __init__(self, bits=0): - super(MockStore, self).__init__(bits, check_alias=False) - - -class MockFilter: - - def __init__(self, name, brk=False, z=None): - self.name = name - if z == None: - h = hashlib.sha256() - h.update(self.name.encode('utf-8')) - z = h.digest() - self.z = z - self.brk = brk - - - def sum(self): - return self.z - - - def common_name(self): - return self.name - - - def filter(self, conn, block, tx): - return self.brk - class TestSync(unittest.TestCase): @@ -64,7 +35,7 @@ class TestSync(unittest.TestCase): def test_sum(self): - store = MockStore(4) + store = MockStore(6) state = SyncState(store) b = b'\x2a' * 32 diff --git a/tests/test_fs.py b/tests/test_fs.py @@ -8,6 +8,13 @@ import os # local imports from chainsyncer.store.fs import SyncFsStore +from chainsyncer.session import SyncSession +from chainsyncer.error import ( + LockError, + FilterDone, + IncompleteFilterError, + ) +from chainsyncer.unittest import MockFilter logging.basicConfig(level=logging.DEBUG) logg = logging.getLogger() @@ -69,8 +76,97 @@ class TestFs(unittest.TestCase): store = SyncFsStore(self.path) store.start(13) self.assertTrue(store.first) + # todo not done + def test_sync_process_nofilter(self): + store = SyncFsStore(self.path) + session = SyncSession(store) + session.start() + o = session.get(0) + with self.assertRaises(FilterDone): + o.advance() + + + def test_sync_process_onefilter(self): + store = SyncFsStore(self.path) + session = SyncSession(store) + + fltr_one = MockFilter('foo') + session.register(fltr_one) + + session.start() + o = session.get(0) + o.advance() + o.release() + + + def test_sync_process_outoforder(self): + store = SyncFsStore(self.path) + session = SyncSession(store) + + fltr_one = MockFilter('foo') + session.register(fltr_one) + fltr_two = MockFilter('two') + session.register(fltr_two) + + session.start() + o = session.get(0) + o.advance() + with self.assertRaises(LockError): + o.advance() + + o.release() + with self.assertRaises(LockError): + o.release() + + o.advance() + o.release() + + + def test_sync_process_interrupt(self): + store = SyncFsStore(self.path) + session = SyncSession(store) + + fltr_one = MockFilter('foo') + session.register(fltr_one) + fltr_two = MockFilter('bar') + session.register(fltr_two) + + session.start() + o = session.get(0) + o.advance() + o.release(interrupt=True) + with self.assertRaises(FilterDone): + o.advance() + + + def test_sync_process_reset(self): + store = SyncFsStore(self.path) + session = SyncSession(store) + + fltr_one = MockFilter('foo') + session.register(fltr_one) + fltr_two = MockFilter('bar') + session.register(fltr_two) + + session.start() + o = session.get(0) + o.advance() + with self.assertRaises(LockError): + o.reset() + o.release() + with self.assertRaises(IncompleteFilterError): + o.reset() + + o.advance() + o.release() + + with self.assertRaises(FilterDone): + o.advance() + + o.reset() + if __name__ == '__main__': unittest.main()