chainsyncer

Blockchain syncer driver
Log | Files | Refs | LICENSE

commit 75a6d2975e4838fffa17914c53ae8581c7358ab5
parent 78bd6ca538d6e20e29ac7bb254c92701133e063a
Author: lash <dev@holbrook.no>
Date:   Fri, 18 Mar 2022 01:11:30 +0000

Implement driver processing

Diffstat:
Mchainsyncer/driver.py | 43+++++++++++++++++++++++++++----------------
Mchainsyncer/session.py | 9++-------
Mchainsyncer/state/base.py | 3+--
Mchainsyncer/store/fs.py | 44+++++++++++++++++++++++++++++++-------------
Mchainsyncer/unittest/base.py | 24++++++++++++++++++++++++
Mtests/test_fs.py | 3++-
Mtests/test_session.py | 27++++++++++++++++++++++-----
7 files changed, 109 insertions(+), 44 deletions(-)

diff --git a/chainsyncer/driver.py b/chainsyncer/driver.py @@ -1,6 +1,7 @@ # standard imports import logging import time +import signal # local imports from chainsyncer.error import ( @@ -17,8 +18,18 @@ NS_DIV = 1000000000 class SyncDriver: running_global = True - - def __init__(self, conn, store, pre_callback=None, post_callback=None, block_callback=None, idle_callback=None): + """If set to false syncer will terminate polling loop.""" + yield_delay=0.005 + """Delay between each processed block.""" + signal_request = [signal.SIGINT, signal.SIGTERM] + """Signals to catch to request shutdown.""" + signal_set = False + """Whether shutdown signal has been received.""" + name = 'base' + """Syncer name, to be overriden for each extended implementation.""" + + + def __init__(self, store, offset=0, target=-1, pre_callback=None, post_callback=None, block_callback=None, idle_callback=None): self.store = store self.running = True self.pre_callback = pre_callback @@ -27,7 +38,7 @@ class SyncDriver: self.idle_callback = idle_callback self.last_start = 0 self.clock_id = time.CLOCK_MONOTONIC_RAW - self.session = SyncSession(self.store) + self.store.start(offset=offset, target=target) def __sig_terminate(self, sig, frame): @@ -43,15 +54,16 @@ class SyncDriver: self.running = False - def run(self): + def run(self, conn): while self.running_global: - item = self.store.next_item() + self.session = SyncSession(self.store) + item = self.session.start() logg.debug('item {}'.format(item)) if item == None: self.running = False self.running_global = False break - self.loop(item) + self.loop(conn, item) def idle(self, interval): @@ -80,20 +92,18 @@ class SyncDriver: time.sleep(interval) - def loop(self, item): + def loop(self, conn, item): + tx_start = item.tx_cursor while self.running and SyncDriver.running_global: self.last_start = time.clock_gettime_ns(self.clock_id) if self.pre_callback != None: self.pre_callback() while True and self.running: - if start_tx > 0: - start_tx -= 1 - continue try: - block = self.get(conn) + block = self.get(conn, item) except SyncDone as e: logg.info('all blocks sumitted for processing: {}'.format(e)) - return self.backend.get() + return except NoBlockForYou as e: break if self.block_callback != None: @@ -101,21 +111,22 @@ class SyncDriver: last_block = block try: - self.process(conn, block) + self.process(conn, item, block, tx_start) except IndexError: - self.backend.set(block.number + 1, 0) - start_tx = 0 + item.next(advance_block=True) + tx_start = 0 time.sleep(self.yield_delay) if self.post_callback != None: self.post_callback() self.idle(interval) + def process_single(self, conn, block, tx): self.session.filter(conn, block, tx) - def process(self, conn, block): + def process(self, conn, block, tx_start): raise NotImplementedError() diff --git a/chainsyncer/session.py b/chainsyncer/session.py @@ -16,17 +16,11 @@ class SyncSession: self.item = None self.filters = self.session_store.filters - -# def register(self, fltr): -# if self.started: -# raise RuntimeError('filters cannot be changed after syncer start') -# self.session_store.register(fltr) -# self.filters.append(fltr) - def start(self, offset=0, target=-1): self.session_store.start(offset=offset, target=target) self.item = self.session_store.next_item() + return self.item def filter(self, conn, block, tx): @@ -43,4 +37,5 @@ class SyncSession: raise BackendError('filter state inconsitent with filter list') except FilterDone: self.item.reset() + self.next() self.session_store.disconnect() diff --git a/chainsyncer/state/base.py b/chainsyncer/state/base.py @@ -47,8 +47,7 @@ class SyncState: s = fltr.common_name() self.state_store.add(s) n = self.state_store.from_name(s) - logg.debug('add {} {} {}'.format(s, n, self)) - + logg.debug('add filter {} {} {}'.format(s, n, self)) def sum(self): diff --git a/chainsyncer/store/fs.py b/chainsyncer/store/fs.py @@ -33,7 +33,8 @@ class SyncFsItem: if started: match_state = self.sync_state.SYNC v = self.sync_state.get(self.state_key) - self.cursor = int.from_bytes(v, 'big') + self.cursor = int.from_bytes(v[:4], 'big') + self.tx_cursor = int.from_bytes(v[4:], 'big') if self.filter_state.state(self.state_key) & self.filter_state.from_name('LOCK') and not ignore_invalid: raise LockError(s) @@ -60,15 +61,26 @@ class SyncFsItem: raise IncompleteFilterError('reset attempt on {} when incomplete'.format(self.state_key)) self.filter_state.move(self.state_key, self.filter_state.from_name('RESET')) + + def next(self, advance_block=False): v = self.sync_state.get(self.state_key) - block_number = int.from_bytes(v, 'big') - block_number += 1 - if self.target >= 0 and block_number > self.target: - raise SyncDone(self.target) + + block_number = int.from_bytes(v[:4], 'big') + tx_index = int.from_bytes(v[4:], 'big') + if advance_block: + block_number += 1 + tx_index = 0 + if self.target >= 0 and block_number > self.target: + raise SyncDone(self.target) + else: + tx_index += 1 + + self.cursor = block_number + self.tx_cursor = tx_index v = block_number.to_bytes(4, 'big') self.sync_state.replace(self.state_key, v) - + def advance(self): if self.skip_filter: @@ -175,7 +187,6 @@ class SyncFsStore: def __load(self, target): - self.state.sync(self.state.NEW) self.state.sync(self.state.SYNC) @@ -220,13 +231,18 @@ class SyncFsStore: def start(self, offset=0, target=-1): + if self.started: + return + self.__load(target) if self.first: block_number = offset - block_number_bytes = block_number.to_bytes(4, 'big') + state_bytes = block_number.to_bytes(4, 'big') + tx_index = 0 + state_bytes += tx_index.to_bytes(4, 'big') block_number_str = str(block_number) - self.state.put(block_number_str, block_number_bytes) + self.state.put(block_number_str, state_bytes) self.filter_state.put(block_number_str) o = SyncFsItem(block_number, target, self.state, self.filter_state) self.items[block_number] = o @@ -239,10 +255,12 @@ class SyncFsStore: def stop(self): - if self.target == 0: - block_number = self.height + 1 - block_number_bytes = block_number.to_bytes(4, 'big') - self.state.put(str(block_number), block_number_bytes) +# if self.target == 0: +# block_number = self.height + 1 +# block_number_bytes = block_number.to_bytes(4, 'big') +# block_number_bytes = block_number.to_bytes(4, 'big') +# self.state.put(str(block_number), block_number_bytes) + pass def get(self, k): diff --git a/chainsyncer/unittest/base.py b/chainsyncer/unittest/base.py @@ -10,6 +10,7 @@ from shep.state import State # local imports #from chainsyncer.driver.history import HistorySyncer from chainsyncer.error import NoBlockForYou +from chainsyncer.driver import SyncDriver logg = logging.getLogger().getChild(__name__) @@ -105,6 +106,29 @@ class MockFilter: return self.brk +class MockDriver(SyncDriver): + + def __init__(self, store, offset=0, target=-1): + super(MockDriver, self).__init__(store, offset=offset, target=target) + self.blocks = {} + + + def add_block(self, block): + self.blocks[block.number] = block + + + def get(self, conn, item): + return self.blocks[item.cursor] + + + def process(self, conn, item, block, tx_start): + i = tx_start + while True: + tx = block.tx(i) + self.process_single(conn, block, tx) + item.next() + i += 1 + #class TestSyncer(HistorySyncer): # """Unittest extension of history syncer driver. diff --git a/tests/test_fs.py b/tests/test_fs.py @@ -183,8 +183,9 @@ class TestFs(unittest.TestCase): o.release() with self.assertRaises(FilterDone): o.advance() + o.reset() with self.assertRaises(SyncDone): - o.reset() + o.next(advance_block=True) if __name__ == '__main__': diff --git a/tests/test_session.py b/tests/test_session.py @@ -20,6 +20,7 @@ from chainsyncer.unittest import ( MockConn, MockTx, MockBlock, + MockDriver, ) from chainsyncer.driver import SyncDriver @@ -53,15 +54,31 @@ class TestFilter(unittest.TestCase): tx_hash = os.urandom(32).hex() tx = MockTx(42, tx_hash) block = MockBlock(1, [tx_hash]) - with self.assertRaises(SyncDone): - session.filter(self.conn, block, tx) + session.filter(self.conn, block, tx) self.assertEqual(len(fltr_one.contents), 2) - def test_driver(self): - drv = SyncDriver(self.conn, self.store) - drv.run() + drv = MockDriver(self.store, target=1) + + tx_hash = os.urandom(32).hex() + tx = MockTx(0, tx_hash) + block = MockBlock(0, [tx_hash]) + drv.add_block(block) + + tx_hash_one = os.urandom(32).hex() + tx = MockTx(0, tx_hash_one) + tx_hash_two = os.urandom(32).hex() + tx = MockTx(1, tx_hash_two) + block = MockBlock(1, [tx_hash_one, tx_hash_two]) + drv.add_block(block) + + fltr_one = MockFilter('foo') + self.store.register(fltr_one) + with self.assertRaises(SyncDone): + drv.run(self.conn) + + self.assertEqual(len(fltr_one.contents), 3) if __name__ == '__main__':