chainsyncer

Blockchain syncer driver
Log | Files | Refs | LICENSE

commit 5f2809c39417104f18049136b10b0ff51b992175
parent e48b62679d79ef998a5a4f2e8b06d0022fa2c753
Author: lash <dev@holbrook.no>
Date:   Sat, 19 Mar 2022 00:52:47 +0000

Correct sync states of live sync ends

Diffstat:
Mchainsyncer/session.py | 4++++
Mchainsyncer/store/fs.py | 52++++++++++++++++++++++++++++++++++++----------------
Mtests/test_fs.py | 17+++++++++++++++++
3 files changed, 57 insertions(+), 16 deletions(-)

diff --git a/chainsyncer/session.py b/chainsyncer/session.py @@ -23,6 +23,10 @@ class SyncSession: return self.item + def stop(self, item): + self.session_store.stop(item) + + def filter(self, conn, block, tx): self.session_store.connect() for fltr in self.filters: diff --git a/chainsyncer/store/fs.py b/chainsyncer/store/fs.py @@ -29,9 +29,8 @@ class SyncFsItem: self.sync_state = sync_state self.filter_state = filter_state self.state_key = str(offset) - match_state = self.sync_state.NEW - if started: - match_state = self.sync_state.SYNC + + logg.debug('get key {}'.format(self.state_key)) v = self.sync_state.get(self.state_key) self.cursor = int.from_bytes(v[:4], 'big') self.tx_cursor = int.from_bytes(v[4:], 'big') @@ -43,7 +42,7 @@ class SyncFsItem: self.skip_filter = False if self.count == 0: self.skip_filter = True - else: + elif not started: self.filter_state.move(self.state_key, self.filter_state.from_name('RESET')) @@ -123,7 +122,7 @@ class SyncFsItem: self.filter_state.set(self.state_key, self.filter_state.from_name('DONE')) return False return True - + def __str__(self): return 'syncitem offset {} target {} cursor {}'.format(self.offset, self.target, self.cursor) @@ -161,6 +160,7 @@ class SyncFsStore: if create_path: self.__create_path(base_path, default_path, session_id=session_id) + self.session_id = os.path.basename(self.session_path) logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path)) @@ -187,7 +187,6 @@ class SyncFsStore: session_id = str(uuid.uuid4()) self.session_path = os.path.join(base_path, session_id) os.makedirs(self.session_path) - self.session_id = os.path.basename(self.session_path) if self.is_default: try: @@ -222,7 +221,7 @@ class SyncFsStore: o = SyncFsItem(block_number, item_target, self.state, self.filter_state, started=True) self.items[block_number] = o self.item_keys.append(block_number) - logg.info('added {}'.format(o)) + logg.info('added existing {}'.format(o)) fp = os.path.join(self.session_path, str(target)) if len(thresholds) == 0: @@ -237,8 +236,6 @@ class SyncFsStore: f.close() self.target = int(v) - logg.debug('target {}'.format(self.target)) - def start(self, offset=0, target=-1): if self.started: @@ -257,6 +254,7 @@ class SyncFsStore: o = SyncFsItem(block_number, target, self.state, self.filter_state) self.items[block_number] = o self.item_keys.append(block_number) + logg.debug('added first {}'.format(o)) elif offset > 0: logg.warning('block number argument {} for start ignored for already initiated sync {}'.format(offset, self.session_id)) self.started = True @@ -264,13 +262,21 @@ class SyncFsStore: self.item_keys.sort() - def stop(self): -# if self.target == 0: -# block_number = self.height + 1 -# block_number_bytes = block_number.to_bytes(4, 'big') -# block_number_bytes = block_number.to_bytes(4, 'big') -# self.state.put(str(block_number), block_number_bytes) - pass + def stop(self, item): + if item.target == -1: + state_bytes = sync_state_serialize(item.cursor, 0, item.cursor) + self.state.replace(str(item.offset), state_bytes) + self.filter_state.put(str(item.cursor)) + + SyncFsItem(item.offset, -1, self.state, self.filter_state) + logg.info('New sync state start at block number {} for next head sync backfill'.format(item.cursor)) + + self.state.move(item.state_key, self.state.DONE) + + state_bytes = sync_state_serialize(item.cursor, 0, -1) + self.state.put(str(item.cursor), state_bytes) + + logg.debug('item {}'.format(self.state.state(item.state_key))) def get(self, k): @@ -291,3 +297,17 @@ class SyncFsStore: def disconnect(self): self.filter_state.disconnect() + + +def sync_state_serialize(block_height, tx_index, block_target): + b = block_height.to_bytes(4, 'big') + b += tx_index.to_bytes(4, 'big') + b += block_target.to_bytes(4, 'big', signed=True) + return b + + +def sync_state_deserialize(b): + block_height = int.from_bytes(b[:4], 'big') + tx_index = int.from_bytes(b[4:8], 'big') + block_target = int.from_bytes(b[8:], 'big', signed=True) + return (block_height, tx_index, block_target,) diff --git a/tests/test_fs.py b/tests/test_fs.py @@ -188,5 +188,22 @@ class TestFs(unittest.TestCase): o.next(advance_block=True) + def test_sync_head_future(self): + store = SyncFsStore(self.path, session_id='foo') + session = SyncSession(store) + + session.start() + logg.debug('list {} {} {}'.format(store.state.list(store.state.SYNC), store.state.list(store.state.DONE), store.state.list(store.state.NEW))) + o = session.get(0) + o.next(advance_block=True) + o.next(advance_block=True) + session.stop(o) + + logg.debug('list {} {} {}'.format(store.state.list(store.state.SYNC), store.state.list(store.state.DONE), store.state.list(store.state.NEW))) + store = SyncFsStore(self.path, session_id='foo') + store.start() + o = store.get(2) + + if __name__ == '__main__': unittest.main()