chainsyncer

Blockchain syncer driver
Log | Files | Refs | LICENSE

commit 97c2d41df341c216e3a073c072ab19346172f17a
parent 891f90ae5f1a2cd38998fdbd6a7e7b5e3791fa35
Author: lash <dev@holbrook.no>
Date:   Wed, 20 Apr 2022 12:31:32 +0000

Factor out sync state scanner

Diffstat:
Mchainsyncer/state/base.py | 26+++++++++++++-------------
Mchainsyncer/store/base.py | 14++++++++++++++
Mchainsyncer/store/fs.py | 15+++++----------
3 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/chainsyncer/state/base.py b/chainsyncer/state/base.py @@ -12,7 +12,7 @@ re_processedname = r'^_?[A-Z,\.]*$' # TODO: properly clarify interface shared with syncfsstore, move to filter module? class SyncState: - def __init__(self, state_store, scan_path=None): + def __init__(self, state_store, scan=None): self.state_store = state_store self.digest = b'\x00' * 32 self.summed = False @@ -36,7 +36,7 @@ class SyncState: self.all = self.state_store.all self.started = False - self.scan_path = scan_path + self.scan = scan def __verify_sum(self, v): @@ -72,17 +72,17 @@ class SyncState: k = self.state_store.from_name(v) self.state_store.sync(k) self.__syncs[v] = True - if self.scan_path != None: - for v in os.listdir(self.scan_path): - if re.match(re_processedname, v): - k = None - try: - k = self.state_store.from_elements(v) - self.state_store.alias(v, k) - except ValueError: - k = self.state_store.from_name(v) - self.state_store.sync(k) - self.__syncs[v] = True + if self.scan != None: + ks = self.scan() + for v in ks: #os.listdir(self.scan_path): + k = None + try: + k = self.state_store.from_elements(v) + self.state_store.alias(v, k) + except ValueError: + k = self.state_store.from_name(v) + self.state_store.sync(k) + self.__syncs[v] = True self.synced = True self.connected = True diff --git a/chainsyncer/store/base.py b/chainsyncer/store/base.py @@ -2,7 +2,9 @@ import logging # local imports +from shep.persist import PersistedState from shep.error import StateInvalid +from chainsyncer.state import SyncState from chainsyncer.error import ( LockError, FilterDone, @@ -154,6 +156,18 @@ class SyncStore: self.started = False + def setup_sync_state(self, factory, event_callback): + self.state = PersistedState(factory.add, 2, event_callback=event_callback) + self.state.add('SYNC') + self.state.add('DONE') + + + def setup_filter_state(self, factory, event_callback): + filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback) + self.filter_state = SyncState(filter_state_backend, scan=factory.ls) + self.filters = [] + + def register(self, fltr): self.filters.append(fltr) self.filter_state.register(fltr) diff --git a/chainsyncer/store/fs.py b/chainsyncer/store/fs.py @@ -5,10 +5,8 @@ import logging # external imports from shep.store.file import SimpleFileStoreFactory -from shep.persist import PersistedState # local imports -from chainsyncer.state import SyncState from chainsyncer.store import ( SyncItem, SyncStore, @@ -45,17 +43,14 @@ class SyncFsStore(SyncStore): logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path)) - factory = SimpleFileStoreFactory(self.session_path, binary=True) - self.state = PersistedState(factory.add, 2, event_callback=state_event_callback) - self.state.add('SYNC') - self.state.add('DONE') + base_sync_path = os.path.join(self.session_path, 'sync') + factory = SimpleFileStoreFactory(base_sync_path, binary=True) + self.setup_sync_state(factory, state_event_callback) base_filter_path = os.path.join(self.session_path, 'filter') factory = SimpleFileStoreFactory(base_filter_path, binary=True) - filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=filter_state_event_callback) - self.filter_state = SyncState(filter_state_backend, scan_path=base_filter_path) - self.filters = [] # used by SyncSession - + self.setup_filter_state(factory, filter_state_event_callback) + def __create_path(self, base_path, default_path, session_id=None): logg.debug('fs store path {} does not exist, creating'.format(self.session_path))