chainsyncer

Blockchain syncer driver
Log | Files | Refs | LICENSE

commit 3a2317d253a9171258ee740a80fe932534e32d34
parent 6647e11df59f5b25384e01ecec83ee91d8bf8746
Author: lash <dev@holbrook.no>
Date:   Wed, 27 Apr 2022 09:43:42 +0000

WIP add lock cli tool

Diffstat:
MCHANGELOG | 3+++
MMANIFEST.in | 2+-
Mchainsyncer/cli/arg.py | 2++
Mchainsyncer/cli/config.py | 3+++
Mchainsyncer/data/config/syncer.ini | 1+
Achainsyncer/runnable/lock.py | 66++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mchainsyncer/settings.py | 12++++++++++--
Mchainsyncer/store/fs.py | 6+++++-
Msetup.cfg | 3++-
9 files changed, 93 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG @@ -1,3 +1,6 @@ +* 0.3.7 + - Remove hard eth dependency in settings rendering + - Add unlock cli tool * 0.3.6 - Add cli arg processing and settings renderer * 0.3.5 diff --git a/MANIFEST.in b/MANIFEST.in @@ -1 +1 @@ -include *requirements.txt LICENSE.txt +include *requirements.txt LICENSE.txt chainsyncer/data/config/* diff --git a/chainsyncer/cli/arg.py b/chainsyncer/cli/arg.py @@ -10,3 +10,5 @@ def process_flags(argparser, flags): if flags & SyncFlag.HEAD > 0: argparser.add_argument('--head', action='store_true', help='Start from latest block as offset') argparser.add_argument('--keep-alive', action='store_true', help='Do not stop syncing when caught up') + + argparser.add_argument('--backend', type=str, help='Backend to use for state store') diff --git a/chainsyncer/cli/config.py b/chainsyncer/cli/config.py @@ -4,6 +4,9 @@ from chainsyncer.cli import SyncFlag def process_config(config, args, flags): args_override = {} + + args_override['SYNCER_BACKEND'] = getattr(args, 'backend') + if flags & SyncFlag.RANGE: args_override['SYNCER_OFFSET'] = getattr(args, 'offset') args_override['SYNCER_LIMIT'] = getattr(args, 'until') diff --git a/chainsyncer/data/config/syncer.ini b/chainsyncer/data/config/syncer.ini @@ -1,3 +1,4 @@ [syncer] offset = 0 limit = 0 +backend = mem diff --git a/chainsyncer/runnable/lock.py b/chainsyncer/runnable/lock.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +# standard imports +import os +import logging +import sys +import importlib + +# external imports +import chainlib.cli + +# local imports +import chainsyncer.cli +from chainsyncer.settings import ChainsyncerSettings + +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() + + +arg_flags = chainlib.cli.argflag_std_base | chainlib.cli.Flag.CHAIN_SPEC +argparser = chainlib.cli.ArgumentParser(arg_flags) +argparser.add_argument('--state-dir', type=str, dest='state_dir', help='State directory') +argparser.add_argument('--session-id', type=str, dest='session_id', help='Session id for state') +argparser.add_argument('--action', type=str, choices=['repeat', 'continue'], help='Action to take on lock. Repeat means re-run the locked filter. Continue means resume execution for next filter.') +argparser.add_positional('tx', type=str, help='Transaction hash to unlock') + +sync_flags = chainsyncer.cli.SyncFlag.RANGE | chainsyncer.cli.SyncFlag.HEAD +chainsyncer.cli.process_flags(argparser, sync_flags) + +args = argparser.parse_args() + +base_config_dir = chainsyncer.cli.config_dir, +config = chainlib.cli.Config.from_args(args, arg_flags, base_config_dir=base_config_dir) +config = chainsyncer.cli.process_config(config, args, sync_flags) +config.add(args.state_dir, '_STATE_DIR', False) +config.add(args.session_id, '_SESSION_ID', False) +logg.debug('config loaded:\n{}'.format(config)) + +settings = ChainsyncerSettings() +settings.process_sync_backend(config) + + +def main(): + state_dir = None + if settings.get('SYNCER_BACKEND') == 'mem': + raise ValueError('cannot unlock volatile state store') + + if settings.get('SYNCER_BACKEND') == 'fs': + syncer_store_module = importlib.import_module('chainsyncer.store.fs') + syncer_store_class = getattr(syncer_store_module, 'SyncFsStore') + elif settings.get('SYNCER_BACKEND') == 'rocksdb': + syncer_store_module = importlib.import_module('chainsyncer.store.rocksdb') + syncer_store_class = getattr(syncer_store_module, 'SyncRocksDbStore') + else: + syncer_store_module = importlib.import_module(settings.get('SYNCER_BACKEND')) + syncer_store_class = getattr(syncer_store_module, 'SyncStore') + + state_dir = os.path.join(config.get('_STATE_DIR'), settings.get('SYNCER_BACKEND')) + sync_path = os.path.join(config.get('_SESSION_ID'), 'sync', 'filter') + sync_store = syncer_store_class(state_dir, session_id=sync_path) + + logg.info('session is {}'.format(sync_store.session_id)) + + +if __name__ == '__main__': + main() diff --git a/chainsyncer/settings.py b/chainsyncer/settings.py @@ -2,7 +2,6 @@ import logging # external imports -from chainlib.eth.block import block_latest from hexathon import ( to_int as hex_to_int, strip_0x, @@ -13,8 +12,17 @@ logg = logging.getLogger(__name__) class ChainsyncerSettings: + def __init__(self): + self.o = {} + self.get = self.o.get + + + def process_sync_backend(self, config): + self.o['SYNCER_BACKEND'] = config.get('SYNCER_BACKEND') + + def process_sync_range(self, config): - o = block_latest() + o = self.o['SYNCER_INTERFACE'].block_latest() r = self.o['RPC'].do(o) block_offset = int(strip_0x(r), 16) + 1 logg.info('network block height at startup is {}'.format(block_offset)) diff --git a/chainsyncer/store/fs.py b/chainsyncer/store/fs.py @@ -33,9 +33,13 @@ class SyncFsStore(SyncStore): factory = SimpleFileStoreFactory(base_sync_path, binary=True) self.setup_sync_state(factory, state_event_callback) + self.setup_filter_state(callback=filter_state_event_callback) + + + def setup_filter_state(self, callback=None): base_filter_path = os.path.join(self.session_path, 'filter') factory = SimpleFileStoreFactory(base_filter_path, binary=True) - self.setup_filter_state(factory, filter_state_event_callback) + super(SyncFsStore, self).setup_filter_state(factory, callback) def __create_path(self, base_path, default_path, session_id=None): diff --git a/setup.cfg b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = chainsyncer -version = 0.3.6 +version = 0.3.7 description = Generic blockchain syncer driver author = Louis Holbrook author_email = dev@holbrook.no @@ -28,6 +28,7 @@ packages = chainsyncer.driver chainsyncer.unittest chainsyncer.store + chainsyncer.cli #[options.package_data] #* =