commit 00725ebc7cd2af583cac0ea9a5920e66f8ab6cd5
parent 88df1418b0c3814f49185cebd7e7021c27ff8125
Author: nolash <dev@holbrook.no>
Date: Thu, 11 Feb 2021 09:02:17 +0100
Rename module
Diffstat:
20 files changed, 586 insertions(+), 608 deletions(-)
diff --git a/chainsyncer/backend.py b/chainsyncer/backend.py
@@ -0,0 +1,216 @@
+# standard imports
+import logging
+import uuid
+
+# local imports
+from chainsyncer.db.models.sync import BlockchainSync
+from chainsyncer.db.models.base import SessionBase
+
+logg = logging.getLogger()
+
+
+class SyncerBackend:
+ """Interface to block and transaction sync state.
+
+ :param chain_spec: Chain spec for the chain that syncer is running for.
+ :type chain_spec: cic_registry.chain.ChainSpec
+ :param object_id: Unique id for the syncer session.
+ :type object_id: number
+ """
+ def __init__(self, chain_spec, object_id):
+ self.db_session = None
+ self.db_object = None
+ self.chain_spec = chain_spec
+ self.object_id = object_id
+ self.connect()
+ self.disconnect()
+
+
+ def connect(self):
+ """Loads the state of the syncer session with the given id.
+ """
+ self.db_session = SessionBase.create_session()
+ q = self.db_session.query(BlockchainSync)
+ q = q.filter(BlockchainSync.id==self.object_id)
+ self.db_object = q.first()
+ if self.db_object == None:
+ raise ValueError('sync entry with id {} not found'.format(self.object_id))
+
+
+ def disconnect(self):
+ """Commits state of sync to backend.
+ """
+ self.db_session.add(self.db_object)
+ self.db_session.commit()
+ self.db_session.close()
+
+
+ def chain(self):
+ """Returns chain spec for syncer
+
+ :returns: Chain spec
+ :rtype chain_spec: cic_registry.chain.ChainSpec
+ """
+ return self.chain_spec
+
+
+ def get(self):
+ """Get the current state of the syncer cursor.
+
+ :returns: Block and block transaction height, respectively
+ :rtype: tuple
+ """
+ self.connect()
+ pair = self.db_object.cursor()
+ self.disconnect()
+ return pair
+
+
+ def set(self, block_height, tx_height):
+ """Update the state of the syncer cursor
+ :param block_height: Block height of cursor
+ :type block_height: number
+ :param tx_height: Block transaction height of cursor
+ :type tx_height: number
+ :returns: Block and block transaction height, respectively
+ :rtype: tuple
+ """
+ self.connect()
+ pair = self.db_object.set(block_height, tx_height)
+ self.disconnect()
+ return pair
+
+
+ def start(self):
+ """Get the initial state of the syncer cursor.
+
+ :returns: Initial block and block transaction height, respectively
+ :rtype: tuple
+ """
+ self.connect()
+ pair = self.db_object.start()
+ self.disconnect()
+ return pair
+
+
+ def target(self):
+ """Get the target state (upper bound of sync) of the syncer cursor.
+
+ :returns: Target block height
+ :rtype: number
+ """
+ self.connect()
+ target = self.db_object.target()
+ self.disconnect()
+ return target
+
+
+ @staticmethod
+ def first(chain):
+ """Returns the model object of the most recent syncer in backend.
+
+ :param chain: Chain spec of chain that syncer is running for.
+ :type chain: cic_registry.chain.ChainSpec
+ :returns: Last syncer object
+ :rtype: cic_eth.db.models.BlockchainSync
+ """
+ return BlockchainSync.first(chain)
+
+
+ @staticmethod
+ def initial(chain, block_height):
+ """Creates a new syncer session and commit its initial state to backend.
+
+ :param chain: Chain spec of chain that syncer is running for.
+ :type chain: cic_registry.chain.ChainSpec
+ :param block_height: Target block height
+ :type block_height: number
+ :returns: New syncer object
+ :rtype: cic_eth.db.models.BlockchainSync
+ """
+ object_id = None
+ session = SessionBase.create_session()
+ o = BlockchainSync(chain, 0, 0, block_height)
+ session.add(o)
+ session.commit()
+ object_id = o.id
+ session.close()
+
+ return SyncerBackend(chain, object_id)
+
+
+ @staticmethod
+ def resume(chain, block_height):
+ """Retrieves and returns all previously unfinished syncer sessions.
+
+
+ :param chain: Chain spec of chain that syncer is running for.
+ :type chain: cic_registry.chain.ChainSpec
+ :param block_height: Target block height
+ :type block_height: number
+ :returns: Syncer objects of unfinished syncs
+ :rtype: list of cic_eth.db.models.BlockchainSync
+ """
+ syncers = []
+
+ session = SessionBase.create_session()
+
+ object_id = None
+
+ for object_id in BlockchainSync.get_unsynced(session=session):
+ logg.debug('block syncer resume added previously unsynced sync entry id {}'.format(object_id))
+ syncers.append(SyncerBackend(chain, object_id))
+
+ (block_resume, tx_resume) = BlockchainSync.get_last_live_height(block_height, session=session)
+ if block_height != block_resume:
+ o = BlockchainSync(chain, block_resume, tx_resume, block_height)
+ session.add(o)
+ session.commit()
+ object_id = o.id
+ syncers.append(SyncerBackend(chain, object_id))
+ logg.debug('block syncer resume added new sync entry from previous run id {}, start{}:{} target {}'.format(object_id, block_resume, tx_resume, block_height))
+
+ session.close()
+
+ return syncers
+
+
+ @staticmethod
+ def live(chain, block_height):
+ """Creates a new open-ended syncer session starting at the given block height.
+
+ :param chain: Chain spec of chain that syncer is running for.
+ :type chain: cic_registry.chain.ChainSpec
+ :param block_height: Target block height
+ :type block_height: number
+ :returns: "Live" syncer object
+ :rtype: cic_eth.db.models.BlockchainSync
+ """
+ object_id = None
+ session = SessionBase.create_session()
+ o = BlockchainSync(chain, block_height, 0, None)
+ session.add(o)
+ session.commit()
+ object_id = o.id
+ session.close()
+
+ return SyncerBackend(chain, object_id)
+
+
+class MemBackend:
+
+ def __init__(self, chain_spec, object_id):
+ self.object_id = object_id
+ self.chain_spec = chain_spec
+ self.block_height = 0
+ self.tx_height = 0
+
+
+ def set(self, block_height, tx_height):
+ logg.debug('stateless backend received {} {}'.format(block_height, tx_height))
+ self.block_height = block_height
+ self.tx_height = tx_height
+
+
+ def get(self):
+ return (self.block_height, self.tx_height)
diff --git a/cic_syncer/client/__init__.py b/chainsyncer/client/__init__.py
diff --git a/cic_syncer/client/block.py b/chainsyncer/client/block.py
diff --git a/chainsyncer/client/evm/response.py b/chainsyncer/client/evm/response.py
@@ -0,0 +1,52 @@
+import json
+
+from chainsyncer.client import translate
+from chainsyncer.client.block import Block
+from chainsyncer.client.tx import Tx
+
+
+translations = {
+ 'block_number': translate.hex_to_int,
+ 'get_block': json.dumps,
+ 'number': translate.hex_to_int,
+ }
+
+
+class EVMResponse:
+
+ def __init__(self, item, response_object):
+ self.response_object = response_object
+ self.item = item
+ self.fn = translations[self.item]
+
+
+ def get_error(self):
+ return self.response_object.get('error')
+
+
+ def get_result(self):
+ r = self.fn(self.response_object.get('result'))
+ if r == 'null':
+ return None
+ return r
+
+
+class EVMTx(Tx):
+
+ def __init__(self, block, tx_number, obj):
+ super(EVMTx, self).__init__(block, tx_number, obj)
+
+
+class EVMBlock(Block):
+
+ def tx(self, idx):
+ o = self.obj['transactions'][idx]
+ return Tx(self, idx, o)
+
+
+ def number(self):
+ return translate.hex_to_int(self.obj['number'])
+
+
+ def __str__(self):
+ return str('block {} {}'.format(self.number(), self.hash))
diff --git a/chainsyncer/client/evm/websocket.py b/chainsyncer/client/evm/websocket.py
@@ -0,0 +1,90 @@
+# standard imports
+import logging
+import uuid
+import json
+
+# third-party imports
+import websocket
+from hexathon import add_0x
+
+# local imports
+from .response import EVMResponse
+from chainsyncer.error import RequestError
+from chainsyncer.client.evm.response import EVMBlock
+
+logg = logging.getLogger()
+
+
+class EVMWebsocketClient:
+
+ def __init__(self, url):
+ self.url = url
+ self.conn = websocket.create_connection(url)
+
+
+ def __del__(self):
+ self.conn.close()
+
+
+ def block_number(self):
+ req_id = str(uuid.uuid4())
+ req = {
+ 'jsonrpc': '2.0',
+ 'method': 'eth_blockNumber',
+ 'id': str(req_id),
+ 'params': [],
+ }
+ self.conn.send(json.dumps(req))
+ r = self.conn.recv()
+ res = EVMResponse('block_number', json.loads(r))
+ err = res.get_error()
+ if err != None:
+ raise RequestError(err)
+
+ return res.get_result()
+
+
+ def block_by_integer(self, n):
+ req_id = str(uuid.uuid4())
+ nhx = '0x' + n.to_bytes(8, 'big').hex()
+ req = {
+ 'jsonrpc': '2.0',
+ 'method': 'eth_getBlockByNumber',
+ 'id': str(req_id),
+ 'params': [nhx, False],
+ }
+ self.conn.send(json.dumps(req))
+ r = self.conn.recv()
+ res = EVMResponse('get_block', json.loads(r))
+ err = res.get_error()
+ if err != None:
+ raise RequestError(err)
+
+ j = res.get_result()
+ if j == None:
+ return None
+ o = json.loads(j)
+ return EVMBlock(o['hash'], o)
+
+
+ def block_by_hash(self, hx_in):
+ req_id = str(uuid.uuid4())
+ hx = add_0x(hx_in)
+ req ={
+ 'jsonrpc': '2.0',
+ 'method': 'eth_getBlockByHash',
+ 'id': str(req_id),
+ 'params': [hx, False],
+ }
+ self.conn.send(json.dumps(req))
+ r = self.conn.recv()
+ res = EVMResponse('get_block', json.loads(r))
+ err = res.get_error()
+ if err != None:
+ raise RequestError(err)
+
+ j = res.get_result()
+ if j == None:
+ return None
+ o = json.loads(j)
+ return EVMBlock(o['hash'], o)
diff --git a/cic_syncer/client/translate.py b/chainsyncer/client/translate.py
diff --git a/cic_syncer/client/tx.py b/chainsyncer/client/tx.py
diff --git a/chainsyncer/db/__init__.py b/chainsyncer/db/__init__.py
@@ -0,0 +1,53 @@
+# standard imports
+import os
+import logging
+
+# local imports
+from chainsyncer.db.models.base import SessionBase
+
+logg = logging.getLogger()
+
+
+def dsn_from_config(config):
+ """Generate a dsn string from the provided config dict.
+
+ The config dict must include all well-known database connection parameters, and must implement the method "get(key)" to retrieve them. Any missing parameters will be be rendered as the literal string "None"
+
+ :param config: Configuration object
+ :type config: Varies
+ :returns: dsn string
+ :rtype: str
+ """
+ scheme = config.get('DATABASE_ENGINE')
+ if config.get('DATABASE_DRIVER') != None:
+ scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
+
+ dsn = ''
+ dsn_out = ''
+ if config.get('DATABASE_ENGINE') == 'sqlite':
+ dsn = '{}:///{}'.format(
+ scheme,
+ config.get('DATABASE_NAME'),
+ )
+ dsn_out = dsn
+
+ else:
+ dsn = '{}://{}:{}@{}:{}/{}'.format(
+ scheme,
+ config.get('DATABASE_USER'),
+ config.get('DATABASE_PASSWORD'),
+ config.get('DATABASE_HOST'),
+ config.get('DATABASE_PORT'),
+ config.get('DATABASE_NAME'),
+ )
+ dsn_out = '{}://{}:{}@{}:{}/{}'.format(
+ scheme,
+ config.get('DATABASE_USER'),
+ '***',
+ config.get('DATABASE_HOST'),
+ config.get('DATABASE_PORT'),
+ config.get('DATABASE_NAME'),
+ )
+ logg.debug('parsed dsn from config: {}'.format(dsn_out))
+ return dsn
+
diff --git a/cic_syncer/db/models/base.py b/chainsyncer/db/models/base.py
diff --git a/cic_syncer/db/models/sync.py b/chainsyncer/db/models/sync.py
diff --git a/chainsyncer/driver.py b/chainsyncer/driver.py
@@ -0,0 +1,81 @@
+# standard imports
+import uuid
+import logging
+import time
+
+logg = logging.getLogger()
+
+
+class Syncer:
+
+ running_global = True
+
+ def __init__(self, backend):
+ self.cursor = None
+ self.running = True
+ self.backend = backend
+ self.filter = []
+
+
+ def chain(self):
+ """Returns the string representation of the chain spec for the chain the syncer is running on.
+
+ :returns: Chain spec string
+ :rtype: str
+ """
+ return self.bc_cache.chain()
+
+
+ def add_filter(self, f):
+ self.filter.append(f)
+
+
+class MinedSyncer(Syncer):
+
+ def __init__(self, backend):
+ super(MinedSyncer, self).__init__(backend)
+
+
+ def loop(self, interval, getter):
+ while self.running and Syncer.running_global:
+ while True:
+ block_hash = self.get(getter)
+ if block_hash == None:
+ break
+ self.process(getter, block_hash)
+ time.sleep(interval)
+
+
+class HeadSyncer(MinedSyncer):
+
+ def __init__(self, backend):
+ super(HeadSyncer, self).__init__(backend)
+
+
+ def process(self, getter, block):
+ logg.debug('process {}'.format(block))
+ block = getter.block_by_hash(block.hash)
+ i = 0
+ tx = None
+ while True:
+ try:
+ #self.filter[0].handle(getter, block, None)
+ tx = block.tx(i)
+ logg.debug('tx {}'.format(tx))
+ self.backend.set(block.number(), i)
+ for f in self.filter:
+ f(getter, block, tx)
+ except IndexError as e:
+ self.backend.set(block.number() + 1, 0)
+ break
+ i += 1
+
+
+ def get(self, getter):
+ (block_number, tx_number) = self.backend.get()
+ block_hash = []
+ uu = uuid.uuid4()
+ res = getter.block_by_integer(block_number)
+ logg.debug('get {}'.format(res))
+
+ return res
diff --git a/cic_syncer/error.py b/chainsyncer/error.py
diff --git a/chainsyncer/runnable/tracker.py b/chainsyncer/runnable/tracker.py
@@ -0,0 +1,93 @@
+# standard imports
+import os
+import sys
+import logging
+import time
+import argparse
+import sys
+import re
+
+# third-party imports
+import confini
+from chainlib.eth.connection import HTTPConnection
+from chainsyncer.driver import HeadSyncer
+from chainsyncer.db import dsn_from_config
+from chainsyncer.db.models.base import SessionBase
+from chainsyncer.backend import SyncerBackend
+from chainsyncer.error import LoopDone
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+config_dir = '/usr/local/etc/cic-syncer'
+
+
+class Handler:
+
+ def __init__(self, method, domain):
+ self.method = method
+ self.domain = domain
+
+ def handle(self, getter, tx, chain):
+ logg.debug('noop tx {} chain {} method {} domain {}'.format(tx, chain, self.method, self.domain))
+handler = getattr(Handler, 'handle')
+
+
+argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
+argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
+argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('-v', help='be verbose', action='store_true')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v == True:
+ logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+# override args
+args_override = {
+ 'CHAIN_SPEC': getattr(args, 'i'),
+ 'ETH_PROVIDER': getattr(args, 'p'),
+ }
+config.dict_override(args_override, 'cli flag')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+#app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
+
+queue = args.q
+
+dsn = dsn_from_config(config)
+SessionBase.connect(dsn)
+
+c = HTTPConnection(config.get('ETH_PROVIDER'))
+chain = config.get('CHAIN_SPEC')
+
+
+def main():
+ block_offset = c.block_number()
+
+ syncer_backend = SyncerBackend.live(chain, 0)
+ syncer = HeadSyncer(syncer_backend)
+
+ try:
+ logg.debug('block offset {} {}'.format(block_offset, c))
+ syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), c)
+ except LoopDone as e:
+ sys.stderr.write("sync '{}' done at block {}\n".format(args.mode, e))
+
+ sys.exit(0)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/cic_syncer/backend.py b/cic_syncer/backend.py
@@ -1,196 +0,0 @@
-# standard imports
-import logging
-
-# local imports
-from cic_syncer.db.models.sync import BlockchainSync
-from cic_syncer.db.models.base import SessionBase
-
-logg = logging.getLogger()
-
-
-class SyncerBackend:
- """Interface to block and transaction sync state.
-
- :param chain_spec: Chain spec for the chain that syncer is running for.
- :type chain_spec: cic_registry.chain.ChainSpec
- :param object_id: Unique id for the syncer session.
- :type object_id: number
- """
- def __init__(self, chain_spec, object_id):
- self.db_session = None
- self.db_object = None
- self.chain_spec = chain_spec
- self.object_id = object_id
- self.connect()
- self.disconnect()
-
-
- def connect(self):
- """Loads the state of the syncer session with the given id.
- """
- self.db_session = SessionBase.create_session()
- q = self.db_session.query(BlockchainSync)
- q = q.filter(BlockchainSync.id==self.object_id)
- self.db_object = q.first()
- if self.db_object == None:
- raise ValueError('sync entry with id {} not found'.format(self.object_id))
-
-
- def disconnect(self):
- """Commits state of sync to backend.
- """
- self.db_session.add(self.db_object)
- self.db_session.commit()
- self.db_session.close()
-
-
- def chain(self):
- """Returns chain spec for syncer
-
- :returns: Chain spec
- :rtype chain_spec: cic_registry.chain.ChainSpec
- """
- return self.chain_spec
-
-
- def get(self):
- """Get the current state of the syncer cursor.
-
- :returns: Block and block transaction height, respectively
- :rtype: tuple
- """
- self.connect()
- pair = self.db_object.cursor()
- self.disconnect()
- return pair
-
-
- def set(self, block_height, tx_height):
- """Update the state of the syncer cursor
- :param block_height: Block height of cursor
- :type block_height: number
- :param tx_height: Block transaction height of cursor
- :type tx_height: number
- :returns: Block and block transaction height, respectively
- :rtype: tuple
- """
- self.connect()
- pair = self.db_object.set(block_height, tx_height)
- self.disconnect()
- return pair
-
-
- def start(self):
- """Get the initial state of the syncer cursor.
-
- :returns: Initial block and block transaction height, respectively
- :rtype: tuple
- """
- self.connect()
- pair = self.db_object.start()
- self.disconnect()
- return pair
-
-
- def target(self):
- """Get the target state (upper bound of sync) of the syncer cursor.
-
- :returns: Target block height
- :rtype: number
- """
- self.connect()
- target = self.db_object.target()
- self.disconnect()
- return target
-
-
- @staticmethod
- def first(chain):
- """Returns the model object of the most recent syncer in backend.
-
- :param chain: Chain spec of chain that syncer is running for.
- :type chain: cic_registry.chain.ChainSpec
- :returns: Last syncer object
- :rtype: cic_eth.db.models.BlockchainSync
- """
- return BlockchainSync.first(chain)
-
-
- @staticmethod
- def initial(chain, block_height):
- """Creates a new syncer session and commit its initial state to backend.
-
- :param chain: Chain spec of chain that syncer is running for.
- :type chain: cic_registry.chain.ChainSpec
- :param block_height: Target block height
- :type block_height: number
- :returns: New syncer object
- :rtype: cic_eth.db.models.BlockchainSync
- """
- object_id = None
- session = SessionBase.create_session()
- o = BlockchainSync(chain, 0, 0, block_height)
- session.add(o)
- session.commit()
- object_id = o.id
- session.close()
-
- return SyncerBackend(chain, object_id)
-
-
- @staticmethod
- def resume(chain, block_height):
- """Retrieves and returns all previously unfinished syncer sessions.
-
-
- :param chain: Chain spec of chain that syncer is running for.
- :type chain: cic_registry.chain.ChainSpec
- :param block_height: Target block height
- :type block_height: number
- :returns: Syncer objects of unfinished syncs
- :rtype: list of cic_eth.db.models.BlockchainSync
- """
- syncers = []
-
- session = SessionBase.create_session()
-
- object_id = None
-
- for object_id in BlockchainSync.get_unsynced(session=session):
- logg.debug('block syncer resume added previously unsynced sync entry id {}'.format(object_id))
- syncers.append(SyncerBackend(chain, object_id))
-
- (block_resume, tx_resume) = BlockchainSync.get_last_live_height(block_height, session=session)
- if block_height != block_resume:
- o = BlockchainSync(chain, block_resume, tx_resume, block_height)
- session.add(o)
- session.commit()
- object_id = o.id
- syncers.append(SyncerBackend(chain, object_id))
- logg.debug('block syncer resume added new sync entry from previous run id {}, start{}:{} target {}'.format(object_id, block_resume, tx_resume, block_height))
-
- session.close()
-
- return syncers
-
-
- @staticmethod
- def live(chain, block_height):
- """Creates a new open-ended syncer session starting at the given block height.
-
- :param chain: Chain spec of chain that syncer is running for.
- :type chain: cic_registry.chain.ChainSpec
- :param block_height: Target block height
- :type block_height: number
- :returns: "Live" syncer object
- :rtype: cic_eth.db.models.BlockchainSync
- """
- object_id = None
- session = SessionBase.create_session()
- o = BlockchainSync(chain, block_height, 0, None)
- session.add(o)
- session.commit()
- object_id = o.id
- session.close()
-
- return SyncerBackend(chain, object_id)
diff --git a/cic_syncer/client/evm/response.py b/cic_syncer/client/evm/response.py
@@ -1,52 +0,0 @@
-import json
-
-from cic_syncer.client import translate
-from cic_syncer.client.block import Block
-from cic_syncer.client.tx import Tx
-
-
-translations = {
- 'block_number': translate.hex_to_int,
- 'get_block': json.dumps,
- 'number': translate.hex_to_int,
- }
-
-
-class EVMResponse:
-
- def __init__(self, item, response_object):
- self.response_object = response_object
- self.item = item
- self.fn = translations[self.item]
-
-
- def get_error(self):
- return self.response_object.get('error')
-
-
- def get_result(self):
- r = self.fn(self.response_object.get('result'))
- if r == 'null':
- return None
- return r
-
-
-class EVMTx(Tx):
-
- def __init__(self, block, tx_number, obj):
- super(EVMTx, self).__init__(block, tx_number, obj)
-
-
-class EVMBlock(Block):
-
- def tx(self, idx):
- o = self.obj['transactions'][idx]
- return Tx(self, idx, o)
-
-
- def number(self):
- return translate.hex_to_int(self.obj['number'])
-
-
- def __str__(self):
- return str('block {} {}'.format(self.number(), self.hash))
diff --git a/cic_syncer/client/evm/websocket.py b/cic_syncer/client/evm/websocket.py
@@ -1,90 +0,0 @@
-# standard imports
-import logging
-import uuid
-import json
-
-# third-party imports
-import websocket
-from hexathon import add_0x
-
-# local imports
-from .response import EVMResponse
-from cic_syncer.error import RequestError
-from cic_syncer.client.evm.response import EVMBlock
-
-logg = logging.getLogger()
-
-
-class EVMWebsocketClient:
-
- def __init__(self, url):
- self.url = url
- self.conn = websocket.create_connection(url)
-
-
- def __del__(self):
- self.conn.close()
-
-
- def block_number(self):
- req_id = str(uuid.uuid4())
- req = {
- 'jsonrpc': '2.0',
- 'method': 'eth_blockNumber',
- 'id': str(req_id),
- 'params': [],
- }
- self.conn.send(json.dumps(req))
- r = self.conn.recv()
- res = EVMResponse('block_number', json.loads(r))
- err = res.get_error()
- if err != None:
- raise RequestError(err)
-
- return res.get_result()
-
-
- def get_block_by_integer(self, n):
- req_id = str(uuid.uuid4())
- nhx = '0x' + n.to_bytes(8, 'big').hex()
- req = {
- 'jsonrpc': '2.0',
- 'method': 'eth_getBlockByNumber',
- 'id': str(req_id),
- 'params': [nhx, False],
- }
- self.conn.send(json.dumps(req))
- r = self.conn.recv()
- res = EVMResponse('get_block', json.loads(r))
- err = res.get_error()
- if err != None:
- raise RequestError(err)
-
- j = res.get_result()
- if j == None:
- return None
- o = json.loads(j)
- return EVMBlock(o['hash'], o)
-
-
- def get_block_by_hash(self, hx_in):
- req_id = str(uuid.uuid4())
- hx = add_0x(hx_in)
- req ={
- 'jsonrpc': '2.0',
- 'method': 'eth_getBlockByHash',
- 'id': str(req_id),
- 'params': [hx, False],
- }
- self.conn.send(json.dumps(req))
- r = self.conn.recv()
- res = EVMResponse('get_block', json.loads(r))
- err = res.get_error()
- if err != None:
- raise RequestError(err)
-
- j = res.get_result()
- if j == None:
- return None
- o = json.loads(j)
- return EVMBlock(o['hash'], o)
diff --git a/cic_syncer/db/__init__.py b/cic_syncer/db/__init__.py
@@ -1,53 +0,0 @@
-# standard imports
-import os
-import logging
-
-# local imports
-from cic_syncer.db.models.base import SessionBase
-
-logg = logging.getLogger()
-
-
-def dsn_from_config(config):
- """Generate a dsn string from the provided config dict.
-
- The config dict must include all well-known database connection parameters, and must implement the method "get(key)" to retrieve them. Any missing parameters will be be rendered as the literal string "None"
-
- :param config: Configuration object
- :type config: Varies
- :returns: dsn string
- :rtype: str
- """
- scheme = config.get('DATABASE_ENGINE')
- if config.get('DATABASE_DRIVER') != None:
- scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
-
- dsn = ''
- dsn_out = ''
- if config.get('DATABASE_ENGINE') == 'sqlite':
- dsn = '{}:///{}'.format(
- scheme,
- config.get('DATABASE_NAME'),
- )
- dsn_out = dsn
-
- else:
- dsn = '{}://{}:{}@{}:{}/{}'.format(
- scheme,
- config.get('DATABASE_USER'),
- config.get('DATABASE_PASSWORD'),
- config.get('DATABASE_HOST'),
- config.get('DATABASE_PORT'),
- config.get('DATABASE_NAME'),
- )
- dsn_out = '{}://{}:{}@{}:{}/{}'.format(
- scheme,
- config.get('DATABASE_USER'),
- '***',
- config.get('DATABASE_HOST'),
- config.get('DATABASE_PORT'),
- config.get('DATABASE_NAME'),
- )
- logg.debug('parsed dsn from config: {}'.format(dsn_out))
- return dsn
-
diff --git a/cic_syncer/driver.py b/cic_syncer/driver.py
@@ -1,79 +0,0 @@
-# standard imports
-import uuid
-import logging
-import time
-
-logg = logging.getLogger()
-
-
-class Syncer:
-
- running_global = True
-
- def __init__(self, backend, handler):
- self.cursor = None
- self.running = True
- self.backend = backend
- self.filter = []
- self.handler = handler
-
-
- def chain(self):
- """Returns the string representation of the chain spec for the chain the syncer is running on.
-
- :returns: Chain spec string
- :rtype: str
- """
- return self.bc_cache.chain()
-
-
-
-class MinedSyncer(Syncer):
-
- def __init__(self, backend, handler):
- super(MinedSyncer, self).__init__(backend, handler)
-
-
- def loop(self, interval, getter):
- while self.running and Syncer.running_global:
- while True:
- block_hash = self.get(getter)
- if block_hash == None:
- break
- self.process(getter, block_hash)
- time.sleep(interval)
-
-
-class HeadSyncer(MinedSyncer):
-
- def __init__(self, backend, handler):
- super(HeadSyncer, self).__init__(backend, handler)
-
-
- def process(self, getter, block):
- logg.debug('process {}'.format(block))
- block = getter.get_block_by_hash(block.hash)
- i = 0
- tx = None
- while True:
- try:
- #self.filter[0].handle(getter, block, None)
- tx = block.tx(i)
- logg.debug('tx {}'.format(tx))
- self.backend.set(block.number(), i)
- for f in self.filter:
- f.handle(getter, block, tx)
- except IndexError as e:
- self.backend.set(block.number() + 1, 0)
- break
- i += 1
-
-
- def get(self, getter):
- (block_number, tx_number) = self.backend.get()
- block_hash = []
- uu = uuid.uuid4()
- res = getter.get_block_by_integer(block_number)
- logg.debug('get {}'.format(res))
-
- return res
diff --git a/cic_syncer/runnable/tracker.py b/cic_syncer/runnable/tracker.py
@@ -1,136 +0,0 @@
-# standard imports
-import os
-import sys
-import logging
-import time
-import argparse
-import sys
-import re
-
-# third-party imports
-import confini
-from cic_syncer.driver import HeadSyncer
-from cic_syncer.db import dsn_from_config
-from cic_syncer.db.models.base import SessionBase
-from cic_syncer.client.evm.websocket import EVMWebsocketClient
-from cic_syncer.backend import SyncerBackend
-from cic_syncer.error import LoopDone
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-config_dir = '/usr/local/etc/cic-syncer'
-
-
-class Handler:
-
- def __init__(self, method, domain):
- self.method = method
- self.domain = domain
-
- def handle(self, getter, tx, chain):
- logg.debug('noop tx {} chain {} method {} domain {}'.format(tx, chain, self.method, self.domain))
-handler = getattr(Handler, 'handle')
-
-
-argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
-argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
-argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
-argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('-v', help='be verbose', action='store_true')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-args = argparser.parse_args(sys.argv[1:])
-
-if args.v == True:
- logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
- logging.getLogger().setLevel(logging.DEBUG)
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-# override args
-args_override = {
- 'CIC_CHAIN_SPEC': getattr(args, 'i'),
- 'ETH_PROVIDER': getattr(args, 'p'),
- }
-config.dict_override(args_override, 'cli flag')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-
-#app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
-
-queue = args.q
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
-
-
-transfer_callbacks = []
-for cb in config.get('TASKS_SYNCER_CALLBACKS', '').split(','):
- task_split = cb.split(':')
- task_queue = queue
- if len(task_split) > 1:
- task_queue = task_split[0]
- task_pair = (task_split[1], task_queue)
- transfer_callbacks.append(task_pair)
-
-
-def tx_filter(w3, tx, rcpt, chain_spec):
- tx_hash_hex = tx.hash.hex()
- otx = Otx.load(tx_hash_hex)
- if otx == None:
- logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
- return None
- logg.info('otx found {}'.format(otx.tx_hash))
- s = celery.signature(
- 'cic_eth.queue.tx.set_final_status',
- [
- tx_hash_hex,
- rcpt.blockNumber,
- rcpt.status == 0,
- ],
- queue=queue,
- )
- t = s.apply_async()
- return t
-
-
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-c = EVMWebsocketClient(config.get('ETH_PROVIDER'))
-chain = config.get('CIC_CHAIN_SPEC')
-
-
-def main():
- block_offset = c.block_number()
-
- #syncer_backend = SyncerBackend.live(chain, block_offset+1)
- syncer_backend = SyncerBackend.live(chain, 0)
- syncer = HeadSyncer(syncer_backend, handler)
-
- for cb in config.get('TASKS_SYNCER_CALLBACKS', '').split(','):
- task_split = cb.split(':')
- task_queue = queue
- if len(task_split) > 1:
- task_queue = task_split[0]
- task_pair = (task_split[1], task_queue)
- h = Handler(task_pair[0], task_pair[1])
- syncer.filter.append(h)
-
- try:
- logg.debug('block offset {} {}'.format(block_offset, c))
- syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), c)
- except LoopDone as e:
- sys.stderr.write("sync '{}' done at block {}\n".format(args.mode, e))
-
- sys.exit(0)
-
-
-if __name__ == '__main__':
- main()
diff --git a/requirements.txt b/requirements.txt
@@ -1,9 +1,8 @@
-websocket-client==0.57.0
psycopg2==2.8.6
SQLAlchemy==1.3.20
py-evm==0.3.0a20
eth-tester==0.5.0b3
-web3==5.12.2
confini==0.3.6b2
semver==2.13.0
hexathon==0.0.1a2
+chainlib=0.0.1a4
