commit efe160dc233864277485194c23784318cfd0cebe
parent cc11ebc2d006921eb4e56b8fc42af7f04768fc2b
Author: lash <dev@holbrook.no>
Date: Mon, 17 Jun 2024 14:30:00 +0100
Adapt adder and implement get/put tx pair in lmdb
Diffstat:
6 files changed, 124 insertions(+), 72 deletions(-)
diff --git a/eth_cache/store/base.py b/eth_cache/store/base.py
@@ -1,3 +1,9 @@
+# external imports
+from chainlib.eth.tx import pack
+import json
+from hexathon import strip_0x
+
+# local imports
from . import StoreAction
class Store:
@@ -20,3 +26,38 @@ class Store:
def to_filepath(self, action, v):
return self.adder[action].to_filepath(v)
+
+
+ def put_tx(self, tx, include_data=False):
+ raw = pack(tx.src, self.chain_spec)
+ #tx_hash_dirnormal = strip_0x(tx.hash).upper()
+ tx_hash_dirnormal = strip_0x(tx.hash)
+ tx_hash_bytes = bytes.fromhex(tx_hash_dirnormal)
+ #self.tx_raw_dir.add(tx_hash_bytes, raw)
+ self.add(StoreAction.TX_RAW, tx_hash_bytes, raw)
+ addresses = []
+ if self.address_rules != None:
+ for a in tx.outputs + tx.inputs:
+ if a not in addresses:
+ addresses.append(a)
+ else:
+ for a in tx.outputs + tx.inputs:
+ addresses.append(a)
+
+ if include_data:
+ src = json.dumps(tx.src).encode('utf-8')
+ #self.tx_dir.add(bytes.fromhex(strip_0x(tx.hash)), src)
+ self.add(StoreAction.TX, bytes.fromhex(strip_0x(tx.hash)), src)
+
+ if tx.result != None:
+ rcpt_src = tx.result.src
+ rcpt_src = json.dumps(rcpt_src).encode('utf-8')
+ #self.rcpt_dir.add(bytes.fromhex(strip_0x(tx.hash)), rcpt_src)
+ self.add(StoreAction.RCPT, bytes.fromhex(strip_0x(tx.hash)), rcpt_src)
+
+ for a in addresses:
+ self.put_address(tx, a)
+
+
+ def put_address(self, tx, address):
+ raise NotImplementedError()
diff --git a/eth_cache/store/file.py b/eth_cache/store/file.py
@@ -7,7 +7,6 @@ import logging
from hexathon import strip_0x
from chainlib.eth.tx import (
Tx,
- pack,
)
from leveldir.numeric import NumDir
from leveldir.hex import HexDir
@@ -19,7 +18,6 @@ from eth_cache.store.base import StoreAction
logg = logging.getLogger(__name__)
-
class FileStore(FsStore):
@@ -28,60 +26,40 @@ class FileStore(FsStore):
address_dir_adder.add_dir(dirhsh, address, b'')
- def put_tx(self, tx, include_data=False):
- raw = pack(tx.src, self.chain_spec)
- tx_hash_dirnormal = strip_0x(tx.hash).upper()
- tx_hash_bytes = bytes.fromhex(tx_hash_dirnormal)
- #self.tx_raw_dir.add(tx_hash_bytes, raw)
- self.add(StoreAction.TX_RAW, tx_hash_bytes, raw)
- addresses = []
- if self.address_rules != None:
- for a in tx.outputs + tx.inputs:
- if a not in addresses:
- addresses.append(a)
- else:
- for a in tx.outputs + tx.inputs:
- addresses.append(a)
-
- for a in addresses:
- a_hex = strip_0x(a).upper()
- a = bytes.fromhex(a_hex)
- #self.address_dir.add_dir(tx_hash_dirnormal, a, b'')
- #address_dir_adder = self.adder[StoreAction.ADDRESS]
- #address_dir_adder.add_dir(tx_hash_dirnormal, a, b'')
- self.add_address_dir(tx_hash_dirnormal, a)
- #dirpath = self.address_dir.to_filepath(a_hex)
- #dirpath = address_dir_adder.to_filepath(a_hex)
- dirpath = self.to_filepath(StoreAction.ADDRESS, a_hex)
- fp = os.path.join(dirpath, '.start')
- num = tx.block.number
- num_compare = 0
- try:
- f = open(fp, 'rb')
- r = f.read(8)
- f.close()
- num_compare = int.from_bytes(r, 'big')
- except FileNotFoundError:
- pass
-
- if num_compare == 0 or num < num_compare:
- logg.debug('recoding new start block {} for {}'.format(num, a))
- num_bytes = num.to_bytes(8, 'big')
- f = open(fp, 'wb')
- f.write(num_bytes)
- f.close()
+ def put_address(self, tx, address):
+ a_hex = strip_0x(address).upper()
+ a = bytes.fromhex(a_hex)
+ #self.address_dir.add_dir(tx_hash_dirnormal, a, b'')
+ #address_dir_adder = self.adder[StoreAction.ADDRESS]
+ #address_dir_adder.add_dir(tx_hash_dirnormal, a, b'')
+ #self.add_address_dir(tx_hash_dirnormal, a)
+ tx_hash = strip_0x(tx.hash).upper()
+ self.add_address_dir(tx_hash, a)
+ #dirpath = self.address_dir.to_filepath(a_hex)
+ #dirpath = address_dir_adder.to_filepath(a_hex)
+ dirpath = self.to_filepath(StoreAction.ADDRESS, a_hex)
+ fp = os.path.join(dirpath, '.start')
+ num = tx.block.number
+ num_compare = 0
+ try:
+ f = open(fp, 'rb')
+ r = f.read(8)
+ f.close()
+ num_compare = int.from_bytes(r, 'big')
+ except FileNotFoundError:
+ pass
+
+ if num_compare == 0 or num < num_compare:
+ logg.debug('recoding new start block {} for {}'.format(num, a))
+ num_bytes = num.to_bytes(8, 'big')
+ f = open(fp, 'wb')
+ f.write(num_bytes)
+ f.close()
- if include_data:
- src = json.dumps(tx.src).encode('utf-8')
- #self.tx_dir.add(bytes.fromhex(strip_0x(tx.hash)), src)
- self.add(StoreAction.TX, bytes.fromhex(strip_0x(tx.hash)), src)
-
- if tx.result != None:
- rcpt_src = tx.result.src
- rcpt_src = json.dumps(rcpt_src).encode('utf-8')
- #self.rcpt_dir.add(bytes.fromhex(strip_0x(tx.hash)), rcpt_src)
- self.add(StoreAction.RCPT, bytes.fromhex(strip_0x(tx.hash)), rcpt_src)
+ def put_tx(self, tx, include_data=False):
+ super(FileStore, self).put_tx(tx, include_data=include_data)
+
def put_block(self, block, include_data=False):
diff --git a/eth_cache/store/fs.py b/eth_cache/store/fs.py
@@ -12,6 +12,8 @@ def chain_dir_for(chain_spec, base_dir=default_base_dir):
chain_dir = os.path.join(base_dir, str(chain_spec).replace(':', '/'))
return os.path.join(chain_dir, 'eth_cache')
+
+
class FsStore(Store):
def __init__(self, chain_spec, cache_root=None, address_rules=None):
diff --git a/eth_cache/store/lmdb.py b/eth_cache/store/lmdb.py
@@ -12,37 +12,64 @@ from chainlib.eth.tx import (
)
# local imports
+from . import StoreAction
from eth_cache.store.fs import FsStore
logg = logging.getLogger(__name__)
+def to_path_key(path, k):
+ if type(k) != bytes:
+ k = k.encode('utf-8')
+ if path[len(path)-1] != '/':
+ path += '/'
+ return path.encode('utf-8') + k
+
+class LmdbStoreAdder:
+
+ def __init__(self, action, db):
+ self.action = action
+ self.db = db
+
+
+ def add(self, k, v):
+ print("adding {} {} {}\n".format(self.action, k, v))
+ dbk = to_path_key(self.action.value, k)
+ with self.db.begin(write=True) as tx:
+ tx.put(dbk, v)
+
+
class LmdbStore(FsStore):
def __init__(self, chain_spec, cache_root=None, address_rules=None):
super(LmdbStore, self).__init__(chain_spec, cache_root=cache_root, address_rules=address_rules)
self.db = lmdb.open(self.cache_dir, create=True)
-
-
- def __to_path_key(self, path, k):
- if path[len(path)-1] != '/':
- path += '/'
- return path.encode('utf-8') + k
+ for action in StoreAction:
+ self.register_adder(action, LmdbStoreAdder(action, self.db))
def put_tx(self, tx, include_data=False):
- raw = pack(tx.src, self.chain_spec)
- tx_hash_dirnormal = strip_0x(tx.hash).upper()
- tx_hash_bytes = bytes.fromhex(tx_hash_dirnormal)
- k = self.__to_path_key('tx_raw', tx_hash_bytes)
- with self.db.begin(write=True) as tx:
- tx.put(k, raw)
+ super(LmdbStore, self).put_tx(tx, include_data=include_data)
+# raw = pack(tx.src, self.chain_spec)
+# tx_hash_dirnormal = strip_0x(tx.hash).upper()
+# tx_hash_bytes = bytes.fromhex(tx_hash_dirnormal)
+# k = self.__to_path_key('tx_raw', tx_hash_bytes)
+# with self.db.begin(write=True) as tx:
+# tx.put(k, raw)
def get_tx(self, tx_hash):
- self.__to_path_key(self, path, key)
+ print("getting {}\n".format(tx_hash))
+ k = bytes.fromhex(tx_hash)
+ k = to_path_key(StoreAction.TX.value, k) #self.adder[Store.ActionTX].get()
+ with self.db.begin() as tx:
+ return tx.get(k)
+
+
+ def put_address(self, tx, address):
+ pass
def __str__(self):
- return 'RockDbStore: root {}'.format(self.cache_dir)
+ return 'LmdbStore: root {}'.format(self.cache_dir)
diff --git a/run_tests.sh b/run_tests.sh
@@ -6,7 +6,7 @@ set -x
default_pythonpath=$PYTHONPATH:.
export PYTHONPATH=${default_pythonpath:-.}
>&2 echo using pythonpath $PYTHONPATH
-for f in `ls tests/*.py`; do
+for f in `ls tests/test_*.py`; do
python $f
done
set +x
diff --git a/tests/test_lmdb.py b/tests/test_lmdb.py
@@ -1,5 +1,9 @@
# standard imports
import unittest
+import json
+
+# external imports
+from chainlib.eth.address import is_same_address
# local imports
from eth_cache.store.lmdb import LmdbStore
@@ -15,9 +19,9 @@ class TestCacheBasic(TestCache):
def test_tx(self):
self.store.put_tx(self.tx, include_data=True)
- #j = self.store.get_tx(self.tx.hash)
- #tx = json.loads(j)
- #self.assertTrue(is_same_address(tx['hash'], self.tx.hash))
+ j = self.store.get_tx(self.tx.hash)
+ tx = json.loads(j)
+ self.assertTrue(is_same_address(tx['hash'], self.tx.hash))
if __name__ == '__main__':
