commit 6a94e28ad87885a17a0d10caf6e52cb8be26396a
parent 6201420ad2c0d0794fa963b2073a1b0e2240ab2a
Author: nolash <dev@holbrook.no>
Date: Thu, 15 Apr 2021 16:23:30 +0200
WIP fix filter handling in file backend
Diffstat:
3 files changed, 77 insertions(+), 27 deletions(-)
diff --git a/chainsyncer/backend/file.py b/chainsyncer/backend/file.py
@@ -156,20 +156,23 @@ class SyncerFileBackend:
def get(self):
- return ((self.block_height_cursor, self.tx_index_cursor), self.filter)
+ logg.debug('filter {}'.format(self.filter.hex()))
+ return ((self.block_height_cursor, self.tx_index_cursor), int.from_bytes(self.filter, 'little'))
def set(self, block_height, tx_index):
self.__set(block_height, tx_index, 'cursor')
- cursor_path = os.path.join(self.object_data_dir, 'filter')
- f = open(cursor_path, 'r+b')
- f.seek(8)
- l = len(self.filter)
- c = 0
- while c < l:
- c += f.write(self.filter)
- f.close()
+# cursor_path = os.path.join(self.object_data_dir, 'filter')
+# f = open(cursor_path, 'r+b')
+# f.seek(8)
+# l = len(self.filter)
+# c = 0
+# while c < l:
+# c += f.write(self.filter[c:])
+# f.close()
+
+ return ((self.block_height_cursor, self.tx_index_cursor), int.from_bytes(self.filter, 'little'))
def __set(self, block_height, tx_index, category):
@@ -198,15 +201,16 @@ class SyncerFileBackend:
o.__set(target_block_height, 0, 'target')
o.__set(start_block_height, 0, 'offset')
- return uu
+ #return uu
+ return o
def target(self):
- return ((self.block_height_target, self.tx_index_target), None,)
+ return (self.block_height_target, 0,)
def start(self):
- return ((self.block_height_offset, self.tx_index_offset), None,)
+ return ((self.block_height_offset, self.tx_index_offset), 0,)
@staticmethod
@@ -289,3 +293,15 @@ class SyncerFileBackend:
b = self.filter_count.to_bytes(8, byteorder='big')
f.write(b)
f.close()
+
+
+ def reset_filter(self):
+ self.filter = b'\x00' * len(self.filter)
+ cursor_path = os.path.join(self.object_data_dir, 'filter')
+ f = open(cursor_path, 'r+b')
+ f.seek(8)
+ l = len(self.filter)
+ c = 0
+ while c < l:
+ c += f.write(self.filter[c:])
+ f.close()
diff --git a/chainsyncer/driver.py b/chainsyncer/driver.py
@@ -170,6 +170,7 @@ class HistorySyncer(HeadSyncer):
if block_number == None:
raise AttributeError('backend has no future target. Use HeadSyner instead')
self.block_target = block_number
+ logg.debug('block target {}'.format(self.block_target))
def get(self, conn):
diff --git a/tests/test_interrupt.py b/tests/test_interrupt.py
@@ -2,6 +2,7 @@
import logging
import unittest
import os
+import tempfile
# external imports
from chainlib.chain import ChainSpec
@@ -9,6 +10,10 @@ from chainlib.chain import ChainSpec
# local imports
from chainsyncer.backend.memory import MemBackend
from chainsyncer.backend.sql import SyncerBackend
+from chainsyncer.backend.file import (
+ SyncerFileBackend,
+ data_dir_for,
+ )
# test imports
from tests.base import TestBase
@@ -61,21 +66,35 @@ class TestInterrupt(TestBase):
def setUp(self):
super(TestInterrupt, self).setUp()
- self.filters = [
+
+ self.backend = None
+ self.conn = MockConn()
+ self.vectors = [
+ [4, 3, 2],
+ [6, 4, 2],
+ [6, 5, 2],
+ [6, 4, 3],
+ ]
+
+
+ def assert_filter_interrupt(self, vector):
+
+ logg.debug('running vector {} {}'.format(str(self.backend), vector))
+
+ z = 0
+ for v in vector:
+ z += v
+
+ syncer = TestSyncer(self.backend, vector)
+
+ filters = [
CountFilter('foo'),
CountFilter('bar'),
NaughtyCountExceptionFilter('xyzzy', croak_on=3),
CountFilter('baz'),
]
- self.backend = None
- self.conn = MockConn()
-
-
- def assert_filter_interrupt(self):
-
- syncer = TestSyncer(self.backend, [4, 3, 2])
- for fltr in self.filters:
+ for fltr in filters:
syncer.add_filter(fltr)
try:
@@ -87,19 +106,33 @@ class TestInterrupt(TestBase):
self.assertGreater(fltr, 0)
syncer.loop(0.1, self.conn)
- for fltr in self.filters:
+ for fltr in filters:
logg.debug('{} {}'.format(str(fltr), fltr.c))
- self.assertEqual(fltr.c, 9)
+ self.assertEqual(fltr.c, z)
+ @unittest.skip('foo')
def test_filter_interrupt_memory(self):
- self.backend = MemBackend(self.chain_spec, None, target_block=4)
- self.assert_filter_interrupt()
+ for vector in self.vectors:
+ self.backend = MemBackend(self.chain_spec, None, target_block=len(vector))
+ self.assert_filter_interrupt(vector)
+ def test_filter_interrpt_file(self):
+ for vector in self.vectors:
+ d = tempfile.mkdtemp()
+ #os.makedirs(data_dir_for(self.chain_spec, 'foo', d))
+ self.backend = SyncerFileBackend.initial(self.chain_spec, len(vector), base_dir=d) #'foo', base_dir=d)
+ self.assert_filter_interrupt(vector)
+
+
+ @unittest.skip('foo')
def test_filter_interrupt_sql(self):
- self.backend = SyncerBackend.initial(self.chain_spec, 4)
- self.assert_filter_interrupt()
+ for vector in self.vectors:
+ self.backend = SyncerBackend.initial(self.chain_spec, len(vector))
+ self.assert_filter_interrupt(vector)
+
+
if __name__ == '__main__':
