chainsyncer

commit 97c2d41df341c216e3a073c072ab19346172f17a
parent 891f90ae5f1a2cd38998fdbd6a7e7b5e3791fa35
Author: lash <dev@holbrook.no>
Date:   Wed, 20 Apr 2022 12:31:32 +0000

Factor out sync state scanner

Diffstat:
M
chainsyncer/state/base.py
 | 
26
+++++++++++++-------------
M
chainsyncer/store/base.py
 | 
14
++++++++++++++
M
chainsyncer/store/fs.py
 | 
15
+++++----------

3 files changed, 32 insertions(+), 23 deletions(-)
diff --git a/chainsyncer/state/base.py b/chainsyncer/state/base.py
@@ -12,7 +12,7 @@ re_processedname = r'^_?[A-Z,\.]*$'
 # TODO: properly clarify interface shared with syncfsstore, move to filter module?
 class SyncState:
 
-    def __init__(self, state_store, scan_path=None):
+    def __init__(self, state_store, scan=None):
         self.state_store = state_store
         self.digest = b'\x00' * 32
         self.summed = False
@@ -36,7 +36,7 @@ class SyncState:
         self.all = self.state_store.all
         self.started = False
 
-        self.scan_path = scan_path
+        self.scan = scan
 
 
     def __verify_sum(self, v):
@@ -72,17 +72,17 @@ class SyncState:
                 k = self.state_store.from_name(v)
                 self.state_store.sync(k)
                 self.__syncs[v] = True
-            if self.scan_path != None:
-                for v in os.listdir(self.scan_path):
-                    if re.match(re_processedname, v):
-                        k = None
-                        try:
-                            k = self.state_store.from_elements(v)
-                            self.state_store.alias(v, k)
-                        except ValueError:
-                            k = self.state_store.from_name(v)
-                        self.state_store.sync(k)
-                        self.__syncs[v] = True
+            if self.scan != None:
+                ks = self.scan()
+                for v in ks: #os.listdir(self.scan_path):
+                    k = None
+                    try:
+                        k = self.state_store.from_elements(v)
+                        self.state_store.alias(v, k)
+                    except ValueError:
+                        k = self.state_store.from_name(v)
+                    self.state_store.sync(k)
+                    self.__syncs[v] = True
             self.synced = True
         self.connected = True
 
diff --git a/chainsyncer/store/base.py b/chainsyncer/store/base.py
@@ -2,7 +2,9 @@
 import logging
 
 # local imports
+from shep.persist import PersistedState
 from shep.error import StateInvalid
+from chainsyncer.state import SyncState
 from chainsyncer.error import (
         LockError,
         FilterDone,
@@ -154,6 +156,18 @@ class SyncStore:
         self.started = False
 
 
+    def setup_sync_state(self, factory, event_callback):
+        self.state = PersistedState(factory.add, 2, event_callback=event_callback)
+        self.state.add('SYNC')
+        self.state.add('DONE')
+
+
+    def setup_filter_state(self, factory, event_callback):
+        filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=event_callback)
+        self.filter_state = SyncState(filter_state_backend, scan=factory.ls)
+        self.filters = []
+
+
     def register(self, fltr):
         self.filters.append(fltr)
         self.filter_state.register(fltr)
diff --git a/chainsyncer/store/fs.py b/chainsyncer/store/fs.py
@@ -5,10 +5,8 @@ import logging
 
 # external imports
 from shep.store.file import SimpleFileStoreFactory
-from shep.persist import PersistedState
 
 # local imports 
-from chainsyncer.state import SyncState
 from chainsyncer.store import (
         SyncItem,
         SyncStore,
@@ -45,17 +43,14 @@ class SyncFsStore(SyncStore):
 
         logg.info('session id {} resolved {} path {}'.format(session_id, self.session_id, self.session_path))
 
-        factory = SimpleFileStoreFactory(self.session_path, binary=True)
-        self.state = PersistedState(factory.add, 2, event_callback=state_event_callback)
-        self.state.add('SYNC')
-        self.state.add('DONE')
+        base_sync_path = os.path.join(self.session_path, 'sync')
+        factory = SimpleFileStoreFactory(base_sync_path, binary=True)
+        self.setup_sync_state(factory, state_event_callback)
 
         base_filter_path = os.path.join(self.session_path, 'filter')
         factory = SimpleFileStoreFactory(base_filter_path, binary=True)
-        filter_state_backend = PersistedState(factory.add, 0, check_alias=False, event_callback=filter_state_event_callback)
-        self.filter_state = SyncState(filter_state_backend, scan_path=base_filter_path)
-        self.filters = [] # used by SyncSession
-   
+        self.setup_filter_state(factory, filter_state_event_callback)
+
 
     def __create_path(self, base_path, default_path, session_id=None):
         logg.debug('fs store path {} does not exist, creating'.format(self.session_path))