Merge pull request #63 from subsquid/octo-gone/fix/substrate-writer-r…

…equest-rate

fix: add timeout if no new blocks returned from ingester

sqa/eth/ingest/util.py

@@ -169,7 +169,7 @@ def transactions_root(transactions: list[Transaction]) -> str:
             ])
         elif tx['type'] == '0x65':
             # https://github.com/OffchainLabs/go-ethereum/blob/7503143fd13f73e46a966ea2c42a058af96f7fcf/core/types/arb_types.go#L43
-            pass
+            raise NotImplementedError('cannot encode tx with type 0x65')
             # trie[path] = b'\x65' + rlp.encode([
             #     qty2int(tx['chainId']),
             #     decode_hex(tx['from']),
@@ -182,7 +182,7 @@ def transactions_root(transactions: list[Transaction]) -> str:
             # ])
         elif tx['type'] == '0x66':
             # https://github.com/OffchainLabs/go-ethereum/blob/7503143fd13f73e46a966ea2c42a058af96f7fcf/core/types/arb_types.go#L104
-            pass
+            raise NotImplementedError('cannot encode tx with type 0x66')
             # trie[path] = b'\x66' + rlp.encode([
             #     qty2int(tx['chainId']),
             #     decode_hex(tx['requestId']),

sqa/writer/ingest.py

@@ -30,6 +30,7 @@ def ingest_from_service(
         data_range['to'] = last_block
 
     while data_range['from'] <= last_block:
+        data_ingested = False
         try:
             with httpx.stream('POST', service_url, json=data_range, timeout=httpx.Timeout(None)) as res:
                 res.raise_for_status()
@@ -38,9 +39,14 @@ def ingest_from_service(
                     height = get_block_height(block)
                     data_range['from'] = height + 1
                     yield block
+                    data_ingested = True
         except (httpx.NetworkError, httpx.RemoteProtocolError):
             LOG.exception('data streaming error, will pause for 5 sec and try again')
             time.sleep(5)
+        else:
+            if not data_ingested:
+                LOG.info('no blocks were found. waiting 5 min for a new try')
+                time.sleep(300)
 
 
 # `res.iter_lines()` uses `str.splitlines()` under the hood, which splits "too much".