csv dest: add improved error handling in tests

ingestr/src/loader.py

@@ -5,11 +5,11 @@
 from contextlib import contextmanager
 from typing import Generator
 
-from pyarrow.parquet import ParquetFile
-
+from pyarrow.parquet import ParquetFile  # type: ignore
 
 PARQUET_BATCH_SIZE = 64
 
+
 class UnsupportedLoaderFileFormat(Exception):
     pass
 
@@ -48,6 +48,7 @@ def jsonlfile(filepath: str):
     def reader(fd):
         for line in fd:
             yield json.loads(line.decode().strip())
+
     with gzip.open(filepath) as fd:
         yield reader(fd)
 
@@ -63,6 +64,6 @@ def parquetfile(filepath: str):
     def reader(pf: ParquetFile):
         for batch in pf.iter_batches(PARQUET_BATCH_SIZE):
             yield from batch.to_pylist()
-        
+
     with open(filepath, "rb") as fd:
         yield reader(ParquetFile(fd))

ingestr/src/loader_test.py

@@ -1,13 +1,17 @@
 import csv
 import gzip
 import json
+import logging
 import os
 import tempfile
 from typing import List
 
-import pyarrow.parquet
+import pyarrow.parquet  # type: ignore
 import pytest
-from loader import load_dlt_file
+
+from ingestr.src.loader import load_dlt_file
+
+logger = logging.getLogger(__name__)
 
 TESTDATA = [
     {"name": "Jhon", "email": "[email protected]"},
@@ -57,7 +61,10 @@ def testfiles():
         yield files
 
         for file in files:
-            os.remove(file)
+            try:
+                os.remove(file)
+            except Exception as e:
+                logger.error(f"error removing temporary file {file}", exc_info=e)
 
 
 def test_loader(testfiles):