Update test- data & setup

* Fix issue with test setup
* Add nox task to import data
* Add nox tasks to start and stop db manually

.gitattributes

@@ -1 +1,2 @@
-poetry.lock     linguist-generated=true
+poetry.lock             linguist-generated=true
+test/data/csv/*.csv     linguist-generated=true

noxfile.py

@@ -66,8 +66,10 @@ def stop_db():
     stop = stop_db if with_db else nop
 
     start()
-    yield
-    stop()
+    try:
+        yield
+    finally:
+        stop()
 
 
 @nox.session(name="unit-tests", python=False)
@@ -88,3 +90,50 @@ def all_tests(session: Session) -> None:
     """Runs all tests (Unit and Integration)"""
     command = _test_command(_ROOT / "test")
     session.run(*command)
+
+
+@nox.session(name="db-start", python=False)
+def start_db(session: Session) -> None:
+    """Start a test database. For more details append '-- -h'"""
+
+    session.run(
+        "itde",
+        "spawn-test-environment",
+        "--environment-name",
+        "test",
+        "--database-port-forward",
+        "8563",
+        "--bucketfs-port-forward",
+        "2580",
+        "--docker-db-image-version",
+        "7.1.17",
+        "--db-mem-size",
+        "4GB",
+        external=True,
+    )
+
+
+@nox.session(name="db-stop", python=False)
+def stop_db(session: Session) -> None:
+    """Stop the test database"""
+    session.run("docker", "kill", "db_container_test", external=True)
+
+
+@nox.session(name="import-data", python=False)
+def import_data(session: Session) -> None:
+    """Import test data into the database"""
+    import sys
+
+    path = _ROOT / "test" / "integration"
+    data_dir = _ROOT / "test" / "data"
+    sys.path.append(f"{path}")
+    from conftest import DockerDataLoader
+
+    loader = DockerDataLoader(
+        dsn="127.0.0.1:8563",
+        username="sys",
+        password="exasol",
+        container_name='db_container_test',
+        data_directory=data_dir
+    )
+    loader.load()

scripts/generate_payments.py

@@ -0,0 +1,108 @@
+import sys
+import csv
+import decimal
+import random
+import datetime
+import argparse
+from pathlib import Path
+
+from faker import Faker
+from faker.providers import BaseProvider
+
+
+class PaymentsProvider(BaseProvider):
+
+    def date(self):
+        start_date = datetime.date(2018, 1, 1)
+        return start_date + datetime.timedelta(random.randint(1, 365))
+
+    def timestamp(self):
+        date = self.date()
+        time = datetime.time(
+            random.randint(0, 23),
+            random.randint(0, 59),
+            random.randint(0, 59),
+            random.randint(0, 999) * 1000
+        )
+        return datetime.datetime.combine(date, time)
+
+    def amount(self):
+        gross = decimal.Decimal(random.randint(0, 1000)) / 100
+        net = gross * decimal.Decimal('0.7')
+        return gross, net
+
+    def user_id(self, min=1, max=10000):
+        return random.randint(min, max)
+
+    def payment_id(self):
+        return '-'.join([
+            str(random.randint(100, 300)),
+            str(random.randint(100, 300)),
+            str(random.randint(100, 300))
+        ])
+
+
+def generate_payments(count, min, max):
+    fake = Faker()
+    fake.add_provider(PaymentsProvider)
+
+    for i in range(count):
+        gross, net = fake.amount()
+        yield (
+            fake.user_id(min, max),
+            fake.payment_id(),
+            fake.timestamp(),
+            gross,
+            net
+        )
+
+
+def _create_parser():
+    parser = argparse.ArgumentParser(
+        description="Generate a CSV file containing payments",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument(
+        'filename',
+        type=Path,
+        help='file the resulting CSV should be written to.'
+    )
+    parser.add_argument(
+        '-n', '--count',
+        type=int, default=10000,
+        help='Number of users to create.'
+    )
+    parser.add_argument(
+        '--min',
+        type=int, default=1,
+        help='Lower bound for user_id used in payments.'
+    )
+    parser.add_argument(
+        '--max',
+        type=int, default=10000,
+        help='Upper bound for user_id used in payments.'
+    )
+
+    return parser
+
+
+FAILURE = -1
+SUCCESS = 0
+
+
+def main(argv=None):
+    parser = _create_parser()
+    args = parser.parse_args(argv)
+    try:
+        with open(args.filename, 'w', newline='') as f:
+            writer = csv.writer(f, delimiter=',')
+            for user in generate_payments(count=args.count, min=args.min, max=args.max):
+                writer.writerow(user)
+    except Exception as ex:
+        print(f"Error while generating users, details: {ex}")
+        return FAILURE
+    return SUCCESS
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/generate_users.py

@@ -4,6 +4,7 @@
 import random
 import datetime
 import argparse
+from pathlib import Path
 
 from faker import Faker
 from faker.providers import BaseProvider
@@ -65,7 +66,7 @@ def _create_parser():
     )
     parser.add_argument(
         'filename',
-        type=argparse.FileType('w', encoding='utf8'),
+        type=Path,
         help='file the resulting CSV should be written to.'
     )
     parser.add_argument(
@@ -85,9 +86,10 @@ def main(argv=None):
     parser = _create_parser()
     args = parser.parse_args(argv)
     try:
-        writer = csv.writer(args.filename, delimiter=',')
-        for user in generate_users(count=args.count):
-            writer.writerow(user)
+        with open(args.filename, 'w', newline='') as f :
+            writer = csv.writer(f, delimiter=',')
+            for user in generate_users(count=args.count):
+                writer.writerow(user)
     except Exception as ex:
         print(f"Error while generating users, details: {ex}")
         return FAILURE