Merge branch 'auto-import' into release-02

src/senaite/timeseries/browser/overrides/auto_import_results.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+
+import datetime
+import os
+from bika.lims import api
+from plone.protect.interfaces import IDisableCSRFProtection
+from senaite.core.exportimport.auto_import_results import AutoImportResultsView as AIRV
+from zope.interface import alsoProvides
+
+CR = "\n"
+LOGFILE = "logs.log"
+INDEXFILE = "imported.csv"
+IGNORE = ",".join([INDEXFILE, LOGFILE])
+
+
+class AutoImportResultsView(AIRV):
+    def __call__(self):
+        # disable CSRF because
+        alsoProvides(self.request, IDisableCSRFProtection)
+        # run auto import of results
+        self.auto_import_results()
+        # return the concatenated logs
+        logs = CR.join(self.logs)
+        return logs
+
+    def auto_import_results(self):
+        """Auto import all new instrument import files"""
+        for brain in self.query_active_instruments():
+            interfaces = []
+            instrument = api.get_object(brain)
+            # instrument_title = api.get_title(instrument)
+
+            # get a valid interface -> folder mapping
+            mapping = self.get_interface_folder_mapping(instrument)
+
+            # If Import Interface ID is specified in request, then auto-import
+            # will run only that interface. Otherwise all available interfaces
+            # of this instruments
+            if self.request.get("interface"):
+                interfaces.append(self.request.get("interface"))
+            else:
+                interfaces = mapping.keys()
+
+            if not interfaces:
+                # self.log("No active interfaces defined", instrument=instrument)
+                continue
+
+            if (
+                "senaite.timeseries.importer.timeseries.timeseries_import"
+                not in interfaces
+            ):
+                continue
+
+            # self.log(
+            #     "Auto import for '%s' started ..." % instrument_title,
+            #     instrument=instrument,
+            #     level="info",
+            # )
+            # import instrument results from all configured interfaces
+            for interface in interfaces:
+                folder = mapping.get(interface)
+                # check if instrument import folder exists
+                if not os.path.exists(folder):
+                    self.log(
+                        "Interface %s: Folder %s does not exist" % (interface, folder),
+                        instrument=instrument,
+                        interface=interface,
+                        level="error",
+                    )
+                    continue
+
+                log_file_path = os.path.join(folder, LOGFILE)
+                # get all files in the instrument folder
+                if os.path.exists(log_file_path):
+                    log_file_mod_date = datetime.datetime.fromtimestamp(
+                        os.path.getmtime(log_file_path)
+                    )
+                    allfiles = self.list_files(
+                        folder, ignore=IGNORE, exclude_before=log_file_mod_date
+                    )
+                else:
+                    allfiles = self.list_files(folder, ignore=IGNORE)
+
+                if len(allfiles) == 0:
+                    self.log(
+                        "Interface '%s': Folder %s has no new files"
+                        % (interface, folder),
+                        instrument=instrument,
+                        interface=interface,
+                        level="info",
+                    )
+                    # crate auto import log object
+                    logobj = self.create_autoimportlog(instrument, interface, "")
+                    # write import logs
+                    self.write_autologs(logobj, self.logs, "info")
+                    continue
+
+                # import results file
+                for f in allfiles:
+                    self.import_results(instrument, interface, folder, f)
+
+            # self.log("Auto-Import finished")
+
+    def list_files(self, folder, ignore="", exclude_before=None):
+        """Returns all files in folder and its subfolders, excluding ignored files and files modified before a given date.
+
+        :param folder: folder path
+        :param ignore: comma-separated list of file names to ignore
+        :param exclude_before: datetime object; exclude files modified before this date
+        """
+        files = []
+        ignore_files = ignore.split(",") if ignore else []
+
+        for root, _, filenames in os.walk(folder):
+            for f in filenames:
+                # skip hidden files
+                if f.startswith("."):
+                    continue
+                # skip ignored files
+                if f in ignore_files:
+                    continue
+
+                file_path = os.path.join(root, f)
+                # skip files modified before the exclude_before date
+                if exclude_before:
+                    file_mod_time = datetime.datetime.fromtimestamp(
+                        os.path.getmtime(file_path)
+                    )
+                    if file_mod_time < exclude_before:
+                        continue
+
+                files.append(file_path)
+
+        return files

src/senaite/timeseries/browser/overrides/overrides.zcml

@@ -1,6 +1,4 @@
 <configure xmlns="http://namespaces.zope.org/zope"
-       xmlns:zcml="http://namespaces.zope.org/browser"
-       xmlns:fss="http://namespaces.zope.org/browser"
        xmlns:plone="http://namespaces.plone.org/plone"
        xmlns:browser="http://namespaces.zope.org/browser"
        xmlns:five="http://namespaces.zope.org/five"
@@ -31,7 +29,15 @@
           name="senaite.core.samplesection.lab_analyses"
           manager="senaite.core.browser.viewlets.interfaces.ISampleSection"
           class=".sampleanalyses.LabAnalysesViewlet"
-          layer="senaite.core.interfaces.ISenaiteCore"
+          layer="senaite.timeseries.interfaces.ISenaiteTimeseriesLayer"
           permission="zope2.View" />
 
+      <!-- Auto Import Results View -->
+      <browser:page
+          for="*"
+          name="auto_import_timeseries_results"
+          class="senaite.timeseries.browser.overrides.auto_import_results.AutoImportResultsView"
+          permission="zope.Public"
+          />
+
 </configure>

src/senaite/timeseries/importer/timeseries.py

@@ -7,6 +7,8 @@
 from bika.lims import bikaMessageFactory as _
 from senaite.core.exportimport.instruments import IInstrumentAutoImportInterface
 from senaite.core.exportimport.instruments import IInstrumentImportInterface
+from senaite.core.exportimport.instruments.importer import ALLOWED_ANALYSIS_STATES
+from senaite.core.exportimport.instruments.importer import ALLOWED_SAMPLE_STATES
 from senaite.core.exportimport.instruments.importer import AnalysisResultsImporter
 from bika.lims.utils import t
 from senaite.instruments.instrument import InstrumentXLSResultsFileParser
@@ -22,7 +24,7 @@ def __init__(self, infile, worksheet=2, encoding=None, instrument_uid=None):
         )
         self._end_header = False
         self._ar_id = None
-        # self._instrument = api.get_object(instrument_uid, None)
+        self._instrument = api.get_object(instrument_uid, None)
         self._analysis_service = None
         self._column_headers = []
         self._result = []
@@ -44,6 +46,9 @@ def parse_headerline(self, line):
 
         if splitted[0] == "Sample ID":
             self._ar_id = splitted[1].strip()
+            if self._ar_id is None or len(self._ar_id) == 0:
+                self.err("Sample ID not found")
+                return -1
 
         if splitted[0] == "Analysis":
             keyword = splitted[1].strip()
@@ -141,36 +146,52 @@ class timeseries_import(object):
     def __init__(self, context):
         self.context = context
         self.request = None
+        self.allowed_sample_states = ALLOWED_SAMPLE_STATES
+        self.allowed_analysis_states = ALLOWED_ANALYSIS_STATES
+        self.errors = []
+        self.logs = []
+        self.warns = []
 
-    def Import(self, context, request):
+    def Import(self, context, request, parser=None):
         """Import Form"""
-        infile = request.form["instrument_results_file"]
-        fileformat = request.form.get("instrument_results_file_format", "xlsx")
-        artoapply = request.form.get("artoapply")
-        override = request.form.get("results_override")
-        instrument_uid = request.form.get("instrument")
-        worksheet = int(request.form.get("worksheet", "2"))
-        errors = []
-        logs = []
-        warns = []
-
-        # Load the most suitable parser according to file extension/options/etc...
-        parser = None
-        if not hasattr(infile, "filename"):
-            errors.append(_("No file selected"))
-        if fileformat in ("xls", "xlsx"):
-            parser = TimeSeriesParser(
-                infile, worksheet, encoding=fileformat, instrument_uid=instrument_uid
-            )
+        if request is not None:
+            infile = request.form["instrument_results_file"]
+            fileformat = request.form.get("instrument_results_file_format", "xlsx")
+            artoapply = request.form.get("artoapply")
+            override = request.form.get("results_override")
+            instrument_uid = request.form.get("instrument")
+            worksheet = int(request.form.get("worksheet", "2"))
+        else:
+            # Auto_importer hack
+            artoapply = "received_tobeverified"
+            override = "overrideempty"
+            instrument_uid = None
+            worksheet = 2
+
+        if hasattr(self, "parser"):
+            # Auto improt hack
+            parser = self.parser
         else:
-            errors.append(
-                t(
-                    _(
-                        "Unrecognized file format ${fileformat}",
-                        mapping={"fileformat": fileformat},
+            # Load the most suitable parser according to file extension/options/etc...
+            parser = None
+            if not hasattr(infile, "filename"):
+                self.errors.append(_("No file selected"))
+            if fileformat in ("xls", "xlsx"):
+                parser = TimeSeriesParser(
+                    infile,
+                    worksheet,
+                    encoding=fileformat,
+                    instrument_uid=instrument_uid,
+                )
+            else:
+                self.errors.append(
+                    t(
+                        _(
+                            "Unrecognized file format ${fileformat}",
+                            mapping={"fileformat": fileformat},
+                        )
                     )
                 )
-            )
 
         if parser:
             # Load the importer
@@ -188,7 +209,7 @@ def Import(self, context, request):
             elif override == "overrideempty":
                 over = [True, True]
 
-            importer = AnalysisResultsImporter(
+            importer = TimeSeriesImporter(
                 parser=parser,
                 context=context,
                 allowed_sample_states=status,
@@ -199,13 +220,69 @@ def Import(self, context, request):
             tbex = ""
             try:
                 importer.process()
-                errors = importer.errors
-                logs = importer.logs
-                warns = importer.warns
+                self.errors.append(importer.errors)
+                self.logs.append(importer.logs)
+                self.warns.append(importer.warns)
             except Exception:
                 tbex = traceback.format_exc()
-                errors.append(tbex)
+                self.errors.append(tbex)
 
-        results = {"errors": errors, "log": logs, "warns": warns}
+        results = {"errors": self.errors, "log": self.logs, "warns": self.warns}
 
         return json.dumps(results)
+
+    def get_automatic_importer(self, instrument, parser, **kw):
+        """Called during automated results import"""
+        # initialize the base class with the required parameters
+        if parser._instrument is None:
+            parser._instrument = instrument
+        self.parser = parser
+        return self
+
+    def get_automatic_parser(self, infile):
+        """Called during automated results import
+
+        Returns the parser to be used by default for the file passed in when
+        automatic results import for this instrument interface is enabled
+        """
+        return TimeSeriesParser(infile, encoding="xlsx")
+
+    def process(self):
+        results = self.Import(self.context, request=None, parser=self.parser)
+        return results
+
+
+class TimeSeriesImporter(AnalysisResultsImporter):
+
+    def __init__(
+        self,
+        parser,
+        context,
+        override,
+        allowed_sample_states=None,
+        allowed_analysis_states=None,
+        instrument_uid="",
+        form=None,
+    ):
+        AnalysisResultsImporter.__init__(
+            self,
+            parser,
+            context,
+            override,
+            allowed_sample_states,
+            allowed_analysis_states,
+            instrument_uid,
+        )
+
+    def parse_results(self):
+        """Parse the results file and return the raw results"""
+        parsed = self.parser.parse()
+
+        self.errors = self.parser.errors
+        self.warns = self.parser.warns
+        self.logs = self.parser.logs
+
+        if not parsed:
+            return {}
+
+        return self.parser.getRawResults()