Merge pull request #280 from george0st/change

Simplify targets

docs/mlrun-iguazio-release-notes.md

@@ -10,9 +10,9 @@ NOTE: You can see detail changes in [MLRun change log](https://docs.mlrun.org/en
 
 ### 1.6 MLRun
  - Model monitoring/drifting for Iguazio (V3IO support only)
- - MLFlow integration
- - Pandas 2.0 support
- - Extend datastore profiles
+ - MLFlow integration (store artifact to MLRun)
+ - Support Pandas 2.0
+ - Extend datastore profiles (add providers)
 
 ### 1.5 MLRun
  - Datastore profiles

qgate-sln-mlrun-private.env

@@ -44,7 +44,7 @@ QGATE_DATASET = 01-size-100
 #agate-kafka
 #QGATE_FILTER_PROJECTS = agate-redis-parquet
 #QGATE_FILTER_PROJECTS = agate-parquet
-QGATE_FILTER_PROJECTS = agate-kafka
+QGATE_FILTER_PROJECTS = agate-redis-csv
 
 # Path to the output directory (as off-line storage, valid for target 'parquet' and 'csv')
 # sample value e.g. ./output

qgate_sln_mlrun/qualityreport.py

@@ -74,19 +74,17 @@ def _project_avoid_testscenarios(self, project):
         # identification of targets (I am focusing on project level)
         # TODO: focus on featureset level also
         spec=self.project_specs[project]
-        for prj_targets in spec["targets"]:
-            targets=spec["targets"][prj_targets]
-            for target in targets:
-                if target in self.TARGET_ONLINE:
-                    online=True
-                if target in self.TARGET_OFFLINE:
-                    offline=True
-
-                # add avoid based on target
-                avoid=self.TARGET_NOT_VALID_TEST.get(target, None)
-                if avoid:
-                    for name in avoid:
-                        all_avoid.add(name)
+        for target in spec["targets"]:
+            if target in self.TARGET_ONLINE:
+                online=True
+            if target in self.TARGET_OFFLINE:
+                offline=True
+
+            # add avoid based on target
+            avoid=self.TARGET_NOT_VALID_TEST.get(target, None)
+            if avoid:
+                for name in avoid:
+                    all_avoid.add(name)
 
         # add avoid TS based on missing On/Off-line targets
         if not offline:

qgate_sln_mlrun/ts/ts201.py

@@ -52,11 +52,6 @@ def _create_featureset(self, testcase_name, project_name, json_file):
             # create feature set only in case, if not exist
             self._create_featureset_content(project_name, name, desc, json_content['spec'])
 
-            # try:
-            #     fstore.get_feature_set(f"{project_name}/{name}")
-            # except:
-            #     self._create_featureset_content(project_name, name, desc, json_content['spec'])
-
     def _create_featureset_content(self, project_name, featureset_name, featureset_desc, json_spec):
         """
         Create featureset based on json spec
@@ -95,32 +90,48 @@ def _create_featureset_content(self, project_name, featureset_name, featureset_d
         # define targets
         count=0
         target_providers=[]
-        for target in json_spec['targets']:
-            # TODO: check if current target == project targets, if yes than secondary cycle
-            target=target.lower().strip()
-
-            # check, if target is not project target
-            project_target = self.get_project_target(project_spec, target)
-            if project_target:
-                # add project targets
-                for sub_target in project_target:
-                    sub_target = sub_target.lower().strip()
-                    if len(sub_target)==0:  # support bypass: switch empty targets
-                        continue
-                    target_provider=self._create_target(sub_target, f"target_{count}", project_name, json_spec)
-                    if target_provider:
-                        target_providers.append(target_provider)
-                    count+=1
-            else:
-                # add target
-                if len(target) == 0:  # support bypass: switch empty targets
-                    continue
-                target_provider = self._create_target(target, f"target_{count}", project_name, json_spec)
-                if target_provider:
-                    target_providers.append(target_provider)
-                count += 1
+
+        for target in project_spec["targets"]:
+            target = target.lower().strip()
+
+            # add target
+            if len(target) == 0:  # support bypass: switch empty targets
+                continue
+            target_provider = self._create_target(target, f"target_{count}", project_name, json_spec)
+            if target_provider:
+                target_providers.append(target_provider)
+            count += 1
         fs.set_targets(target_providers, with_defaults=False)
 
+
+        # for target in json_spec['targets']:     # target from FeatureSet
+        #     # TODO: check if current target == project targets, if yes than secondary cycle
+        #     target=target.lower().strip()
+        #
+        #     # check, if target is not project target
+        #     project_target = self.get_project_target(project_spec, target)
+        #     if project_target:
+        #         # add project targets
+        #         for sub_target in project_target:
+        #             sub_target = sub_target.lower().strip()
+        #
+        #
+        #             if len(sub_target)==0:  # support bypass: switch empty targets
+        #                 continue
+        #             target_provider=self._create_target(sub_target, f"target_{count}", project_name, json_spec)
+        #             if target_provider:
+        #                 target_providers.append(target_provider)
+        #             count+=1
+        #     else:
+        #         # add target
+        #         if len(target) == 0:  # support bypass: switch empty targets
+        #             continue
+        #         target_provider = self._create_target(target, f"target_{count}", project_name, json_spec)
+        #         if target_provider:
+        #             target_providers.append(target_provider)
+        #         count += 1
+        # fs.set_targets(target_providers, with_defaults=False)
+
         fs.save()
         return fs
 

qgate_sln_mlrun/version.py

@@ -1,3 +1,3 @@
 # Store the version here so:
 
-__version__ = '0.1.9'
+__version__ = '0.2.0'