diff --git a/docs/mlrun-iguazio-release-notes.md b/docs/mlrun-iguazio-release-notes.md index f918c62a..72209364 100644 --- a/docs/mlrun-iguazio-release-notes.md +++ b/docs/mlrun-iguazio-release-notes.md @@ -10,9 +10,9 @@ NOTE: You can see detail changes in [MLRun change log](https://docs.mlrun.org/en ### 1.6 MLRun - Model monitoring/drifting for Iguazio (V3IO support only) - - MLFlow integration - - Pandas 2.0 support - - Extend datastore profiles + - MLFlow integration (store artifact to MLRun) + - Support Pandas 2.0 + - Extend datastore profiles (add providers) ### 1.5 MLRun - Datastore profiles diff --git a/qgate-sln-mlrun-private.env b/qgate-sln-mlrun-private.env index a935456b..9dbd18d2 100644 --- a/qgate-sln-mlrun-private.env +++ b/qgate-sln-mlrun-private.env @@ -44,7 +44,7 @@ QGATE_DATASET = 01-size-100 #agate-kafka #QGATE_FILTER_PROJECTS = agate-redis-parquet #QGATE_FILTER_PROJECTS = agate-parquet -QGATE_FILTER_PROJECTS = agate-kafka +QGATE_FILTER_PROJECTS = agate-redis-csv # Path to the output directory (as off-line storage, valid for target 'parquet' and 'csv') # sample value e.g. ./output diff --git a/qgate_sln_mlrun/model_changes/01.7-agate-kafka.json b/qgate_sln_mlrun/model_changes/01.x-xgate-xx.json similarity index 100% rename from qgate_sln_mlrun/model_changes/01.7-agate-kafka.json rename to qgate_sln_mlrun/model_changes/01.x-xgate-xx.json diff --git a/qgate_sln_mlrun/qualityreport.py b/qgate_sln_mlrun/qualityreport.py index d01eadb5..2cf78bcf 100644 --- a/qgate_sln_mlrun/qualityreport.py +++ b/qgate_sln_mlrun/qualityreport.py @@ -74,19 +74,17 @@ def _project_avoid_testscenarios(self, project): # identification of targets (I am focusing on project level) # TODO: focus on featureset level also spec=self.project_specs[project] - for prj_targets in spec["targets"]: - targets=spec["targets"][prj_targets] - for target in targets: - if target in self.TARGET_ONLINE: - online=True - if target in self.TARGET_OFFLINE: - offline=True - - # add avoid based on target - avoid=self.TARGET_NOT_VALID_TEST.get(target, None) - if avoid: - for name in avoid: - all_avoid.add(name) + for target in spec["targets"]: + if target in self.TARGET_ONLINE: + online=True + if target in self.TARGET_OFFLINE: + offline=True + + # add avoid based on target + avoid=self.TARGET_NOT_VALID_TEST.get(target, None) + if avoid: + for name in avoid: + all_avoid.add(name) # add avoid TS based on missing On/Off-line targets if not offline: diff --git a/qgate_sln_mlrun/ts/ts201.py b/qgate_sln_mlrun/ts/ts201.py index eea02724..529e8943 100644 --- a/qgate_sln_mlrun/ts/ts201.py +++ b/qgate_sln_mlrun/ts/ts201.py @@ -52,11 +52,6 @@ def _create_featureset(self, testcase_name, project_name, json_file): # create feature set only in case, if not exist self._create_featureset_content(project_name, name, desc, json_content['spec']) - # try: - # fstore.get_feature_set(f"{project_name}/{name}") - # except: - # self._create_featureset_content(project_name, name, desc, json_content['spec']) - def _create_featureset_content(self, project_name, featureset_name, featureset_desc, json_spec): """ Create featureset based on json spec @@ -95,32 +90,48 @@ def _create_featureset_content(self, project_name, featureset_name, featureset_d # define targets count=0 target_providers=[] - for target in json_spec['targets']: - # TODO: check if current target == project targets, if yes than secondary cycle - target=target.lower().strip() - - # check, if target is not project target - project_target = self.get_project_target(project_spec, target) - if project_target: - # add project targets - for sub_target in project_target: - sub_target = sub_target.lower().strip() - if len(sub_target)==0: # support bypass: switch empty targets - continue - target_provider=self._create_target(sub_target, f"target_{count}", project_name, json_spec) - if target_provider: - target_providers.append(target_provider) - count+=1 - else: - # add target - if len(target) == 0: # support bypass: switch empty targets - continue - target_provider = self._create_target(target, f"target_{count}", project_name, json_spec) - if target_provider: - target_providers.append(target_provider) - count += 1 + + for target in project_spec["targets"]: + target = target.lower().strip() + + # add target + if len(target) == 0: # support bypass: switch empty targets + continue + target_provider = self._create_target(target, f"target_{count}", project_name, json_spec) + if target_provider: + target_providers.append(target_provider) + count += 1 fs.set_targets(target_providers, with_defaults=False) + + # for target in json_spec['targets']: # target from FeatureSet + # # TODO: check if current target == project targets, if yes than secondary cycle + # target=target.lower().strip() + # + # # check, if target is not project target + # project_target = self.get_project_target(project_spec, target) + # if project_target: + # # add project targets + # for sub_target in project_target: + # sub_target = sub_target.lower().strip() + # + # + # if len(sub_target)==0: # support bypass: switch empty targets + # continue + # target_provider=self._create_target(sub_target, f"target_{count}", project_name, json_spec) + # if target_provider: + # target_providers.append(target_provider) + # count+=1 + # else: + # # add target + # if len(target) == 0: # support bypass: switch empty targets + # continue + # target_provider = self._create_target(target, f"target_{count}", project_name, json_spec) + # if target_provider: + # target_providers.append(target_provider) + # count += 1 + # fs.set_targets(target_providers, with_defaults=False) + fs.save() return fs diff --git a/qgate_sln_mlrun/version.py b/qgate_sln_mlrun/version.py index 923f05d9..6db6df12 100644 --- a/qgate_sln_mlrun/version.py +++ b/qgate_sln_mlrun/version.py @@ -1,3 +1,3 @@ # Store the version here so: -__version__ = '0.1.9' \ No newline at end of file +__version__ = '0.2.0' \ No newline at end of file