From 338959e5acc25cb8abeff8c77be539b35866ee05 Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:06:15 +0200 Subject: [PATCH 1/5] Update 01.x-xgate-xx.json --- .../model_changes/{01.7-agate-kafka.json => 01.x-xgate-xx.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename qgate_sln_mlrun/model_changes/{01.7-agate-kafka.json => 01.x-xgate-xx.json} (100%) diff --git a/qgate_sln_mlrun/model_changes/01.7-agate-kafka.json b/qgate_sln_mlrun/model_changes/01.x-xgate-xx.json similarity index 100% rename from qgate_sln_mlrun/model_changes/01.7-agate-kafka.json rename to qgate_sln_mlrun/model_changes/01.x-xgate-xx.json From 5e79a248fc816a2e3c6f1081c4bed4483977bb0d Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:20:42 +0200 Subject: [PATCH 2/5] Update ts201.py --- qgate_sln_mlrun/ts/ts201.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/qgate_sln_mlrun/ts/ts201.py b/qgate_sln_mlrun/ts/ts201.py index eea02724..16fa09cc 100644 --- a/qgate_sln_mlrun/ts/ts201.py +++ b/qgate_sln_mlrun/ts/ts201.py @@ -52,11 +52,6 @@ def _create_featureset(self, testcase_name, project_name, json_file): # create feature set only in case, if not exist self._create_featureset_content(project_name, name, desc, json_content['spec']) - # try: - # fstore.get_feature_set(f"{project_name}/{name}") - # except: - # self._create_featureset_content(project_name, name, desc, json_content['spec']) - def _create_featureset_content(self, project_name, featureset_name, featureset_desc, json_spec): """ Create featureset based on json spec From 81f4e25f31a34d18cb9bfa9aac3420c1ed2f0230 Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:26:46 +0200 Subject: [PATCH 3/5] Update mlrun-iguazio-release-notes.md --- docs/mlrun-iguazio-release-notes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/mlrun-iguazio-release-notes.md b/docs/mlrun-iguazio-release-notes.md index f918c62a..72209364 100644 --- a/docs/mlrun-iguazio-release-notes.md +++ b/docs/mlrun-iguazio-release-notes.md @@ -10,9 +10,9 @@ NOTE: You can see detail changes in [MLRun change log](https://docs.mlrun.org/en ### 1.6 MLRun - Model monitoring/drifting for Iguazio (V3IO support only) - - MLFlow integration - - Pandas 2.0 support - - Extend datastore profiles + - MLFlow integration (store artifact to MLRun) + - Support Pandas 2.0 + - Extend datastore profiles (add providers) ### 1.5 MLRun - Datastore profiles From 72da7cda145c5c839df944e6986935f3b1d46744 Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:50:19 +0200 Subject: [PATCH 4/5] Simplify targets --- qgate-sln-mlrun-private.env | 2 +- qgate_sln_mlrun/qualityreport.py | 24 ++++++------ qgate_sln_mlrun/ts/ts201.py | 64 ++++++++++++++++++++------------ 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/qgate-sln-mlrun-private.env b/qgate-sln-mlrun-private.env index a935456b..9dbd18d2 100644 --- a/qgate-sln-mlrun-private.env +++ b/qgate-sln-mlrun-private.env @@ -44,7 +44,7 @@ QGATE_DATASET = 01-size-100 #agate-kafka #QGATE_FILTER_PROJECTS = agate-redis-parquet #QGATE_FILTER_PROJECTS = agate-parquet -QGATE_FILTER_PROJECTS = agate-kafka +QGATE_FILTER_PROJECTS = agate-redis-csv # Path to the output directory (as off-line storage, valid for target 'parquet' and 'csv') # sample value e.g. ./output diff --git a/qgate_sln_mlrun/qualityreport.py b/qgate_sln_mlrun/qualityreport.py index d01eadb5..2cf78bcf 100644 --- a/qgate_sln_mlrun/qualityreport.py +++ b/qgate_sln_mlrun/qualityreport.py @@ -74,19 +74,17 @@ def _project_avoid_testscenarios(self, project): # identification of targets (I am focusing on project level) # TODO: focus on featureset level also spec=self.project_specs[project] - for prj_targets in spec["targets"]: - targets=spec["targets"][prj_targets] - for target in targets: - if target in self.TARGET_ONLINE: - online=True - if target in self.TARGET_OFFLINE: - offline=True - - # add avoid based on target - avoid=self.TARGET_NOT_VALID_TEST.get(target, None) - if avoid: - for name in avoid: - all_avoid.add(name) + for target in spec["targets"]: + if target in self.TARGET_ONLINE: + online=True + if target in self.TARGET_OFFLINE: + offline=True + + # add avoid based on target + avoid=self.TARGET_NOT_VALID_TEST.get(target, None) + if avoid: + for name in avoid: + all_avoid.add(name) # add avoid TS based on missing On/Off-line targets if not offline: diff --git a/qgate_sln_mlrun/ts/ts201.py b/qgate_sln_mlrun/ts/ts201.py index 16fa09cc..529e8943 100644 --- a/qgate_sln_mlrun/ts/ts201.py +++ b/qgate_sln_mlrun/ts/ts201.py @@ -90,32 +90,48 @@ def _create_featureset_content(self, project_name, featureset_name, featureset_d # define targets count=0 target_providers=[] - for target in json_spec['targets']: - # TODO: check if current target == project targets, if yes than secondary cycle - target=target.lower().strip() - - # check, if target is not project target - project_target = self.get_project_target(project_spec, target) - if project_target: - # add project targets - for sub_target in project_target: - sub_target = sub_target.lower().strip() - if len(sub_target)==0: # support bypass: switch empty targets - continue - target_provider=self._create_target(sub_target, f"target_{count}", project_name, json_spec) - if target_provider: - target_providers.append(target_provider) - count+=1 - else: - # add target - if len(target) == 0: # support bypass: switch empty targets - continue - target_provider = self._create_target(target, f"target_{count}", project_name, json_spec) - if target_provider: - target_providers.append(target_provider) - count += 1 + + for target in project_spec["targets"]: + target = target.lower().strip() + + # add target + if len(target) == 0: # support bypass: switch empty targets + continue + target_provider = self._create_target(target, f"target_{count}", project_name, json_spec) + if target_provider: + target_providers.append(target_provider) + count += 1 fs.set_targets(target_providers, with_defaults=False) + + # for target in json_spec['targets']: # target from FeatureSet + # # TODO: check if current target == project targets, if yes than secondary cycle + # target=target.lower().strip() + # + # # check, if target is not project target + # project_target = self.get_project_target(project_spec, target) + # if project_target: + # # add project targets + # for sub_target in project_target: + # sub_target = sub_target.lower().strip() + # + # + # if len(sub_target)==0: # support bypass: switch empty targets + # continue + # target_provider=self._create_target(sub_target, f"target_{count}", project_name, json_spec) + # if target_provider: + # target_providers.append(target_provider) + # count+=1 + # else: + # # add target + # if len(target) == 0: # support bypass: switch empty targets + # continue + # target_provider = self._create_target(target, f"target_{count}", project_name, json_spec) + # if target_provider: + # target_providers.append(target_provider) + # count += 1 + # fs.set_targets(target_providers, with_defaults=False) + fs.save() return fs From 342efdc63e410fd3959001ef3c8cd9ca6911f8af Mon Sep 17 00:00:00 2001 From: jist <95856749+george0st@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:50:47 +0200 Subject: [PATCH 5/5] Update version.py --- qgate_sln_mlrun/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qgate_sln_mlrun/version.py b/qgate_sln_mlrun/version.py index 923f05d9..6db6df12 100644 --- a/qgate_sln_mlrun/version.py +++ b/qgate_sln_mlrun/version.py @@ -1,3 +1,3 @@ # Store the version here so: -__version__ = '0.1.9' \ No newline at end of file +__version__ = '0.2.0' \ No newline at end of file