From 9703be00ea759418136a25b245568eb62245e677 Mon Sep 17 00:00:00 2001 From: Narek Hovhannisyan Date: Sun, 6 Oct 2024 14:17:19 +0400 Subject: [PATCH] =?UTF-8?q?chore(release):=20v0.7.0=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitmodules | 3 - Refactor-migration-guide.md | 150 +- github-processes.md | 12 +- grafana/IF_GRAFANA_SETUP.md | 71 - grafana/if_grafana_config.json | 370 ---- .../failure-invalid-config-input-param.yml | 6 +- .../failure-output-param-is-null.yaml | 4 +- .../examples/builtins/coefficient/success.yml | 4 +- .../failure-invalid-instance-type.yaml | 3 +- .../failure-invalid-vendor.yaml | 7 +- .../failure-missing-cloud-vendor.yml | 5 +- .../csv-lookup/cloud-metadata/success.yml | 3 +- .../failure-missing-column.yml | 2 +- .../failure-missing-output.yml | 4 +- .../region-metadata/success-renaming.yml | 2 +- .../csv-lookup/region-metadata/success.yml | 2 +- .../failure-missing-input-param.yml | 3 +- ...failure-unsupported-physical-processor.yml | 3 +- .../csv-lookup/tdp-finder/success.yml | 3 +- .../divide/failure-denominator-equal-zero.yml | 38 - .../failure-invalid-config-denominator.yml | 8 +- .../divide/failure-missing-numerator.yml | 6 +- .../examples/builtins/divide/success.yml | 4 +- .../examples/builtins/exponent/success.yml | 2 +- .../builtins/interpolation/interpolation.yml | 24 - .../builtins/interpolation/success.yml | 2 +- .../failure-invalid-config-cpu-range.yml | 4 +- ...ilure-invalid-memory-utilization-range.yml | 2 +- .../failure-missing-timestamp-from-param.yml | 2 +- .../builtins/mock-observations/success.yml | 2 +- .../failure-input-parameter-is-missing.yml | 2 +- .../multiply/success-with-multiple-inputs.yml | 2 +- .../examples/builtins/multiply/success.yml | 2 +- .../regex/failure-missing-input-param.yml | 2 +- .../regex/failure-not-matching-with-regex.yml | 4 +- manifests/examples/builtins/regex/success.yml | 2 +- ...failure-invalid-default-emission-value.yml | 23 - .../failure-missing-expected-lifespan.yml | 23 - .../builtins/sci-embodied/scenario-1.yml | 27 + .../builtins/sci-embodied/scenario-2.yml | 37 + .../builtins/sci-embodied/success.yml | 13 + .../sci/failure-invalid-config-value.yml | 7 +- .../sci/failure-missing-input-param.yml | 2 +- manifests/examples/builtins/sci/success.yml | 2 +- .../shell/failure-invalid-command.yml | 4 +- manifests/examples/builtins/shell/success.yml | 4 +- .../examples/builtins/subtract/success.yml | 2 +- .../sum/failure-missing-input-param.yml | 4 +- .../sum/failure-missing-output-param.yml | 4 +- manifests/examples/builtins/sum/success.yml | 2 +- .../builtins/time-converter/success.yaml | 6 +- .../failure-config-start-later-end.yml | 22 +- ...-config.yml => failure-missing-config.yml} | 11 +- .../examples/builtins/time-sync/success.yml | 4 +- .../failure-missing-cloud-instance-type.yml | 2 +- .../examples/features/regroup/success.yml | 2 +- manifests/examples/pipelines/generics.yml | 16 +- .../examples/pipelines/instance-metadata.yml | 4 +- manifests/examples/pipelines/nesting.yml | 108 +- .../pipeline-with-aggregate.yaml | 288 +-- .../outputs-if-diff/pipeline-with-mocks.yaml | 200 +- .../examples/pipelines/pipeline-teads-sci.yml | 30 +- .../pipelines/pipeline-with-aggregate.yml | 84 +- .../pipelines/pipeline-with-mocks.yml | 140 +- manifests/examples/pipelines/scenario-1.yml | 34 - manifests/examples/pipelines/scenario-2.yml | 52 - manifests/examples/pipelines/scenario-3.yml | 2 +- manifests/examples/pipelines/scenario-4.yml | 10 +- manifests/examples/pipelines/scenario-5.yml | 6 +- manifests/examples/pipelines/sci.yml | 24 +- manifests/examples/pipelines/teads-curve.yml | 20 +- manifests/examples/pipelines/zeros.yml | 46 +- .../bugs/aggregation-error-wrong-metric.yaml | 80 +- .../bugs/input-error-missing-duration.yaml | 2 +- ...observations-failure-duration-is-zero.yaml | 4 +- .../bugs/pipeline-error-naming-mismatch.yaml | 2 +- .../pipeline-error-uninitialized-plugin.yaml | 2 +- .../outputs/bugs/pipeline-ordering-error.yaml | 12 +- .../sci-embodied-missing-resources-total.yaml | 72 - .../failure-invalid-config-input-param.yaml | 4 +- .../failure-output-param-is-null.yaml | 2 +- .../outputs/builtins/coefficient/success.yaml | 45 +- .../failure-invalid-instance-type.yaml | 2 +- .../failure-invalid-vendor.yaml | 76 + .../failure-missing-cloud-vendor.yaml | 43 +- .../csv-lookup/cloud-metadata/success.yaml | 2 +- .../failure-missing-column.yaml | 2 +- .../failure-missing-output.yaml | 41 +- .../region-metadata/success-renaming.yaml | 2 +- .../csv-lookup/region-metadata/success.yaml | 2 +- .../failure-missing-input-param.yaml | 2 +- ...ailure-unsupported-physical-processor.yaml | 2 +- .../csv-lookup/tdp-finder/success.yaml | 2 +- .../failure-invalid-config-denominator.yaml | 66 +- .../divide/failure-missing-numerator.yaml | 55 +- .../success-denominator-equal-zero.yaml | 4 +- .../outputs/builtins/divide/success.yaml | 4 +- .../outputs/builtins/exponent/success.yaml | 2 +- .../builtins/interpolation/interpolation.yaml | 2 +- .../builtins/interpolation/success.yaml | 2 +- .../failure-invalid-config-cpu-range.yaml | 6 +- ...lure-invalid-memory-utilization-range.yaml | 4 +- .../failure-missing-timestamp-from-param.yaml | 2 +- .../builtins/mock-observations/success.yaml | 239 +-- .../failure-input-parameter-is-missing.yaml | 2 +- .../success-with-multiple-inputs.yaml | 2 +- .../outputs/builtins/multiply/success.yaml | 2 +- .../regex/failure-missing-input-param.yaml | 2 +- .../failure-not-matching-with-regex.yaml | 72 + manifests/outputs/builtins/regex/success.yaml | 2 +- ...ailure-invalid-default-emission-value.yaml | 48 +- .../failure-missing-expected-lifespan.yaml | 69 - .../builtins/sci-embodied/scenario-1.yaml | 100 + .../builtins/sci-embodied/scenario-2.yaml | 101 + .../builtins/sci-embodied/success.yaml | 64 +- .../sci/failure-invalid-config-value.yaml | 45 +- .../sci/failure-missing-input-param.yaml | 2 +- manifests/outputs/builtins/sci/success.yaml | 2 +- .../shell/failure-invalid-command.yaml | 4 +- manifests/outputs/builtins/shell/success.yaml | 2 +- .../outputs/builtins/subtract/success.yaml | 2 +- .../sum/failure-missing-input-param.yaml | 4 +- .../sum/failure-missing-output-param.yaml | 4 +- manifests/outputs/builtins/sum/success.yaml | 2 +- .../builtins/time-converter/success.yaml | 73 + .../failure-config-start-later-end.yaml | 6 +- .../time-sync/failure-missing-config.yaml | 74 + .../outputs/builtins/time-sync/success.yaml | 109 +- .../aggregate-failure-invalid-metrics.yaml | 2 +- ...gate-failure-missing-metric-in-inputs.yaml | 2 +- .../features/aggregate-horizontal.yaml | 6 +- .../outputs/features/aggregate-vertical.yaml | 6 +- manifests/outputs/features/aggregate.yaml | 6 +- .../regroup/failure-invalid-regroup.yaml | 87 + .../failure-missing-cloud-instance-type.yaml | 87 + .../pipelines/cloud-metadata-divide.yaml | 4 +- manifests/outputs/pipelines/generics.yaml | 56 +- .../outputs/pipelines/instance-metadata.yaml | 28 +- .../outputs/pipelines/mock-obs-time-sync.yaml | 449 ----- manifests/outputs/pipelines/nesting.yaml | 1199 +++++++----- .../outputs/pipelines/pipeline-teads-sci.yaml | 95 +- manifests/outputs/pipelines/scenario-4.yaml | 105 + manifests/outputs/pipelines/sci.yaml | 88 +- manifests/outputs/pipelines/teads-curve.yaml | 36 +- manifests/outputs/pipelines/zeros.yaml | 54 +- package-lock.json | 12 +- package.json | 8 +- scripts/impact-test.sh | 12 - scripts/run-yamls.sh | 8 - scripts/yaml-to-csv/yaml-to-csv.md | 73 - scripts/yaml-to-csv/yaml-to-csv.py | 102 - src/__mocks__/builtins/export-yaml.ts | 14 +- src/__mocks__/fs/index.ts | 7 +- src/__mocks__/mock-manifest.yaml | 3 +- src/__tests__/common/util/helpers.test.ts | 2 +- src/__tests__/if-merge/util/helpers.test.ts | 10 +- .../if-run/builtins/CommonGenerator.test.ts | 6 +- .../if-run/builtins/RandIntGenerator.test.ts | 10 +- .../if-run/builtins/coefficient.test.ts | 161 +- .../if-run/builtins/copy-param.test.ts | 127 +- .../if-run/builtins/csv-lookup.test.ts | 161 +- src/__tests__/if-run/builtins/divide.test.ts | 155 +- .../if-run/builtins/exponent.test.ts | 170 +- .../if-run/builtins/interpolation.test.ts | 232 ++- .../if-run/builtins/mock-observations.test.ts | 223 ++- .../if-run/builtins/multiply.test.ts | 166 +- src/__tests__/if-run/builtins/regex.test.ts | 96 +- .../if-run/builtins/sci-embodied.test.ts | 305 ++- src/__tests__/if-run/builtins/sci.test.ts | 192 +- src/__tests__/if-run/builtins/shell.test.ts | 28 +- .../if-run/builtins/subtract.test.ts | 165 +- src/__tests__/if-run/builtins/sum.test.ts | 171 +- .../if-run/builtins/time-converter.test.ts | 154 +- .../if-run/builtins/time-sync.test.ts | 1692 ++++++++++------- src/__tests__/if-run/lib/aggregate.test.ts | 16 +- src/__tests__/if-run/lib/compute.test.ts | 182 +- src/__tests__/if-run/lib/environment.test.ts | 2 +- src/__tests__/if-run/lib/explain.test.ts | 205 +- src/__tests__/if-run/lib/initialize.test.ts | 14 +- src/__tests__/if-run/lib/regroup.test.ts | 78 +- .../if-run/util/aggregation-helper.test.ts | 54 +- src/__tests__/if-run/util/helpers.test.ts | 102 +- .../if-run/util/plugin-storage.test.ts | 2 +- src/common/config/strings.ts | 4 + src/common/types/manifest.ts | 5 +- src/common/util/debug-logger.ts | 20 +- src/common/util/validations.ts | 54 +- src/if-env/config/env-template.yml | 3 +- src/if-run/builtins/coefficient/README.md | 58 +- src/if-run/builtins/coefficient/index.ts | 125 +- src/if-run/builtins/copy-param/README.md | 36 +- src/if-run/builtins/copy-param/index.ts | 121 +- src/if-run/builtins/csv-lookup/README.md | 42 +- src/if-run/builtins/csv-lookup/index.ts | 363 ++-- src/if-run/builtins/divide/README.md | 46 +- src/if-run/builtins/divide/index.ts | 140 +- src/if-run/builtins/exponent/README.md | 40 +- src/if-run/builtins/exponent/index.ts | 112 +- src/if-run/builtins/interpolation/README.md | 59 +- src/if-run/builtins/interpolation/index.ts | 286 ++- .../builtins/mock-observations/README.md | 34 +- .../helpers/common-generator.ts | 8 +- .../helpers/rand-int-generator.ts | 12 +- .../builtins/mock-observations/index.ts | 258 ++- src/if-run/builtins/multiply/README.md | 36 +- src/if-run/builtins/multiply/index.ts | 102 +- src/if-run/builtins/regex/README.md | 42 +- src/if-run/builtins/regex/index.ts | 131 +- src/if-run/builtins/sci-embodied/README.md | 151 +- src/if-run/builtins/sci-embodied/index.ts | 337 ++-- src/if-run/builtins/sci/README.md | 37 +- src/if-run/builtins/sci/index.ts | 142 +- src/if-run/builtins/shell/README.md | 22 +- src/if-run/builtins/shell/index.ts | 96 +- src/if-run/builtins/subtract/README.md | 39 +- src/if-run/builtins/subtract/index.ts | 99 +- src/if-run/builtins/sum/README.md | 61 +- src/if-run/builtins/sum/index.ts | 121 +- src/if-run/builtins/time-converter/README.md | 16 +- src/if-run/builtins/time-converter/index.ts | 136 +- src/if-run/builtins/time-sync/README.md | 66 +- src/if-run/builtins/time-sync/index.ts | 828 ++++---- src/if-run/config/config.ts | 8 +- src/if-run/config/strings.ts | 53 +- src/if-run/index.ts | 17 +- src/if-run/lib/aggregate.ts | 45 +- src/if-run/lib/compute.ts | 104 +- src/if-run/lib/exhaust.ts | 2 +- src/if-run/lib/explain.ts | 78 +- src/if-run/lib/initialize.ts | 27 +- src/if-run/lib/regroup.ts | 61 +- src/if-run/types/aggregation.ts | 12 +- src/if-run/types/compute.ts | 4 + src/if-run/types/exhaust-plugin-interface.ts | 8 - src/if-run/types/explain.ts | 12 +- src/if-run/types/interface.ts | 6 - src/if-run/types/plugin-storage.ts | 2 +- src/if-run/types/process-args.ts | 2 + src/if-run/types/time-sync.ts | 20 - src/if-run/util/aggregation-helper.ts | 48 +- src/if-run/util/args.ts | 2 + src/if-run/util/helpers.ts | 32 - src/if-run/util/plugin-storage.ts | 2 +- 243 files changed, 8928 insertions(+), 6945 deletions(-) delete mode 100644 .gitmodules delete mode 100644 grafana/IF_GRAFANA_SETUP.md delete mode 100644 grafana/if_grafana_config.json delete mode 100644 manifests/examples/builtins/divide/failure-denominator-equal-zero.yml delete mode 100644 manifests/examples/builtins/interpolation/interpolation.yml delete mode 100644 manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml delete mode 100644 manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml create mode 100644 manifests/examples/builtins/sci-embodied/scenario-1.yml create mode 100644 manifests/examples/builtins/sci-embodied/scenario-2.yml rename manifests/examples/builtins/time-sync/{failure-missing-global-config.yml => failure-missing-config.yml} (69%) delete mode 100644 manifests/examples/pipelines/scenario-1.yml delete mode 100644 manifests/examples/pipelines/scenario-2.yml delete mode 100644 manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml create mode 100644 manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml create mode 100644 manifests/outputs/builtins/regex/failure-not-matching-with-regex.yaml delete mode 100644 manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml create mode 100644 manifests/outputs/builtins/sci-embodied/scenario-1.yaml create mode 100644 manifests/outputs/builtins/sci-embodied/scenario-2.yaml create mode 100644 manifests/outputs/builtins/time-converter/success.yaml create mode 100644 manifests/outputs/builtins/time-sync/failure-missing-config.yaml create mode 100644 manifests/outputs/features/regroup/failure-invalid-regroup.yaml create mode 100644 manifests/outputs/features/regroup/failure-missing-cloud-instance-type.yaml delete mode 100644 manifests/outputs/pipelines/mock-obs-time-sync.yaml create mode 100644 manifests/outputs/pipelines/scenario-4.yaml delete mode 100755 scripts/impact-test.sh delete mode 100644 scripts/run-yamls.sh delete mode 100644 scripts/yaml-to-csv/yaml-to-csv.md delete mode 100644 scripts/yaml-to-csv/yaml-to-csv.py delete mode 100644 src/if-run/types/exhaust-plugin-interface.ts delete mode 100644 src/if-run/types/interface.ts delete mode 100644 src/if-run/types/time-sync.ts diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 8f4f4893f..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "third-party/ccf-coefficients"] - path = third-party/ccf-coefficients - url = https://github.com/cloud-carbon-footprint/ccf-coefficients diff --git a/Refactor-migration-guide.md b/Refactor-migration-guide.md index 95283c841..abffd6884 100644 --- a/Refactor-migration-guide.md +++ b/Refactor-migration-guide.md @@ -105,10 +105,10 @@ There have also been some changes to the structure of manifest files. Some of th method: SciEmbodied ``` -- **Global config** - We have introduced the concept of global config to the plugins. This is truly global configuration data that should be kept constant regardless of where the plugin is invoked across the manifest file. +- **Config** + We have introduced the concept of config to the plugins. This is truly configuration data that should be kept constant regardless of where the plugin is invoked across the manifest file. - A good example is the `interpolation` method to use in the Teads curve plugin - this is not expected to vary from component to component and can therefore be defined in global config. The plugin code itself must expect the global config. Then, the config can be passed in the `Initialize` block, for example: + A good example is the `interpolation` method to use in the Teads curve plugin - this is not expected to vary from component to component and can therefore be defined in config. The plugin code itself must expect the config. Then, the config can be passed in the `Initialize` block, for example: ```yaml initialize: @@ -116,34 +116,13 @@ There have also been some changes to the structure of manifest files. Some of th 'time-sync': method: TimeSync path: 'builtin' - global-config: + config: start-time: '2023-12-12T00:00:00.000Z' end-time: '2023-12-12T00:01:00.000Z' interval: 5 allow-padding: true ``` -- **Node level config** - - We have also introduced the concept of node-level config. This is designed for pluin configuration that might vary between components in the tree. For example, for each child in the tree you might wish to use the `regroup` feature to group the outputs according to a different set of keys. - - ```yaml - tree: - children: - child-1: - pipeline: - compute: - - teads-curve - - sci-e - - sci-embodied - - sci-o - - time-sync - - sci - regroup: - - region - - cloud/instance-type - ``` - - **Defaults** We have also introduced the concept of `defaults`. This is a section in each component's definition that can be used to provide fallbacks for missing input data. For example, perhaps you have a value arriving from an external API that should be present in every observation in your inputs array, but for soem reason the API fails to deliver a value for some timestamps. In this case, IF would fallback to the value provided for that metric in the `defaults` section of the manifest for that component. @@ -178,7 +157,7 @@ There have also been some changes to the structure of manifest files. Some of th Technically time-sync is not a new feature as it was present in IF before the refactor, but there are some tweaks to how the plugin is configured that are worth explaining here. Time sync snaps all input arrays across an entire graph to a common time grid. -This means you have to define a global start time, end time and interval to use everywhere. There is also a boolean to toggle whether you should allow the time sync model to pad the start and end of your time series with zeroes. You should default to `true` unless you have a specific reason not to. In the refactored IF we expect this information to be provided in global config, as follows: +This means you have to define a start time, end time and interval to use everywhere. There is also a boolean to toggle whether you should allow the time sync model to pad the start and end of your time series with zeroes. You should default to `true` unless you have a specific reason not to. In the refactored IF we expect this information to be provided in config, as follows: ```yaml initialize: @@ -186,7 +165,7 @@ initialize: 'time-sync': method: TimeSync path: 'builtin' - global-config: + config: start-time: '2023-12-12T00:00:00.000Z' end-time: '2023-12-12T00:01:00.000Z' interval: 5 @@ -220,68 +199,75 @@ Details tbc... ## Plugins -The plugins themselves require some changes to keep them compatible with the refactored IF. +Plugins require some modifications to remain compatible with the refactored IF interface. -Instead of the old class-based model, plugins are now functions. They conform to the following interface: +Each plugin follows the `PluginFactory` interface, which is a higher-order function that accepts a `params` object of type `PluginFactoryParams`. This function returns another function (the inner function), which handles the plugin’s `config`, `parametersMetadata`, and `mapping`. ```ts -export type PluginInterface = { - execute: ( - inputs: PluginParams[], - config?: Record - ) => PluginParams[]; - metadata: { - kind: string; - }; - [key: string]: any; -}; +export const PluginFactory = + (params: PluginFactoryParams) => + ( + config: ConfigParams = {}, + parametersMetadata: PluginParametersMetadata, + mapping: MappingParams + ) => ({ + metadata: { + inputs: {...params.metadata.inputs, ...parametersMetadata?.inputs}, + outputs: parametersMetadata?.outputs || params.metadata.outputs, + }, + execute: async (inputs: PluginParams[]) => { + // Generic plugin functionality goes here + // E.g., mapping, arithmetic operations, validation + // Process inputs and mapping logic + }); + }) ``` -The plugin still requires an execute function. This is where you implement the plugin logic. +Inner Function Parameters: + +- `config`: This is of type `ConfigParams` and has a default value of an empty object ({}). This might hold configuration settings for the plugin. +- `parametersMetadata`: A `PluginParametersMetadata` object that describes the metadata for the plugin’s parameters. +- `mapping`: A `MappingParams` object, describing parameters are mapped. + +Implementation Function: -Here's a minimal example for a plugin that sums some inputs defined in global config - see inline comments for some important notes: +The plugin requires an `implementation` function, where the actual plugin logic is defined. +Here’s a minimal example of a plugin that sums inputs as defined in the config. See the inline comments for further clarification. ```ts -// Here's the function definition - notice that global config is passed in here! -export const Sum = (globalConfig: SumConfig): PluginInterface => { - const inputParameters = globalConfig['input-parameters'] || []; - const outputParameter = globalConfig['output-parameter']; - - // we also return metadata now too - you can add more or just use this default - const metadata = { - kind: 'execute', - }; - - /** - * Calculate the sum of the input metrics for each timestamp. - */ - const execute = async (inputs: PluginParams[]): Promise => { - inputs.map(input => { - return calculateSum(input, inputParameters, outputParameter); +// Here's the function definition! +export const Sum = PluginFactory({ + configValidation: z.object({ + 'input-parameters': z.array(z.string()), + 'output-parameter': z.string().min(1), + }), + inputValidation: (input: PluginParams, config: ConfigParams) => { + return validate(validationSchema, inputData); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + const { + 'input-parameters': inputParameters, + 'output-parameter': outputParameter, + } = config; + + return inputs.map(input => { + const calculatedResult = calculateSum(input, inputParameters); + + return { + ...input, + [outputParameter]: calculatedResult, + }; }); - return inputs; - }; - - /** - * Calculates the sum of the energy components. - */ - const calculateSum = ( - input: PluginParams, - inputParameters: string[], - outputParameter: string - ) => { - input[outputParameter] = inputParameters.reduce( - (accumulator, metricToSum) => { - return accumulator + input[metricToSum]; - }, - 0 - ); - }; - - // return the metadata and the execute function - return { - metadata, - execute, - }; -}; + }, + allowArithmeticExpressions: [], +}); + +/** + * Calculates the sum of the energy components. + */ +const calculateSum = (input: PluginParams, inputParameters: string[]) => + inputParameters.reduce( + (accumulator, metricToSum) => accumulator + input[metricToSum], + 0 + ); ``` diff --git a/github-processes.md b/github-processes.md index 81f2301d5..032115558 100644 --- a/github-processes.md +++ b/github-processes.md @@ -3,24 +3,18 @@ - [`if`](https://github.com/Green-Software-Foundation/if) - source code for the IF +- [`if-core`](https://github.com/Green-Software-Foundation/if-core) + - helper types, interfaces and utilities for IF and plugin development - [`if-plugins`](https://github.com/Green-Software-Foundation/if-plugins) **DEPRECATED** - source code for standard library of plugins - IF core team commit to maintaining these plugins -- [`if-unofficial-plugins`](https://github.com/Green-Software-Foundation/if-unofficial-plugins) +- [`if-unofficial-plugins`](https://github.com/Green-Software-Foundation/if-unofficial-plugins) **DEPRECATED** - source code for plugins relying on third-party data/APIs - intended to be deprecated and removed in mid-term future - plugins in this repo should be handed over to relevant organizations to maintain - [`if-plugin-template`](https://github.com/Green-Software-Foundation/if-plugin-template) - template for new plugins - intended for builders to bootstrap IF-compatible plugin development -- [`if-standards`](https://github.com/Green-Software-Foundation/if-standards) - - not currently used, but intended to be the home of params.ts - - will have a dedicated discussion board and governance to set IF standards -- [`if-exhaust plugins`](https://github.com/Green-Software-Foundation/if-exhaust-plugins) - - not currently used - - intended to become a separate repo just for exhaust plugins - - requires strict rules from if - ## Branch names and purposes diff --git a/grafana/IF_GRAFANA_SETUP.md b/grafana/IF_GRAFANA_SETUP.md deleted file mode 100644 index 99b0575d8..000000000 --- a/grafana/IF_GRAFANA_SETUP.md +++ /dev/null @@ -1,71 +0,0 @@ -# Setting up the Impact Framework Grafana dashboard -(for any questions please contact paz.barda@intel.com / pazbarda@gmail.com) - -## Download and Install Grafana -https://grafana.com/get/?plcmt=top-nav&cta=downloads&tab=self-managed - -This is the self managed version: you install it and can run it on your local machine. - -## Open Grafana on your local machine -Web browser: localhost:3000 - -You should see the Grafana welcome page. - -## Download and Install Grafana CSV plugin -https://grafana.com/grafana/plugins/marcusolsson-csv-datasource/?tab=installation - -After installation, go to Menu -> Plugins, search for "CSV" and make sure CSV plugin is there - -## Import the dashboard json config - -Menu -> Dashboards -> New -> Import - -1. Drag and drop "grafana_config.json" from this folder to Grafana webpage -2. Optional - change the name of the dashboard and the details text on the right -3. Click import. - -Dashboard should now appear, but with no data fed into the charts (yet). - -## Create a data source for your OMPL CSV file -Menu -> Connections -> Data Sources -> Add data source - -1. Search for "CSV", you should see the CSV plugin. Click it. -2. Name you new data source. -3. Switch from "HTTP" to "Local". -4. Type/paste in the path to your ompl csv ("your/path/to/csvs/if-iee-demo.csv" in our example) - -Click Save & Test - - - -## Connect your OMPL CSV data source to your dashboard - -Menu -> Dashboards -> Select your new dashboard - -For each blank chart: - -1. Click the chart menu -> edit -2. Below the chart go to Quary tab -3. At the top of the tab select the ompl CSV datasource you created -4. Go to Transform tab, and select the fields you'd like to show on the chart. -5. Start with "timestamp" and convert it to "Time". -6. Any other numeric value you'd like to show should be converted to "Number" - -Click Apply - -Click Save - -NOTE: when you select a CSV file (step 3) it might initially not show the columns in the transformation dropdown. if that happens - save the dashboard, exit it refresh the Grafana webpage (localhost:3000). Once you go into the dashboard again it should show the columns for transformation. - - -## Enable auto-refresh of the dashboard - -On the top right of the dashboard, look for the "refresh dashboard" button - -Click the dropdown next to it, and choose the auto-refresh interval - -Click Save - -## Your dashboard is now ready to go! -With every change to you CSV file you should see it reflect on the dashboard momentarily. - diff --git a/grafana/if_grafana_config.json b/grafana/if_grafana_config.json deleted file mode 100644 index 2fe00915b..000000000 --- a/grafana/if_grafana_config.json +++ /dev/null @@ -1,370 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 5, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "marcusolsson-csv-datasource", - "uid": "" - }, - "gridPos": { - "h": 3, - "w": 18, - "x": 0, - "y": 0 - }, - "id": 3, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "

\n Dashboard Title goes here", - "mode": "html" - }, - "pluginVersion": "10.1.1", - "targets": [ - { - "datasource": { - "type": "marcusolsson-csv-datasource", - "uid": "" - }, - "decimalSeparator": ".", - "delimiter": ",", - "header": true, - "ignoreUnknown": false, - "refId": "A", - "schema": [ - { - "name": "", - "type": "string" - } - ], - "skipRows": 0 - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "marcusolsson-csv-datasource", - "uid": "" - }, - "description": "", - "gridPos": { - "h": 3, - "w": 5, - "x": 19, - "y": 0 - }, - "id": 2, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "

\n Some detailed text:
goes here

", - "mode": "html" - }, - "pluginVersion": "10.1.1", - "targets": [ - { - "datasource": { - "type": "marcusolsson-csv-datasource", - "uid": "" - }, - "decimalSeparator": ".", - "delimiter": ",", - "header": true, - "ignoreUnknown": false, - "refId": "A", - "schema": [ - { - "name": "", - "type": "string" - } - ], - "skipRows": 0 - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": { - "type": "marcusolsson-csv-datasource", - "uid": "" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "marcusolsson-csv-datasource", - "uid": "" - }, - "decimalSeparator": ".", - "delimiter": ",", - "header": true, - "ignoreUnknown": false, - "refId": "A", - "schema": [ - { - "name": "", - "type": "string" - } - ], - "skipRows": 0 - } - ], - "title": "Energy [kWh]", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "timestamp" - }, - { - "destinationType": "number", - "targetField": "energy" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "marcusolsson-csv-datasource", - "uid": "" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "marcusolsson-csv-datasource", - "uid": "" - }, - "decimalSeparator": ".", - "delimiter": ",", - "header": true, - "ignoreUnknown": false, - "refId": "A", - "schema": [ - { - "name": "", - "type": "string" - } - ], - "skipRows": 0 - } - ], - "title": "Carbon [gCO2]", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "timestamp" - }, - { - "destinationType": "number", - "targetField": "carbon" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "2023-11-02T08:35:31.000Z", - "to": "2023-11-02T08:35:42.000Z" - }, - "timepicker": {}, - "timezone": "", - "title": "IF_dashboard", - "uid": "", - "version": 40, - "weekStart": "" -} \ No newline at end of file diff --git a/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml b/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml index 5d497b948..17672dcdc 100644 --- a/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml +++ b/manifests/examples/builtins/coefficient/failure-invalid-config-input-param.yml @@ -1,12 +1,12 @@ name: coefficient-demo -description: failure with ivalid `global-config.input-parameter` +description: failure with ivalid `config.input-parameter` tags: initialize: plugins: coefficient: method: Coefficient path: "builtin" - global-config: + config: input-parameter: 4 coefficient: 3 output-parameter: "carbon-product" @@ -16,8 +16,6 @@ tree: pipeline: compute: - coefficient - config: - sum: inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml b/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml index 1ef932df0..1a790e205 100644 --- a/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml +++ b/manifests/examples/builtins/coefficient/failure-output-param-is-null.yaml @@ -6,7 +6,7 @@ initialize: coefficient: method: Coefficient path: "builtin" - global-config: + config: input-parameter: "carbon" coefficient: 3 output-parameter: @@ -16,8 +16,6 @@ tree: pipeline: compute: - coefficient - config: - sum: inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/coefficient/success.yml b/manifests/examples/builtins/coefficient/success.yml index 69dd764eb..227f556e4 100644 --- a/manifests/examples/builtins/coefficient/success.yml +++ b/manifests/examples/builtins/coefficient/success.yml @@ -6,7 +6,7 @@ initialize: coefficient: method: Coefficient path: "builtin" - global-config: + config: input-parameter: "carbon" coefficient: 3 output-parameter: "carbon-product" @@ -16,8 +16,6 @@ tree: pipeline: compute: - coefficient - config: - sum: inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index 21f9769b4..60709dced 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -18,7 +18,6 @@ tree: pipeline: compute: - cloud-metadata - config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred cloud/vendor: aws diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml index c9fbb8296..5d2fa16b2 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml @@ -6,19 +6,18 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: - instance-class: cloud/instance-type - output: ["cpu-cores-utilized", "vcpus-allocated"] + instance-class: cloud/vendor + output: ["cpu-cores-utilized", "cloud/instance-type"] tree: children: child: pipeline: compute: - cloud-metadata - config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred cloud/vendor: gcp diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml index 62de6150d..4d9b4c2df 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml @@ -6,11 +6,11 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: - instance-class: cloud/instance-type + instance-class: cloud/vendor output: ["cpu-cores-utilized", "vcpus-allocated"] tree: children: @@ -18,7 +18,6 @@ tree: pipeline: compute: - cloud-metadata - config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred #cloud/vendor: aws diff --git a/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml b/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml index 84e10112b..34a2eae6c 100644 --- a/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml +++ b/manifests/examples/builtins/csv-lookup/cloud-metadata/success.yml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -18,7 +18,6 @@ tree: pipeline: compute: - cloud-metadata - config: inputs: - timestamp: 2023-07-06T00:00 # [KEYWORD] [NO-SUBFIELDS] time when measurement occurred cloud/vendor: aws diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml index 5b4e9583b..216836083 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-column.yml @@ -6,7 +6,7 @@ initialize: cloud-metadata: method: CSVLookup path: "builtin" - global-config: + config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: cloud-provider: "nonexistant" diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml index 71417932d..2a3d363b3 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml @@ -6,12 +6,12 @@ initialize: cloud-metadata: method: CSVLookup path: "builtin" - global-config: + config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: cloud-provider: "cloud/provider" cloud-region: "cloud/region" - output: "*" + output: tree: children: child: diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml b/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml index 4c2767a3a..3c9fe7e03 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/success-renaming.yml @@ -6,7 +6,7 @@ initialize: cloud-metadata: method: CSVLookup path: "builtin" - global-config: + config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: cloud-provider: "cloud/provider" diff --git a/manifests/examples/builtins/csv-lookup/region-metadata/success.yml b/manifests/examples/builtins/csv-lookup/region-metadata/success.yml index 71417932d..501bde3ce 100644 --- a/manifests/examples/builtins/csv-lookup/region-metadata/success.yml +++ b/manifests/examples/builtins/csv-lookup/region-metadata/success.yml @@ -6,7 +6,7 @@ initialize: cloud-metadata: method: CSVLookup path: "builtin" - global-config: + config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: cloud-provider: "cloud/provider" diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml index 991bdbb8b..070b4f7b0 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yml @@ -6,7 +6,7 @@ initialize: tdp-finder: method: CSVLookup path: "builtin" - global-config: + config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv query: name: physical-processor @@ -17,7 +17,6 @@ tree: pipeline: compute: - tdp-finder - config: inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml index 3d433d6b2..4c9401e02 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yml @@ -6,7 +6,7 @@ initialize: tdp-finder: method: CSVLookup path: "builtin" - global-config: + config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv query: name: physical-processor @@ -17,7 +17,6 @@ tree: pipeline: compute: - tdp-finder - config: inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml b/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml index a04288e0c..aab0ae3ef 100644 --- a/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml +++ b/manifests/examples/builtins/csv-lookup/tdp-finder/success.yml @@ -6,7 +6,7 @@ initialize: tdp-finder: method: CSVLookup path: "builtin" - global-config: + config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv query: name: physical-processor @@ -17,7 +17,6 @@ tree: pipeline: compute: - tdp-finder - config: inputs: - timestamp: 2023-07-06T00:00 duration: 300 diff --git a/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml b/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml deleted file mode 100644 index 96d627aee..000000000 --- a/manifests/examples/builtins/divide/failure-denominator-equal-zero.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: divide -description: denominator is invalid, denominator is -tags: -initialize: - plugins: - cloud-metadata: - path: builtin - method: CSVLookup - global-config: - filepath: >- - https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv - query: - instance-class: cloud/instance-type - output: ["cpu-cores-utilized", "vcpus-allocated"] - divide: - method: Divide - path: "builtin" - global-config: - numerator: vcpus-allocated - denominator: 0 - output: cpu/number-cores -tree: - children: - child: - pipeline: - compute: - - cloud-metadata - - divide - config: - divide: - defaults: - cloud/vendor: aws - cloud/instance-type: m5n.large - cpu/name: Intel® Core™ i7-1185G7 - inputs: - - timestamp: 2023-08-06T00:00 - duration: 3600 - cpu/utilization: 80 diff --git a/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml b/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml index fef8a1a7c..f43489d72 100644 --- a/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml +++ b/manifests/examples/builtins/divide/failure-invalid-config-denominator.yml @@ -1,12 +1,12 @@ name: divide -description: failure when `global-config.denominator` is string +description: failure when `config.denominator` is string tags: initialize: plugins: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -15,7 +15,7 @@ initialize: divide: method: Divide path: "builtin" - global-config: + config: numerator: vcpus-allocated denominator: "vcpus" output: cpu/number-cores @@ -26,8 +26,6 @@ tree: compute: - cloud-metadata - divide - config: - divide: defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/examples/builtins/divide/failure-missing-numerator.yml b/manifests/examples/builtins/divide/failure-missing-numerator.yml index 5645f0ecf..a50b90d6c 100644 --- a/manifests/examples/builtins/divide/failure-missing-numerator.yml +++ b/manifests/examples/builtins/divide/failure-missing-numerator.yml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -15,7 +15,7 @@ initialize: divide: method: Divide path: "builtin" - global-config: + config: #numerator: vcpus-allocated denominator: 2 output: cpu/number-cores @@ -26,8 +26,6 @@ tree: compute: - cloud-metadata - divide - config: - divide: defaults: cloud/vendor: aws cloud/instance-type: m5n.large diff --git a/manifests/examples/builtins/divide/success.yml b/manifests/examples/builtins/divide/success.yml index 32b247a87..10d27bd7f 100644 --- a/manifests/examples/builtins/divide/success.yml +++ b/manifests/examples/builtins/divide/success.yml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -15,7 +15,7 @@ initialize: divide: method: Divide path: "builtin" - global-config: + config: numerator: vcpus-allocated denominator: 2 output: cpu/number-cores diff --git a/manifests/examples/builtins/exponent/success.yml b/manifests/examples/builtins/exponent/success.yml index 163dd1460..a05692ddb 100644 --- a/manifests/examples/builtins/exponent/success.yml +++ b/manifests/examples/builtins/exponent/success.yml @@ -6,7 +6,7 @@ initialize: exponent: method: Exponent path: "builtin" - global-config: + config: input-parameter: "cpu/energy" exponent: 2 output-parameter: "energy" diff --git a/manifests/examples/builtins/interpolation/interpolation.yml b/manifests/examples/builtins/interpolation/interpolation.yml deleted file mode 100644 index 394946467..000000000 --- a/manifests/examples/builtins/interpolation/interpolation.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: interpolation-demo -description: simple demo of interpolation plugin -tags: -initialize: - plugins: - interpolation: - method: Interpolation - path: "builtin" - global-config: - method: linear - x: [0, 10, 50, 100] - y: [0.12, 0.32, 0.75, 1.02] - input-parameter: "cpu/utilization" - output-parameter: "result" -tree: - children: - child: - pipeline: - compute: - - interpolation - inputs: - - timestamp: 2023-07-06T00:00 - duration: 3600 - cpu/utilization: 45 diff --git a/manifests/examples/builtins/interpolation/success.yml b/manifests/examples/builtins/interpolation/success.yml index 394946467..8f4fc946e 100644 --- a/manifests/examples/builtins/interpolation/success.yml +++ b/manifests/examples/builtins/interpolation/success.yml @@ -6,7 +6,7 @@ initialize: interpolation: method: Interpolation path: "builtin" - global-config: + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] diff --git a/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml b/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml index 472f797fd..d4e44fe22 100644 --- a/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml +++ b/manifests/examples/builtins/mock-observations/failure-invalid-config-cpu-range.yml @@ -1,5 +1,5 @@ name: mock-observation-demo -description: failure with `global-config->generators->randint->cpu/utilization->min` is greater than `max` +description: failure with `config->generators->randint->cpu/utilization->min` is greater than `max` tags: initialize: plugins: @@ -7,7 +7,7 @@ initialize: kind: plugin method: MockObservations path: "builtin" - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 60 diff --git a/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml b/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml index 85e3f566b..b0e2e801f 100644 --- a/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml +++ b/manifests/examples/builtins/mock-observations/failure-invalid-memory-utilization-range.yml @@ -7,7 +7,7 @@ initialize: kind: plugin method: MockObservations path: "builtin" - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 60 diff --git a/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml b/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml index 58545dcec..86078ad31 100644 --- a/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml +++ b/manifests/examples/builtins/mock-observations/failure-missing-timestamp-from-param.yml @@ -7,7 +7,7 @@ initialize: kind: plugin method: MockObservations path: "builtin" - global-config: + config: #timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 60 diff --git a/manifests/examples/builtins/mock-observations/success.yml b/manifests/examples/builtins/mock-observations/success.yml index 5ca2d1942..cb2151fcf 100644 --- a/manifests/examples/builtins/mock-observations/success.yml +++ b/manifests/examples/builtins/mock-observations/success.yml @@ -7,7 +7,7 @@ initialize: kind: plugin method: MockObservations path: "builtin" - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 60 diff --git a/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml b/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml index 988a33fc2..acb59098d 100644 --- a/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml +++ b/manifests/examples/builtins/multiply/failure-input-parameter-is-missing.yml @@ -6,7 +6,7 @@ initialize: multiply: method: Multiply path: "builtin" - global-config: + config: input-parameters: ["cpu/energy", "network/energy"] output-parameter: "energy-product" tree: diff --git a/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml b/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml index e6e138723..63ac84183 100644 --- a/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml +++ b/manifests/examples/builtins/multiply/success-with-multiple-inputs.yml @@ -6,7 +6,7 @@ initialize: multiply: method: Multiply path: "builtin" - global-config: + config: input-parameters: ["cpu/energy", "network/energy"] output-parameter: "energy-product" tree: diff --git a/manifests/examples/builtins/multiply/success.yml b/manifests/examples/builtins/multiply/success.yml index c5d53e046..ee7f8875b 100644 --- a/manifests/examples/builtins/multiply/success.yml +++ b/manifests/examples/builtins/multiply/success.yml @@ -7,7 +7,7 @@ initialize: multiply: method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu/energy", "network/energy"] output-parameter: "energy-product" tree: diff --git a/manifests/examples/builtins/regex/failure-missing-input-param.yml b/manifests/examples/builtins/regex/failure-missing-input-param.yml index 84adc0cb2..95e055eda 100644 --- a/manifests/examples/builtins/regex/failure-missing-input-param.yml +++ b/manifests/examples/builtins/regex/failure-missing-input-param.yml @@ -6,7 +6,7 @@ initialize: regex: method: Regex path: "builtin" - global-config: + config: parameter: physical-processor match: ^(.*), output: cpu/name diff --git a/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml b/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml index 8341d08e6..f79303efb 100644 --- a/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml +++ b/manifests/examples/builtins/regex/failure-not-matching-with-regex.yml @@ -6,9 +6,9 @@ initialize: regex: method: Regex path: "builtin" - global-config: + config: parameter: physical-processor - match: ^ + match: ^$ output: cpu/name tree: children: diff --git a/manifests/examples/builtins/regex/success.yml b/manifests/examples/builtins/regex/success.yml index 425dd9e67..81a3432f2 100644 --- a/manifests/examples/builtins/regex/success.yml +++ b/manifests/examples/builtins/regex/success.yml @@ -6,7 +6,7 @@ initialize: regex: method: Regex path: "builtin" - global-config: + config: parameter: physical-processor match: ^(.*), output: cpu/name diff --git a/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml b/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml deleted file mode 100644 index 503300f84..000000000 --- a/manifests/examples/builtins/sci-embodied/failure-invalid-default-emission-value.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: sci-embodied -description: failure with `defaults.device/emissions-embodied` being string instead of number -tags: -initialize: - plugins: - "sci-embodied": # a model that calculates m from te, tir, el, rr and rtor - method: SciEmbodied - path: "builtin" -tree: - children: - child: - pipeline: - compute: - - sci-embodied # duration & config -> embodied - defaults: - device/emissions-embodied: "fail" # gCO2eq - time-reserved: 3600 # 1hr in seconds - device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 - inputs: - - timestamp: 2023-07-06T00:00 - duration: 3600 diff --git a/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml b/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml deleted file mode 100644 index 8fd3e5784..000000000 --- a/manifests/examples/builtins/sci-embodied/failure-missing-expected-lifespan.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: sci-embodied -description: missing device/expected-lifespan -tags: -initialize: - plugins: - "sci-embodied": # a model that calculates m from te, tir, el, rr and rtor - method: SciEmbodied - path: "builtin" -tree: - children: - child: - pipeline: - compute: - - sci-embodied # duration & config -> embodied - defaults: - device/emissions-embodied: 1533.120 # gCO2eq - time-reserved: 3600 # 1hr in seconds - #device/expected-lifespan: 94608000 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 - inputs: - - timestamp: 2023-07-06T00:00 - duration: 3600 diff --git a/manifests/examples/builtins/sci-embodied/scenario-1.yml b/manifests/examples/builtins/sci-embodied/scenario-1.yml new file mode 100644 index 000000000..57bfdd5cf --- /dev/null +++ b/manifests/examples/builtins/sci-embodied/scenario-1.yml @@ -0,0 +1,27 @@ +name: embodied-carbon demo +description: +tags: +aggregation: + metrics: + - embodied-carbon + type: "both" +initialize: + plugins: + embodied-carbon: + method: SciEmbodied + path: builtin + config: + output-parameter: "embodied-carbon" +tree: + children: + child: + pipeline: + compute: + - embodied-carbon + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + hdd: 2 + - timestamp: 2023-08-06T10:00 + duration: 3600 + hdd: 2 diff --git a/manifests/examples/builtins/sci-embodied/scenario-2.yml b/manifests/examples/builtins/sci-embodied/scenario-2.yml new file mode 100644 index 000000000..d4c2640ff --- /dev/null +++ b/manifests/examples/builtins/sci-embodied/scenario-2.yml @@ -0,0 +1,37 @@ +name: embodied-carbon demo +description: +tags: +initialize: + plugins: + embodied-carbon: + method: SciEmbodied + path: builtin + config: + baseline-vcpus: 1 + baseline-memory: 16 + lifespan: 157680000 + baseline-emissions: 2000000 + vcpu-emissions-constant: 100000 + memory-emissions-constant: 1172 + ssd-emissions-constant: 50000 + hdd-emissions-constant: 100000 + gpu-emissions-constant: 150000 + output-parameter: "embodied-carbon" +tree: + children: + child: + pipeline: + compute: + - embodied-carbon + defaults: + vCPUs: 4 + memory: 32 + ssd: 1 + hdd: 1 + gpu: 1 + total-vcpus: 16 + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + - timestamp: 2023-08-06T10:00 + duration: 3600 diff --git a/manifests/examples/builtins/sci-embodied/success.yml b/manifests/examples/builtins/sci-embodied/success.yml index 991569404..cff80fe05 100644 --- a/manifests/examples/builtins/sci-embodied/success.yml +++ b/manifests/examples/builtins/sci-embodied/success.yml @@ -3,6 +3,15 @@ description: successful path tags: initialize: plugins: + "csv-lookup": + path: builtin + method: CSVLookup + config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: cloud/instance-type + output: ["cpu-cores-utilized", "vcpus-allocated"] "sci-embodied": # a model that calculates m from te, tir, el, rr and rtor method: SciEmbodied path: "builtin" @@ -11,6 +20,7 @@ tree: child: pipeline: compute: + - csv-lookup - sci-embodied # duration & config -> embodied defaults: device/emissions-embodied: 1533.120 # gCO2eq @@ -21,3 +31,6 @@ tree: inputs: - timestamp: 2023-07-06T00:00 duration: 3600 + cloud/vendor: intel + cloud/instance-type: Standard_A1_v2 + cpu/utilization: 10 diff --git a/manifests/examples/builtins/sci/failure-invalid-config-value.yml b/manifests/examples/builtins/sci/failure-invalid-config-value.yml index 5882111dd..961a28786 100644 --- a/manifests/examples/builtins/sci/failure-invalid-config-value.yml +++ b/manifests/examples/builtins/sci/failure-invalid-config-value.yml @@ -7,7 +7,7 @@ initialize: kind: plugin method: Sci path: "builtin" - # global-config: + # config: # functional-unit: 1 minute tree: children: @@ -15,9 +15,8 @@ tree: pipeline: compute: - sci - config: - sci: - functional-unit: 999 # factor to convert per time to per f.unit + defaults: + functional-unit: 999 # factor to convert per time to per f.unit inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/examples/builtins/sci/failure-missing-input-param.yml b/manifests/examples/builtins/sci/failure-missing-input-param.yml index 0a7677261..efa2d3d96 100644 --- a/manifests/examples/builtins/sci/failure-missing-input-param.yml +++ b/manifests/examples/builtins/sci/failure-missing-input-param.yml @@ -8,7 +8,7 @@ initialize: kind: plugin method: Sci path: "builtin" - global-config: + config: functional-unit: requests tree: children: diff --git a/manifests/examples/builtins/sci/success.yml b/manifests/examples/builtins/sci/success.yml index 85dd19db2..1ed4c72fb 100644 --- a/manifests/examples/builtins/sci/success.yml +++ b/manifests/examples/builtins/sci/success.yml @@ -7,7 +7,7 @@ initialize: kind: plugin method: Sci path: "builtin" - global-config: + config: functional-unit: requests tree: children: diff --git a/manifests/examples/builtins/shell/failure-invalid-command.yml b/manifests/examples/builtins/shell/failure-invalid-command.yml index e67aff103..0ced9f6ec 100644 --- a/manifests/examples/builtins/shell/failure-invalid-command.yml +++ b/manifests/examples/builtins/shell/failure-invalid-command.yml @@ -1,12 +1,12 @@ name: shell -description: falure with `global-config.command` being number instead od string +description: falure with `config.command` being number instead od string tags: initialize: plugins: shell: method: Shell path: "builtin" - global-config: + config: command: 1000 tree: children: diff --git a/manifests/examples/builtins/shell/success.yml b/manifests/examples/builtins/shell/success.yml index a26af274d..969e6ae92 100644 --- a/manifests/examples/builtins/shell/success.yml +++ b/manifests/examples/builtins/shell/success.yml @@ -5,8 +5,8 @@ initialize: plugins: shell: method: Shell - path: 'builtin' - global-config: + path: "builtin" + config: command: python3 /usr/local/bin/sampler tree: children: diff --git a/manifests/examples/builtins/subtract/success.yml b/manifests/examples/builtins/subtract/success.yml index 745ceb067..a227f7217 100644 --- a/manifests/examples/builtins/subtract/success.yml +++ b/manifests/examples/builtins/subtract/success.yml @@ -6,7 +6,7 @@ initialize: subtract: method: Subtract path: "builtin" - global-config: + config: input-parameters: ["cpu/energy", "network/energy"] output-parameter: "energy/diff" tree: diff --git a/manifests/examples/builtins/sum/failure-missing-input-param.yml b/manifests/examples/builtins/sum/failure-missing-input-param.yml index 1e963b694..a82f8748a 100644 --- a/manifests/examples/builtins/sum/failure-missing-input-param.yml +++ b/manifests/examples/builtins/sum/failure-missing-input-param.yml @@ -1,12 +1,12 @@ name: sum -description: failure with `inputs[0]` misses one of `global-config.input-parameters` +description: failure with `inputs[0]` misses one of `config.input-parameters` tags: initialize: plugins: sum: method: Sum path: "builtin" - global-config: + config: input-parameters: ["cpu/energy", "network/energy"] output-parameter: "energy" tree: diff --git a/manifests/examples/builtins/sum/failure-missing-output-param.yml b/manifests/examples/builtins/sum/failure-missing-output-param.yml index 0248bf449..0c8c323cb 100644 --- a/manifests/examples/builtins/sum/failure-missing-output-param.yml +++ b/manifests/examples/builtins/sum/failure-missing-output-param.yml @@ -1,12 +1,12 @@ name: sum -description: missing `output-parameter` in global-config +description: missing `output-parameter` in config tags: initialize: plugins: sum: method: Sum path: "builtin" - global-config: + config: input-parameters: ["cpu/energy", "network/energy"] # output-parameter: "energy" tree: diff --git a/manifests/examples/builtins/sum/success.yml b/manifests/examples/builtins/sum/success.yml index 366ae5580..8c30b20b8 100644 --- a/manifests/examples/builtins/sum/success.yml +++ b/manifests/examples/builtins/sum/success.yml @@ -6,7 +6,7 @@ initialize: sum: method: Sum path: "builtin" - global-config: + config: input-parameters: ["cpu/energy", "network/energy"] output-parameter: "energy" tree: diff --git a/manifests/examples/builtins/time-converter/success.yaml b/manifests/examples/builtins/time-converter/success.yaml index 30c5d987e..7b2fa1a61 100644 --- a/manifests/examples/builtins/time-converter/success.yaml +++ b/manifests/examples/builtins/time-converter/success.yaml @@ -6,7 +6,7 @@ initialize: time-converter: method: TimeConverter path: builtin - global-config: + config: input-parameter: "energy-per-year" original-time-unit: "year" new-time-unit: "duration" @@ -15,8 +15,8 @@ tree: children: child: pipeline: - - time-converter - config: + compute: + - time-converter defaults: energy-per-year: 10000 inputs: diff --git a/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml b/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml index ffed32274..6a268a2d2 100644 --- a/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml +++ b/manifests/examples/builtins/time-sync/failure-config-start-later-end.yml @@ -1,16 +1,14 @@ name: time-sync -description: failure with `global-config.start-time` being later than `global-config.end-time` +description: failure with `config.start-time` being later than `config.end-time` tags: initialize: - output: - - yaml plugins: - 'time-sync': + "time-sync": method: TimeSync path: "builtin" - global-config: - start-time: '2023-12-12T00:01:00.000Z' - end-time: '2023-12-12T00:00:00.000Z' + config: + start-time: "2023-12-12T00:01:00.000Z" + end-time: "2023-12-12T00:00:00.000Z" interval: 5 allow-padding: true tree: @@ -20,15 +18,15 @@ tree: compute: - time-sync inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 energy-cpu: 0.001 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 - energy-cpu: 0.001 \ No newline at end of file + energy-cpu: 0.001 diff --git a/manifests/examples/builtins/time-sync/failure-missing-global-config.yml b/manifests/examples/builtins/time-sync/failure-missing-config.yml similarity index 69% rename from manifests/examples/builtins/time-sync/failure-missing-global-config.yml rename to manifests/examples/builtins/time-sync/failure-missing-config.yml index c5715a26a..d0ab31a05 100644 --- a/manifests/examples/builtins/time-sync/failure-missing-global-config.yml +++ b/manifests/examples/builtins/time-sync/failure-missing-config.yml @@ -1,5 +1,5 @@ name: time-sync -description: missing global config +description: missing config tags: initialize: output: @@ -7,12 +7,7 @@ initialize: plugins: 'time-sync': method: TimeSync - path: "builtin" - global-config: - # start-time: '2023-12-12T00:00:00.000Z' - # end-time: '2023-12-12T00:01:00.000Z' - # interval: 5 - # allow-padding: true + path: 'builtin' tree: children: child: @@ -31,4 +26,4 @@ tree: energy-cpu: 0.001 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 - energy-cpu: 0.001 \ No newline at end of file + energy-cpu: 0.001 diff --git a/manifests/examples/builtins/time-sync/success.yml b/manifests/examples/builtins/time-sync/success.yml index 8aac13740..0fd0cbcf3 100644 --- a/manifests/examples/builtins/time-sync/success.yml +++ b/manifests/examples/builtins/time-sync/success.yml @@ -7,8 +7,8 @@ initialize: plugins: 'time-sync': method: TimeSync - path: "builtin" - global-config: + path: 'builtin' + config: start-time: '2023-12-12T00:00:00.000Z' end-time: '2023-12-12T00:01:00.000Z' interval: 5 diff --git a/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml index 8b8b44faf..708250e27 100644 --- a/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml +++ b/manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml @@ -1,7 +1,7 @@ name: regroup description: initialize: - plugins: + plugins: {} tree: children: my-app: diff --git a/manifests/examples/features/regroup/success.yml b/manifests/examples/features/regroup/success.yml index 6863dc0d6..de6571919 100644 --- a/manifests/examples/features/regroup/success.yml +++ b/manifests/examples/features/regroup/success.yml @@ -1,7 +1,7 @@ name: regroup description: successful path initialize: - plugins: + plugins: {} tree: children: my-app: diff --git a/manifests/examples/pipelines/generics.yml b/manifests/examples/pipelines/generics.yml index f9a19a6b3..78c254838 100644 --- a/manifests/examples/pipelines/generics.yml +++ b/manifests/examples/pipelines/generics.yml @@ -6,7 +6,7 @@ initialize: "interpolate": method: Interpolation path: "builtin" - global-config: + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] @@ -15,47 +15,47 @@ initialize: "cpu-factor-to-wattage": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" "wattage-times-duration": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" "wattage-to-energy-kwh": method: Divide path: "builtin" - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw "calculate-vcpu-ratio": method: Divide path: "builtin" - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio "correct-cpu-energy-for-vcpu-ratio": method: Divide path: "builtin" - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh "coefficient": path: "builtin" method: Coefficient - global-config: + config: input-parameter: cpu-energy-kwh coefficient: 2 output-parameter: energy-doubled "multiply": path: "builtin" method: Multiply - global-config: + config: input-parameters: ["cpu/utilization", "duration"] output-parameter: "cpu-times-duration" tree: diff --git a/manifests/examples/pipelines/instance-metadata.yml b/manifests/examples/pipelines/instance-metadata.yml index 0a64e9ac3..58b4c9613 100644 --- a/manifests/examples/pipelines/instance-metadata.yml +++ b/manifests/examples/pipelines/instance-metadata.yml @@ -6,7 +6,7 @@ initialize: cloud-instance-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv query: @@ -15,7 +15,7 @@ initialize: extract-processor-name: path: builtin method: Regex - global-config: + config: parameter: cpu-model-name match: /^([^,])+/g output: cpu/name diff --git a/manifests/examples/pipelines/nesting.yml b/manifests/examples/pipelines/nesting.yml index baf12a688..f62105c87 100644 --- a/manifests/examples/pipelines/nesting.yml +++ b/manifests/examples/pipelines/nesting.yml @@ -13,7 +13,7 @@ initialize: "interpolate": method: Interpolation path: "builtin" - global-config: + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] @@ -24,16 +24,20 @@ initialize: cpu/utilization: unit: percentage description: refers to CPU utilization. - aggregation-method: avg + aggregation-method: + time: avg + component: sum outputs: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg "cpu-factor-to-wattage": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" parameter-metadata: @@ -41,26 +45,32 @@ initialize: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu/thermal-design-power: unit: kWh description: thermal design power for a processor - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-wattage: unit: kWh description: the energy used by the CPU - aggregation-method: sum + aggregation-method: + time: sum + component: sum "wattage-times-duration": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" "wattage-to-energy-kwh": method: Divide path: "builtin" - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw @@ -69,16 +79,20 @@ initialize: cpu-wattage-times-duration: unit: kWh description: CPU wattage multiplied by duration - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: cpu-energy-raw: unit: kWh description: Raw energy used by CPU in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum "calculate-vcpu-ratio": method: Divide path: "builtin" - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio @@ -87,11 +101,13 @@ initialize: vcpu-ratio: unit: none description: Ratio of vCPUs - aggregation-method: none + aggregation-method: + time: copy + component: copy "correct-cpu-energy-for-vcpu-ratio": method: Divide path: "builtin" - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -101,7 +117,7 @@ initialize: "operational-carbon": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" parameter-metadata: @@ -109,54 +125,68 @@ initialize: cpu-energy-kwh: unit: kWh description: Corrected CPU energy in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum grid/carbon-intensity: unit: gCO2eq/kWh description: Carbon intensity for the grid - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: carbon-operational: unit: gCO2eq description: Operational carbon footprint - aggregation-method: sum + aggregation-method: + time: sum + component: sum sci: path: "builtin" method: Sci - global-config: + config: functional-unit: "requests" parameter-metadata: inputs: requests: unit: none description: expressed the final SCI value - aggregation-method: sum + aggregation-method: + time: sum + component: sum "sum-carbon": path: "builtin" method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon parameter-metadata: inputs: carbon-operational: description: Operational carbon footprint unit: gCO2eq - aggregation-method: sum - carbon-embodied: + aggregation-method: + time: sum + component: sum + embodied-carbon: description: Embodied carbon footprint unit: gCO2eq - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: carbon: description: Total carbon footprint unit: gCO2eq - aggregation-method: sum + aggregation-method: + time: sum + component: sum time-sync: method: TimeSync path: "builtin" - global-config: + config: start-time: "2023-12-12T00:00:00.000Z" end-time: "2023-12-12T00:01:00.000Z" interval: 5 @@ -166,27 +196,39 @@ initialize: timestamp: unit: RFC3339 description: refers to the time of occurrence of the input - aggregation-method: none + aggregation-method: + time: none + component: none duration: unit: seconds description: refers to the duration of the input - aggregation-method: sum + aggregation-method: + time: sum + component: sum cloud/instance-type: unit: none description: type of Cloud Instance name used in the cloud provider APIs - aggregation-method: none + aggregation-method: + time: copy + component: copy cloud/region: unit: none description: region cloud instance - aggregation-method: none + aggregation-method: + time: copy + component: copy time-reserved: unit: seconds description: time reserved for a component - aggregation-method: avg + aggregation-method: + time: avg + component: avg network/energy: description: "Energy consumed by the Network of the component" unit: "kWh" - aggregation-method: "sum" + aggregation-method: + time: sum + component: sum tree: children: child-0: diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml index c1f92da04..6a96cbfc8 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-aggregate.yaml @@ -10,14 +10,14 @@ initialize: interpolate: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -29,16 +29,20 @@ initialize: cpu/utilization: unit: percentage description: refers to CPU utilization. - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu-factor-to-wattage: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-factor - cpu/thermal-design-power @@ -48,20 +52,26 @@ initialize: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu/thermal-design-power: unit: kWh description: thermal design power for a processor - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-wattage: unit: kWh description: the energy used by the CPU - aggregation-method: sum + aggregation-method: + time: sum + component: sum wattage-times-duration: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-wattage - duration @@ -69,7 +79,7 @@ initialize: wattage-to-energy-kwh: path: builtin method: Divide - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw @@ -78,16 +88,20 @@ initialize: cpu-wattage-times-duration: unit: kWh description: CPU wattage multiplied by duration - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: cpu-energy-raw: unit: kWh description: Raw energy used by CPU in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum calculate-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio @@ -96,20 +110,26 @@ initialize: vcpus-total: unit: count description: total number of vcpus available on a particular resource - aggregation-method: none + aggregation-method: + time: none + component: none vcpus-allocated: unit: count description: number of vcpus allocated to particular resource - aggregation-method: none + aggregation-method: + time: none + component: none outputs: vcpu-ratio: unit: none description: Ratio of vCPUs - aggregation-method: none + aggregation-method: + time: none + component: none correct-cpu-energy-for-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -119,7 +139,7 @@ initialize: operational-carbon: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-energy-kwh - grid/carbon-intensity @@ -129,56 +149,70 @@ initialize: cpu-energy-kwh: unit: kWh description: Corrected CPU energy in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum grid/carbon-intensity: unit: gCO2eq/kWh description: Carbon intensity for the grid - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: carbon-operational: unit: gCO2eq description: Operational carbon footprint - aggregation-method: sum + aggregation-method: + time: sum + component: sum sci: path: builtin method: Sci - global-config: + config: functional-unit: requests parameter-metadata: inputs: requests: unit: none description: expressed the final SCI value - aggregation-method: sum + aggregation-method: + time: sum + component: sum sum-carbon: path: builtin method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon parameter-metadata: inputs: carbon-operational: description: Operational carbon footprint unit: gCO2eq - aggregation-method: sum - carbon-embodied: + aggregation-method: + time: sum + component: sum + embodied-carbon: description: Embodied carbon footprint unit: gCO2eq - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: carbon: description: Total carbon footprint unit: gCO2eq - aggregation-method: sum + aggregation-method: + time: sum + component: sum time-sync: path: builtin method: TimeSync - global-config: - start-time: '2023-12-12T00:00:00.000Z' - end-time: '2023-12-12T00:01:00.000Z' + config: + start-time: "2023-12-12T00:00:00.000Z" + end-time: "2023-12-12T00:01:00.000Z" interval: 5 allow-padding: true parameter-metadata: @@ -186,23 +220,33 @@ initialize: timestamp: unit: RFC3339 description: refers to the time of occurrence of the input - aggregation-method: none + aggregation-method: + time: none + component: none duration: unit: seconds description: refers to the duration of the input - aggregation-method: sum + aggregation-method: + time: sum + component: sum cloud/instance-type: unit: none description: type of Cloud Instance name used in the cloud provider APIs - aggregation-method: none + aggregation-method: + time: none + component: none cloud/region: unit: none description: region cloud instance - aggregation-method: none + aggregation-method: + time: none + component: none time-reserved: unit: seconds description: time reserved for a component - aggregation-method: avg + aggregation-method: + time: avg + component: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -212,20 +256,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-31T12:41:31.920Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.16' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.16" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -265,7 +309,7 @@ tree: - operational-carbon - sum-carbon - time-sync - - sci + - sci defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -275,32 +319,32 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 10 requests: 10 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west requests: 5 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west requests: 15 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west cpu/utilization: 15 requests: 30 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -319,11 +363,11 @@ tree: cpu-energy-raw: 0.0000563888888888889 vcpu-ratio: 8 cpu-energy-kwh: 0.000007048611111111113 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.0056388888888888895 carbon: 0.005649016996448503 sci: 0.000403501214032036 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -342,11 +386,11 @@ tree: cpu-energy-raw: 0.00005340277777777778 vcpu-ratio: 8 cpu-energy-kwh: 0.000006675347222222222 - carbon-embodied: 0.000010128107559614407 + embodied-carbon: 0.000010128107559614407 carbon-operational: 0.005340277777777777 carbon: 0.005350405885337391 sci: 0.0005589976298113692 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -365,11 +409,11 @@ tree: cpu-energy-raw: 0.00005190972222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.0000064887152777777775 - carbon-embodied: 0.000010128107559614407 + embodied-carbon: 0.000010128107559614407 carbon-operational: 0.005190972222222222 carbon: 0.0052011003297818366 sci: 0.0006170797001436077 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -388,11 +432,11 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0010402200659563674 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -411,11 +455,11 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0010402200659563674 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -434,11 +478,11 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0010402200659563674 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -457,11 +501,11 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0010402200659563674 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -480,11 +524,11 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.005190972222222222 carbon: 0.005201100329781837 sci: 0.0010402200659563674 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -503,11 +547,11 @@ tree: cpu-energy-raw: 0.000031145833333333336 vcpu-ratio: 8 cpu-energy-kwh: 0.000003893229166666667 - carbon-embodied: 0.000006076864535768645 + embodied-carbon: 0.000006076864535768645 carbon-operational: 0.0031145833333333334 carbon: 0.003120660197869102 sci: 0.0010402200659563674 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -526,11 +570,11 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -549,11 +593,11 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -572,11 +616,11 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cloud/instance-type: A1 cloud/region: uk-west @@ -595,7 +639,7 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 @@ -627,32 +671,32 @@ tree: vcpus-total: 8 vcpus-allocated: 1 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 cpu/utilization: 30 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 28 cloud/instance-type: A1 cloud/region: uk-west requests: 150 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 40 cloud/instance-type: A1 cloud/region: uk-west requests: 110 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cpu/utilization: 33 cloud/instance-type: A1 cloud/region: uk-west requests: 180 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 5 cpu/utilization: 22.8 cloud/instance-type: A1 @@ -671,11 +715,11 @@ tree: cpu-energy-raw: 0.00007191666666666668 vcpu-ratio: 8 cpu-energy-kwh: 0.000008989583333333334 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.007191666666666666 carbon: 0.007201794774226282 sci: 0.00003273543079193765 - - timestamp: '2023-12-12T00:00:05.000Z' + - timestamp: "2023-12-12T00:00:05.000Z" duration: 5 cpu/utilization: 29.6 cloud/instance-type: A1 @@ -694,11 +738,11 @@ tree: cpu-energy-raw: 0.00008565277777777778 vcpu-ratio: 8 cpu-energy-kwh: 0.000010706597222222223 - carbon-embodied: 0.000010128107559614407 + embodied-carbon: 0.000010128107559614407 carbon-operational: 0.008565277777777778 carbon: 0.008575405885337391 sci: 0.00009235052491901808 - - timestamp: '2023-12-12T00:00:10.000Z' + - timestamp: "2023-12-12T00:00:10.000Z" duration: 5 cpu/utilization: 30.6 cloud/instance-type: A1 @@ -717,11 +761,11 @@ tree: cpu-energy-raw: 0.00008505555555555556 vcpu-ratio: 8 cpu-energy-kwh: 0.000010631944444444445 - carbon-embodied: 0.000010128107559614407 + embodied-carbon: 0.000010128107559614407 carbon-operational: 0.008505555555555556 carbon: 0.00851568366311517 sci: 0.0001439849894729618 - - timestamp: '2023-12-12T00:00:15.000Z' + - timestamp: "2023-12-12T00:00:15.000Z" duration: 5 cpu/utilization: 26.4 cloud/instance-type: A1 @@ -740,11 +784,11 @@ tree: cpu-energy-raw: 0.00007878472222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.000009848090277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.007878472222222222 carbon: 0.007888600329781836 sci: 0.0002629533443260612 - - timestamp: '2023-12-12T00:00:20.000Z' + - timestamp: "2023-12-12T00:00:20.000Z" duration: 5 cpu/utilization: 26.4 cloud/instance-type: A1 @@ -763,11 +807,11 @@ tree: cpu-energy-raw: 0.00007878472222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.000009848090277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.007878472222222222 carbon: 0.007888600329781836 sci: 0.0002629533443260612 - - timestamp: '2023-12-12T00:00:25.000Z' + - timestamp: "2023-12-12T00:00:25.000Z" duration: 5 cpu/utilization: 26.4 cloud/instance-type: A1 @@ -786,11 +830,11 @@ tree: cpu-energy-raw: 0.00007878472222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.000009848090277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.007878472222222222 carbon: 0.007888600329781836 sci: 0.0002629533443260612 - - timestamp: '2023-12-12T00:00:30.000Z' + - timestamp: "2023-12-12T00:00:30.000Z" duration: 5 cpu/utilization: 26.4 cloud/instance-type: A1 @@ -809,11 +853,11 @@ tree: cpu-energy-raw: 0.00007878472222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.000009848090277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.007878472222222222 carbon: 0.007888600329781836 sci: 0.0002629533443260612 - - timestamp: '2023-12-12T00:00:35.000Z' + - timestamp: "2023-12-12T00:00:35.000Z" duration: 5 cpu/utilization: 26.4 cloud/instance-type: A1 @@ -832,11 +876,11 @@ tree: cpu-energy-raw: 0.00007878472222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.000009848090277777778 - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: 0.007878472222222222 carbon: 0.007888600329781836 sci: 0.0002629533443260612 - - timestamp: '2023-12-12T00:00:40.000Z' + - timestamp: "2023-12-12T00:00:40.000Z" duration: 5 cpu/utilization: 19.8 cloud/instance-type: A1 @@ -855,11 +899,11 @@ tree: cpu-energy-raw: 0.00004727083333333333 vcpu-ratio: 8 cpu-energy-kwh: 0.000005908854166666666 - carbon-embodied: 0.000006076864535768645 + embodied-carbon: 0.000006076864535768645 carbon-operational: 0.004727083333333333 carbon: 0.0047331601978691015 sci: 0.00026295334432606117 - - timestamp: '2023-12-12T00:00:45.000Z' + - timestamp: "2023-12-12T00:00:45.000Z" duration: 5 cpu/utilization: 0 cloud/instance-type: A1 @@ -878,11 +922,11 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:50.000Z' + - timestamp: "2023-12-12T00:00:50.000Z" duration: 5 cpu/utilization: 0 cloud/instance-type: A1 @@ -901,11 +945,11 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:00:55.000Z' + - timestamp: "2023-12-12T00:00:55.000Z" duration: 5 cpu/utilization: 0 cloud/instance-type: A1 @@ -924,11 +968,11 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: '2023-12-12T00:01:00.000Z' + - timestamp: "2023-12-12T00:01:00.000Z" duration: 1 cpu/utilization: 0 cloud/instance-type: A1 @@ -947,7 +991,7 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 @@ -955,43 +999,43 @@ tree: carbon: 0.06846904616945712 outputs: - carbon: 0.012850811770674785 - timestamp: '2023-12-12T00:00:00.000Z' + timestamp: "2023-12-12T00:00:00.000Z" duration: 5 - carbon: 0.013925811770674782 - timestamp: '2023-12-12T00:00:05.000Z' + timestamp: "2023-12-12T00:00:05.000Z" duration: 5 - carbon: 0.013716783992897007 - timestamp: '2023-12-12T00:00:10.000Z' + timestamp: "2023-12-12T00:00:10.000Z" duration: 5 - carbon: 0.013089700659563674 - timestamp: '2023-12-12T00:00:15.000Z' + timestamp: "2023-12-12T00:00:15.000Z" duration: 5 - carbon: 0.013089700659563674 - timestamp: '2023-12-12T00:00:20.000Z' + timestamp: "2023-12-12T00:00:20.000Z" duration: 5 - carbon: 0.013089700659563674 - timestamp: '2023-12-12T00:00:25.000Z' + timestamp: "2023-12-12T00:00:25.000Z" duration: 5 - carbon: 0.013089700659563674 - timestamp: '2023-12-12T00:00:30.000Z' + timestamp: "2023-12-12T00:00:30.000Z" duration: 5 - carbon: 0.013089700659563674 - timestamp: '2023-12-12T00:00:35.000Z' + timestamp: "2023-12-12T00:00:35.000Z" duration: 5 - carbon: 0.007853820395738204 - timestamp: '2023-12-12T00:00:40.000Z' + timestamp: "2023-12-12T00:00:40.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:45.000Z' + timestamp: "2023-12-12T00:00:45.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:50.000Z' + timestamp: "2023-12-12T00:00:50.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:00:55.000Z' + timestamp: "2023-12-12T00:00:55.000Z" duration: 5 - carbon: 0 - timestamp: '2023-12-12T00:01:00.000Z' + timestamp: "2023-12-12T00:01:00.000Z" duration: 1 aggregated: carbon: 0.11379573122780316 diff --git a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml index c86d6fec1..3a744410c 100644 --- a/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml +++ b/manifests/examples/pipelines/outputs-if-diff/pipeline-with-mocks.yaml @@ -10,7 +10,7 @@ initialize: mock-observations: path: builtin method: MockObservations - global-config: + config: timestamp-from: 2023-12-12T00:00 timestamp-to: 2023-12-12T00:10 duration: 60 @@ -28,23 +28,31 @@ initialize: timestamp: unit: RFC3339 description: refers to the time of occurrence of the input - aggregation-method: none + aggregation-method: + time: none + component: none duration: unit: seconds description: refers to the duration of the input - aggregation-method: sum + aggregation-method: + time: sum + component: sum cloud/instance-type: unit: none description: type of Cloud Instance name used in the cloud provider APIs - aggregation-method: none + aggregation-method: + time: none + component: none cloud/region: unit: none description: region cloud instance - aggregation-method: none + aggregation-method: + time: none + component: none interpolate: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 @@ -63,16 +71,20 @@ initialize: cpu/utilization: unit: percentage description: refers to CPU utilization. - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu-factor-to-wattage: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-factor - cpu/thermal-design-power @@ -82,20 +94,26 @@ initialize: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu/thermal-design-power: unit: kWh description: thermal design power for a processor - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-wattage: unit: kWh description: the energy used by the CPU - aggregation-method: sum + aggregation-method: + time: sum + component: sum wattage-times-duration: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-wattage - duration @@ -105,20 +123,26 @@ initialize: cpu-wattage: unit: kWh description: Energy used by the CPU - aggregation-method: sum + aggregation-method: + time: sum + component: sum duration: unit: seconds description: Duration of the observation - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: cpu-wattage-times-duration: unit: kWh description: CPU wattage multiplied by duration - aggregation-method: sum + aggregation-method: + time: sum + component: sum wattage-to-energy-kwh: path: builtin method: Divide - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw @@ -127,16 +151,20 @@ initialize: cpu-wattage-times-duration: unit: kWh description: CPU wattage multiplied by duration - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: cpu-energy-raw: unit: kWh description: Raw energy used by CPU in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum calculate-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio @@ -145,20 +173,26 @@ initialize: vcpus-total: unit: count description: total number of vcpus available on a particular resource - aggregation-method: none + aggregation-method: + time: none + component: none vcpus-allocated: unit: count description: number of vcpus allocated to particular resource - aggregation-method: none + aggregation-method: + time: none + component: none outputs: vcpu-ratio: unit: none description: Ratio of vCPUs - aggregation-method: none + aggregation-method: + time: none + component: none correct-cpu-energy-for-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -167,23 +201,29 @@ initialize: cpu-energy-raw: unit: kWh description: Raw energy used by CPU in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum vcpu-ratio: unit: none description: Ratio of vCPUs - aggregation-method: none + aggregation-method: + time: none + component: none outputs: cpu-energy-kwh: unit: kWh description: Corrected CPU energy in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum sci-embodied: path: builtin method: SciEmbodied operational-carbon: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-energy-kwh - grid/carbon-intensity @@ -193,59 +233,75 @@ initialize: cpu-energy-kwh: unit: kWh description: Corrected CPU energy in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum grid/carbon-intensity: unit: gCO2eq/kWh description: Carbon intensity for the grid - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: carbon-operational: unit: gCO2eq description: Operational carbon footprint - aggregation-method: sum + aggregation-method: + time: sum + component: sum sum-carbon: path: builtin method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon parameter-metadata: inputs: carbon-operational: unit: gCO2eq description: Operational carbon footprint - aggregation-method: sum - carbon-embodied: + aggregation-method: + time: sum + component: sum + embodied-carbon: unit: gCO2eq description: Embodied carbon footprint - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: carbon: unit: gCO2eq description: Total carbon footprint - aggregation-method: sum + aggregation-method: + time: sum + component: sum sci: path: builtin method: Sci - global-config: + config: functional-unit: requests parameter-metadata: inputs: requests: unit: none description: expressed the final SCI value - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: sci: unit: none description: Scientific Carbon Intensity - aggregation-method: none + aggregation-method: + time: none + component: none time-sync: path: builtin method: TimeSync - global-config: + config: start-time: "2023-12-12T00:00:00.000Z" end-time: "2023-12-12T00:01:00.000Z" interval: 5 @@ -255,12 +311,16 @@ initialize: time-reserved: unit: seconds description: time reserved for a component - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: synced-time: unit: none description: Synced time - aggregation-method: none + aggregation-method: + time: none + component: none execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node @@ -379,7 +439,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -402,7 +462,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -425,7 +485,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -448,7 +508,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -471,7 +531,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -494,7 +554,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -517,7 +577,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -540,7 +600,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -563,7 +623,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -586,7 +646,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -609,7 +669,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -632,7 +692,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.000010128107559614409 + embodied-carbon: 0.000010128107559614409 carbon-operational: "*" carbon: "*" sci: "*" @@ -655,7 +715,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: 0.0000020256215119228817 + embodied-carbon: 0.0000020256215119228817 carbon-operational: "*" carbon: "*" sci: "*" @@ -733,7 +793,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -756,7 +816,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -779,7 +839,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -802,7 +862,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -825,7 +885,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -848,7 +908,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -871,7 +931,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -894,7 +954,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -917,7 +977,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -940,7 +1000,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -963,7 +1023,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -986,7 +1046,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -1009,7 +1069,7 @@ tree: cpu-energy-raw: "*" vcpu-ratio: 8 cpu-energy-kwh: "*" - carbon-embodied: "*" + embodied-carbon: "*" carbon-operational: "*" carbon: "*" sci: "*" @@ -1056,4 +1116,4 @@ tree: timestamp: "2023-12-12T00:01:00.000Z" duration: 1 aggregated: - carbon: "*" + carbon: "*" \ No newline at end of file diff --git a/manifests/examples/pipelines/pipeline-teads-sci.yml b/manifests/examples/pipelines/pipeline-teads-sci.yml index d80d0e89e..2a20ce1b7 100644 --- a/manifests/examples/pipelines/pipeline-teads-sci.yml +++ b/manifests/examples/pipelines/pipeline-teads-sci.yml @@ -5,43 +5,43 @@ initialize: plugins: "interpolate": method: Interpolation - path: 'builtin' - global-config: + path: "builtin" + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] - input-parameter: 'cpu/utilization' - output-parameter: 'cpu-factor' + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" "cpu-factor-to-wattage": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" "wattage-times-duration": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" "wattage-to-energy-kwh": method: Divide path: "builtin" - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw "calculate-vcpu-ratio": method: Divide path: "builtin" - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio "correct-cpu-energy-for-vcpu-ratio": method: Divide path: "builtin" - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -51,26 +51,26 @@ initialize: "operational-carbon": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" "sci": path: "builtin" method: Sci - global-config: + config: functional-unit: "component" "sum-carbon": path: "builtin" method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon "time-sync": method: TimeSync path: "builtin" - global-config: + config: start-time: "2023-12-12T00:00:00.000Z" end-time: "2023-12-12T00:01:00.000Z" interval: 5 @@ -123,4 +123,4 @@ tree: cloud/instance-type: A1 cloud/region: uk-west cpu/utilization: 15 - network/energy: 0.000001 + network/energy: 0.000001 \ No newline at end of file diff --git a/manifests/examples/pipelines/pipeline-with-aggregate.yml b/manifests/examples/pipelines/pipeline-with-aggregate.yml index 6f7822ad7..c95047f49 100644 --- a/manifests/examples/pipelines/pipeline-with-aggregate.yml +++ b/manifests/examples/pipelines/pipeline-with-aggregate.yml @@ -10,7 +10,7 @@ initialize: "interpolate": method: Interpolation path: "builtin" - global-config: + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] @@ -21,16 +21,20 @@ initialize: cpu/utilization: unit: percentage description: refers to CPU utilization. - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg "cpu-factor-to-wattage": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" parameter-metadata: @@ -38,26 +42,32 @@ initialize: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu/thermal-design-power: unit: kWh description: thermal design power for a processor - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-wattage: unit: kWh description: the energy used by the CPU - aggregation-method: sum + aggregation-method: + time: sum + component: sum "wattage-times-duration": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" "wattage-to-energy-kwh": method: Divide path: "builtin" - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw @@ -66,16 +76,20 @@ initialize: cpu-wattage-times-duration: unit: kWh description: CPU wattage multiplied by duration - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: cpu-energy-raw: unit: kWh description: Raw energy used by CPU in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum "calculate-vcpu-ratio": method: Divide path: "builtin" - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio @@ -84,20 +98,26 @@ initialize: vcpus-total: unit: count description: total number of vcpus available on a particular resource - aggregation-method: none + aggregation-method: + time: copy + component: copy vcpus-allocated: unit: count description: number of vcpus allocated to particular resource - aggregation-method: none + aggregation-method: + time: copy + component: copy outputs: vcpu-ratio: unit: none description: Ratio of vCPUs - aggregation-method: none + aggregation-method: + time: copy + component: copy "correct-cpu-energy-for-vcpu-ratio": method: Divide path: "builtin" - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -107,7 +127,7 @@ initialize: "operational-carbon": method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" parameter-metadata: @@ -115,45 +135,55 @@ initialize: cpu-energy-kwh: unit: kWh description: Corrected CPU energy in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum grid/carbon-intensity: unit: gCO2eq/kWh description: Carbon intensity for the grid - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: carbon-operational: unit: gCO2eq description: Operational carbon footprint - aggregation-method: sum + aggregation-method: + time: sum + component: sum "sci": path: "builtin" method: Sci - global-config: + config: functional-unit: requests # factor to convert per time to per f.unit parameter-metadata: inputs: requests: unit: none description: expressed the final SCI value - aggregation-method: sum + aggregation-method: + time: sum + component: sum "sum-carbon": path: "builtin" method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon parameter-metadata: outputs: carbon: unit: gCO2eq description: product of carbon - aggregation-method: sum + aggregation-method: + time: sum + component: sum "time-sync": method: TimeSync path: "builtin" - global-config: + config: start-time: "2023-12-12T00:00:00.000Z" end-time: "2023-12-12T00:01:00.000Z" interval: 5 @@ -259,4 +289,4 @@ tree: cpu/utilization: 33 cloud/instance-type: A1 cloud/region: uk-west - requests: 180 + requests: 180 \ No newline at end of file diff --git a/manifests/examples/pipelines/pipeline-with-mocks.yml b/manifests/examples/pipelines/pipeline-with-mocks.yml index fc6f81cb0..0358525c2 100644 --- a/manifests/examples/pipelines/pipeline-with-mocks.yml +++ b/manifests/examples/pipelines/pipeline-with-mocks.yml @@ -11,7 +11,7 @@ initialize: kind: plugin method: MockObservations path: "builtin" - global-config: + config: timestamp-from: "2023-12-12T00:00:00.000Z" timestamp-to: "2023-12-12T00:00:13.000Z" duration: 30 @@ -29,23 +29,31 @@ initialize: timestamp: description: refers to the time of occurrence of the input unit: RFC3339 - aggregation-method: none + aggregation-method: + time: none + component: none duration: description: refers to the duration of the input unit: seconds - aggregation-method: sum + aggregation-method: + time: sum + component: sum cloud/instance-type: description: type of Cloud Instance name used in the cloud provider APIs unit: none - aggregation-method: none + aggregation-method: + time: none + component: none cloud/region: description: region cloud instance unit: none - aggregation-method: none + aggregation-method: + time: none + component: none interpolate: method: Interpolation path: "builtin" - global-config: + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] @@ -56,16 +64,20 @@ initialize: cpu/utilization: description: refers to CPU utilization. unit: percentage - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-factor: description: result of interpolate unit: kWh - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu-factor-to-wattage: method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" parameter-metadata: @@ -73,20 +85,26 @@ initialize: cpu-factor: description: result of interpolate unit: kWh - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu/thermal-design-power: description: thermal design power for a processor unit: kWh - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-wattage: description: the energy used by the CPU unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum wattage-times-duration: method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" parameter-metadata: @@ -94,20 +112,26 @@ initialize: cpu-wattage: description: Energy used by the CPU unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum duration: description: Duration of the observation unit: seconds - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: cpu-wattage-times-duration: description: CPU wattage multiplied by duration unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum wattage-to-energy-kwh: method: Divide path: "builtin" - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw @@ -116,16 +140,20 @@ initialize: cpu-wattage-times-duration: description: CPU wattage multiplied by duration unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: cpu-energy-raw: description: Raw energy used by CPU in kWh unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum calculate-vcpu-ratio: method: Divide path: "builtin" - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio @@ -134,20 +162,26 @@ initialize: vcpus-total: description: total number of vcpus available on a particular resource unit: count - aggregation-method: none + aggregation-method: + time: none + component: none vcpus-allocated: description: number of vcpus allocated to particular resource unit: count - aggregation-method: none + aggregation-method: + time: none + component: none outputs: vcpu-ratio: description: Ratio of vCPUs unit: none - aggregation-method: none + aggregation-method: + time: none + component: none correct-cpu-energy-for-vcpu-ratio: method: Divide path: "builtin" - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -156,23 +190,29 @@ initialize: cpu-energy-raw: description: Raw energy used by CPU in kWh unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum vcpu-ratio: description: Ratio of vCPUs unit: none - aggregation-method: none + aggregation-method: + time: none + component: none outputs: cpu-energy-kwh: description: Corrected CPU energy in kWh unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum sci-embodied: path: "builtin" method: SciEmbodied operational-carbon: method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-energy-kwh", "grid/carbon-intensity"] output-parameter: "carbon-operational" parameter-metadata: @@ -180,59 +220,75 @@ initialize: cpu-energy-kwh: description: Corrected CPU energy in kWh unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum grid/carbon-intensity: description: Carbon intensity for the grid unit: gCO2eq/kWh - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: carbon-operational: description: Operational carbon footprint unit: gCO2eq - aggregation-method: sum + aggregation-method: + time: sum + component: sum sum-carbon: path: "builtin" method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon parameter-metadata: inputs: carbon-operational: description: Operational carbon footprint unit: gCO2eq - aggregation-method: sum - carbon-embodied: + aggregation-method: + time: sum + component: sum + embodied-carbon: description: Embodied carbon footprint unit: gCO2eq - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: carbon: description: Total carbon footprint unit: gCO2eq - aggregation-method: sum + aggregation-method: + time: sum + component: sum sci: path: "builtin" method: Sci - global-config: + config: functional-unit: "requests" parameter-metadata: inputs: requests: description: expressed the final SCI value unit: none - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: sci: description: Scientific Carbon Intensity unit: none - aggregation-method: none + aggregation-method: + time: none + component: none time-sync: method: TimeSync path: "builtin" - global-config: + config: start-time: "2023-12-12T00:00:00.000Z" end-time: "2023-12-12T00:01:00.000Z" interval: 5 @@ -266,6 +322,7 @@ tree: device/expected-lifespan: 94608000 # 3 years in seconds vcpus-total: 8 vcpus-allocated: 1 + requests: 50 inputs: child-2: pipeline: @@ -294,4 +351,5 @@ tree: device/expected-lifespan: 94608000 # 3 years in seconds vcpus-total: 8 vcpus-allocated: 1 + requests: 50 inputs: diff --git a/manifests/examples/pipelines/scenario-1.yml b/manifests/examples/pipelines/scenario-1.yml deleted file mode 100644 index 9ab2d20f4..000000000 --- a/manifests/examples/pipelines/scenario-1.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: demo -description: demo for observe feat -tags: -initialize: - plugins: - mock-observations: - kind: plugin - method: MockObservations - path: "builtin" - global-config: - timestamp-from: 2023-07-06T00:00 - timestamp-to: 2023-07-06T00:01 - duration: 60 - components: - - cloud/instance-type: A1 - - cloud/instance-type: B1 - generators: - common: - region: uk-west - common-key: common-val - randint: - cpu/utilization: - min: 1 - max: 99 - memory/utilization: - min: 1 - max: 99 -tree: - children: - child: - pipeline: - observe: - - mock-observations - inputs: null diff --git a/manifests/examples/pipelines/scenario-2.yml b/manifests/examples/pipelines/scenario-2.yml deleted file mode 100644 index 569cbcb05..000000000 --- a/manifests/examples/pipelines/scenario-2.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: regroup demo -description: -initialize: - plugins: - interpolate: - method: Interpolation - path: "builtin" - global-config: - method: linear - x: [0, 10, 50, 100] - y: [0.12, 0.32, 0.75, 1.02] - input-parameter: "cpu/utilization" - output-parameter: "cpu-factor" -tree: - children: - child: - pipeline: - observe: - regroup: - - cloud/region - - cloud/instance-type - inputs: - - timestamp: 2023-07-06T00:00 - duration: 300 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 99 - - timestamp: 2023-07-06T05:00 - duration: 300 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 23 - - timestamp: 2023-07-06T10:00 - duration: 300 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 12 - - timestamp: 2023-07-06T00:00 # note this time restarts at the start timstamp - duration: 300 - cloud/instance-type: B1 - cloud/region: uk-west - cpu/utilization: 11 - - timestamp: 2023-07-06T05:00 - duration: 300 - cloud/instance-type: B1 - cloud/region: uk-west - cpu/utilization: 67 - - timestamp: 2023-07-06T10:00 - duration: 300 - cloud/instance-type: B1 - cloud/region: uk-west - cpu/utilization: 1 diff --git a/manifests/examples/pipelines/scenario-3.yml b/manifests/examples/pipelines/scenario-3.yml index f5710b553..8d8eb6c09 100644 --- a/manifests/examples/pipelines/scenario-3.yml +++ b/manifests/examples/pipelines/scenario-3.yml @@ -5,7 +5,7 @@ initialize: "sum": path: "builtin" method: Sum - global-config: + config: input-parameters: - cpu/energy - network/energy diff --git a/manifests/examples/pipelines/scenario-4.yml b/manifests/examples/pipelines/scenario-4.yml index 151db09be..52f9e2bd8 100644 --- a/manifests/examples/pipelines/scenario-4.yml +++ b/manifests/examples/pipelines/scenario-4.yml @@ -1,12 +1,12 @@ name: demo -description: +description: tags: initialize: plugins: "sum": path: "builtin" method: Sum - global-config: + config: input-parameters: - cpu/energy - network/energy @@ -14,14 +14,14 @@ initialize: "coefficient": path: "builtin" method: Coefficient - global-config: + config: input-parameter: energy coefficient: 2 output-parameter: energy-doubled "multiply": path: "builtin" method: Multiply - global-config: + config: input-parameters: ["cpu/utilization", "duration"] output-parameter: "cpu-times-duration" tree: @@ -29,7 +29,7 @@ tree: child-1: pipeline: observe: - compute: + compute: - sum - coefficient - multiply diff --git a/manifests/examples/pipelines/scenario-5.yml b/manifests/examples/pipelines/scenario-5.yml index 6990c33e3..adb0fe139 100644 --- a/manifests/examples/pipelines/scenario-5.yml +++ b/manifests/examples/pipelines/scenario-5.yml @@ -1,5 +1,5 @@ name: demo -description: +description: tags: initialize: plugins: @@ -7,7 +7,7 @@ initialize: kind: plugin method: MockObservations path: "builtin" - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:01 duration: 60 @@ -28,7 +28,7 @@ initialize: sum: path: "builtin" method: Sum - global-config: + config: input-parameters: - cpu/utilization - memory/utilization diff --git a/manifests/examples/pipelines/sci.yml b/manifests/examples/pipelines/sci.yml index 949d9e7df..4fca75ca7 100644 --- a/manifests/examples/pipelines/sci.yml +++ b/manifests/examples/pipelines/sci.yml @@ -6,7 +6,7 @@ initialize: interpolate: method: Interpolation path: "builtin" - global-config: + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] @@ -15,40 +15,40 @@ initialize: cpu-factor-to-wattage: method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-factor", "cpu/thermal-design-power"] output-parameter: "cpu-wattage" wattage-times-duration: method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" wattage-to-energy-kwh: method: Divide path: "builtin" - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw calculate-vcpu-ratio: method: Divide path: "builtin" - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio correct-cpu-energy-for-vcpu-ratio: method: Divide path: "builtin" - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu/energy sum-energy-components: path: "builtin" method: Sum - global-config: + config: input-parameters: - cpu/energy - network/energy @@ -59,21 +59,21 @@ initialize: "operational-carbon": method: Multiply path: builtin - global-config: + config: input-parameters: ["energy", "grid/carbon-intensity"] output-parameter: "carbon-operational" "sum-carbon": path: "builtin" method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon "sci": path: "builtin" method: Sci - global-config: + config: functional-unit: "component" tree: children: @@ -126,4 +126,4 @@ tree: cloud/instance-type: A1 cloud/region: uk-west cpu/utilization: 15 - network/energy: 0.000001 + network/energy: 0.000001 \ No newline at end of file diff --git a/manifests/examples/pipelines/teads-curve.yml b/manifests/examples/pipelines/teads-curve.yml index 1679a0976..f0a650aa9 100644 --- a/manifests/examples/pipelines/teads-curve.yml +++ b/manifests/examples/pipelines/teads-curve.yml @@ -1,47 +1,47 @@ name: carbon-intensity plugin demo -description: +description: tags: initialize: plugins: interpolate: method: Interpolation - path: 'builtin' - global-config: + path: "builtin" + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] - input-parameter: 'cpu/utilization' - output-parameter: 'cpu-factor' + input-parameter: "cpu/utilization" + output-parameter: "cpu-factor" cpu-factor-to-wattage: method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-factor", "thermal-design-power"] output-parameter: "cpu-wattage" wattage-times-duration: method: Multiply path: builtin - global-config: + config: input-parameters: ["cpu-wattage", "duration"] output-parameter: "cpu-wattage-times-duration" wattage-to-energy-kwh: method: Divide path: "builtin" - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw calculate-vcpu-ratio: method: Divide path: "builtin" - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio correct-cpu-energy-for-vcpu-ratio: method: Divide path: "builtin" - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh diff --git a/manifests/examples/pipelines/zeros.yml b/manifests/examples/pipelines/zeros.yml index f76a2c79f..958aa2b99 100644 --- a/manifests/examples/pipelines/zeros.yml +++ b/manifests/examples/pipelines/zeros.yml @@ -6,7 +6,7 @@ initialize: "sum-zero-and-one": path: "builtin" method: Sum - global-config: + config: input-parameters: - some-value - zero-value @@ -14,7 +14,7 @@ initialize: "sum-zero-and-zero": path: "builtin" method: Sum - global-config: + config: input-parameters: - zero-value - zero-value @@ -22,7 +22,7 @@ initialize: "subtract-one-and-zero": path: "builtin" method: Subtract - global-config: + config: input-parameters: - some-value - zero-value @@ -30,7 +30,7 @@ initialize: "subtract-zero-and-zero": path: "builtin" method: Sum - global-config: + config: input-parameters: - zero-value - zero-value @@ -38,7 +38,7 @@ initialize: "subtract-zero-and-one": path: "builtin" method: Subtract - global-config: + config: input-parameters: - zero-value - some-value @@ -46,61 +46,61 @@ initialize: "coefficient-one-times-zero": path: "builtin" method: Coefficient - global-config: + config: input-parameter: zero-value coefficient: 1 output-parameter: zero-times-one-coefficient "coefficient-zero-times-one": path: "builtin" method: Coefficient - global-config: + config: input-parameter: some-value coefficient: 0 output-parameter: one-times-zero-coefficient "coefficient-zero-times-zero": path: "builtin" method: Coefficient - global-config: + config: input-parameter: zero-value coefficient: 0 output-parameter: zero-times-zero-coefficient "multiply-one-times-zero": path: "builtin" method: Multiply - global-config: + config: input-parameters: ["some-value", "zero-value"] output-parameter: "one-times-zero" "multiply-zero-times-one": path: "builtin" method: Multiply - global-config: + config: input-parameters: ["zero-value", "zero-value"] output-parameter: "zero-times-one" exponent-one-to-zero: method: Exponent - path: 'builtin' - global-config: - input-parameter: 'some-value' + path: "builtin" + config: + input-parameter: "some-value" exponent: 0 - output-parameter: 'one-raised-to-zero-power' + output-parameter: "one-raised-to-zero-power" exponent-zero-to-zero: method: Exponent - path: 'builtin' - global-config: - input-parameter: 'zero-value' + path: "builtin" + config: + input-parameter: "zero-value" exponent: 0 - output-parameter: 'zero-raised-to-zero-power' + output-parameter: "zero-raised-to-zero-power" exponent-zero-to-one: method: Exponent - path: 'builtin' - global-config: - input-parameter: 'zero-value' + path: "builtin" + config: + input-parameter: "zero-value" exponent: 1 - output-parameter: 'zero-raised-to-first-power' + output-parameter: "zero-raised-to-first-power" "sci": path: "builtin" method: Sci - global-config: + config: functional-unit: "zero-value" tree: children: diff --git a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml index ed4c00385..d45f433e0 100644 --- a/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml +++ b/manifests/outputs/bugs/aggregation-error-wrong-metric.yaml @@ -12,14 +12,14 @@ initialize: interpolate: method: Interpolation path: builtin - global-config: + config: method: linear x: - 0 - 10 - 50 - 100 - "y": + 'y': - 0.12 - 0.32 - 0.75 @@ -29,7 +29,7 @@ initialize: cpu-factor-to-wattage: method: Multiply path: builtin - global-config: + config: input-parameters: - cpu-factor - cpu/thermal-design-power @@ -37,7 +37,7 @@ initialize: wattage-times-duration: method: Multiply path: builtin - global-config: + config: input-parameters: - cpu-wattage - duration @@ -45,21 +45,21 @@ initialize: wattage-to-energy-kwh: method: Divide path: builtin - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw calculate-vcpu-ratio: method: Divide path: builtin - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio correct-cpu-energy-for-vcpu-ratio: method: Divide path: builtin - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -69,7 +69,7 @@ initialize: operational-carbon: method: Multiply path: builtin - global-config: + config: input-parameters: - cpu-energy-kwh - grid/carbon-intensity @@ -77,40 +77,39 @@ initialize: sci: path: builtin method: Sci - global-config: + config: functional-unit: requests time-sync: method: TimeSync path: builtin - global-config: - start-time: "2023-12-12T00:00:00.000Z" - end-time: "2023-12-12T00:01:00.000Z" + config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' interval: 5 allow-padding: true execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/bugs/aggregation-error-wrong-metric.yml -o - manifests/outputs/bugs/aggregation-error-wrong-metric + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/outputs/bugs/aggregation-error-wrong-metric.yaml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-01T19:25:34.759Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T08:38:25.343Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.25' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -130,9 +129,10 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- - MissingInputDataError: `functional-unit` value is missing from input data or it is not a positive integer + MissingAggregationParamError: Aggregation metric dummy-param is not found in + inputs[0]. tree: children: child-1: @@ -160,25 +160,25 @@ tree: vcpus-allocated: 1 vcpus-total: 8 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 10 requests: 100 - - timestamp: "2023-12-12T00:00:01.000Z" + - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: "2023-12-12T00:00:06.000Z" + - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: "2023-12-12T00:00:13.000Z" + - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -209,25 +209,25 @@ tree: vcpus-allocated: 1 vcpus-total: 8 inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 cpu/utilization: 30 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: "2023-12-12T00:00:01.000Z" + - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 cpu/utilization: 28 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: "2023-12-12T00:00:06.000Z" + - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 cpu/utilization: 40 cloud/instance-type: A1 cloud/region: uk-west requests: 100 - - timestamp: "2023-12-12T00:00:13.000Z" + - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 cpu/utilization: 33 cloud/instance-type: A1 diff --git a/manifests/outputs/bugs/input-error-missing-duration.yaml b/manifests/outputs/bugs/input-error-missing-duration.yaml index 1006c1c0c..3119c72d9 100644 --- a/manifests/outputs/bugs/input-error-missing-duration.yaml +++ b/manifests/outputs/bugs/input-error-missing-duration.yaml @@ -8,7 +8,7 @@ initialize: interpolate: method: Interpolation path: builtin - global-config: + config: method: linear x: - 0 diff --git a/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml b/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml index d137975b5..a3c760849 100644 --- a/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml +++ b/manifests/outputs/bugs/mock-observations-failure-duration-is-zero.yaml @@ -7,7 +7,7 @@ initialize: kind: plugin method: MockObservations path: builtin - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 0 @@ -69,7 +69,7 @@ execution: - typescript@5.2.2 - winston@3.11.0 - zod@3.22.4 - error: "InputValidationError: \"duration\" parameter is number must be greater than 0. Error code: too_small." + error: 'InputValidationError: "duration" parameter is number must be greater than 0. Error code: too_small.' tree: children: child: diff --git a/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml b/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml index fef0fa246..39227bd84 100644 --- a/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml +++ b/manifests/outputs/bugs/pipeline-error-naming-mismatch.yaml @@ -8,7 +8,7 @@ initialize: interpolate: method: Interpolation path: builtin - global-config: + config: method: linear x: - 0 diff --git a/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml b/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml index 7a0060013..bb47be31e 100644 --- a/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml +++ b/manifests/outputs/bugs/pipeline-error-uninitialized-plugin.yaml @@ -8,7 +8,7 @@ initialize: interpolate: method: Interpolation path: builtin - global-config: + config: method: linear x: - 0 diff --git a/manifests/outputs/bugs/pipeline-ordering-error.yaml b/manifests/outputs/bugs/pipeline-ordering-error.yaml index d1bd27d27..7df81f61a 100644 --- a/manifests/outputs/bugs/pipeline-ordering-error.yaml +++ b/manifests/outputs/bugs/pipeline-ordering-error.yaml @@ -8,7 +8,7 @@ initialize: interpolate: method: Interpolation path: builtin - global-config: + config: method: linear x: - 0 @@ -25,7 +25,7 @@ initialize: cpu-factor-to-wattage: method: Multiply path: builtin - global-config: + config: input-parameters: - cpu-factor - cpu/thermal-design-power @@ -33,7 +33,7 @@ initialize: wattage-times-duration: method: Multiply path: builtin - global-config: + config: input-parameters: - cpu-wattage - duration @@ -41,21 +41,21 @@ initialize: wattage-to-energy-kwh: method: Divide path: builtin - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw calculate-vcpu-ratio: method: Divide path: builtin - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio correct-cpu-energy-for-vcpu-ratio: method: Divide path: builtin - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh diff --git a/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml b/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml deleted file mode 100644 index 486947ebd..000000000 --- a/manifests/outputs/bugs/sci-embodied-missing-resources-total.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: sci-embodied -description: >- - receiving incorrect error message when running sci-embodied without - `resources-total` issue -tags: null -initialize: - plugins: - sci-embodied: - method: SciEmbodied - path: builtin -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/bugs/sci-embodied-missing-resources-total.yml -o - manifests/outputs/bugs/sci-embodied-missing-resources-total - environment: - if-version: 0.4.0 - os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-01T20:17:30.390Z (UTC) - dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - InputValidationError: "vcpus-allocated" parameter is required. Error code: - invalid_union. -tree: - children: - child: - pipeline: - compute: - - sci-embodied - defaults: - device/emissions-embodied: 1533.12 - time-reserved: 3600 - device/expected-lifespan: 94608000 - resources-reserved: 1 - inputs: - - timestamp: 2023-07-06T00:00 - duration: 3600 diff --git a/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml b/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml index 7b6051dce..e8239c914 100644 --- a/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml +++ b/manifests/outputs/builtins/coefficient/failure-invalid-config-input-param.yaml @@ -1,12 +1,12 @@ name: coefficient-demo -description: failure with ivalid `global-config.input-parameter` +description: failure with ivalid `config.input-parameter` tags: null initialize: plugins: coefficient: method: Coefficient path: builtin - global-config: + config: input-parameter: 4 coefficient: 3 output-parameter: carbon-product diff --git a/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml b/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml index 847e984e7..9482011f8 100644 --- a/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml +++ b/manifests/outputs/builtins/coefficient/failure-output-param-is-null.yaml @@ -6,7 +6,7 @@ initialize: coefficient: method: Coefficient path: builtin - global-config: + config: input-parameter: carbon coefficient: 3 output-parameter: null diff --git a/manifests/outputs/builtins/coefficient/success.yaml b/manifests/outputs/builtins/coefficient/success.yaml index 14a235709..d34222918 100644 --- a/manifests/outputs/builtins/coefficient/success.yaml +++ b/manifests/outputs/builtins/coefficient/success.yaml @@ -6,33 +6,33 @@ initialize: coefficient: path: builtin method: Coefficient - global-config: + config: input-parameter: carbon coefficient: 3 output-parameter: carbon-product execution: command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/coefficient/success.yml -o - manifests/outputs/plugins/coefficient/success + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/coefficient/success.yml -o + manifests/outputs/builtins/coefficient/success.yaml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T05:37:08.297Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-11T11:37:21.738Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -52,13 +52,14 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 status: success tree: children: child: pipeline: - - coefficient + compute: + - coefficient inputs: - timestamp: 2023-08-06T00:00 duration: 3600 @@ -67,4 +68,4 @@ tree: - timestamp: 2023-08-06T00:00 duration: 3600 carbon: 30 - carbon-product: 90 + carbon-product: 90 \ No newline at end of file diff --git a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml index 6945e6012..e8cc6170e 100644 --- a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-instance-type.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: diff --git a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml new file mode 100644 index 000000000..aa83f102b --- /dev/null +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml @@ -0,0 +1,76 @@ +name: cloud-metadata +description: failure with invalid `inputs.cloud/vendor` +tags: null +initialize: + plugins: + cloud-metadata: + path: builtin + method: CSVLookup + config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/vendor + output: + - cpu-cores-utilized + - cloud/instance-type +execution: + status: fail + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml + -o + manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-invalid-vendor.yaml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T07:42:52.156Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + error: >- + QueryDataNotFoundError: One or more of the given query parameters are not + found in the target CSV file column headers. +tree: + children: + child: + pipeline: + compute: + - cloud-metadata + inputs: + - timestamp: 2023-07-06T00:00 + cloud/vendor: gcp + cloud/instance-type: m5n.large + duration: 100 + cpu/utilization: 10 diff --git a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml index 333d9e1ce..0cbfb31db 100644 --- a/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml @@ -6,39 +6,39 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: - instance-class: cloud/instance-type + instance-class: cloud/vendor output: - cpu-cores-utilized - vcpus-allocated execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yml -o - manifests/outputs/plugins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor1 + manifests/outputs/builtins/csv-lookup/cloud-metadata/failure-missing-cloud-vendor.yaml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-05T09:07:35.386Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T08:24:01.971Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -58,7 +58,7 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- QueryDataNotFoundError: One or more of the given query parameters are not found in the target CSV file column headers. @@ -70,5 +70,6 @@ tree: - cloud-metadata inputs: - timestamp: 2023-07-06T00:00 + cloud/instance-type: m5n.large duration: 100 cpu/utilization: 10 diff --git a/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml b/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml index 37fab5816..a4e5cdf21 100644 --- a/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/cloud-metadata/success.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: diff --git a/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml index 842080a77..e1128e36b 100644 --- a/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-column.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: method: CSVLookup path: builtin - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: diff --git a/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml index 32ca690d9..92c9b4c79 100644 --- a/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml @@ -6,37 +6,38 @@ initialize: cloud-metadata: method: CSVLookup path: builtin - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: cloud-provider: cloud/provider cloud-region: cloud/region + output: null execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-output.yml + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/csv-lookup/region-metadata/failure-missing-output.yml -o - manifests/outputs/plugins/csv-lookup/region-metadata/failure-missing-output + manifests/outputs/builtins/csv-lookup/region-metadata/failure-missing-output.yaml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T21:26:09.874Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T08:47:18.608Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -56,7 +57,7 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- InputValidationError: "output" parameter is invalid input. Error code: invalid_union. diff --git a/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml index 2f6f23dcf..7fc116809 100644 --- a/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/success-renaming.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: diff --git a/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml b/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml index f8bd9fdba..bb099e8ee 100644 --- a/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/region-metadata/success.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: diff --git a/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml index ce268cd02..1b97a0c30 100644 --- a/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-missing-input-param.yaml @@ -6,7 +6,7 @@ initialize: tdp-finder: method: CSVLookup path: builtin - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv query: diff --git a/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml index 28dc0b1a0..aec123ef4 100644 --- a/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/failure-unsupported-physical-processor.yaml @@ -6,7 +6,7 @@ initialize: tdp-finder: method: CSVLookup path: builtin - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv query: diff --git a/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml b/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml index f085664ff..1d8dd9797 100644 --- a/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml +++ b/manifests/outputs/builtins/csv-lookup/tdp-finder/success.yaml @@ -6,7 +6,7 @@ initialize: tdp-finder: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/tdp-data-1.csv query: diff --git a/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml b/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml index b1149c505..3709451e3 100644 --- a/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml +++ b/manifests/outputs/builtins/divide/failure-invalid-config-denominator.yaml @@ -1,39 +1,50 @@ name: divide -description: failure when `global-config.denominator` is string +description: failure when `config.denominator` is string tags: null initialize: plugins: + cloud-metadata: + path: builtin + method: CSVLookup + config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: + - cpu-cores-utilized + - vcpus-allocated divide: method: Divide path: builtin - global-config: - numerator: cpu/utilization - denominator: test - output: cpu/divided-two + config: + numerator: vcpus-allocated + denominator: vcpus + output: cpu/number-cores execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/divide/failure-invalid-config-denominator.yml -o - manifests/outputs/plugins/divide/failure-invalid-config-denominator.yml + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/divide/failure-invalid-config-denominator.yml -o + manifests/outputs/builtins/divide/failure-invalid-config-denominator.yml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T06:02:25.409Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T06:15:13.478Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -53,16 +64,21 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- - MissingInputDataError: test is missing from the input array, or has nullish + MissingInputDataError: vcpus is missing from the input array, or has nullish value. tree: children: child: pipeline: compute: + - cloud-metadata - divide + defaults: + cloud/vendor: aws + cloud/instance-type: m5n.large + cpu/name: Intel® Core™ i7-1185G7 inputs: - timestamp: 2023-08-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/divide/failure-missing-numerator.yaml b/manifests/outputs/builtins/divide/failure-missing-numerator.yaml index 6c714bf18..2a34a9513 100644 --- a/manifests/outputs/builtins/divide/failure-missing-numerator.yaml +++ b/manifests/outputs/builtins/divide/failure-missing-numerator.yaml @@ -3,36 +3,47 @@ description: success path tags: null initialize: plugins: + cloud-metadata: + path: builtin + method: CSVLookup + config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv + query: + instance-class: cloud/instance-type + output: + - cpu-cores-utilized + - vcpus-allocated divide: method: Divide path: builtin - global-config: + config: denominator: 2 output: cpu/number-cores execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/divide/failure-missing-numerator.yml -o - manifests/outputs/plugins/divide/failure-missing-numerator + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/divide/failure-missing-numerator.yml -o + manifests/outputs/builtins/divide/failure-missing-numerator.yml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T05:49:51.802Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T06:15:22.702Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -52,7 +63,7 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- InputValidationError: "numerator" parameter is required. Error code: invalid_type. @@ -61,9 +72,8 @@ tree: child: pipeline: compute: + - cloud-metadata - divide - config: - divide: null defaults: cloud/vendor: aws cloud/instance-type: m5n.large @@ -72,4 +82,3 @@ tree: - timestamp: 2023-08-06T00:00 duration: 3600 cpu/utilization: 80 - vcpus-allocated: 8 diff --git a/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml b/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml index 25033c32c..940b042e1 100644 --- a/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml +++ b/manifests/outputs/builtins/divide/success-denominator-equal-zero.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -17,7 +17,7 @@ initialize: divide: path: builtin method: Divide - global-config: + config: numerator: vcpus-allocated denominator: 0 output: cpu/number-cores diff --git a/manifests/outputs/builtins/divide/success.yaml b/manifests/outputs/builtins/divide/success.yaml index f7bab8b04..625e81d4e 100644 --- a/manifests/outputs/builtins/divide/success.yaml +++ b/manifests/outputs/builtins/divide/success.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -17,7 +17,7 @@ initialize: divide: path: builtin method: Divide - global-config: + config: numerator: vcpus-allocated denominator: 2 output: cpu/number-cores diff --git a/manifests/outputs/builtins/exponent/success.yaml b/manifests/outputs/builtins/exponent/success.yaml index 8586b57f6..04ac4e6de 100644 --- a/manifests/outputs/builtins/exponent/success.yaml +++ b/manifests/outputs/builtins/exponent/success.yaml @@ -6,7 +6,7 @@ initialize: exponent: path: builtin method: Exponent - global-config: + config: input-parameter: cpu/energy exponent: 2 output-parameter: energy diff --git a/manifests/outputs/builtins/interpolation/interpolation.yaml b/manifests/outputs/builtins/interpolation/interpolation.yaml index 2569b5eb4..a67ab0a1e 100644 --- a/manifests/outputs/builtins/interpolation/interpolation.yaml +++ b/manifests/outputs/builtins/interpolation/interpolation.yaml @@ -6,7 +6,7 @@ initialize: interpolation: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 diff --git a/manifests/outputs/builtins/interpolation/success.yaml b/manifests/outputs/builtins/interpolation/success.yaml index 80def13d9..5eb41320d 100644 --- a/manifests/outputs/builtins/interpolation/success.yaml +++ b/manifests/outputs/builtins/interpolation/success.yaml @@ -6,7 +6,7 @@ initialize: interpolation: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 diff --git a/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml b/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml index 272d9940e..f335e47b7 100644 --- a/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-invalid-config-cpu-range.yaml @@ -1,6 +1,6 @@ name: mock-observation-demo description: >- - failure with `global-config->generators->randint->cpu/utilization->min` is + failure with `config->generators->randint->cpu/utilization->min` is greater than `max` tags: null initialize: @@ -9,7 +9,7 @@ initialize: kind: plugin method: MockObservations path: builtin - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 60 @@ -73,7 +73,7 @@ execution: - winston@3.11.0 - zod@3.22.4 error: >- - GlobalConfigError: Min value should not be greater than or equal to max + ConfigError: Min value should not be greater than or equal to max value of cpu/utilization tree: children: diff --git a/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml b/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml index e7c7e6180..90a3e7429 100644 --- a/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-invalid-memory-utilization-range.yaml @@ -7,7 +7,7 @@ initialize: kind: plugin method: MockObservations path: builtin - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 60 @@ -71,7 +71,7 @@ execution: - winston@3.11.0 - zod@3.22.4 error: >- - GlobalConfigError: Min value should not be greater than or equal to max + ConfigError: Min value should not be greater than or equal to max value of memory/utilization tree: children: diff --git a/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml b/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml index 86f4afd9e..733ac0a97 100644 --- a/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml +++ b/manifests/outputs/builtins/mock-observations/failure-missing-timestamp-from-param.yaml @@ -7,7 +7,7 @@ initialize: kind: plugin method: MockObservations path: builtin - global-config: + config: timestamp-to: 2023-07-06T00:10 duration: 60 components: diff --git a/manifests/outputs/builtins/mock-observations/success.yaml b/manifests/outputs/builtins/mock-observations/success.yaml index bcc5ee034..f099a5cb5 100644 --- a/manifests/outputs/builtins/mock-observations/success.yaml +++ b/manifests/outputs/builtins/mock-observations/success.yaml @@ -6,7 +6,7 @@ initialize: mock-observations: path: builtin method: MockObservations - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 60 @@ -26,22 +26,22 @@ initialize: max: 99 execution: command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m manifests/examples/builtins/mock-observations/success.yml -o - manifests/outputs/builtins/mock-observations/success + manifests/outputs/builtins/mock-observations/success.yaml environment: - if-version: 0.5.0 + if-version: 0.6.0 os: macOS - os-version: '14.5' - node-version: 18.14.2 - date-time: 2024-08-02T15:04:18.262Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T10:54:25.979Z (UTC) dependencies: - '@babel/core@7.22.10' - '@babel/preset-typescript@7.23.3' - '@commitlint/cli@18.6.0' - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.16' + - '@grnsft/if-core@0.0.22' - '@jest/globals@29.7.0' - '@types/jest@29.5.8' - '@types/js-yaml@4.0.9' @@ -80,279 +80,138 @@ tree: cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:01:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:02:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:03:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:04:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:05:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:06:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:07:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:08:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:09:00.000Z' duration: 60 cloud/instance-type: A1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:00:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:01:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:02:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:03:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:04:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:05:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:06:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:07:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:08:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" - timestamp: '2023-07-06T00:09:00.000Z' duration: 60 cloud/instance-type: B1 region: uk-west common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - outputs: - - timestamp: '2023-07-06T00:00:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:01:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:02:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:03:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:04:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:05:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:06:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:07:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:08:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:09:00.000Z' - duration: 60 - cloud/instance-type: A1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:00:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:01:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:02:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:03:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:04:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:05:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:06:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:07:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:08:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' - - timestamp: '2023-07-06T00:09:00.000Z' - duration: 60 - cloud/instance-type: B1 - region: uk-west - common-key: common-val - cpu/utilization: '*' - memory/utilization: '*' + cpu/utilization: "*" + memory/utilization: "*" diff --git a/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml b/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml index 80738e80c..04388331f 100644 --- a/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml +++ b/manifests/outputs/builtins/multiply/failure-input-parameter-is-missing.yaml @@ -6,7 +6,7 @@ initialize: multiply: method: Multiply path: builtin - global-config: + config: input-parameters: - cpu/energy - network/energy diff --git a/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml b/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml index 359e25a06..60daf4c76 100644 --- a/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml +++ b/manifests/outputs/builtins/multiply/success-with-multiple-inputs.yaml @@ -6,7 +6,7 @@ initialize: multiply: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu/energy - network/energy diff --git a/manifests/outputs/builtins/multiply/success.yaml b/manifests/outputs/builtins/multiply/success.yaml index 1f93a9140..62b8e4be4 100644 --- a/manifests/outputs/builtins/multiply/success.yaml +++ b/manifests/outputs/builtins/multiply/success.yaml @@ -6,7 +6,7 @@ initialize: multiply: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu/energy - network/energy diff --git a/manifests/outputs/builtins/regex/failure-missing-input-param.yaml b/manifests/outputs/builtins/regex/failure-missing-input-param.yaml index cd578218f..44505edde 100644 --- a/manifests/outputs/builtins/regex/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/regex/failure-missing-input-param.yaml @@ -6,7 +6,7 @@ initialize: regex: method: Regex path: builtin - global-config: + config: parameter: physical-processor match: ^(.*), output: cpu/name diff --git a/manifests/outputs/builtins/regex/failure-not-matching-with-regex.yaml b/manifests/outputs/builtins/regex/failure-not-matching-with-regex.yaml new file mode 100644 index 000000000..e18b408b1 --- /dev/null +++ b/manifests/outputs/builtins/regex/failure-not-matching-with-regex.yaml @@ -0,0 +1,72 @@ +name: regex +description: physical processor doesn't match the regex expression +tags: null +initialize: + plugins: + regex: + method: Regex + path: builtin + config: + parameter: physical-processor + match: ^$ + output: cpu/name +execution: + status: fail + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/regex/failure-not-matching-with-regex.yml -o + manifests/outputs/builtins/regex/failure-not-matching-with-regex.yaml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T11:17:40.549Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + error: >- + RegexMismatchError: `Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 + GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz` does + not match the /^$/ regex expression +tree: + children: + child: + pipeline: + compute: + - regex + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + physical-processor: >- + Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® + E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz diff --git a/manifests/outputs/builtins/regex/success.yaml b/manifests/outputs/builtins/regex/success.yaml index 22988eead..c4f0838bd 100644 --- a/manifests/outputs/builtins/regex/success.yaml +++ b/manifests/outputs/builtins/regex/success.yaml @@ -6,7 +6,7 @@ initialize: regex: path: builtin method: Regex - global-config: + config: parameter: physical-processor match: ^(.*), output: cpu/name diff --git a/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml b/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml index f6af79041..4a11178b3 100644 --- a/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml +++ b/manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml @@ -1,7 +1,6 @@ name: sci-embodied description: >- - failure with `defaults.device/emissions-embodied` being string instead of - number + failure with `vCPUs` being string instead of number tags: null initialize: plugins: @@ -11,28 +10,26 @@ initialize: execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/sci-embodied/failure-invalid-default-emission-value.yml - -o - manifests/outputs/plugins/sci-embodied/failure-invalid-default-emission-value + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/outputs/builtins/sci-embodied/failure-invalid-default-emission-value.yaml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T20:49:08.280Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T08:59:52.608Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.25' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -52,10 +49,10 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 error: >- - InputValidationError: "device/emissions-embodied" parameter is invalid - number. please provide it as `gco2e` to input. Error code: invalid_union. + InputValidationError: "vCPUs" parameter is expected number, received string + at index 0. Error code: invalid_type. tree: children: child: @@ -63,9 +60,8 @@ tree: compute: - sci-embodied defaults: - device/emissions-embodied: fail + vCPUs: fail time-reserved: 3600 - device/expected-lifespan: 94608000 resources-reserved: 1 resources-total: 8 inputs: diff --git a/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml b/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml deleted file mode 100644 index 43f22163e..000000000 --- a/manifests/outputs/builtins/sci-embodied/failure-missing-expected-lifespan.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: sci-embodied -description: missing device/expected-lifespan -tags: null -initialize: - plugins: - sci-embodied: - method: SciEmbodied - path: builtin -execution: - status: fail - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/sci-embodied/failure-missing-expected-lifespan.yml - -o manifests/outputs/plugins/sci-embodied/failure-missing-expected-lifespan - environment: - if-version: 0.4.0 - os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T20:42:51.951Z (UTC) - dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - error: >- - InputValidationError: "device/expected-lifespan" parameter is required. - Error code: invalid_union. -tree: - children: - child: - pipeline: - compute: - - sci-embodied - defaults: - device/emissions-embodied: 1533.12 - time-reserved: 3600 - resources-reserved: 1 - resources-total: 8 - inputs: - - timestamp: 2023-07-06T00:00 - duration: 3600 diff --git a/manifests/outputs/builtins/sci-embodied/scenario-1.yaml b/manifests/outputs/builtins/sci-embodied/scenario-1.yaml new file mode 100644 index 000000000..878c88fe3 --- /dev/null +++ b/manifests/outputs/builtins/sci-embodied/scenario-1.yaml @@ -0,0 +1,100 @@ +name: embodied-carbon demo +description: null +tags: null +aggregation: + metrics: + - embodied-carbon + type: both +initialize: + plugins: + embodied-carbon: + path: builtin + method: SciEmbodied + config: + output-parameter: embodied-carbon +execution: + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/outputs/builtins/sci-embodied/scenario-1.yaml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T09:06:05.353Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.25' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child: + pipeline: + compute: + - embodied-carbon + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + hdd: 2 + - timestamp: 2023-08-06T10:00 + duration: 3600 + hdd: 2 + outputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + hdd: 2 + vCPUs: 1 + memory: 16 + ssd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 34.24657534246575 + - timestamp: 2023-08-06T10:00 + duration: 3600 + hdd: 2 + vCPUs: 1 + memory: 16 + ssd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 34.24657534246575 + aggregated: + embodied-carbon: 68.4931506849315 + outputs: + - embodied-carbon: 34.24657534246575 + timestamp: 2023-08-06T00:00 + duration: 3600 + - embodied-carbon: 34.24657534246575 + timestamp: 2023-08-06T10:00 + duration: 3600 + aggregated: + embodied-carbon: 68.4931506849315 diff --git a/manifests/outputs/builtins/sci-embodied/scenario-2.yaml b/manifests/outputs/builtins/sci-embodied/scenario-2.yaml new file mode 100644 index 000000000..a6d3f7eeb --- /dev/null +++ b/manifests/outputs/builtins/sci-embodied/scenario-2.yaml @@ -0,0 +1,101 @@ +name: embodied-carbon demo +description: null +tags: null +initialize: + plugins: + embodied-carbon: + path: builtin + method: SciEmbodied + config: + baseline-vcpus: 1 + baseline-memory: 16 + lifespan: 157680000 + baseline-emissions: 2000000 + vcpu-emissions-constant: 100000 + memory-emissions-constant: 1172 + ssd-emissions-constant: 50000 + hdd-emissions-constant: 100000 + gpu-emissions-constant: 150000 + output-parameter: embodied-carbon +execution: + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/outputs/builtins/sci-embodied/scenario-2.yaml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T09:08:03.615Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.25' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child: + pipeline: + compute: + - embodied-carbon + defaults: + vCPUs: 4 + memory: 32 + ssd: 1 + hdd: 1 + gpu: 1 + total-vcpus: 16 + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + - timestamp: 2023-08-06T10:00 + duration: 3600 + outputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + vCPUs: 4 + memory: 32 + ssd: 1 + hdd: 1 + gpu: 1 + total-vcpus: 16 + usage-ratio: 1 + embodied-carbon: 487.48858447488584 + - timestamp: 2023-08-06T10:00 + duration: 3600 + vCPUs: 4 + memory: 32 + ssd: 1 + hdd: 1 + gpu: 1 + total-vcpus: 16 + usage-ratio: 1 + embodied-carbon: 487.48858447488584 diff --git a/manifests/outputs/builtins/sci-embodied/success.yaml b/manifests/outputs/builtins/sci-embodied/success.yaml index e91ab9212..023cf59ea 100644 --- a/manifests/outputs/builtins/sci-embodied/success.yaml +++ b/manifests/outputs/builtins/sci-embodied/success.yaml @@ -3,32 +3,42 @@ description: successful path tags: null initialize: plugins: + csv-lookup: + path: builtin + method: CSVLookup + config: + filepath: >- + https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv + query: + instance-class: cloud/instance-type + output: + - cpu-cores-utilized + - vcpus-allocated sci-embodied: path: builtin method: SciEmbodied execution: command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/sci-embodied/success.yml -o - manifests/outputs/plugins/sci-embodied/success + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/outputs/builtins/sci-embodied/success.yaml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T20:42:03.186Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T09:08:28.940Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.25' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -48,13 +58,14 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 status: success tree: children: child: pipeline: compute: + - csv-lookup - sci-embodied defaults: device/emissions-embodied: 1533.12 @@ -65,12 +76,25 @@ tree: inputs: - timestamp: 2023-07-06T00:00 duration: 3600 + cloud/vendor: intel + cloud/instance-type: Standard_A1_v2 + cpu/utilization: 10 outputs: - timestamp: 2023-07-06T00:00 duration: 3600 + cloud/vendor: intel + cloud/instance-type: Standard_A1_v2 + cpu/utilization: 10 device/emissions-embodied: 1533.12 time-reserved: 3600 device/expected-lifespan: 94608000 resources-reserved: 1 resources-total: 8 - carbon-embodied: 0.007292237442922374 + vcpus-allocated: 1 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 28.538812785388128 diff --git a/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml b/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml index 384fe14b1..8d46b1964 100644 --- a/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml +++ b/manifests/outputs/builtins/sci/failure-invalid-config-value.yaml @@ -10,27 +10,27 @@ initialize: execution: status: fail command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/sci/failure-invalid-config-value.yml -o - manifests/outputs/plugins/sci/failure-invalid-config-value + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/sci/failure-invalid-config-value.yml -o + manifests/outputs/builtins/sci/failure-invalid-config-value.yml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T20:38:15.858Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T06:15:31.434Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -50,17 +50,16 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 - error: "InputValidationError: Required" + - zod@3.23.8 + error: 'ConfigError: Config is not provided.' tree: children: child: pipeline: compute: - sci - config: - sci: - functional-unit: 999 + defaults: + functional-unit: 999 inputs: - timestamp: 2023-07-06T00:00 duration: 3600 diff --git a/manifests/outputs/builtins/sci/failure-missing-input-param.yaml b/manifests/outputs/builtins/sci/failure-missing-input-param.yaml index b0fbce2f8..08c482991 100644 --- a/manifests/outputs/builtins/sci/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/sci/failure-missing-input-param.yaml @@ -7,7 +7,7 @@ initialize: kind: plugin method: Sci path: builtin - global-config: + config: functional-unit: requests execution: status: fail diff --git a/manifests/outputs/builtins/sci/success.yaml b/manifests/outputs/builtins/sci/success.yaml index 9f36fc9bd..1f384be28 100644 --- a/manifests/outputs/builtins/sci/success.yaml +++ b/manifests/outputs/builtins/sci/success.yaml @@ -6,7 +6,7 @@ initialize: sci: path: builtin method: Sci - global-config: + config: functional-unit: requests execution: command: >- diff --git a/manifests/outputs/builtins/shell/failure-invalid-command.yaml b/manifests/outputs/builtins/shell/failure-invalid-command.yaml index 97d8bb6af..9c1c3b84b 100644 --- a/manifests/outputs/builtins/shell/failure-invalid-command.yaml +++ b/manifests/outputs/builtins/shell/failure-invalid-command.yaml @@ -1,12 +1,12 @@ name: shell -description: falure with `global-config.command` being number instead od string +description: falure with `config.command` being number instead od string tags: null initialize: plugins: shell: method: Shell path: builtin - global-config: + config: command: 1000 execution: status: fail diff --git a/manifests/outputs/builtins/shell/success.yaml b/manifests/outputs/builtins/shell/success.yaml index b85cbfb95..cd7a41989 100644 --- a/manifests/outputs/builtins/shell/success.yaml +++ b/manifests/outputs/builtins/shell/success.yaml @@ -6,7 +6,7 @@ initialize: shell: path: builtin method: Shell - global-config: + config: command: python3 /usr/local/bin/sampler execution: command: >- diff --git a/manifests/outputs/builtins/subtract/success.yaml b/manifests/outputs/builtins/subtract/success.yaml index 5eb0a7bd3..bcc9c1ae3 100644 --- a/manifests/outputs/builtins/subtract/success.yaml +++ b/manifests/outputs/builtins/subtract/success.yaml @@ -6,7 +6,7 @@ initialize: subtract: path: builtin method: Subtract - global-config: + config: input-parameters: - cpu/energy - network/energy diff --git a/manifests/outputs/builtins/sum/failure-missing-input-param.yaml b/manifests/outputs/builtins/sum/failure-missing-input-param.yaml index e2aaf8158..3eb324c4a 100644 --- a/manifests/outputs/builtins/sum/failure-missing-input-param.yaml +++ b/manifests/outputs/builtins/sum/failure-missing-input-param.yaml @@ -1,12 +1,12 @@ name: sum -description: failure with `inputs[0]` misses one of `global-config.input-parameters` +description: failure with `inputs[0]` misses one of `config.input-parameters` tags: null initialize: plugins: sum: method: Sum path: builtin - global-config: + config: input-parameters: - cpu/energy - network/energy diff --git a/manifests/outputs/builtins/sum/failure-missing-output-param.yaml b/manifests/outputs/builtins/sum/failure-missing-output-param.yaml index d7a4d9302..f72ef14f8 100644 --- a/manifests/outputs/builtins/sum/failure-missing-output-param.yaml +++ b/manifests/outputs/builtins/sum/failure-missing-output-param.yaml @@ -1,12 +1,12 @@ name: sum -description: missing `output-parameter` in global-config +description: missing `output-parameter` in config tags: null initialize: plugins: sum: method: Sum path: builtin - global-config: + config: input-parameters: - cpu/energy - network/energy diff --git a/manifests/outputs/builtins/sum/success.yaml b/manifests/outputs/builtins/sum/success.yaml index 6b5b4d973..e044ddb4b 100644 --- a/manifests/outputs/builtins/sum/success.yaml +++ b/manifests/outputs/builtins/sum/success.yaml @@ -6,7 +6,7 @@ initialize: sum: path: builtin method: Sum - global-config: + config: input-parameters: - cpu/energy - network/energy diff --git a/manifests/outputs/builtins/time-converter/success.yaml b/manifests/outputs/builtins/time-converter/success.yaml new file mode 100644 index 000000000..012fdbbbb --- /dev/null +++ b/manifests/outputs/builtins/time-converter/success.yaml @@ -0,0 +1,73 @@ +name: time-converter demo +description: successful path +tags: null +initialize: + plugins: + time-converter: + path: builtin + method: TimeConverter + config: + input-parameter: energy-per-year + original-time-unit: year + new-time-unit: duration + output-parameter: energy-per-duration +execution: + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/time-converter/success.yaml -o + manifests/outputs/builtins/time-converter/success.yaml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T06:14:08.350Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child: + pipeline: + compute: + - time-converter + defaults: + energy-per-year: 10000 + inputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + outputs: + - timestamp: 2023-08-06T00:00 + duration: 3600 + energy-per-year: 10000 + energy-per-duration: 1.140795 diff --git a/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml b/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml index bb5ae79fb..e0c633680 100644 --- a/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml +++ b/manifests/outputs/builtins/time-sync/failure-config-start-later-end.yaml @@ -1,14 +1,14 @@ name: time-sync description: >- - failure with `global-config.start-time` being later than - `global-config.end-time` + failure with `config.start-time` being later than + `config.end-time` tags: null initialize: plugins: time-sync: method: TimeSync path: builtin - global-config: + config: start-time: "2023-12-12T00:01:00.000Z" end-time: "2023-12-12T00:00:00.000Z" interval: 5 diff --git a/manifests/outputs/builtins/time-sync/failure-missing-config.yaml b/manifests/outputs/builtins/time-sync/failure-missing-config.yaml new file mode 100644 index 000000000..58339a417 --- /dev/null +++ b/manifests/outputs/builtins/time-sync/failure-missing-config.yaml @@ -0,0 +1,74 @@ +name: time-sync +description: missing config +tags: null +initialize: + output: + - yaml + plugins: + time-sync: + method: TimeSync + path: builtin +execution: + status: fail + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/time-sync/failure-missing-config.yml -o + manifests/outputs/builtins/time-sync/failure-missing-config.yaml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T13:07:11.870Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.25' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + error: 'ConfigError: Config is not provided.' +tree: + children: + child: + pipeline: + compute: + - time-sync + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + duration: 3 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:01.000Z' + duration: 5 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:06.000Z' + duration: 7 + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:13.000Z' + duration: 30 + energy-cpu: 0.001 diff --git a/manifests/outputs/builtins/time-sync/success.yaml b/manifests/outputs/builtins/time-sync/success.yaml index 7b1d6bbb8..b49acb084 100644 --- a/manifests/outputs/builtins/time-sync/success.yaml +++ b/manifests/outputs/builtins/time-sync/success.yaml @@ -6,40 +6,34 @@ initialize: time-sync: path: builtin method: TimeSync - global-config: - start-time: "2023-12-12T00:00:00.000Z" - end-time: "2023-12-12T00:01:00.000Z" + config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' interval: 5 allow-padding: true - parameter-metadata: - outputs: - energy-cpu: - unit: KWH - description: energy - aggregation-method: sum execution: command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/plugins/time-sync/success.yml -o - manifests/outputs/plugins/time-sync/success -s + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/builtins/time-sync/success.yml -o + manifests/outputs/builtins/time-sync/success.yml environment: - if-version: 0.4.0 + if-version: 0.6.0 os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T21:12:32.629Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T06:16:13.676Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -59,7 +53,7 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 status: success tree: children: @@ -68,55 +62,52 @@ tree: compute: - time-sync inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' duration: 1 energy-cpu: 0.001 - - timestamp: "2023-12-12T00:00:01.000Z" + - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 energy-cpu: 0.001 - - timestamp: "2023-12-12T00:00:06.000Z" + - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 energy-cpu: 0.001 - - timestamp: "2023-12-12T00:00:13.000Z" + - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 energy-cpu: 0.001 outputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' duration: 5 - energy-cpu: 0.0018000000000000004 - - timestamp: "2023-12-12T00:00:05.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:05.000Z' duration: 5 - energy-cpu: 0.0007714285714285716 - - timestamp: "2023-12-12T00:00:10.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:10.000Z' duration: 5 - energy-cpu: 0.0004952380952380952 - - timestamp: "2023-12-12T00:00:15.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:15.000Z' duration: 5 - energy-cpu: 0.0001666666666666667 - - timestamp: "2023-12-12T00:00:20.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:20.000Z' duration: 5 - energy-cpu: 0.0001666666666666667 - - timestamp: "2023-12-12T00:00:25.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:25.000Z' duration: 5 - energy-cpu: 0.0001666666666666667 - - timestamp: "2023-12-12T00:00:30.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:30.000Z' duration: 5 - energy-cpu: 0.0001666666666666667 - - timestamp: "2023-12-12T00:00:35.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:35.000Z' duration: 5 - energy-cpu: 0.0001666666666666667 - - timestamp: "2023-12-12T00:00:40.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:40.000Z' duration: 5 - energy-cpu: 0.0001 - - timestamp: "2023-12-12T00:00:45.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:45.000Z' duration: 5 - energy-cpu: 0 - - timestamp: "2023-12-12T00:00:50.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:50.000Z' duration: 5 - energy-cpu: 0 - - timestamp: "2023-12-12T00:00:55.000Z" + energy-cpu: 0.001 + - timestamp: '2023-12-12T00:00:55.000Z' duration: 5 - energy-cpu: 0 - - timestamp: "2023-12-12T00:01:00.000Z" - duration: 1 - energy-cpu: 0 + energy-cpu: 0.001 diff --git a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml index aef8ef6c5..e369846ff 100644 --- a/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml +++ b/manifests/outputs/features/aggregate-failure-invalid-metrics.yaml @@ -9,7 +9,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: diff --git a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml index 2dfc9fe80..fc53a6859 100644 --- a/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml +++ b/manifests/outputs/features/aggregate-failure-missing-metric-in-inputs.yaml @@ -9,7 +9,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: diff --git a/manifests/outputs/features/aggregate-horizontal.yaml b/manifests/outputs/features/aggregate-horizontal.yaml index d595d5bdb..f08a645e3 100644 --- a/manifests/outputs/features/aggregate-horizontal.yaml +++ b/manifests/outputs/features/aggregate-horizontal.yaml @@ -9,7 +9,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -22,7 +22,9 @@ initialize: cpu/utilization: unit: percentage description: refers to CPU utilization. - aggregation-method: avg + aggregation-method: + time: avg + component: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/features/aggregate-vertical.yaml b/manifests/outputs/features/aggregate-vertical.yaml index 9213522af..e559d715a 100644 --- a/manifests/outputs/features/aggregate-vertical.yaml +++ b/manifests/outputs/features/aggregate-vertical.yaml @@ -9,7 +9,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -22,7 +22,9 @@ initialize: cpu/utilization: unit: percentage description: refers to CPU utilization. - aggregation-method: avg + aggregation-method: + time: avg + component: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/features/aggregate.yaml b/manifests/outputs/features/aggregate.yaml index cb9ed29dc..d00adde49 100644 --- a/manifests/outputs/features/aggregate.yaml +++ b/manifests/outputs/features/aggregate.yaml @@ -9,7 +9,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -22,7 +22,9 @@ initialize: cpu/utilization: unit: percentage description: refers to CPU utilization. - aggregation-method: avg + aggregation-method: + time: avg + component: avg execution: command: >- /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node diff --git a/manifests/outputs/features/regroup/failure-invalid-regroup.yaml b/manifests/outputs/features/regroup/failure-invalid-regroup.yaml new file mode 100644 index 000000000..2a40fe2fb --- /dev/null +++ b/manifests/outputs/features/regroup/failure-invalid-regroup.yaml @@ -0,0 +1,87 @@ +name: regroup +description: failure when `regroup` is not an array +initialize: + plugins: {} +execution: + status: fail + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/features/regroup/failure-invalid-regroup.yml -o + manifests/outputs/features/regroup/failure-invalid-regroup.yml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T06:13:09.837Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + error: >- + InputValidationError: "regroup" parameter is not an array or should contain + at least one key. Error code: invalid_type. +tree: + children: + my-app: + pipeline: + regroup: cloud/region + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 99 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 1 diff --git a/manifests/outputs/features/regroup/failure-missing-cloud-instance-type.yaml b/manifests/outputs/features/regroup/failure-missing-cloud-instance-type.yaml new file mode 100644 index 000000000..65fa5a113 --- /dev/null +++ b/manifests/outputs/features/regroup/failure-missing-cloud-instance-type.yaml @@ -0,0 +1,87 @@ +name: regroup +description: null +initialize: + plugins: {} +execution: + status: fail + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/features/regroup/failure-missing-cloud-instance-type.yml + -o + manifests/outputs/features/regroup/failure-missing-cloud-instance-type.yml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T06:13:14.590Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + error: 'InvalidGroupingError: Invalid group cloud/instance-type.' +tree: + children: + my-app: + pipeline: + regroup: + - cloud/region + - cloud/instance-type + inputs: + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 99 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 23 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/instance-type: A1 + cloud/region: uk-west + cpu/utilization: 12 + - timestamp: 2023-07-06T00:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 11 + - timestamp: 2023-07-06T05:00 + duration: 300 + cloud/instance-type: B1 + cloud/region: uk-west + cpu/utilization: 67 + - timestamp: 2023-07-06T10:00 + duration: 300 + cloud/region: uk-west + cpu/utilization: 1 diff --git a/manifests/outputs/pipelines/cloud-metadata-divide.yaml b/manifests/outputs/pipelines/cloud-metadata-divide.yaml index 02ff19b86..40afe668c 100644 --- a/manifests/outputs/pipelines/cloud-metadata-divide.yaml +++ b/manifests/outputs/pipelines/cloud-metadata-divide.yaml @@ -6,7 +6,7 @@ initialize: cloud-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv query: @@ -17,7 +17,7 @@ initialize: divide: path: builtin method: Divide - global-config: + config: numerator: vcpus-allocated denominator: 2 output: cpu/number-cores diff --git a/manifests/outputs/pipelines/generics.yaml b/manifests/outputs/pipelines/generics.yaml index 613a65f83..addba9510 100644 --- a/manifests/outputs/pipelines/generics.yaml +++ b/manifests/outputs/pipelines/generics.yaml @@ -8,14 +8,14 @@ initialize: interpolate: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -25,7 +25,7 @@ initialize: cpu-factor-to-wattage: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-factor - cpu/thermal-design-power @@ -33,7 +33,7 @@ initialize: wattage-times-duration: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-wattage - duration @@ -41,35 +41,35 @@ initialize: wattage-to-energy-kwh: path: builtin method: Divide - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw calculate-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio correct-cpu-energy-for-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh coefficient: path: builtin method: Coefficient - global-config: + config: input-parameter: cpu-energy-kwh coefficient: 2 output-parameter: energy-doubled multiply: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu/utilization - duration @@ -83,20 +83,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-17T20:30:54.004Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -136,28 +136,28 @@ tree: vcpus-allocated: 1 vcpus-total: 8 inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 10 energy: 5 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 10 energy: 5 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 10 energy: 5 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -165,7 +165,7 @@ tree: network/energy: 10 energy: 5 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" cloud/instance-type: A1 cloud/region: uk-west duration: 1 @@ -183,7 +183,7 @@ tree: cpu-energy-kwh: 0.0000026041666666666666 energy-doubled: 0.000005208333333333333 cpu-times-duration: 50 - - timestamp: '2023-12-12T00:00:01.000Z' + - timestamp: "2023-12-12T00:00:01.000Z" duration: 5 cpu/utilization: 20 cloud/instance-type: A1 @@ -201,7 +201,7 @@ tree: cpu-energy-kwh: 0.000007421875 energy-doubled: 0.00001484375 cpu-times-duration: 100 - - timestamp: '2023-12-12T00:00:06.000Z' + - timestamp: "2023-12-12T00:00:06.000Z" duration: 7 cpu/utilization: 15 cloud/instance-type: A1 @@ -219,7 +219,7 @@ tree: cpu-energy-kwh: 0.000009084201388888889 energy-doubled: 0.000018168402777777778 cpu-times-duration: 105 - - timestamp: '2023-12-12T00:00:13.000Z' + - timestamp: "2023-12-12T00:00:13.000Z" duration: 30 cloud/instance-type: A1 cloud/region: uk-west diff --git a/manifests/outputs/pipelines/instance-metadata.yaml b/manifests/outputs/pipelines/instance-metadata.yaml index 369d54c27..b38876e08 100644 --- a/manifests/outputs/pipelines/instance-metadata.yaml +++ b/manifests/outputs/pipelines/instance-metadata.yaml @@ -6,16 +6,16 @@ initialize: cloud-instance-metadata: path: builtin method: CSVLookup - global-config: + config: filepath: >- https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-azure-instances.csv query: instance-class: cloud/instance-type - output: '*' + output: "*" extract-processor-name: path: builtin method: Regex - global-config: + config: parameter: cpu-model-name match: /^([^,])+/g output: cpu/name @@ -28,20 +28,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-19T06:31:27.411Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 diff --git a/manifests/outputs/pipelines/mock-obs-time-sync.yaml b/manifests/outputs/pipelines/mock-obs-time-sync.yaml deleted file mode 100644 index 04eb881ec..000000000 --- a/manifests/outputs/pipelines/mock-obs-time-sync.yaml +++ /dev/null @@ -1,449 +0,0 @@ -name: Mock observation and time sync integration -description: Integration of `mock observation` + `time sync` -tags: null -initialize: - plugins: - mock-observations: - path: builtin - method: MockObservations - global-config: - timestamp-from: 2023-12-12T00:00 - timestamp-to: 2023-12-12T00:10 - duration: 60 - components: - - cloud/instance-type: A1 - generators: - common: - cloud/region: uk-west - randint: - cpu/utilization: - min: 1 - max: 99 - parameter-metadata: - inputs: - timestamp: - description: refers to the time of occurrence of the input - unit: RFC3339 - aggregation-method: none - duration: - description: refers to the duration of the input - unit: seconds - aggregation-method: sum - cloud/instance-type: - description: type of Cloud Instance name used in the cloud provider APIs - unit: none - aggregation-method: none - cloud/region: - description: region cloud instance - unit: none - aggregation-method: none - interpolate: - path: builtin - method: Interpolation - global-config: - method: linear - x: - - 0 - - 10 - - 50 - - 100 - "y": - - 0.12 - - 0.32 - - 0.75 - - 1.02 - input-parameter: cpu/utilization - output-parameter: cpu-factor - parameter-metadata: - inputs: - cpu/utilization: - description: refers to CPU utilization. - unit: percentage - aggregation-method: avg - cpu-factor-to-wattage: - path: builtin - method: Multiply - global-config: - input-parameters: - - cpu-factor - - cpu/thermal-design-power - output-parameter: cpu-wattage - parameter-metadata: - inputs: - cpu/thermal-design-power: - description: thermal design power for a processor - unit: kwh - aggregation-method: avg - wattage-times-duration: - path: builtin - method: Multiply - global-config: - input-parameters: - - cpu-wattage - - duration - output-parameter: cpu-wattage-times-duration - wattage-to-energy-kwh: - path: builtin - method: Divide - global-config: - numerator: cpu-wattage-times-duration - denominator: 3600000 - output: cpu-energy-raw - calculate-vcpu-ratio: - path: builtin - method: Divide - global-config: - numerator: vcpus-total - denominator: vcpus-allocated - output: vcpu-ratio - parameter-metadata: - inputs: - vcpus-total: - description: total number of vcpus available on a particular resource - unit: count - aggregation-method: none - vcpus-allocated: - description: number of vcpus allocated to particular resource - unit: count - aggregation-method: none - correct-cpu-energy-for-vcpu-ratio: - path: builtin - method: Divide - global-config: - numerator: cpu-energy-raw - denominator: vcpu-ratio - output: cpu-energy-kwh - time-sync: - path: builtin - method: TimeSync - global-config: - start-time: "2023-12-12T00:00:00.000Z" - end-time: "2023-12-12T00:01:00.000Z" - interval: 5 - allow-padding: true -execution: - command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/index.ts -m - manifests/outputs/pipelines/mock-obs-time-sync.yml -o - manifests/outputs/pipelines/mock-obs-time-sync - environment: - if-version: 0.4.0 - os: macOS - os-version: "13.2" - node-version: 18.14.2 - date-time: 2024-07-02T05:29:47.787Z (UTC) - dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.10" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" - - axios-mock-adapter@1.22.0 - - axios@1.7.2 - - cross-env@7.0.3 - - csv-parse@5.5.6 - - csv-stringify@6.4.6 - - fixpack@4.0.0 - - gts@5.2.0 - - husky@8.0.3 - - jest@29.7.0 - - js-yaml@4.1.0 - - lint-staged@15.2.2 - - luxon@3.4.4 - - release-it@16.3.0 - - rimraf@5.0.5 - - ts-command-line-args@2.5.1 - - ts-jest@29.1.1 - - typescript-cubic-spline@1.0.1 - - typescript@5.2.2 - - winston@3.11.0 - - zod@3.22.4 - status: success -tree: - children: - child-1: - pipeline: - observe: - - mock-observations - compute: - - interpolate - - cpu-factor-to-wattage - - wattage-times-duration - - wattage-to-energy-kwh - - calculate-vcpu-ratio - - correct-cpu-energy-for-vcpu-ratio - - time-sync - defaults: - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - inputs: - - timestamp: '2023-12-12T00:00:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:01:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:02:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:03:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:04:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:05:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:06:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:07:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:08:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - - timestamp: '2023-12-12T00:09:00.000Z' - cloud/instance-type: A1 - cloud/region: uk-west - duration: 60 - cpu/utilization: '*' - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - outputs: - - timestamp: "2023-12-12T00:00:00.000Z" - cloud/instance-type: A1 - cloud/region: uk-west - duration: 5 - cpu/utilization: '*' - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:05.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:10.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:15.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:20.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:25.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:30.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:35.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:40.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:45.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:50.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:00:55.000Z" - duration: 5 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 80 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' - - timestamp: "2023-12-12T00:01:00.000Z" - duration: 1 - cpu/utilization: '*' - cloud/instance-type: A1 - cloud/region: uk-west - cpu/thermal-design-power: 100 - vcpus-total: 8 - vcpus-allocated: 1 - cpu-factor: '*' - cpu-wattage: '*' - cpu-wattage-times-duration: '*' - cpu-energy-raw: '*' - vcpu-ratio: '*' - cpu-energy-kwh: '*' diff --git a/manifests/outputs/pipelines/nesting.yaml b/manifests/outputs/pipelines/nesting.yaml index 792f41243..85f69f515 100644 --- a/manifests/outputs/pipelines/nesting.yaml +++ b/manifests/outputs/pipelines/nesting.yaml @@ -13,14 +13,14 @@ initialize: interpolate: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 - 10 - 50 - 100 - "y": + 'y': - 0.12 - 0.32 - 0.75 @@ -32,16 +32,20 @@ initialize: cpu/utilization: unit: percentage description: refers to CPU utilization. - aggregation-method: avg + aggregation-method: + time: avg + component: sum outputs: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu-factor-to-wattage: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-factor - cpu/thermal-design-power @@ -51,20 +55,26 @@ initialize: cpu-factor: unit: kWh description: result of interpolate - aggregation-method: avg + aggregation-method: + time: avg + component: avg cpu/thermal-design-power: unit: kWh description: thermal design power for a processor - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: cpu-wattage: unit: kWh description: the energy used by the CPU - aggregation-method: sum + aggregation-method: + time: sum + component: sum wattage-times-duration: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-wattage - duration @@ -72,7 +82,7 @@ initialize: wattage-to-energy-kwh: path: builtin method: Divide - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw @@ -81,16 +91,20 @@ initialize: cpu-wattage-times-duration: unit: kWh description: CPU wattage multiplied by duration - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: cpu-energy-raw: unit: kWh description: Raw energy used by CPU in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum calculate-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio @@ -99,11 +113,13 @@ initialize: vcpu-ratio: unit: none description: Ratio of vCPUs - aggregation-method: none + aggregation-method: + time: copy + component: copy correct-cpu-energy-for-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -113,7 +129,7 @@ initialize: operational-carbon: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-energy-kwh - grid/carbon-intensity @@ -123,56 +139,70 @@ initialize: cpu-energy-kwh: unit: kWh description: Corrected CPU energy in kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum grid/carbon-intensity: unit: gCO2eq/kWh description: Carbon intensity for the grid - aggregation-method: avg + aggregation-method: + time: avg + component: avg outputs: carbon-operational: unit: gCO2eq description: Operational carbon footprint - aggregation-method: sum + aggregation-method: + time: sum + component: sum sci: path: builtin method: Sci - global-config: + config: functional-unit: requests parameter-metadata: inputs: requests: unit: none description: expressed the final SCI value - aggregation-method: sum + aggregation-method: + time: sum + component: sum sum-carbon: path: builtin method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon parameter-metadata: inputs: carbon-operational: + unit: gCO2eq description: Operational carbon footprint + aggregation-method: + time: sum + component: sum + embodied-carbon: unit: gCO2eq - aggregation-method: sum - carbon-embodied: description: Embodied carbon footprint - unit: gCO2eq - aggregation-method: sum + aggregation-method: + time: sum + component: sum outputs: carbon: - description: Total carbon footprint unit: gCO2eq - aggregation-method: sum + description: Total carbon footprint + aggregation-method: + time: sum + component: sum time-sync: path: builtin method: TimeSync - global-config: - start-time: "2023-12-12T00:00:00.000Z" - end-time: "2023-12-12T00:01:00.000Z" + config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' interval: 5 allow-padding: true parameter-metadata: @@ -180,50 +210,61 @@ initialize: timestamp: unit: RFC3339 description: refers to the time of occurrence of the input - aggregation-method: none + aggregation-method: + time: none + component: none duration: unit: seconds description: refers to the duration of the input - aggregation-method: sum + aggregation-method: + time: sum + component: sum cloud/instance-type: unit: none description: type of Cloud Instance name used in the cloud provider APIs - aggregation-method: none + aggregation-method: + time: copy + component: copy cloud/region: unit: none description: region cloud instance - aggregation-method: none + aggregation-method: + time: copy + component: copy time-reserved: unit: seconds description: time reserved for a component - aggregation-method: avg + aggregation-method: + time: avg + component: avg network/energy: unit: kWh description: Energy consumed by the Network of the component - aggregation-method: sum + aggregation-method: + time: sum + component: sum execution: command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m - manifests/examples/pipelines/nesting.yml -o - manifests/outputs/pipelines/nesting-1.yaml + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/outputs/pipelines/nesting.yaml environment: - if-version: 0.5.0 + if-version: 0.6.0 os: macOS - os-version: "14.5" - node-version: 18.14.2 - date-time: 2024-07-31T13:17:29.944Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T11:05:09.727Z (UTC) dependencies: - - "@babel/core@7.22.10" - - "@babel/preset-typescript@7.23.3" - - "@commitlint/cli@18.6.0" - - "@commitlint/config-conventional@18.6.0" - - "@grnsft/if-core@0.0.16" - - "@jest/globals@29.7.0" - - "@types/jest@29.5.8" - - "@types/js-yaml@4.0.9" - - "@types/luxon@3.4.2" - - "@types/node@20.9.0" + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.25' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -270,28 +311,28 @@ tree: - time-sync - sci inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: "2023-12-12T00:00:01.000Z" + - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 60 - - timestamp: "2023-12-12T00:00:06.000Z" + - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 70 - - timestamp: "2023-12-12T00:00:13.000Z" + - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -299,7 +340,7 @@ tree: network/energy: 0.000001 requests: 55 outputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -308,9 +349,9 @@ tree: requests: 98 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 2759.6159999999995 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 170294400 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.4065 @@ -319,11 +360,17 @@ tree: cpu-energy-raw: 0.00006833333333333335 vcpu-ratio: 8 cpu-energy-kwh: 0.000008541666666666668 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.006833333333333334 - carbon: 0.0068434614408929475 - sci: 0.00006983123919278518 - - timestamp: "2023-12-12T00:00:05.000Z" + carbon: 0.04647057331303907 + sci: 0.0004741895236024395 + - timestamp: '2023-12-12T00:00:05.000Z' duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -332,9 +379,9 @@ tree: requests: 52 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 1182.6925714285712 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 72983314.28571428 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.30975 @@ -343,11 +390,17 @@ tree: cpu-energy-raw: 0.00005340277777777778 vcpu-ratio: 8 cpu-energy-kwh: 0.000006675347222222222 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.005340277777777777 - carbon: 0.005350405885337391 - sci: 0.0001028924208718729 - - timestamp: "2023-12-12T00:00:10.000Z" + carbon: 0.044977517757483515 + sci: 0.0008649522645669907 + - timestamp: '2023-12-12T00:00:10.000Z' duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -356,9 +409,9 @@ tree: requests: 33.666666666666664 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 759.2594285714285 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 46853485.71428572 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -367,11 +420,17 @@ tree: cpu-energy-raw: 0.00005190972222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.0000064887152777777775 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.005190972222222222 - carbon: 0.0052011003297818366 - sci: 0.0001544881286073813 - - timestamp: "2023-12-12T00:00:15.000Z" + carbon: 0.04482821220192795 + sci: 0.0013315310555028106 + - timestamp: '2023-12-12T00:00:15.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -380,9 +439,9 @@ tree: requests: 9.166666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -391,11 +450,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0005673927632489277 - - timestamp: "2023-12-12T00:00:20.000Z" + carbon: 0.04482821220192795 + sci: 0.004890350422028504 + - timestamp: '2023-12-12T00:00:20.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -404,9 +469,9 @@ tree: requests: 9.166666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -415,11 +480,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0005673927632489277 - - timestamp: "2023-12-12T00:00:25.000Z" + carbon: 0.04482821220192795 + sci: 0.004890350422028504 + - timestamp: '2023-12-12T00:00:25.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -428,9 +499,9 @@ tree: requests: 9.166666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -439,11 +510,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0005673927632489277 - - timestamp: "2023-12-12T00:00:30.000Z" + carbon: 0.04482821220192795 + sci: 0.004890350422028504 + - timestamp: '2023-12-12T00:00:30.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -452,9 +529,9 @@ tree: requests: 9.166666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -463,11 +540,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0005673927632489277 - - timestamp: "2023-12-12T00:00:35.000Z" + carbon: 0.04482821220192795 + sci: 0.004890350422028504 + - timestamp: '2023-12-12T00:00:35.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -476,9 +559,9 @@ tree: requests: 9.166666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -487,11 +570,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0005673927632489277 - - timestamp: "2023-12-12T00:00:40.000Z" + carbon: 0.04482821220192795 + sci: 0.004890350422028504 + - timestamp: '2023-12-12T00:00:40.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -500,9 +589,9 @@ tree: requests: 5.5 cpu/thermal-design-power: 60 grid/carbon-intensity: 480 - device/emissions-embodied: 153.312 + device/emissions-embodied: 1533.12 time-reserved: 2160.2 - device/expected-lifespan: 9460800 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.22425 @@ -511,11 +600,17 @@ tree: cpu-energy-raw: 0.000031145833333333336 vcpu-ratio: 8 cpu-energy-kwh: 0.000003893229166666667 - carbon-embodied: 0.000006076864535768645 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.02378234398782344 carbon-operational: 0.0031145833333333334 - carbon: 0.003120660197869102 - sci: 0.0005673927632489276 - - timestamp: "2023-12-12T00:00:45.000Z" + carbon: 0.02689692732115677 + sci: 0.004890350422028503 + - timestamp: '2023-12-12T00:00:45.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -524,9 +619,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -535,11 +630,17 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: "2023-12-12T00:00:50.000Z" + - timestamp: '2023-12-12T00:00:50.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -548,9 +649,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -559,11 +660,17 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: "2023-12-12T00:00:55.000Z" + - timestamp: '2023-12-12T00:00:55.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -572,33 +679,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 - vcpus-allocated: 1 - vcpus-total: 8 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 8 - cpu-energy-kwh: 0 - carbon-embodied: 0 - carbon-operational: 0 - carbon: 0 - sci: 0 - - timestamp: "2023-12-12T00:01:00.000Z" - duration: 1 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 0 - network/energy: 0 - requests: 0 - cpu/thermal-design-power: 0 - grid/carbon-intensity: 0 - device/emissions-embodied: 0 - time-reserved: 1 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -607,12 +690,18 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 aggregated: - carbon: 0.04652112950279046 + carbon: 0.3873142916032471 child-1: defaults: cpu/thermal-design-power: 100 @@ -636,28 +725,28 @@ tree: - time-sync - sci inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 10 - - timestamp: "2023-12-12T00:00:01.000Z" + - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 90 - - timestamp: "2023-12-12T00:00:06.000Z" + - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 30 - - timestamp: "2023-12-12T00:00:13.000Z" + - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -665,7 +754,7 @@ tree: network/energy: 0.000001 requests: 22 outputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -674,9 +763,9 @@ tree: requests: 82 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 2759.6159999999995 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 170294400 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.4065 @@ -685,11 +774,17 @@ tree: cpu-energy-raw: 0.00006833333333333335 vcpu-ratio: 8 cpu-energy-kwh: 0.000008541666666666668 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.006833333333333334 - carbon: 0.0068434614408929475 - sci: 0.0000834568468401579 - - timestamp: "2023-12-12T00:00:05.000Z" + carbon: 0.04647057331303907 + sci: 0.0005667143086955984 + - timestamp: '2023-12-12T00:00:05.000Z' duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -698,9 +793,9 @@ tree: requests: 35.14285714285714 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 1182.6925714285712 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 72983314.28571428 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.30975 @@ -709,11 +804,17 @@ tree: cpu-energy-raw: 0.00005340277777777778 vcpu-ratio: 8 cpu-energy-kwh: 0.000006675347222222222 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.005340277777777777 - carbon: 0.005350405885337391 - sci: 0.00015224732194049487 - - timestamp: "2023-12-12T00:00:10.000Z" + carbon: 0.044977517757483515 + sci: 0.0012798480662698562 + - timestamp: '2023-12-12T00:00:10.000Z' duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -722,9 +823,9 @@ tree: requests: 14.323809523809523 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 759.2594285714285 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 46853485.71428572 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -733,11 +834,17 @@ tree: cpu-energy-raw: 0.00005190972222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.0000064887152777777775 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.005190972222222222 - carbon: 0.0052011003297818366 - sci: 0.000363108733129716 - - timestamp: "2023-12-12T00:00:15.000Z" + carbon: 0.04482821220192795 + sci: 0.0031296291763314066 + - timestamp: '2023-12-12T00:00:15.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -746,9 +853,9 @@ tree: requests: 3.6666666666666665 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -757,11 +864,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0014184819081223194 - - timestamp: "2023-12-12T00:00:20.000Z" + carbon: 0.04482821220192795 + sci: 0.01222587605507126 + - timestamp: '2023-12-12T00:00:20.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -770,9 +883,9 @@ tree: requests: 3.6666666666666665 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -781,11 +894,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0014184819081223194 - - timestamp: "2023-12-12T00:00:25.000Z" + carbon: 0.04482821220192795 + sci: 0.01222587605507126 + - timestamp: '2023-12-12T00:00:25.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -794,9 +913,9 @@ tree: requests: 3.6666666666666665 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -805,11 +924,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0014184819081223194 - - timestamp: "2023-12-12T00:00:30.000Z" + carbon: 0.04482821220192795 + sci: 0.01222587605507126 + - timestamp: '2023-12-12T00:00:30.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -818,9 +943,9 @@ tree: requests: 3.6666666666666665 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -829,11 +954,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0014184819081223194 - - timestamp: "2023-12-12T00:00:35.000Z" + carbon: 0.04482821220192795 + sci: 0.01222587605507126 + - timestamp: '2023-12-12T00:00:35.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -842,9 +973,9 @@ tree: requests: 3.6666666666666665 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -853,11 +984,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0014184819081223194 - - timestamp: "2023-12-12T00:00:40.000Z" + carbon: 0.04482821220192795 + sci: 0.01222587605507126 + - timestamp: '2023-12-12T00:00:40.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -866,9 +1003,9 @@ tree: requests: 2.1999999999999997 cpu/thermal-design-power: 60 grid/carbon-intensity: 480 - device/emissions-embodied: 153.312 + device/emissions-embodied: 1533.12 time-reserved: 2160.2 - device/expected-lifespan: 9460800 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.22425 @@ -877,11 +1014,17 @@ tree: cpu-energy-raw: 0.000031145833333333336 vcpu-ratio: 8 cpu-energy-kwh: 0.000003893229166666667 - carbon-embodied: 0.000006076864535768645 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.02378234398782344 carbon-operational: 0.0031145833333333334 - carbon: 0.003120660197869102 - sci: 0.0014184819081223194 - - timestamp: "2023-12-12T00:00:45.000Z" + carbon: 0.02689692732115677 + sci: 0.01222587605507126 + - timestamp: '2023-12-12T00:00:45.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -890,9 +1033,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -901,11 +1044,17 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: "2023-12-12T00:00:50.000Z" + - timestamp: '2023-12-12T00:00:50.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -914,9 +1063,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -925,11 +1074,17 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: "2023-12-12T00:00:55.000Z" + - timestamp: '2023-12-12T00:00:55.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -938,33 +1093,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 - vcpus-allocated: 1 - vcpus-total: 8 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 8 - cpu-energy-kwh: 0 - carbon-embodied: 0 - carbon-operational: 0 - carbon: 0 - sci: 0 - - timestamp: "2023-12-12T00:01:00.000Z" - duration: 1 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 0 - network/energy: 0 - requests: 0 - cpu/thermal-design-power: 0 - grid/carbon-intensity: 0 - device/emissions-embodied: 0 - time-reserved: 1 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -973,12 +1104,18 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 aggregated: - carbon: 0.04652112950279046 + carbon: 0.3873142916032471 child-2: children: child-2-0: @@ -1004,28 +1141,28 @@ tree: - time-sync - sci inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: "2023-12-12T00:00:01.000Z" + - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 65 - - timestamp: "2023-12-12T00:00:06.000Z" + - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 80 - - timestamp: "2023-12-12T00:00:13.000Z" + - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -1033,7 +1170,7 @@ tree: network/energy: 0.000001 requests: 40 outputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -1042,9 +1179,9 @@ tree: requests: 102 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 2759.6159999999995 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 170294400 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.4065 @@ -1053,11 +1190,17 @@ tree: cpu-energy-raw: 0.00006833333333333335 vcpu-ratio: 8 cpu-energy-kwh: 0.000008541666666666668 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.006833333333333334 - carbon: 0.0068434614408929475 - sci: 0.00006709275922444067 - - timestamp: "2023-12-12T00:00:05.000Z" + carbon: 0.04647057331303907 + sci: 0.00045559385601018696 + - timestamp: '2023-12-12T00:00:05.000Z' duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -1066,9 +1209,9 @@ tree: requests: 58.71428571428572 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 1182.6925714285712 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 72983314.28571428 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.30975 @@ -1077,11 +1220,17 @@ tree: cpu-energy-raw: 0.00005340277777777778 vcpu-ratio: 8 cpu-energy-kwh: 0.000006675347222222222 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.005340277777777777 - carbon: 0.005350405885337391 - sci: 0.0000911261343001502 - - timestamp: "2023-12-12T00:00:10.000Z" + carbon: 0.044977517757483515 + sci: 0.0007660404484242933 + - timestamp: '2023-12-12T00:00:10.000Z' duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -1090,9 +1239,9 @@ tree: requests: 36.952380952380956 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 759.2594285714285 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 46853485.71428572 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1101,11 +1250,17 @@ tree: cpu-energy-raw: 0.00005190972222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.0000064887152777777775 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.005190972222222222 - carbon: 0.0052011003297818366 - sci: 0.00014075142645028166 - - timestamp: "2023-12-12T00:00:15.000Z" + carbon: 0.04482821220192795 + sci: 0.0012131346085573285 + - timestamp: '2023-12-12T00:00:15.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1114,9 +1269,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1125,11 +1280,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:20.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:20.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1138,9 +1299,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1149,11 +1310,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:25.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:25.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1162,9 +1329,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1173,11 +1340,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:30.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:30.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1186,9 +1359,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1197,11 +1370,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:35.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:35.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1210,9 +1389,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1221,11 +1400,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:40.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:40.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1234,9 +1419,9 @@ tree: requests: 4 cpu/thermal-design-power: 60 grid/carbon-intensity: 480 - device/emissions-embodied: 153.312 + device/emissions-embodied: 1533.12 time-reserved: 2160.2 - device/expected-lifespan: 9460800 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.22425 @@ -1245,11 +1430,17 @@ tree: cpu-energy-raw: 0.000031145833333333336 vcpu-ratio: 8 cpu-energy-kwh: 0.000003893229166666667 - carbon-embodied: 0.000006076864535768645 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.02378234398782344 carbon-operational: 0.0031145833333333334 - carbon: 0.003120660197869102 - sci: 0.0007801650494672755 - - timestamp: "2023-12-12T00:00:45.000Z" + carbon: 0.02689692732115677 + sci: 0.006724231830289192 + - timestamp: '2023-12-12T00:00:45.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1258,9 +1449,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -1269,11 +1460,17 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: "2023-12-12T00:00:50.000Z" + - timestamp: '2023-12-12T00:00:50.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1282,9 +1479,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -1293,11 +1490,17 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: "2023-12-12T00:00:55.000Z" + - timestamp: '2023-12-12T00:00:55.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1306,9 +1509,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -1317,36 +1520,18 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 - carbon-operational: 0 - carbon: 0 - sci: 0 - - timestamp: "2023-12-12T00:01:00.000Z" - duration: 1 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 0 - network/energy: 0 - requests: 0 - cpu/thermal-design-power: 0 - grid/carbon-intensity: 0 - device/emissions-embodied: 0 - time-reserved: 1 - device/expected-lifespan: 0 - vcpus-allocated: 1 - vcpus-total: 8 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 8 - cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 aggregated: - carbon: 0.04652112950279046 + carbon: 0.3873142916032471 child-2-1: defaults: cpu/thermal-design-power: 100 @@ -1370,28 +1555,28 @@ tree: - time-sync - sci inputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 1 cpu/utilization: 50 network/energy: 0.000001 requests: 50 - - timestamp: "2023-12-12T00:00:01.000Z" + - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 cpu/utilization: 20 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 50 - - timestamp: "2023-12-12T00:00:06.000Z" + - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 cpu/utilization: 15 cloud/instance-type: A1 cloud/region: uk-west network/energy: 0.000001 requests: 60 - - timestamp: "2023-12-12T00:00:13.000Z" + - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 cloud/instance-type: A1 cloud/region: uk-west @@ -1399,7 +1584,7 @@ tree: network/energy: 0.000001 requests: 40 outputs: - - timestamp: "2023-12-12T00:00:00.000Z" + - timestamp: '2023-12-12T00:00:00.000Z' cloud/instance-type: A1 cloud/region: uk-west duration: 5 @@ -1408,9 +1593,9 @@ tree: requests: 90 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 2759.6159999999995 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 170294400 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.4065 @@ -1419,11 +1604,17 @@ tree: cpu-energy-raw: 0.00006833333333333335 vcpu-ratio: 8 cpu-energy-kwh: 0.000008541666666666668 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.006833333333333334 - carbon: 0.0068434614408929475 - sci: 0.00007603846045436609 - - timestamp: "2023-12-12T00:00:05.000Z" + carbon: 0.04647057331303907 + sci: 0.0005163397034782119 + - timestamp: '2023-12-12T00:00:05.000Z' duration: 5 cpu/utilization: 13 cloud/instance-type: A1 @@ -1432,9 +1623,9 @@ tree: requests: 44.28571428571428 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 1182.6925714285712 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 72983314.28571428 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.30975 @@ -1443,11 +1634,17 @@ tree: cpu-energy-raw: 0.00005340277777777778 vcpu-ratio: 8 cpu-energy-kwh: 0.000006675347222222222 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.005340277777777777 - carbon: 0.005350405885337391 - sci: 0.00012081561676568304 - - timestamp: "2023-12-12T00:00:10.000Z" + carbon: 0.044977517757483515 + sci: 0.00101562136871737 + - timestamp: '2023-12-12T00:00:10.000Z' duration: 5 cpu/utilization: 12 cloud/instance-type: A1 @@ -1456,9 +1653,9 @@ tree: requests: 28.38095238095238 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 759.2594285714285 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 46853485.71428572 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1467,11 +1664,17 @@ tree: cpu-energy-raw: 0.00005190972222222222 vcpu-ratio: 8 cpu-energy-kwh: 0.0000064887152777777775 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.005190972222222222 - carbon: 0.0052011003297818366 - sci: 0.0001832602465191587 - - timestamp: "2023-12-12T00:00:15.000Z" + carbon: 0.04482821220192795 + sci: 0.0015795175440276629 + - timestamp: '2023-12-12T00:00:15.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1480,9 +1683,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1491,11 +1694,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:20.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:20.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1504,9 +1713,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1515,11 +1724,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:25.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:25.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1528,9 +1743,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1539,11 +1754,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:30.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:30.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1552,9 +1773,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1563,11 +1784,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:35.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:35.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1576,9 +1803,9 @@ tree: requests: 6.666666666666666 cpu/thermal-design-power: 80 grid/carbon-intensity: 640 - device/emissions-embodied: 255.51999999999998 + device/emissions-embodied: 1533.12 time-reserved: 2880 - device/expected-lifespan: 15768000 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.29900000000000004 @@ -1587,11 +1814,17 @@ tree: cpu-energy-raw: 0.00005190972222222223 vcpu-ratio: 8 cpu-energy-kwh: 0.000006488715277777778 - carbon-embodied: 0.000010128107559614409 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970573 carbon-operational: 0.005190972222222222 - carbon: 0.005201100329781837 - sci: 0.0007801650494672756 - - timestamp: "2023-12-12T00:00:40.000Z" + carbon: 0.04482821220192795 + sci: 0.006724231830289193 + - timestamp: '2023-12-12T00:00:40.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1600,9 +1833,9 @@ tree: requests: 4 cpu/thermal-design-power: 60 grid/carbon-intensity: 480 - device/emissions-embodied: 153.312 + device/emissions-embodied: 1533.12 time-reserved: 2160.2 - device/expected-lifespan: 9460800 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0.22425 @@ -1611,11 +1844,17 @@ tree: cpu-energy-raw: 0.000031145833333333336 vcpu-ratio: 8 cpu-energy-kwh: 0.000003893229166666667 - carbon-embodied: 0.000006076864535768645 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.02378234398782344 carbon-operational: 0.0031145833333333334 - carbon: 0.003120660197869102 - sci: 0.0007801650494672755 - - timestamp: "2023-12-12T00:00:45.000Z" + carbon: 0.02689692732115677 + sci: 0.006724231830289192 + - timestamp: '2023-12-12T00:00:45.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1624,9 +1863,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -1635,11 +1874,17 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: "2023-12-12T00:00:50.000Z" + - timestamp: '2023-12-12T00:00:50.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1648,9 +1893,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -1659,11 +1904,17 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 - - timestamp: "2023-12-12T00:00:55.000Z" + - timestamp: '2023-12-12T00:00:55.000Z' duration: 5 cloud/instance-type: A1 cloud/region: uk-west @@ -1672,9 +1923,9 @@ tree: requests: 0 cpu/thermal-design-power: 0 grid/carbon-intensity: 0 - device/emissions-embodied: 0 + device/emissions-embodied: 1533.12 time-reserved: 0.8 - device/expected-lifespan: 0 + device/expected-lifespan: 94608000 vcpus-allocated: 1 vcpus-total: 8 cpu-factor: 0 @@ -1683,117 +1934,93 @@ tree: cpu-energy-raw: 0 vcpu-ratio: 8 cpu-energy-kwh: 0 - carbon-embodied: 0 - carbon-operational: 0 - carbon: 0 - sci: 0 - - timestamp: "2023-12-12T00:01:00.000Z" - duration: 1 - cloud/instance-type: A1 - cloud/region: uk-west - cpu/utilization: 0 - network/energy: 0 - requests: 0 - cpu/thermal-design-power: 0 - grid/carbon-intensity: 0 - device/emissions-embodied: 0 - time-reserved: 1 - device/expected-lifespan: 0 - vcpus-allocated: 1 - vcpus-total: 8 - cpu-factor: 0 - cpu-wattage: 0 - cpu-wattage-times-duration: 0 - cpu-energy-raw: 0 - vcpu-ratio: 8 - cpu-energy-kwh: 0 - carbon-embodied: 0 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0 carbon-operational: 0 carbon: 0 sci: 0 aggregated: - carbon: 0.04652112950279046 + carbon: 0.3873142916032471 outputs: - - carbon: 0.013686922881785895 - timestamp: "2023-12-12T00:00:00.000Z" + - carbon: 0.09294114662607814 + timestamp: '2023-12-12T00:00:00.000Z' duration: 5 - - carbon: 0.010700811770674782 - timestamp: "2023-12-12T00:00:05.000Z" + - carbon: 0.08995503551496703 + timestamp: '2023-12-12T00:00:05.000Z' duration: 5 - - carbon: 0.010402200659563673 - timestamp: "2023-12-12T00:00:10.000Z" + - carbon: 0.0896564244038559 + timestamp: '2023-12-12T00:00:10.000Z' duration: 5 - - carbon: 0.010402200659563675 - timestamp: "2023-12-12T00:00:15.000Z" + - carbon: 0.0896564244038559 + timestamp: '2023-12-12T00:00:15.000Z' duration: 5 - - carbon: 0.010402200659563675 - timestamp: "2023-12-12T00:00:20.000Z" + - carbon: 0.0896564244038559 + timestamp: '2023-12-12T00:00:20.000Z' duration: 5 - - carbon: 0.010402200659563675 - timestamp: "2023-12-12T00:00:25.000Z" + - carbon: 0.0896564244038559 + timestamp: '2023-12-12T00:00:25.000Z' duration: 5 - - carbon: 0.010402200659563675 - timestamp: "2023-12-12T00:00:30.000Z" + - carbon: 0.0896564244038559 + timestamp: '2023-12-12T00:00:30.000Z' duration: 5 - - carbon: 0.010402200659563675 - timestamp: "2023-12-12T00:00:35.000Z" + - carbon: 0.0896564244038559 + timestamp: '2023-12-12T00:00:35.000Z' duration: 5 - - carbon: 0.006241320395738204 - timestamp: "2023-12-12T00:00:40.000Z" + - carbon: 0.05379385464231354 + timestamp: '2023-12-12T00:00:40.000Z' duration: 5 - carbon: 0 - timestamp: "2023-12-12T00:00:45.000Z" + timestamp: '2023-12-12T00:00:45.000Z' duration: 5 - carbon: 0 - timestamp: "2023-12-12T00:00:50.000Z" + timestamp: '2023-12-12T00:00:50.000Z' duration: 5 - carbon: 0 - timestamp: "2023-12-12T00:00:55.000Z" + timestamp: '2023-12-12T00:00:55.000Z' duration: 5 - - carbon: 0 - timestamp: "2023-12-12T00:01:00.000Z" - duration: 1 aggregated: - carbon: 0.09304225900558093 + carbon: 0.7746285832064942 outputs: - - carbon: 0.02737384576357179 - timestamp: "2023-12-12T00:00:00.000Z" + - carbon: 0.18588229325215627 + timestamp: '2023-12-12T00:00:00.000Z' duration: 5 - - carbon: 0.021401623541349564 - timestamp: "2023-12-12T00:00:05.000Z" + - carbon: 0.17991007102993406 + timestamp: '2023-12-12T00:00:05.000Z' duration: 5 - - carbon: 0.020804401319127346 - timestamp: "2023-12-12T00:00:10.000Z" + - carbon: 0.1793128488077118 + timestamp: '2023-12-12T00:00:10.000Z' duration: 5 - - carbon: 0.02080440131912735 - timestamp: "2023-12-12T00:00:15.000Z" + - carbon: 0.1793128488077118 + timestamp: '2023-12-12T00:00:15.000Z' duration: 5 - - carbon: 0.02080440131912735 - timestamp: "2023-12-12T00:00:20.000Z" + - carbon: 0.1793128488077118 + timestamp: '2023-12-12T00:00:20.000Z' duration: 5 - - carbon: 0.02080440131912735 - timestamp: "2023-12-12T00:00:25.000Z" + - carbon: 0.1793128488077118 + timestamp: '2023-12-12T00:00:25.000Z' duration: 5 - - carbon: 0.02080440131912735 - timestamp: "2023-12-12T00:00:30.000Z" + - carbon: 0.1793128488077118 + timestamp: '2023-12-12T00:00:30.000Z' duration: 5 - - carbon: 0.02080440131912735 - timestamp: "2023-12-12T00:00:35.000Z" + - carbon: 0.1793128488077118 + timestamp: '2023-12-12T00:00:35.000Z' duration: 5 - - carbon: 0.012482640791476408 - timestamp: "2023-12-12T00:00:40.000Z" + - carbon: 0.10758770928462708 + timestamp: '2023-12-12T00:00:40.000Z' duration: 5 - carbon: 0 - timestamp: "2023-12-12T00:00:45.000Z" + timestamp: '2023-12-12T00:00:45.000Z' duration: 5 - carbon: 0 - timestamp: "2023-12-12T00:00:50.000Z" + timestamp: '2023-12-12T00:00:50.000Z' duration: 5 - carbon: 0 - timestamp: "2023-12-12T00:00:55.000Z" + timestamp: '2023-12-12T00:00:55.000Z' duration: 5 - - carbon: 0 - timestamp: "2023-12-12T00:01:00.000Z" - duration: 1 aggregated: - carbon: 0.18608451801116185 + carbon: 1.5492571664129884 diff --git a/manifests/outputs/pipelines/pipeline-teads-sci.yaml b/manifests/outputs/pipelines/pipeline-teads-sci.yaml index 1c2a9d005..aca6ca4b9 100644 --- a/manifests/outputs/pipelines/pipeline-teads-sci.yaml +++ b/manifests/outputs/pipelines/pipeline-teads-sci.yaml @@ -8,7 +8,7 @@ initialize: interpolate: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 @@ -25,7 +25,7 @@ initialize: cpu-factor-to-wattage: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-factor - cpu/thermal-design-power @@ -33,7 +33,7 @@ initialize: wattage-times-duration: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-wattage - duration @@ -41,21 +41,21 @@ initialize: wattage-to-energy-kwh: path: builtin method: Divide - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw calculate-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio correct-cpu-energy-for-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -65,7 +65,7 @@ initialize: operational-carbon: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-energy-kwh - grid/carbon-intensity @@ -73,34 +73,42 @@ initialize: sci: path: builtin method: Sci - global-config: + config: functional-unit: component sum-carbon: path: builtin method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon + time-sync: + path: builtin + method: TimeSync + config: + start-time: '2023-12-12T00:00:00.000Z' + end-time: '2023-12-12T00:01:00.000Z' + interval: 5 + allow-padding: true execution: command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m manifests/examples/pipelines/pipeline-teads-sci.yml -o - manifests/outputs/pipelines/pipeline-teads-sci + manifests/outputs/pipelines/pipeline-teads-sci.yaml environment: - if-version: 0.5.0 + if-version: 0.6.0 os: macOS - os-version: '14.5' - node-version: 18.14.2 - date-time: 2024-07-19T06:32:50.994Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T09:52:09.777Z (UTC) dependencies: - '@babel/core@7.22.10' - '@babel/preset-typescript@7.23.3' - '@commitlint/cli@18.6.0' - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' + - '@grnsft/if-core@0.0.25' - '@jest/globals@29.7.0' - '@types/jest@29.5.8' - '@types/js-yaml@4.0.9' @@ -125,7 +133,7 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 status: success tree: children: @@ -142,7 +150,6 @@ tree: - operational-carbon - sum-carbon - sci - config: null defaults: cpu/thermal-design-power: 100 grid/carbon-intensity: 800 @@ -198,10 +205,16 @@ tree: cpu-energy-raw: 0.000020833333333333333 vcpu-ratio: 8 cpu-energy-kwh: 0.0000026041666666666666 - carbon-embodied: 0.0000020256215119228817 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.007927447995941146 carbon-operational: 0.0020833333333333333 - carbon: 0.002085358954845256 - sci: 0.002085358954845256 + carbon: 0.010010781329274479 + sci: 0.010010781329274479 - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 cpu/utilization: 20 @@ -222,10 +235,16 @@ tree: cpu-energy-raw: 0.000059375 vcpu-ratio: 8 cpu-energy-kwh: 0.000007421875 - carbon-embodied: 0.000010128107559614407 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.0059375 - carbon: 0.005947628107559615 - sci: 0.005947628107559615 + carbon: 0.045574739979705736 + sci: 0.045574739979705736 - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 cpu/utilization: 15 @@ -246,10 +265,16 @@ tree: cpu-energy-raw: 0.00007267361111111111 vcpu-ratio: 8 cpu-energy-kwh: 0.000009084201388888889 - carbon-embodied: 0.00001417935058346017 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.05549213597158803 carbon-operational: 0.007267361111111111 - carbon: 0.007281540461694571 - sci: 0.007281540461694571 + carbon: 0.06275949708269914 + sci: 0.06275949708269914 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 cloud/instance-type: A1 @@ -270,7 +295,13 @@ tree: cpu-energy-raw: 0.00031145833333333335 vcpu-ratio: 8 cpu-energy-kwh: 0.00003893229166666667 - carbon-embodied: 0.00006076864535768645 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.2378234398782344 carbon-operational: 0.031145833333333334 - carbon: 0.03120660197869102 - sci: 0.03120660197869102 + carbon: 0.2689692732115677 + sci: 0.2689692732115677 diff --git a/manifests/outputs/pipelines/scenario-4.yaml b/manifests/outputs/pipelines/scenario-4.yaml new file mode 100644 index 000000000..fde5bcbf0 --- /dev/null +++ b/manifests/outputs/pipelines/scenario-4.yaml @@ -0,0 +1,105 @@ +name: demo +description: null +tags: null +initialize: + plugins: + sum: + path: builtin + method: Sum + config: + input-parameters: + - cpu/energy + - network/energy + output-parameter: energy-sum + coefficient: + path: builtin + method: Coefficient + config: + input-parameter: energy + coefficient: 2 + output-parameter: energy-doubled + multiply: + path: builtin + method: Multiply + config: + input-parameters: + - cpu/utilization + - duration + output-parameter: cpu-times-duration +execution: + command: >- + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/pipelines/scenario-4.yml -o + manifests/outputs/pipelines/scenario-4.yml + environment: + if-version: 0.6.0 + os: macOS + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-09-12T06:13:40.278Z (UTC) + dependencies: + - '@babel/core@7.22.10' + - '@babel/preset-typescript@7.23.3' + - '@commitlint/cli@18.6.0' + - '@commitlint/config-conventional@18.6.0' + - '@grnsft/if-core@0.0.22' + - '@jest/globals@29.7.0' + - '@types/jest@29.5.8' + - '@types/js-yaml@4.0.9' + - '@types/luxon@3.4.2' + - '@types/node@20.9.0' + - axios-mock-adapter@1.22.0 + - axios@1.7.2 + - cross-env@7.0.3 + - csv-parse@5.5.6 + - csv-stringify@6.4.6 + - fixpack@4.0.0 + - gts@5.2.0 + - husky@8.0.3 + - jest@29.7.0 + - js-yaml@4.1.0 + - lint-staged@15.2.2 + - luxon@3.4.4 + - release-it@16.3.0 + - rimraf@5.0.5 + - ts-command-line-args@2.5.1 + - ts-jest@29.1.1 + - typescript-cubic-spline@1.0.1 + - typescript@5.2.2 + - winston@3.11.0 + - zod@3.23.8 + status: success +tree: + children: + child-1: + pipeline: + observe: null + compute: + - sum + - coefficient + - multiply + defaults: + cpu/thermal-design-power: 100 + inputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + cpu/energy: 20 + network/energy: 10 + energy: 5 + outputs: + - timestamp: '2023-12-12T00:00:00.000Z' + cloud/instance-type: A1 + cloud/region: uk-west + duration: 1 + cpu/utilization: 50 + cpu/energy: 20 + network/energy: 10 + energy: 5 + cpu/thermal-design-power: 100 + energy-sum: 30 + energy-doubled: 10 + cpu-times-duration: 50 diff --git a/manifests/outputs/pipelines/sci.yaml b/manifests/outputs/pipelines/sci.yaml index c4fa641c2..7f080e67c 100644 --- a/manifests/outputs/pipelines/sci.yaml +++ b/manifests/outputs/pipelines/sci.yaml @@ -8,7 +8,7 @@ initialize: interpolate: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 @@ -25,7 +25,7 @@ initialize: cpu-factor-to-wattage: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-factor - cpu/thermal-design-power @@ -33,7 +33,7 @@ initialize: wattage-times-duration: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-wattage - duration @@ -41,28 +41,28 @@ initialize: wattage-to-energy-kwh: path: builtin method: Divide - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw calculate-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio correct-cpu-energy-for-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu/energy sum-energy-components: path: builtin method: Sum - global-config: + config: input-parameters: - cpu/energy - network/energy @@ -73,7 +73,7 @@ initialize: operational-carbon: path: builtin method: Multiply - global-config: + config: input-parameters: - energy - grid/carbon-intensity @@ -81,33 +81,33 @@ initialize: sum-carbon: path: builtin method: Sum - global-config: + config: input-parameters: - carbon-operational - - carbon-embodied + - embodied-carbon output-parameter: carbon sci: path: builtin method: Sci - global-config: + config: functional-unit: component execution: command: >- - /Users/mariamkhalatova/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node - /Users/mariamkhalatova/Projects/UK/if/src/if-run/index.ts -m - manifests/examples/pipelines/sci.yml -o manifests/outputs/pipelines/sci + /Users/manushak/.npm/_npx/1bf7c3c15bf47d04/node_modules/.bin/ts-node + /Users/manushak/Documents/Projects/Green-Software/if/src/if-run/index.ts -m + manifests/examples/pipelines/sci.yml -o manifests/outputs/pipelines/sci.yaml environment: - if-version: 0.5.0 + if-version: 0.6.0 os: macOS - os-version: '14.5' - node-version: 18.14.2 - date-time: 2024-07-19T06:34:45.027Z (UTC) + os-version: 14.6.1 + node-version: 18.20.4 + date-time: 2024-10-04T09:57:49.899Z (UTC) dependencies: - '@babel/core@7.22.10' - '@babel/preset-typescript@7.23.3' - '@commitlint/cli@18.6.0' - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' + - '@grnsft/if-core@0.0.25' - '@jest/globals@29.7.0' - '@types/jest@29.5.8' - '@types/js-yaml@4.0.9' @@ -132,7 +132,7 @@ execution: - typescript-cubic-spline@1.0.1 - typescript@5.2.2 - winston@3.11.0 - - zod@3.22.4 + - zod@3.23.8 status: success tree: children: @@ -210,10 +210,16 @@ tree: vcpu-ratio: 4 cpu/energy: 0.000005208333333333333 energy: 0.000006208333333333333 - carbon-embodied: 0.000004051243023845763 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.007927447995941146 carbon-operational: 0.004966666666666666 - carbon: 0.004970717909690512 - sci: 0.004970717909690512 + carbon: 0.012894114662607812 + sci: 0.012894114662607812 - timestamp: '2023-12-12T00:00:01.000Z' duration: 5 cpu/utilization: 20 @@ -237,10 +243,16 @@ tree: vcpu-ratio: 4 cpu/energy: 0.00001484375 energy: 0.00001584375 - carbon-embodied: 0.000020256215119228814 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.03963723997970574 carbon-operational: 0.012674999999999999 - carbon: 0.012695256215119228 - sci: 0.012695256215119228 + carbon: 0.05231223997970574 + sci: 0.05231223997970574 - timestamp: '2023-12-12T00:00:06.000Z' duration: 7 cpu/utilization: 15 @@ -264,10 +276,16 @@ tree: vcpu-ratio: 4 cpu/energy: 0.000018168402777777778 energy: 0.000019168402777777778 - carbon-embodied: 0.00002835870116692034 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.05549213597158803 carbon-operational: 0.015334722222222222 - carbon: 0.015363080923389142 - sci: 0.015363080923389142 + carbon: 0.07082685819381025 + sci: 0.07082685819381025 - timestamp: '2023-12-12T00:00:13.000Z' duration: 30 cloud/instance-type: A1 @@ -291,7 +309,13 @@ tree: vcpu-ratio: 4 cpu/energy: 0.00007786458333333334 energy: 0.00007886458333333333 - carbon-embodied: 0.0001215372907153729 + vCPUs: 1 + memory: 16 + ssd: 0 + hdd: 0 + gpu: 0 + usage-ratio: 1 + embodied-carbon: 0.2378234398782344 carbon-operational: 0.06309166666666667 - carbon: 0.06321320395738204 - sci: 0.06321320395738204 + carbon: 0.30091510654490106 + sci: 0.30091510654490106 diff --git a/manifests/outputs/pipelines/teads-curve.yaml b/manifests/outputs/pipelines/teads-curve.yaml index 26885e1c7..ea3cab0ef 100644 --- a/manifests/outputs/pipelines/teads-curve.yaml +++ b/manifests/outputs/pipelines/teads-curve.yaml @@ -6,14 +6,14 @@ initialize: interpolate: path: builtin method: Interpolation - global-config: + config: method: linear x: - 0 - 10 - 50 - 100 - 'y': + "y": - 0.12 - 0.32 - 0.75 @@ -23,7 +23,7 @@ initialize: cpu-factor-to-wattage: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-factor - thermal-design-power @@ -31,7 +31,7 @@ initialize: wattage-times-duration: path: builtin method: Multiply - global-config: + config: input-parameters: - cpu-wattage - duration @@ -39,21 +39,21 @@ initialize: wattage-to-energy-kwh: path: builtin method: Divide - global-config: + config: numerator: cpu-wattage-times-duration denominator: 3600000 output: cpu-energy-raw calculate-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: vcpus-total denominator: vcpus-allocated output: vcpu-ratio correct-cpu-energy-for-vcpu-ratio: path: builtin method: Divide - global-config: + config: numerator: cpu-energy-raw denominator: vcpu-ratio output: cpu-energy-kwh @@ -66,20 +66,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-19T06:35:33.728Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 diff --git a/manifests/outputs/pipelines/zeros.yaml b/manifests/outputs/pipelines/zeros.yaml index 3efd3c0fb..d7513f077 100644 --- a/manifests/outputs/pipelines/zeros.yaml +++ b/manifests/outputs/pipelines/zeros.yaml @@ -8,7 +8,7 @@ initialize: sum-zero-and-one: path: builtin method: Sum - global-config: + config: input-parameters: - some-value - zero-value @@ -16,7 +16,7 @@ initialize: sum-zero-and-zero: path: builtin method: Sum - global-config: + config: input-parameters: - zero-value - zero-value @@ -24,7 +24,7 @@ initialize: subtract-one-and-zero: path: builtin method: Subtract - global-config: + config: input-parameters: - some-value - zero-value @@ -32,7 +32,7 @@ initialize: subtract-zero-and-zero: path: builtin method: Sum - global-config: + config: input-parameters: - zero-value - zero-value @@ -40,7 +40,7 @@ initialize: subtract-zero-and-one: path: builtin method: Subtract - global-config: + config: input-parameters: - zero-value - some-value @@ -48,28 +48,28 @@ initialize: coefficient-one-times-zero: path: builtin method: Coefficient - global-config: + config: input-parameter: zero-value coefficient: 1 output-parameter: zero-times-one-coefficient coefficient-zero-times-one: path: builtin method: Coefficient - global-config: + config: input-parameter: some-value coefficient: 0 output-parameter: one-times-zero-coefficient coefficient-zero-times-zero: path: builtin method: Coefficient - global-config: + config: input-parameter: zero-value coefficient: 0 output-parameter: zero-times-zero-coefficient multiply-one-times-zero: path: builtin method: Multiply - global-config: + config: input-parameters: - some-value - zero-value @@ -77,7 +77,7 @@ initialize: multiply-zero-times-one: path: builtin method: Multiply - global-config: + config: input-parameters: - zero-value - zero-value @@ -85,28 +85,28 @@ initialize: exponent-one-to-zero: path: builtin method: Exponent - global-config: + config: input-parameter: some-value exponent: 0 output-parameter: one-raised-to-zero-power exponent-zero-to-zero: path: builtin method: Exponent - global-config: + config: input-parameter: zero-value exponent: 0 output-parameter: zero-raised-to-zero-power exponent-zero-to-one: path: builtin method: Exponent - global-config: + config: input-parameter: zero-value exponent: 1 output-parameter: zero-raised-to-first-power sci: path: builtin method: Sci - global-config: + config: functional-unit: zero-value execution: command: >- @@ -116,20 +116,20 @@ execution: environment: if-version: 0.5.0 os: macOS - os-version: '14.5' + os-version: "14.5" node-version: 18.14.2 date-time: 2024-07-19T06:36:00.790Z (UTC) dependencies: - - '@babel/core@7.22.10' - - '@babel/preset-typescript@7.23.3' - - '@commitlint/cli@18.6.0' - - '@commitlint/config-conventional@18.6.0' - - '@grnsft/if-core@0.0.10' - - '@jest/globals@29.7.0' - - '@types/jest@29.5.8' - - '@types/js-yaml@4.0.9' - - '@types/luxon@3.4.2' - - '@types/node@20.9.0' + - "@babel/core@7.22.10" + - "@babel/preset-typescript@7.23.3" + - "@commitlint/cli@18.6.0" + - "@commitlint/config-conventional@18.6.0" + - "@grnsft/if-core@0.0.10" + - "@jest/globals@29.7.0" + - "@types/jest@29.5.8" + - "@types/js-yaml@4.0.9" + - "@types/luxon@3.4.2" + - "@types/node@20.9.0" - axios-mock-adapter@1.22.0 - axios@1.7.2 - cross-env@7.0.3 @@ -171,13 +171,13 @@ tree: - exponent-zero-to-zero - sci inputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 some-value: 1 zero-value: 0 carbon: 10 outputs: - - timestamp: '2023-12-12T00:00:00.000Z' + - timestamp: "2023-12-12T00:00:00.000Z" duration: 1 some-value: 1 zero-value: 0 diff --git a/package-lock.json b/package-lock.json index 80f89027a..43766c712 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,17 @@ { "name": "@grnsft/if", - "version": "0.6.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@grnsft/if", - "version": "0.6.0", + "version": "0.7.0", "license": "MIT", "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.16", + "@grnsft/if-core": "^0.0.25", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -1186,9 +1186,9 @@ } }, "node_modules/@grnsft/if-core": { - "version": "0.0.16", - "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.16.tgz", - "integrity": "sha512-Ep/YRk8rpFK7+kgD3iKon6PtY8jEj8H3ihYglw9Jli5lPszObwIMb4e6aHXmW2kcCndpBQKuSXaruGTgQ/d9ww==", + "version": "0.0.25", + "resolved": "https://registry.npmjs.org/@grnsft/if-core/-/if-core-0.0.25.tgz", + "integrity": "sha512-1W4SXsXhXos06q4SBPc8QpgQPDhHEc03njrGcd/X2UiJyh0ycBKTqGCjuRPRipEayGgUxO0DwRNOgNcgzzcDkA==", "dependencies": { "typescript": "^5.1.6", "zod": "^3.23.8" diff --git a/package.json b/package.json index f8bec2d83..6aaed3cf1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@grnsft/if", "description": "Impact Framework", - "version": "0.6.0", + "version": "0.7.0", "author": { "name": "Green Software Foundation", "email": "info@gsf.com" @@ -20,7 +20,7 @@ "dependencies": { "@commitlint/cli": "^18.6.0", "@commitlint/config-conventional": "^18.6.0", - "@grnsft/if-core": "^0.0.16", + "@grnsft/if-core": "^0.0.25", "axios": "^1.7.2", "csv-parse": "^5.5.6", "csv-stringify": "^6.4.6", @@ -58,11 +58,12 @@ "homepage": "https://greensoftware.foundation", "keywords": [ "engine", + "framework", "green software foundation", "greensoftware", "if", "impact", - "models" + "plugins" ], "license": "MIT", "publishConfig": { @@ -86,7 +87,6 @@ "lint": "gts lint", "pre-commit": "lint-staged", "prepare": "husky install", - "prepublish": "npm run build", "release": "release-it", "test": "jest --verbose --testPathPattern=src/__tests__/" }, diff --git a/scripts/impact-test.sh b/scripts/impact-test.sh deleted file mode 100755 index 2cefa3d17..000000000 --- a/scripts/impact-test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -echo "Starting impact tests" -prefix="examples/manifests/"; -for file in examples/manifests/*; -do -echo "" -echo executing $file, outfile is ${file#"$prefix"} -echo "" -npx ts-node ./src --manifest $file --output examples/outputs/${file#"$prefix"} -done -exit 0 diff --git a/scripts/run-yamls.sh b/scripts/run-yamls.sh deleted file mode 100644 index fbe394c6b..000000000 --- a/scripts/run-yamls.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -echo 'Running all manifests' - - -for f in ./examples/manifests/*.yml; do - echo "Processing $f file..."; - npm run if-run -- --manifest $f - done diff --git a/scripts/yaml-to-csv/yaml-to-csv.md b/scripts/yaml-to-csv/yaml-to-csv.md deleted file mode 100644 index 02979cb39..000000000 --- a/scripts/yaml-to-csv/yaml-to-csv.md +++ /dev/null @@ -1,73 +0,0 @@ -# YML to CSV shell plugin - -This is a shell plugin (python script) that produces a CSV file from an output YML file - -## Usage - -> python path/to/your/if-yaml-to-csv.py -c path/to/your/output.csv - -In this default usage: - -1. The script will listen on STDIN for input impl. -2. The output CSV will be created / overwritten in the specified path. -3. By default, only _timestamp_, _duration_, _energy_ and _carbon_ fields are projected as columns to the CSV file. To change this, see 'Optional arguments' - -### Optional arguments: - -> _-y_ - -Path to input yml file (output). Using this option will override the default input method of listening on STDIN. - -> _-p_ - -Comma separated (no spaces!) names of fields to project from input yml to output CSV as columns. Default = _timestamp,duration,energy,carbon_. Putting an emply list here (""") will project all output output fields. - -> _-j_ - -Left-join the resulting CSV data to existing CSV file. Boolean switch, no argument values. - -- Default join keys are _timestamp,duration_. To change this, use the _-jk_ option. -- In case of identical column names between existing and new data, new column names will be added with _"\_new"_ suffix. -- If there is no pre-existing file at the path specified under _-c_, _-j_ is ignored (along with other join-related options) - -> _-jk_ - -Comma separated (no spaces!) names of columns to join by. Default = _timestamp,duration_. Relevant only when using --join (-j) option. - -> _-js_ - -Suffix to add to ALL projected columns in the new data CSV data. Relevant only when using --join (-j) option. - -#### Example: - -> python path/to/your/if-yaml-to-csv.py -y path/to/your/output.yml -c path/to/your/output.csv -p timestamp,duration,energy,carbon,location -j -jk duration,location -js "\_MY_SUFFIX" - -This will: - -- Convert the content of _path/to/your/output.yml_ file into CSV format data, instead of listening to input on SDTIN. -- project the _location_ field to the CSV data, alongside the default timestamp, duration, energy and carbon fields. -- left-join the resulting CSV data to the data already existing in the path specified under _-c_, using duration, location columns as join keys. -- will add "\_MY_SUFFIX" suffix to ALL columns in the new CSV data. - -## Integrating the script in your IMPL as a shell plugin - - initialize: - plugins: - ... - ... - yaml-to-csv: - method: Shell - path: "@grnsft/if-plugins" - - graph: - children: - child: - pipeline: - ... - ... - - yaml-to-csv - config: - ... - ... - yaml-to-csv: - executable: python path/to/your/if-yaml-to-csv.py -c path/to/your/output.csv -j diff --git a/scripts/yaml-to-csv/yaml-to-csv.py b/scripts/yaml-to-csv/yaml-to-csv.py deleted file mode 100644 index aa269d9ed..000000000 --- a/scripts/yaml-to-csv/yaml-to-csv.py +++ /dev/null @@ -1,102 +0,0 @@ -import sys -import yaml -import pandas -import os -import argparse - - -default_projection_list = 'timestamp,duration,energy,carbon' -default_join_keys = 'timestamp,duration' - - -def parse_arguments(): - parser = argparse.ArgumentParser(description='Impact Framework yaml-to-csv parser') - parser.add_argument('-y', '--yml', type=str, help='Path to input yml file') - parser.add_argument('-c', '--csv', type=str, help='Path to output csv file') - parser.add_argument('-p', '--project', type=str, default=default_projection_list, help=f'Comma separated (no spaces!) names of fields to project from input yml to output CSV as columns. Default ={default_projection_list}') - parser.add_argument('-j', '--join', action='store_true', help='Join the resulting CSV data to existing CSV file') - parser.add_argument('-jk', '--join_keys', type=str, default=default_join_keys, help=f'Comma separated (no spaces!) names of columns to join by. Default ={default_join_keys}. Relevant only when using --join (-j) option') - parser.add_argument('-js', '--join_suffix', type=str, help='Suffix to add to projected columns in resulting CSV data. Relevant only when using --join (-j) option') - args = parser.parse_args() - return args - - -def get_yaml_data(input_yaml_path): - if input_yaml_path is not None: - return read_yaml_file(input_yaml_path) - else: - input_yaml_string = sys.stdin.read() - return input_yaml_string - - -def read_yaml_file(input_yaml): - try: - with open(input_yaml, 'r') as yaml_file: - yaml_string = yaml_file.read() - yaml_data = yaml.safe_load(yaml_string) - return yaml_data["graph"]["children"]["child"] - except FileNotFoundError: - print(f"Input YAML file '{input_yaml}' not found.") - sys.exit(1) - - -def read_and_project_yaml_data(yaml_data, projection_list): - yaml_obj = yaml.safe_load(yaml_data) - output_yaml_data = yaml_obj["inputs"] - outputs_df = pandas.json_normalize(output_yaml_data) - filtered_df = outputs_df[projection_list] if projection_list else outputs_df - return filtered_df - - -def write_to_csv_file(df_to_write, output_csv_path): - csv_data = df_to_write.to_csv() - with open(output_csv_path, 'w', newline='') as csv_file: - csv_file.write(csv_data) - - -def validate_file_exists(file_path): - if not os.path.exists(file_path): - raise Exception(f"unable to join: file {file_path} doesn't exist") - - -def rename_columns(df, join_keys, projection_list, suff): - cols_to_rename = list(filter(lambda x: x not in join_keys, projection_list)) - new_column_names = [col_name + "_" + suff for col_name in cols_to_rename] - rename_dict = dict(zip(cols_to_rename, new_column_names)) - df_result = df.rename(columns=rename_dict) - return df_result - - -def do_new(yaml_data, output_csv_path, projection_list): - filtered_df = read_and_project_yaml_data(yaml_data, projection_list) - write_to_csv_file(filtered_df, output_csv_path) - - -def do_join(yaml_data, output_csv_path, projection_list, join_keys, suff=""): - if not os.path.exists(output_csv_path): - do_new(yaml_data, output_csv_path, projection_list) - else: - filtered_df = read_and_project_yaml_data(yaml_data, projection_list) - if suff is None or len(suff) == 0: - outputs_df_to_join = filtered_df - else: - outputs_df_to_join = rename_columns(filtered_df, join_keys, projection_list, suff) - df_existing = pandas.read_csv(output_csv_path) - df_merged = df_existing.merge(outputs_df_to_join, on=join_keys, how="left", suffixes=("", "_new")) - write_to_csv_file(df_merged, output_csv_path) - - -args = parse_arguments() - -input_yaml_path = args.yml -output_csv_path = args.csv -projection_list = args.project.split(',') - -yaml_data = get_yaml_data(input_yaml_path) -if args.join: - join_keys = args.join_keys.split(',') - join_suffix = args.join_suffix - do_join(yaml_data, output_csv_path, projection_list, join_keys, join_suffix) -else: - do_new(yaml_data, output_csv_path, projection_list) -sys.stdout.write(str(yaml_data)) diff --git a/src/__mocks__/builtins/export-yaml.ts b/src/__mocks__/builtins/export-yaml.ts index 85f54e966..5d23a8705 100644 --- a/src/__mocks__/builtins/export-yaml.ts +++ b/src/__mocks__/builtins/export-yaml.ts @@ -6,7 +6,6 @@ export const tree = { children: { 'child-1': { pipeline: ['teads-curve', 'sum', 'sci-embodied', 'sci-o', 'sci'], - config: null, defaults: { 'cpu/thermal-design-power': 100, 'grid/carbon-intensity': 800, @@ -44,8 +43,8 @@ export const tree = { 'resources-reserved': 1, 'resources-total': 8, 'cpu/energy': 0.000008888888888888888, - "carbon-plus-energy'": 10.000008888888889, - 'carbon-embodied': 0.0000020256215119228817, + 'carbon-plus-energy': 10.000008888888889, + 'embodied-carbon': 0.0000020256215119228817, 'carbon-operational': 4000, carbon: 4000.0000020256216, sci: 240000.0001215373, @@ -55,7 +54,6 @@ export const tree = { }, 'child-2': { pipeline: ['teads-curve', 'sum', 'sci-embodied', 'sci-o', 'sci'], - config: null, defaults: { 'cpu/thermal-design-power': 100, 'grid/carbon-intensity': 800, @@ -94,7 +92,7 @@ export const tree = { 'resources-total': 8, 'cpu/energy': 0.00001650338753387534, "carbon-plus-energy'": 10.000016503387533, - 'carbon-embodied': 0.0000020256215119228817, + 'embodied-carbon': 0.0000020256215119228817, 'carbon-operational': 4000, carbon: 4000.0000020256216, sci: 240000.0001215373, @@ -126,14 +124,14 @@ export const context: Context = { 'teads-curve': { path: '@grnsft/if-unofficial-plugins', method: 'TeadsCurve', - 'global-config': { + config: { interpolation: 'spline', }, }, sum: { path: '@grnsft/if-plugins', method: 'Sum', - 'global-config': { + config: { 'input-parameters': ['cpu/energy', 'network/energy'], 'output-parameter': "carbon-plus-energy'", }, @@ -149,7 +147,7 @@ export const context: Context = { sci: { path: '@grnsft/if-plugins', method: 'Sci', - 'global-config': { + config: { 'functional-unit': 'requests', }, }, diff --git a/src/__mocks__/fs/index.ts b/src/__mocks__/fs/index.ts index a0520879c..61bd51adf 100644 --- a/src/__mocks__/fs/index.ts +++ b/src/__mocks__/fs/index.ts @@ -55,10 +55,9 @@ cpu-cores-available,cpu-cores-utilized,cpu-manufacturer,cpu-model-name,cpu-tdp,g pipeline: compute: - boavizta-cpu - config: - boavizta-cpu: - core-units: 24 - processor: Intel® Core™ i7-1185G7 + defaults: + core-units: 24 + processor: Intel® Core™ i7-1185G7 inputs: - timestamp: 2023-07-06T00:00 duration: 3600 # Secs diff --git a/src/__mocks__/mock-manifest.yaml b/src/__mocks__/mock-manifest.yaml index a69009d1e..1b9d38ac6 100644 --- a/src/__mocks__/mock-manifest.yaml +++ b/src/__mocks__/mock-manifest.yaml @@ -6,7 +6,7 @@ initialize: memory-energy-from-memory-util: path: builtin method: Coefficient - global-config: + config: input-parameter: memory/utilization coefficient: 0.0001 output-parameter: memory/energy @@ -59,7 +59,6 @@ tree: pipeline: compute: - memory-energy-from-memory-util - config: null inputs: - timestamp: 2023-12-12T00:00:00.000Z duration: 3600 diff --git a/src/__tests__/common/util/helpers.test.ts b/src/__tests__/common/util/helpers.test.ts index 92b4f9dab..11838100c 100644 --- a/src/__tests__/common/util/helpers.test.ts +++ b/src/__tests__/common/util/helpers.test.ts @@ -37,7 +37,7 @@ describe('common/util/helpers: ', () => { const response = await parseManifestFromStdin(); const expectedMessage = '\nname: mock-name\ndescription: mock-description\n'; - + expect.assertions(1); expect(response).toEqual(expectedMessage); }); }); diff --git a/src/__tests__/if-merge/util/helpers.test.ts b/src/__tests__/if-merge/util/helpers.test.ts index 1f4c0609f..9dd77f547 100644 --- a/src/__tests__/if-merge/util/helpers.test.ts +++ b/src/__tests__/if-merge/util/helpers.test.ts @@ -27,7 +27,7 @@ jest.mock('../../../if-run/builtins/export-yaml', () => ({ multiply: { path: 'builtin', method: 'Multiply', - 'global-config': { + config: { 'input-parameters': ['cpu/utilization', 'duration'], 'output-parameter': 'cpu-times-duration', }, @@ -50,10 +50,10 @@ jest.mock('../../../if-run/builtins/export-yaml', () => ({ })); describe('if-merge/util/helpers: ', () => { - const consopleSpy = jest.spyOn(global.console, 'log'); + const consoleSpy = jest.spyOn(global.console, 'log'); beforeEach(() => { - consopleSpy.mockReset(); + consoleSpy.mockReset(); }); describe('mergeManifests(): ', () => { @@ -103,7 +103,7 @@ describe('if-merge/util/helpers: ', () => { multiply: { path: 'builtin', method: 'Multiply', - 'global-config': { + config: { 'input-parameters': ['cpu/utilization', 'duration'], 'output-parameter': 'cpu-times-duration', }, @@ -149,7 +149,7 @@ describe('if-merge/util/helpers: ', () => { await mergeManifests(mockCommandArgs); expect.assertions(1); - expect(consopleSpy).toHaveBeenCalledTimes(1); + expect(consoleSpy).toHaveBeenCalledTimes(1); }); it('gets YAML files when there is only one manifest.', async () => { diff --git a/src/__tests__/if-run/builtins/CommonGenerator.test.ts b/src/__tests__/if-run/builtins/CommonGenerator.test.ts index 87b8457a8..62a1b52da 100644 --- a/src/__tests__/if-run/builtins/CommonGenerator.test.ts +++ b/src/__tests__/if-run/builtins/CommonGenerator.test.ts @@ -4,8 +4,8 @@ import {CommonGenerator} from '../../../if-run/builtins/mock-observations/helper import {STRINGS} from '../../../if-run/config'; -const {GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; describe('builtins/mock-observations/CommonGenerator: ', () => { describe('initialize: ', () => { @@ -17,7 +17,7 @@ describe('builtins/mock-observations/CommonGenerator: ', () => { try { commonGenerator.next([]); } catch (error) { - expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); + expect(error).toEqual(new ConfigError(MISSING_CONFIG)); } }); }); diff --git a/src/__tests__/if-run/builtins/RandIntGenerator.test.ts b/src/__tests__/if-run/builtins/RandIntGenerator.test.ts index 05fdb37f2..7ffbfb581 100644 --- a/src/__tests__/if-run/builtins/RandIntGenerator.test.ts +++ b/src/__tests__/if-run/builtins/RandIntGenerator.test.ts @@ -4,8 +4,8 @@ import {RandIntGenerator} from '../../../if-run/builtins/mock-observations/helpe import {STRINGS} from '../../../if-run/config'; -const {GlobalConfigError} = ERRORS; -const {INVALID_NAME, MISSING_MIN_MAX, MISSING_GLOBAL_CONFIG} = STRINGS; +const {ConfigError} = ERRORS; +const {INVALID_NAME, MISSING_MIN_MAX, MISSING_CONFIG} = STRINGS; describe('builtins/mock-observations/RandIntGenerator: ', () => { describe('initialize', () => { @@ -14,7 +14,7 @@ describe('builtins/mock-observations/RandIntGenerator: ', () => { try { RandIntGenerator('', {}); } catch (error) { - expect(error).toEqual(new GlobalConfigError(INVALID_NAME)); + expect(error).toEqual(new ConfigError(INVALID_NAME)); } }); @@ -23,7 +23,7 @@ describe('builtins/mock-observations/RandIntGenerator: ', () => { try { RandIntGenerator('generator-name', {}); } catch (error) { - expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); + expect(error).toEqual(new ConfigError(MISSING_CONFIG)); } }); @@ -35,7 +35,7 @@ describe('builtins/mock-observations/RandIntGenerator: ', () => { try { RandIntGenerator('random', config); } catch (error) { - expect(error).toEqual(new GlobalConfigError(MISSING_MIN_MAX)); + expect(error).toEqual(new ConfigError(MISSING_MIN_MAX)); } }); }); diff --git a/src/__tests__/if-run/builtins/coefficient.test.ts b/src/__tests__/if-run/builtins/coefficient.test.ts index 8d99c3e87..4a3f8562f 100644 --- a/src/__tests__/if-run/builtins/coefficient.test.ts +++ b/src/__tests__/if-run/builtins/coefficient.test.ts @@ -4,12 +4,12 @@ import {Coefficient} from '../../../if-run/builtins/coefficient'; import {STRINGS} from '../../../if-run/config'; -const {InputValidationError, GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; +const {InputValidationError, ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; describe('builtins/coefficient: ', () => { describe('Coefficient: ', () => { - const globalConfig = { + const config = { 'input-parameter': 'carbon', coefficient: 3, 'output-parameter': 'carbon-product', @@ -18,7 +18,7 @@ describe('builtins/coefficient: ', () => { inputs: {}, outputs: {}, }; - const coefficient = Coefficient(globalConfig, parametersMetadata); + const coefficient = Coefficient(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -28,7 +28,7 @@ describe('builtins/coefficient: ', () => { }); describe('execute(): ', () => { - it('successfully applies coefficient strategy to given input.', () => { + it('successfully applies coefficient strategy to given input.', async () => { expect.assertions(1); const expectedResult = [ @@ -40,7 +40,7 @@ describe('builtins/coefficient: ', () => { }, ]; - const result = coefficient.execute([ + const result = await coefficient.execute([ { duration: 3600, carbon: 3, @@ -53,14 +53,141 @@ describe('builtins/coefficient: ', () => { expect(result).toStrictEqual(expectedResult); }); - it('throws an error when global config is not provided.', () => { + it('succcessfully executes when the mapping has data.', async () => { + const mapping = { + carbon: 'carbon-for-production', + }; + + const coefficient = Coefficient(config, parametersMetadata, mapping); + expect.assertions(1); + + const expectedResult = [ + { + duration: 3600, + 'carbon-for-production': 3, + 'carbon-product': 9, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await coefficient.execute([ + { + duration: 3600, + 'carbon-for-production': 3, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect.assertions(1); + + expect(result).toStrictEqual(expectedResult); + }); + + it('succcessfully executes when the mapping map output parameter.', async () => { + const mapping = { + 'carbon-product': 'carbon-result', + }; + + const coefficient = Coefficient(config, parametersMetadata, mapping); + expect.assertions(1); + + const expectedResult = [ + { + duration: 3600, + carbon: 3, + 'carbon-result': 9, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await coefficient.execute([ + { + duration: 3600, + carbon: 3, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect.assertions(1); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executes when a parameter has an arithmetic expression.', async () => { + expect.assertions(1); + const config = { + 'input-parameter': '=3*carbon', + coefficient: 3, + 'output-parameter': 'carbon-product', + }; + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const coefficient = Coefficient(config, parametersMetadata, {}); + + const expectedResult = [ + { + duration: 3600, + carbon: 3, + 'carbon-product': 27, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await coefficient.execute([ + { + duration: 3600, + carbon: 3, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect.assertions(1); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error when the `coefficient` has wrong arithmetic expression.', async () => { + const config = { + 'input-parameter': 'carbon', + coefficient: 'mock-param', + 'output-parameter': 'carbon-product', + }; + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const coefficient = Coefficient(config, parametersMetadata, {}); + + expect.assertions(2); + + try { + await coefficient.execute([ + { + duration: 3600, + carbon: 'some-param', + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new InputValidationError( + '"coefficient" parameter is expected number, received string. Error code: invalid_type.' + ) + ); + } + }); + + it('throws an error when config is not provided.', async () => { const config = undefined; - const coefficient = Coefficient(config!, parametersMetadata); + const coefficient = Coefficient(config!, parametersMetadata, {}); expect.assertions(1); try { - coefficient.execute([ + await coefficient.execute([ { duration: 3600, timestamp: '2021-01-01T00:00:00Z', @@ -68,26 +195,24 @@ describe('builtins/coefficient: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual( - new GlobalConfigError(MISSING_GLOBAL_CONFIG) - ); + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); } }); - it('throws an error on missing `input-parameter` param in input.', () => { + it('throws an error on missing `input-parameter` param in input.', async () => { const invalidConfig = { 'input-parameter': '', coefficient: 3, 'output-parameter': 'carbon-product', }; - const coefficient = Coefficient(invalidConfig, parametersMetadata); + const coefficient = Coefficient(invalidConfig, parametersMetadata, {}); const expectedMessage = '"input-parameter" parameter is string must contain at least 1 character(s). Error code: too_small.'; expect.assertions(1); try { - coefficient.execute([ + await coefficient.execute([ { duration: 3600, timestamp: '2021-01-01T00:00:00Z', @@ -101,19 +226,19 @@ describe('builtins/coefficient: ', () => { } }); - it('throws an error on missing `output-parameter` param in input.', () => { + it('throws an error on missing `output-parameter` param in input.', async () => { const invalidConfig = { 'input-parameter': 'carbon', coefficient: 10, 'output-parameter': '', }; - const coefficient = Coefficient(invalidConfig, parametersMetadata); + const coefficient = Coefficient(invalidConfig, parametersMetadata, {}); const expectedMessage = '"output-parameter" parameter is string must contain at least 1 character(s). Error code: too_small.'; expect.assertions(1); try { - coefficient.execute([ + await coefficient.execute([ { duration: 3600, timestamp: '2021-01-01T00:00:00Z', diff --git a/src/__tests__/if-run/builtins/copy-param.test.ts b/src/__tests__/if-run/builtins/copy-param.test.ts index 952546505..6edfc7a17 100644 --- a/src/__tests__/if-run/builtins/copy-param.test.ts +++ b/src/__tests__/if-run/builtins/copy-param.test.ts @@ -4,12 +4,12 @@ import {Copy} from '../../../if-run/builtins/copy-param'; import {STRINGS} from '../../../if-run/config'; -const {GlobalConfigError, InputValidationError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; +const {ConfigError, InputValidationError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; describe('builtins/copy: ', () => { describe('Copy: ', () => { - const globalConfig = { + const config = { 'keep-existing': true, from: 'original', to: 'copy', @@ -18,7 +18,7 @@ describe('builtins/copy: ', () => { inputs: {}, outputs: {}, }; - const copy = Copy(globalConfig, parametersMetadata); + const copy = Copy(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -28,7 +28,7 @@ describe('builtins/copy: ', () => { }); describe('execute(): ', () => { - it('successfully applies Copy strategy to given input.', () => { + it('successfully applies Copy strategy to given input.', async () => { expect.assertions(1); const expectedResult = [ @@ -40,7 +40,7 @@ describe('builtins/copy: ', () => { }, ]; - const result = copy.execute([ + const result = await copy.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -51,14 +51,70 @@ describe('builtins/copy: ', () => { expect(result).toStrictEqual(expectedResult); }); - it('throws an error when global config is not provided.', () => { + it('successfully executed when `mapping` has valid data.', async () => { + expect.assertions(1); + + const mapping = { + original: 'from', + }; + + const copy = Copy(config, parametersMetadata, mapping); + const expectedResult = [ + { + duration: 3600, + from: 'hello', + copy: 'hello', + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await copy.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + from: 'hello', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executed when the `mapping` map output parameter.', async () => { + expect.assertions(1); + + const mapping = { + copy: 'result', + }; + + const copy = Copy(config, parametersMetadata, mapping); + const expectedResult = [ + { + duration: 3600, + original: 'hello', + result: 'hello', + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await copy.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + original: 'hello', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error when config is not provided.', async () => { const config = undefined; - const copy = Copy(config!, parametersMetadata); + const copy = Copy(config!, parametersMetadata, {}); expect.assertions(1); try { - copy.execute([ + await copy.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -66,23 +122,21 @@ describe('builtins/copy: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual( - new GlobalConfigError(MISSING_GLOBAL_CONFIG) - ); + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); } }); - it('throws an error on missing params in input.', () => { - const globalConfig = { + it('throws an error on missing params in input.', async () => { + const config = { 'keep-existing': true, from: 'original', to: 'copy', }; - const copy = Copy(globalConfig, parametersMetadata); + const copy = Copy(config, parametersMetadata, {}); expect.assertions(1); try { - copy.execute([ + await copy.execute([ { duration: 3600, timestamp: '2021-01-01T00:00:00Z', @@ -91,20 +145,21 @@ describe('builtins/copy: ', () => { } catch (error) { expect(error).toStrictEqual( new InputValidationError( - '"original" parameter is required. Error code: invalid_type.' + '"original" parameter is required. Error code: invalid_union.' ) ); } }); - it('does not persist the original value when keep-existing==false.', () => { + + it('does not persist the original value when keep-existing==false.', async () => { expect.assertions(1); - const globalConfig = { + const config = { 'keep-existing': false, from: 'original', to: 'copy', }; - const copy = Copy(globalConfig, parametersMetadata); + const copy = Copy(config, parametersMetadata, {}); const expectedResult = [ { duration: 3600, @@ -113,7 +168,7 @@ describe('builtins/copy: ', () => { }, ]; - const result = copy.execute([ + const result = await copy.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -123,6 +178,36 @@ describe('builtins/copy: ', () => { expect(result).toStrictEqual(expectedResult); }); + + it('successfully executes when the `from` contains arithmetic expression.', async () => { + const config = { + 'keep-existing': false, + from: '=3*size', + to: 'if-size', + }; + const copy = Copy(config, parametersMetadata, {}); + + const inputs = [ + { + timestamp: '2024-07-05T13:45:48.398Z', + duration: 3600, + size: 0.05, + }, + ]; + + const expectedResult = [ + { + timestamp: '2024-07-05T13:45:48.398Z', + duration: 3600, + 'if-size': 0.15000000000000002, + }, + ]; + + expect.assertions(1); + const result = await copy.execute(inputs); + + expect(result).toEqual(expectedResult); + }); }); }); }); diff --git a/src/__tests__/if-run/builtins/csv-lookup.test.ts b/src/__tests__/if-run/builtins/csv-lookup.test.ts index 49e4d45bb..6861f9c39 100644 --- a/src/__tests__/if-run/builtins/csv-lookup.test.ts +++ b/src/__tests__/if-run/builtins/csv-lookup.test.ts @@ -9,37 +9,39 @@ import {CSVLookup} from '../../../if-run/builtins'; import {STRINGS} from '../../../if-run/config'; const { - GlobalConfigError, + ConfigError, ReadFileError, FetchingFileError, QueryDataNotFoundError, MissingCSVColumnError, CSVParseError, } = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_CSV_COLUMN, NO_QUERY_DATA} = STRINGS; +const {MISSING_CONFIG, MISSING_CSV_COLUMN, NO_QUERY_DATA} = STRINGS; describe('builtins/CSVLookup: ', () => { - const parametersMetadata = { - inputs: {}, - outputs: {}, - }; const mock = new AxiosMockAdapter(axios); describe('CSVLookup: ', () => { + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; afterEach(() => { mock.reset(); }); describe('init: ', () => { it('successfully initalized.', () => { - const globalConfig = { + const config = { filepath: '', query: { 'cpu-cores-available': 'cpu/available', }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + + const csvLookup = CSVLookup(config, parametersMetadata, {}); + expect(csvLookup).toHaveProperty('metadata'); expect(csvLookup).toHaveProperty('execute'); }); @@ -48,7 +50,7 @@ describe('builtins/CSVLookup: ', () => { describe('execute(): ', () => { it('successfully applies CSVLookup `url` strategy to given input.', async () => { expect.assertions(1); - const globalConfig = { + const config = { filepath: 'https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv', query: { @@ -58,12 +60,12 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const responseData = `cpu-cores-available,cpu-cores-utilized,cpu-manufacturer,cpu-model-name,cpu-tdp,gpu-count,gpu-model-name,Hardware Information on AWS Documentation & Comments,instance-class,instance-storage,memory-available,platform-memory,release-date,storage-drives 16,8,AWS,AWS Graviton,150.00,N/A,N/A,AWS Graviton (ARM),a1.2xlarge,EBS-Only,16,32,November 2018,0 16,16,AWS,AWS Graviton,150.00,N/A,N/A,AWS Graviton (ARM),a1.4xlarge,EBS-Only,32,32,November 2018,0`; - mock.onGet(globalConfig.filepath).reply(200, responseData); + mock.onGet(config.filepath).reply(200, responseData); const result = await csvLookup.execute([ { @@ -88,7 +90,7 @@ describe('builtins/CSVLookup: ', () => { it('successfully applies CSVLookup `local file` strategy to given input.', async () => { expect.assertions(1); - const globalConfig = { + const config = { filepath: './file.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -97,8 +99,8 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const result = await csvLookup.execute([ { timestamp: '2024-03-01', @@ -120,8 +122,84 @@ describe('builtins/CSVLookup: ', () => { expect(result).toStrictEqual(expectedResult); }); + it('successfully executes when `mapping` has valid data.', async () => { + expect.assertions(1); + const config = { + filepath: './file.csv', + query: { + 'cpu-cores-available': 'cpu/available', + 'cpu-cores-utilized': 'cpu/utilized', + 'cpu-manufacturer': 'cpu/manufacturer', + }, + output: ['cpu-tdp', 'tdp'], + }; + const parameterMetadata = {inputs: {}, outputs: {}}; + const mapping = { + 'cpu/utilized': 'cpu/util', + }; + const csvLookup = CSVLookup(config, parameterMetadata, mapping); + + const result = await csvLookup.execute([ + { + timestamp: '2024-03-01', + 'cpu/available': 16, + 'cpu/util': 16, + 'cpu/manufacturer': 'AWS', + }, + ]); + const expectedResult = [ + { + timestamp: '2024-03-01', + 'cpu/available': 16, + 'cpu/util': 16, + 'cpu/manufacturer': 'AWS', + tdp: 150, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executes when the `mapping` map output parameter.', async () => { + expect.assertions(1); + const config = { + filepath: './file.csv', + query: { + 'cpu-cores-available': 'cpu/available', + 'cpu-cores-utilized': 'cpu/utilized', + 'cpu-manufacturer': 'cpu/manufacturer', + }, + output: ['cpu-tdp', 'tdp'], + }; + const parameterMetadata = {inputs: {}, outputs: {}}; + const mapping = { + tdp: 'tdp-finder', + }; + const csvLookup = CSVLookup(config, parameterMetadata, mapping); + + const result = await csvLookup.execute([ + { + timestamp: '2024-03-01', + 'cpu/available': 16, + 'cpu/utilized': 16, + 'cpu/manufacturer': 'AWS', + }, + ]); + const expectedResult = [ + { + timestamp: '2024-03-01', + 'cpu/available': 16, + 'cpu/utilized': 16, + 'cpu/manufacturer': 'AWS', + 'tdp-finder': 150, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); + it('rejects with file not found error.', async () => { - const globalConfig = { + const config = { filepath: './file-fail.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -130,7 +208,8 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + + const csvLookup = CSVLookup(config, parametersMetadata, {}); const input = [ { timestamp: '2024-03-01', @@ -150,7 +229,7 @@ describe('builtins/CSVLookup: ', () => { }); it('rejects with file not found error.', async () => { - const globalConfig = { + const config = { filepath: './file-fail.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -159,7 +238,7 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const input = [ { timestamp: '2024-03-01', @@ -179,7 +258,7 @@ describe('builtins/CSVLookup: ', () => { }); it('rejects with axios error.', async () => { - const globalConfig = { + const config = { filepath: 'https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv', query: { @@ -189,9 +268,9 @@ describe('builtins/CSVLookup: ', () => { }, output: ['cpu-tdp', 'tdp'], }; - mock.onGet(globalConfig.filepath).reply(404); + mock.onGet(config.filepath).reply(404); - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const input = [ { timestamp: '2024-03-01', @@ -212,7 +291,7 @@ describe('builtins/CSVLookup: ', () => { it('successfully applies CSVLookup if output is `*`.', async () => { expect.assertions(1); - const globalConfig = { + const config = { filepath: './file.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -221,8 +300,7 @@ describe('builtins/CSVLookup: ', () => { }, output: '*', }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); - + const csvLookup = CSVLookup(config, parametersMetadata, {}); const result = await csvLookup.execute([ { timestamp: '2024-03-01', @@ -257,7 +335,7 @@ describe('builtins/CSVLookup: ', () => { it('successfully applies CSVLookup if output is matrix.', async () => { expect.assertions(1); - const globalConfig = { + const config = { filepath: './file.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -269,8 +347,8 @@ describe('builtins/CSVLookup: ', () => { ['gpu-model-name', 'gpumodel'], ], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const result = await csvLookup.execute([ { timestamp: '2024-03-01', @@ -295,7 +373,7 @@ describe('builtins/CSVLookup: ', () => { it('successfully applies CSVLookup if output is exact string.', async () => { expect.assertions(1); - const globalConfig = { + const config = { filepath: './file.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -304,8 +382,8 @@ describe('builtins/CSVLookup: ', () => { }, output: 'gpu-count', }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const result = await csvLookup.execute([ { timestamp: '2024-03-01', @@ -329,7 +407,7 @@ describe('builtins/CSVLookup: ', () => { it('rejects with query data not found.', async () => { expect.assertions(2); - const globalConfig = { + const config = { filepath: './file.csv', query: { 'fail-cpu-cores-available': 'cpu/available', @@ -339,7 +417,7 @@ describe('builtins/CSVLookup: ', () => { output: ['cpu-tdp', 'tdp'], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const input = [ { timestamp: '2024-03-01', @@ -378,8 +456,8 @@ describe('builtins/CSVLookup: ', () => { await csvLookup.execute(input); } catch (error) { if (error instanceof Error) { - expect(error).toBeInstanceOf(GlobalConfigError); - expect(error.message).toEqual(MISSING_GLOBAL_CONFIG); + expect(error).toBeInstanceOf(ConfigError); + expect(error.message).toEqual(MISSING_CONFIG); } } }); @@ -387,7 +465,7 @@ describe('builtins/CSVLookup: ', () => { it('rejects with no such column in csv error.', async () => { expect.assertions(2); - const globalConfig = { + const config = { filepath: './file.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -396,7 +474,7 @@ describe('builtins/CSVLookup: ', () => { }, output: 'mock', }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const input = [ { timestamp: '2024-03-01', @@ -411,16 +489,14 @@ describe('builtins/CSVLookup: ', () => { } catch (error) { if (error instanceof Error) { expect(error).toBeInstanceOf(MissingCSVColumnError); - expect(error.message).toEqual( - MISSING_CSV_COLUMN(globalConfig.output) - ); + expect(error.message).toEqual(MISSING_CSV_COLUMN(config.output)); } } }); it('successfully applies CSVLookup if output is array with string.', async () => { expect.assertions(1); - const globalConfig = { + const config = { filepath: './file.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -429,8 +505,8 @@ describe('builtins/CSVLookup: ', () => { }, output: ['gpu-count'], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const result = await csvLookup.execute([ { timestamp: '2024-03-01', @@ -454,7 +530,7 @@ describe('builtins/CSVLookup: ', () => { it('successfully applies CSVLookup if output is matrix with strings.', async () => { expect.assertions(1); - const globalConfig = { + const config = { filepath: './file.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -463,8 +539,8 @@ describe('builtins/CSVLookup: ', () => { }, output: [['gpu-count']], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + const csvLookup = CSVLookup(config, parametersMetadata, {}); const result = await csvLookup.execute([ { timestamp: '2024-03-01', @@ -490,7 +566,7 @@ describe('builtins/CSVLookup: ', () => { it('rejects with CSV parse error', async () => { process.env.csv = 'fail'; expect.assertions(1); - const globalConfig = { + const config = { filepath: './fail-csv-reader.csv', query: { 'cpu-cores-available': 'cpu/available', @@ -499,7 +575,8 @@ describe('builtins/CSVLookup: ', () => { }, output: [['gpu-count']], }; - const csvLookup = CSVLookup(globalConfig, parametersMetadata); + + const csvLookup = CSVLookup(config, parametersMetadata, {}); try { await csvLookup.execute([ diff --git a/src/__tests__/if-run/builtins/divide.test.ts b/src/__tests__/if-run/builtins/divide.test.ts index e0a472998..ae48c4a61 100644 --- a/src/__tests__/if-run/builtins/divide.test.ts +++ b/src/__tests__/if-run/builtins/divide.test.ts @@ -4,12 +4,12 @@ import {Divide} from '../../../if-run/builtins'; import {STRINGS} from '../../../if-run/config'; -const {InputValidationError, GlobalConfigError, MissingInputDataError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA} = STRINGS; +const {InputValidationError, ConfigError, MissingInputDataError} = ERRORS; +const {MISSING_CONFIG, MISSING_INPUT_DATA} = STRINGS; describe('builtins/divide: ', () => { describe('Divide: ', () => { - const globalConfig = { + const config = { numerator: 'vcpus-allocated', denominator: 2, output: 'cpu/number-cores', @@ -18,7 +18,7 @@ describe('builtins/divide: ', () => { inputs: {}, outputs: {}, }; - const divide = Divide(globalConfig, parametersMetadata); + const divide = Divide(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -51,15 +51,71 @@ describe('builtins/divide: ', () => { expect(result).toStrictEqual(expectedResult); }); - it('returns a result when `denominator` is provded in input.', async () => { + it('successfully executes when `mapping` has valid data.', async () => { expect.assertions(1); - const globalConfig = { + const mapping = { + 'vcpus-allocated': 'vcpus-distributed', + }; + + const divide = Divide(config, parametersMetadata, mapping); + + const expectedResult = [ + { + duration: 3600, + 'vcpus-distributed': 24, + 'cpu/number-cores': 12, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await divide.execute([ + { + duration: 3600, + 'vcpus-distributed': 24, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executes when the `mapping` map output parameter.', async () => { + expect.assertions(1); + const mapping = { + 'cpu/number-cores': 'cpu-number-cores', + }; + + const divide = Divide(config, parametersMetadata, mapping); + + const expectedResult = [ + { + duration: 3600, + 'vcpus-allocated': 24, + 'cpu-number-cores': 12, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await divide.execute([ + { + duration: 3600, + 'vcpus-allocated': 24, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('returns a result when `denominator` is provided in input.', async () => { + expect.assertions(1); + const config = { numerator: 'vcpus-allocated', denominator: 'duration', output: 'vcpus-allocated-per-second', }; - const divide = Divide(globalConfig, parametersMetadata); + const divide = Divide(config, parametersMetadata, {}); const input = [ { timestamp: '2021-01-01T00:00:00Z', @@ -81,16 +137,76 @@ describe('builtins/divide: ', () => { expect(response).toEqual(expectedResult); }); + it('successfully executes when a parameter contains arithmetic expression.', async () => { + expect.assertions(1); + + const config = { + numerator: '=3*"vcpus-allocated"', + denominator: 'duration', + output: 'vcpus-allocated-per-second', + }; + + const divide = Divide(config, parametersMetadata, {}); + const input = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'vcpus-allocated': 24, + }, + ]; + const response = await divide.execute(input); + + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'vcpus-allocated': 24, + 'vcpus-allocated-per-second': 72 / 3600, + }, + ]; + + expect(response).toEqual(expectedResult); + }); + + it('throws an error the `numerator` parameter has wrong arithmetic expression.', async () => { + const config = { + numerator: '3*"vcpus-allocated"', + denominator: 'duration', + output: 'vcpus-allocated-per-second', + }; + + const divide = Divide(config, parametersMetadata, {}); + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'vcpus-allocated': 24, + }, + ]; + expect.assertions(2); + try { + await divide.execute(inputs); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new InputValidationError( + 'The `numerator` contains an invalid arithmetic expression. It should start with `=` and include the symbols `*`, `+`, `-` and `/`.' + ) + ); + } + }); + it('throws an error on missing params in input.', async () => { const expectedMessage = '"vcpus-allocated" parameter is required. Error code: invalid_type.'; - const globalConfig = { + const config = { numerator: 'vcpus-allocated', denominator: 3600, output: 'vcpus-allocated-per-second', }; - const divide = Divide(globalConfig, parametersMetadata); + + const divide = Divide(config, parametersMetadata, {}); expect.assertions(1); @@ -109,9 +225,9 @@ describe('builtins/divide: ', () => { }); }); - it('throws an error on missing global config.', async () => { + it('throws an error on missing config.', async () => { const config = undefined; - const divide = Divide(config!, parametersMetadata); + const divide = Divide(config!, parametersMetadata, {}); expect.assertions(1); @@ -123,19 +239,17 @@ describe('builtins/divide: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual( - new GlobalConfigError(MISSING_GLOBAL_CONFIG) - ); + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); } }); it('throws an error when `denominator` is 0.', async () => { - const globalConfig = { + const config = { numerator: 'vcpus-allocated', denominator: 0, output: 'vcpus-allocated-per-second', }; - const divide = Divide(globalConfig, parametersMetadata); + const divide = Divide(config, parametersMetadata, {}); expect.assertions(1); @@ -158,12 +272,13 @@ describe('builtins/divide: ', () => { }); it('throws an error when `denominator` is string.', async () => { - const globalConfig = { + const config = { numerator: 'vcpus-allocated', denominator: '10', output: 'vcpus-allocated-per-second', }; - const divide = Divide(globalConfig, parametersMetadata); + + const divide = Divide(config, parametersMetadata, {}); expect.assertions(1); @@ -177,9 +292,7 @@ describe('builtins/divide: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new MissingInputDataError( - MISSING_INPUT_DATA(globalConfig.denominator) - ) + new MissingInputDataError(MISSING_INPUT_DATA(config.denominator)) ); } }); diff --git a/src/__tests__/if-run/builtins/exponent.test.ts b/src/__tests__/if-run/builtins/exponent.test.ts index 2e0419686..3c2c977bf 100644 --- a/src/__tests__/if-run/builtins/exponent.test.ts +++ b/src/__tests__/if-run/builtins/exponent.test.ts @@ -1,12 +1,15 @@ import {ERRORS} from '@grnsft/if-core/utils'; +import {STRINGS} from '../../../if-run/config'; import {Exponent} from '../../../if-run/builtins/exponent'; -const {InputValidationError} = ERRORS; +const {InputValidationError, ConfigError} = ERRORS; + +const {MISSING_CONFIG} = STRINGS; describe('builtins/exponent: ', () => { describe('Exponent: ', () => { - const globalConfig = { + const config = { 'input-parameter': 'energy/base', exponent: 3, 'output-parameter': 'energy', @@ -15,7 +18,8 @@ describe('builtins/exponent: ', () => { inputs: {}, outputs: {}, }; - const exponent = Exponent(globalConfig, parametersMetadata); + + const exponent = Exponent(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -48,6 +52,70 @@ describe('builtins/exponent: ', () => { expect(result).toStrictEqual(expectedResult); }); + it('successfully executes when `mapping` has valid data.', async () => { + expect.assertions(1); + const mapping = { + 'energy/base': 'energy/main', + }; + const config = { + 'input-parameter': 'energy/base', + exponent: 3, + 'output-parameter': 'energy', + }; + const exponent = Exponent(config, parametersMetadata, mapping); + const expectedResult = [ + { + duration: 3600, + 'energy/main': 2, + energy: 8, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await exponent.execute([ + { + duration: 3600, + 'energy/main': 2, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executes when the `mapping` maps output parameter.', async () => { + expect.assertions(1); + + const mapping = { + energy: 'energy-result', + }; + const config = { + 'input-parameter': 'energy/base', + exponent: 3, + 'output-parameter': 'energy', + }; + + const exponent = Exponent(config, parametersMetadata, mapping); + const expectedResult = [ + { + duration: 3600, + 'energy/base': 2, + 'energy-result': 8, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await exponent.execute([ + { + duration: 3600, + 'energy/base': 2, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + it('throws an error on missing params in input.', async () => { expect.assertions(1); @@ -61,7 +129,7 @@ describe('builtins/exponent: ', () => { } catch (error) { expect(error).toStrictEqual( new InputValidationError( - '"input-parameter" parameter is required. Error code: invalid_type.' + '"energy/base" parameter is required. Error code: invalid_type.' ) ); } @@ -69,7 +137,7 @@ describe('builtins/exponent: ', () => { it('throws an error on input param value not numeric.', async () => { expect.assertions(1); - const input = [ + const inputs = [ { duration: 3600, 'energy/base': 'i-am-not-a-number', @@ -78,11 +146,11 @@ describe('builtins/exponent: ', () => { ]; try { - await exponent.execute(input); + await exponent.execute(inputs); } catch (error) { expect(error).toStrictEqual( new InputValidationError( - '"input-parameter" parameter is expected number, received string. Error code: invalid_type.' + '"energy/base" parameter is expected number, received string. Error code: invalid_type.' ) ); } @@ -95,7 +163,7 @@ describe('builtins/exponent: ', () => { exponent: 4, 'output-parameter': 'carbon', }; - const exponent = Exponent(newConfig, parametersMetadata); + const exponent = Exponent(newConfig, parametersMetadata, {}); const data = [ { @@ -117,6 +185,92 @@ describe('builtins/exponent: ', () => { expect(response).toEqual(expectedResult); }); + + it('successfully executes when a parameter contains arithmetic expression.', async () => { + const config = { + 'input-parameter': "=2*'energy/base'", + exponent: 3, + 'output-parameter': 'energy', + }; + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + + const exponent = Exponent(config, parametersMetadata, {}); + + expect.assertions(1); + + const expectedResult = [ + { + duration: 3600, + 'energy/base': 4, + energy: 512, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await exponent.execute([ + { + duration: 3600, + 'energy/base': 4, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error when the `exponent` has wrong arithmetic expression.', async () => { + const config = { + 'input-parameter': "=2*'energy/base'", + exponent: "3*'mock-param'", + 'output-parameter': 'energy', + }; + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + + const exponent = Exponent(config, parametersMetadata, {}); + + expect.assertions(2); + + try { + await exponent.execute([ + { + duration: 3600, + 'energy/base': 4, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new InputValidationError( + 'The `exponent` contains an invalid arithmetic expression. It should start with `=` and include the symbols `*`, `+`, `-` and `/`.' + ) + ); + } + }); + }); + + it('throws an error on missing config.', async () => { + const config = undefined; + const exponent = Exponent(config!, parametersMetadata, {}); + + expect.assertions(1); + + try { + await exponent.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + }, + ]); + } catch (error) { + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); + } }); }); }); diff --git a/src/__tests__/if-run/builtins/interpolation.test.ts b/src/__tests__/if-run/builtins/interpolation.test.ts index 6634556dd..be379b3d3 100644 --- a/src/__tests__/if-run/builtins/interpolation.test.ts +++ b/src/__tests__/if-run/builtins/interpolation.test.ts @@ -5,17 +5,13 @@ import {Interpolation} from '../../../if-run/builtins'; import {STRINGS} from '../../../if-run/config'; -const {InputValidationError, GlobalConfigError} = ERRORS; -const { - MISSING_GLOBAL_CONFIG, - WITHIN_THE_RANGE, - ARRAY_LENGTH_NON_EMPTY, - X_Y_EQUAL, -} = STRINGS; +const {InputValidationError, ConfigError} = ERRORS; +const {MISSING_CONFIG, WITHIN_THE_RANGE, ARRAY_LENGTH_NON_EMPTY, X_Y_EQUAL} = + STRINGS; describe('builtins/interpolation: ', () => { describe('Interpolation: ', () => { - const globalConfig = { + const config = { method: Method.LINEAR, x: [0, 10, 50, 100], y: [0.12, 0.32, 0.75, 1.02], @@ -33,7 +29,8 @@ describe('builtins/interpolation: ', () => { 'cpu/utilization': 45, }, ]; - const plugin = Interpolation(globalConfig, parametersMetadata); + + const plugin = Interpolation(config, parametersMetadata, {}); describe('init Interpolation: ', () => { it('initalizes object with properties.', async () => { @@ -43,7 +40,7 @@ describe('builtins/interpolation: ', () => { }); describe('execute(): ', () => { - it('returns result when all parameters are valid.', () => { + it('returns result when all parameters are valid.', async () => { const outputs = [ { timestamp: '2023-07-06T00:00', @@ -53,18 +50,86 @@ describe('builtins/interpolation: ', () => { }, ]; - expect(plugin.execute(inputs)).toEqual(outputs); + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); }); - it('returns result when the `method` is not provided in the global config.', () => { - const globalConfig = { + it('returns result when `mapping` has valid data.', async () => { + const mapping = { + 'cpu/utilization': 'cpu/util', + }; + const config = { + method: Method.LINEAR, + x: [0, 10, 50, 100], + y: [0.12, 0.32, 0.75, 1.02], + 'input-parameter': 'cpu/utilization', + 'output-parameter': 'result', + }; + const inputs = [ + { + timestamp: '2023-07-06T00:00', + duration: 3600, + 'cpu/util': 45, + }, + ]; + const plugin = Interpolation(config, parametersMetadata, mapping); + const outputs = [ + { + timestamp: '2023-07-06T00:00', + duration: 3600, + 'cpu/util': 45, + result: 0.69625, + }, + ]; + + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); + }); + + it('returns result when the `mapping` maps output parameter.', async () => { + const mapping = { + 'interpolation-result': 'result', + }; + const config = { + method: Method.LINEAR, + x: [0, 10, 50, 100], + y: [0.12, 0.32, 0.75, 1.02], + 'input-parameter': 'cpu/utilization', + 'output-parameter': 'interpolation-result', + }; + const inputs = [ + { + timestamp: '2023-07-06T00:00', + duration: 3600, + 'cpu/utilization': 45, + }, + ]; + const plugin = Interpolation(config, parametersMetadata, mapping); + const outputs = [ + { + timestamp: '2023-07-06T00:00', + duration: 3600, + 'cpu/utilization': 45, + result: 0.69625, + }, + ]; + + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); + }); + + it('returns result when the `method` is not provided in the config.', async () => { + const config = { x: [0, 10, 50, 100], y: [0.12, 0.32, 0.75, 1.02], 'input-parameter': 'cpu/utilization', 'output-parameter': 'interpolation-result', }; - const plugin = Interpolation(globalConfig, parametersMetadata); + const plugin = Interpolation(config, parametersMetadata, {}); const outputs = [ { timestamp: '2023-07-06T00:00', @@ -74,12 +139,14 @@ describe('builtins/interpolation: ', () => { }, ]; - expect(plugin.execute(inputs)).toEqual(outputs); + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); }); - it('returns result when the `method` is `spline`.', () => { - const config = Object.assign({}, globalConfig, {method: Method.SPLINE}); - const plugin = Interpolation(config, parametersMetadata); + it('returns result when the `method` is `spline`.', async () => { + const newConfig = Object.assign({}, config, {method: Method.SPLINE}); + const plugin = Interpolation(newConfig, parametersMetadata, {}); const outputs = [ { @@ -90,14 +157,16 @@ describe('builtins/interpolation: ', () => { }, ]; - expect(plugin.execute(inputs)).toEqual(outputs); + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); }); - it('returns result when the `method` is `polynomial`.', () => { - const config = Object.assign({}, globalConfig, { + it('returns result when the `method` is `polynomial`.', async () => { + const newConfig = Object.assign({}, config, { method: Method.POLYNOMIAL, }); - const plugin = Interpolation(config, parametersMetadata); + const plugin = Interpolation(newConfig, parametersMetadata, {}); const outputs = [ { @@ -108,15 +177,16 @@ describe('builtins/interpolation: ', () => { }, ]; - expect(plugin.execute(inputs)).toEqual(outputs); + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); }); - it('returns result when the elements of `x` is not in acsending order.', () => { - const config = Object.assign({}, globalConfig, { + it('returns result when the elements of `x` is not in acsending order.', async () => { + const newConfig = Object.assign({}, config, { x: [0, 10, 100, 50], }); - const plugin = Interpolation(config, parametersMetadata); - + const plugin = Interpolation(newConfig, parametersMetadata, {}); const outputs = [ { timestamp: '2023-07-06T00:00', @@ -126,10 +196,12 @@ describe('builtins/interpolation: ', () => { }, ]; - expect(plugin.execute(inputs)).toEqual(outputs); + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); }); - it('returns result when the `cpu/utilization` is equal to one of the `x` points element.', () => { + it('returns result when the `cpu/utilization` is equal to one of the `x` points element.', async () => { const inputs = [ { timestamp: '2023-07-06T00:00', @@ -146,39 +218,103 @@ describe('builtins/interpolation: ', () => { }, ]; - expect(plugin.execute(inputs)).toEqual(outputs); + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); + }); + + it('successfully executes when the config parameter contains an arithmetic expression.', async () => { + const config = { + method: Method.LINEAR, + x: [0, 10, 50, 100], + y: [0.12, 0.32, 0.75, 1.02], + 'input-parameter': "=2*'cpu/utilization'", + 'output-parameter': 'interpolation-result', + }; + const inputs = [ + { + timestamp: '2023-07-06T00:00', + duration: 3600, + 'cpu/utilization': 90, + }, + ]; + + const plugin = Interpolation(config, parametersMetadata, {}); + const outputs = [ + { + timestamp: '2023-07-06T00:00', + duration: 3600, + 'cpu/utilization': 90, + 'interpolation-result': 0, + }, + ]; + + expect.assertions(1); + const result = await plugin.execute(inputs); + + expect(result).toEqual(outputs); + }); + + it('throws an error the config parameter contains wrong arithmetic expression.', async () => { + const config = { + method: Method.LINEAR, + x: [0, 10, 50, 100], + y: [0.12, 0.32, 0.75, 1.02], + 'input-parameter': "2*'cpu/utilization'", + 'output-parameter': 'interpolation-result', + }; + const inputs = [ + { + timestamp: '2023-07-06T00:00', + duration: 3600, + 'cpu/utilization': 90, + }, + ]; + + const plugin = Interpolation(config, parametersMetadata, {}); + + expect.assertions(2); + try { + await plugin.execute(inputs); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new InputValidationError( + 'The `input-parameter` contains an invalid arithmetic expression. It should start with `=` and include the symbols `*`, `+`, `-` and `/`.' + ) + ); + } }); - it('throws an when the global config is not provided.', () => { + it('throws an when the config is not provided.', async () => { const config = undefined; - const plugin = Interpolation(config!, parametersMetadata); + const plugin = Interpolation(config!, parametersMetadata, {}); expect.assertions(2); try { - plugin.execute(inputs); + await plugin.execute(inputs); } catch (error) { - expect(error).toBeInstanceOf(GlobalConfigError); - expect(error).toEqual(new GlobalConfigError(MISSING_GLOBAL_CONFIG)); + expect(error).toBeInstanceOf(ConfigError); + expect(error).toEqual(new ConfigError(MISSING_CONFIG)); } }); - it('throws an error when `x` and `y` points not equal.', () => { - const config = Object.assign({}, globalConfig, { + it('throws an error when `x` and `y` points not equal.', async () => { + const newConfig = Object.assign({}, config, { x: [0, 10, 100], }); - - const plugin = Interpolation(config, parametersMetadata); + const plugin = Interpolation(newConfig, parametersMetadata, {}); expect.assertions(2); try { - plugin.execute(inputs); + await plugin.execute(inputs); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual(new InputValidationError(X_Y_EQUAL)); } }); - it('throws an error when `cpu/utilization` is out of the range of `x` elements.', () => { + it('throws an error when `cpu/utilization` is out of the range of `x` elements.', async () => { const inputs = [ { timestamp: '2023-07-06T00:00', @@ -188,21 +324,23 @@ describe('builtins/interpolation: ', () => { ]; expect.assertions(2); try { - plugin.execute(inputs); + await plugin.execute(inputs); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual(new InputValidationError(WITHIN_THE_RANGE)); } }); - it('throws an error when the the length of the input arrays is <2', () => { - const globalConfig = { + + it('throws an error when the the length of the input arrays is <2', async () => { + const basicConfig = { x: [0], y: [0.12], 'input-parameter': 'cpu/utilization', 'output-parameter': 'interpolation-result', }; - const config = Object.assign({}, globalConfig, {method: Method.SPLINE}); - const plugin = Interpolation(config, parametersMetadata); + + const config = Object.assign({}, basicConfig, {method: Method.SPLINE}); + const plugin = Interpolation(config, parametersMetadata, {}); const inputs = [ { timestamp: '2023-07-06T00:00', @@ -212,7 +350,7 @@ describe('builtins/interpolation: ', () => { ]; expect.assertions(2); try { - plugin.execute(inputs); + await plugin.execute(inputs); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( diff --git a/src/__tests__/if-run/builtins/mock-observations.test.ts b/src/__tests__/if-run/builtins/mock-observations.test.ts index 0569b4c93..046a066f6 100644 --- a/src/__tests__/if-run/builtins/mock-observations.test.ts +++ b/src/__tests__/if-run/builtins/mock-observations.test.ts @@ -4,7 +4,7 @@ import {MockObservations} from '../../../if-run/builtins/mock-observations'; import {STRINGS} from '../../../if-run/config'; -const {InputValidationError, GlobalConfigError} = ERRORS; +const {InputValidationError, ConfigError} = ERRORS; const {INVALID_MIN_MAX} = STRINGS; describe('builtins/mock-observations: ', () => { @@ -12,28 +12,25 @@ describe('builtins/mock-observations: ', () => { inputs: {}, outputs: {}, }; - describe('init: ', () => { it('successfully initalized.', () => { - const mockObservations = MockObservations( - { - 'timestamp-from': '2023-07-06T00:00', - 'timestamp-to': '2023-07-06T00:01', - duration: 5, - components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], - generators: { - common: { - region: 'uk-west', - 'common-key': 'common-val', - }, - randint: { - 'cpu/utilization': {min: 10, max: 95}, - 'memory/utilization': {min: 10, max: 85}, - }, + const config = { + 'timestamp-from': '2023-07-06T00:00', + 'timestamp-to': '2023-07-06T00:01', + duration: 5, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, }, }, - parametersMetadata - ); + }; + const mockObservations = MockObservations(config, parametersMetadata, {}); expect(mockObservations).toHaveProperty('metadata'); expect(mockObservations).toHaveProperty('execute'); @@ -57,7 +54,7 @@ describe('builtins/mock-observations: ', () => { }, }, }; - const mockObservations = MockObservations(config, parametersMetadata); + const mockObservations = MockObservations(config, parametersMetadata, {}); const result = await mockObservations.execute([]); expect.assertions(1); @@ -98,6 +95,70 @@ describe('builtins/mock-observations: ', () => { ]); }); + it('executes successfully when `mapping` is provided.', async () => { + const config = { + 'timestamp-from': '2023-07-06T00:00', + 'timestamp-to': '2023-07-06T00:01', + duration: 30, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/util': {min: 10, max: 11}, + }, + }, + }; + const mapping = { + 'cpu/utilization': 'cpu/util', + }; + const mockObservations = MockObservations( + config, + parametersMetadata, + mapping + ); + const result = await mockObservations.execute([]); + + expect.assertions(1); + + expect(result).toStrictEqual([ + { + timestamp: '2023-07-06T00:00:00.000Z', + duration: 30, + 'common-key': 'common-val', + 'instance-type': 'A1', + region: 'uk-west', + 'cpu/util': 10, + }, + { + timestamp: '2023-07-06T00:00:30.000Z', + duration: 30, + 'common-key': 'common-val', + 'instance-type': 'A1', + region: 'uk-west', + 'cpu/util': 10, + }, + { + timestamp: '2023-07-06T00:00:00.000Z', + duration: 30, + 'common-key': 'common-val', + 'instance-type': 'B1', + region: 'uk-west', + 'cpu/util': 10, + }, + { + timestamp: '2023-07-06T00:00:30.000Z', + duration: 30, + 'common-key': 'common-val', + 'instance-type': 'B1', + region: 'uk-west', + 'cpu/util': 10, + }, + ]); + }); + it('throws an error when the `min` is greater then `max` of `randint` config.', async () => { const config = { 'timestamp-from': '2023-07-06T00:00', @@ -117,13 +178,13 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); - const mockObservations = MockObservations(config, parametersMetadata); + const mockObservations = MockObservations(config, parametersMetadata, {}); try { await mockObservations.execute([]); } catch (error) { - expect(error).toBeInstanceOf(GlobalConfigError); + expect(error).toBeInstanceOf(ConfigError); expect(error).toEqual( - new GlobalConfigError(INVALID_MIN_MAX('cpu/utilization')) + new ConfigError(INVALID_MIN_MAX('cpu/utilization')) ); } }); @@ -135,11 +196,14 @@ describe('builtins/mock-observations: ', () => { duration: 5, components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], }; - expect.assertions(2); try { - const mockObservations = MockObservations(config, parametersMetadata); + const mockObservations = MockObservations( + config, + parametersMetadata, + {} + ); await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -169,11 +233,14 @@ describe('builtins/mock-observations: ', () => { }, }, }; - expect.assertions(2); try { - const mockObservations = MockObservations(config, parametersMetadata); + const mockObservations = MockObservations( + config, + parametersMetadata, + {} + ); await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -185,23 +252,25 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); try { - const mockObservations = MockObservations( - { - 'timestamp-from': '2023-07-06T00:00', - 'timestamp-to': '2023-07-06T00:01', - components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], - generators: { - common: { - region: 'uk-west', - 'common-key': 'common-val', - }, - randint: { - 'cpu/utilization': {min: 10, max: 95}, - 'memory/utilization': {min: 10, max: 85}, - }, + const config = { + 'timestamp-from': '2023-07-06T00:00', + 'timestamp-to': '2023-07-06T00:01', + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, }, }, - parametersMetadata + }; + const mockObservations = MockObservations( + config, + parametersMetadata, + {} ); await mockObservations.execute([]); } catch (error) { @@ -218,25 +287,26 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); try { - const mockObservations = MockObservations( - { - 'timestamp-from': '2023-07-06T00:00', - duration: 5, - components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], - generators: { - common: { - region: 'uk-west', - 'common-key': 'common-val', - }, - randint: { - 'cpu/utilization': {min: 10, max: 95}, - 'memory/utilization': {min: 10, max: 85}, - }, + const config = { + 'timestamp-from': '2023-07-06T00:00', + duration: 5, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, }, }, - parametersMetadata + }; + const mockObservations = MockObservations( + config, + parametersMetadata, + {} ); - await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -252,25 +322,26 @@ describe('builtins/mock-observations: ', () => { expect.assertions(2); try { - const mockObservations = MockObservations( - { - 'timestamp-to': '2023-07-06T00:01', - duration: 5, - components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], - generators: { - common: { - region: 'uk-west', - 'common-key': 'common-val', - }, - randint: { - 'cpu/utilization': {min: 10, max: 95}, - 'memory/utilization': {min: 10, max: 85}, - }, + const config = { + 'timestamp-to': '2023-07-06T00:01', + duration: 5, + components: [{'instance-type': 'A1'}, {'instance-type': 'B1'}], + generators: { + common: { + region: 'uk-west', + 'common-key': 'common-val', + }, + randint: { + 'cpu/utilization': {min: 10, max: 95}, + 'memory/utilization': {min: 10, max: 85}, }, }, - parametersMetadata + }; + const mockObservations = MockObservations( + config, + parametersMetadata, + {} ); - await mockObservations.execute([]); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); @@ -296,7 +367,7 @@ describe('builtins/mock-observations: ', () => { randint: null, }, }; - const mockObservations = MockObservations(config, parametersMetadata); + const mockObservations = MockObservations(config, parametersMetadata, {}); expect.assertions(2); @@ -326,7 +397,7 @@ describe('builtins/mock-observations: ', () => { }, }, }; - const mockObservations = MockObservations(config, parametersMetadata); + const mockObservations = MockObservations(config, parametersMetadata, {}); expect.assertions(2); diff --git a/src/__tests__/if-run/builtins/multiply.test.ts b/src/__tests__/if-run/builtins/multiply.test.ts index b3856dfd4..d7b7a9147 100644 --- a/src/__tests__/if-run/builtins/multiply.test.ts +++ b/src/__tests__/if-run/builtins/multiply.test.ts @@ -1,12 +1,16 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Multiply} from '../../../if-run/builtins/multiply'; +import {STRINGS} from '../../../if-run/config'; -const {InputValidationError} = ERRORS; +const {InputValidationError, ConfigError, WrongArithmeticExpressionError} = + ERRORS; + +const {MISSING_CONFIG} = STRINGS; describe('builtins/multiply: ', () => { describe('Multiply: ', () => { - const globalConfig = { + const config = { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], 'output-parameter': 'energy', }; @@ -14,7 +18,8 @@ describe('builtins/multiply: ', () => { inputs: {}, outputs: {}, }; - const multiply = Multiply(globalConfig, parametersMetadata); + + const multiply = Multiply(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -51,6 +56,78 @@ describe('builtins/multiply: ', () => { expect(result).toStrictEqual(expectedResult); }); + it('successfully executes when `mapping` is provided.', async () => { + expect.assertions(1); + const mapping = { + 'cpu/energy': 'energy-from-cpu', + 'network/energy': 'energy-from-network', + 'memory/energy': 'energy-from-memory', + }; + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': 'energy', + }; + const multiply = Multiply(config, parametersMetadata, mapping); + + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-from-cpu': 2, + 'energy-from-network': 2, + 'energy-from-memory': 2, + energy: 8, + }, + ]; + + const result = await multiply.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-from-cpu': 2, + 'energy-from-network': 2, + 'energy-from-memory': 2, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executes when the `mapping` maps output parameter.', async () => { + expect.assertions(1); + const mapping = { + energy: 'total/energy', + }; + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': 'energy', + }; + const multiply = Multiply(config, parametersMetadata, mapping); + + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 2, + 'network/energy': 2, + 'memory/energy': 2, + 'total/energy': 8, + }, + ]; + + const result = await multiply.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 2, + 'network/energy': 2, + 'memory/energy': 2, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + it('throws an error on missing params in input.', async () => { expect.assertions(1); @@ -76,7 +153,7 @@ describe('builtins/multiply: ', () => { 'input-parameters': ['carbon', 'other-carbon'], 'output-parameter': 'carbon-product', }; - const multiply = Multiply(newConfig, parametersMetadata); + const multiply = Multiply(newConfig, parametersMetadata, {}); const data = [ { @@ -100,6 +177,87 @@ describe('builtins/multiply: ', () => { expect(response).toEqual(expectedResult); }); + + it('successfully executes when the config output parameter contains arithmetic expression.', async () => { + expect.assertions(1); + + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': '=2*energy', + }; + const multiply = Multiply(config, parametersMetadata, {}); + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 2, + 'network/energy': 2, + 'memory/energy': 2, + }, + ]; + const response = await multiply.execute(inputs); + + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 2, + 'network/energy': 2, + 'memory/energy': 2, + energy: 16, + }, + ]; + + expect(response).toEqual(expectedResult); + }); + + it('throws an error the config output parameter has wrong arithmetic expression.', async () => { + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': '2*energy', + }; + + const multiply = Multiply(config, parametersMetadata, {}); + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 2, + 'network/energy': 2, + 'memory/energy': 2, + }, + ]; + expect.assertions(2); + + try { + await multiply.execute(inputs); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new WrongArithmeticExpressionError( + 'The output parameter `2*energy` contains an invalid arithmetic expression. It should start with `=` and include the symbols `*`, `+`, `-` and `/`.' + ) + ); + } + }); + + it('throws an error on missing config.', async () => { + const config = undefined; + const multiply = Multiply(config!, parametersMetadata, {}); + + expect.assertions(1); + + try { + await multiply.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + }, + ]); + } catch (error) { + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); + } + }); }); }); }); diff --git a/src/__tests__/if-run/builtins/regex.test.ts b/src/__tests__/if-run/builtins/regex.test.ts index ea45c49d1..d509c8918 100644 --- a/src/__tests__/if-run/builtins/regex.test.ts +++ b/src/__tests__/if-run/builtins/regex.test.ts @@ -4,12 +4,12 @@ import {Regex} from '../../../if-run/builtins/regex'; import {STRINGS} from '../../../if-run/config'; -const {GlobalConfigError, MissingInputDataError, RegexMismatchError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; +const {ConfigError, MissingInputDataError, RegexMismatchError} = ERRORS; +const {MISSING_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; describe('builtins/regex: ', () => { describe('Regex: ', () => { - const globalConfig = { + const config = { parameter: 'physical-processor', match: '^[^,]+', output: 'cpu/name', @@ -18,7 +18,7 @@ describe('builtins/regex: ', () => { inputs: {}, outputs: {}, }; - const regex = Regex(globalConfig, parametersMetadata); + const regex = Regex(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -53,14 +53,13 @@ describe('builtins/regex: ', () => { expect(result).toStrictEqual(expectedResult); }); - it('successfully applies regex strategy with multiple matches', async () => { - const globalConfig = { + it('successfully applies regex strategy with multiple matches.', async () => { + const config = { parameter: 'cloud/instance-type', match: '/(?<=_)[^_]+?(?=_|$)/g', output: 'cloud/instance-type', }; - const regex = Regex(globalConfig, parametersMetadata); - + const regex = Regex(config, parametersMetadata, {}); const expectedResult = [ { timestamp: '2023-08-06T00:00', @@ -80,18 +79,80 @@ describe('builtins/regex: ', () => { expect(result).toStrictEqual(expectedResult); }); + it('successfully applies regex when `mapping` has valid data.', async () => { + const config = { + parameter: 'cloud/instance-type', + match: '/(?<=_)[^_]+?(?=_|$)/g', + output: 'cloud/instance-type', + }; + + const mapping = { + 'cloud/instance-type': 'instance-type', + }; + const regex = Regex(config, parametersMetadata, mapping); + + const expectedResult = [ + { + timestamp: '2023-08-06T00:00', + duration: 3600, + 'instance-type': 'DS1 v2', + }, + ]; + + const result = await regex.execute([ + { + timestamp: '2023-08-06T00:00', + duration: 3600, + 'instance-type': 'Standard_DS1_v2', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully applies regex when the `mapping` maps output parameter.', async () => { + const config = { + parameter: 'cloud/instance-type', + match: '/(?<=_)[^_]+?(?=_|$)/g', + output: 'cloud/instance-name', + }; + + const mapping = { + 'cloud/instance-name': 'instance-name', + }; + const regex = Regex(config, parametersMetadata, mapping); + + const expectedResult = [ + { + timestamp: '2023-08-06T00:00', + duration: 3600, + 'cloud/instance-type': 'Standard_DS1_v2', + 'instance-name': 'DS1 v2', + }, + ]; + + const result = await regex.execute([ + { + timestamp: '2023-08-06T00:00', + duration: 3600, + 'cloud/instance-type': 'Standard_DS1_v2', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + it('returns a result when regex is not started and ended with ``.', async () => { const physicalProcessor = 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz'; expect.assertions(1); - const globalConfig = { + const config = { parameter: 'physical-processor', match: '[^,]+/', output: 'cpu/name', }; - const regex = Regex(globalConfig, parametersMetadata); - + const regex = Regex(config, parametersMetadata, {}); const expectedResult = [ { timestamp: '2021-01-01T00:00:00Z', @@ -116,12 +177,13 @@ describe('builtins/regex: ', () => { const physicalProcessor = 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz'; - const globalConfig = { + const config = { parameter: 'physical-processor', match: '^(^:)+', output: 'cpu/name', }; - const regex = Regex(globalConfig, parametersMetadata); + + const regex = Regex(config, parametersMetadata, {}); expect.assertions(1); @@ -142,9 +204,9 @@ describe('builtins/regex: ', () => { } }); - it('throws an error on missing global config.', async () => { + it('throws an error on missing config.', async () => { const config = undefined; - const regex = Regex(config!, parametersMetadata); + const regex = Regex(config!, parametersMetadata, {}); expect.assertions(1); @@ -156,9 +218,7 @@ describe('builtins/regex: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual( - new GlobalConfigError(MISSING_GLOBAL_CONFIG) - ); + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); } }); diff --git a/src/__tests__/if-run/builtins/sci-embodied.test.ts b/src/__tests__/if-run/builtins/sci-embodied.test.ts index 0e2c234f7..33261ea30 100644 --- a/src/__tests__/if-run/builtins/sci-embodied.test.ts +++ b/src/__tests__/if-run/builtins/sci-embodied.test.ts @@ -2,10 +2,7 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {SciEmbodied} from '../../../if-run/builtins/sci-embodied'; -import {STRINGS} from '../../../if-run/config'; - const {InputValidationError} = ERRORS; -const {SCI_EMBODIED_ERROR} = STRINGS; describe('builtins/sci-embodied:', () => { describe('SciEmbodied: ', () => { @@ -13,7 +10,7 @@ describe('builtins/sci-embodied:', () => { inputs: {}, outputs: {}, }; - const sciEmbodied = SciEmbodied(parametersMetadata); + const sciEmbodied = SciEmbodied({}, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -27,19 +24,13 @@ describe('builtins/sci-embodied:', () => { const inputs = [ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 1, - 'resources-total': 1, + duration: 3600, + vCPUs: 2, }, { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30 * 2, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 1, - 'resources-total': 1, + duration: 3600, + vCPUs: 4, }, ]; @@ -50,75 +41,44 @@ describe('builtins/sci-embodied:', () => { expect(result).toStrictEqual([ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 1, - 'resources-total': 1, - 'carbon-embodied': 4.10958904109589, - }, - { - timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30 * 2, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 1, - 'resources-total': 1, - 'carbon-embodied': 4.10958904109589 * 2, - }, - ]); - }); - - it('returns a result when `vcpus-allocated` and `vcpus-total` are in the input.', async () => { - const inputs = [ - { - timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'vcpus-allocated': 1, - 'vcpus-total': 1, + duration: 3600, + vCPUs: 2, + memory: 16, + gpu: 0, + hdd: 0, + ssd: 0, + 'usage-ratio': 1, + 'embodied-carbon': 31.39269406392694, }, - ]; - - const result = await sciEmbodied.execute(inputs); - - expect.assertions(1); - - expect(result).toStrictEqual([ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'vcpus-allocated': 1, - 'vcpus-total': 1, - 'carbon-embodied': 4.10958904109589, + duration: 3600, + vCPUs: 4, + memory: 16, + gpu: 0, + hdd: 0, + ssd: 0, + 'usage-ratio': 1, + 'embodied-carbon': 37.10045662100457, }, ]); }); - it('returns a result when `vcpus-allocated` and `vcpus-total` are preferred to `resources-reserved` and `resources-total`.', async () => { + it('executes when `mapping` has valid data.', async () => { + const mapping = { + vCPUs: 'device/cpu-cores', + }; + const sciEmbodied = SciEmbodied({}, parametersMetadata, mapping); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 2, - 'resources-total': 2, - 'vcpus-allocated': 1, - 'vcpus-total': 1, + duration: 3600, + 'device/cpu-cores': 1, }, { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30 * 2, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'vcpus-allocated': 1, - 'vcpus-total': 1, - 'resources-reserved': 2, - 'resources-total': 2, + duration: 3600, + 'device/cpu-cores': 2, }, ]; @@ -129,46 +89,42 @@ describe('builtins/sci-embodied:', () => { expect(result).toStrictEqual([ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'vcpus-allocated': 1, - 'vcpus-total': 1, - 'carbon-embodied': 4.10958904109589, - 'resources-reserved': 2, - 'resources-total': 2, + duration: 3600, + memory: 16, + gpu: 0, + hdd: 0, + ssd: 0, + 'usage-ratio': 1, + 'device/cpu-cores': 1, + 'embodied-carbon': 28.538812785388128, }, { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30 * 2, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'vcpus-allocated': 1, - 'vcpus-total': 1, - 'carbon-embodied': 4.10958904109589 * 2, - 'resources-reserved': 2, - 'resources-total': 2, + duration: 3600, + 'device/cpu-cores': 2, + memory: 16, + gpu: 0, + hdd: 0, + ssd: 0, + 'usage-ratio': 1, + 'embodied-carbon': 31.39269406392694, }, ]); }); - it('returns a result when `vcpus-allocated` and `vcpus-total` are miised.', async () => { + it('executes when the `mapping` maps output parameter.', async () => { + const mapping = { + 'embodied-carbon': 'carbon', + }; + const sciEmbodied = SciEmbodied({}, parametersMetadata, mapping); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 1, - 'resources-total': 1, + duration: 3600, }, { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30 * 2, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 1, - 'resources-total': 1, + duration: 3600, }, ]; @@ -179,42 +135,40 @@ describe('builtins/sci-embodied:', () => { expect(result).toStrictEqual([ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'carbon-embodied': 4.10958904109589, - 'resources-reserved': 1, - 'resources-total': 1, + duration: 3600, + vCPUs: 1, + memory: 16, + gpu: 0, + hdd: 0, + ssd: 0, + 'usage-ratio': 1, + carbon: 28.538812785388128, }, { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30 * 2, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'carbon-embodied': 4.10958904109589 * 2, - 'resources-reserved': 1, - 'resources-total': 1, + duration: 3600, + vCPUs: 1, + memory: 16, + gpu: 0, + hdd: 0, + ssd: 0, + 'usage-ratio': 1, + carbon: 28.538812785388128, }, ]); }); - it('throws an error when `device/emissions-embodied` is string.', async () => { + it('throws an error when `vCPUs` is string.', async () => { const inputs = [ { timestamp: '2021-01-01T00:00:00Z', duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': '10,00', - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 1, - 'resources-total': 1, + vCPUs: 'string', }, { timestamp: '2021-01-01T00:00:00Z', duration: 60 * 60 * 24 * 30 * 2, - 'device/emissions-embodied': 200, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'resources-reserved': 1, - 'resources-total': 1, + vCPUs: 'string', }, ]; @@ -224,69 +178,94 @@ describe('builtins/sci-embodied:', () => { } catch (error) { expect(error).toStrictEqual( new InputValidationError( - `"device/emissions-embodied" parameter is ${SCI_EMBODIED_ERROR( - 'gco2e' - )}. Error code: invalid_union.` + '"vCPUs" parameter is expected number, received string at index 0. Error code: invalid_type.' ) ); expect(error).toBeInstanceOf(InputValidationError); } }); - it('throws an exception on missing `device/emissions-embodied`.', async () => { - const errorMessage = - '"device/emissions-embodied" parameter is required. Error code: invalid_union.'; + it('successfully executes when a parameter contains arithmetic expression.', async () => { + const config = { + 'baseline-vcpus': 1, + 'baseline-memory': 16, + lifespan: 157680000, + 'baseline-emissions': 2000000, + 'vcpu-emissions-constant': 100000, + 'memory-emissions-constant': 1172, + 'ssd-emissions-constant': 50000, + 'hdd-emissions-constant': 1 * 100000, + 'gpu-emissions-constant': '= 2 * "mock-param"', + 'output-parameter': 'embodied-carbon', + }; + const sciEmbodied = SciEmbodied(config, parametersMetadata, {}); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, - 'vcpus-allocated': 1, - 'vcpus-total': 1, + duration: 3600, + vCPUs: 2, + 'mock-param': 150000, + }, + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + vCPUs: 4, + 'mock-param': 100000, }, ]; - expect.assertions(2); + const result = await sciEmbodied.execute(inputs); - try { - await sciEmbodied.execute(inputs); - } catch (error) { - expect(error).toStrictEqual(new InputValidationError(errorMessage)); - expect(error).toBeInstanceOf(InputValidationError); - } - }); + expect.assertions(1); - it('throws an exception on missing `device/expected-lifespan`.', async () => { - const errorMessage = - '"device/expected-lifespan" parameter is required. Error code: invalid_union.'; - const inputs = [ + expect(result).toStrictEqual([ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': 200, - 'vcpus-allocated': 1, - 'vcpus-total': 1, + duration: 3600, + vCPUs: 2, + gpu: 0, + hdd: 0, + memory: 16, + ssd: 0, + 'usage-ratio': 1, + 'embodied-carbon': 47.945205479452056, + 'mock-param': 150000, }, - ]; - - expect.assertions(2); - - try { - await sciEmbodied.execute(inputs); - } catch (error) { - expect(error).toStrictEqual(new InputValidationError(errorMessage)); - expect(error).toBeInstanceOf(InputValidationError); - } + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + vCPUs: 4, + gpu: 0, + hdd: 0, + memory: 16, + ssd: 0, + 'usage-ratio': 1, + 'embodied-carbon': 52.51141552511416, + 'mock-param': 100000, + }, + ]); }); - it('throws an exception on invalid values.', async () => { + it('throws an error the `gpu-emissions-constant` parameter has wrong arithmetic expression.', async () => { + const config = { + 'baseline-vcpus': 1, + 'baseline-memory': 16, + lifespan: 157680000, + 'baseline-emissions': 2000000, + 'vcpu-emissions-constant': 100000, + 'memory-emissions-constant': 1172, + 'ssd-emissions-constant': 50000, + 'hdd-emissions-constant': 1 * 100000, + 'gpu-emissions-constant': '2 * "mock-param"', + 'output-parameter': 'embodied-carbon', + }; + const sciEmbodied = SciEmbodied(config, parametersMetadata, {}); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', - duration: 60 * 60 * 24 * 30, - 'device/emissions-embodied': '200', - 'vcpus-allocated': true, - 'vcpus-total': 1, + duration: 3600, + vCPUs: 2, + 'mock-param': 150000, }, ]; @@ -295,12 +274,10 @@ describe('builtins/sci-embodied:', () => { try { await sciEmbodied.execute(inputs); } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toStrictEqual( + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( new InputValidationError( - `"device/emissions-embodied" parameter is ${SCI_EMBODIED_ERROR( - 'gco2e' - )}. Error code: invalid_union.` + 'The `gpu-emissions-constant` contains an invalid arithmetic expression. It should start with `=` and include the symbols `*`, `+`, `-` and `/`.' ) ); } diff --git a/src/__tests__/if-run/builtins/sci.test.ts b/src/__tests__/if-run/builtins/sci.test.ts index 0360149a9..4d4621ad4 100644 --- a/src/__tests__/if-run/builtins/sci.test.ts +++ b/src/__tests__/if-run/builtins/sci.test.ts @@ -2,15 +2,17 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Sci} from '../../../if-run/builtins/sci'; -const {MissingInputDataError} = ERRORS; +import {STRINGS} from '../../../if-run/config'; + +const {MissingInputDataError, ConfigError, InputValidationError} = ERRORS; + +const {MISSING_CONFIG} = STRINGS; describe('builtins/sci:', () => { describe('Sci: ', () => { - const parametersMetadata = { - inputs: {}, - outputs: {}, - }; - const sci = Sci({'functional-unit': 'users'}, parametersMetadata); + const config = {'functional-unit': 'users'}; + const parametersMetadata = {inputs: {}, outputs: {}}; + const sci = Sci(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -21,13 +23,21 @@ describe('builtins/sci:', () => { describe('execute():', () => { it('returns a result with valid inputs.', async () => { - const sci = Sci( + const inputs = [ { - 'functional-unit': 'users', + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.02, + 'carbon-embodied': 5, + carbon: 5.02, + users: 100, + duration: 1, }, - parametersMetadata - ); - const inputs = [ + ]; + const result = await sci.execute(inputs); + + expect.assertions(1); + + expect(result).toStrictEqual([ { timestamp: '2021-01-01T00:00:00Z', 'carbon-operational': 0.02, @@ -35,6 +45,24 @@ describe('builtins/sci:', () => { carbon: 5.02, users: 100, duration: 1, + sci: 0.050199999999999995, + }, + ]); + }); + + it('successfully executes when `mapping` has valid data.', async () => { + const mapping = { + 'carbon-footprint': 'carbon-embodied', + }; + const sci = Sci(config, parametersMetadata, mapping); + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 1, + 'carbon-operational': 0.02, + 'carbon-embodied': 5, + carbon: 5.02, + users: 100, }, ]; const result = await sci.execute(inputs); @@ -54,13 +82,41 @@ describe('builtins/sci:', () => { ]); }); - it('returns the same result regardless of input duration.', async () => { - const sci = Sci( + it('successfully executes when the `mapping` maps output parameter.', async () => { + const mapping = { + sci: 'sci-result', + }; + const sci = Sci(config, parametersMetadata, mapping); + const inputs = [ { - 'functional-unit': 'requests', + timestamp: '2021-01-01T00:00:00Z', + duration: 1, + 'carbon-operational': 0.02, + 'carbon-footprint': 5, + carbon: 5.02, + users: 100, }, - parametersMetadata - ); + ]; + const result = await sci.execute(inputs); + + expect.assertions(1); + + expect(result).toStrictEqual([ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.02, + 'carbon-footprint': 5, + carbon: 5.02, + users: 100, + duration: 1, + 'sci-result': 0.050199999999999995, + }, + ]); + }); + + it('returns the same result regardless of input duration.', async () => { + const config = {'functional-unit': 'requests'}; + const sci = Sci(config, parametersMetadata, {}); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -106,12 +162,8 @@ describe('builtins/sci:', () => { }); it('throws exception on invalid functional unit data.', async () => { - const sci = Sci( - { - 'functional-unit': 'requests', - }, - parametersMetadata - ); + const config = {'functional-unit': 'requests'}; + const sci = Sci(config, parametersMetadata, {}); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -131,12 +183,8 @@ describe('builtins/sci:', () => { }); it('throws exception if functional unit value is not positive integer.', async () => { - const sci = Sci( - { - 'functional-unit': 'requests', - }, - parametersMetadata - ); + const config = {'functional-unit': 'requests'}; + const sci = Sci(config, parametersMetadata, {}); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -158,12 +206,8 @@ describe('builtins/sci:', () => { }); it('fallbacks to carbon value, if functional unit is 0.', async () => { - const sci = Sci( - { - 'functional-unit': 'requests', - }, - parametersMetadata - ); + const config = {'functional-unit': 'requests'}; + const sci = Sci(config, parametersMetadata, {}); const inputs = [ { timestamp: '2021-01-01T00:00:00Z', @@ -180,5 +224,85 @@ describe('builtins/sci:', () => { expect(result).toStrictEqual([{...inputs[0], sci: inputs[0].carbon}]); }); + + it('throws an error on missing config.', async () => { + const config = undefined; + const sci = Sci(config!, parametersMetadata, {}); + + expect.assertions(1); + + try { + await sci.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + }, + ]); + } catch (error) { + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); + } + }); + + it('successfully executes when a parameter contains arithmetic expression.', async () => { + const config = {'functional-unit': '=10*users'}; + const sci = Sci(config, parametersMetadata, {}); + expect.assertions(1); + + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.02, + 'carbon-embodied': 5, + carbon: 5.02, + users: 100, + duration: 1, + }, + ]; + const result = await sci.execute(inputs); + + expect.assertions(1); + expect(result).toStrictEqual([ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.02, + 'carbon-embodied': 5, + carbon: 5.02, + users: 100, + duration: 1, + // eslint-disable-next-line @typescript-eslint/no-loss-of-precision + sci: 0.0050199999999999995, + }, + ]); + }); + + it('throws an error the `functional-unit` parameter has wrong arithmetic expression.', async () => { + const config = {'functional-unit': '10*users'}; + const sci = Sci(config, parametersMetadata, {}); + expect.assertions(1); + + const inputs = [ + { + timestamp: '2021-01-01T00:00:00Z', + 'carbon-operational': 0.02, + 'carbon-embodied': 5, + carbon: 5.02, + users: 100, + duration: 1, + }, + ]; + + expect.assertions(2); + + try { + await sci.execute(inputs); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new InputValidationError( + 'The `functional-unit` contains an invalid arithmetic expression. It should start with `=` and include the symbols `*`, `+`, `-` and `/`.' + ) + ); + } + }); }); }); diff --git a/src/__tests__/if-run/builtins/shell.test.ts b/src/__tests__/if-run/builtins/shell.test.ts index 20b6a7e1c..d295b21a4 100644 --- a/src/__tests__/if-run/builtins/shell.test.ts +++ b/src/__tests__/if-run/builtins/shell.test.ts @@ -10,12 +10,13 @@ jest.mock('child_process'); jest.mock('js-yaml'); describe('builtins/shell', () => { - const parametersMetadata = { - inputs: {}, - outputs: {}, - }; describe('Shell', () => { - const shell = Shell({}, parametersMetadata); + const config = {command: 'python3 /path/to/script.py'}; + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const shell = Shell(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -25,11 +26,7 @@ describe('builtins/shell', () => { }); describe('execute(): ', () => { - it('execute with valid inputs and command', async () => { - const shell = Shell( - {command: 'python3 /path/to/script.py'}, - parametersMetadata - ); + it('executes with valid inputs and command.', async () => { const mockSpawnSync = spawnSync as jest.MockedFunction< typeof spawnSync >; @@ -59,11 +56,13 @@ describe('builtins/shell', () => { expect(mockLoadAll).toHaveBeenCalledWith('mocked stdout'); }); - it('throw an error if validation fails', async () => { + it('throws an error if validation fails.', async () => { + const shell = Shell({}, parametersMetadata, {}); const invalidInputs = [ {duration: 3600, timestamp: '2022-01-01T00:00:00Z', command: 123}, ]; + expect.assertions(2); try { await shell.execute(invalidInputs); } catch (error) { @@ -76,11 +75,8 @@ describe('builtins/shell', () => { } }); - it('throw an error when shell could not run command.', async () => { - const shell = Shell( - {command: 'python3 /path/to/script.py'}, - parametersMetadata - ); + it('throws an error when shell could not run command.', async () => { + const shell = Shell(config, parametersMetadata, {}); (spawnSync as jest.Mock).mockImplementation(() => { throw new InputValidationError('Could not run the command'); }); diff --git a/src/__tests__/if-run/builtins/subtract.test.ts b/src/__tests__/if-run/builtins/subtract.test.ts index 134cebfa7..3e283f51e 100644 --- a/src/__tests__/if-run/builtins/subtract.test.ts +++ b/src/__tests__/if-run/builtins/subtract.test.ts @@ -2,11 +2,16 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Subtract} from '../../../if-run/builtins/subtract'; -const {InputValidationError} = ERRORS; +import {STRINGS} from '../../../if-run/config'; + +const {InputValidationError, ConfigError, WrongArithmeticExpressionError} = + ERRORS; + +const {MISSING_CONFIG} = STRINGS; describe('builtins/subtract: ', () => { describe('Subtract: ', () => { - const globalConfig = { + const config = { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], 'output-parameter': 'energy/diff', }; @@ -14,7 +19,7 @@ describe('builtins/subtract: ', () => { inputs: {}, outputs: {}, }; - const subtract = Subtract(globalConfig, parametersMetadata); + const subtract = Subtract(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -51,6 +56,76 @@ describe('builtins/subtract: ', () => { expect(result).toStrictEqual(expectedResult); }); + it('successfully executes when `mapping` is provided.', async () => { + const mapping = { + 'cpu/energy': 'energy-for-cpu', + }; + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': 'energy/diff', + }; + const subtract = Subtract(config, parametersMetadata, mapping); + expect.assertions(1); + + const expectedResult = [ + { + duration: 3600, + 'energy-for-cpu': 4, + 'network/energy': 2, + 'memory/energy': 1, + 'energy/diff': 1, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await subtract.execute([ + { + duration: 3600, + 'energy-for-cpu': 4, + 'network/energy': 2, + 'memory/energy': 1, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executes when the `mapping` maps output parameter.', async () => { + const mapping = { + 'energy/diff': 'diff/energy', + }; + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': 'energy/diff', + }; + const subtract = Subtract(config, parametersMetadata, mapping); + expect.assertions(1); + + const expectedResult = [ + { + duration: 3600, + 'cpu/energy': 4, + 'network/energy': 2, + 'memory/energy': 1, + 'diff/energy': 1, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await subtract.execute([ + { + duration: 3600, + 'cpu/energy': 4, + 'network/energy': 2, + 'memory/energy': 1, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + it('throws an error on missing params in input.', async () => { expect.assertions(1); @@ -76,7 +151,7 @@ describe('builtins/subtract: ', () => { 'input-parameters': ['carbon', 'other-carbon'], 'output-parameter': 'carbon-diff', }; - const subtract = Subtract(newConfig, parametersMetadata); + const subtract = Subtract(newConfig, parametersMetadata, {}); const data = [ { @@ -101,5 +176,87 @@ describe('builtins/subtract: ', () => { expect(response).toEqual(expectedResult); }); }); + + it('successfully executes when the config output parameter contains an arithmetic expression.', async () => { + expect.assertions(1); + + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': "= 2 * 'energy/diff'", + }; + const subtract = Subtract(config, parametersMetadata, {}); + + const expectedResult = [ + { + duration: 3600, + 'cpu/energy': 4, + 'network/energy': 2, + 'memory/energy': 1, + 'energy/diff': 2, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await subtract.execute([ + { + duration: 3600, + 'cpu/energy': 4, + 'network/energy': 2, + 'memory/energy': 1, + timestamp: '2021-01-01T00:00:00Z', + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error the config output parameter has wrong arithmetic expression.', async () => { + expect.assertions(2); + + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': "=2 & 'energy/diff'", + }; + const subtract = Subtract(config, parametersMetadata, {}); + + const inputs = [ + { + duration: 3600, + 'cpu/energy': 4, + 'network/energy': 2, + 'memory/energy': 1, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + try { + await subtract.execute(inputs); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new WrongArithmeticExpressionError( + "The output parameter `=2 & 'energy/diff'` contains an invalid arithmetic expression. It should start with `=` and include the symbols `*`, `+`, `-` and `/`." + ) + ); + } + }); + + it('throws an error on missing config.', async () => { + const config = undefined; + const subtract = Subtract(config!, parametersMetadata, {}); + + expect.assertions(1); + + try { + await subtract.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + }, + ]); + } catch (error) { + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); + } + }); }); }); diff --git a/src/__tests__/if-run/builtins/sum.test.ts b/src/__tests__/if-run/builtins/sum.test.ts index 9ccc64378..31e49a27b 100644 --- a/src/__tests__/if-run/builtins/sum.test.ts +++ b/src/__tests__/if-run/builtins/sum.test.ts @@ -2,14 +2,11 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {Sum} from '../../../if-run/builtins/sum'; -import {STRINGS} from '../../../if-run/config'; - -const {GlobalConfigError, InputValidationError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; +const {InputValidationError, WrongArithmeticExpressionError} = ERRORS; describe('builtins/sum: ', () => { describe('Sum: ', () => { - const globalConfig = { + const config = { 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], 'output-parameter': 'energy', }; @@ -17,7 +14,7 @@ describe('builtins/sum: ', () => { inputs: {}, outputs: {}, }; - const sum = Sum(globalConfig, parametersMetadata); + const sum = Sum(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -27,7 +24,7 @@ describe('builtins/sum: ', () => { }); describe('execute(): ', () => { - it('successfully applies Sum strategy to given input.', () => { + it('successfully applies Sum strategy to given input.', async () => { expect.assertions(1); const expectedResult = [ @@ -41,7 +38,7 @@ describe('builtins/sum: ', () => { }, ]; - const result = sum.execute([ + const result = await sum.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -54,14 +51,89 @@ describe('builtins/sum: ', () => { expect(result).toStrictEqual(expectedResult); }); - it('throws an error when global config is not provided.', () => { + it('successfully executes when `mapping` has valid data.', async () => { + expect.assertions(1); + + const mapping = { + 'cpu/energy': 'energy-from-cpu', + 'network/energy': 'energy-from-network', + }; + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': 'energy', + }; + + const sum = Sum(config, parametersMetadata, mapping); + + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-from-cpu': 1, + 'energy-from-network': 1, + 'memory/energy': 1, + energy: 3, + }, + ]; + + const result = await sum.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-from-cpu': 1, + 'energy-from-network': 1, + 'memory/energy': 1, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executes when the `mapping` maps output parameter.', async () => { + expect.assertions(1); + + const mapping = { + energy: 'total/energy', + }; + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': 'energy', + }; + + const sum = Sum(config, parametersMetadata, mapping); + + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 1, + 'network/energy': 1, + 'memory/energy': 1, + 'total/energy': 3, + }, + ]; + + const result = await sum.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 1, + 'network/energy': 1, + 'memory/energy': 1, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error when config is not provided.', async () => { const config = undefined; - const sum = Sum(config!, parametersMetadata); + const sum = Sum(config!, parametersMetadata, {}); expect.assertions(1); try { - sum.execute([ + await sum.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -72,16 +144,18 @@ describe('builtins/sum: ', () => { ]); } catch (error) { expect(error).toStrictEqual( - new GlobalConfigError(MISSING_GLOBAL_CONFIG) + new InputValidationError( + '"input-parameters" parameter is required. Error code: invalid_type.,"output-parameter" parameter is required. Error code: invalid_type.' + ) ); } }); - it('throws an error on missing params in input.', () => { + it('throws an error on missing params in input.', async () => { expect.assertions(1); try { - sum.execute([ + await sum.execute([ { duration: 3600, timestamp: '2021-01-01T00:00:00Z', @@ -96,13 +170,13 @@ describe('builtins/sum: ', () => { } }); - it('returns a result with input params not related to energy.', () => { + it('returns a result with input params not related to energy.', async () => { expect.assertions(1); const newConfig = { 'input-parameters': ['carbon', 'other-carbon'], 'output-parameter': 'carbon-sum', }; - const sum = Sum(newConfig, parametersMetadata); + const sum = Sum(newConfig, parametersMetadata, {}); const data = [ { @@ -112,7 +186,7 @@ describe('builtins/sum: ', () => { 'other-carbon': 2, }, ]; - const response = sum.execute(data); + const response = await sum.execute(data); const expectedResult = [ { @@ -126,6 +200,69 @@ describe('builtins/sum: ', () => { expect(response).toEqual(expectedResult); }); + + it('successfully executes when the config output parameter contains an arithmetic expression.', async () => { + expect.assertions(1); + + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': "=2*'energy'", + }; + + const sum = Sum(config, parametersMetadata, {}); + const expectedResult = [ + { + duration: 3600, + 'cpu/energy': 1, + 'network/energy': 1, + 'memory/energy': 1, + energy: 6, + timestamp: '2021-01-01T00:00:00Z', + }, + ]; + + const result = await sum.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 1, + 'network/energy': 1, + 'memory/energy': 1, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error the config output parameter has wrong arithmetic expression.', async () => { + expect.assertions(2); + + const config = { + 'input-parameters': ['cpu/energy', 'network/energy', 'memory/energy'], + 'output-parameter': "2*'energy'", + }; + + const sum = Sum(config, parametersMetadata, {}); + + try { + await sum.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'cpu/energy': 1, + 'network/energy': 1, + 'memory/energy': 1, + }, + ]); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new WrongArithmeticExpressionError( + `The output parameter \`${config['output-parameter']}\` contains an invalid arithmetic expression. It should start with \`=\` and include the symbols \`*\`, \`+\`, \`-\` and \`/\`.` + ) + ); + } + }); }); }); }); diff --git a/src/__tests__/if-run/builtins/time-converter.test.ts b/src/__tests__/if-run/builtins/time-converter.test.ts index ae55daf5d..8b2ce8ea1 100644 --- a/src/__tests__/if-run/builtins/time-converter.test.ts +++ b/src/__tests__/if-run/builtins/time-converter.test.ts @@ -4,12 +4,12 @@ import {TimeConverter} from '../../../if-run/builtins/time-converter'; import {STRINGS} from '../../../if-run/config'; -const {GlobalConfigError, InputValidationError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; +const {ConfigError, InputValidationError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; describe('builtins/time-converter: ', () => { describe('TimeConverter: ', () => { - const globalConfig = { + const config = { 'input-parameter': 'energy-per-year', 'original-time-unit': 'year', 'new-time-unit': 'duration', @@ -19,7 +19,7 @@ describe('builtins/time-converter: ', () => { inputs: {}, outputs: {}, }; - const timeConverter = TimeConverter(globalConfig, parametersMetadata); + const timeConverter = TimeConverter(config, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -29,7 +29,7 @@ describe('builtins/time-converter: ', () => { }); describe('execute(): ', () => { - it('successfully applies TimeConverter strategy to given input.', () => { + it('successfully applies TimeConverter strategy to given input.', async () => { expect.assertions(1); const expectedResult = [ @@ -41,7 +41,7 @@ describe('builtins/time-converter: ', () => { }, ]; - const result = timeConverter.execute([ + const result = await timeConverter.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -52,14 +52,76 @@ describe('builtins/time-converter: ', () => { expect(result).toStrictEqual(expectedResult); }); - it('throws an error when global config is not provided.', () => { + it('successfully executes when the `mapping` is not empty object.', async () => { + expect.assertions(1); + + const mapping = { + 'energy-per-year': 'energy/year', + }; + const timeConverter = TimeConverter( + config, + parametersMetadata, + mapping + ); + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy/year': 10000, + 'energy-per-duration': 1.140795, + }, + ]; + + const result = await timeConverter.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy/year': 10000, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('successfully executes when the `mapping` maps output parameter.', async () => { + expect.assertions(1); + + const mapping = { + 'energy-per-duration': 'energy/duration', + }; + const timeConverter = TimeConverter( + config, + parametersMetadata, + mapping + ); + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-per-year': 10000, + 'energy/duration': 1.140795, + }, + ]; + + const result = await timeConverter.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-per-year': 10000, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error when config is not provided.', async () => { const config = undefined; - const timeConverter = TimeConverter(config!, parametersMetadata); + const timeConverter = TimeConverter(config!, parametersMetadata, {}); expect.assertions(1); try { - timeConverter.execute([ + await timeConverter.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -67,17 +129,15 @@ describe('builtins/time-converter: ', () => { }, ]); } catch (error) { - expect(error).toStrictEqual( - new GlobalConfigError(MISSING_GLOBAL_CONFIG) - ); + expect(error).toStrictEqual(new ConfigError(MISSING_CONFIG)); } }); - it('throws an error on missing params in input.', () => { + it('throws an error on missing params in input.', async () => { expect.assertions(1); try { - timeConverter.execute([ + await timeConverter.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -92,7 +152,7 @@ describe('builtins/time-converter: ', () => { } }); - it('returns a result when `new-time-unit` is a different time unit than `duration`.', () => { + it('returns a result when `new-time-unit` is a different time unit than `duration`.', async () => { expect.assertions(1); const newConfig = { 'input-parameter': 'energy-per-year', @@ -100,7 +160,7 @@ describe('builtins/time-converter: ', () => { 'new-time-unit': 'month', 'output-parameter': 'energy-per-duration', }; - const timeConverter = TimeConverter(newConfig, parametersMetadata); + const timeConverter = TimeConverter(newConfig, parametersMetadata, {}); const data = [ { @@ -109,7 +169,7 @@ describe('builtins/time-converter: ', () => { 'energy-per-year': 10000, }, ]; - const response = timeConverter.execute(data); + const response = await timeConverter.execute(data); const expectedResult = [ { timestamp: '2021-01-01T00:00:00Z', @@ -121,6 +181,66 @@ describe('builtins/time-converter: ', () => { expect(response).toEqual(expectedResult); }); + + it('successfully executes when the config output parameter contains an arithmetic expression.', async () => { + expect.assertions(1); + + const config = { + 'input-parameter': '=2 * "energy-per-year"', + 'original-time-unit': 'year', + 'new-time-unit': 'duration', + 'output-parameter': 'energy-per-duration', + }; + + const timeConverter = TimeConverter(config, parametersMetadata, {}); + const expectedResult = [ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-per-year': 10000, + 'energy-per-duration': 2.281589, + }, + ]; + + const result = await timeConverter.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-per-year': 10000, + }, + ]); + + expect(result).toStrictEqual(expectedResult); + }); + + it('throws an error the config input parameter has wrong arithmetic expression.', async () => { + expect.assertions(2); + const config = { + 'input-parameter': '2*"energy-per-year"', + 'original-time-unit': 'year', + 'new-time-unit': 'duration', + 'output-parameter': 'energy-per-duration', + }; + + const timeConverter = TimeConverter(config, parametersMetadata, {}); + + try { + await timeConverter.execute([ + { + timestamp: '2021-01-01T00:00:00Z', + duration: 3600, + 'energy-per-year': 10000, + }, + ]); + } catch (error) { + expect(error).toBeInstanceOf(Error); + expect(error).toEqual( + new InputValidationError( + 'The `input-parameter` contains an invalid arithmetic expression. It should start with `=` and include the symbols `*`, `+`, `-` and `/`.' + ) + ); + } + }); }); }); }); diff --git a/src/__tests__/if-run/builtins/time-sync.test.ts b/src/__tests__/if-run/builtins/time-sync.test.ts index a9fa13cd1..58863272b 100644 --- a/src/__tests__/if-run/builtins/time-sync.test.ts +++ b/src/__tests__/if-run/builtins/time-sync.test.ts @@ -1,3 +1,4 @@ +import {AGGREGATION_METHODS} from '@grnsft/if-core/consts'; import {ERRORS} from '@grnsft/if-core/utils'; import {Settings, DateTime} from 'luxon'; @@ -7,22 +8,20 @@ import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {TimeSync} from '../../../if-run/builtins/time-sync'; import {STRINGS} from '../../../if-run/config'; -import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; Settings.defaultZone = 'utc'; const { InputValidationError, InvalidPaddingError, - InvalidDateInInputError, InvalidInputError, - GlobalConfigError, + ConfigError, } = ERRORS; const { + INCOMPATIBLE_RESOLUTION_WITH_INTERVAL, + INCOMPATIBLE_RESOLUTION_WITH_GAPS, INVALID_OBSERVATION_OVERLAP, - INVALID_TIME_NORMALIZATION, AVOIDING_PADDING_BY_EDGES, - INVALID_DATE_TYPE, } = STRINGS; jest.mock('luxon', () => { @@ -34,17 +33,17 @@ jest.mock('luxon', () => { fromDateTimes: jest.fn((start, end) => ({ start, end, - splitBy: jest.fn(() => { + splitBy: jest.fn(duration => { const intervals = []; let current = start; while (current < end) { intervals.push({ start: process.env.MOCK_INTERVAL === 'true' ? null : current, - end: current.plus({seconds: 1}), + end: current.plus(duration), }); - current = current.plus({seconds: 1}); + current = current.plus(duration); } return intervals; @@ -66,16 +65,15 @@ describe('builtins/time-sync:', () => { type: 'horizontal', }; const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], + [metric]: { + time: AGGREGATION_METHODS[2], + component: AGGREGATION_METHODS[2], + }, })); storeAggregationMetrics(...convertedMetrics); }); describe('time-sync: ', () => { - const parametersMetadata = { - inputs: {}, - outputs: {}, - }; const basicConfig = { 'start-time': '2023-12-12T00:01:00.000Z', 'end-time': '2023-12-12T00:01:00.000Z', @@ -83,7 +81,11 @@ describe('builtins/time-sync:', () => { 'allow-padding': true, }; - const timeSync = TimeSync(basicConfig, parametersMetadata); + const parametersMetadata = { + inputs: {}, + outputs: {}, + }; + const timeSync = TimeSync(basicConfig, parametersMetadata, {}); describe('init: ', () => { it('successfully initalized.', () => { @@ -91,719 +93,1009 @@ describe('builtins/time-sync:', () => { expect(timeSync).toHaveProperty('execute'); }); }); - }); -}); - -describe('execute(): ', () => { - const parametersMetadata = { - inputs: {}, - outputs: {}, - }; - - it('throws error if `start-time` is missing.', async () => { - const invalidStartTimeConfig = { - 'start-time': '', - 'end-time': '2023-12-12T00:01:00.000Z', - interval: 5, - 'allow-padding': true, - }; - - const timeModel = TimeSync(invalidStartTimeConfig, parametersMetadata); - - expect.assertions(1); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 10, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InputValidationError( - '"start-time" parameter is invalid datetime. Error code: invalid_string.' - ) - ); - } - }); - - it('throws error if `end-time` is missing.', async () => { - const errorMessage = - '"end-time" parameter is invalid datetime. Error code: invalid_string.,`start-time` should be lower than `end-time`'; - const invalidEndTimeConfig = { - 'start-time': '2023-12-12T00:01:00.000Z', - 'end-time': '', - interval: 5, - 'allow-padding': true, - }; - const timeModel = TimeSync(invalidEndTimeConfig, parametersMetadata); - - expect.assertions(1); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 10, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual(new InputValidationError(errorMessage)); - } - }); - - it('fails if `start-time` is not a valid ISO date.', async () => { - const invalidStartTimeConfig = { - 'start-time': '0023-X', - 'end-time': '2023-12-12T00:01:00.000Z', - interval: 5, - 'allow-padding': true, - }; - const timeModel = TimeSync(invalidStartTimeConfig, parametersMetadata); - expect.assertions(1); - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 10, - 'cpu/utilization': 10, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InputValidationError( - '"start-time" parameter is invalid datetime. Error code: invalid_string.' - ) - ); - } - }); - - it('fails if `end-time` is not a valid ISO date.', async () => { - const invalidEndTimeConfig = { - 'start-time': '2023-12-12T00:01:00.000Z', - 'end-time': '20XX', - interval: 5, - 'allow-padding': true, - }; - const timeModel = TimeSync(invalidEndTimeConfig, parametersMetadata); - - expect.assertions(1); - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 10, - 'cpu/utilization': 10, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InputValidationError( - '"end-time" parameter is invalid datetime. Error code: invalid_string.' - ) - ); - } - }); - - it('throws error on missing global config.', async () => { - const config = undefined; - const timeModel = TimeSync(config!, parametersMetadata); - - expect.assertions(1); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 15, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new GlobalConfigError(INVALID_TIME_NORMALIZATION) - ); - } - }); - - it('throws error if interval is invalid.', async () => { - const invalidIntervalConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:01:00.000Z', - interval: 0, - 'allow-padding': true, - }; - - const timeModel = TimeSync(invalidIntervalConfig, parametersMetadata); - - expect.assertions(1); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 15, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InvalidInputError(INVALID_OBSERVATION_OVERLAP) - ); - } - }); - it('throws error if timestamps overlap.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:01:00.000Z', - interval: 5, - 'allow-padding': true, - }; - - const timeModel = TimeSync(basicConfig, parametersMetadata); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 15, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InvalidInputError(INVALID_OBSERVATION_OVERLAP) - ); - } - }); - - it('throws error if `timestamp` is missing.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:01:00.000Z', - interval: 5, - 'allow-padding': true, - }; - - const timeModel = TimeSync(basicConfig, parametersMetadata); - - try { - await timeModel.execute([ - { - duration: 15, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toStrictEqual( - new InputValidationError( - '"timestamp" parameter is required in input[0]. Error code: invalid_union.' - ) - ); - } - }); - - it('throws error if the seconds `timestamp` is above 60.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:01:00.000Z', - interval: 5, - 'allow-padding': true, - }; - - const timeModel = TimeSync(basicConfig, parametersMetadata); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:90.000Z', - duration: 15, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toStrictEqual( - new InputValidationError( - '"timestamp" parameter is invalid datetime in input[0]. Error code: invalid_string.' - ) - ); - } - }); - - it('throws an error if the `timestamp` is not valid date.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:01:00.000Z', - interval: 10, - 'allow-padding': true, - }; - const data = [ - { - timestamp: 45, - duration: 10, - 'cpu/utilization': 10, - }, - ]; - - const timeModel = TimeSync(basicConfig, parametersMetadata); - expect.assertions(2); - - try { - await timeModel.execute(data); - } catch (error) { - expect(error).toBeInstanceOf(InvalidDateInInputError); - expect(error).toStrictEqual( - new InvalidDateInInputError(INVALID_DATE_TYPE(data[0].timestamp)) - ); - } - }); - - it('throws error if end is before start in global config.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:10.000Z', - 'end-time': '2023-12-12T00:00:00.000Z', - interval: 5, - 'allow-padding': true, - }; - - const timeModel = TimeSync(basicConfig, parametersMetadata); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 15, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InputValidationError('`start-time` should be lower than `end-time`') - ); - } - }); - - it('converts Date objects to string outputs.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:01.000Z', - interval: 1, - 'allow-padding': false, - }; - - const timeModel = TimeSync(basicConfig, parametersMetadata); - - const result = await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 1, - 'cpu/utilization': 10, - }, - { - timestamp: new Date('2023-12-12T00:00:01.000Z'), - duration: 1, - 'cpu/utilization': 10, - }, - ]); - - const expectedResult = [ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 1, - }, - { - timestamp: '2023-12-12T00:00:01.000Z', - duration: 1, - }, - ]; - - expect(result).toStrictEqual(expectedResult); - }); - - it('checks that metric (carbon) with aggregation-method == sum is properly spread over interpolated time points.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:10.000Z', - interval: 1, - 'allow-padding': true, - }; - storeAggregationMetrics({carbon: 'sum'}); - - const timeModel = TimeSync(basicConfig, parametersMetadata); + describe('execute(): ', () => { + it('throws error if `start-time` is missing.', async () => { + const invalidStartTimeConfig = { + 'start-time': '', + 'end-time': '2023-12-12T00:01:00.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync( + invalidStartTimeConfig, + parametersMetadata, + {} + ); + + expect.assertions(1); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 10, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InputValidationError( + '"start-time" parameter is invalid datetime. Error code: invalid_string.' + ) + ); + } + }); - const result = await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 10, - carbon: 10, - }, - ]); + it('throws error if `end-time` is missing.', async () => { + const errorMessage = + '"end-time" parameter is invalid datetime. Error code: invalid_string.,`start-time` should be lower than `end-time`'; + const invalidEndTimeConfig = { + 'start-time': '2023-12-12T00:01:00.000Z', + 'end-time': '', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync( + invalidEndTimeConfig, + parametersMetadata, + {} + ); + + expect.assertions(1); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 10, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual(new InputValidationError(errorMessage)); + } + }); - const expectedResult = [ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:01.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:02.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:03.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:04.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:05.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:06.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:07.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:08.000Z', - duration: 1, - carbon: 1, - }, - { - timestamp: '2023-12-12T00:00:09.000Z', - duration: 1, - carbon: 1, - }, - ]; + it('fails if `start-time` is not a valid ISO date.', async () => { + const invalidStartTimeConfig = { + 'start-time': '0023-X', + 'end-time': '2023-12-12T00:01:00.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync( + invalidStartTimeConfig, + parametersMetadata, + {} + ); + expect.assertions(1); + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 10, + 'cpu/utilization': 10, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InputValidationError( + '"start-time" parameter is invalid datetime. Error code: invalid_string.' + ) + ); + } + }); - expect(result).toStrictEqual(expectedResult); - }); + it('fails if `end-time` is not a valid ISO date.', async () => { + const invalidEndTimeConfig = { + 'start-time': '2023-12-12T00:01:00.000Z', + 'end-time': '20XX', + interval: 5, + 'allow-padding': true, + }; + + const timeModel = TimeSync( + invalidEndTimeConfig, + parametersMetadata, + {} + ); + + expect.assertions(1); + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 10, + 'cpu/utilization': 10, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InputValidationError( + '"end-time" parameter is invalid datetime. Error code: invalid_string.' + ) + ); + } + }); - it('checks that constants are copied to results unchanged.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:09.000Z', - interval: 5, - 'allow-padding': true, - }; + it('throws error on missing config.', async () => { + const config = undefined; + const timeModel = TimeSync(config!, parametersMetadata, {}); + + expect.assertions(1); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 15, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new ConfigError('Config is not provided.') + ); + } + }); - const timeModel = TimeSync(basicConfig, parametersMetadata); + it('throws error if interval is invalid.', async () => { + const invalidIntervalConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:01:00.000Z', + interval: 0, + 'allow-padding': true, + }; + const timeModel = TimeSync( + invalidIntervalConfig, + parametersMetadata, + {} + ); + + expect.assertions(1); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 15, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InvalidInputError(INVALID_OBSERVATION_OVERLAP) + ); + } + }); - const result = await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 3, - 'resources-total': 10, - }, - { - timestamp: '2023-12-12T00:00:05.000Z', - duration: 3, - 'resources-total': 10, - }, - ]); + it('throws error if timestamps overlap.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:01:00.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 15, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InvalidInputError(INVALID_OBSERVATION_OVERLAP) + ); + } + }); - const expectedResult = [ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 5, - }, - { - timestamp: '2023-12-12T00:00:05.000Z', - duration: 5, - }, - ]; + it('throws error if `timestamp` is missing.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:01:00.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + try { + await timeModel.execute([ + { + duration: 15, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toStrictEqual( + new InputValidationError( + '"timestamp" parameter is required at index 0. Error code: invalid_union.' + ) + ); + } + }); - expect(result).toStrictEqual(expectedResult); - }); + it('throws error if the seconds `timestamp` is above 60.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:01:00.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:90.000Z', + duration: 15, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toStrictEqual( + new InputValidationError( + '"timestamp" parameter is invalid datetime at index 0. Error code: invalid_string.' + ) + ); + } + }); - it('returns a result when `time-reserved` persists.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:09.000Z', - interval: 5, - 'allow-padding': true, - }; - storeAggregationMetrics({'time-reserved': 'avg'}); - storeAggregationMetrics({'resources-total': 'sum'}); + it('throws an error if the `timestamp` is not valid date.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:01:00.000Z', + interval: 10, + 'allow-padding': true, + }; + const data = [ + { + timestamp: 45, + duration: 10, + 'cpu/utilization': 10, + }, + ]; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + expect.assertions(2); + + try { + await timeModel.execute(data); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toStrictEqual( + new InputValidationError( + '"timestamp" parameter is expected string, received number at index 0. Error code: invalid_union.' + ) + ); + } + }); - const timeModel = TimeSync(basicConfig, parametersMetadata); + it('throws error if end is before start in global config.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:10.000Z', + 'end-time': '2023-12-12T00:00:00.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 15, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InputValidationError( + '`start-time` should be lower than `end-time`' + ) + ); + } + }); - const result = await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 3, - 'time-reserved': 5, - 'resources-total': 10, - }, - { - timestamp: '2023-12-12T00:00:05.000Z', - duration: 3, - 'time-reserved': 5, - 'resources-total': 10, - }, - ]); - - const expectedResult = [ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 5, - 'resources-total': 10, - 'time-reserved': 3.2, - }, - { - timestamp: '2023-12-12T00:00:05.000Z', - duration: 5, - 'resources-total': 10, - 'time-reserved': 3.2, - }, - ]; + it('converts Date objects to string outputs.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:01.000Z', + interval: 1, + 'allow-padding': false, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 1, + 'cpu/utilization': 10, + }, + { + timestamp: new Date('2023-12-12T00:00:01.000Z'), + duration: 1, + 'cpu/utilization': 10, + }, + ]); + + const expectedResult = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 1, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:01.000Z', + duration: 1, + 'cpu/utilization': 10, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); - expect(result).toStrictEqual(expectedResult); - }); + it('checks that metric (carbon) with aggregation-method == sum is properly spread over interpolated time points.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:10.000Z', + interval: 1, + 'allow-padding': true, + }; + + storeAggregationMetrics({ + carbon: { + time: 'sum', + component: 'sum', + }, + }); + + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 10, + carbon: 10, + }, + ]); + + const expectedResult = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:01.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:02.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:03.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:04.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:06.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:07.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:08.000Z', + duration: 1, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:09.000Z', + duration: 1, + carbon: 1, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); - it('throws an error when `start-time` is wrong.', async () => { - process.env.MOCK_INTERVAL = 'true'; - const basicConfig = { - 'start-time': '2023-12-12T00:00:90.000Z', - 'end-time': '2023-12-12T00:01:09.000Z', - interval: 5, - 'allow-padding': true, - }; + it('checks that constants are copied to results unchanged.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:09.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 3, + 'resources-total': 10, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 3, + 'resources-total': 10, + }, + ]); + + /**In each 5 second interval, 60% of the time cpu/utilization = 10, 40% of the time it is 0, so cpu/utilization in the averaged result be 6 */ + const expectedResult = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 5, + 'resources-total': 10, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 4, + 'resources-total': 10, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); - const timeModel = TimeSync(basicConfig, parametersMetadata); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toStrictEqual( - new InputValidationError( - '"start-time" parameter is invalid datetime. Error code: invalid_string.' - ) - ); - } - }); + it('returns a result when `time-reserved` persists.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:09.000Z', + interval: 5, + 'allow-padding': true, + }; + + storeAggregationMetrics({ + 'time-reserved': { + time: 'avg', + component: 'avg', + }, + }); + storeAggregationMetrics({ + 'resources-total': { + time: 'sum', + component: 'sum', + }, + }); + + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 3, + 'time-reserved': 5, + 'resources-total': 10, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 3, + 'time-reserved': 5, + 'resources-total': 10, + }, + ]); + + const expectedResult = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 5, + 'resources-total': 10, + 'time-reserved': 3.2, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 4, + 'resources-total': 10, + 'time-reserved': 3.75, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); - it('returns a result when the first timestamp in the input has time padding.', async () => { - process.env.MOCK_INTERVAL = 'false'; - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:09.000Z', - interval: 5, - 'allow-padding': true, - }; - storeAggregationMetrics({'resources-total': 'none'}); + it('returns a result when `mapping` has valid data.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:09.000Z', + interval: 5, + 'allow-padding': true, + }; + const mapping = { + 'time-reserved': 'time-allocated', + }; + + storeAggregationMetrics({ + 'time-allocated': { + time: 'avg', + component: 'avg', + }, + }); + storeAggregationMetrics({ + 'resources-total': { + time: 'sum', + component: 'sum', + }, + }); + + const timeModel = TimeSync(basicConfig, parametersMetadata, mapping); + + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 3, + 'time-allocated': 5, + 'resources-total': 10, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 3, + 'time-allocated': 5, + 'resources-total': 10, + }, + ]); + + const expectedResult = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 5, + 'resources-total': 10, + 'time-allocated': 3.2, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 4, + 'resources-total': 10, + 'time-allocated': 3.75, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); - const timeModel = TimeSync(basicConfig, parametersMetadata); + it('throws an error when `start-time` is wrong.', async () => { + process.env.MOCK_INTERVAL = 'true'; + const basicConfig = { + 'start-time': '2023-12-12T00:00:90.000Z', + 'end-time': '2023-12-12T00:01:09.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toStrictEqual( + new InputValidationError( + '"start-time" parameter is invalid datetime. Error code: invalid_string.' + ) + ); + } + }); - const result = await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:05.000Z', - duration: 3, - 'resources-total': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 3, - 'resources-total': 10, - }, - ]); + it('returns a result when the first timestamp in the input has time padding.', async () => { + process.env.MOCK_INTERVAL = 'false'; + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:09.000Z', + interval: 5, + 'allow-padding': true, + }; + + storeAggregationMetrics({ + 'resources-total': { + time: 'none', + component: 'none', + }, + }); + + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 3, + 'resources-total': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 3, + 'resources-total': 10, + }, + ]); + + const expectedResult = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 5, + 'resources-total': null, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 5, + 'resources-total': null, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); - const expectedResult = [ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 5, - 'resources-total': 10, - }, - { - timestamp: '2023-12-12T00:00:05.000Z', - duration: 5, - 'resources-total': 10, - }, - ]; + it('throws error if padding is required at start while allow-padding = false.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:10.000Z', + interval: 5, + 'allow-padding': false, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:02.000Z', + duration: 15, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(true, false)) + ); + } + }); - expect(result).toStrictEqual(expectedResult); - }); + it('throws error if padding is required at end while allow-padding = false.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:10.000Z', + interval: 5, + 'allow-padding': false, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 10, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 30, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InputValidationError('Avoiding padding at end') + ); + } + }); - it('throws error if padding is required at start while allow-padding = false.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:10.000Z', - interval: 5, - 'allow-padding': false, - }; + it('throws error if padding is required at start and end while allow-padding = false.', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:10.000Z', + interval: 5, + 'allow-padding': false, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:02.000Z', + duration: 10, + 'cpu/utilization': 10, + }, + { + timestamp: '2023-12-12T00:00:08.000Z', + duration: 1, + 'cpu/utilization': 20, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(true, true)) + ); + } + }); - const timeModel = TimeSync(basicConfig, parametersMetadata); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:02.000Z', - duration: 15, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(true, false)) - ); - } - }); + it('checks that timestamps in return object are ISO 8061 and timezone UTC.', async () => { + process.env.MOCK_INTERVAL = 'false'; + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:03.000Z', + interval: 1, + 'allow-padding': true, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 1, + carbon: 1, + }, + ]); + expect( + DateTime.fromISO(result[0].timestamp).zone.valueOf() === + 'FixedOffsetZone { fixed: 0 }' + ); + expect(DateTime.fromISO(result[0].timestamp).offset === 0); + }); - it('throws error if padding is required at end while allow-padding = false.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:10.000Z', - interval: 5, - 'allow-padding': false, - }; + it('successfully executes when the `duration` contains an arithmetic expression.', async () => { + expect.assertions(1); + + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:10.000Z', + interval: 5, + 'allow-padding': true, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 3, + 'resources-total': 10, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 3 * 2, + 'resources-total': 10, + }, + ]); + + const expectedResult = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 5, + 'resources-total': null, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 5, + 'resources-total': null, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 1, + 'resources-total': null, + }, + ]; + + expect(result).toStrictEqual(expectedResult); + }); - const timeModel = TimeSync(basicConfig, parametersMetadata); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 10, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:10.000Z', - duration: 30, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InputValidationError('Avoiding padding at end') - ); - } - }); + it('should throw an error if the upsampling resolution is not compatible with the interval', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:03.000Z', + interval: 3, + 'allow-padding': true, + 'upsampling-resolution': 2, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + expect.assertions(1); + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:02.000Z', + duration: 10, + 'cpu/utilization': 10, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new ConfigError(INCOMPATIBLE_RESOLUTION_WITH_INTERVAL) + ); + } + }); - it('throws error if padding is required at start and end while allow-padding = false.', async () => { - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:10.000Z', - interval: 5, - 'allow-padding': false, - }; + it('should throw an error if the upsampling resolution is not compatible with paddings', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:12.000Z', + interval: 2, + 'allow-padding': true, + 'upsampling-resolution': 2, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + expect.assertions(1); + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 10, + 'cpu/utilization': 10, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new ConfigError(INCOMPATIBLE_RESOLUTION_WITH_GAPS) + ); + } + }); - const timeModel = TimeSync(basicConfig, parametersMetadata); - - try { - await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:02.000Z', - duration: 10, - 'cpu/utilization': 10, - }, - { - timestamp: '2023-12-12T00:00:08.000Z', - duration: 1, - 'cpu/utilization': 20, - }, - ]); - } catch (error) { - expect(error).toStrictEqual( - new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(true, true)) - ); - } - }); + it('should throw an error if the upsampling resolution is not compatible with gaps', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:12.000Z', + interval: 5, + 'allow-padding': true, + 'upsampling-resolution': 5, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + expect.assertions(1); + try { + await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 5, + }, + { + timestamp: '2023-12-12T00:00:07.000Z', + duration: 5, + }, + ]); + } catch (error) { + expect(error).toStrictEqual( + new ConfigError(INCOMPATIBLE_RESOLUTION_WITH_GAPS) + ); + } + }); - it('checks that timestamps in return object are ISO 8061 and timezone UTC.', async () => { - process.env.MOCK_INTERVAL = 'false'; - const basicConfig = { - 'start-time': '2023-12-12T00:00:00.000Z', - 'end-time': '2023-12-12T00:00:03.000Z', - interval: 1, - 'allow-padding': true, - }; + it('should upsample and resample correctly with a custom upsampling resolution given', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:20.000Z', + interval: 5, + 'allow-padding': true, + 'upsampling-resolution': 5, + }; + + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 15, + }, + ]); + const expected = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 5, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 5, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 5, + }, + { + timestamp: '2023-12-12T00:00:15.000Z', + duration: 5, + }, + ]; + expect(result).toEqual(expected); + }); - const timeModel = TimeSync(basicConfig, parametersMetadata); - const result = await timeModel.execute([ - { - timestamp: '2023-12-12T00:00:00.000Z', - duration: 1, - carbon: 1, - }, - ]); - expect( - DateTime.fromISO(result[0].timestamp).zone.valueOf() === - 'FixedOffsetZone { fixed: 0 }' - ); - expect(DateTime.fromISO(result[0].timestamp).offset === 0); + it('checks that metric carbon with aggregation == sum is properly spread over interpolated time points with custom upsampling resolution given', async () => { + const basicConfig = { + 'start-time': '2023-12-12T00:00:00.000Z', + 'end-time': '2023-12-12T00:00:15.000Z', + interval: 5, + 'allow-padding': true, + 'upsampling-resolution': 5, + }; + const timeModel = TimeSync(basicConfig, parametersMetadata, {}); + const result = await timeModel.execute([ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 15, + carbon: 3, + }, + ]); + + const expected = [ + { + timestamp: '2023-12-12T00:00:00.000Z', + duration: 5, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:05.000Z', + duration: 5, + carbon: 1, + }, + { + timestamp: '2023-12-12T00:00:10.000Z', + duration: 5, + carbon: 1, + }, + ]; + expect(result).toEqual(expected); + }); + }); }); }); diff --git a/src/__tests__/if-run/lib/aggregate.test.ts b/src/__tests__/if-run/lib/aggregate.test.ts index 00d9c6d0e..f39efdfcd 100644 --- a/src/__tests__/if-run/lib/aggregate.test.ts +++ b/src/__tests__/if-run/lib/aggregate.test.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ +import {AGGREGATION_METHODS} from '@grnsft/if-core/consts'; import {AggregationParams} from '../../../common/types/manifest'; @@ -6,7 +7,6 @@ import { aggregate, storeAggregationMetrics, } from '../../../if-run/lib/aggregate'; -import {AGGREGATION_METHODS} from '../../../if-run/types/aggregation'; describe('lib/aggregate: ', () => { beforeAll(() => { @@ -15,13 +15,25 @@ describe('lib/aggregate: ', () => { type: 'horizontal', }; const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], + [metric]: { + time: AGGREGATION_METHODS[2], + component: AGGREGATION_METHODS[2], + }, })); storeAggregationMetrics(...convertedMetrics); }); describe('aggregate(): ', () => { + beforeAll(() => { + storeAggregationMetrics({ + carbon: { + time: 'sum', + component: 'sum', + }, + }); + }); + it('returns tree if aggregation is missing.', () => { const tree = {}; const aggregation = undefined; diff --git a/src/__tests__/if-run/lib/compute.test.ts b/src/__tests__/if-run/lib/compute.test.ts index 912793381..9c883f961 100644 --- a/src/__tests__/if-run/lib/compute.test.ts +++ b/src/__tests__/if-run/lib/compute.test.ts @@ -15,18 +15,14 @@ describe('lib/compute: ', () => { return input; }), - metadata: { - kind: 'execute', - }, + metadata: {}, }); const mockObservePlugin = () => ({ execute: () => [ {timestamp: '2024-09-02', duration: 40, 'cpu/utilization': 30}, {timestamp: '2024-09-03', duration: 60, 'cpu/utilization': 40}, ], - metadata: { - kind: 'execute', - }, + metadata: {}, }); const mockObservePluginTimeSync = () => ({ execute: () => [ @@ -41,9 +37,7 @@ describe('lib/compute: ', () => { 'cpu/utilization': 40, }, ], - metadata: { - kind: 'execute', - }, + metadata: {}, }); const mockTimeSync = () => ({ execute: () => [ @@ -68,9 +62,7 @@ describe('lib/compute: ', () => { 'cpu/utilization': 40, }, ], - metadata: { - kind: 'execute', - }, + metadata: {}, }); /** * Compute params. @@ -94,6 +86,7 @@ describe('lib/compute: ', () => { .set('mock-observe-time-sync', mockObservePluginTimeSync()) .set('time-sync', mockTimeSync()), }; + const paramsExecuteWithAppend = {...paramsExecute, append: true}; describe('compute(): ', () => { it('computes simple tree with execute plugin.', async () => { @@ -117,24 +110,28 @@ describe('lib/compute: ', () => { expect(response.children.mockChild.outputs).toEqual(expectedResult); }); - it('computes simple tree with groupby plugin.', async () => { + it('computes simple tree with regroup on inputs only (no compute).', async () => { const tree = { children: { mockChild: { - pipeline: {regroup: ['duration']}, + pipeline: {regroup: ['region']}, inputs: [ - {timestamp: 'mock-timestamp-1', duration: 10}, - {timestamp: 'mock-timestamp-2', duration: 10}, + {timestamp: 'mock-timestamp-1', region: 'uk-west'}, + {timestamp: 'mock-timestamp-2', region: 'uk-east'}, + {timestamp: 'mock-timestamp-3', region: 'uk-east'}, ], }, }, }; const response = await compute(tree, paramsExecute); const expectedResponse = { - '10': { + 'uk-west': { + inputs: [{region: 'uk-west', timestamp: 'mock-timestamp-1'}], + }, + 'uk-east': { inputs: [ - {duration: 10, timestamp: 'mock-timestamp-1'}, - {duration: 10, timestamp: 'mock-timestamp-2'}, + {region: 'uk-east', timestamp: 'mock-timestamp-2'}, + {region: 'uk-east', timestamp: 'mock-timestamp-3'}, ], }, }; @@ -142,6 +139,53 @@ describe('lib/compute: ', () => { expect(response.children.mockChild.children).toEqual(expectedResponse); }); + it('computes simple tree with regroup, grouping inputs and outputs.', async () => { + const tree = { + children: { + mockChild: { + pipeline: {regroup: ['region'], compute: ['mock']}, + inputs: [ + {timestamp: 'mock-timestamp-1', region: 'uk-west'}, + {timestamp: 'mock-timestamp-2', region: 'uk-east'}, + {timestamp: 'mock-timestamp-3', region: 'uk-east'}, + ], + }, + }, + }; + const response = await compute(tree, paramsExecute); + const expectedResponse = { + 'uk-west': { + inputs: [{region: 'uk-west', timestamp: 'mock-timestamp-1'}], + outputs: [ + { + region: 'uk-west', + timestamp: 'mock-timestamp-1', + newField: 'mock-newField', + }, + ], + }, + 'uk-east': { + inputs: [ + {region: 'uk-east', timestamp: 'mock-timestamp-2'}, + {region: 'uk-east', timestamp: 'mock-timestamp-3'}, + ], + outputs: [ + { + region: 'uk-east', + timestamp: 'mock-timestamp-2', + newField: 'mock-newField', + }, + { + region: 'uk-east', + timestamp: 'mock-timestamp-3', + newField: 'mock-newField', + }, + ], + }, + }; + expect(response.children.mockChild.children).toEqual(expectedResponse); + }); + it('computes simple tree with defaults and execute plugin.', async () => { const tree = { children: { @@ -218,7 +262,7 @@ describe('lib/compute: ', () => { ); }); - it('computes simple tree with no defaults and no inputs with execue plugin.', async () => { + it('computes simple tree with no defaults and no inputs with execute plugin.', async () => { const tree = { children: { mockChild: { @@ -233,7 +277,7 @@ describe('lib/compute: ', () => { expect(response.children.mockChild.outputs).toBeUndefined(); }); - it('computes simple tree with defaults and no inputs with execue plugin.', async () => { + it('computes simple tree with defaults and no inputs with execute plugin.', async () => { const tree = { children: { mockChild: { @@ -253,32 +297,100 @@ describe('lib/compute: ', () => { expect(response.children.mockChild.outputs).toEqual(expectedResult); }); - it('computes simple tree with node config and execute plugin.', async () => { + it('computes simple tree with append, preserving existing outputs.', async () => { const tree = { children: { mockChild: { - pipeline: { - compute: ['mock'], - }, - config: { - 'cpu/name': 'Intel CPU', - }, + pipeline: {compute: ['mock']}, inputs: [ - {timestamp: 'mock-timestamp-1', duration: 10}, - {timestamp: 'mock-timestamp-2', duration: 10}, + {timestamp: 'mock-timestamp-1', region: 'eu-west'}, + {timestamp: 'mock-timestamp-2', region: 'eu-west'}, + ], + outputs: [ + { + timestamp: 'mock-timestamp-preexisting-1', + newField: 'mock-newField', + region: 'eu-west', + }, + { + timestamp: 'mock-timestamp-preexisting-2', + newField: 'mock-newField', + region: 'eu-west', + }, ], }, }, }; - const response = await compute(tree, paramsExecute); - const expectedResult = mockExecutePlugin().execute( - tree.children.mockChild.inputs - ); - + const response = await compute(tree, paramsExecuteWithAppend); + const expectedResult = [ + ...tree.children.mockChild.outputs, + ...mockExecutePlugin().execute(tree.children.mockChild.inputs), + ]; + expect(response.children.mockChild.outputs).toHaveLength(4); expect(response.children.mockChild.outputs).toEqual(expectedResult); }); }); + it('computes simple tree with regroup and append, with existing outputs preserved and regrouped without re-computing.', async () => { + const tree = { + children: { + mockChild: { + pipeline: {regroup: ['region'], compute: ['mock']}, + inputs: [{timestamp: 'mock-timestamp-1', region: 'uk-east'}], + outputs: [ + {timestamp: 'mock-timestamp-preexisting-1', region: 'uk-east'}, + ], + }, + }, + }; + const response = await compute(tree, paramsExecuteWithAppend); + const expectedResponse = { + 'uk-east': { + inputs: [{region: 'uk-east', timestamp: 'mock-timestamp-1'}], + outputs: [ + { + region: 'uk-east', + timestamp: 'mock-timestamp-preexisting-1', + }, + { + region: 'uk-east', + timestamp: 'mock-timestamp-1', + newField: 'mock-newField', + }, + ], + }, + }; + expect(response.children.mockChild.children).toEqual(expectedResponse); + }); + + it('computes simple tree with regroup and no append, with existing outputs that are removed.', async () => { + const tree = { + children: { + mockChild: { + pipeline: {regroup: ['region'], compute: ['mock']}, + inputs: [{timestamp: 'mock-timestamp-1', region: 'uk-east'}], + outputs: [ + {timestamp: 'mock-timestamp-preexisting-1', region: 'uk-east'}, + ], + }, + }, + }; + const response = await compute(tree, paramsExecute); + const expectedResponse = { + 'uk-east': { + inputs: [{region: 'uk-east', timestamp: 'mock-timestamp-1'}], + outputs: [ + { + region: 'uk-east', + timestamp: 'mock-timestamp-1', + newField: 'mock-newField', + }, + ], + }, + }; + expect(response.children.mockChild.children).toEqual(expectedResponse); + }); + it('computes simple tree with observe plugin.', async () => { const tree = { children: { diff --git a/src/__tests__/if-run/lib/environment.test.ts b/src/__tests__/if-run/lib/environment.test.ts index 324bd1da6..b7b6ab60b 100644 --- a/src/__tests__/if-run/lib/environment.test.ts +++ b/src/__tests__/if-run/lib/environment.test.ts @@ -2,7 +2,7 @@ import {injectEnvironment} from '../../../if-run/lib/environment'; -describe('lib/envirnoment: ', () => { +describe('lib/environment: ', () => { describe('injectEnvironment(): ', () => { const context = {}; diff --git a/src/__tests__/if-run/lib/explain.test.ts b/src/__tests__/if-run/lib/explain.test.ts index 506c62669..df86448ac 100644 --- a/src/__tests__/if-run/lib/explain.test.ts +++ b/src/__tests__/if-run/lib/explain.test.ts @@ -1,8 +1,15 @@ /* eslint-disable @typescript-eslint/ban-ts-comment */ +import {ERRORS} from '@grnsft/if-core/utils'; + +import {STRINGS} from '../../../common/config'; + import {explain, addExplainData} from '../../../if-run/lib/explain'; +const {ManifestValidationError} = ERRORS; +const {AGGREGATION_UNITS_NOT_MATCH, AGGREGATION_METHODS_NOT_MATCH} = STRINGS; + describe('lib/explain: ', () => { - it('successfully adds explain data if `inputs` and `outputs` of `metadata` are `undefined`.', () => { + it('missing explain data if `inputs` and `outputs` of `metadata` are `undefined`.', () => { const mockData = { pluginName: 'divide', metadata: {kind: 'execute', inputs: undefined, outputs: undefined}, @@ -11,19 +18,11 @@ describe('lib/explain: ', () => { method: 'Divide', }, }; - const expectedResult = { - divide: { - method: 'Divide', - path: 'builtin', - inputs: 'undefined', - outputs: 'undefined', - }, - }; addExplainData(mockData); const result = explain(); expect.assertions(1); - expect(result).toEqual(expectedResult); + expect(result).toEqual({}); }); it('successfully adds explain data if `inputs` and `outputs` of `metadata` are valid data.', () => { @@ -56,36 +55,99 @@ describe('lib/explain: ', () => { method: 'Sum', }, }; + const expectedResult = { - divide: { - method: 'Divide', - path: 'builtin', - inputs: 'undefined', - outputs: 'undefined', + 'cpu/energy': { + plugins: ['sum'], + unit: 'kWh', + description: 'energy consumed by the cpu', + 'aggregation-method': 'sum', }, - sum: { - method: 'Sum', - path: 'builtin', + 'network/energy': { + plugins: ['sum'], + unit: 'kWh', + description: 'energy consumed by data ingress and egress', + 'aggregation-method': 'sum', + }, + 'energy-sum': { + plugins: ['sum'], + unit: 'kWh', + description: 'sum of energy components', + 'aggregation-method': 'sum', + }, + }; + + // @ts-ignore + addExplainData(mockData); + + const result = explain(); + + expect.assertions(1); + expect(result).toEqual(expectedResult); + }); + + it('successfully adds explain data if the parameter is using more than one plugin.', () => { + const mockData = { + pluginName: 'sum-energy', + metadata: { + kind: 'execute', inputs: { 'cpu/energy': { unit: 'kWh', description: 'energy consumed by the cpu', 'aggregation-method': 'sum', }, - 'network/energy': { + 'memory/energy': { unit: 'kWh', - description: 'energy consumed by data ingress and egress', + description: 'energy consumed by data from memory', 'aggregation-method': 'sum', }, }, outputs: { - 'energy-sum': { + 'total/energy': { unit: 'kWh', description: 'sum of energy components', 'aggregation-method': 'sum', }, }, }, + pluginData: { + path: 'builtin', + method: 'Sum', + }, + }; + + const expectedResult = { + 'cpu/energy': { + plugins: ['sum', 'sum-energy'], + unit: 'kWh', + description: 'energy consumed by the cpu', + 'aggregation-method': 'sum', + }, + 'network/energy': { + plugins: ['sum'], + unit: 'kWh', + description: 'energy consumed by data ingress and egress', + 'aggregation-method': 'sum', + }, + 'energy-sum': { + plugins: ['sum'], + unit: 'kWh', + description: 'sum of energy components', + 'aggregation-method': 'sum', + }, + 'memory/energy': { + plugins: ['sum-energy'], + unit: 'kWh', + description: 'energy consumed by data from memory', + 'aggregation-method': 'sum', + }, + 'total/energy': { + plugins: ['sum-energy'], + unit: 'kWh', + description: 'sum of energy components', + 'aggregation-method': 'sum', + }, }; // @ts-ignore @@ -96,4 +158,105 @@ describe('lib/explain: ', () => { expect.assertions(1); expect(result).toEqual(expectedResult); }); + + it('throws an error if `unit` of the parameter is not matched.', () => { + const mockData = { + pluginName: 'sum-of-energy', + metadata: { + kind: 'execute', + inputs: { + 'cpu/energy': { + unit: 'co2q', + description: 'energy consumed by the cpu', + 'aggregation-method': 'sum', + }, + 'memory/energy': { + unit: 'kWh', + description: 'energy consumed by data from memory', + 'aggregation-method': 'sum', + }, + }, + outputs: { + 'total/energy': { + unit: 'kWh', + description: 'sum of energy components', + 'aggregation-method': 'sum', + }, + }, + }, + pluginData: { + path: 'builtin', + method: 'Sum', + }, + }; + + expect.assertions(2); + try { + // @ts-ignore + addExplainData(mockData); + explain(); + } catch (error) { + if (error instanceof Error) { + expect(error).toBeInstanceOf(ManifestValidationError); + expect(error.message).toEqual( + AGGREGATION_UNITS_NOT_MATCH('cpu/energy') + ); + } + } + }); + + it('throws an error if `aggregation-method` of the parameter is not matched.', () => { + const mockData = { + pluginName: 'sum-of-energy', + metadata: { + kind: 'execute', + inputs: { + 'cpu/energy': { + unit: 'kWh', + description: 'energy consumed by the cpu', + 'aggregation-method': { + time: 'avg', + component: 'avg', + }, + }, + 'memory/energy': { + unit: 'kWh', + description: 'energy consumed by data from memory', + 'aggregation-method': { + time: 'sum', + component: 'sum', + }, + }, + }, + outputs: { + 'total/energy': { + unit: 'kWh', + description: 'sum of energy components', + 'aggregation-method': { + time: 'sum', + component: 'sum', + }, + }, + }, + }, + pluginData: { + path: 'builtin', + method: 'Sum', + }, + }; + + expect.assertions(2); + try { + // @ts-ignore + addExplainData(mockData); + explain(); + } catch (error) { + if (error instanceof Error) { + expect(error).toBeInstanceOf(ManifestValidationError); + expect(error.message).toEqual( + AGGREGATION_METHODS_NOT_MATCH('cpu/energy') + ); + } + } + }); }); diff --git a/src/__tests__/if-run/lib/initialize.test.ts b/src/__tests__/if-run/lib/initialize.test.ts index a1eda8924..3b2c82fdc 100644 --- a/src/__tests__/if-run/lib/initialize.test.ts +++ b/src/__tests__/if-run/lib/initialize.test.ts @@ -60,14 +60,14 @@ describe('lib/initalize: ', () => { expect(mockLog).toHaveBeenCalledTimes(1); // checks if logger is called }); - it('checks if plugin is initalized with global config and has execute and metadata.', async () => { + it('checks if plugin is initalized with config and has execute and metadata.', async () => { const context = { initialize: { plugins: { mockavizta: { path: 'mockavizta', method: 'Mockavizta', - 'global-config': { + config: { verbose: true, }, }, @@ -89,7 +89,7 @@ describe('lib/initalize: ', () => { plugins: { mockavizta: { method: 'Mockavizta', - 'global-config': { + config: { verbose: true, }, }, @@ -115,7 +115,7 @@ describe('lib/initalize: ', () => { plugins: { mockavizta: { path: 'mockavizta', - 'global-config': { + config: { verbose: true, }, }, @@ -142,7 +142,7 @@ describe('lib/initalize: ', () => { mockavizta: { path: 'builtin', method: 'Mockavizta', - 'global-config': { + config: { verbose: true, }, }, @@ -165,7 +165,7 @@ describe('lib/initalize: ', () => { mockavizta: { path: 'https://github.com/mockavizta', method: 'Mockavizta', - 'global-config': { + config: { verbose: true, }, }, @@ -188,7 +188,7 @@ describe('lib/initalize: ', () => { mockavizta: { path: 'failing-mock', method: 'Mockavizta', - 'global-config': { + config: { verbose: true, }, }, diff --git a/src/__tests__/if-run/lib/regroup.test.ts b/src/__tests__/if-run/lib/regroup.test.ts index 67ff72e53..b4ba074c6 100644 --- a/src/__tests__/if-run/lib/regroup.test.ts +++ b/src/__tests__/if-run/lib/regroup.test.ts @@ -54,7 +54,75 @@ describe('lib/regroup: ', () => { }, }; - const result = Regroup(inputs, groups); + const result = Regroup(inputs, [], groups); + expect(result).toEqual(expectedOutput); + }); + + it('groups inputs combined with outputs correctly.', () => { + const inputs = [ + { + timestamp: '2023-07-06T00:00', + region: 'uk-west', + }, + { + timestamp: '2023-07-06T05:00', + region: 'uk-east1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-east1', + }, + ]; + const outputs = [ + { + timestamp: '2022-06-06T00:00', + region: 'uk-west', + }, + { + timestamp: '2022-06-06T05:00', + region: 'uk-east2', + }, + ]; + const groups = ['region']; + + const expectedOutput = { + 'uk-west': { + inputs: [ + { + region: 'uk-west', + timestamp: '2023-07-06T00:00', + }, + ], + outputs: [ + { + timestamp: '2022-06-06T00:00', + region: 'uk-west', + }, + ], + }, + 'uk-east1': { + inputs: [ + { + timestamp: '2023-07-06T05:00', + region: 'uk-east1', + }, + { + timestamp: '2023-07-06T10:00', + region: 'uk-east1', + }, + ], + }, + 'uk-east2': { + outputs: [ + { + timestamp: '2022-06-06T05:00', + region: 'uk-east2', + }, + ], + }, + }; + + const result = Regroup(inputs, outputs, groups); expect(result).toEqual(expectedOutput); }); @@ -81,7 +149,7 @@ describe('lib/regroup: ', () => { expect.assertions(2); try { - Regroup(inputs, groups!); + Regroup(inputs, [], groups!); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( @@ -113,7 +181,7 @@ describe('lib/regroup: ', () => { expect.assertions(2); try { - Regroup(inputs, groups); + Regroup(inputs, [], groups); } catch (error) { expect(error).toBeInstanceOf(InvalidGroupingError); expect(error).toEqual( @@ -130,7 +198,7 @@ describe('lib/regroup: ', () => { expect.assertions(2); try { - Regroup(inputs, groups); + Regroup(inputs, [], groups); } catch (error) { expect(error).toBeInstanceOf(InputValidationError); expect(error).toEqual( @@ -149,7 +217,7 @@ describe('lib/regroup: ', () => { expect.assertions(2); try { - Regroup(inputs, groups); + Regroup(inputs, [], groups); } catch (error) { expect(error).toBeInstanceOf(InvalidGroupingError); expect(error).toEqual( diff --git a/src/__tests__/if-run/util/aggregation-helper.test.ts b/src/__tests__/if-run/util/aggregation-helper.test.ts index f83536f39..f27eb7874 100644 --- a/src/__tests__/if-run/util/aggregation-helper.test.ts +++ b/src/__tests__/if-run/util/aggregation-helper.test.ts @@ -1,13 +1,10 @@ +import {AGGREGATION_METHODS} from '@grnsft/if-core/consts'; import {ERRORS} from '@grnsft/if-core/utils'; import {PluginParams} from '@grnsft/if-core/types'; import {AggregationParams} from '../../../common/types/manifest'; -import {aggregateInputsIntoOne} from '../../../if-run/util/aggregation-helper'; -import { - AGGREGATION_METHODS, - AggregationMetric, -} from '../../../if-run/types/aggregation'; +import {aggregateOutputsIntoOne} from '../../../if-run/util/aggregation-helper'; import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; import {STRINGS} from '../../../if-run/config'; @@ -22,21 +19,30 @@ describe('util/aggregation-helper: ', () => { type: 'horizontal', }; const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], + [metric]: { + time: AGGREGATION_METHODS[2], + component: AGGREGATION_METHODS[2], + }, })); storeAggregationMetrics(...convertedMetrics); + storeAggregationMetrics({ + carbon: { + time: 'sum', + component: 'sum', + }, + }); }); - describe('aggregateInputsIntoOne(): ', () => { + describe('aggregateOutputsIntoOne(): ', () => { it('throws error if aggregation criteria is not found in input.', () => { const inputs: PluginParams[] = [{timestamp: '', duration: 10}]; - const metrics: AggregationMetric[] = [{'cpu/utilization': 'sum'}]; + const metrics: string[] = ['cpu/utilization']; const isTemporal = false; expect.assertions(2); try { - aggregateInputsIntoOne(inputs, metrics, isTemporal); + aggregateOutputsIntoOne(inputs, metrics, isTemporal); } catch (error) { expect(error).toBeInstanceOf(MissingAggregationParamError); @@ -47,11 +53,17 @@ describe('util/aggregation-helper: ', () => { }); it('passes `timestamp`, `duration` to aggregator if aggregation is temporal.', () => { + storeAggregationMetrics({ + carbon: { + time: 'sum', + component: 'sum', + }, + }); const inputs: PluginParams[] = [ {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric[] = [{carbon: 'sum'}]; + const metrics: string[] = ['carbon']; const isTemporal = true; const expectedValue = { @@ -59,7 +71,7 @@ describe('util/aggregation-helper: ', () => { duration: 10, carbon: inputs[0].carbon + inputs[1].carbon, }; - const aggregated = aggregateInputsIntoOne(inputs, metrics, isTemporal); + const aggregated = aggregateOutputsIntoOne(inputs, metrics, isTemporal); expect(aggregated).toEqual(expectedValue); }); @@ -68,13 +80,13 @@ describe('util/aggregation-helper: ', () => { {timestamp: '', duration: 10, carbon: 10}, {timestamp: '', duration: 10, carbon: 20}, ]; - const metrics: AggregationMetric[] = [{carbon: 'sum'}]; + const metrics: string[] = ['carbon']; const isTemporal = false; const expectedValue = { carbon: inputs[0].carbon + inputs[1].carbon, }; - const aggregated = aggregateInputsIntoOne(inputs, metrics, isTemporal); + const aggregated = aggregateOutputsIntoOne(inputs, metrics, isTemporal); expect(aggregated).toEqual(expectedValue); }); @@ -84,16 +96,24 @@ describe('util/aggregation-helper: ', () => { type: 'horizontal', }; const convertedMetrics = metricStorage.metrics.map((metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], + [metric]: { + time: AGGREGATION_METHODS[2], + component: AGGREGATION_METHODS[2], + }, })); storeAggregationMetrics(...convertedMetrics); - storeAggregationMetrics({'cpu/utilization': 'avg'}); + storeAggregationMetrics({ + 'cpu/utilization': { + time: 'avg', + component: 'avg', + }, + }); const inputs: PluginParams[] = [ {timestamp: '', duration: 10, 'cpu/utilization': 10}, {timestamp: '', duration: 10, 'cpu/utilization': 90}, ]; - const metrics: AggregationMetric[] = [{'cpu/utilization': 'avg'}]; + const metrics: string[] = ['cpu/utilization']; const isTemporal = false; const expectedValue = { @@ -101,7 +121,7 @@ describe('util/aggregation-helper: ', () => { (inputs[0]['cpu/utilization'] + inputs[1]['cpu/utilization']) / inputs.length, }; - const aggregated = aggregateInputsIntoOne(inputs, metrics, isTemporal); + const aggregated = aggregateOutputsIntoOne(inputs, metrics, isTemporal); expect(aggregated).toEqual(expectedValue); expect(aggregated.timestamp).toBeUndefined(); expect(aggregated.duration).toBeUndefined(); diff --git a/src/__tests__/if-run/util/helpers.test.ts b/src/__tests__/if-run/util/helpers.test.ts index 796f6b201..dedc76634 100644 --- a/src/__tests__/if-run/util/helpers.test.ts +++ b/src/__tests__/if-run/util/helpers.test.ts @@ -4,15 +4,7 @@ const mockError = jest.fn(); import {ERRORS} from '@grnsft/if-core/utils'; -import {GlobalPlugins} from '../../../common/types/manifest'; - -import {storeAggregationMetrics} from '../../../if-run/lib/aggregate'; - -import { - andHandle, - mergeObjects, - storeAggregationMethods, -} from '../../../if-run/util/helpers'; +import {andHandle, mergeObjects} from '../../../if-run/util/helpers'; const {WriteFileError} = ERRORS; @@ -179,96 +171,4 @@ describe('if-run/util/helpers: ', () => { expect(result).toEqual(expectedResult); }); }); - - describe('storeAggregationMethods(): ', () => { - // @typescript-eslint/no-unused-vars - const mockPluginStorage = { - get: jest.fn(), - set: jest.fn(() => {}), - }; - - const mockPlugins: GlobalPlugins = { - multiply: { - path: 'builtin', - method: 'Multiply', - }, - sci: { - path: 'builtin', - method: 'Sci', - }, - }; - - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('succefully executes with correct metrics.', () => { - const mockPlugin1 = { - execute: () => [{}], - metadata: { - kind: 'execute', - inputs: { - carbon: { - description: 'mock description', - unit: 'none', - 'aggregation-method': 'sum', - }, - }, - outputs: { - cpu: { - description: 'mock description', - unit: 'none', - 'aggregation-method': 'avg', - }, - }, - }, - }; - - const mockPlugin2 = { - metadata: { - inputs: {}, - outputs: { - carbon: {'aggregation-method': 'none'}, - }, - }, - }; - - mockPluginStorage.get - .mockReturnValueOnce(mockPlugin1) - .mockReturnValueOnce(mockPlugin2); - - // @ts-ignore - storeAggregationMethods(mockPlugins, mockPluginStorage); - - expect(storeAggregationMetrics).toHaveBeenCalledTimes(3); - expect(storeAggregationMetrics).toHaveBeenNthCalledWith(1, { - carbon: 'sum', - }); - expect(storeAggregationMetrics).toHaveBeenNthCalledWith(2, { - cpu: 'avg', - }); - expect(storeAggregationMetrics).toHaveBeenNthCalledWith(3, { - carbon: 'none', - }); - }); - - it('does not execute if there are no inputs or outputs.', () => { - mockPluginStorage.get.mockReturnValueOnce({ - execute: () => [{}], - metadata: {}, - }); - - const mockPlugin = { - execute: () => [{}], - metadata: { - kind: 'execute', - }, - }; - - mockPluginStorage.get.mockReturnValueOnce(mockPlugin); - // @ts-ignore - storeAggregationMethods(mockPlugins, mockPluginStorage); - expect(storeAggregationMetrics).not.toHaveBeenCalled(); - }); - }); }); diff --git a/src/__tests__/if-run/util/plugin-storage.test.ts b/src/__tests__/if-run/util/plugin-storage.test.ts index 885ccb107..90709ea78 100644 --- a/src/__tests__/if-run/util/plugin-storage.test.ts +++ b/src/__tests__/if-run/util/plugin-storage.test.ts @@ -20,7 +20,7 @@ describe('util/pluginStorage: ', () => { const pluginName = 'mock-plugin'; const pluginBody = { execute: () => [{}], - metadata: {kind: 'mock-kind'}, + metadata: {}, }; describe('get(): ', () => { diff --git a/src/common/config/strings.ts b/src/common/config/strings.ts index 82592463f..e5884852b 100644 --- a/src/common/config/strings.ts +++ b/src/common/config/strings.ts @@ -10,4 +10,8 @@ Incubation projects are experimental, offer no support guarantee, have minimal g SUCCESS_MESSAGE: 'The environment is successfully setup!', MANIFEST_IS_MISSING: 'Manifest is missing.', DIRECTORY_NOT_FOUND: 'Directory not found.', + AGGREGATION_UNITS_NOT_MATCH: (param: string) => + `Your manifest uses two instances of ${param} with different units. Please check that you are using consistent units for ${param} throughout your manifest.`, + AGGREGATION_METHODS_NOT_MATCH: (param: string) => + `Your manifest uses two instances of ${param} with different 'aggregation-method'. Please check that you are using right 'aggregation-method' for ${param} throughout your manifest.`, }; diff --git a/src/common/types/manifest.ts b/src/common/types/manifest.ts index 1efab523d..483af708c 100644 --- a/src/common/types/manifest.ts +++ b/src/common/types/manifest.ts @@ -1,6 +1,5 @@ import {z} from 'zod'; - -import {AggregationMethodTypes} from '../../if-run/types/aggregation'; +import {AggregationOptions} from '@grnsft/if-core/types'; import {manifestSchema} from '../util/validations'; @@ -12,7 +11,7 @@ export type PluginOptions = GlobalPlugins[string]; export type AggregationParams = Manifest['aggregation']; export type AggregationMetricsWithMethod = { - [key: string]: AggregationMethodTypes; + [key: string]: AggregationOptions; }; export type AggregationParamsSure = Extract; diff --git a/src/common/util/debug-logger.ts b/src/common/util/debug-logger.ts index 35dc77ea7..9caab3083 100644 --- a/src/common/util/debug-logger.ts +++ b/src/common/util/debug-logger.ts @@ -11,6 +11,9 @@ const logMessagesKeys: (keyof typeof STRINGS)[] = [ 'INITIALIZING_PLUGIN', 'LOADING_PLUGIN_FROM_PATH', 'COMPUTING_PIPELINE_FOR_NODE', + 'COMPUTING_COMPONENT_PIPELINE', + 'REGROUPING', + 'OBSERVING', 'MERGING_DEFAULTS_WITH_INPUT_DATA', 'AGGREGATING_OUTPUTS', 'AGGREGATING_NODE', @@ -96,16 +99,25 @@ const debugLog = (level: LogLevel, args: any[], debugMode: boolean) => { return; } - if (args[0].includes('# start')) { + if (typeof args[0] === 'string' && args[0].includes('# start')) { originalConsole.log(...args); return; } + if (args[0] === '\n') { + originalConsole.log(); + return; + } + const date = new Date().toISOString(); const plugin = pluginNameManager.currentPluginName; - const formattedMessage = `${level}: ${date}: ${ - plugin ? plugin + ': ' : '' - }${args.join(', ')}`; + const isExeption = + typeof args[0] === 'string' && args[0].includes('**Computing'); + const message = `${level}: ${date}: ${plugin ? plugin + ': ' : ''}${args.join( + ', ' + )}`; + + const formattedMessage = isExeption ? args.join(', ') : message; if (debugMode) { switch (level) { diff --git a/src/common/util/validations.ts b/src/common/util/validations.ts index 5cdbec5db..5d0ac758b 100644 --- a/src/common/util/validations.ts +++ b/src/common/util/validations.ts @@ -1,4 +1,5 @@ import {ZodIssue, ZodIssueCode, ZodSchema, z} from 'zod'; +import {AGGREGATION_METHODS} from '@grnsft/if-core/consts'; import {ERRORS} from '@grnsft/if-core/utils'; import {STRINGS} from '../../if-run/config'; @@ -22,32 +23,35 @@ export const allDefined = (obj: Record) => Object.values(obj).every(v => v !== undefined); /** - * Schema for parameter metadata. + * Reusabe aggregation method schema for parameter metadata. + */ +const aggregationMethodSchema = z.object({ + time: z.enum(AGGREGATION_METHODS), + component: z.enum(AGGREGATION_METHODS), +}); + +/** + * Reusable metadata schema. + */ +const metadataSchema = z + .record( + z.string(), + z.object({ + unit: z.string(), + description: z.string(), + 'aggregation-method': aggregationMethodSchema, + }) + ) + .optional() + .nullable(); + +/** + * Reusable parameter metadata schema. */ const parameterMetadataSchema = z .object({ - inputs: z - .record( - z.string(), - z.object({ - unit: z.string(), - description: z.string(), - 'aggregation-method': z.string(), - }) - ) - .optional() - .nullable(), - outputs: z - .record( - z.string(), - z.object({ - unit: z.string(), - description: z.string(), - 'aggregation-method': z.string(), - }) - ) - .optional() - .nullable(), + inputs: metadataSchema, + outputs: metadataSchema, }) .optional(); @@ -71,6 +75,7 @@ export const manifestSchema = z.object({ .object({ metrics: z.array(z.string()), type: z.enum(AGGREGATION_TYPES), + 'skip-components': z.array(z.string()).optional(), }) .optional() .nullable(), @@ -81,7 +86,8 @@ export const manifestSchema = z.object({ .object({ path: z.string(), method: z.string(), - 'global-config': z.record(z.string(), z.any()).optional(), + mapping: z.record(z.string(), z.string()).optional(), + config: z.record(z.string(), z.any()).optional(), 'parameter-metadata': parameterMetadataSchema, }) .optional() diff --git a/src/if-env/config/env-template.yml b/src/if-env/config/env-template.yml index dc801cbc1..4dd25ad63 100644 --- a/src/if-env/config/env-template.yml +++ b/src/if-env/config/env-template.yml @@ -6,7 +6,7 @@ initialize: memory-energy-from-memory-util: # you can name this any way you like! method: Coefficient # the name of the function exported from the plugin path: "builtin" # the import path - global-config: # anmy config required by the plugin + config: # any config required by the plugin input-parameter: "memory/utilization" coefficient: 0.0001 #kwH/GB output-parameter: "memory/energy" @@ -16,7 +16,6 @@ tree: pipeline: # the pipeline is an ordered list of plugins you want to execute compute: - memory-energy-from-memory-util # must match the name in initialize! - config: # any plugin specific, node-level config inputs: - timestamp: 2023-12-12T00:00:00.000Z # ISO 8061 string duration: 3600 # units of seconds diff --git a/src/if-run/builtins/coefficient/README.md b/src/if-run/builtins/coefficient/README.md index 903a15496..2b82fa78c 100644 --- a/src/if-run/builtins/coefficient/README.md +++ b/src/if-run/builtins/coefficient/README.md @@ -8,9 +8,9 @@ For example, you could multiply `cpu/energy` by 10 and name the result `energy-p ## Parameters -### Plugin global config +### Plugin config -Three parameters are required in global config: `input-parameter`, `coefficient` and `output-parameter`. +Three parameters are required in config: `input-parameter`, `coefficient` and `output-parameter`. - `input-parameter`: a string matching an existing key in the `inputs` array - `coefficient`: the value to multiply `input-parameter` by. @@ -21,16 +21,32 @@ Three parameters are required in global config: `input-parameter`, `coefficient` The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe parameters of the `input-parameter` of the global config. Each parameter has: +- `inputs`: describe parameters of the `input-parameter` of the config. Each parameter has: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. -- `outputs`: describe parameters of the `output-parameter` of the global config. Each parameter has: +- `outputs`: describe parameters of the `output-parameter` of the config. Each parameter has: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +coefficient: + method: Coefficient + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -38,7 +54,7 @@ All of `input-parameters` must be available in the input array. ## Returns -- `output-parameter`: the product of all `input-parameters` with the parameter name defined by `output-parameter` in global config. +- `output-parameter`: the product of all `input-parameters` with the parameter name defined by `output-parameter` in config. ## Calculation @@ -56,9 +72,11 @@ const config = { coefficient: 10, 'output-parameter': 'carbon-product', }; +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = {}; -const coeff = Coefficient(config); -const result = coeff.execute([ +const coeff = Coefficient(config, parametersMetadata, mapping); +const result = await coeff.execute([ { duration: 3600, timestamp: '2021-01-01T00:00:00Z', @@ -80,19 +98,25 @@ initialize: coefficient: method: Coefficient path: 'builtin' - global-config: + config: input-parameter: 'carbon' coefficient: 3 output-parameter: 'carbon-product' - parameter-metadata: + parameter-metadata: inputs: carbon: - description: "an amount of carbon emitted into the atmosphere" - unit: "gCO2e" + description: 'an amount of carbon emitted into the atmosphere' + unit: 'gCO2e' + aggregation-method: + time: sum + component: sum outputs: carbon-product: - description: "a product of cabon property and the coefficient" - unit: "gCO2e" + description: 'a product of cabon property and the coefficient' + unit: 'gCO2e' + aggregation-method: + time: sum + component: sum tree: children: child: @@ -117,9 +141,9 @@ The results will be saved to a new `yaml` file in `./examples/outputs` `Coefficient` exposes one of the IF error classes. -### GlobalConfigError +### ConfigError -You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. +You will receive an error starting `ConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: diff --git a/src/if-run/builtins/coefficient/index.ts b/src/if-run/builtins/coefficient/index.ts index 177eeece3..99895482c 100644 --- a/src/if-run/builtins/coefficient/index.ts +++ b/src/if-run/builtins/coefficient/index.ts @@ -1,92 +1,67 @@ -import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; -import { - CoefficientConfig, - ExecutePlugin, - PluginParametersMetadata, - PluginParams, -} from '@grnsft/if-core/types'; +import {z, ZodType} from 'zod'; + +import {ERRORS, validateArithmeticExpression} from '@grnsft/if-core/utils'; +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; -const {GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; - -export const Coefficient = ( - globalConfig: CoefficientConfig, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; - /** - * Calculate the product of each input parameter. - */ - const execute = (inputs: PluginParams[]) => { - const safeGlobalConfig = validateGlobalConfig(); - const inputParameter = safeGlobalConfig['input-parameter']; - const outputParameter = safeGlobalConfig['output-parameter']; - const coefficient = safeGlobalConfig['coefficient']; - - return inputs.map(input => { - validateSingleInput(input, inputParameter); +export const Coefficient = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); + } - return { - ...input, - [outputParameter]: calculateProduct(input, inputParameter, coefficient), - }; + const configSchema = z.object({ + coefficient: z.preprocess( + value => validateArithmeticExpression('coefficient', value, 'number'), + z.number() + ), + 'input-parameter': z.string().min(1), + 'output-parameter': z.string().min(1), }); - }; - /** - * Checks for required fields in input. - */ - const validateSingleInput = (input: PluginParams, inputParameter: string) => { + return validate>( + configSchema as ZodType, + config + ); + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { const inputData = { - 'input-parameter': input[inputParameter], + 'input-parameter': input[config['input-parameter']], }; const validationSchema = z.record(z.string(), z.number()); validate(validationSchema, inputData); return input; - }; - - /** - * Calculates the product of the energy components. - */ - const calculateProduct = ( - input: PluginParams, - inputParameter: string, - coefficient: number - ) => input[inputParameter] * coefficient; + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + const { + 'input-parameter': inputParameter, + 'output-parameter': outputParameter, + coefficient, + } = config; - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - if (!globalConfig) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); - } - - const globalConfigSchema = z.object({ - coefficient: z.number(), - 'input-parameter': z.string().min(1), - 'output-parameter': z.string().min(1), - }); - - return validate>( - globalConfigSchema, - globalConfig - ); - }; + return inputs.map(input => ({ + ...input, + [outputParameter]: calculateProduct(input, inputParameter, coefficient), + })); + }, + allowArithmeticExpressions: ['input-parameter', 'coefficient'], +}); - return { - metadata, - execute, - }; -}; +/** + * Calculates the product of the energy components. + */ +const calculateProduct = ( + input: PluginParams, + inputParameter: string | number, + coefficient: number +) => + (isNaN(Number(inputParameter)) ? input[inputParameter] : inputParameter) * + coefficient; diff --git a/src/if-run/builtins/copy-param/README.md b/src/if-run/builtins/copy-param/README.md index 2f76c98da..a13b0915b 100644 --- a/src/if-run/builtins/copy-param/README.md +++ b/src/if-run/builtins/copy-param/README.md @@ -43,16 +43,32 @@ Three parameters are required in config: `from` and `to` and `keep-existing`. The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe the parameter of the `from` of the global config. The parameter has the following attributes: +- `inputs`: describe the parameter of the `from` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. -- `outputs`: describe the parameters of the `to` of the global config. The parameter has the following attributes: +- `outputs`: describe the parameters of the `to` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +copy-param: + path: builtin + method: Copy + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -69,13 +85,17 @@ To run the plugin, you must first create an instance of `Copy`. Then, you can ca ```typescript import {Copy} from '.'; -const plugin = Copy({ +const config = { 'keep-existing': true, from: 'from-param', to: 'to-param', -}); +}; +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = {}; + +const plugin = Copy(config, parametersMetadata, mapping); -const result = plugin.execute([ +const result = await plugin.execute([ { timestamp: '2023-12-12T00:00:13.000Z', duration: 30, @@ -99,7 +119,7 @@ initialize: copy-param: path: builtin method: Copy - global-config: + config: keep-existing: true from: original to: copy diff --git a/src/if-run/builtins/copy-param/index.ts b/src/if-run/builtins/copy-param/index.ts index 7f10bf696..3e0dfb761 100644 --- a/src/if-run/builtins/copy-param/index.ts +++ b/src/if-run/builtins/copy-param/index.ts @@ -1,99 +1,72 @@ import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; + import { - ExecutePlugin, - PluginParametersMetadata, - PluginParams, -} from '@grnsft/if-core/types'; + ERRORS, + getParameterFromArithmeticExpression, +} from '@grnsft/if-core/utils'; +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; -const {MISSING_GLOBAL_CONFIG} = STRINGS; -const {GlobalConfigError} = ERRORS; -// keep-existing: true/false (whether to remove the parameter you are copying from) -// from-param: the parameter you are copying from (e.g. cpu/name) -// to-field: the parameter you are copying to (e.g. cpu/processor-name) +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; -export const Copy = ( - globalConfig: Record, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; +/** + * keep-existing: true/false (whether to remove the parameter you are copying from) + * from-param: the parameter you are copying from (e.g. cpu/name) + * to-field: the parameter you are copying to (e.g. cpu/processor-name) + */ - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - if (!globalConfig) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); +export const Copy = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); } - const globalConfigSchema = z.object({ + const configSchema = z.object({ 'keep-existing': z.boolean(), from: z.string().min(1), to: z.string().min(1), }); - return validate>( - globalConfigSchema, - globalConfig - ); - }; - - /** - * Checks for required fields in input. - */ - const validateSingleInput = ( - input: PluginParams, - inputParameters: string[] - ) => { - const inputData = inputParameters.reduce( - (acc, param) => { - acc[param] = input[param]; - - return acc; - }, - {} as Record - ); - - const validationSchema = z.record(z.string(), z.string()); - - validate(validationSchema, inputData); - - return input; - }; - - const execute = (inputs: PluginParams[]) => { - const safeGlobalConfig = validateGlobalConfig(); - const keepExisting = safeGlobalConfig['keep-existing'] === true; - const from = safeGlobalConfig['from']; - const to = safeGlobalConfig['to']; + const extractedFrom = getParameterFromArithmeticExpression(config.from); + const updatedConfig = config['keep-existing'] + ? config + : {...config, 'pure-from': extractedFrom}; + + validate>(configSchema, updatedConfig); + + return updatedConfig; + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { + const from = config.from; + const inputData = { + [from]: input[from], + }; + const validationSchema = z.record(z.string(), z.string().or(z.number())); + + return validate(validationSchema, inputData); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + const keepExisting = config['keep-existing'] === true; + const from = config['from']; + const to = config['to']; return inputs.map(input => { - const safeInput = validateSingleInput(input, [from]); + const outputValue = !isNaN(from) ? from : input[from]; - const outputValue = safeInput[from]; - if (safeInput[from]) { - if (!keepExisting) { - delete safeInput[from]; - } + if (input[from] || (!isNaN(from) && !keepExisting)) { + delete input[config['pure-from']]; } return { - ...safeInput, // need to return or what you provide won't be outputted, don't be evil! + ...input, [to]: outputValue, }; }); - }; - - return { - metadata, - execute, - }; -}; + }, + allowArithmeticExpressions: ['from'], +}); diff --git a/src/if-run/builtins/csv-lookup/README.md b/src/if-run/builtins/csv-lookup/README.md index 434e234a2..d9d8c3903 100644 --- a/src/if-run/builtins/csv-lookup/README.md +++ b/src/if-run/builtins/csv-lookup/README.md @@ -61,12 +61,28 @@ The `parameter-metadata` section contains information about `description`, `unit - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. - `outputs`: describe the parameters in the `output` of the config block. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +cloud-metadata: + method: CSVLookup + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -78,7 +94,7 @@ The input data with the requested csv content appended to it. ## Plugin logic -1. Validates global config which contains `filepath`, `query` and `output`. +1. Validates config which contains `filepath`, `query` and `output`. 2. Tries to retrieve given file (with url or local path). 3. Parses given CSV. 4. Filters requested information from CSV. @@ -90,7 +106,7 @@ The input data with the requested csv content appended to it. To run the plugin, you must first create an instance of `CSVLookup`. Then, you can call `execute()`. ```typescript -const globalConfig = { +const config = { filepath: 'https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/cloud-metdata-aws-instances.csv', query: { 'cloud-provider': 'cloud/provider' @@ -99,9 +115,11 @@ const globalConfig = { }, output: ['cpu-tdp', 'tdp'], }; -const csvLookup = CSVLookup(globalConfig); +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = {}; +const csvLookup = CSVLookup(config, parametersMetadata, mapping); -const input = [ +const result = await csvLookup.execute([ { timestamp: '2023-08-06T00:00' duration: 3600 @@ -109,7 +127,7 @@ const input = [ 'cloud/provider': gcp 'cloud/region': asia-east }, -]; +]); ``` ## Example manifest @@ -125,12 +143,14 @@ initialize: cloud-metadata: method: CSVLookup path: 'builtin' - global-config: + config: filepath: https://raw.githubusercontent.com/Green-Software-Foundation/if-data/main/region-metadata.csv query: cloud-provider: 'cloud/provider' cloud-region: 'cloud/region' output: '*' + mapping: + cloud/region: cloud/area tree: children: child: @@ -141,7 +161,7 @@ tree: - timestamp: 2023-08-06T00:00 duration: 3600 cloud/provider: Google Cloud - cloud/region: europe-north1 + cloud/area: europe-north1 ``` You can run this example by saving it as `./examples/manifests/csv-lookup.yml` and executing the following command from the project root: @@ -177,9 +197,9 @@ This error is caused by the `CsvLookup` plugin failing to find data that matches This error arises due to problems parsing CSV data into IF. This can occur when the CSV data is incorrectly formatted or contains unexpected characters that IF does not recognize. These errors are expected to be unusual edge cases as incorrectly formatted data will usually be identified during file loading and cause a `ReadFileError`. To debug, check your CSV file for any unexpected formatting or unusual characters. -### GlobalConfigError +### ConfigError -You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. +You will receive an error starting `ConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: diff --git a/src/if-run/builtins/csv-lookup/index.ts b/src/if-run/builtins/csv-lookup/index.ts index e227eaadf..08f7760c6 100644 --- a/src/if-run/builtins/csv-lookup/index.ts +++ b/src/if-run/builtins/csv-lookup/index.ts @@ -1,17 +1,12 @@ /* eslint-disable eqeqeq */ import {readFile} from 'fs/promises'; - import axios from 'axios'; import {z} from 'zod'; import {parse} from 'csv-parse/sync'; -import {ERRORS} from '@grnsft/if-core/utils'; -import { - ExecutePlugin, - PluginParametersMetadata, - PluginParams, -} from '@grnsft/if-core/types'; -import {validate} from '../../../common/util/validations'; +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; +import {ERRORS, validate} from '@grnsft/if-core/utils'; import {STRINGS} from '../../config'; @@ -19,8 +14,8 @@ const { FILE_FETCH_FAILED, FILE_READ_FAILED, MISSING_CSV_COLUMN, + MISSING_CONFIG, NO_QUERY_DATA, - MISSING_GLOBAL_CONFIG, } = STRINGS; const { @@ -28,178 +23,34 @@ const { ReadFileError, MissingCSVColumnError, QueryDataNotFoundError, - GlobalConfigError, + ConfigError, CSVParseError, } = ERRORS; -export const CSVLookup = ( - globalConfig: any, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; - - /** - * Checks if given string is URL. - */ - const isURL = (filepath: string) => { - try { - new URL(filepath); - return true; - } catch (error) { - return false; - } - }; - - /** - * Checks if given `filepath` is url, then tries to fetch it. - * Otherwise tries to read file. - */ - const retrieveFile = async (filepath: string) => { - if (isURL(filepath)) { - const {data} = await axios.get(filepath).catch(error => { - throw new FetchingFileError( - FILE_FETCH_FAILED(filepath, error.response.message) - ); - }); - - return data; +export const CSVLookup = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); } - return readFile(filepath).catch(error => { - throw new ReadFileError(FILE_READ_FAILED(filepath, error)); + const configSchema = z.object({ + filepath: z.string(), + query: z.record(z.string(), z.string()), + output: z + .string() + .or(z.array(z.string())) + .or(z.array(z.array(z.string()))), }); - }; - - /** - * Checks if value is invalid: `undefined`, `null` or an empty string, then sets `nan` instead. - */ - const setNanValue = (value: any) => - value == null || value === '' ? 'nan' : value; - - /** - * Converts empty values to `nan`. - */ - const nanifyEmptyValues = (object: any) => { - if (typeof object === 'object') { - const keys = Object.keys(object); - - keys.forEach(key => { - const value = object[key]; - object[key] = setNanValue(value); - }); - - return object; - } - - return setNanValue(object); - }; - - /** - * If `field` is missing from `object`, then reject with error. - * Otherwise nanify empty values and return data. - */ - const fieldAccessor = (field: string, object: any) => { - if (!(`${field}` in object)) { - throw new MissingCSVColumnError(MISSING_CSV_COLUMN(field)); - } - - return nanifyEmptyValues(object[field]); - }; - - /** - * 1. If output is anything, then removes query data from csv record to escape duplicates. - * 2. Otherwise checks if it's a miltidimensional array, then grabs multiple fields (). - * 3. If not, then returns single field. - * 4. In case if it's string, then - */ - const filterOutput = ( - dataFromCSV: any, - params: { - output: string | string[] | string[][]; - query: Record; - } - ) => { - const {output, query} = params; - - if (output === '*') { - const keys = Object.keys(query); - - keys.forEach(key => { - delete dataFromCSV[key]; - }); - - return nanifyEmptyValues(dataFromCSV); - } - - if (Array.isArray(output)) { - /** Check if it's a multidimensional array. */ - if (Array.isArray(output[0])) { - const result: any = {}; - - output.forEach(outputField => { - /** Check if there is no renaming request, then export as is */ - const outputTitle = outputField[1] || outputField[0]; - result[outputTitle] = fieldAccessor(outputField[0], dataFromCSV); - }); - - return result; - } - - const outputTitle = output[1] || output[0]; - - return { - [outputTitle as string]: fieldAccessor(output[0], dataFromCSV), - }; - } - - return { - [output]: fieldAccessor(output, dataFromCSV), - }; - }; - - /** - * Asserts CSV record with query data. - */ - const withCriteria = (queryData: Record) => (csvRecord: any) => { - const ifMatchesCriteria = Object.keys(queryData).map( - (key: string) => csvRecord[key] == queryData[key] - ); - - return ifMatchesCriteria.every(value => value === true); - }; - - /** - * Parses CSV file. - */ - const parseCSVFile = (file: string | Buffer) => { - try { - const parsedCSV: any[] = parse(file, { - columns: true, - skip_empty_lines: true, - cast: true, - }); - - return parsedCSV; - } catch (error: any) { - console.error(error); - throw new CSVParseError(error); - } - }; - - /** - * 1. Validates global config. - * 2. Tries to retrieve given file (with url or local path). - * 3. Parses given CSV. - * 4. Filters requested information from CSV. - */ - const execute = async (inputs: PluginParams[]) => { - const safeGlobalConfig = validateGlobalConfig(); - const {filepath, query, output} = safeGlobalConfig; + return validate>(configSchema, config); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + /** + * 1. Tries to retrieve given file (with url or local path). + * 2. Parses given CSV. + * 3. Filters requested information from CSV. + */ + const {filepath, query, output} = config; const file = await retrieveFile(filepath); const parsedCSV = parseCSVFile(file); @@ -224,33 +75,153 @@ export const CSVLookup = ( ...filterOutput(relatedData, {output, query}), }; }); - }; + }, +}); + +/** + * Checks if given string is URL. + */ +const isURL = (filepath: string) => { + try { + new URL(filepath); + return true; + } catch (error) { + return false; + } +}; - /** - * Checks for `filepath`, `query` and `output` fields in global config. - */ - const validateGlobalConfig = () => { - if (!globalConfig) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); - } +/** + * Checks if given `filepath` is url, then tries to fetch it. + * Otherwise tries to read file. + */ +const retrieveFile = async (filepath: string) => { + if (isURL(filepath)) { + const {data} = await axios.get(filepath).catch(error => { + throw new FetchingFileError( + FILE_FETCH_FAILED(filepath, error.response.message) + ); + }); - const globalConfigSchema = z.object({ - filepath: z.string(), - query: z.record(z.string(), z.string()), - output: z - .string() - .or(z.array(z.string())) - .or(z.array(z.array(z.string()))), + return data; + } + + return readFile(filepath).catch(error => { + throw new ReadFileError(FILE_READ_FAILED(filepath, error)); + }); +}; + +/** + * Checks if value is invalid: `undefined`, `null` or an empty string, then sets `nan` instead. + */ +const setNanValue = (value: any) => + value == null || value === '' ? 'nan' : value; + +/** + * Converts empty values to `nan`. + */ +const nanifyEmptyValues = (object: any) => { + if (typeof object === 'object') { + const keys = Object.keys(object); + + keys.forEach(key => { + const value = object[key]; + object[key] = setNanValue(value); }); - return validate>( - globalConfigSchema, - globalConfig - ); - }; + return object; + } + + return setNanValue(object); +}; + +/** + * If `field` is missing from `object`, then reject with error. + * Otherwise nanify empty values and return data. + */ +const fieldAccessor = (field: string, object: any) => { + if (!(`${field}` in object)) { + throw new MissingCSVColumnError(MISSING_CSV_COLUMN(field)); + } + + return nanifyEmptyValues(object[field]); +}; + +/** + * 1. If output is anything, then removes query data from csv record to escape duplicates. + * 2. Otherwise checks if it's a miltidimensional array, then grabs multiple fields (). + * 3. If not, then returns single field. + * 4. In case if it's string, then + */ +const filterOutput = ( + dataFromCSV: any, + params: { + output: string | string[] | string[][]; + query: Record; + } +) => { + const {output, query} = params; + + if (output === '*') { + const keys = Object.keys(query); + + keys.forEach(key => { + delete dataFromCSV[key]; + }); + + return nanifyEmptyValues(dataFromCSV); + } + + if (Array.isArray(output)) { + /** Check if it's a multidimensional array. */ + if (Array.isArray(output[0])) { + const result: any = {}; + + output.forEach(outputField => { + /** Check if there is no renaming request, then export as is */ + const outputTitle = outputField[1] || outputField[0]; + result[outputTitle] = fieldAccessor(outputField[0], dataFromCSV); + }); + + return result; + } + + const outputTitle = output[1] || output[0]; + + return { + [outputTitle as string]: fieldAccessor(output[0], dataFromCSV), + }; + } return { - metadata, - execute, + [output]: fieldAccessor(output, dataFromCSV), }; }; + +/** + * Asserts CSV record with query data. + */ +const withCriteria = (queryData: Record) => (csvRecord: any) => { + const ifMatchesCriteria = Object.keys(queryData).map( + (key: string) => csvRecord[key] == queryData[key] + ); + + return ifMatchesCriteria.every(value => value === true); +}; + +/** + * Parses CSV file. + */ +const parseCSVFile = (file: string | Buffer) => { + try { + const parsedCSV: any[] = parse(file, { + columns: true, + skip_empty_lines: true, + cast: true, + }); + + return parsedCSV; + } catch (error: any) { + console.error(error); + throw new CSVParseError(error); + } +}; diff --git a/src/if-run/builtins/divide/README.md b/src/if-run/builtins/divide/README.md index abb534c8f..6d536696f 100644 --- a/src/if-run/builtins/divide/README.md +++ b/src/if-run/builtins/divide/README.md @@ -16,16 +16,32 @@ You provide the names of the values you want to divide, and a name to use to add The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe the parameter of the `numerator` of the global config. The parameter has the following attributes: +- `inputs`: describe the parameter of the `numerator` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. - `outputs`: describe the parameter of the `denominator` of the global config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +divide: + method: Divide + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -35,7 +51,7 @@ The `parameter-metadata` section contains information about `description`, `unit ## Returns -- `output`: the division of `numerator` with the parameter name into `denominator` with the parameter name defined by `output` in global config. +- `output`: the division of `numerator` with the parameter name into `denominator` with the parameter name defined by `output` in config. The plugin throws an exception if the division result is not a number. @@ -52,20 +68,24 @@ output = input0 / input1 To run the plugin, you must first create an instance of `Divide`. Then, you can call `execute()`. ```typescript -const globalConfig = { +const config = { numerator: 'vcpus-allocated', denominator: 2, output: 'cpu/number-cores', }; -const divide = Divide(globalConfig, parametersMetadata); +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = { + 'vcpus-allocated': 'vcpus-distributed', +}; +const divide = Divide(config, parametersMetadata, mapping); -const input = [ +const result = await divide.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, 'vcpus-allocated': 24, }, -]; +]); ``` ## Example manifest @@ -81,10 +101,12 @@ initialize: divide: method: Divide path: 'builtin' - global-config: + config: numerator: vcpus-allocated denominator: 2 output: cpu/number-cores + mapping: + vcpus-allocated: vcpus-distributed tree: children: child: @@ -94,7 +116,7 @@ tree: inputs: - timestamp: 2023-08-06T00:00 duration: 3600 - vcpus-allocated: 24 + vcpus-distributed: 24 ``` You can run this example by saving it as `./examples/manifests/divide.yml` and executing the following command from the project root: @@ -110,9 +132,9 @@ The results will be saved to a new `yaml` file in `./examples/outputs`. `Divide` exposes two of IF's error classes. -### GlobalConfigError +### ConfigError -You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. +You will receive an error starting `ConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: diff --git a/src/if-run/builtins/divide/index.ts b/src/if-run/builtins/divide/index.ts index 3d09c130c..9252bc305 100644 --- a/src/if-run/builtins/divide/index.ts +++ b/src/if-run/builtins/divide/index.ts @@ -1,56 +1,20 @@ import {z} from 'zod'; + +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; import {ERRORS} from '@grnsft/if-core/utils'; -import { - ExecutePlugin, - PluginParams, - ConfigParams, - PluginParametersMetadata, -} from '@grnsft/if-core/types'; import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; -const {GlobalConfigError, MissingInputDataError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, ZERO_DIVISION} = STRINGS; - -export const Divide = ( - globalConfig: ConfigParams, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; - - /** - * Calculate the division of each input parameter. - */ - const execute = (inputs: PluginParams[]) => { - const safeGlobalConfig = validateGlobalConfig(); - const {numerator, denominator, output} = safeGlobalConfig; +const {MissingInputDataError, ConfigError} = ERRORS; +const {MISSING_INPUT_DATA, ZERO_DIVISION, MISSING_CONFIG} = STRINGS; - return inputs.map((input, index) => { - const safeInput = Object.assign( - {}, - input, - validateSingleInput(input, {numerator, denominator}) - ); - - return { - ...input, - [output]: calculateDivide(safeInput, index, {numerator, denominator}), - }; - }); - }; - - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - if (!globalConfig) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); +export const Divide = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); } const schema = z.object({ @@ -59,20 +23,10 @@ export const Divide = ( output: z.string(), }); - return validate>(schema, globalConfig); - }; - - /** - * Checks for required fields in input. - */ - const validateSingleInput = ( - input: PluginParams, - params: { - numerator: string; - denominator: number | string; - } - ) => { - const {numerator, denominator} = params; + return validate>(schema, config); + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { + const {numerator, denominator} = config; const schema = z .object({ @@ -88,32 +42,48 @@ export const Divide = ( }); return validate>(schema, input); - }; - - /** - * Calculates the division of the given parameter. - */ - const calculateDivide = ( - input: PluginParams, - index: number, - params: { - numerator: string; - denominator: number | string; - } - ) => { - const {denominator, numerator} = params; - const finalDenominator = input[denominator] || denominator; + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + const {numerator, denominator, output} = config; - if (finalDenominator === 0) { - console.warn(ZERO_DIVISION(Divide.name, index)); - return input[numerator]; - } - - return input[numerator] / finalDenominator; - }; + return inputs.map((input, index) => { + const calculatedResult = calculateDivide(input, index, { + numerator: input.numerator || numerator, + denominator: input.denominator || denominator, + }); - return { - metadata, - execute, - }; + return { + ...input, + [output]: calculatedResult, + }; + }); + }, + allowArithmeticExpressions: ['numerator', 'denominator'], +}); + +/** + * Calculates the division of the given parameter. + */ +const calculateDivide = ( + input: PluginParams, + index: number, + params: { + numerator: number | string; + denominator: number | string; + } +) => { + const {denominator, numerator} = params; + const finalDenominator = + typeof denominator === 'number' + ? denominator + : input[denominator] || denominator; + const finalNumerator = + typeof numerator === 'number' ? numerator : input[numerator]; + + if (finalDenominator === 0) { + console.warn(ZERO_DIVISION(Divide.name, index)); + return finalNumerator; + } + + return finalNumerator / finalDenominator; }; diff --git a/src/if-run/builtins/exponent/README.md b/src/if-run/builtins/exponent/README.md index fddb55ffa..8306be0b4 100644 --- a/src/if-run/builtins/exponent/README.md +++ b/src/if-run/builtins/exponent/README.md @@ -10,7 +10,7 @@ For example, you use `cpu/energy` as base and `network/energy` as and name the r ### Plugin config -Three parameters are required in global config: `input-parameter`, `exponent` and `output-parameter`. +Three parameters are required in config: `input-parameter`, `exponent` and `output-parameter`. `input-parameter`: a string defining the base. Must match an existing key in the `inputs` array `exponent`: a number defining the exponent. @@ -20,16 +20,32 @@ Three parameters are required in global config: `input-parameter`, `exponent` an The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe the parameter of the `input-parameter` of the global config. The parameter has the following attributes: +- `inputs`: describe the parameter of the `input-parameter` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. -- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes:: +- `outputs`: describe the parameter of the `output-parameter` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +exponent: + method: Exponent + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -37,7 +53,7 @@ The `parameter-metadata` section contains information about `description`, `unit ## Returns -- `output-parameter`: `input-parameter` raised by `exponent` with the parameter name defined by `output-parameter` in global config. +- `output-parameter`: `input-parameter` raised by `exponent` with the parameter name defined by `output-parameter` in config. ## Calculation @@ -53,12 +69,14 @@ To run the plugin, you must first create an instance of `Exponent`. Then, you ca import {Exponent} from 'builtins'; const config = { - inputParameter: ['cpu/energy'], - exponent: 2 - outputParameter: 'energy', + inputParameter: ['cpu/energy'], + exponent: 2 + outputParameter: 'energy', }; +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = {}; -const exponent = Exponent(config); +const exponent = Exponent(config, parametersMetadata, mapping); const result = await exponent.execute([ { duration: 3600, @@ -82,7 +100,7 @@ initialize: exponent: method: Exponent path: 'builtin' - global-config: + config: input-parameter: 'cpu/energy' exponent: 2 output-parameter: 'energy' diff --git a/src/if-run/builtins/exponent/index.ts b/src/if-run/builtins/exponent/index.ts index a821c3a14..464ce9367 100644 --- a/src/if-run/builtins/exponent/index.ts +++ b/src/if-run/builtins/exponent/index.ts @@ -1,87 +1,81 @@ -import {z} from 'zod'; -import { - ExecutePlugin, - PluginParams, - ExponentConfig, - PluginParametersMetadata, -} from '@grnsft/if-core/types'; +import {z, ZodType} from 'zod'; + +import {PluginParams, ConfigParams} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; +import {ERRORS} from '@grnsft/if-core/utils'; import {validate} from '../../../common/util/validations'; -export const Exponent = ( - globalConfig: ExponentConfig, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; +import {STRINGS} from '../../config'; + +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - const globalConfigSchema = z.object({ +export const Exponent = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); + } + + const configSchema = z.object({ 'input-parameter': z.string().min(1), exponent: z.number(), 'output-parameter': z.string().min(1), }); - return validate>( - globalConfigSchema, - globalConfig + return validate>( + configSchema as ZodType, + config ); - }; - - /** - * Checks for required fields in input. - */ - const validateSingleInput = (input: PluginParams, inputParameter: string) => { + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { + const inputParameter = config['input-parameter']; const inputData = { - 'input-parameter': input[inputParameter], + [inputParameter]: + typeof inputParameter === 'number' + ? inputParameter + : input[inputParameter], }; const validationSchema = z.record(z.string(), z.number()); - validate(validationSchema, inputData); - return input; - }; - - /** - * Calculate the input param raised by to the power of the given exponent. - */ - const execute = (inputs: PluginParams[]): PluginParams[] => { + return validate>( + validationSchema, + inputData + ); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams = {}) => { const { 'input-parameter': inputParameter, exponent, 'output-parameter': outputParameter, - } = validateGlobalConfig(); + } = config; return inputs.map(input => { - validateSingleInput(input, inputParameter); + const calculatedResult = calculateExponent( + input, + inputParameter, + exponent + ); return { ...input, - [outputParameter]: calculateExponent(input, inputParameter, exponent), + [outputParameter]: calculatedResult, }; }); - }; - - /** - * Calculates the input param raised by the power of a given exponent. - */ - const calculateExponent = ( - input: PluginParams, - inputParameter: string, - exponent: number - ) => { - const base = input[inputParameter]; + }, + allowArithmeticExpressions: ['input-parameter', 'exponent'], +}); - return Math.pow(base, exponent); - }; +/** + * Calculates the input param raised by the power of a given exponent. + */ +const calculateExponent = ( + input: PluginParams, + inputParameter: string | number, + exponent: number +) => { + const base = + typeof inputParameter === 'number' ? inputParameter : input[inputParameter]; - return { - metadata, - execute, - }; + return Math.pow(base, exponent); }; diff --git a/src/if-run/builtins/interpolation/README.md b/src/if-run/builtins/interpolation/README.md index 243b70e70..d328dc9aa 100644 --- a/src/if-run/builtins/interpolation/README.md +++ b/src/if-run/builtins/interpolation/README.md @@ -9,13 +9,13 @@ This plugin provides the `y` value at a given `x` by interpolating between known To employ the `Interpolation` plugin, adhere to these steps: -1. **Initialize Plugin**: Import the `Interpolation` function and initialize it with global configuration parameters `method`, `x`, `y`, `input-parameter` and `output-parameter`. +1. **Initialize Plugin**: Import the `Interpolation` function and initialize it with configuration parameters `method`, `x`, `y`, `input-parameter` and `output-parameter`. 2. **Execute Plugin**: Invoke the `execute` method of the initialized plugin instance with an array of input parameters. Each input parameter should include a `timestamp`, `duration` and `[input-parameter]` information. 3. **Result**: The plugin will return an array of plugin parameters enriched with the calculated average carbon intensity for each input. -## Global Config +## Config - `method`: specifies the interpolation method for the data. Acceptable values are 'linear', 'spline', or 'polynomial'. The default method is linear. (optional) - `x`: array of x points. Numbers should be in ascending order (required). @@ -29,16 +29,32 @@ To employ the `Interpolation` plugin, adhere to these steps: The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe the parameter of the `input-parameter` of the global config. The parameter has the following attributes: +- `inputs`: describe the parameter of the `input-parameter` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. -- `outputs`: describe the parameters of the `output-parameter` of the global config. The parameter has the following attributes: +- `outputs`: describe the parameters of the `output-parameter` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +interpolation: + method: Interpolation + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ## Input Parameters @@ -46,7 +62,7 @@ The plugin expects the following input parameters: - `timestamp`: a timestamp for the input (required) - `duration`: the amount of time, in seconds, that the input covers. (required) -- `[input-parameter]` - a field whose name matches the string provided to input-parameter in global config (i.e. if the input-parameter in global config is cpu/utilisation then cpu-utilisation must exist in the input data) +- `[input-parameter]` - a field whose name matches the string provided to input-parameter in config (i.e. if the input-parameter in config is cpu/utilisation then cpu-utilisation must exist in the input data) ## Output @@ -60,7 +76,7 @@ The plugin conducts input validation using the `zod` library and may throw error 1. **Execution**: - - Validate Global config + - Validate config - `method` - validates if the method is one of these methods: `linear`, `spline`, or `polynomial`. If the method isn’t provided, it sets to `linear`. - `x` and `y` should be arrays of numbers, the length should be equal, and elements should be ordered in the ascendant order. @@ -84,28 +100,27 @@ The plugin conducts input validation using the `zod` library and may throw error ### TypeScript Usage ```ts -const globalConfig = { +const config = { method: 'linear', x: [0, 10, 50, 100], y: [0.12, 0.32, 0.75, 1.02], - 'input-parameter': 'cpu/utilization' - 'output-parameter': 'cpu/energy' - + 'input-parameter': 'cpu/utilization', + 'output-parameter': 'cpu/energy', }; - -const interpolationPlugin = Interpolation(globalConfig); +const parametersMetadata = {inputs: {}, outputs: {}}; +const interpolationPlugin = Interpolation(config, parametersMetadata, {}); const inputs = [ { timestamp: '2024-04-16T12:00:00Z', duration: 3600, - 'cpu/utilization': 45 + 'cpu/utilization': 45, }, ]; -const results = interpolationPlugin.execute(inputs); +const result = await interpolationPlugin.execute(inputs); -console.log(results); +console.log(result); ``` ### Manifest Usage @@ -121,7 +136,7 @@ initialize: interpolation: method: Interpolation path: 'builtin' - global-config: + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] @@ -150,7 +165,7 @@ initialize: interpolation: method: Interpolation path: 'builtin' - global-config: + config: method: linear x: [0, 10, 50, 100] y: [0.12, 0.32, 0.75, 1.02] @@ -184,11 +199,11 @@ if-run --manifest ./manifests/examples/interpolation.yml --output ./manifests/ou `Interpolation` exposes one of IF's error classes. -## `GlobalConfigError` +## `ConfigError` -### GlobalConfigError +### ConfigError -You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. +You will receive an error starting `ConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: diff --git a/src/if-run/builtins/interpolation/index.ts b/src/if-run/builtins/interpolation/index.ts index 60fe90e5b..30226e5a6 100644 --- a/src/if-run/builtins/interpolation/index.ts +++ b/src/if-run/builtins/interpolation/index.ts @@ -1,144 +1,23 @@ import Spline from 'typescript-cubic-spline'; import {z} from 'zod'; + +import {PluginParams, ConfigParams, Method} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; import {ERRORS} from '@grnsft/if-core/utils'; -import { - ExecutePlugin, - PluginParams, - ConfigParams, - Method, - PluginParametersMetadata, -} from '@grnsft/if-core/types'; import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; -const {GlobalConfigError} = ERRORS; -const { - MISSING_GLOBAL_CONFIG, - X_Y_EQUAL, - ARRAY_LENGTH_NON_EMPTY, - WITHIN_THE_RANGE, -} = STRINGS; - -export const Interpolation = ( - globalConfig: ConfigParams, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; - - /** - * Executes the energy consumption calculation for an array of input parameters. - */ - const execute = (inputs: PluginParams[]) => { - const validatedConfig = validateConfig(); - - return inputs.map((input, index) => { - const safeInput = validateInput(input, index); - const result = calculateResult(validatedConfig, safeInput); - - return { - ...input, - [validatedConfig['output-parameter']]: result, - }; - }); - }; - - /** - * Calculates the appropriate interpolation value based on the specified method type in the config and input parameters. - */ - const calculateResult = (config: ConfigParams, input: PluginParams) => { - const methodType: {[key: string]: number} = { - linear: getLinearInterpolation(config, input), - spline: getSplineInterpolation(config, input), - polynomial: getPolynomialInterpolation(config, input), - }; - - return methodType[config.method]; - }; +const {X_Y_EQUAL, ARRAY_LENGTH_NON_EMPTY, WITHIN_THE_RANGE, MISSING_CONFIG} = + STRINGS; - /** - * Calculates the interpolation when the method is linear. - */ - const getLinearInterpolation = ( - config: ConfigParams, - input: PluginParams - ) => { - const parameter = input[globalConfig['input-parameter']]; - const xPoints: number[] = config.x; - const yPoints: number[] = config.y; - - const result = xPoints.reduce( - (acc, xPoint, i) => { - if (parameter === xPoint) { - acc.baseCpu = xPoint; - acc.baseRate = yPoints[i]; - } else if (parameter > xPoint && parameter < xPoints[i + 1]) { - acc.baseCpu = xPoint; - acc.baseRate = yPoints[i]; - acc.ratio = (yPoints[i + 1] - yPoints[i]) / (xPoints[i + 1] - xPoint); - } - - return acc; - }, - {baseRate: 0, baseCpu: 0, ratio: 0} - ); - - return result.baseRate + (parameter - result.baseCpu) * result.ratio; - }; - - /** - * Calculates the interpolation when the method is spline. - */ - const getSplineInterpolation = ( - config: ConfigParams, - input: PluginParams - ) => { - const parameter = input[globalConfig['input-parameter']]; - const xPoints: number[] = config.x; - const yPoints: number[] = config.y; - const spline: any = new Spline(xPoints, yPoints); - - return spline.at(parameter); - }; - - /** - * Calculates the interpolation when the method is polynomial. - */ - const getPolynomialInterpolation = ( - config: ConfigParams, - input: PluginParams - ) => { - const parameter = input[globalConfig['input-parameter']]; - const xPoints: number[] = config.x; - const yPoints: number[] = config.y; - - const result = xPoints.reduce((acc, x, i) => { - const term = - yPoints[i] * - xPoints.reduce((prod, xPoint, j) => { - if (j !== i) { - return (prod * (parameter - xPoint)) / (x - xPoint); - } - return prod; - }, 1); - return acc + term; - }, 0); - - return result; - }; +const {ConfigError} = ERRORS; - /** - * Validates global config parameters. - * Sorts elements of `x` and `y`. - */ - const validateConfig = () => { - if (!globalConfig) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); +export const Interpolation = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); } const schema = z @@ -156,30 +35,21 @@ export const Interpolation = ( message: ARRAY_LENGTH_NON_EMPTY, }); - const defaultMethod = globalConfig.method ?? Method.LINEAR; - const updatedConfig = Object.assign( - {}, - {method: defaultMethod}, - globalConfig, - { - x: sortPoints(globalConfig.x), - y: sortPoints(globalConfig.y), - } - ); + const defaultMethod = config.method ?? Method.LINEAR; + const updatedConfig = Object.assign({}, {method: defaultMethod}, config, { + x: sortPoints(config.x), + y: sortPoints(config.y), + }); return validate>(schema, updatedConfig); - }; - - const sortPoints = (items: number[]) => - items.sort((a: number, b: number) => { - return a - b; - }); + }, + inputValidation: ( + input: PluginParams, + config: ConfigParams, + index: number | undefined + ) => { + const inputParameter = config['input-parameter']; - /** - * Validates inputes parameters. - */ - const validateInput = (input: PluginParams, index: number) => { - const inputParameter = globalConfig['input-parameter']; const schema = z .object({ timestamp: z.string().or(z.date()), @@ -188,18 +58,118 @@ export const Interpolation = ( }) .refine( data => - data[inputParameter] >= globalConfig.x[0] && - data[inputParameter] <= globalConfig.x[globalConfig.x.length - 1], + data[inputParameter] >= config.x[0] && + data[inputParameter] <= config.x[config.x.length - 1], { message: WITHIN_THE_RANGE, } ); return validate>(schema, input, index); - }; + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + const {'output-parameter': outputParameter} = config; + + return inputs.map(input => { + const calculatedResult = calculateResult(config, input); - return { - metadata, - execute, + return { + ...input, + [outputParameter]: calculatedResult, + }; + }); + }, + allowArithmeticExpressions: ['input-parameter'], +}); + +/** + * Calculates the appropriate interpolation value based on the specified method type in the config and input parameters. + */ +const calculateResult = (config: ConfigParams, input: PluginParams) => { + const methodType: {[key: string]: number} = { + linear: getLinearInterpolation(config, input), + spline: getSplineInterpolation(config, input), + polynomial: getPolynomialInterpolation(config, input), }; + + return methodType[config.method]; }; + +/** + * Calculates the interpolation when the method is linear. + */ +const getLinearInterpolation = (config: ConfigParams, input: PluginParams) => { + const parameter = + typeof config['input-parameter'] === 'number' + ? config['input-parameter'] + : input[config['input-parameter']]; + const xPoints: number[] = config.x; + const yPoints: number[] = config.y; + + const result = xPoints.reduce( + (acc, xPoint, i) => { + if (parameter === xPoint) { + acc.baseCpu = xPoint; + acc.baseRate = yPoints[i]; + } else if (parameter > xPoint && parameter < xPoints[i + 1]) { + acc.baseCpu = xPoint; + acc.baseRate = yPoints[i]; + acc.ratio = (yPoints[i + 1] - yPoints[i]) / (xPoints[i + 1] - xPoint); + } + + return acc; + }, + {baseRate: 0, baseCpu: 0, ratio: 0} + ); + + return result.baseRate + (parameter - result.baseCpu) * result.ratio; +}; + +/** + * Calculates the interpolation when the method is spline. + */ +const getSplineInterpolation = (config: ConfigParams, input: PluginParams) => { + const parameter = + typeof config['input-parameter'] === 'number' + ? config['input-parameter'] + : input[config['input-parameter']]; + const xPoints: number[] = config.x; + const yPoints: number[] = config.y; + const spline: any = new Spline(xPoints, yPoints); + + return spline.at(parameter); +}; + +/** + * Calculates the interpolation when the method is polynomial. + */ +const getPolynomialInterpolation = ( + config: ConfigParams, + input: PluginParams +) => { + const parameter = + typeof config['input-parameter'] === 'number' + ? config['input-parameter'] + : input[config['input-parameter']]; + const xPoints: number[] = config.x; + const yPoints: number[] = config.y; + + const result = xPoints.reduce((acc, x, i) => { + const term = + yPoints[i] * + xPoints.reduce((prod, xPoint, j) => { + if (j !== i) { + return (prod * (parameter - xPoint)) / (x - xPoint); + } + return prod; + }, 1); + return acc + term; + }, 0); + + return result; +}; + +const sortPoints = (items: number[]) => + items.sort((a: number, b: number) => { + return a - b; + }); diff --git a/src/if-run/builtins/mock-observations/README.md b/src/if-run/builtins/mock-observations/README.md index 53bd89924..8dd8a963f 100644 --- a/src/if-run/builtins/mock-observations/README.md +++ b/src/if-run/builtins/mock-observations/README.md @@ -11,7 +11,7 @@ The mode currently mocks 2 types of observation data: - Common key-value pairs, that are generated statically and are the same for each generated observation/input (see 'helpers/CommonGenerator.ts') - Randomly generated integer values for predefined keys (see 'helpers/RandIntGenerator.ts') -### Plugin global config +### Plugin config - `timestamp-from`, `timestamp-to` and `duration` define time buckets for which to generate observations. - `generators` define which fields to generate for each observation @@ -25,12 +25,29 @@ The `parameter-metadata` section contains information about `description`, `unit - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. - `outputs`: describe the output parameters. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +mock-observations: + kind: plugin + method: MockObservations + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Authentication @@ -38,13 +55,13 @@ N/A ### Inputs -The plugin's `global-config` section in the manifest file determines its behaviour. +The plugin's `config` section in the manifest file determines its behaviour. 'inputs' section is ignored. ### Typescript Usage ```typescript -const mockObservations = MockObservations({ +const config = { 'timestamp-from': '2023-07-06T00:00', 'timestamp-to': '2023-07-06T00:10', duration: 60, @@ -56,7 +73,10 @@ const mockObservations = MockObservations({ region: 'uk-west', }, }, -}); +}; +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = {}; +const mockObservations = MockObservations(config, parametersMetadata, mapping); const result = await mockObservations.execute([]); ``` @@ -74,7 +94,7 @@ initialize: kind: plugin method: MockObservations path: 'builtin' - global-config: + config: timestamp-from: 2023-07-06T00:00 timestamp-to: 2023-07-06T00:10 duration: 60 diff --git a/src/if-run/builtins/mock-observations/helpers/common-generator.ts b/src/if-run/builtins/mock-observations/helpers/common-generator.ts index 8283cf1bd..1aceb59f6 100644 --- a/src/if-run/builtins/mock-observations/helpers/common-generator.ts +++ b/src/if-run/builtins/mock-observations/helpers/common-generator.ts @@ -5,18 +5,18 @@ import {STRINGS} from '../../../config'; import {Generator} from '../interfaces'; -const {GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; export const CommonGenerator = (config: ConfigParams): Generator => { /** * Generates next value by copying the validated config. * Validates the provided config is not null or empty. - * Returns a copy of the validated config, otherwise throws an GlobalConfigError. + * Returns a copy of the validated config, otherwise throws an ConfigError. */ const validateConfig = (config: object) => { if (!config || Object.keys(config).length === 0) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); + throw new ConfigError(MISSING_CONFIG); } return structuredClone(config); diff --git a/src/if-run/builtins/mock-observations/helpers/rand-int-generator.ts b/src/if-run/builtins/mock-observations/helpers/rand-int-generator.ts index 1cb4d161f..9194fe4b1 100644 --- a/src/if-run/builtins/mock-observations/helpers/rand-int-generator.ts +++ b/src/if-run/builtins/mock-observations/helpers/rand-int-generator.ts @@ -5,9 +5,9 @@ import {STRINGS} from '../../../config'; import {Generator} from '../interfaces'; -const {GlobalConfigError} = ERRORS; +const {ConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_MIN_MAX, INVALID_MIN_MAX, INVALID_NAME} = +const {MISSING_CONFIG, MISSING_MIN_MAX, INVALID_MIN_MAX, INVALID_NAME} = STRINGS; export const RandIntGenerator = ( @@ -20,7 +20,7 @@ export const RandIntGenerator = ( const validateName = (name: string | null): string => { if (!name || name.trim() === '') { - throw new GlobalConfigError(INVALID_NAME); + throw new ConfigError(INVALID_NAME); } return name; @@ -28,15 +28,15 @@ export const RandIntGenerator = ( const validateConfig = (config: ConfigParams): {min: number; max: number} => { if (!config || Object.keys(config).length === 0) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); + throw new ConfigError(MISSING_CONFIG); } if (!config.min || !config.max) { - throw new GlobalConfigError(MISSING_MIN_MAX); + throw new ConfigError(MISSING_MIN_MAX); } if (config.min >= config.max) { - throw new GlobalConfigError(INVALID_MIN_MAX(validatedName)); + throw new ConfigError(INVALID_MIN_MAX(validatedName)); } return {min: config.min, max: config.max}; diff --git a/src/if-run/builtins/mock-observations/index.ts b/src/if-run/builtins/mock-observations/index.ts index facc78353..138386940 100644 --- a/src/if-run/builtins/mock-observations/index.ts +++ b/src/if-run/builtins/mock-observations/index.ts @@ -1,36 +1,48 @@ import {DateTime, Duration} from 'luxon'; import {z} from 'zod'; + +import {PluginFactory} from '@grnsft/if-core/interfaces'; import { - ExecutePlugin, PluginParams, ConfigParams, ObservationParams, - PluginParametersMetadata, } from '@grnsft/if-core/types'; +import {ERRORS} from '@grnsft/if-core/utils'; import {validate} from '../../../common/util/validations'; +import {STRINGS} from '../../config'; + import {CommonGenerator} from './helpers/common-generator'; import {RandIntGenerator} from './helpers/rand-int-generator'; import {Generator} from './interfaces/index'; -export const MockObservations = ( - globalConfig: ConfigParams, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; - /** - * Generate sets of mocked observations based on config. - */ - const execute = (inputs: PluginParams[]) => { +export const MockObservations = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); + } + + const schema = z.object({ + 'timestamp-from': z.string(), + 'timestamp-to': z.string(), + duration: z.number().gt(0), + components: z.array(z.record(z.string())), + generators: z.object({ + common: z.record(z.string().or(z.number())), + randint: z.record(z.object({min: z.number(), max: z.number()})), + }), + }); + + return validate>(schema, config); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { const {duration, timeBuckets, components, generators} = - generateParamsFromConfig(); + generateParamsFromConfig(config); const generatorToHistory = new Map(); generators.forEach(generator => { @@ -40,7 +52,7 @@ export const MockObservations = ( const defaults = inputs && inputs[0]; return Object.entries(components).reduce((acc: PluginParams[], item) => { - const component = item[1]; + const component: any = item[1]; timeBuckets.forEach(timeBucket => { const observation = createObservation( {duration, component, timeBucket, generators}, @@ -52,133 +64,111 @@ export const MockObservations = ( return acc; }, []); - }; - - /** - * Validates global config parameters. - */ - const validateGlobalConfig = () => { - const schema = z.object({ - 'timestamp-from': z.string(), - 'timestamp-to': z.string(), - duration: z.number().gt(0), - components: z.array(z.record(z.string())), - generators: z.object({ - common: z.record(z.string().or(z.number())), - randint: z.record(z.object({min: z.number(), max: z.number()})), - }), - }); + }, +}); + +/** + * Configures the MockObservations Plugin for IF + */ +const generateParamsFromConfig = (config: ConfigParams) => { + const { + 'timestamp-from': timestampFrom, + 'timestamp-to': timestampTo, + duration, + generators, + components, + } = config; + + const convertedTimestampFrom = DateTime.fromISO(timestampFrom, { + zone: 'UTC', + }); + const convertedTimestampTo = DateTime.fromISO(timestampTo, {zone: 'UTC'}); - return validate>(schema, globalConfig); + return { + duration, + timeBuckets: createTimeBuckets( + convertedTimestampFrom, + convertedTimestampTo, + duration + ), + generators: createGenerators(generators), + components, }; +}; - /** - * Configures the MockObservations Plugin for IF - */ - const generateParamsFromConfig = () => { - const { - 'timestamp-from': timestampFrom, - 'timestamp-to': timestampTo, +/* + * Creates time buckets based on start time, end time and duration of each bucket. + */ +const createTimeBuckets = ( + timestampFrom: DateTime, + timestampTo: DateTime, + duration: number, + timeBuckets: DateTime[] = [] +): DateTime[] => { + if ( + timestampFrom < timestampTo || + timestampFrom.plus(Duration.fromObject({seconds: duration})) < timestampTo + ) { + return createTimeBuckets( + timestampFrom.plus(Duration.fromObject({seconds: duration})), + timestampTo, duration, - generators, - components, - } = validateGlobalConfig(); - const convertedTimestampFrom = DateTime.fromISO(timestampFrom, { - zone: 'UTC', - }); - const convertedTimestampTo = DateTime.fromISO(timestampTo, {zone: 'UTC'}); + [...timeBuckets, timestampFrom] + ); + } + return timeBuckets; +}; - return { - duration, - timeBuckets: createTimeBuckets( - convertedTimestampFrom, - convertedTimestampTo, - duration - ), - generators: createGenerators(generators), - components, - }; - }; +/* + * Creates generators based on a given config + */ +const createGenerators = (generatorsConfig: object): Generator[] => { + const createCommonGenerator = (config: any): Generator[] => [ + CommonGenerator(config), + ]; + + const createRandIntGenerators = (config: any): Generator[] => + Object.entries(config).map(([fieldToPopulate, value]) => + RandIntGenerator(fieldToPopulate, value as Record) + ); - /* - * create time buckets based on start time, end time and duration of each bucket. - */ - const createTimeBuckets = ( - timestampFrom: DateTime, - timestampTo: DateTime, - duration: number, - timeBuckets: DateTime[] = [] - ): DateTime[] => { - if ( - timestampFrom < timestampTo || - timestampFrom.plus(Duration.fromObject({seconds: duration})) < timestampTo - ) { - return createTimeBuckets( - timestampFrom.plus(Duration.fromObject({seconds: duration})), - timestampTo, - duration, - [...timeBuckets, timestampFrom] - ); - } - return timeBuckets; - }; + return Object.entries(generatorsConfig).flatMap(([key, value]) => + key === 'randint' + ? createRandIntGenerators(value).flat() + : createCommonGenerator(value) + ); +}; - /* - * create generators based on a given config - */ - const createGenerators = (generatorsConfig: object): Generator[] => { - const createCommonGenerator = (config: any): Generator[] => [ - CommonGenerator(config), - ]; - - const createRandIntGenerators = (config: any): Generator[] => - Object.entries(config).map(([fieldToPopulate, value]) => - RandIntGenerator(fieldToPopulate, value as Record) - ); - - return Object.entries(generatorsConfig).flatMap(([key, value]) => - key === 'randint' - ? createRandIntGenerators(value).flat() - : createCommonGenerator(value) - ); - }; +/* + * Creates time buckets based on start time, end time and duration of each bucket. + */ +const createObservation = ( + observationParams: ObservationParams, + generatorToHistory: Map +): PluginParams => { + const {duration, component, timeBucket, generators} = observationParams; + const timestamp = timeBucket.toISO(); - /* - * Creates time buckets based on start time, end time and duration of each bucket. - */ - const createObservation = ( - observationParams: ObservationParams, - generatorToHistory: Map - ): PluginParams => { - const {duration, component, timeBucket, generators} = observationParams; - const timestamp = timeBucket.toISO(); - - const generateObservation = (generator: Generator) => { - const history = generatorToHistory.get(generator) || []; - const generated: Record = generator.next(history); - - generatorToHistory.set(generator, [...history, generated.value]); - - return generated; - }; - - const generateObservations = (gen: Generator) => generateObservation(gen); - const generatedValues = generators.map(generateObservations); - const initialObservation: PluginParams = { - timestamp, - duration, - ...component, - }; - const generatedObservation = generatedValues.reduce( - (observation, generated) => Object.assign(observation, generated), - initialObservation - ); + const generateObservation = (generator: Generator) => { + const history = generatorToHistory.get(generator) || []; + const generated: Record = generator.next(history); + + generatorToHistory.set(generator, [...history, generated.value]); - return generatedObservation as PluginParams; + return generated; }; - return { - metadata, - execute, + const generateObservations = (gen: Generator) => generateObservation(gen); + const generatedValues = generators.map(generateObservations); + const initialObservation: PluginParams = { + timestamp, + duration, + ...component, }; + const generatedObservation = generatedValues.reduce( + (observation, generated) => Object.assign(observation, generated), + initialObservation + ); + + return generatedObservation as PluginParams; }; diff --git a/src/if-run/builtins/multiply/README.md b/src/if-run/builtins/multiply/README.md index dc6cde7c7..a624c7256 100644 --- a/src/if-run/builtins/multiply/README.md +++ b/src/if-run/builtins/multiply/README.md @@ -10,7 +10,7 @@ For example, you could multiply `cpu/energy` and `network/energy` and name the r ### Plugin config -Two parameters are required in global config: `input-parameters` and `output-parameter`. +Two parameters are required in config: `input-parameters` and `output-parameter`. `input-parameters`: an array of strings. Each string should match an existing key in the `inputs` array `output-parameter`: a string defining the name to use to add the product of the input parameters to the output array. @@ -19,16 +19,32 @@ Two parameters are required in global config: `input-parameters` and `output-par The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe parameters of the `input-parameters` of the global config. Each parameter has: +- `inputs`: describe parameters of the `input-parameters` of the config. Each parameter has: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. -- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes: +- `outputs`: describe the parameter of the `output-parameter` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +multiply: + method: Multiply + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -36,7 +52,7 @@ All of `input-parameters` must be available in the input array. ## Returns -- `output-parameter`: the product of all `input-parameters` with the parameter name defined by `output-parameter` in global config. +- `output-parameter`: the product of all `input-parameters` with the parameter name defined by `output-parameter` in config. ## Calculation @@ -56,7 +72,9 @@ const config = { outputParameter: 'energy-product', }; -const multiply = Multiply(config, parametersMetadata); +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = {}; +const multiply = Multiply(config, parametersMetadata, mapping); const result = await multiply.execute([ { duration: 3600, @@ -69,7 +87,7 @@ const result = await multiply.execute([ ## Example manifest -IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by `ie` and does not have to be done explicitly by the user. The following is an example manifest that calls `multiply`: +IF users will typically call the plugin as part of a pipeline defined in a manifest file. In this case, instantiating the plugin is handled by `if-run` and does not have to be done explicitly by the user. The following is an example manifest that calls `multiply`: ```yaml name: multiply-demo @@ -80,7 +98,7 @@ initialize: multiply: method: Multiply path: 'builtin' - global-config: + config: input-parameters: ['cpu/energy', 'network/energy'] output-parameter: 'energy-product' tree: diff --git a/src/if-run/builtins/multiply/index.ts b/src/if-run/builtins/multiply/index.ts index b51c1a426..544b87add 100644 --- a/src/if-run/builtins/multiply/index.ts +++ b/src/if-run/builtins/multiply/index.ts @@ -1,47 +1,34 @@ import {z} from 'zod'; -import { - ExecutePlugin, - PluginParams, - MultiplyConfig, - PluginParametersMetadata, -} from '@grnsft/if-core/types'; + +import {PluginParams, ConfigParams} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; +import {ERRORS} from '@grnsft/if-core/utils'; import {validate} from '../../../common/util/validations'; -export const Multiply = ( - globalConfig: MultiplyConfig, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; +import {STRINGS} from '../../config'; + +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - const globalConfigSchema = z.object({ +export const Multiply = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); + } + + const configSchema = z.object({ 'input-parameters': z.array(z.string()), 'output-parameter': z.string().min(1), }); - return validate>( - globalConfigSchema, - globalConfig - ); - }; + return validate>(configSchema, config); + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { + const inputParameters = config['input-parameters']; - /** - * Checks for required fields in input. - */ - const validateSingleInput = ( - input: PluginParams, - inputParameters: string[] - ) => { const inputData = inputParameters.reduce( - (acc, param) => { + (acc: {[x: string]: any}, param: string | number) => { acc[param] = input[param]; return acc; @@ -51,40 +38,31 @@ export const Multiply = ( const validationSchema = z.record(z.string(), z.number()); - validate(validationSchema, inputData); - - return input; - }; - - /** - * Calculate the product of each input parameter. - */ - const execute = (inputs: PluginParams[]): PluginParams[] => { - const safeGlobalConfig = validateGlobalConfig(); - const inputParameters = safeGlobalConfig['input-parameters']; - const outputParameter = safeGlobalConfig['output-parameter']; + return validate(validationSchema, inputData); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + const { + 'input-parameters': inputParameters, + 'output-parameter': outputParameter, + } = config; return inputs.map(input => { - validateSingleInput(input, inputParameters); + const calculatedResult = calculateProduct(input, inputParameters); return { ...input, - [outputParameter]: calculateProduct(input, inputParameters), + [outputParameter]: calculatedResult, }; }); - }; - - /** - * Calculates the product of the components. - */ - const calculateProduct = (input: PluginParams, inputParameters: string[]) => - inputParameters.reduce( - (accumulator, metricToMultiply) => accumulator * input[metricToMultiply], - 1 - ); + }, + allowArithmeticExpressions: [], +}); - return { - metadata, - execute, - }; -}; +/** + * Calculates the product of the components. + */ +const calculateProduct = (input: PluginParams, inputParameters: string[]) => + inputParameters.reduce( + (accumulator, metricToMultiply) => accumulator * input[metricToMultiply], + 1 + ); diff --git a/src/if-run/builtins/regex/README.md b/src/if-run/builtins/regex/README.md index 2c6a4cd88..37d60bcd0 100644 --- a/src/if-run/builtins/regex/README.md +++ b/src/if-run/builtins/regex/README.md @@ -20,16 +20,32 @@ Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-26 The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe the parameter of the `parameter` value of the global config. The parameter has the following attributes: +- `inputs`: describe the parameter of the `parameter` value of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. -- `outputs`: describe the parameters of the `output` of the global config. The parameter has the following attributes: +- `outputs`: describe the parameters of the `output` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +regex: + method: Regex + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -37,21 +53,23 @@ The `parameter-metadata` section contains information about `description`, `unit ## Returns -- `output`: The match of the `parameter` value using the `match` regex defined in the global config. If the `match` regex includes the global flag (`g`), a string containing all matches separated by spaces. +- `output`: The match of the `parameter` value using the `match` regex defined in the config. If the `match` regex includes the global flag (`g`), a string containing all matches separated by spaces. ## Implementation To run the plugin, you must first create an instance of `Regex`. Then, you can call `execute()`. ```typescript -const globalConfig = { +const config = { parameter: 'physical-processor', match: '^[^,]+', output: 'cpu/name', }; -const regex = Regex(globalConfig); +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = {}; +const regex = Regex(config, parametersMetadata, mapping); -const input = [ +const inputs = [ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -59,6 +77,8 @@ const input = [ 'Intel® Xeon® Platinum 8272CL,Intel® Xeon® 8171M 2.1 GHz,Intel® Xeon® E5-2673 v4 2.3 GHz,Intel® Xeon® E5-2673 v3 2.4 GHz', }, ]; + +const result = await regex.execute(inputs); ``` ## Example manifest @@ -74,7 +94,7 @@ initialize: regex: method: Regex path: 'builtin' - global-config: + config: parameter: physical-processor match: ^[^,]+ output: cpu/name @@ -112,9 +132,9 @@ Every element in the `inputs` array must contain: - `duration` - whatever value you passed to `parameter` -### `GlobalConfigError` +### `ConfigError` -You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. +You will receive an error starting `ConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: diff --git a/src/if-run/builtins/regex/index.ts b/src/if-run/builtins/regex/index.ts index 0076b6cfe..ed904d25f 100644 --- a/src/if-run/builtins/regex/index.ts +++ b/src/if-run/builtins/regex/index.ts @@ -1,35 +1,20 @@ import {z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; -import { - ExecutePlugin, - PluginParams, - ConfigParams, - PluginParametersMetadata, -} from '@grnsft/if-core/types'; + +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; -const {MissingInputDataError, GlobalConfigError, RegexMismatchError} = ERRORS; -const {MISSING_GLOBAL_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; - -export const Regex = ( - globalConfig: ConfigParams, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; +const {MissingInputDataError, ConfigError, RegexMismatchError} = ERRORS; +const {MISSING_CONFIG, MISSING_INPUT_DATA, REGEX_MISMATCH} = STRINGS; - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - if (!globalConfig) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); +export const Regex = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); } const schema = z.object({ @@ -38,69 +23,49 @@ export const Regex = ( output: z.string(), }); - return validate>(schema, globalConfig); - }; + return validate>(schema, config); + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { + const parameter = config['parameter']; - /** - * Checks for required fields in input. - */ - const validateSingleInput = (input: PluginParams, parameter: string) => { if (!input[parameter]) { throw new MissingInputDataError(MISSING_INPUT_DATA(parameter)); } return input; - }; - - /** - * Executes the regex of the given parameter. - */ - const execute = (inputs: PluginParams[]) => { - const safeGlobalConfig = validateGlobalConfig(); - const {parameter: parameter, match, output} = safeGlobalConfig; - - return inputs.map(input => { - const safeInput = Object.assign( - {}, - input, - validateSingleInput(input, parameter) - ); - - return { - ...input, - [output]: extractMatching(safeInput, parameter, match), - }; - }); - }; - - /** - * Extracts a substring from the given input parameter that matches the provided regular expression pattern. - */ - const extractMatching = ( - input: PluginParams, - parameter: string, - match: string - ) => { - if (!match.startsWith('/')) { - match = '/' + match; - } - - if (!match.endsWith('/g') && !match.endsWith('/')) { - match += '/'; - } - - const regex = eval(match); - const matchedItems = input[parameter].match(regex); - - if (!matchedItems || matchedItems.length === 0) { - throw new RegexMismatchError(REGEX_MISMATCH(input[parameter], match)); - } - - return matchedItems.join(' '); - }; - - return { - metadata, - execute, - }; + }, + implementation: async (inputs, config) => { + const {parameter: parameter, match, output} = config; + + return inputs.map(input => ({ + ...input, + [output]: extractMatching(input, parameter, match), + })); + }, +}); + +/** + * Extracts a substring from the given input parameter that matches the provided regular expression pattern. + */ +const extractMatching = ( + input: PluginParams, + parameter: string, + match: string +) => { + if (!match.startsWith('/')) { + match = '/' + match; + } + + if (!match.endsWith('/g') && !match.endsWith('/')) { + match += '/'; + } + + const regex = eval(match); + const matchedItems = input[parameter].match(regex); + + if (!matchedItems || matchedItems.length === 0) { + throw new RegexMismatchError(REGEX_MISMATCH(input[parameter], match)); + } + + return matchedItems.join(' '); }; diff --git a/src/if-run/builtins/sci-embodied/README.md b/src/if-run/builtins/sci-embodied/README.md index a4c304c6d..80baeab06 100644 --- a/src/if-run/builtins/sci-embodied/README.md +++ b/src/if-run/builtins/sci-embodied/README.md @@ -2,135 +2,136 @@ Software systems cause emissions through the hardware that they operate on, both through the energy that the physical hardware consumes and the emissions associated with manufacturing the hardware. Embodied carbon refers to the carbon emitted during the manufacture and eventual disposal of a component. It is added to the operational carbon (carbon emitted when a component is used) to give an overall SCI score. -Read more on [embodied carbon](https://github.com/Green-Software-Foundation/sci/blob/main/Software_Carbon_Intensity/Software_Carbon_Intensity_Specification.md#embodied-emissions) +Read more on [embodied carbon](https://github.com/Green-Software-Foundation/sci/blob/main/Software_Carbon_Intensity/Software_Carbon_Intensity_Specification.md#embodied-emissions). -## Parameters +Our plugin follows the Cloud Carbon Footprint methodology for calculating embodied carbon and extends it to scale down the total embodied carbon for a piece of hardware by the portion of it that should be allocated to a particular application, using a `usage-ratio` and `time`. The `usage-ratio` is a term that can be used to scale by, for example, the storage you actually use on a shared server, rather than the total storage available for that hardware, or the time you are active compared to the hardware lifespan. -### Plugin config -Not Needed +## Parameters -### Plugin parameter metadata +### Plugin Configuration -The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs +The `SciEmbodied` plugin requires a configuration object and parameter metadata (optional) to do the calculation. -- `inputs`: describe the parameters of the `inputs`. Each parameter has: +### Plugin Parameter Metadata - - `description`: description of the parameter - - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) +The `parameter-metadata` section contains information about the `description`, `unit`, and `aggregation-method` of the input and output parameters. -- `outputs`: describe the `carbon-embodied` parameter. The parameter has the following attributes: - - `description`: description of the parameter - - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) +- `inputs`: Describes the parameters for the input data. Each parameter includes: + - `description`: A brief description of the parameter. + - `unit`: The unit of measurement for the parameter. + - `aggregation-method`: The method used to aggregate this parameter (`sum`, `avg`, or `none`). -### Inputs +- `outputs`: Describes the `embodied-carbon` parameter, which includes: + - `description`: A brief description of the parameter. + - `unit`: The unit of measurement for the parameter. + - `aggregation-method`: The method used to aggregate this parameter (`sum`, `avg`, or `none`). -- `device/emissions-embodied`: the sum of Life Cycle Assessment (LCA) emissions for the component -- `device/expected-lifespan`: the length of time, in seconds, between a component's manufacture and its disposal -- `resources-reserved`: the number of resources reserved for use by the software -- `resources-total`: the total number of resources available -- `duration`: the amount of time covered by an observation, in this context it is used as the share of the total life span of the hardware reserved for use by an application, in seconds. +### Mapping -> Note that if you have a plugin pipeline that adds `vcpus-allocated` and `vcpus-total` to each observation, such as the `cloud-metadata` plugin, those values will be used **in preference** to the given `resources-reserved` and `resources-total` fields. +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: -## Returns +```yaml +sci-embodied: + method: SciEmbodied + path: 'builtins' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` -- `carbon-embodied`: the carbon emitted in manufacturing and disposing of a component, in gCO2eq +### Config -## Calculation +`baseline-vcpus`: the number of CPUs to use in the baseline server, defaults tothe CCF value of 1, +`baseline-memory`: the amount of memory to use in the baseline server, defaults tothe CCF value of 16, +`baseline-emissions`: the embodied carbon assumed to represent a baseline server, in g +`lifespan`: the lifespan of the device, in seconds. Defaults to 4 years (126144000 seconds), +`time`: the time to consider when scaling the total device embodied carbon, if not given defaults to `duration` +`vcpu-emissions-constant`: emissions for a CPU in gCO2e. Defaults tothe CCF value (100000), +`memory-emissions-constant`: value used in calculating emissions due to memory, defaults to the CCf value of 533/384 +`ssd-emissions-constant`: emissions for a SSD in gCO2e. Defaults tothe CCF value (50000), +`hdd-emissions-constant`: emissions for a CPU in gCO2e. Defaults tothe CCF value (100000), +`gpu-emissions-constant`: emissions for a GPU in gCO2e. Defaults tothe CCF value (150000), +`output-parameter`: name to give the output value, defaults to `embodied-carbon` -To calculate the embodied carbon, `m` for a software application, use the equation: +Note that if you do not provide any config at all, we will fallback to defaults for everything, equivalent to setting the baseline server equal to the CCF version, which has 1000000g of embodied emissions. -``` -m = te * ts * rs -``` +### Inputs -Where: +- `vCPUs`: number of CPUs available on device +- `memory`: amount of RAM available on device, in GB +- `ssd`: number of SSD drives mounted on device +- `hdd`: number of HDD drives mounted on device +- `gpu`: number of GPUs available on device +- `duration`: The length of time the hardware is reserved for use by the software, in seconds. +- `time`: the time to use for scalign the total embodied carbon per timestap, if you do not want to use `duration` +- `usage-ratio`: the ratio by which to scale down the total embodied carbon according to your usage, e.g. for a shared storage server the total storage divided by your actual storage. -- `device/emissions-embodied` = Total embodied emissions; the sum of Life Cycle Assessment (LCA) emissions for the component. +Note that if you do not provide any inputs at all, we fall back to defaults that are equivalent to using the full resources of the baseline server, scaled only by `duration`. -- `timeShare` = Time-share; the share of the total life span of the hardware reserved for use by an application. +### Outputs - - `timeShare` is calculated as `duration/'device/expected-lifespan'`, where: - - `duration` = the length of time the hardware is reserved for use by the software. - - `device/expected-lifespan` = Expected lifespan: the length of time, in seconds, between a component's manufacture and its disposal. +- `embodied-carbon`: The total embodied emissions for the component, measured in gCO2e, per timestep. -- `resourceShare` = Resource-share; the share of the total available resources of the hardware reserved for use by an application. - - `resourceShare` is calculated as `resources-reserved/resources-total`, where: - - `resources-reserved` = Resources reserved; the number of resources reserved for use by the software. - - `resources-total` = Total Resources; the total number of resources available. +## Calculation -## Implementation +The plugin calculates the total embodied carbon emissions using the following steps: -IF implements the plugin based on the logic described above. To run the plugin, you must first create an instance of `SciEmbodied`. Then, you can call `execute()` to return `m`. + - CPU emissions (`cpuE`) are calculated based on the difference between allocated vCPUs and baseline vCPUs. + - Memory emissions (`memoryE`) are calculated based on the difference between allocated memory and baseline memory. + - Emissions for HDD, SSD, and GPU are also calculated based on their respective differences from baseline values. + - The total embodied emissions are calculated by summing the baseline emissions with the above components scaling by the usage ratio and time. -## Usage +## Implementation -The following snippet demonstrates how to call the `sci-embodied` plugin from Typescript. +The plugin can be instantiated and executed as follows: ```typescript import {SciEmbodied} from 'builtins'; -const sciEmbodied = SciEmbodied(); +const sciEmbodied = SciEmbodied(config, parametersMetadata, {}); const results = await sciEmbodied.execute([ { - 'device/emissions-embodied': 200, // in gCO2e for total resource units - duration: 60 * 60 * 24 * 30, // time reserved in seconds, can point to another field "duration" - 'device/expected-lifespan': 60 * 60 * 24 * 365 * 4, // lifespan in seconds (4 years) - 'resources-reserved': 1, // resource units reserved / used - 'resources-total': 1, // total resource units available + duration: 3600, // time reserved in seconds + vCPUs: 2, // allocated vCPUs + memory: 32, // allocated memory in GB + ssd: 100, // allocated SSD storage in GB + hdd: 1000, // allocated HDD storage in GB + gpu: 1, // allocated GPUs }, ]); + +console.log(results); ``` -## Example manifest +# Example Manifest -IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `ie` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `sci-embodied`: +IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `if-run` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `sci-embodied`: ```yaml name: sci-embodied -description: simple demo invoking sci-embodied +description: demo invoking sci-embodied tags: initialize: plugins: sci-embodied: method: SciEmbodied path: 'builtins' + mapping: + device/emissions-embodied: device/carbon-footprint tree: children: child: pipeline: compute: - - sci-embodied # duration & config -> embodied - defaults: - device/emissions-embodied: 1533.120 # gCO2eq - device/expected-lifespan: 3 # 3 years in seconds - resources-reserved: 1 - resources-total: 8 + - sci-embodied inputs: - - timestamp: 2023-07-06T00:00 + - timestamp: 2024-08-19T00:00 duration: 3600 ``` -You can run this example `manifest` by executing the following command from the project root: +To run this example manifest, use the following command: -```sh +```bash npm i -g @grnsft/if -if-run --manifest manifests/plugins/sci-embodied.yml --output manifests/outputs/sci-embodied.yml +if-run --manifest manifests/plugins/sci-embodied/success.yml --output manifests/outputs/success.yml ``` - -The results will be saved to a new `yaml` file in `./examples/outputs`. - -## Errors - -`SciEmbodied` uses one of IF's error classes - -### `SciEmbodiedError` - -This error class is used to describe a problem with one of the input values to `sci-embodied`. This is typically due to an incorrect type or a reference to a value that is not available. - -You will receive a specific error message explaining which parameter is problematic, and you can check and replace where appropriate. - -For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors diff --git a/src/if-run/builtins/sci-embodied/index.ts b/src/if-run/builtins/sci-embodied/index.ts index b304d4ae1..0a45b419d 100644 --- a/src/if-run/builtins/sci-embodied/index.ts +++ b/src/if-run/builtins/sci-embodied/index.ts @@ -1,180 +1,193 @@ import {z} from 'zod'; -import { - ExecutePlugin, - ParameterMetadata, - PluginParametersMetadata, - PluginParams, -} from '@grnsft/if-core/types'; -import {validate, allDefined} from '../../../common/util/validations'; +import {validateArithmeticExpression} from '@grnsft/if-core/utils'; +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; -import {STRINGS} from '../../config'; - -const {SCI_EMBODIED_ERROR} = STRINGS; - -export const SciEmbodied = ( - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', +export const SciEmbodied = PluginFactory({ + metadata: { inputs: { - ...({ - 'device/emissions-embodied': { - description: 'total embodied emissions of some component', - unit: 'gCO2e', - 'aggregation-method': 'sum', + vCPUs: { + description: 'number of CPUs allocated to an application', + unit: 'CPUs', + 'aggregation-method': { + time: 'copy', + component: 'copy', }, - 'device/expected-lifespan': { - description: 'Total Expected Lifespan of the Component in Seconds', - unit: 'seconds', - 'aggregation-method': 'sum', + }, + memory: { + description: 'RAM available for a resource, in GB', + unit: 'GB', + 'aggregation-method': { + time: 'copy', + component: 'copy', }, - 'resources-reserved': { - description: 'resources reserved for an application', - unit: 'count', - 'aggregation-method': 'none', + }, + ssd: { + description: 'number of SSDs available for a resource', + unit: 'SSDs', + 'aggregation-method': { + time: 'copy', + component: 'copy', }, - 'resources-total': { - description: 'total resources available', - unit: 'count', - 'aggregation-method': 'none', + }, + hdd: { + description: 'number of HDDs available for a resource', + unit: 'HDDs', + 'aggregation-method': { + time: 'copy', + component: 'copy', }, - 'vcpus-allocated': { - description: 'number of vcpus allocated to particular resource', - unit: 'count', - 'aggregation-method': 'none', + }, + gpu: { + description: 'number of GPUs available for a resource', + unit: 'GPUs', + 'aggregation-method': { + time: 'copy', + component: 'copy', + }, + }, + 'usage-ratio': { + description: + 'a scaling factor that can be used to describe the ratio of actual resource usage comapred to real device usage, e.g. 0.25 if you are using 2 out of 8 vCPUs, 0.1 if you are responsible for 1 out of 10 GB of storage, etc', + unit: 'dimensionless', + 'aggregation-method': { + time: 'copy', + component: 'copy', }, - 'vcpus-total': { - description: - 'total number of vcpus available on a particular resource', - unit: 'count', - 'aggregation-method': 'none', + }, + time: { + description: + 'a time unit to scale the embodied carbon by, in seconds. If not provided,time defaults to the value of the timestep duration.', + unit: 'seconds', + 'aggregation-method': { + time: 'copy', + component: 'copy', }, - } as ParameterMetadata), - ...parametersMetadata?.inputs, + }, }, - outputs: parametersMetadata?.outputs || { - 'carbon-embodied': { - description: 'embodied emissions of the component', - unit: 'gCO2e', - 'aggregation-method': 'sum', + outputs: { + 'embodied-carbon': { + description: 'embodied carbon for a resource, scaled by usage', + unit: 'gCO2eq', + 'aggregation-method': { + time: 'sum', + component: 'sum', + }, }, }, - }; - - const METRICS = [ - 'device/emissions-embodied', - 'device/expected-lifespan', - 'resources-reserved', - 'vcpus-allocated', - 'resources-total', - 'vcpus-total', - ]; - - /** - * Calculate the Embodied carbon for a list of inputs. - */ - const execute = (inputs: PluginParams[]) => - inputs.map(input => { - const safeInput = validateInput(input); + }, + configValidation: z.object({ + 'baseline-vcpus': z.preprocess( + value => validateArithmeticExpression('baseline-vcpus', value, 'number'), + z.number().gte(0).default(1) + ), + 'baseline-memory': z.preprocess( + value => validateArithmeticExpression('baseline-memory', value, 'number'), + z.number().gte(0).default(16) + ), + 'baseline-emissions': z.preprocess( + value => + validateArithmeticExpression('baseline-emissions', value, 'number'), + z.number().gte(0).default(1000000) + ), + lifespan: z.preprocess( + value => validateArithmeticExpression('lifespan', value, 'number'), + z.number().gt(0).default(126144000) + ), + 'vcpu-emissions-constant': z.preprocess( + value => + validateArithmeticExpression( + 'vcpu-emissions-constant', + value, + 'number' + ), + z.number().gte(0).default(100000) + ), + 'memory-emissions-constant': z.preprocess( + value => + validateArithmeticExpression( + 'memory-emissions-constant', + value, + 'number' + ), + z + .number() + .gte(0) + .default(533 / 384) + ), + 'ssd-emissions-constant': z.preprocess( + value => + validateArithmeticExpression('ssd-emissions-constant', value, 'number'), + z.number().gte(0).default(50000) + ), + 'hdd-emissions-constant': z.preprocess( + value => + validateArithmeticExpression('hdd-emissions-constant', value, 'number'), + z.number().gte(0).default(100000) + ), + 'gpu-emissions-constant': z.preprocess( + value => + validateArithmeticExpression('gpu-emissions-constant', value, 'number'), + z.number().gte(0).default(150000) + ), + 'output-parameter': z.string().optional(), + }), + inputValidation: z.object({ + duration: z.number().gt(0), + vCPUs: z.number().gt(0).default(1), + memory: z.number().gt(0).default(16), + ssd: z.number().gte(0).default(0), + hdd: z.number().gte(0).default(0), + gpu: z.number().gte(0).default(0), + 'usage-ratio': z.number().gt(0).default(1), + time: z.number().gt(0).optional(), + }), + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + /** + * 1. Validates configuration and assigns defaults values if not provided. + * 2. Maps through observations and validates them. + * 3. Calculates total embodied carbon by substracting and the difference between baseline server and given one. + */ + return inputs.map(input => { + const cpuE = + (input.vCPUs - config['baseline-vcpus']) * + config['vcpu-emissions-constant']; + const memoryE = + (input.memory - config['baseline-memory']) * + ((config['memory-emissions-constant'] * config['baseline-memory']) / + 16) * + 1000; + const hddE = input.hdd * config['hdd-emissions-constant']; + const gpuE = input.gpu * config['gpu-emissions-constant']; + const ssdE = input.ssd * config['ssd-emissions-constant']; + const time = input['time'] || input.duration; + + const totalEmbodied = + config['baseline-emissions'] + cpuE + memoryE + ssdE + hddE + gpuE; + + const totalEmbodiedScaledByUsage = totalEmbodied * input['usage-ratio']; + + const totalEmbodiedScaledByUsageAndTime = + totalEmbodiedScaledByUsage * (time / config['lifespan']); + + const embodiedCarbonKey = config['output-parameter'] || 'embodied-carbon'; return { ...input, - 'carbon-embodied': calculateEmbodiedCarbon(safeInput), + [embodiedCarbonKey]: totalEmbodiedScaledByUsageAndTime, }; }); - - /** - * Calculate the Embodied carbon for the input. - * M = totalEmissions * (duration/ExpectedLifespan) * (resourcesReserved/totalResources) - */ - const calculateEmbodiedCarbon = (input: PluginParams) => { - const totalEmissions = input['device/emissions-embodied']; - const duration = input['duration']; - const expectedLifespan = input['device/expected-lifespan']; - const resourcesReserved = - input['vcpus-allocated'] || input['resources-reserved']; - const totalResources = input['vcpus-total'] || input['resources-total']; - - return ( - totalEmissions * - (duration / expectedLifespan) * - (resourcesReserved / totalResources) - ); - }; - - /** - * Checks for required fields in input. - */ - const validateInput = (input: PluginParams) => { - const commonSchemaPart = (errorMessage: (unit: string) => string) => ({ - 'device/emissions-embodied': z - .number({ - invalid_type_error: errorMessage('gCO2e'), - }) - .gte(0) - .min(0), - 'device/expected-lifespan': z - .number({ - invalid_type_error: errorMessage('gCO2e'), - }) - .gte(0) - .min(0), - duration: z - .number({ - invalid_type_error: errorMessage('seconds'), - }) - .gte(1), - }); - - const vcpusSchemaPart = { - 'vcpus-allocated': z - .number({ - invalid_type_error: SCI_EMBODIED_ERROR('count'), - }) - .gte(0) - .min(0), - 'vcpus-total': z - .number({ - invalid_type_error: SCI_EMBODIED_ERROR('count'), - }) - .gte(0) - .min(0), - }; - - const resourcesSchemaPart = { - 'resources-reserved': z - .number({ - invalid_type_error: SCI_EMBODIED_ERROR('count'), - }) - .gte(0) - .min(0), - 'resources-total': z - .number({ - invalid_type_error: SCI_EMBODIED_ERROR('count'), - }) - .gte(0) - .min(0), - }; - - const schemaWithVcpus = z.object({ - ...commonSchemaPart(SCI_EMBODIED_ERROR), - ...vcpusSchemaPart, - }); - const schemaWithResources = z.object({ - ...commonSchemaPart(SCI_EMBODIED_ERROR), - ...resourcesSchemaPart, - }); - - const schema = schemaWithVcpus.or(schemaWithResources).refine(allDefined, { - message: `All ${METRICS} should be present.`, - }); - - return validate>(schema, input); - }; - - return { - metadata, - execute, - }; -}; + }, + allowArithmeticExpressions: [ + 'baseline-vcpus', + 'baseline-memory', + 'baseline-emissions', + 'lifespan', + 'vcpu-emissions-constant', + 'memory-emissions-constant', + 'ssd-emissions-constant', + 'hdd-emissions-constant', + 'gpu-emissions-constant', + ], +}); diff --git a/src/if-run/builtins/sci/README.md b/src/if-run/builtins/sci/README.md index 31b15afed..b628a5eb9 100644 --- a/src/if-run/builtins/sci/README.md +++ b/src/if-run/builtins/sci/README.md @@ -4,7 +4,7 @@ ## Parameters -### Plugin global config +### Plugin config - `functional-unit`: the name of the functional unit in which to express the carbon impact (required) @@ -16,17 +16,34 @@ The `parameter-metadata` section contains information about `description`, `unit - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. - `outputs`: describe the `sci` parameter which has the following attributes: + - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +sci: + method: Sci + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs - `carbon`: total carbon in gCO2eq (required) -- `functional-unit`: whatever `functional-unit` you define in global config also has to be present in each input, for example if you provide `functional-unit: requests` in global config, `requests` must be present in your input data. +- `functional-unit`: whatever `functional-unit` you define in config also has to be present in each input, for example if you provide `functional-unit: requests` in config, `requests` must be present in your input data. ## Returns @@ -49,11 +66,15 @@ To run the plugin, you must first create an instance of `Sci`. Then, you can cal ```typescript import {Sci} from 'builtins'; -const sci = Sci({'functional-unit': 'requests'}); +const config = {'functional-unit': 'requests'} +const parametersMetadata = {inputs: {}, outputs: {}}; + +const sci = Sci(config, parametersMetadata, {}); + const results = await sci.execute( [ { - 'carbon': 5' + 'carbon': 5 duration: 1, requests: 100, }, @@ -63,7 +84,7 @@ const results = await sci.execute( ## Example manifest -IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `ie` and does not have to be done explicitly by the user. +IF users will typically call the plugin as part of a pipeline defined in a `manifest` file. In this case, instantiating the plugin is handled by `if-run` and does not have to be done explicitly by the user. The following is an example `manifest` that calls `sci`: @@ -76,7 +97,7 @@ initialize: sci: method: Sci path: 'builtin' - global-config: + config: functional-unit: 'requests' tree: children: diff --git a/src/if-run/builtins/sci/index.ts b/src/if-run/builtins/sci/index.ts index 1106f3725..4238878dd 100644 --- a/src/if-run/builtins/sci/index.ts +++ b/src/if-run/builtins/sci/index.ts @@ -1,60 +1,62 @@ import {z} from 'zod'; + +import {PluginFactory} from '@grnsft/if-core/interfaces'; +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; + +import {validate} from '../../../common/util/validations'; + import {ERRORS} from '@grnsft/if-core/utils'; -import { - ExecutePlugin, - PluginParams, - ConfigParams, - PluginParametersMetadata, - ParameterMetadata, -} from '@grnsft/if-core/types'; -import {validate, allDefined} from '../../../common/util/validations'; +import {allDefined} from '../../../common/util/validations'; import {STRINGS} from '../../config'; -const {MissingInputDataError} = ERRORS; +const {MissingInputDataError, ConfigError} = ERRORS; const { MISSING_FUNCTIONAL_UNIT_CONFIG, MISSING_FUNCTIONAL_UNIT_INPUT, SCI_MISSING_FN_UNIT, ZERO_DIVISION, + MISSING_CONFIG, } = STRINGS; -export const Sci = ( - globalConfig: ConfigParams, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', +export const Sci = PluginFactory({ + metadata: { inputs: { - ...({ - carbon: { - description: 'an amount of carbon emitted into the atmosphere', - unit: 'gCO2e', - 'aggregation-method': 'sum', + carbon: { + description: 'an amount of carbon emitted into the atmosphere', + unit: 'gCO2e', + 'aggregation-method': { + time: 'sum', + component: 'sum', }, - 'functional-unit': { - description: - 'the name of the functional unit in which the final SCI value should be expressed, e.g. requests, users', - unit: 'none', - 'aggregation-method': 'sum', + }, + 'functional-unit': { + description: + 'the name of the functional unit in which the final SCI value should be expressed, e.g. requests, users', + unit: 'none', + 'aggregation-method': { + time: 'sum', + component: 'sum', }, - } as ParameterMetadata), - ...parametersMetadata?.inputs, + }, }, - outputs: parametersMetadata?.outputs || { + outputs: { sci: { description: 'carbon expressed in terms of the given functional unit', unit: 'gCO2e', - 'aggregation-method': 'sum', + 'aggregation-method': { + time: 'avg', + component: 'sum', + }, }, }, - }; + }, + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); + } - /** - * Validates node and gloabl configs. - */ - const validateConfig = (config?: ConfigParams) => { const schema = z .object({ 'functional-unit': z.string(), @@ -64,43 +66,11 @@ export const Sci = ( }); return validate>(schema, config); - }; - - /** - * Calculate the total emissions for a list of inputs. - */ - const execute = (inputs: PluginParams[]): PluginParams[] => - inputs.map((input, index) => { - const safeInput = validateInput(input); - const functionalUnit = input[globalConfig['functional-unit']]; + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { + const functionalUnit = config['functional-unit']; - if (functionalUnit === 0) { - console.warn(ZERO_DIVISION(Sci.name, index)); - - return { - ...input, - sci: safeInput['carbon'], - }; - } - - return { - ...input, - sci: safeInput['carbon'] / functionalUnit, - }; - }); - - /** - * Checks for fields in input. - */ - const validateInput = (input: PluginParams) => { - const validatedConfig = validateConfig(globalConfig); - - if ( - !( - validatedConfig['functional-unit'] in input && - input[validatedConfig['functional-unit']] >= 0 - ) - ) { + if (!(functionalUnit in input && input[functionalUnit] >= 0)) { throw new MissingInputDataError(MISSING_FUNCTIONAL_UNIT_INPUT); } @@ -110,14 +80,32 @@ export const Sci = ( duration: z.number().gte(1), }) .refine(allDefined, { - message: SCI_MISSING_FN_UNIT(globalConfig['functional-unit']), + message: SCI_MISSING_FN_UNIT(config['functional-unit']), }); return validate>(schema, input); - }; + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + return inputs.map((input, index) => { + const functionalUnit = isNaN(config['functional-unit']) + ? input[config['functional-unit']] + : config['functional-unit']; - return { - metadata, - execute, - }; -}; + if (functionalUnit === 0) { + console.warn(ZERO_DIVISION(Sci.name, index)); + + return { + ...input, + sci: input['carbon'], + }; + } + const calculatedResult = input['carbon'] / functionalUnit; + + return { + ...input, + sci: calculatedResult, + }; + }); + }, + allowArithmeticExpressions: ['functional-unit'], +}); diff --git a/src/if-run/builtins/shell/README.md b/src/if-run/builtins/shell/README.md index d114adb3b..3eb98c5e6 100644 --- a/src/if-run/builtins/shell/README.md +++ b/src/if-run/builtins/shell/README.md @@ -4,7 +4,7 @@ The `shell` is a wrapper enabling plugins implemented in any other programming l ## Parameters -### Plugin global config +### Plugin config The plugin should be initialized as follows: @@ -14,7 +14,7 @@ initialize: shell: method: Shell path: 'builtin' - global-config: + config: command: python3 /usr/local/bin/sampler ``` @@ -30,12 +30,16 @@ The `parameter-metadata` section contains information about `description`, `unit - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. - `outputs`: describe the output parameter. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. ### Inputs @@ -53,7 +57,11 @@ The specific return types depend on the plugin being invoked. Typically, we woul To run the plugin, you must first create an instance of `Shell` and call its `execute()` to run the external plugin. ```typescript -const output = Shell({command: '/usr/local/bin/sampler'}); +const config = { + command: '/usr/local/bin/sampler', +}; +const parametersMetadata = {inputs: {}, outputs: {}}; +const output = Shell(config, parametersMetadata); const result = await output.execute([ { timestamp: '2021-01-01T00:00:00Z', @@ -85,7 +93,7 @@ initialize: sampler: method: Shell path: 'builtin' - global-config: + config: command: python3 /usr/local/bin/sampler tree: children: @@ -112,7 +120,7 @@ initialize: sampler: method: Shell path: 'builtin' - global-config: + config: command: python3 /usr/local/bin/sampler tree: children: diff --git a/src/if-run/builtins/shell/index.ts b/src/if-run/builtins/shell/index.ts index 9abe8363c..a6752f3a0 100644 --- a/src/if-run/builtins/shell/index.ts +++ b/src/if-run/builtins/shell/index.ts @@ -2,74 +2,56 @@ import {spawnSync, SpawnSyncReturns} from 'child_process'; import {loadAll, dump} from 'js-yaml'; import {z} from 'zod'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; import {ERRORS} from '@grnsft/if-core/utils'; -import { - ExecutePlugin, - PluginParams, - ConfigParams, - PluginParametersMetadata, -} from '@grnsft/if-core/types'; +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; import {validate} from '../../../common/util/validations'; -const {ProcessExecutionError} = ERRORS; +import {STRINGS} from '../../config'; -export const Shell = ( - globalConfig: ConfigParams, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; +const {ProcessExecutionError, ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; - /** - * Calculate the total emissions for a list of inputs. - */ - const execute = (inputs: PluginParams[]): any[] => { - const inputWithConfig = Object.assign({}, inputs[0], validateConfig()); - const command = inputWithConfig.command; - const inputAsString: string = dump(inputs, {indent: 2}); - const results = runModelInShell(inputAsString, command); - - return results?.outputs?.flat(); - }; +export const Shell = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config) { + throw new ConfigError(MISSING_CONFIG); + } - /** - * Checks for required fields in input. - */ - const validateConfig = () => { const schema = z.object({ command: z.string(), }); - return validate>(schema, globalConfig); - }; - - /** - * Runs the model in a shell. Spawns a child process to run an external IMP, - * an executable with a CLI exposing two methods: `--execute` and `--manifest`. - * The shell command then calls the `--command` method passing var manifest as the path to the desired manifest file. - */ - const runModelInShell = (input: string, command: string) => { - try { - const [executable, ...args] = command.split(' '); - - const result: SpawnSyncReturns = spawnSync(executable, args, { - input, - encoding: 'utf8', - }); - const outputs = loadAll(result.stdout); + return validate>(schema, config); + }, + implementation: async (inputs, config) => { + const inputWithConfig = Object.assign({}, inputs[0], config); + const command = inputWithConfig.command; + const inputAsString: string = dump(inputs, {indent: 2}); + const results = runModelInShell(inputAsString, command); - return {outputs}; - } catch (error: any) { - throw new ProcessExecutionError(error.message); - } - }; + return results?.outputs?.flat() as PluginParams[]; + }, +}); + +/** + * Runs the model in a shell. Spawns a child process to run an external IMP, + * an executable with a CLI exposing two methods: `--execute` and `--manifest`. + * The shell command then calls the `--command` method passing var manifest as the path to the desired manifest file. + */ +const runModelInShell = (input: string, command: string) => { + try { + const [executable, ...args] = command.split(' '); + + const result: SpawnSyncReturns = spawnSync(executable, args, { + input, + encoding: 'utf8', + }); + const outputs = loadAll(result.stdout); - return { - metadata, - execute, - }; + return {outputs}; + } catch (error: any) { + throw new ProcessExecutionError(error.message); + } }; diff --git a/src/if-run/builtins/subtract/README.md b/src/if-run/builtins/subtract/README.md index 776526c7e..a36b4ba11 100644 --- a/src/if-run/builtins/subtract/README.md +++ b/src/if-run/builtins/subtract/README.md @@ -10,7 +10,7 @@ For example, you could subtract `cpu/energy` and `network/energy` and name the r ### Plugin config -Two parameters are required in global config: `input-parameters` and `output-parameter`. +Two parameters are required in config: `input-parameters` and `output-parameter`. `input-parameters`: an array of strings. Each string should match an existing key in the `inputs` array `output-parameter`: a string defining the name to use to add the result of the diff to the output array. @@ -19,16 +19,32 @@ Two parameters are required in global config: `input-parameters` and `output-par The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe parameters of the `input-parameters` of the global config. Each parameter has the following attributes: +- `inputs`: describe parameters of the `input-parameters` of the config. Each parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. -- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes: +- `outputs`: describe the parameter of the `output-parameter` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +subtract: + method: Subtract + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -36,7 +52,7 @@ All of `input-parameters` must be available in the input array. ## Returns -- `output-parameter`: the subtraction of all `input-parameters` with the parameter name defined by `output-parameter` in global config. +- `output-parameter`: the subtraction of all `input-parameters` with the parameter name defined by `output-parameter` in config. ## Calculation @@ -55,9 +71,10 @@ const config = { inputParameters: ['cpu/energy', 'network/energy'], outputParameter: 'offset/energy', }; - -const subtract = Subtract(config); -const result = subtract subtract.execute([ +const parametersMetadata = {inputs: {}, outputs: {}}; +const mapping = {}; +const subtract = Subtract(config, parametersMetadata, mapping); +const result = await subtract.execute([ { duration: 3600, timestamp: '2021-01-01T00:00:00Z', @@ -80,7 +97,7 @@ initialize: subtract: method: Subtract path: 'builtin' - global-config: + config: input-parameters: ['cpu/energy', 'network/energy'] output-parameter: 'energy/diff' tree: @@ -113,4 +130,4 @@ The results will be saved to a new `yaml` file in `manifests/outputs`. This error arises when an invalid value is passed to `Subtract`. Typically, this can occur when a non-numeric value (such as a string made of alphabetic characters) is passed where a number or numeric string is expected. Please check that the types are correct for all the relevant fields in your `inputs` array. -For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors +For more information on our error classes, please visit [our docs](https://if.greensoftware.foundation/reference/errors) diff --git a/src/if-run/builtins/subtract/index.ts b/src/if-run/builtins/subtract/index.ts index 2598ecb8a..3bd852b0a 100644 --- a/src/if-run/builtins/subtract/index.ts +++ b/src/if-run/builtins/subtract/index.ts @@ -1,47 +1,34 @@ import {z} from 'zod'; -import { - ExecutePlugin, - PluginParametersMetadata, - PluginParams, - SubtractConfig, -} from '@grnsft/if-core/types'; + +import {PluginFactory} from '@grnsft/if-core/interfaces'; +import {ConfigParams, PluginParams} from '@grnsft/if-core/types'; +import {ERRORS} from '@grnsft/if-core/utils'; import {validate} from '../../../common/util/validations'; -export const Subtract = ( - globalConfig: SubtractConfig, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; +import {STRINGS} from '../../config'; + +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - const globalConfigSchema = z.object({ +export const Subtract = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); + } + + const configSchema = z.object({ 'input-parameters': z.array(z.string()), 'output-parameter': z.string().min(1), }); - return validate>( - globalConfigSchema, - globalConfig - ); - }; + return validate>(configSchema, config); + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { + const inputParameters = config['input-parameters']; - /** - * Checks for required fields in input. - */ - const validateSingleInput = ( - input: PluginParams, - inputParameters: string[] - ) => { const inputData = inputParameters.reduce( - (acc, param) => { + (acc: {[x: string]: any}, param: string | number) => { acc[param] = input[param]; return acc; @@ -51,44 +38,34 @@ export const Subtract = ( const validationSchema = z.record(z.string(), z.number()); - validate(validationSchema, inputData); - - return input; - }; - - /** - * Subtract items from inputParams[1..n] from inputParams[0] and write the result in a new param outputParam. - */ - const execute = (inputs: PluginParams[]): PluginParams[] => { + return validate(validationSchema, inputData); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { const { 'input-parameters': inputParameters, 'output-parameter': outputParameter, - } = validateGlobalConfig(); + } = config; return inputs.map(input => { - validateSingleInput(input, inputParameters); + const calculatedResult = calculateDiff(input, inputParameters); return { ...input, - [outputParameter]: calculateDiff(input, inputParameters), + [outputParameter]: calculatedResult, }; }); - }; + }, + allowArithmeticExpressions: [], +}); - /** - * Calculates the diff between the 1st item in the inputs nad the rest of the items - */ - const calculateDiff = (input: PluginParams, inputParameters: string[]) => { - const [firstItem, ...restItems] = inputParameters; - - return restItems.reduce( - (accumulator, metricToSubtract) => accumulator - input[metricToSubtract], - input[firstItem] // Starting accumulator with the value of the first item - ); - }; +/** + * Calculates the diff between the 1st item in the inputs nad the rest of the items + */ +const calculateDiff = (input: PluginParams, inputParameters: string[]) => { + const [firstItem, ...restItems] = inputParameters; - return { - metadata, - execute, - }; + return restItems.reduce( + (accumulator, metricToSubtract) => accumulator - input[metricToSubtract], + input[firstItem] // Starting accumulator with the value of the first item + ); }; diff --git a/src/if-run/builtins/sum/README.md b/src/if-run/builtins/sum/README.md index 14ad1c336..f140b486e 100644 --- a/src/if-run/builtins/sum/README.md +++ b/src/if-run/builtins/sum/README.md @@ -10,7 +10,7 @@ For example, you could add `cpu/energy` and `network/energy` and name the result ### Plugin config -Two parameters are required in global config: `input-parameters` and `output-parameter`. +Two parameters are required in config: `input-parameters` and `output-parameter`. `input-parameters`: an array of strings. Each string should match an existing key in the `inputs` array `output-parameter`: a string defining the name to use to add the result of summing the input parameters to the output array. @@ -19,16 +19,32 @@ Two parameters are required in global config: `input-parameters` and `output-par The `parameter-metadata` section contains information about `description`, `unit` and `aggregation-method` of the parameters of the inputs and outputs -- `inputs`: describe parameters of the `input-parameters` of the global config. Each parameter has: +- `inputs`: describe parameters of the `input-parameters` of the config. Each parameter has: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. -- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes: +- `outputs`: describe the parameter of the `output-parameter` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - - `aggregation-method`: aggregation method of the parameter (it can be `sum`, `avg` or `none`) + - `aggregation-method`: aggregation method object of the parameter + - `time`: this value is used for `horizontal` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + - `component`: this value is used for `vertical` aggregation. It can be of the following values: `sum`, `avg`, `copy`, or `none`. + +### Mapping + +The `mapping` block is an optional block. It is added in the plugin section and allows the plugin to receive a parameter from the input with a different name than the one the plugin uses for data manipulation. The parameter with the mapped name will not appear in the outputs. It also maps the output parameter of the plugin. The structure of the `mapping` block is: + +```yaml +sum: + method: Sum + path: 'builtin' + mapping: + 'parameter-name-in-the-plugin': 'parameter-name-in-the-input' +``` ### Inputs @@ -36,7 +52,7 @@ All of `input-parameters` must be available in the input array. ## Returns -- `output-parameter`: the sum of all `input-parameters` with the parameter name defined by `output-parameter` in global config. +- `output-parameter`: the sum of all `input-parameters` with the parameter name defined by `output-parameter` in config. ## Calculation @@ -53,9 +69,14 @@ const config = { inputParameters: ['cpu/energy', 'network/energy'], outputParameter: 'energy', }; +const parametersMetadata = {inputs: {}, outputs: {}}; +const = mapping { + 'cpu/energy': 'energy-from-cpu', + 'network/energy': 'energy-from-network', +}; -const sum = Sum(config, parametersMetadata); -const result = sum.execute([ +const sum = Sum(config, parametersMetadata, mapping); +const result = await sum.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -78,7 +99,7 @@ initialize: sum: method: Sum path: 'builtin' - global-config: + config: input-parameters: ['cpu/energy', 'network/energy'] output-parameter: 'energy' parameter-metadata: @@ -86,16 +107,28 @@ initialize: cpu/energy: description: energy consumed by the cpu unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum + aggregation-method: + time: sum + component: sum network/energy: description: energy consumed by data ingress and egress unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum + aggregation-method: + time: sum + component: sum outputs: energy: description: sum of energy components unit: kWh - aggregation-method: sum + aggregation-method: + time: sum + component: sum tree: children: child: @@ -121,9 +154,9 @@ The results will be saved to a new `yaml` file in `./examples/outputs`. `Sum` exposes two of the IF error classes. -### GlobalConfigError +### ConfigError -You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. +You will receive an error starting `ConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: diff --git a/src/if-run/builtins/sum/index.ts b/src/if-run/builtins/sum/index.ts index da642dd13..5081f0c1d 100644 --- a/src/if-run/builtins/sum/index.ts +++ b/src/if-run/builtins/sum/index.ts @@ -1,75 +1,19 @@ import {z} from 'zod'; -import {ERRORS} from '@grnsft/if-core/utils'; -import { - ExecutePlugin, - PluginParams, - SumConfig, - PluginParametersMetadata, -} from '@grnsft/if-core/types'; -import {validate} from '../../../common/util/validations'; - -import {STRINGS} from '../../config'; - -const {GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; - -export const Sum = ( - globalConfig: SumConfig, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; - - /** - * Calculate the sum of each input-paramters. - */ - const execute = (inputs: PluginParams[]) => { - const safeGlobalConfig = validateGlobalConfig(); - const inputParameters = safeGlobalConfig['input-parameters']; - const outputParameter = safeGlobalConfig['output-parameter']; - - return inputs.map(input => { - validateSingleInput(input, inputParameters); - - return { - ...input, - [outputParameter]: calculateSum(input, inputParameters), - }; - }); - }; +import {PluginFactory} from '@grnsft/if-core/interfaces'; +import {PluginParams, ConfigParams} from '@grnsft/if-core/types'; - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - if (!globalConfig) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); - } - - const globalConfigSchema = z.object({ - 'input-parameters': z.array(z.string()), - 'output-parameter': z.string().min(1), - }); - - return validate>( - globalConfigSchema, - globalConfig - ); - }; +import {validate} from '../../../common/util/validations'; - /** - * Checks for required fields in input. - */ - const validateSingleInput = ( - input: PluginParams, - inputParameters: string[] - ) => { +export const Sum = PluginFactory({ + configValidation: z.object({ + 'input-parameters': z.array(z.string()), + 'output-parameter': z.string().min(1), + }), + inputValidation: (input: PluginParams, config: ConfigParams) => { + const inputParameters = config['input-parameters']; const inputData = inputParameters.reduce( - (acc, param) => { + (acc: {[x: string]: any}, param: string | number) => { acc[param] = input[param]; return acc; @@ -77,22 +21,31 @@ export const Sum = ( {} as Record ); const validationSchema = z.record(z.string(), z.number()); - validate(validationSchema, inputData); - - return input; - }; + return validate(validationSchema, inputData); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + const { + 'input-parameters': inputParameters, + 'output-parameter': outputParameter, + } = config; - /** - * Calculates the sum of the energy components. - */ - const calculateSum = (input: PluginParams, inputParameters: string[]) => - inputParameters.reduce( - (accumulator, metricToSum) => accumulator + input[metricToSum], - 0 - ); + return inputs.map(input => { + const calculatedResult = calculateSum(input, inputParameters); - return { - metadata, - execute, - }; -}; + return { + ...input, + [outputParameter]: calculatedResult, + }; + }); + }, + allowArithmeticExpressions: [], +}); + +/** + * Calculates the sum of the energy components. + */ +const calculateSum = (input: PluginParams, inputParameters: string[]) => + inputParameters.reduce( + (accumulator, metricToSum) => accumulator + input[metricToSum], + 0 + ); diff --git a/src/if-run/builtins/time-converter/README.md b/src/if-run/builtins/time-converter/README.md index b6c945c20..6b4ca9dc2 100644 --- a/src/if-run/builtins/time-converter/README.md +++ b/src/if-run/builtins/time-converter/README.md @@ -10,7 +10,7 @@ For example, you could add `energy-per-year`, the time unit `year`, and the new ### Plugin config -These parameters are required in global config: +These parameters are required in config: - `input-parameter`: a string that should match an existing key in the `inputs` array - `original-time-unit`: a string that defines the time unit of the `input-parameter`. The original time unit should be a valid unit, like `year`, `month`, `day`, `hour` and so on @@ -21,13 +21,13 @@ These parameters are required in global config: The `parameter-metadata` section contains information about `description` and `unit` of the parameters of the inputs and outputs -- `inputs`: describe parameters of the `input-parameter` of the global config. Each parameter has: +- `inputs`: describe parameters of the `input-parameter` of the config. Each parameter has: - `description`: description of the parameter - `unit`: unit of the parameter - `aggregation-method`: the aggregation method of the parameter (can be `sum`, `avg` or `none`) -- `outputs`: describe the parameter of the `output-parameter` of the global config. The parameter has the following attributes: +- `outputs`: describe the parameter of the `output-parameter` of the config. The parameter has the following attributes: - `description`: description of the parameter - `unit`: unit of the parameter - `aggregation-method`: the aggregation method of the parameter (can be `sum`, `avg` or `none`) @@ -38,7 +38,7 @@ The `input-parameter` must be available in the input array. ## Returns -- `output-parameter`: the converted energy of the `input-parameter` with the parameter name defined by `output-parameter` in global config. +- `output-parameter`: the converted energy of the `input-parameter` with the parameter name defined by `output-parameter` in config. ## Calculation @@ -59,7 +59,7 @@ const config = { }; const timeConverter = TimeConverter(config, parametersMetadata); -const result = timeConverter.execute([ +const result = await timeConverter.execute([ { timestamp: '2021-01-01T00:00:00Z', duration: 3600, @@ -81,7 +81,7 @@ initialize: time-converter: method: TimeConverter path: builtin - global-config: + config: input-parameter: 'energy-per-year' original-time-unit: 'year' new-time-unit: 'duration' @@ -111,9 +111,9 @@ The results will be saved to a new `yaml` file in `./examples/outputs`. `TimeConverter` exposes two of the IF error classes. -### GlobalConfigError +### ConfigError -You will receive an error starting `GlobalConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. +You will receive an error starting `ConfigError: ` if you have not provided the expected configuration data in the plugin's `initialize` block. The required parameters are: diff --git a/src/if-run/builtins/time-converter/index.ts b/src/if-run/builtins/time-converter/index.ts index cb25951a9..f50c51614 100644 --- a/src/if-run/builtins/time-converter/index.ts +++ b/src/if-run/builtins/time-converter/index.ts @@ -1,80 +1,22 @@ import {z} from 'zod'; + +import {PluginParams, ConfigParams} from '@grnsft/if-core/types'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; import {ERRORS} from '@grnsft/if-core/utils'; -import { - ExecutePlugin, - PluginParams, - PluginParametersMetadata, - ConfigParams, -} from '@grnsft/if-core/types'; import {validate} from '../../../common/util/validations'; -import {STRINGS} from '../../config'; - import {TIME_UNITS_IN_SECONDS} from './config'; -const {GlobalConfigError} = ERRORS; -const {MISSING_GLOBAL_CONFIG} = STRINGS; - -export const TimeConverter = ( - globalConfig: ConfigParams, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', - inputs: parametersMetadata?.inputs, - outputs: parametersMetadata?.outputs, - }; - - const execute = (inputs: PluginParams[]) => { - const safeGlobalConfig = validateGlobalConfig(); - const inputParameter = safeGlobalConfig['input-parameter']; - const outputParameter = safeGlobalConfig['output-parameter']; - - return inputs.map(input => { - validateInput(input, inputParameter); - - return { - ...input, - [outputParameter]: calculateEnergy(input), - }; - }); - }; - - /** - * Calculates the energy for given period. - */ - const calculateEnergy = (input: PluginParams) => { - const originalTimeUnit = globalConfig['original-time-unit']; - const originalTimeUnitInSeoncds = TIME_UNITS_IN_SECONDS[originalTimeUnit]; - const energyPerPeriod = input[globalConfig['input-parameter']]; - const newTimeUnit = - globalConfig['new-time-unit'] === 'duration' - ? input.duration - : TIME_UNITS_IN_SECONDS[globalConfig['new-time-unit']]; - const result = (energyPerPeriod / originalTimeUnitInSeoncds) * newTimeUnit; - - return Number(result.toFixed(6)); - }; - - /** - * Checks for required fields in input. - */ - const validateInput = (input: PluginParams, inputParameter: string) => { - const schema = z.object({ - duration: z.number().gte(1), - [inputParameter]: z.number(), - }); +import {STRINGS} from '../../config'; - return validate>(schema, input); - }; - - /** - * Checks global config value are valid. - */ - const validateGlobalConfig = () => { - if (!globalConfig) { - throw new GlobalConfigError(MISSING_GLOBAL_CONFIG); +const {ConfigError} = ERRORS; +const {MISSING_CONFIG} = STRINGS; + +export const TimeConverter = PluginFactory({ + configValidation: (config: ConfigParams) => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); } const timeUnitsValues = Object.keys(TIME_UNITS_IN_SECONDS); @@ -84,20 +26,56 @@ export const TimeConverter = ( ] as const; const originalTimeUnitValues = timeUnitsValues as [string, ...string[]]; - const globalConfigSchema = z.object({ + const configSchema = z.object({ 'input-parameter': z.string(), 'original-time-unit': z.enum(originalTimeUnitValues), 'new-time-unit': z.enum(originalTimeUnitValuesWithDuration), 'output-parameter': z.string().min(1), }); - return validate>( - globalConfigSchema, - globalConfig - ); - }; - return { - metadata, - execute, - }; + return validate>(configSchema, config); + }, + inputValidation: (input: PluginParams, config: ConfigParams) => { + const inputParameter = config['input-parameter']; + + const schema = z.object({ + duration: z.number().gte(1), + [inputParameter]: z.number(), + }); + + return validate>(schema, input); + }, + implementation: async (inputs: PluginParams[], config: ConfigParams) => { + const outputParameter = config['output-parameter']; + + return inputs.map(input => ({ + ...input, + [outputParameter]: calculateEnergy(input, config), + })); + }, + allowArithmeticExpressions: ['input-parameter'], +}); + +/** + * Calculates the energy for given period. + */ +const calculateEnergy = (input: PluginParams, config: ConfigParams) => { + const { + 'original-time-unit': originalTimeUnit, + 'input-parameter': inputParameter, + 'new-time-unit': newTimeUnit, + } = config; + + const originalTimeUnitInSeoncds = TIME_UNITS_IN_SECONDS[originalTimeUnit]; + const energyPerPeriod = isNaN(Number(inputParameter)) + ? input[inputParameter] + : inputParameter; + const timeUnit = + newTimeUnit === 'duration' + ? input.duration + : TIME_UNITS_IN_SECONDS[newTimeUnit]; + + const result = (energyPerPeriod / originalTimeUnitInSeoncds) * timeUnit; + + return Number(result.toFixed(6)); }; diff --git a/src/if-run/builtins/time-sync/README.md b/src/if-run/builtins/time-sync/README.md index 335866546..dd09a89f2 100644 --- a/src/if-run/builtins/time-sync/README.md +++ b/src/if-run/builtins/time-sync/README.md @@ -9,9 +9,10 @@ Time sync standardizes the start time, end time and temporal resolution of all o The following should be defined in the plugin initialization: - `start-time`: global start time as ISO 8061 string -- `stop`: global end time as ISO 8061 string +- `end-time`: global end time as ISO 8061 string - `interval`: temporal resolution in seconds -- `error-on-padding`: avoid zero/'zeroish' padding (if needed) and error out instead. `False` by defult. +- `allow-padding`: avoid zero/'zeroish' padding (if needed) and error out instead. +- `upsampling-resolution`: temporal resolution at which observations will be upsampled, in seconds. Defaults to 1. #### Inputs: @@ -28,7 +29,7 @@ A manifest file for a tree might contain many nodes each representing some diffe We do this by implementing the following logic: - Shift readings to nearest whole seconds -- Upsample the time series to a base resolution (1s) +- Upsample the time series to a base resolution. - Resample to desired resolution by batching 1s entries - Extrapolate or trim to ensure all time series share global start and end dates @@ -39,6 +40,7 @@ The next section explains each stage in more detail. ##### Upsampling rules A set of `inputs` is naturally a time series because all `observations` include a `timestamp` and a `duration`, measured in seconds. + For each `observation` in `inputs` we check whether the duration is greater than 1 second. If `duration` is greater than 1 second, we create N new `observation` objects, where N is equal to `duration`. This means we have an `observation` for every second between the initial timestamp and the end of the observation period. Each new object receives a timestamp incremented by one second. This looks as follows: @@ -54,6 +56,7 @@ This looks as follows: {timestamp: '2023-12-12T00:00:04.000Z', duration: 1} {timestamp: '2023-12-12T00:00:05.000Z', duration: 1} ] + ``` Each `observation` actually includes many key-value pairs. The precise content of the `observation` is not known until runtime because it depends on which plugins have been included in the pipeline. Different values have to be treated differently when we upsample in time. The method we use to upsample depends on the `aggregation-method` defined for each key in `units.yml`. @@ -118,7 +121,7 @@ Note that when `error-on-padding` is `true` no gap-filling is performed and the ##### Trimming and padding -To ensure parity across all the components in a tree, we need to synchronize the start and end times for all time series. To do this, we pass the `time-sync` plugin plugin some global config: `startTime`, `endTime` and `interval`. The `startTime` is the timestamp where _all_ input arrays across the entire tree should begin, and `endTime` is the timestamp where _all_ input arrays across the entire tree should end. `interval` is the time resolution we ultimately want to resample to. +To ensure parity across all the components in a tree, we need to synchronize the start and end times for all time series. To do this, we pass the `time-sync` plugin plugin some config: `startTime`, `endTime` and `interval`. The `startTime` is the timestamp where _all_ input arrays across the entire tree should begin, and `endTime` is the timestamp where _all_ input arrays across the entire tree should end. `interval` is the time resolution we ultimately want to resample to. To synchronize the time series start and end we check the first element of `inputs` for each node in the tree and determine whether it is earlier, later or equal to the global start time. If it is equal then no action is required. If the `input` start time is earlier than the global start time, we simply discard entries from the front of the array until the start times are aligned. If the `input` start time is after the global start time, then we pad with our "zero-observation" object - one for every second separating the global start time from the `input` start time. The same process is repeated for the end time - we either trim away `input` data or pad it out with "zero-observation" objects. @@ -151,12 +154,31 @@ For example, for `startTime = 2023-12-12T00:00:00.000Z` and `endTime = 2023-12-1 ] ``` -Note that when `error-on-padding` is `true` no padding is performed and the plugin will error out instead. +Note that when `allow-padding` is `true` no padding is performed and the plugin will error out instead. ##### Resampling rules Now we have synchronized, continuous, high resolution time series data, we can resample. To achieve this, we use `interval`, which sets the global temporal resolution for the final, processed time series. `interval` is expressed in units of seconds, which means we can simply batch `observations` together in groups of size `interval`. For each value in each object we either sum, average or copy the values into one single summary object representing each time bucket of size `interval` depending on their `aggregation-method` defined in `aggregation` section in the manifest file. The returned array is the final, synchronized time series at the desired temporal resolution. +#### Setting a custom upsampling resolution + +The model defaults to upsampling observations to a 1-second resolution. However, this can lead to unnecessary effort, as upsampling at a coarser resolution is often sufficient, provided it doesn't interfere with the accuracy of resampling. To optimize performance, we can set the `upsampling-resolution` parameter in the configuration to a more appropriate value. The chosen value should meet the following criteria : + +- It should evenly divide all observation durations within the dataset. +- It must be a divisor of the `interval`. +- It should also divide any gaps between observations, as well as the start and end paddings. + +For example, for `interval = 10` and this time-series + +```ts +[ + {timestamp: '2023-12-12T00:00:00.000Z', duration: 300}, +] +```` +setting the `upsampling-resolution` to `10s` is preferable to the default behavior. +If the default behavior were used, the model would create `300` samples of `1s` each, which would be inefficient. By setting a custom `upsampling-resolution` of `10s`, the model only generates `30` samples, each representing `10s`. + + #### Assumptions and limitations To do time synchronization, we assume: @@ -170,12 +192,13 @@ To run the plugin, you must first create an instance of `TimeSync`. Then, you can call `execute()`. ```typescript -const globalConfig = { +const config = { 'start-time': '2023-12-12T00:00:00.000Z', 'end-time': '2023-12-12T00:00:30.000Z', - interval: 10 + interval: 10, + 'allow-padding': true, } -const timeSync = TimeSync(globalConfig); +const timeSync = TimeSync(config); const results = timeSync.execute([ { timestamp: '2023-12-12T00:00:00.000Z' @@ -213,8 +236,8 @@ initialize: method: TeadsCurve path: '@grnsft/if-unofficial-plugins' sci-e: - method: SciE - path: '@grnsft/if-plugins' + method: SciEmbodied + path: 'builtin' sci-embodied: path: 'builtin' method: SciEmbodied @@ -224,10 +247,11 @@ initialize: time-sync: method: TimeSync path: builtin - global-config: + config: start-time: '2023-12-12T00:00:00.000Z' # ISO timestamp end-time: '2023-12-12T00:01:00.000Z' # ISO timestamp interval: 5 # seconds + allow-padding: true tree: children: child: # an advanced grouping node @@ -238,19 +262,16 @@ tree: - sci-embodied - sci-o - time-sync - config: - teads-curve: - cpu/thermal-design-power: 65 - sci-embodied: - device/emissions-embodied: 251000 # gCO2eq - time-reserved: 3600 # 1 hour in s - device/expected-lifespan: 126144000 # 4 years in seconds - resources-reserved: 1 - resources-total: 1 - sci-o: - grid/carbon-intensity: 457 # gCO2/kwh children: child-1: + defaults: + device/emissions-embodied: 251000 # gCO2eq + time-reserved: 3600 # 1 hour in s + device/expected-lifespan: 126144000 # 4 years in seconds + resources-reserved: 1 + resources-total: 1 + grid/carbon-intensity: 457 # gCO2/kwh + cpu/thermal-design-power: 65 inputs: - timestamp: '2023-12-12T00:00:00.000Z' duration: 10 @@ -260,6 +281,7 @@ tree: requests: 300 - timestamp: '2023-12-12T00:00:10.000Z' duration: 10 + cpu/thermal-design-power: 65 cpu/utilization: 20 carbon: 200 energy: 200 diff --git a/src/if-run/builtins/time-sync/index.ts b/src/if-run/builtins/time-sync/index.ts index 27f0aac8e..0c1ecfb22 100644 --- a/src/if-run/builtins/time-sync/index.ts +++ b/src/if-run/builtins/time-sync/index.ts @@ -4,37 +4,38 @@ import {Settings, DateTime, DateTimeMaybeValid, Interval} from 'luxon'; import {z} from 'zod'; import {ERRORS} from '@grnsft/if-core/utils'; import { - ExecutePlugin, PluginParams, PaddingReceipt, - TimeNormalizerConfig, + MappingParams, + ConfigParams, TimeParams, - PluginParametersMetadata, - ParameterMetadata, + TimeNormalizerConfig, } from '@grnsft/if-core/types'; import {validate} from '../../../common/util/validations'; import {STRINGS} from '../../config'; -import {getAggregationMethod} from '../../lib/aggregate'; +import {getAggregationInfoFor} from '../../lib/aggregate'; +import {PluginFactory} from '@grnsft/if-core/interfaces'; Settings.defaultZone = 'utc'; const { - GlobalConfigError, + ConfigError, InvalidDateInInputError, InvalidPaddingError, InvalidInputError, } = ERRORS; const { - INVALID_TIME_NORMALIZATION, + INCOMPATIBLE_RESOLUTION_WITH_INTERVAL, + INCOMPATIBLE_RESOLUTION_WITH_GAPS, + INCOMPATIBLE_RESOLUTION_WITH_INPUTS, INVALID_OBSERVATION_OVERLAP, AVOIDING_PADDING_BY_EDGES, INVALID_DATE_TYPE, START_LOWER_END, - TIMESTAMP_REQUIRED, - INVALID_DATETIME, + MISSING_CONFIG, } = STRINGS; /** @@ -52,454 +53,521 @@ const { * allow-padding: true * ``` */ -export const TimeSync = ( - globalConfig: TimeNormalizerConfig, - parametersMetadata: PluginParametersMetadata -): ExecutePlugin => { - const metadata = { - kind: 'execute', +export const TimeSync = PluginFactory({ + metadata: { inputs: { - ...({ - timestamp: { - description: 'refers to the time of occurrence of the input', - unit: 'RFC3339', - 'aggregation-method': 'none', + timestamp: { + description: 'refers to the time of occurrence of the input', + unit: 'RFC3339', + 'aggregation-method': { + time: 'none', + component: 'none', }, - duration: { - description: 'refers to the duration of the input', - unit: 'seconds', - 'aggregation-method': 'sum', + }, + duration: { + description: 'refers to the duration of the input', + unit: 'seconds', + 'aggregation-method': { + time: 'sum', + component: 'none', }, - } as ParameterMetadata), - ...parametersMetadata?.inputs, + }, }, - outputs: parametersMetadata?.outputs, - }; - - /** - * Take input array and return time-synchronized input array. - */ - const execute = (inputs: PluginParams[]): PluginParams[] => { - const validatedConfig = validateGlobalConfig(); - const timeParams = { - startTime: DateTime.fromISO(validatedConfig['start-time']), - endTime: DateTime.fromISO(validatedConfig['end-time']), - interval: validatedConfig.interval, - allowPadding: validatedConfig['allow-padding'], + }, + configValidation: (config: ConfigParams): TimeNormalizerConfig => { + if (!config || !Object.keys(config)?.length) { + throw new ConfigError(MISSING_CONFIG); + } + + const schema = z + .object({ + 'start-time': z.string().datetime(), + 'end-time': z.string().datetime(), + interval: z.number(), + 'allow-padding': z.boolean(), + 'upsampling-resolution': z.number().min(1).optional(), + }) + .refine(data => data['start-time'] < data['end-time'], { + message: START_LOWER_END, + }); + + return validate>(schema, config); + }, + inputValidation: (input: PluginParams, _config: any, index?: number) => { + const schema = z.object({ + timestamp: z.string().datetime({}).or(z.date()), + duration: z.number(), + }); + + return validate>(schema, input, index); + }, + implementation: async ( + inputs: PluginParams[], + config, + mapping?: MappingParams + ) => { + /** + * Checks if a given duration is compatible with a given timeStep. If not, throws an error + */ + const validateIntervalForResample = ( + duration: number, + timeStep: number, + errorMessage: string + ) => { + if (duration % timeStep !== 0) { + throw new ConfigError(errorMessage); + } }; - const pad = checkForPadding(inputs, timeParams); - validatePadding(pad, timeParams); + /** + * Dates are passed to `time-sync` both in ISO 8601 format + * and as a Date object (from the deserialization of a YAML file). + * If the YAML parser fails to identify as a date, it passes as a string. + */ + const parseDate = (date: Date | string) => { + if (!date) { + return DateTime.invalid('Invalid date'); + } - const paddedInputs = padInputs(inputs, pad, timeParams); + if (isDate(date)) { + return DateTime.fromJSDate(date); + } - const flattenInputs = paddedInputs.reduce( - (acc: PluginParams[], input, index) => { - const safeInput = Object.assign({}, input, validateInput(input, index)); - const currentMoment = parseDate(safeInput.timestamp); + if (typeof date === 'string') { + return DateTime.fromISO(date); + } - /** Checks if not the first input, then check consistency with previous ones. */ - if (index > 0) { - const previousInput = paddedInputs[index - 1]; - const previousInputTimestamp = parseDate(previousInput.timestamp); + throw new InvalidDateInInputError(INVALID_DATE_TYPE(date)); + }; - /** Checks for timestamps overlap. */ - if ( - parseDate(previousInput.timestamp).plus({ - seconds: previousInput.duration, - }) > currentMoment - ) { - throw new InvalidInputError(INVALID_OBSERVATION_OVERLAP); - } + /** + * Calculates minimal factor. + */ + const convertPerInterval = ( + value: number, + duration: number, + timeStep: number + ) => { + const samplesNumber = duration / timeStep; + + return value / samplesNumber; + }; - const compareableTime = previousInputTimestamp.plus({ - seconds: previousInput.duration, - }); + /** + * Normalize time per given second. + */ + const normalizeTimePerSecond = ( + currentRoundMoment: Date | string, + i: number + ) => { + const thisMoment = parseDate(currentRoundMoment).startOf('second'); - const timelineGapSize = currentMoment - .diff(compareableTime) - .as('seconds'); + return thisMoment.plus({seconds: i}); + }; - /** Checks if there is gap in timeline. */ - if (timelineGapSize > 1) { - acc.push( - ...getZeroishInputPerSecondBetweenRange( - compareableTime, - currentMoment, - safeInput - ) - ); - } + /** + * Breaks down input per minimal time unit. + */ + const breakDownInput = ( + input: PluginParams, + i: number, + params: TimeParams + ) => { + const metrics = Object.keys(input); + const timeStep = params.upsamplingResolution; + + return metrics.reduce((acc, metric) => { + const aggregationParams = getAggregationInfoFor(metric); + + if (metric === 'timestamp') { + const perSecond = normalizeTimePerSecond(input.timestamp, i); + acc[metric] = perSecond.toUTC().toISO() ?? ''; + + return acc; } - /** Break down current observation. */ - for (let i = 0; i < safeInput.duration; i++) { - const normalizedInput = breakDownInput(safeInput, i); - acc.push(normalizedInput); + if (metric === 'duration') { + acc[metric] = timeStep; + + return acc; } - return trimInputsByGlobalTimeline(acc, timeParams); - }, - [] as PluginParams[] - ); + if (aggregationParams.time === 'none') { + acc[metric] = null; - const sortedInputs = flattenInputs.sort((a, b) => - parseDate(a.timestamp).diff(parseDate(b.timestamp)).as('seconds') - ); + return acc; + } - return resampleInputs(sortedInputs, timeParams) as PluginParams[]; - }; - - /** - * Dates are passed to `time-sync` both in ISO 8601 format - * and as a Date object (from the deserialization of a YAML file). - * If the YAML parser fails to identify as a date, it passes as a string. - */ - const parseDate = (date: Date | string) => { - if (!date) { - return DateTime.invalid('Invalid date'); - } + acc[metric] = + aggregationParams.time === 'sum' + ? convertPerInterval(input[metric], input['duration'], timeStep) + : input[metric]; - if (isDate(date)) { - return DateTime.fromJSDate(date); - } + return acc; + }, {} as PluginParams); + }; - if (typeof date === 'string') { - return DateTime.fromISO(date); - } + /** + * Populates object to fill the gaps in observational timeline using zeroish values. + */ + const fillWithZeroishInput = ( + input: PluginParams, + missingTimestamp: DateTimeMaybeValid, + timeStep: number + ) => { + const metrics = Object.keys(input); + return metrics.reduce((acc, metric) => { + if (metric === 'timestamp') { + acc[metric] = + missingTimestamp.startOf('second').toUTC().toISO() ?? ''; - throw new InvalidDateInInputError(INVALID_DATE_TYPE(date)); - }; + return acc; + } - /** - * Validates input parameters. - */ - const validateInput = (input: PluginParams, index: number) => { - const schema = z.object({ - timestamp: z - .string({ - required_error: TIMESTAMP_REQUIRED(index), - }) - .datetime({ - message: INVALID_DATETIME(index), - }) - .or(z.date()), - duration: z.number(), - }); + if (metric === 'duration') { + acc[metric] = timeStep; - return validate>(schema, input); - }; + return acc; + } - /** - * Validates global config parameters. - */ - const validateGlobalConfig = () => { - if (globalConfig === undefined) { - throw new GlobalConfigError(INVALID_TIME_NORMALIZATION); - } + if ( + metric === 'time-reserved' || + (mapping && + mapping['time-reserved'] && + metric === mapping['time-reserved']) + ) { + acc[metric] = acc['duration']; - const schema = z - .object({ - 'start-time': z.string().datetime(), - 'end-time': z.string().datetime(), - interval: z.number(), - 'allow-padding': z.boolean(), - }) - .refine(data => data['start-time'] < data['end-time'], { - message: START_LOWER_END, - }); + return acc; + } - return validate>(schema, globalConfig); - }; - - /** - * Calculates minimal factor. - */ - const convertPerInterval = (value: number, duration: number) => - value / duration; - - /** - * Normalize time per given second. - */ - const normalizeTimePerSecond = ( - currentRoundMoment: Date | string, - i: number - ) => { - const thisMoment = parseDate(currentRoundMoment).startOf('second'); + const aggregationParams = getAggregationInfoFor(metric); + + if (aggregationParams.time === 'none') { + acc[metric] = null; - return thisMoment.plus({seconds: i}); - }; + return acc; + } - /** - * Breaks down input per minimal time unit. - */ - const breakDownInput = (input: PluginParams, i: number) => { - const inputKeys = Object.keys(input); + if ( + aggregationParams.time === 'avg' || + aggregationParams.time === 'sum' + ) { + acc[metric] = 0; - return inputKeys.reduce((acc, key) => { - const method = getAggregationMethod(key); + return acc; + } - if (key === 'timestamp') { - const perSecond = normalizeTimePerSecond(input.timestamp, i); - acc[key] = perSecond.toUTC().toISO() ?? ''; + if (aggregationParams.time === 'copy') { + acc[metric] = input[metric]; + return acc; + } return acc; - } + }, {} as PluginParams); + }; - /** @todo use user defined resolution later */ - if (key === 'duration') { - acc[key] = 1; + /** + * Checks if `error on padding` is enabled and padding is needed. If so, then throws error. + */ + const validatePadding = (pad: PaddingReceipt, params: TimeParams): void => { + const {start, end} = pad; + const isPaddingNeeded = start || end; - return acc; + if (!params.allowPadding && isPaddingNeeded) { + throw new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(start, end)); } + }; - acc[key] = - method === 'sum' - ? convertPerInterval(input[key], input['duration']) - : input[key]; - - return acc; - }, {} as PluginParams); - }; - - /** - * Populates object to fill the gaps in observational timeline using zeroish values. - */ - const fillWithZeroishInput = ( - input: PluginParams, - missingTimestamp: DateTimeMaybeValid - ) => { - const metrics = Object.keys(input); + /** + * Checks if padding is needed either at start of the timeline or the end and returns status. + */ + const checkForPadding = ( + inputs: PluginParams[], + params: TimeParams + ): PaddingReceipt => { + const startDiffInSeconds = parseDate(inputs[0].timestamp) + .diff(params.startTime) + .as('seconds'); - return metrics.reduce((acc, metric) => { - if (metric === 'timestamp') { - acc[metric] = missingTimestamp.startOf('second').toUTC().toISO() ?? ''; + const lastInput = inputs[inputs.length - 1]; - return acc; - } + const endDiffInSeconds = parseDate(lastInput.timestamp) + .plus({second: eval(lastInput.duration)}) + .diff(params.endTime) + .as('seconds'); + return { + start: startDiffInSeconds > 0, + end: endDiffInSeconds < 0, + }; + }; - /** @todo later will be changed to user defined interval */ - if (metric === 'duration') { - acc[metric] = 1; + /** + * Iterates over given inputs frame, meanwhile checking if aggregation method is `sum`, then calculates it. + * For methods is `avg` and `none` calculating average of the frame. + */ + const resampleInputFrame = (inputsInTimeslot: PluginParams[]) => + inputsInTimeslot.reduce((acc, input, index, inputs) => { + const metrics = Object.keys(input); - return acc; - } + metrics.forEach(metric => { + const aggregationParams = getAggregationInfoFor(metric); - if (metric === 'time-reserved') { - acc[metric] = acc['duration']; + if (metric === 'timestamp') { + acc[metric] = inputs[0][metric]; - return acc; - } + return; + } - const method = getAggregationMethod(metric); + if (metric === 'duration') { + aggregationParams.time = 'sum'; + } - if (method === 'avg' || method === 'sum') { - acc[metric] = 0; + if (aggregationParams.time === 'none') { + acc[metric] = null; + return; + } - return acc; - } + acc[metric] = acc[metric] ?? 0; - acc[metric] = input[metric]; + if (aggregationParams.time === 'sum') { + acc[metric] += input[metric]; - return acc; - }, {} as PluginParams); - }; + return; + } - /** - * Checks if `error on padding` is enabled and padding is needed. If so, then throws error. - */ - const validatePadding = (pad: PaddingReceipt, params: TimeParams): void => { - const {start, end} = pad; - const isPaddingNeeded = start || end; + if (aggregationParams.time === 'copy') { + acc[metric] = input[metric]; - if (!params.allowPadding && isPaddingNeeded) { - throw new InvalidPaddingError(AVOIDING_PADDING_BY_EDGES(start, end)); - } - }; + return; + } - /** - * Checks if padding is needed either at start of the timeline or the end and returns status. - */ - const checkForPadding = ( - inputs: PluginParams[], - params: TimeParams - ): PaddingReceipt => { - const startDiffInSeconds = parseDate(inputs[0].timestamp) - .diff(params.startTime) - .as('seconds'); - - const lastInput = inputs[inputs.length - 1]; - - const endDiffInSeconds = parseDate(lastInput.timestamp) - .plus({second: lastInput.duration}) - .diff(params.endTime) - .as('seconds'); - - return { - start: startDiffInSeconds > 0, - end: endDiffInSeconds < 0, - }; - }; - - /** - * Iterates over given inputs frame, meanwhile checking if aggregation method is `sum`, then calculates it. - * For methods is `avg` and `none` calculating average of the frame. - */ - const resampleInputFrame = (inputsInTimeslot: PluginParams[]) => - inputsInTimeslot.reduce((acc, input, index, inputs) => { - const metrics = Object.keys(input); + /** + * If timeslot contains records more than one, then divide each metric by the timeslot length, + * so that their sum yields the timeslot average. + */ + if ( + inputsInTimeslot.length > 1 && + index === inputsInTimeslot.length - 1 + ) { + acc[metric] /= inputsInTimeslot.length; - metrics.forEach(metric => { - let method = getAggregationMethod(metric); + return; + } - if (metric === 'timestamp') { - acc[metric] = inputs[0][metric]; + acc[metric] += input[metric]; + }); - return; + return acc; + }, {} as PluginParams); + + /** + * Takes each array frame with interval length, then aggregating them together as from units.yaml file. + */ + const resampleInputs = (inputs: PluginParams[], params: TimeParams) => + inputs.reduce((acc: PluginParams[], _input, index, inputs) => { + const frameStart = + (index * params.interval) / params.upsamplingResolution; + const frameEnd = + ((index + 1) * params.interval) / params.upsamplingResolution; + + const inputsFrame = inputs.slice(frameStart, frameEnd); + const resampledInput = resampleInputFrame(inputsFrame); + + /** Checks if resampled input is not empty, then includes in result. */ + if (Object.keys(resampledInput).length > 0) { + acc.push(resampledInput); } - if (metric === 'duration') { - method = 'sum'; - } + return acc; + }, [] as PluginParams[]); + + /** + * Pads zeroish inputs from the beginning or at the end of the inputs if needed. + */ + const padInputs = ( + inputs: PluginParams[], + pad: PaddingReceipt, + params: TimeParams + ): PluginParams[] => { + const {start, end} = pad; + const paddedFromBeginning = []; + + if (start) { + paddedFromBeginning.push( + ...getZeroishInputPerSecondBetweenRange( + { + startDate: params.startTime, + endDate: parseDate(inputs[0].timestamp), + timeStep: params.upsamplingResolution, + }, + inputs[0] + ) + ); + } - if (!method) { - return; - } + const paddedArray = paddedFromBeginning.concat(inputs); + + if (end) { + const lastInput = inputs[inputs.length - 1]; + const lastInputEnd = parseDate(lastInput.timestamp).plus({ + seconds: eval(lastInput.duration), + }); + paddedArray.push( + ...getZeroishInputPerSecondBetweenRange( + { + startDate: lastInputEnd, + endDate: params.endTime, + timeStep: params.upsamplingResolution, + }, + lastInput + ) + ); + } - acc[metric] = acc[metric] ?? 0; + return paddedArray; + }; - if (method === 'sum') { - acc[metric] += input[metric]; + /** + * Brakes down the given range by 1 second, and generates zeroish values. + */ + const getZeroishInputPerSecondBetweenRange = ( + params: PluginParams, + input: PluginParams + ) => { + const array: PluginParams[] = []; + validateIntervalForResample( + params.endDate.diff(params.startDate).as('seconds'), + params.timeStep, + INCOMPATIBLE_RESOLUTION_WITH_GAPS + ); + const dateRange = Interval.fromDateTimes( + params.startDate, + params.endDate + ); - return; - } + for (const interval of dateRange.splitBy({second: params.timeStep})) { + array.push( + fillWithZeroishInput( + input, + // as far as I can tell, start will never be null + // because if we pass an invalid start/endDate to + // Interval, we get a zero length array as the range + interval.start || DateTime.invalid('not expected - start is null'), + params.timeStep + ) + ); + } - if (method === 'none') { - acc[metric] = input[metric]; + return array; + }; - return; - } + /* + * Checks if input's timestamp is included in global specified period then leaves it, otherwise. + */ + const trimInputsByGlobalTimeline = ( + inputs: PluginParams[], + params: TimeParams + ): PluginParams[] => + inputs.reduce((acc: PluginParams[], item) => { + const {timestamp} = item; - /** - * If timeslot contains records more than one, then divide each metric by the timeslot length, - * so that their sum yields the timeslot average. - */ if ( - inputsInTimeslot.length > 1 && - index === inputsInTimeslot.length - 1 + parseDate(timestamp) >= params.startTime && + parseDate(timestamp) <= params.endTime ) { - acc[metric] /= inputsInTimeslot.length; - - return; + acc.push(item); } - acc[metric] += input[metric]; - }); - - return acc; - }, {} as PluginParams); + return acc; + }, [] as PluginParams[]); - /** - * Takes each array frame with interval length, then aggregating them together as from units.yaml file. - */ - const resampleInputs = (inputs: PluginParams[], params: TimeParams) => - inputs.reduce((acc: PluginParams[], _input, index, inputs) => { - const frameStart = index * params.interval; - const frameEnd = (index + 1) * params.interval; - const inputsFrame = inputs.slice(frameStart, frameEnd); + /** Implementation */ + const timeParams = { + startTime: DateTime.fromISO(config['start-time'] as string), + endTime: DateTime.fromISO(config['end-time'] as string), + interval: config.interval, + allowPadding: config['allow-padding'], + upsamplingResolution: config['upsampling-resolution'] + ? config['upsampling-resolution'] + : 1, + }; + validateIntervalForResample( + timeParams.interval, + timeParams.upsamplingResolution, + INCOMPATIBLE_RESOLUTION_WITH_INTERVAL + ); + const pad = checkForPadding(inputs, timeParams); + validatePadding(pad, timeParams); + const paddedInputs = padInputs(inputs, pad, timeParams); - const resampledInput = resampleInputFrame(inputsFrame); + const flattenInputs = paddedInputs.reduce( + (acc: PluginParams[], input, index) => { + const currentMoment = parseDate(input.timestamp); - /** Checks if resampled input is not empty, then includes in result. */ - if (Object.keys(resampledInput).length > 0) { - acc.push(resampledInput); - } + /** Checks if not the first input, then check consistency with previous ones. */ + if (index > 0) { + const previousInput = paddedInputs[index - 1]; + const previousInputTimestamp = parseDate(previousInput.timestamp); - return acc; - }, [] as PluginParams[]); + /** Checks for timestamps overlap. */ + if ( + parseDate(previousInput.timestamp).plus({ + seconds: eval(previousInput.duration), + }) > currentMoment + ) { + throw new InvalidInputError(INVALID_OBSERVATION_OVERLAP); + } - /** - * Pads zeroish inputs from the beginning or at the end of the inputs if needed. - */ - const padInputs = ( - inputs: PluginParams[], - pad: PaddingReceipt, - params: TimeParams - ): PluginParams[] => { - const {start, end} = pad; - const paddedFromBeginning = []; - - if (start) { - paddedFromBeginning.push( - ...getZeroishInputPerSecondBetweenRange( - params.startTime, - parseDate(inputs[0].timestamp), - inputs[0] - ) - ); - } + const compareableTime = previousInputTimestamp.plus({ + seconds: eval(previousInput.duration), + }); - const paddedArray = paddedFromBeginning.concat(inputs); + const timelineGapSize = currentMoment + .diff(compareableTime) + .as('seconds'); - if (end) { - const lastInput = inputs[inputs.length - 1]; - const lastInputEnd = parseDate(lastInput.timestamp).plus({ - seconds: lastInput.duration, - }); - paddedArray.push( - ...getZeroishInputPerSecondBetweenRange( - lastInputEnd, - params.endTime.plus({seconds: 1}), - lastInput - ) - ); - } + validateIntervalForResample( + input.duration, + timeParams.upsamplingResolution, + INCOMPATIBLE_RESOLUTION_WITH_INPUTS + ); - return paddedArray; - }; + if (timelineGapSize > 1) { + /** Checks if there is gap in timeline. */ + acc.push( + ...getZeroishInputPerSecondBetweenRange( + { + startDate: compareableTime, + endDate: currentMoment, + timeStep: timeParams.upsamplingResolution, + }, + input + ) + ); + } + } - const getZeroishInputPerSecondBetweenRange = ( - startDate: DateTimeMaybeValid, - endDate: DateTimeMaybeValid, - templateInput: PluginParams - ) => { - const array: PluginParams[] = []; - const dateRange = Interval.fromDateTimes(startDate, endDate); - - for (const interval of dateRange.splitBy({second: 1})) { - array.push( - fillWithZeroishInput( - templateInput, - // as far as I can tell, start will never be null - // because if we pass an invalid start/endDate to - // Interval, we get a zero length array as the range - interval.start || DateTime.invalid('not expected - start is null') - ) - ); - } + /** Break down current observation. */ + for ( + let i = 0; + i <= input.duration - timeParams.upsamplingResolution; + i += timeParams.upsamplingResolution + ) { + const normalizedInput = breakDownInput(input, i, timeParams); - return array; - }; + acc.push(normalizedInput); + } - /* - * Checks if input's timestamp is included in global specified period then leaves it, otherwise. - */ - const trimInputsByGlobalTimeline = ( - inputs: PluginParams[], - params: TimeParams - ): PluginParams[] => - inputs.reduce((acc: PluginParams[], item) => { - const {timestamp} = item; - - if ( - parseDate(timestamp) >= params.startTime && - parseDate(timestamp) <= params.endTime - ) { - acc.push(item); - } + return trimInputsByGlobalTimeline(acc, timeParams); + }, + [] as PluginParams[] + ); - return acc; - }, [] as PluginParams[]); + const sortedInputs = flattenInputs.sort((a, b) => + parseDate(a.timestamp).diff(parseDate(b.timestamp)).as('seconds') + ); - return {metadata, execute}; -}; + return resampleInputs(sortedInputs, timeParams) as PluginParams[]; + }, +}); diff --git a/src/if-run/config/config.ts b/src/if-run/config/config.ts index 9bd39d89f..4ec43ee29 100644 --- a/src/if-run/config/config.ts +++ b/src/if-run/config/config.ts @@ -38,6 +38,12 @@ export const CONFIG = { alias: 'h', description: '[prints out the above help instruction]', }, + append: { + type: Boolean, + optional: true, + alias: 'a', + description: '[append to outputs, instead of overwriting]', + }, debug: { type: Boolean, optional: true, @@ -71,5 +77,5 @@ export const CONFIG = { } as ParseOptions, GITHUB_PATH: 'https://github.com', NATIVE_PLUGIN: 'if-plugins', - AGGREGATION_ADDITIONAL_PARAMS: ['timestamp', 'duration'], + AGGREGATION_TIME_METRICS: ['timestamp', 'duration'], }; diff --git a/src/if-run/config/strings.ts b/src/if-run/config/strings.ts index 6231e9ff7..41b9dc013 100644 --- a/src/if-run/config/strings.ts +++ b/src/if-run/config/strings.ts @@ -1,32 +1,28 @@ export const STRINGS = { MISSING_METHOD: "Initalization param 'method' is missing.", MISSING_PATH: "Initalization param 'path' is missing.", - UNSUPPORTED_PLUGIN: - "Plugin interface doesn't implement 'execute' or 'metadata' methods.", NOT_NATIVE_PLUGIN: (path: string) => ` You are using plugin ${path} which is not part of the Impact Framework standard library. You should do your own research to ensure the plugins are up to date and accurate. They may not be actively maintained.`, INVALID_MODULE_PATH: (path: string, error?: any) => `Provided module \`${path}\` is invalid or not found. ${error ?? ''} `, - INVALID_TIME_NORMALIZATION: 'Start time or end time is missing.', + INCOMPATIBLE_RESOLUTION_WITH_INTERVAL: + 'The upsampling resolution must be a divisor of the given interval, but the provided value does not satisfy this criteria.', + INCOMPATIBLE_RESOLUTION_WITH_INPUTS: + 'The upsampling resolution must be a divisor of all inputs durations, but the provided values do not satisfy this criteria.', + INCOMPATIBLE_RESOLUTION_WITH_GAPS: + 'The upsampling resolution must be a divisor of gaps and paddings in the time-series, but the provided values do not satisfy this criteria.', UNEXPECTED_TIME_CONFIG: 'Unexpected node-level config provided for time-sync plugin.', - INVALID_TIME_INTERVAL: 'Interval is missing.', - AVOIDING_PADDING: (description: string) => - `Avoiding padding at ${description}`, AVOIDING_PADDING_BY_EDGES: (start: boolean, end: boolean) => `Avoiding padding at ${ start && end ? 'start and end' : start ? 'start' : 'end' }`, - INVALID_AGGREGATION_METHOD: (metric: string) => - `Aggregation is not possible for given ${metric} since method is 'none'.`, METRIC_MISSING: (metric: string, index: number) => `Aggregation metric ${metric} is not found in inputs[${index}].`, INVALID_GROUP_KEY: (key: string) => `Invalid group ${key}.`, REGROUP_ERROR: 'not an array or should contain at least one key', - INVALID_EXHAUST_PLUGIN: (pluginName: string) => - `Invalid exhaust plugin: ${pluginName}.`, UNKNOWN_PARAM: (name: string) => `Unknown parameter: ${name}. Omitting from the output.`, NOT_INITALIZED_PLUGIN: (name: string) => @@ -46,15 +42,24 @@ Note that for the '--output' option you also need to define the output type in y CHECKING_AGGREGATION_METHOD: (unitName: string) => `Checking aggregation method for ${unitName}`, INITIALIZING_PLUGINS: 'Initializing plugins', - INITIALIZING_PLUGIN: (pluginName: string) => `Initializing ${pluginName}`, + INITIALIZING_PLUGIN: (pluginName: string) => + `Initializing \`${pluginName}\` instance`, LOADING_PLUGIN_FROM_PATH: (pluginName: string, path: string) => `Loading ${pluginName} from ${path}`, COMPUTING_PIPELINE_FOR_NODE: (nodeName: string) => - `Computing pipeline for \`${nodeName}\``, + `Running compute pipeline: \`${nodeName}\` plugin`, + COMPUTING_COMPONENT_PIPELINE: (component: string) => + `**Computing \`${component}\` pipeline**`, + REGROUPING: 'Regrouping', + OBSERVING: (nodeName: string) => + `Running observe pipeline: \`${nodeName}\` plugin`, MERGING_DEFAULTS_WITH_INPUT_DATA: 'Merging defaults with input data', AGGREGATING_OUTPUTS: 'Aggregating outputs', AGGREGATING_NODE: (nodeName: string) => `Aggregating node ${nodeName}`, - PREPARING_OUTPUT_DATA: 'Preparing output data', + PREPARING_OUTPUT_DATA: () => { + console.debug('\n'); + return 'Preparing output data'; + }, EXPORTING_TO_YAML_FILE: (savepath: string) => `Exporting to yaml file: ${savepath}`, EMPTY_PIPELINE: `You're using an old style manifest. Please update for phased execution. More information can be found here: @@ -63,11 +68,8 @@ https://if.greensoftware.foundation/major-concepts/manifest-file`, OUTPUT_REQUIRED: 'Output path is required, please make sure output is configured properly.', /** Plugins messages */ - INVALID_NAME: - '`name` global config parameter is empty or contains all spaces', + INVALID_NAME: '`name` config parameter is empty or contains all spaces', START_LOWER_END: '`start-time` should be lower than `end-time`', - TIMESTAMP_REQUIRED: (index: number) => `required in input[${index}]`, - INVALID_DATETIME: (index: number) => `invalid datetime in input[${index}]`, X_Y_EQUAL: 'The length of `x` and `y` should be equal', ARRAY_LENGTH_NON_EMPTY: 'the length of the input arrays must be greater than 1', @@ -84,13 +86,11 @@ https://if.greensoftware.foundation/major-concepts/manifest-file`, SCI_MISSING_FN_UNIT: (functionalUnit: string) => `'carbon' and ${functionalUnit} should be present in your input data.`, MISSING_FUNCTIONAL_UNIT_CONFIG: - '`functional-unit` should be provided in your global config', + '`functional-unit` should be provided in your config', MISSING_FUNCTIONAL_UNIT_INPUT: '`functional-unit` value is missing from input data or it is not a positive integer', REGEX_MISMATCH: (input: any, match: string) => `\`${input}\` does not match the ${match} regex expression`, - SCI_EMBODIED_ERROR: (unit: string) => - `invalid number. please provide it as \`${unit}\` to input`, MISSING_MIN_MAX: 'Config is missing min or max value', INVALID_MIN_MAX: (name: string) => `Min value should not be greater than or equal to max value of ${name}`, @@ -106,7 +106,18 @@ ${message}`, ${error}`, ZERO_DIVISION: (moduleName: string, index: number) => `-- SKIPPING -- DivisionByZero: you are attempting to divide by zero in ${moduleName} plugin : inputs[${index}]\n`, - MISSING_GLOBAL_CONFIG: 'Global config is not provided.', + MISSING_CONFIG: 'Config is not provided.', MISSING_INPUT_DATA: (param: string) => `${param} is missing from the input array, or has nullish value.`, + CONFIG_WARN: (plugins: string, isMore: boolean) => { + const withoutPlugins = + 'You have included node-level config in your manifest. IF no longer supports node-level config. The manifest should be refactored to accept all its node-level config from config or input data.'; + const withPlugins = `You have included node-level config in your manifest to support \`${plugins}\` plugin${ + isMore ? 's' : '' + }. IF no longer supports node-level config. \`${plugins}\` plugin${ + isMore ? 's' : '' + } should be refactored to accept all its config from config or input data.`; + + return plugins.length ? withPlugins : withoutPlugins; + }, }; diff --git a/src/if-run/index.ts b/src/if-run/index.ts index 6d9f61a78..0a29409a5 100644 --- a/src/if-run/index.ts +++ b/src/if-run/index.ts @@ -5,15 +5,13 @@ import {debugLogger} from '../common/util/debug-logger'; import {logger} from '../common/util/logger'; import {load} from '../common/lib/load'; -import {aggregate, storeAggregationMetrics} from './lib/aggregate'; +import {aggregate} from './lib/aggregate'; import {injectEnvironment} from './lib/environment'; import {initialize} from './lib/initialize'; import {compute} from './lib/compute'; import {exhaust} from './lib/exhaust'; import {explain} from './lib/explain'; -import {AGGREGATION_METHODS} from './types/aggregation'; - import {parseIfRunProcessArgs} from './util/args'; import {andHandle} from './util/helpers'; @@ -31,6 +29,7 @@ const impactEngine = async () => { observe, regroup, compute: computeFlag, + append, } = options; debugLogger.overrideConsoleMethods(!!debug); @@ -44,16 +43,6 @@ const impactEngine = async () => { try { const {tree, ...context} = validateManifest(envManifest); - if (context.aggregation) { - const convertMetrics = context.aggregation?.metrics.map( - (metric: string) => ({ - [metric]: AGGREGATION_METHODS[2], - }) - ); - - storeAggregationMetrics(...convertMetrics); - } - const pluginStorage = await initialize(context); const computedTree = await compute(tree, { context, @@ -61,6 +50,7 @@ const impactEngine = async () => { observe, regroup, compute: computeFlag, + append, }); const aggregatedTree = aggregate(computedTree, context.aggregation); @@ -69,6 +59,7 @@ const impactEngine = async () => { await exhaust(aggregatedTree, context, outputOptions); } catch (error) { if (error instanceof Error) { + /** Execution block exists because manifest is already processed. Set's status to `fail`. */ envManifest.execution!.status = 'fail'; envManifest.execution!.error = error.toString(); logger.error(error); diff --git a/src/if-run/lib/aggregate.ts b/src/if-run/lib/aggregate.ts index 508d7df25..ef3142b38 100644 --- a/src/if-run/lib/aggregate.ts +++ b/src/if-run/lib/aggregate.ts @@ -1,3 +1,4 @@ +import {AGGREGATION_METHODS} from '@grnsft/if-core/consts'; import {PluginParams} from '@grnsft/if-core/types'; import {debugLogger} from '../../common/util/debug-logger'; @@ -8,11 +9,9 @@ import { AggregationMetricsWithMethod, } from '../../common/types/manifest'; -import {aggregateInputsIntoOne} from '../util/aggregation-helper'; +import {aggregateOutputsIntoOne} from '../util/aggregation-helper'; import {memoizedLog} from '../util/log-memoize'; -import {AggregationMetric} from '../types/aggregation'; - import {STRINGS} from '../config/strings'; const { @@ -39,13 +38,13 @@ const getIthElementsFromChildren = (children: any, i: number) => { * 1. Gets the i'th element from each childrens outputs (treating children as rows and we are after a column of data). * 2. Now we just aggregate over the `ithSliceOfOutputs` the same as we did for the normal outputs. */ -const temporalAggregation = (node: any, metrics: AggregationMetric[]) => { +const temporalAggregation = (node: any, metrics: string[]) => { const outputs: PluginParams[] = []; const values: any = Object.values(node.children); for (let i = 0; i < values[0].outputs.length; i++) { const ithSliceOfOutputs = getIthElementsFromChildren(node.children, i); - outputs.push(aggregateInputsIntoOne(ithSliceOfOutputs, metrics, true)); + outputs.push(aggregateOutputsIntoOne(ithSliceOfOutputs, metrics, true)); } return outputs; @@ -63,9 +62,7 @@ const temporalAggregation = (node: any, metrics: AggregationMetric[]) => { * 5. Now a grouping node has it's own outputs, it can horizotnally aggregate them. */ const aggregateNode = (node: any, aggregationParams: AggregationParamsSure) => { - const metrics: AggregationMetric[] = aggregationParams!.metrics.map( - metric => ({[metric]: 'none'}) - ); + const metrics = aggregationParams!.metrics; const type = aggregationParams!.type; if (node.children) { @@ -77,14 +74,16 @@ const aggregateNode = (node: any, aggregationParams: AggregationParamsSure) => { } if (!node.children) { - if (type === 'horizontal' || type === 'both') { - node.aggregated = aggregateInputsIntoOne(node.outputs, metrics); + /** `time` aggregation is the new name of `horizontal`. */ + if (type === 'horizontal' || type === 'time' || type === 'both') { + node.aggregated = aggregateOutputsIntoOne(node.outputs, metrics); } } else { - if (type === 'vertical' || type === 'both') { + /** `component` aggregation is the new name of `vertical`. */ + if (type === 'vertical' || type === 'component' || type === 'both') { const outputs = temporalAggregation(node, metrics); node.outputs = outputs; - node.aggregated = aggregateInputsIntoOne(outputs, metrics); + node.aggregated = aggregateOutputsIntoOne(outputs, metrics); } } }; @@ -126,13 +125,13 @@ export const storeAggregationMetrics = ( * Creates an encapsulated object to retrieve the metrics. */ const metricManager = (() => { - let metric: AggregationMetric; + let metric: AggregationMetricsWithMethod; const manager = { get metrics() { return metric; }, - set metrics(value: AggregationMetric) { + set metrics(value: AggregationMetricsWithMethod) { metric = value; }, }; @@ -141,21 +140,25 @@ const metricManager = (() => { })(); /** - * Returns aggregation method for given `unitName`. If doesn't exist then returns value `sum`. + * Returns aggregation method for given `metric`. */ -export const getAggregationMethod = (unitName: string) => { +export const getAggregationInfoFor = (metric: string) => { debugLogger.setExecutingPluginName(); - memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(unitName)); + memoizedLog(console.debug, '\n'); + memoizedLog(console.debug, CHECKING_AGGREGATION_METHOD(metric)); const aggregationMetricsStorage = storeAggregationMetrics(); if ( aggregationMetricsStorage && - Object.keys(aggregationMetricsStorage).includes(unitName) + Object.keys(aggregationMetricsStorage).includes(metric) ) { - return aggregationMetricsStorage[unitName]; + return aggregationMetricsStorage[metric]; } - memoizedLog(logger.warn, UNKNOWN_PARAM(unitName)); + memoizedLog(logger.warn, UNKNOWN_PARAM(metric)); - return undefined; + return { + time: AGGREGATION_METHODS[3], + component: AGGREGATION_METHODS[3], + }; }; diff --git a/src/if-run/lib/compute.ts b/src/if-run/lib/compute.ts index e8f15e077..5e73cbbb4 100644 --- a/src/if-run/lib/compute.ts +++ b/src/if-run/lib/compute.ts @@ -3,22 +3,31 @@ import {PluginParams} from '@grnsft/if-core/types'; import {Regroup} from './regroup'; import {addExplainData} from './explain'; -import {mergeObjects} from '../util/helpers'; import {debugLogger} from '../../common/util/debug-logger'; import {logger} from '../../common/util/logger'; +import {mergeObjects} from '../util/helpers'; + import {STRINGS} from '../config/strings'; import {ComputeParams, Node, PhasedPipeline} from '../types/compute'; -import {isExecute} from '../types/interface'; -const {MERGING_DEFAULTS_WITH_INPUT_DATA, EMPTY_PIPELINE} = STRINGS; +const { + MERGING_DEFAULTS_WITH_INPUT_DATA, + EMPTY_PIPELINE, + CONFIG_WARN, + COMPUTING_PIPELINE_FOR_NODE, + COMPUTING_COMPONENT_PIPELINE, + REGROUPING, + OBSERVING, +} = STRINGS; /** * Traverses all child nodes based on children grouping. */ const traverse = async (children: any, params: ComputeParams) => { for (const child in children) { + console.debug(COMPUTING_COMPONENT_PIPELINE(child)); await computeNode(children[child], params); } }; @@ -38,11 +47,24 @@ const mergeDefaults = ( return response; } - console.debug(MERGING_DEFAULTS_WITH_INPUT_DATA); + console.debug(MERGING_DEFAULTS_WITH_INPUT_DATA, '\n'); return defaults ? [defaults] : []; }; +/** + * Warns if the `config` is provided in the manifest. + */ +const warnIfConfigProvided = (node: any) => { + if ('config' in node) { + const plugins = Object.keys(node.config || {}); + const joinedPlugins = plugins.join(', '); + const isMore = plugins.length > 1; + + logger.warn(CONFIG_WARN(joinedPlugins, isMore)); + } +}; + /** * 1. If the node has it's own pipeline, defaults or config then use that, * otherwise use whatever has been passed down from further up the tree. @@ -65,6 +87,9 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { const defaults = node.defaults || params.defaults; const noFlags = !params.observe && !params.regroup && !params.compute; + debugLogger.setExecutingPluginName(); + warnIfConfigProvided(node); + if (node.children) { return traverse(node.children, { ...params, @@ -94,20 +119,21 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { if ((noFlags || params.observe) && pipelineCopy.observe) { while (pipelineCopy.observe.length !== 0) { const pluginName = pipelineCopy.observe.shift() as string; + console.debug(OBSERVING(pluginName)); + debugLogger.setExecutingPluginName(pluginName); + const plugin = params.pluginStorage.get(pluginName); const nodeConfig = config && config[pluginName]; - if (isExecute(plugin)) { - inputStorage = await plugin.execute(inputStorage, nodeConfig); - node.inputs = inputStorage; - - if (params.context.explainer) { - addExplainData({ - pluginName, - metadata: plugin.metadata, - pluginData: params.context.initialize!.plugins[pluginName], - }); - } + inputStorage = await plugin.execute(inputStorage, nodeConfig); + node.inputs = inputStorage; + + if (params.context.explainer) { + addExplainData({ + pluginName, + metadata: plugin.metadata, + pluginData: params.context.initialize!.plugins[pluginName], + }); } } } @@ -116,10 +142,20 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { * If regroup is requested, execute regroup strategy, delete child's inputs, outputs and empty regroup array. */ if ((noFlags || params.regroup) && pipelineCopy.regroup) { - node.children = Regroup(inputStorage, pipelineCopy.regroup); + const originalOutputs = params.append ? node.outputs || [] : []; + + node.children = Regroup( + inputStorage, + originalOutputs, + pipelineCopy.regroup + ); + delete node.inputs; delete node.outputs; + debugLogger.setExecutingPluginName(); + console.debug(REGROUPING); + return traverse(node.children, { ...params, pipeline: { @@ -131,30 +167,42 @@ const computeNode = async (node: Node, params: ComputeParams): Promise => { }); } + console.debug('\n'); + /** * If iteration is on compute plugin, then executes compute plugins and sets the outputs value. */ if ((noFlags || params.compute) && pipelineCopy.compute) { + const originalOutputs = params.append ? node.outputs || [] : []; + while (pipelineCopy.compute.length !== 0) { const pluginName = pipelineCopy.compute.shift() as string; const plugin = params.pluginStorage.get(pluginName); const nodeConfig = config && config[pluginName]; - if (isExecute(plugin)) { - inputStorage = await plugin.execute(inputStorage, nodeConfig); - node.outputs = inputStorage; - - if (params.context.explainer) { - addExplainData({ - pluginName, - metadata: plugin.metadata, - pluginData: params.context.initialize!.plugins[pluginName], - }); - } - debugLogger.setExecutingPluginName(); + console.debug(COMPUTING_PIPELINE_FOR_NODE(pluginName)); + debugLogger.setExecutingPluginName(pluginName); + + inputStorage = await plugin.execute(inputStorage, nodeConfig); + debugLogger.setExecutingPluginName(); + + node.outputs = inputStorage; + + if (params.context.explainer) { + addExplainData({ + pluginName, + metadata: plugin.metadata, + pluginData: params.context.initialize!.plugins[pluginName], + }); } } + + if (params.append) { + node.outputs = originalOutputs.concat(node.outputs || []); + } } + + console.debug('\n'); }; /** diff --git a/src/if-run/lib/exhaust.ts b/src/if-run/lib/exhaust.ts index bed36f405..f38977f4b 100644 --- a/src/if-run/lib/exhaust.ts +++ b/src/if-run/lib/exhaust.ts @@ -20,7 +20,7 @@ export const exhaust = async ( context: Context, outputOptions: Options ) => { - console.debug(PREPARING_OUTPUT_DATA); + console.debug(PREPARING_OUTPUT_DATA(), '\n'); if (!outputOptions.noOutput && !outputOptions.outputPath) { ExportLog().execute(tree, context); diff --git a/src/if-run/lib/explain.ts b/src/if-run/lib/explain.ts index dce49b75a..470eed2e7 100644 --- a/src/if-run/lib/explain.ts +++ b/src/if-run/lib/explain.ts @@ -1,26 +1,33 @@ -import {ExplainParams} from '../types/explain'; +import {ERRORS} from '@grnsft/if-core/utils'; + +import {STRINGS} from '../../common/config'; + +import {ExplainParams, ExplainStorageType} from '../types/explain'; + +const {ManifestValidationError} = ERRORS; +const {AGGREGATION_UNITS_NOT_MATCH, AGGREGATION_METHODS_NOT_MATCH} = STRINGS; /** * Retrieves stored explain data. */ -export const explain = () => storeExplainData.plugins; +export const explain = () => storeExplainData.parameters; /** * Manages the storage of explain data. */ const storeExplainData = (() => { - let plugin = {}; + let parameter: ExplainStorageType = {}; - const pluginManager = { - get plugins() { - return plugin; + const parameterManager = { + get parameters() { + return parameter; }, - set plugins(value: object) { - plugin = value; + set parameters(value: ExplainStorageType) { + parameter = value; }, }; - return pluginManager; + return parameterManager; })(); /** @@ -28,17 +35,48 @@ const storeExplainData = (() => { */ export const addExplainData = (params: ExplainParams) => { const {pluginName, pluginData, metadata} = params; - const plugin = { - [pluginName]: { - method: pluginData!.method, - path: pluginData!.path, - inputs: metadata?.inputs || 'undefined', - outputs: metadata?.outputs || 'undefined', - }, - }; + const parameterMetadata = pluginData?.['parameter-metadata'] || metadata; + const parameters = storeExplainData.parameters; + const allParameters = { + ...parameterMetadata?.inputs, + ...parameterMetadata?.outputs, + } as ExplainStorageType; + + Object.entries(allParameters).forEach(([name, meta]) => { + const existingParameter = parameters[name]; + + if (parameters[name]?.plugins?.includes(pluginName)) { + return; + } + + if (existingParameter) { + if (meta.unit !== existingParameter.unit) { + throw new ManifestValidationError(AGGREGATION_UNITS_NOT_MATCH(name)); + } + + if ( + meta['aggregation-method'].component !== + existingParameter['aggregation-method'].component || + meta['aggregation-method'].time !== + existingParameter['aggregation-method'].time + ) { + throw new ManifestValidationError(AGGREGATION_METHODS_NOT_MATCH(name)); + } + + existingParameter.plugins.push(pluginName); + existingParameter.description = + meta.description || existingParameter.description; + } else { + parameters[name] = { + plugins: [pluginName], + unit: meta.unit, + description: meta.description, + 'aggregation-method': meta['aggregation-method'], + }; + } + }); - storeExplainData.plugins = { - ...storeExplainData.plugins, - ...plugin, + storeExplainData.parameters = { + ...parameters, }; }; diff --git a/src/if-run/lib/initialize.ts b/src/if-run/lib/initialize.ts index 181e88d15..554b4f9c0 100644 --- a/src/if-run/lib/initialize.ts +++ b/src/if-run/lib/initialize.ts @@ -1,6 +1,7 @@ import * as path from 'node:path'; import {ERRORS} from '@grnsft/if-core/utils'; +import {PluginInterface} from '@grnsft/if-core/types'; import {logger} from '../../common/util/logger'; import {memoizedLog} from '../util/log-memoize'; @@ -8,7 +9,6 @@ import {pluginStorage} from '../util/plugin-storage'; import {CONFIG, STRINGS} from '../config'; -import {PluginInterface} from '../types/interface'; import {Context, PluginOptions} from '../../common/types/manifest'; import {PluginStorageInterface} from '../types/plugin-storage'; import {storeAggregationMetrics} from './aggregate'; @@ -42,7 +42,7 @@ const importModuleFrom = async (path: string) => { }; /** - * Imports `module` from given `path`, then checks if it's `ModelPluginInterface` extension. + * Imports `module` from given `path` and returns requested `method`. */ const importAndVerifyModule = async (method: string, path: string) => { const pluginModule = await importModuleFrom(path); @@ -56,7 +56,7 @@ const importAndVerifyModule = async (method: string, path: string) => { * Imports module, then checks if it's a valid plugin. */ const handModule = (method: string, pluginPath: string) => { - console.debug(LOADING_PLUGIN_FROM_PATH(method, pluginPath)); + console.debug(LOADING_PLUGIN_FROM_PATH(method, pluginPath), '\n'); if (pluginPath === 'builtin') { pluginPath = path.normalize(`${__dirname}/../builtins`); @@ -75,7 +75,7 @@ const handModule = (method: string, pluginPath: string) => { }; /** - * Initializes plugin with global config. + * Initializes plugin with config. */ const initPlugin = async ( initPluginParams: PluginOptions @@ -83,12 +83,11 @@ const initPlugin = async ( const { method, path, - 'global-config': globalConfig, + mapping, + config, 'parameter-metadata': parameterMetadata, } = initPluginParams!; - console.debug(INITIALIZING_PLUGIN(method)); - if (!method) { throw new MissingPluginMethodError(MISSING_METHOD); } @@ -99,26 +98,32 @@ const initPlugin = async ( const plugin = await handModule(method, path); - return plugin(globalConfig, parameterMetadata); + return plugin(config, parameterMetadata, mapping); }; /** * Registers all plugins from `manifest`.`initialize` property. + * 1. Initalizes plugin storage. + * 2. Iterates over plugin names array. + * 3. While iteration, initalizes current plugin, gathers it's parameters (input/output). + * Then stores the aggregation metrics for each parameter to override stub values. */ export const initialize = async ( context: Context ): Promise => { - console.debug(INITIALIZING_PLUGINS); + console.debug(INITIALIZING_PLUGINS, '\n'); const {plugins} = context.initialize; const storage = pluginStorage(); for await (const pluginName of Object.keys(plugins)) { + console.debug(INITIALIZING_PLUGIN(pluginName)); + const plugin = await initPlugin(plugins[pluginName]); const parameters = {...plugin.metadata.inputs, ...plugin.metadata.outputs}; - Object.keys(parameters).forEach(key => { + Object.keys(parameters).forEach(current => { storeAggregationMetrics({ - [key]: parameters[key]['aggregation-method'], + [current]: parameters[current]['aggregation-method'], }); }); diff --git a/src/if-run/lib/regroup.ts b/src/if-run/lib/regroup.ts index affd51307..7e39b1c89 100644 --- a/src/if-run/lib/regroup.ts +++ b/src/if-run/lib/regroup.ts @@ -13,11 +13,20 @@ const {INVALID_GROUP_KEY, REGROUP_ERROR} = STRINGS; /** * Grouping strategy. */ -export const Regroup = (inputs: PluginParams[], groups: string[]) => { +export const Regroup = ( + inputs: PluginParams[], + outputs: PluginParams[], + groups: string[] +) => { /** * Creates structure to insert inputs by groups. */ - const appendGroup = (value: PluginParams, object: any, groups: string[]) => { + const appendGroup = ( + value: PluginParams, + object: any, + target: string, + groups: string[] + ) => { if (groups.length > 0) { const group = groups.shift() as string; @@ -26,16 +35,16 @@ export const Regroup = (inputs: PluginParams[], groups: string[]) => { if (groups.length === 0) { if ( - object.children[group].inputs && - object.children[group].inputs.length > 0 + object.children[group][target] && + object.children[group][target].length > 0 ) { - object.children[group].inputs.push(value); + object.children[group][target].push(value); } else { - object.children[group].inputs = [value]; + object.children[group][target] = [value]; } } - appendGroup(value, object.children[group], groups); + appendGroup(value, object.children[group], target, groups); } return object; @@ -60,21 +69,31 @@ export const Regroup = (inputs: PluginParams[], groups: string[]) => { * Interates over inputs, grabs group values for each one. * Based on grouping, initializes the structure. */ - return inputs.reduce((acc, input) => { - const validtedGroups = validateGroups(groups); - const groupsWithData = validtedGroups.map(groupType => { - if (!input[groupType]) { - throw new InvalidGroupingError(INVALID_GROUP_KEY(groupType)); - } - return input[groupType]; - }); + const validatedGroups = validateGroups(groups); + + const lookupGroupKey = (input: PluginParams, groupKey: string) => { + if (!input[groupKey]) { + throw new InvalidGroupingError(INVALID_GROUP_KEY(groupKey)); + } + + return input[groupKey]; + }; + + let acc = {} as any; + for (const input of inputs) { + const groupsWithData = validatedGroups.map(groupKey => + lookupGroupKey(input, groupKey) + ); + acc = appendGroup(input, acc, 'inputs', groupsWithData); + } - acc = { - ...acc, - ...appendGroup(input, acc, groupsWithData), - }; + for (const output of outputs) { + const groupsWithData = validatedGroups.map(groupKey => + lookupGroupKey(output, groupKey) + ); + acc = appendGroup(output, acc, 'outputs', groupsWithData); + } - return acc; - }, {} as any).children; + return acc.children; }; diff --git a/src/if-run/types/aggregation.ts b/src/if-run/types/aggregation.ts index c3b143a1f..811833c7b 100644 --- a/src/if-run/types/aggregation.ts +++ b/src/if-run/types/aggregation.ts @@ -1,7 +1,9 @@ export type AggregationResult = Record; -export const AGGREGATION_TYPES = ['horizontal', 'vertical', 'both'] as const; -export const AGGREGATION_METHODS = ['sum', 'avg', 'none'] as const; - -export type AggregationMethodTypes = 'sum' | 'avg' | 'none'; -export type AggregationMetric = Record; +export const AGGREGATION_TYPES = [ + 'horizontal', + 'time', + 'vertical', + 'component', + 'both', +] as const; diff --git a/src/if-run/types/compute.ts b/src/if-run/types/compute.ts index 6777bf553..22b80b9cc 100644 --- a/src/if-run/types/compute.ts +++ b/src/if-run/types/compute.ts @@ -3,6 +3,9 @@ import {PluginParams} from '@grnsft/if-core/types'; import {PluginStorageInterface} from './plugin-storage'; import {Context} from '../../common/types/manifest'; +/** + * @todo: remove NodeConfig after some period + */ export type NodeConfig = { [key: string]: Record; }; @@ -22,6 +25,7 @@ export type ComputeParams = { observe?: Boolean; regroup?: Boolean; compute?: Boolean; + append?: boolean; }; export type Node = { diff --git a/src/if-run/types/exhaust-plugin-interface.ts b/src/if-run/types/exhaust-plugin-interface.ts deleted file mode 100644 index de6183842..000000000 --- a/src/if-run/types/exhaust-plugin-interface.ts +++ /dev/null @@ -1,8 +0,0 @@ -import {Context} from '../../common/types/manifest'; - -export interface ExhaustPluginInterface { - /** - * Execute exhaust based on `context` and `tree`, produce output to a file in `outputPath`. - */ - execute(tree: any, context: Context, outputPath?: string): void; -} diff --git a/src/if-run/types/explain.ts b/src/if-run/types/explain.ts index c7f1a6d38..caedd3992 100644 --- a/src/if-run/types/explain.ts +++ b/src/if-run/types/explain.ts @@ -1,4 +1,4 @@ -import {ParameterMetadata} from '@grnsft/if-core/types'; +import {AggregationOptions, ParameterMetadata} from '@grnsft/if-core/types'; import {PluginOptions} from '../../common/types/manifest'; @@ -7,3 +7,13 @@ export type ExplainParams = { pluginData: PluginOptions; metadata: {inputs?: ParameterMetadata; outputs?: ParameterMetadata}; }; + +export type ExplainStorageType = Record< + string, + { + plugins: string[]; + unit: string; + description: string; + 'aggregation-method': AggregationOptions; + } +>; diff --git a/src/if-run/types/interface.ts b/src/if-run/types/interface.ts deleted file mode 100644 index 3884305a2..000000000 --- a/src/if-run/types/interface.ts +++ /dev/null @@ -1,6 +0,0 @@ -import {ExecutePlugin} from '@grnsft/if-core/types'; - -export type PluginInterface = ExecutePlugin; - -export const isExecute = (plugin: ExecutePlugin): plugin is ExecutePlugin => - (plugin as ExecutePlugin).metadata.kind === 'execute'; diff --git a/src/if-run/types/plugin-storage.ts b/src/if-run/types/plugin-storage.ts index 0d3d865c5..70a317483 100644 --- a/src/if-run/types/plugin-storage.ts +++ b/src/if-run/types/plugin-storage.ts @@ -1,5 +1,5 @@ +import {PluginInterface} from '@grnsft/if-core/types'; import {pluginStorage} from '../util/plugin-storage'; -import {PluginInterface} from './interface'; export type PluginStorage = { [key: string]: PluginInterface; diff --git a/src/if-run/types/process-args.ts b/src/if-run/types/process-args.ts index 298cadda6..60deb686b 100644 --- a/src/if-run/types/process-args.ts +++ b/src/if-run/types/process-args.ts @@ -6,6 +6,7 @@ export interface IfRunArgs { observe?: boolean; regroup?: boolean; compute?: boolean; + append?: boolean; } export interface ProcessArgsOutputs { @@ -19,6 +20,7 @@ export interface ProcessArgsOutputs { observe?: boolean; regroup?: boolean; compute?: boolean; + append?: boolean; } export interface Options { diff --git a/src/if-run/types/time-sync.ts b/src/if-run/types/time-sync.ts deleted file mode 100644 index 505a91f82..000000000 --- a/src/if-run/types/time-sync.ts +++ /dev/null @@ -1,20 +0,0 @@ -import {DateTime} from 'luxon'; - -export type TimeNormalizerConfig = { - 'start-time': Date | string; - 'end-time': Date | string; - interval: number; - 'allow-padding': boolean; -}; - -export type PaddingReceipt = { - start: boolean; - end: boolean; -}; - -export type TimeParams = { - startTime: DateTime; - endTime: DateTime; - interval: number; - allowPadding: boolean; -}; diff --git a/src/if-run/util/aggregation-helper.ts b/src/if-run/util/aggregation-helper.ts index e8fe63de5..defa71726 100644 --- a/src/if-run/util/aggregation-helper.ts +++ b/src/if-run/util/aggregation-helper.ts @@ -3,51 +3,57 @@ import {PluginParams} from '@grnsft/if-core/types'; import {CONFIG, STRINGS} from '../config'; -import {AggregationMetric, AggregationResult} from '../types/aggregation'; +import {AggregationResult} from '../types/aggregation'; -import {getAggregationMethod} from '../lib/aggregate'; +import {getAggregationInfoFor} from '../lib/aggregate'; const {MissingAggregationParamError} = ERRORS; const {METRIC_MISSING} = STRINGS; -const {AGGREGATION_ADDITIONAL_PARAMS} = CONFIG; +const {AGGREGATION_TIME_METRICS} = CONFIG; /** - * Aggregates child node level metrics. Validates if metric aggregation type is `none`, then rejects with error. - * Appends aggregation additional params to metrics. Otherwise iterates over inputs by aggregating per given `metrics`. + * Aggregates child node level metrics. Appends aggregation additional params to metrics. + * Otherwise iterates over outputs by aggregating per given `metrics`. */ -export const aggregateInputsIntoOne = ( - inputs: PluginParams[], - metrics: AggregationMetric[], +export const aggregateOutputsIntoOne = ( + outputs: PluginParams[], + metrics: string[], isTemporal?: boolean ) => { - const metricsKeys: string[] = metrics.map(metric => Object.keys(metric)[0]); - const extendedMetrics = [...metricsKeys, ...AGGREGATION_ADDITIONAL_PARAMS]; + const metricsWithTime = metrics.concat(AGGREGATION_TIME_METRICS); - return inputs.reduce((acc, input, index) => { - for (const metric of extendedMetrics) { - if (!(metric in input)) { + return outputs.reduce((acc, output, index) => { + for (const metric of metricsWithTime) { + if (!(metric in output)) { throw new MissingAggregationParamError(METRIC_MISSING(metric, index)); } /** Checks if metric is timestamp or duration, then adds to aggregated value. */ - if (AGGREGATION_ADDITIONAL_PARAMS.includes(metric)) { + if (AGGREGATION_TIME_METRICS.includes(metric)) { if (isTemporal) { - acc[metric] = input[metric]; + acc[metric] = output[metric]; } } else { - const method = getAggregationMethod(metric); + const aggregationParams = getAggregationInfoFor(metric); + /** Checks either its a temporal aggregation (vertical), then chooses `component`, otherwise `time`. */ + const aggregationType = isTemporal ? 'component' : 'time'; - if (!method) { + if (aggregationParams[aggregationType] === 'none') { + return acc; + } + + if (aggregationParams[aggregationType] === 'copy') { + acc[metric] = output[metric]; return acc; } acc[metric] = acc[metric] ?? 0; - acc[metric] += parseFloat(input[metric]); + acc[metric] += parseFloat(output[metric]); /** Checks for the last iteration. */ - if (index === inputs.length - 1) { - if (method === 'avg') { - acc[metric] /= inputs.length; + if (index === outputs.length - 1) { + if (aggregationParams[aggregationType] === 'avg') { + acc[metric] /= outputs.length; } } } diff --git a/src/if-run/util/args.ts b/src/if-run/util/args.ts index 538d37a36..b0cf90b8f 100644 --- a/src/if-run/util/args.ts +++ b/src/if-run/util/args.ts @@ -48,6 +48,7 @@ export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { observe, regroup, compute, + append, } = validateAndParseProcessArgs(); if (!output && noOutput) { @@ -66,6 +67,7 @@ export const parseIfRunProcessArgs = (): ProcessArgsOutputs => { observe, regroup, compute, + ...(append && {append}), }; } diff --git a/src/if-run/util/helpers.ts b/src/if-run/util/helpers.ts index 2939022e2..4a578955b 100644 --- a/src/if-run/util/helpers.ts +++ b/src/if-run/util/helpers.ts @@ -1,9 +1,6 @@ import {ERRORS} from '@grnsft/if-core/utils'; import {logger} from '../../common/util/logger'; -import {GlobalPlugins} from '../../common/types/manifest'; -import {PluginStorageInterface} from '../types/plugin-storage'; -import {storeAggregationMetrics} from '../lib/aggregate'; import {STRINGS} from '../config'; @@ -42,32 +39,3 @@ export const mergeObjects = (defaults: any, input: any) => { return merged; }; - -/** - * Stores `'aggregation-method'` of the plugins in the pipeline. - */ -export const storeAggregationMethods = ( - plugins: GlobalPlugins, - pluginStorage: PluginStorageInterface -) => { - Object.keys(plugins).forEach(pluginName => { - const plugin = pluginStorage.get(pluginName); - - if ('inputs' in plugin.metadata || 'outputs' in plugin.metadata) { - const pluginParameters = - {...plugin.metadata.inputs, ...plugin.metadata.outputs} || {}; - - Object.entries(pluginParameters).forEach( - ([parameterName, parameterMetadata]) => { - const {'aggregation-method': aggregationMethod} = parameterMetadata; - - if (aggregationMethod) { - const metrics = {[parameterName]: aggregationMethod}; - - storeAggregationMetrics(metrics); - } - } - ); - } - }); -}; diff --git a/src/if-run/util/plugin-storage.ts b/src/if-run/util/plugin-storage.ts index ca2907fcc..06caac434 100644 --- a/src/if-run/util/plugin-storage.ts +++ b/src/if-run/util/plugin-storage.ts @@ -1,9 +1,9 @@ import {ERRORS} from '@grnsft/if-core/utils'; +import {PluginInterface} from '@grnsft/if-core/types'; import {STRINGS} from '../config'; import {PluginStorage} from '../types/plugin-storage'; -import {PluginInterface} from '../types/interface'; const {PluginInitializationError} = ERRORS; const {NOT_INITALIZED_PLUGIN} = STRINGS;