diff --git a/README.md b/README.md index dd9a60eb..7c2fbb82 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ See more dashboards on the [workshop page](https://catalog.workshops.aws/awscid/ pip3 install --upgrade cid-cmd ``` -#### Dasbhoard Deployment +#### Dashboard Deployment ```bash cid-cmd deploy @@ -71,6 +71,7 @@ Update dashboard and all dependencies (Datasets and Athena View). WARNING: this ```bash cid-cmd update --force --recursive ``` + #### Show Dashboard Status Show dashboards status @@ -80,8 +81,6 @@ cid-cmd status [status](https://www.youtube.com/watch?v=ivr1MoGaApM) - - #### Share QuickSight resources ```bash cid-cmd share @@ -95,18 +94,43 @@ cid-cmd init-qs ``` #### Initialize CUR -One time action to initialize Athena table and Crawler from s3 with CUR data. +One time action to initialize Athena table and Crawler from s3 with CUR data. Currently only CUR1 supported. ```bash cid-cmd init-cur ``` +#### Create CUR proxy +There are 2 CUR formats that `cid-cmd` supports. CUR1 and CUR2 have slightly different fields structure. Each dashboard can be developed using CUR2 or CUR1, but on installation `cid-cmd` will deploy a special Athena View "CUR proxy". This View will play a role of compatibility layer between the CUR version used by Dashboard and available CUR. This View can be extended as new dashboard request CUR fields. + +Please note that Cost Allocation Tags and Cost Categories are not present in CUR Proxy by default. You will need to add these fields by modifying CUR Proxy in Athena or using `cid-cmd` tool: + +For CUR2 proxy (using CUR1 data as source) +```bash +cid-cmd create-cur-proxy -vv \ + --cur-version '2' \ + --cur-table-name 'mycur1' \ + --athena-workgroup 'primary' \ + --fields "resource_tags['user_cost_center'],resource_tags['user_owner']" +``` + +For CUR1 proxy (using CUR2 data as source) +```bash +cid-cmd create-cur-proxy -vv \ + --cur-version '1' \ + --cur-table-name 'mycur2' \ + --athena-workgroup 'primary' \ + --fields "resource_tags_user_cost_center,resource_tags_user_owner" +``` + + #### Delete Dashboard and all dependencies unused by other Delete Dashboards and all dependencies unused by other CID-managed dashboards.(including QuickSight datasets, Athena views and tables) ```bash cid-cmd delete ``` + #### Delete Command Options: ``` --dashboard-id TEXT QuickSight dashboard id diff --git a/cfn-templates/cid-admin-policies.yaml b/cfn-templates/cid-admin-policies.yaml index 9f905290..f32661d4 100644 --- a/cfn-templates/cid-admin-policies.yaml +++ b/cfn-templates/cid-admin-policies.yaml @@ -11,6 +11,7 @@ Metadata: default: Permissions for the Data Collection Account (where the Dashboards will be deployed) Parameters: - CloudIntelligenceDashboardsCFNManagement + - CloudIntelligenceDashboardsManagement - CURDestination - QuickSightManagement - QuickSightAdmin @@ -41,6 +42,11 @@ Parameters: Description: "Choose Yes only when deploying this stack in the account where your Dashboards will be deployed. This grants the specified role above rights to deploy the Cloud Intelligence Dashboards via CloudFormation" Default: 'no' AllowedValues: ["yes", "no"] + CloudIntelligenceDashboardsManagement: + Type: String + Description: "Choose Yes only when deploying this stack in the account where your Dashboards will be deployed. This grants the specified role above rights to deploy Athena, Glue, QuickSight resources via CloudFormation" + Default: 'no' + AllowedValues: ["yes", "no"] CURDestination: Type: String Description: "Choose Yes only when deploying this stack in the account where your Dashboards will be deployed. This grants the specified role above rights to create an S3 bucket for replicated CUR data Data Collection Account" @@ -55,6 +61,7 @@ Parameters: Conditions: CreateQuickSightManagementPolicy: !Equals [ !Ref QuickSightManagement, "yes" ] CreateCloudIntelligenceDashboardsCFNManagementPolicy: !Equals [ !Ref CloudIntelligenceDashboardsCFNManagement, "yes" ] + CreateCloudIntelligenceDashboardsManagementPolicy: !Equals [ !Ref CloudIntelligenceDashboardsManagement, "yes" ] CreateQuickSightAdminPolicy: !Equals [ !Ref QuickSightAdmin, "yes" ] CreateCURDestinationPolicy: !Equals [ !Ref CURDestination, "yes" ] CreateCURReplicationPolicy: !Equals [ !Ref CURReplication, "yes" ] @@ -108,18 +115,27 @@ Resources: - ds:UnauthorizeApplication Resource: '*' # as per doc https://docs.aws.amazon.com/quicksight/latest/user/iam-policy-examples.html#security_iam_id-based-policy-examples-all-access-enterprise-edition - - Sid: IAM + - Sid: IAMRole Action: - iam:AttachRolePolicy - - iam:CreatePolicy - - iam:CreatePolicyVersion + - iam:DetachRolePolicy - iam:CreateRole - iam:PassRole + Effect: Allow + Resource: + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/service-role/aws-quicksight-service-role-v0 + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/service-role/CidQuickSightDataSourceRole + + - Sid: IAMPolicy + Action: + - iam:CreatePolicy + - iam:DeletePolicy + - iam:CreatePolicyVersion - iam:DeletePolicyVersion - - iam:DetachRolePolicy Effect: Allow Resource: - - !Sub arn:aws:iam::${AWS::AccountId}:role/service-role/aws-quicksight-service-role-v0 + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/cid* + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/CID* - Sid: QuickSight Action: @@ -147,10 +163,14 @@ Resources: - s3:ListMultipartUploadParts Effect: Allow Resource: - - !Sub arn:aws:s3:::cid-${AWS::AccountId}-shared - - !Sub arn:aws:s3:::aws-athena-query-results-cid-${AWS::AccountId}-${AWS::Region} - - !Sub arn:aws:s3:::athena-query-result-${AWS::AccountId} - + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-shared + - !Sub arn:${AWS::Partition}:s3:::aws-athena-query-results-cid-${AWS::AccountId}-${AWS::Region} + - !Sub arn:${AWS::Partition}:s3:::aws-athena-query-results-cidcmd-${AWS::AccountId}-${AWS::Region} + - !Sub arn:${AWS::Partition}:s3:::athena-query-result-${AWS::AccountId}-${AWS::Region} + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-shared/* + - !Sub arn:${AWS::Partition}:s3:::aws-athena-query-results-cid-${AWS::AccountId}-${AWS::Region}/* + - !Sub arn:${AWS::Partition}:s3:::aws-athena-query-results-cidcmd-${AWS::AccountId}-${AWS::Region}/* + - !Sub arn:${AWS::Partition}:s3:::athena-query-result-${AWS::AccountId}-${AWS::Region}/* - Sid: ReadOnly Action: - ds:CheckAlias @@ -190,16 +210,6 @@ Resources: PolicyDocument: Version: '2012-10-17' Statement: - - Sid: Athena - Action: - - athena:CreateWorkGroup - - athena:UpdateWorkGroup - - athena:DeleteWorkGroup - - athena:GetWorkGroup - Effect: Allow - Resource: - - !Sub arn:aws:athena:${AWS::Region}:${AWS::AccountId}:workgroup/CID - - Sid: CloudFormation Action: - cloudformation:CreateStack @@ -214,7 +224,133 @@ Resources: - cloudformation:UpdateStack Effect: Allow Resource: - - !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/Cloud-Intelligence-Dashboards/* + - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/Cloud-Intelligence-Dashboards/* + + - Sid: CloudFormationListExports + Action: + - cloudformation:ListExports + Effect: Allow + Resource: + - "*" # Cannot restrict this + + - Sid: IAMForCFN + Action: + - iam:AttachRolePolicy + - iam:CreateRole + - iam:DeleteRole + - iam:DeleteRolePolicy + - iam:DetachRolePolicy + - iam:GetRole + - iam:GetRolePolicy + - iam:PassRole + - iam:PutRolePolicy + Effect: Allow + Resource: + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/CidSpiceRefreshExecutionRole + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-* #Roles created by CFN stack. Name is hardcoded here + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/CidQuickSightDataSourceRole + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/CidExecRole + + - Sid: IAMPolicy + Action: + - iam:CreatePolicy + - iam:DeletePolicy + - iam:CreatePolicyVersion + - iam:DeletePolicyVersion + Effect: Allow + Resource: + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/cid* + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/CID* + + - Sid: LambdaForCFN + Action: + - lambda:AddPermission + - lambda:CreateFunction + - lambda:DeleteFunction + - lambda:GetFunction + - lambda:InvokeFunction + - lambda:RemovePermission + - lambda:UpdateFunctionConfiguration + - lambda:UpdateFunctionCode + - lambda:PublishLayerVersion + - lambda:GetLayerVersion + - lambda:DeleteLayerVersion + Effect: Allow + Resource: + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:Cid* + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics-DataExports + # - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:CidProcessPath-DoNotRun + # - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceProcessPath-DoNotRun + # - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceDashboard + # - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceFunctionInit-DoNotRun + # - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:CidInitialSetup-DoNotRun #legacy + # - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:CidSpiceRefreshLambda + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:layer:CidLambdaLayer* + + - Sid: ReadLambdaLayerS3 + Action: + - s3:GetObject + Effect: Allow + Resource: + - !Sub arn:${AWS::Partition}:s3:::aws-managed-cost-intelligence-dashboards-${AWS::Region}/cid-resource-lambda-layer/* + + - Sid: ReadOnlyCFN + Action: + - cloudformation:GetTemplate + - cloudformation:GetTemplateSummary + - cloudformation:ListStacks + - cloudformation:ValidateTemplate + Effect: Allow + Resource: '*' # Read only Rights needed for CFN operator + + Metadata: + cfn_nag: + rules_to_suppress: + - id: W13 + reason: "Some resources are not possible to restrict. See comments." + - id: W28 + reason: "No Replacement needed" + + CloudIntelligenceDashboardsManagementPolicy: + Type: AWS::IAM::ManagedPolicy + Condition: CreateCloudIntelligenceDashboardsManagementPolicy + Properties: + ManagedPolicyName: CidCloudIntelligenceDashboardsManagementPolicy + Description: 'CloudIntelligenceDashboards Policy for Athena, Glue, Quicksight deployment' + Roles: + - !Ref RoleName + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: Athena + Action: + - athena:CreateWorkGroup + - athena:UpdateWorkGroup + - athena:DeleteWorkGroup + - athena:GetWorkGroup + Effect: Allow + Resource: + - !Sub arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:workgroup/CID + + - Sid: BCMDataExports + Action: + - bcm-data-exports:CreateExport + - bcm-data-exports:DeleteExport + - bcm-data-exports:GetExport + - bcm-data-exports:ListExports + - bcm-data-exports:ListTagsForResource + - bcm-data-exports:UpdateExport + Effect: Allow + Resource: + - !Sub arn:${AWS::Partition}:bcm-data-exports:${AWS::Region}:${AWS::AccountId}:export/* + - !Sub arn:${AWS::Partition}:bcm-data-exports:${AWS::Region}:${AWS::AccountId}:table/* + + - Sid: CurDataAccess + Action: + - cur:PutReportDefinition + Effect: Allow + Resource: + - !Sub arn:${AWS::Partition}:cur:${AWS::Region}:${AWS::AccountId}:* #Required as per https://docs.aws.amazon.com/cur/latest/userguide/bcm-data-exports-access.html#bcm-data-exports-access-examples - Sid: EventBridge Action: @@ -226,7 +362,7 @@ Resources: - events:RemoveTargets Effect: Allow Resource: - - !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/Cloud-Intelligence-Dashboards-SpiceRefreshRule-* + - !Sub arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/Cloud-Intelligence-Dashboards-SpiceRefreshRule-* - Sid: GlueCrawler Action: @@ -236,7 +372,8 @@ Resources: - glue:StopCrawler Effect: Allow Resource: - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:crawler/CidCrawler + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/CidCrawler + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:crawler/cid-DataExport* - Sid: GlueDatabase Action: @@ -247,11 +384,14 @@ Resources: - glue:GetDatabases Effect: Allow Resource: - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:database/cid_cur - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:database/cid_cur/* - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:table/cid_cur/* - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:catalog - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:userDefinedFunction/cid_cur/* + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/cid_cur + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/cid_cur/* + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/cid_cur/* + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:userDefinedFunction/cid_cur/* + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/cid_data_export + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/cid_data_export/* + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:userDefinedFunction/cid_data_export/* - Sid: GlueTable Action: @@ -260,51 +400,11 @@ Resources: - glue:DeleteTable Effect: Allow Resource: - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:database/cid_cur - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:table/cid_cur/* - - !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:catalog - - - Sid: IAMForCFN - Action: - - iam:AttachRolePolicy - - iam:CreateRole - - iam:DeleteRole - - iam:DeleteRolePolicy - - iam:DetachRolePolicy - - iam:GetRole - - iam:GetRolePolicy - - iam:PassRole - - iam:PutRolePolicy - Effect: Allow - Resource: - - !Sub arn:aws:iam::${AWS::AccountId}:role/CidSpiceRefreshExecutionRole - - !Sub arn:aws:iam::${AWS::AccountId}:role/Cloud-Intelligence-*-* #Roles created by CFN stack. Name is hardcoded here - - !Sub arn:aws:iam::${AWS::AccountId}:role/CidQuickSightDataSourceRole - - !Sub arn:aws:iam::${AWS::AccountId}:role/CidExecRole - - - Sid: LambdaForCFN - Action: - - lambda:AddPermission - - lambda:CreateFunction - - lambda:DeleteFunction - - lambda:GetFunction - - lambda:InvokeFunction - - lambda:RemovePermission - - lambda:UpdateFunctionConfiguration - - lambda:UpdateFunctionCode - - lambda:PublishLayerVersion - - lambda:GetLayerVersion - - lambda:DeleteLayerVersion - Effect: Allow - Resource: - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:Cid* - # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidProcessPath-DoNotRun - # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceProcessPath-DoNotRun - # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceDashboard - # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidCustomResourceFunctionInit-DoNotRun - # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidInitialSetup-DoNotRun #legacy - # - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:CidSpiceRefreshLambda - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:layer:CidLambdaLayer* + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/cid_cur + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/cid_cur/* + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/cid_data_export + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/cid_data_export/* - Sid: QuickSightDashboard Action: @@ -314,12 +414,12 @@ Resources: - quicksight:DescribeDashboard Effect: Allow Resource: - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cudos* - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cost_intelligence_dashboard - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/kpi_dashboard - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/ta-organizational-view - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/trends-dashboard - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/compute-optimizer-dashboard + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cudos* + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/cost_intelligence_dashboard + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/kpi_dashboard + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/ta-organizational-view + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/trends-dashboard + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:dashboard/compute-optimizer-dashboard - Sid: QuickSightCreateDataSource Action: @@ -333,7 +433,7 @@ Resources: - quicksight:UpdateDataSourcePermissions Effect: Allow Resource: - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:datasource/CID-* + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:datasource/CID-* - Sid: QuickSightDataSet Action: @@ -345,7 +445,7 @@ Resources: - quicksight:UpdateDataSetPermissions Effect: Allow Resource: - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dataset/* # DataSetIDs are dynamic + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:dataset/* # DataSetIDs are dynamic - Sid: QuickSightDataSetSchedule Action: @@ -360,7 +460,7 @@ Resources: - quicksight:DeleteDataSetRefreshProperties Effect: Allow Resource: - - !Sub arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:dataset/*/refresh-schedule/* # DataSetIDs are dynamic as well as schedule ids + - !Sub arn:${AWS::Partition}:quicksight:${AWS::Region}:${AWS::AccountId}:dataset/*/refresh-schedule/* # DataSetIDs are dynamic as well as schedule ids - Sid: CreateQueryResultsBucketS3 Action: @@ -379,24 +479,10 @@ Resources: - s3:PutObject Effect: Allow Resource: - - !Sub arn:aws:s3:::aws-athena-query-results-cid-${AWS::AccountId}-${AWS::Region} - - !Sub arn:aws:s3:::aws-athena-query-results-cid-${AWS::AccountId}-${AWS::Region}/* - - - Sid: ReadLambdaLayerS3 - Action: - - s3:GetObject - Effect: Allow - Resource: - - !Sub arn:aws:s3:::aws-managed-cost-intelligence-dashboards-${AWS::Region}/cid-resource-lambda-layer/* - - - Sid: ReadOnlyCFN - Action: - - cloudformation:GetTemplate - - cloudformation:GetTemplateSummary - - cloudformation:ListStacks - - cloudformation:ValidateTemplate - Effect: Allow - Resource: '*' # Read only Rights needed for CFN operator + - !Sub arn:${AWS::Partition}:s3:::aws-athena-query-results-cid-${AWS::AccountId}-${AWS::Region} + - !Sub arn:${AWS::Partition}:s3:::aws-athena-query-results-cid-${AWS::AccountId}-${AWS::Region}/* + - !Sub arn:${AWS::Partition}:s3:::aws-athena-query-results-cidcmd-${AWS::AccountId}-${AWS::Region} + - !Sub arn:${AWS::Partition}:s3:::aws-athena-query-results-cidcmd-${AWS::AccountId}-${AWS::Region}/* Metadata: cfn_nag: @@ -406,7 +492,6 @@ Resources: - id: W28 reason: "No Replacement needed" - CURDestinationPolicy: Type: AWS::IAM::ManagedPolicy Condition: CreateCURDestinationPolicy @@ -431,7 +516,7 @@ Resources: - cloudformation:UpdateStack Effect: Allow Resource: - - !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/CID-CUR-Destination/* + - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/CID-CUR-Destination/* - Sid: IAMForCFN Action: @@ -441,11 +526,16 @@ Resources: - iam:GetRole - iam:GetRolePolicy - iam:PutRolePolicy + - iam:GetPolicy + - iam:DeletePolicy + - iam:ListPolicyVersions + - iam:DeletePolicyVersion - iam:PassRole Effect: Allow Resource: - - !Sub arn:aws:iam::${AWS::AccountId}:role/cid/CID-CUR-Destination-CIDLambdaAnalyticsRole-* - - !Sub arn:aws:iam::${AWS::AccountId}:role/cid/CID-CUR-Destination-CIDLambdaCURCreatorRole-* + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cid/CID-CUR-Destination-* + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/CID-CUR-Destination-* + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/cidDataExportsReadAccess # hardcoded prefix - Sid: Lambda Action: @@ -457,8 +547,8 @@ Resources: - lambda:UpdateFunctionCode Effect: Allow Resource: - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-CURCreator + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-CURCreator - Sid: S3 Action: @@ -479,8 +569,10 @@ Resources: - s3:PutObject Effect: Allow Resource: - - !Sub arn:aws:s3:::cid-${AWS::AccountId}-shared - - !Sub arn:aws:s3:::cid-${AWS::AccountId}-shared/* + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-shared + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-shared/* + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-data-exports + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-data-exports/* - Sid: CostUsageReport Action: @@ -489,14 +581,14 @@ Resources: - cur:PutReportDefinition Effect: Allow Resource: - - !Sub arn:aws:cur:*:${AWS::AccountId}:definition/cid + - !Sub arn:${AWS::Partition}:cur:*:${AWS::AccountId}:definition/cid - Sid: CostUsageReportForDelete Action: - cur:DescribeReportDefinitions Effect: Allow Resource: - - !Sub arn:aws:cur:*:${AWS::AccountId}:definition/ + - !Sub arn:${AWS::Partition}:cur:*:${AWS::AccountId}:definition/ - Sid: ReadOnly Action: @@ -507,6 +599,7 @@ Resources: Effect: Allow Resource: '*' # ReadOnly Policy for CFN + Metadata: cfn_nag: rules_to_suppress: @@ -540,7 +633,7 @@ Resources: - cloudformation:UpdateStack Effect: Allow Resource: - - !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/CID-CUR-Replication/* + - !Sub arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/CID-CUR-Replication/* - Sid: CostUsageReport Action: @@ -549,14 +642,14 @@ Resources: - cur:PutReportDefinition Effect: Allow Resource: - - !Sub arn:aws:cur:*:${AWS::AccountId}:definition/cid + - !Sub arn:${AWS::Partition}:cur:*:${AWS::AccountId}:definition/cid - Sid: CostUsageReportForDelete Action: - cur:DescribeReportDefinitions Effect: Allow Resource: - - !Sub arn:aws:cur:*:${AWS::AccountId}:definition/ + - !Sub arn:${AWS::Partition}:cur:*:${AWS::AccountId}:definition/ - Sid: IAMForCFN Action: @@ -570,7 +663,7 @@ Resources: - iam:PutRolePolicy Effect: Allow Resource: - - !Sub arn:aws:iam::${AWS::AccountId}:role/cid/CID-CUR-Replication-* + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cid/CID-CUR-Replication-* - Sid: LambdaForCFN Action: @@ -584,8 +677,8 @@ Resources: - lambda:UpdateFunctionCode Effect: Allow Resource: - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics - - !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-CURCreator + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-Analytics + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:cid-CID-CURCreator - Sid: S3 Action: @@ -610,8 +703,10 @@ Resources: - s3:PutReplicationConfiguration Effect: Allow Resource: - - !Sub arn:aws:s3:::cid-${AWS::AccountId}-local - - !Sub arn:aws:s3:::cid-${AWS::AccountId}-local/* + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-local + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-local/* + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-data-local + - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-data-local/* - Sid: ReadOnly Action: diff --git a/cfn-templates/cid-cfn.yml b/cfn-templates/cid-cfn.yml index 579034f0..a1972e23 100644 --- a/cfn-templates/cid-cfn.yml +++ b/cfn-templates/cid-cfn.yml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: '2010-09-09' -Description: Deployment of Cloud Intelligence Dashboards v0.3.15 +Description: Deployment of Cloud Intelligence Dashboards v4.0.0 Metadata: AWS::CloudFormation::Interface: ParameterGroups: @@ -13,7 +13,7 @@ Metadata: - Label: default: CUDOS, Cost-Intelligence-Dashboard and KPI-Dashboard. Require deployment of CUR via CloudFormation (cur-aggregation.yaml) or manually (Dashboard data will appear within 24h after CUR creation). Parameters: - - CURBucketPath + - CURVersion - DeployCUDOSv5 - DeployCostIntelligenceDashboard - DeployKPIDashboard @@ -31,7 +31,6 @@ Metadata: - AthenaWorkgroup - AthenaQueryResultsBucket - DatabaseName - - CURTableName - GlueDataCatalog - Suffix - QuickSightDataSourceRoleName @@ -40,6 +39,12 @@ Metadata: - DeployCUDOSDashboard - DataBucketsKmsKeysArns - ShareDashboard + - Label: + default: 'Legacy' + Parameters: + - KeepLegacyCURTable + - CURBucketPath + - CURTableName ParameterLabels: PrerequisitesQuickSight: default: "I have enabled QuickSight Enterprise Edition AND I have a SPICE capacity in the current region." @@ -50,7 +55,7 @@ Metadata: QuickSightUser: default: "User name of QuickSight user (as displayed in QuickSight admin panel). Dashboards created by this template will be owned by this user." CURBucketPath: - default: "Path to Cost and Usage Report" + default: "Path to Legacy Cost and Usage Report (only if Legacy CUR used)" DeployCUDOSDashboard: default: "CUDOS Dashboard v4 - Deprecated [set 'no' to delete]" DeployCUDOSv5: @@ -91,6 +96,8 @@ Metadata: default: "Primary Tag for Compute Optimizer dashboard" SecondaryTagName: default: "Secondary Tag for Compute Optimizer dashboard" + KeepLegacyCURTable: + default: "Keep Legacy CUR Table" cfn-lint: config: ignore_checks: @@ -126,6 +133,11 @@ Parameters: Default: 'yes' Description: "Make Dashboards visible by all users who access this QuickSight account." AllowedValues: ["yes", "no"] + CURVersion: + Type: String + Default: '2.0' + AllowedValues: ["1.0", "2.0"] + Description: "We support 2 options: CUR 2.0 from DataExports stack managed by CID or CUR 1(Legacy)." CURBucketPath: Type: String MinLength: 3 @@ -224,6 +236,11 @@ Parameters: Type: String Default: '/' Description: Path for roles where PermissionBoundaries can limit location + KeepLegacyCURTable: + Type: String + Description: Choose 'yes' if you want to keep the Legacy CUR table + Default: "no" + AllowedValues: ["yes", "no"] Conditions: NeedCUDOSDashboard: !Equals [ !Ref DeployCUDOSDashboard, "yes" ] @@ -231,29 +248,23 @@ Conditions: NeedCostIntelligenceDashboard: !Equals [ !Ref DeployCostIntelligenceDashboard, "yes" ] NeedKPIDashboard: !Equals [ !Ref DeployKPIDashboard, "yes" ] NeedTAODashboard: !Equals [ !Ref DeployTAODashboard, "yes" ] + NeedLegacyCUR: !Equals [!Ref KeepLegacyCURTable, "yes"] NeedComputeOptimizerDashboard: !Equals [ !Ref DeployComputeOptimizerDashboard, "yes" ] - NeedCUR: - Fn::Or: - - !Equals [ !Ref DeployCUDOSDashboard, "yes" ] - - !Equals [ !Ref DeployCUDOSv5, "yes" ] - - !Equals [ !Ref DeployCostIntelligenceDashboard, "yes" ] - - !Equals [ !Ref DeployKPIDashboard, "yes" ] - NeedDataCollectionLab: - Fn::Or: - - !Equals [ !Ref DeployTAODashboard, "yes" ] - - !Equals [ !Ref DeployComputeOptimizerDashboard, "yes" ] - NeedAthenaWorkgroup: !Equals [ !Ref AthenaWorkgroup, "" ] - NeedAthenaQueryResultsBucket: !Equals [ !Ref AthenaQueryResultsBucket, "" ] - NeedDatabase: + UseCUR2: Fn::And: - - !Equals [ !Ref DatabaseName, "" ] + - !Equals [!Ref CURVersion, '2.0'] - Fn::Or: - - !Condition NeedDataCollectionLab - - !Condition NeedCUR + - !Equals [ !Ref DeployCUDOSDashboard, "yes" ] + - !Equals [ !Ref DeployCUDOSv5, "yes" ] + - !Equals [ !Ref DeployCostIntelligenceDashboard, "yes" ] + - !Equals [ !Ref DeployKPIDashboard, "yes" ] + NeedAthenaWorkgroup: !Equals [ !Ref AthenaWorkgroup, "" ] + NeedAthenaQueryResultsBucket: !Equals [ !Ref AthenaQueryResultsBucket, "" ] + NeedDatabase: !Equals [ !Ref DatabaseName, "" ] NeedCURTable: Fn::And: - !Equals [ !Ref CURTableName, "" ] - - !Condition NeedCUR + - !Condition NeedLegacyCUR NeedRefreshDatasets: !Not [ !Equals [ !Ref QuickSightDataSetRefreshSchedule, ""] ] NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ] NeedDataBucketsKmsAndNeedCURTable: @@ -274,10 +285,10 @@ Conditions: - !Condition NeedCURTable UseQuickSightDataSourceRole: !Not [!Equals [ !Ref QuickSightDataSourceRoleName, "" ]] NeedQuickSightDataSourceRole: !Equals [ !Ref QuickSightDataSourceRoleName, "CidQuickSightDataSourceRole" ] - NeedQuickSightDataSourceRoleAndCUR: + NeedQuickSightDataSourceRoleAndLegacyCUR: Fn::And: - !Condition NeedQuickSightDataSourceRole - - !Condition NeedCUR + - !Condition NeedLegacyCUR NeedQuickSightDataSourceKMS: Fn::And: - !Condition NeedQuickSightDataSourceRole @@ -289,6 +300,31 @@ Conditions: - !Equals [!Ref "AWS::Region", "cn-northwest-1"] - !Equals [!Ref "AWS::Region", "us-gov-east-1"] - !Equals [!Ref "AWS::Region", "us-gov-west-1"] + LambdaLayerBucketPrefixIsManaged: !Equals [!Ref LambdaLayerBucketPrefix, 'aws-managed-cost-intelligence-dashboards'] + +Mappings: + RegionMap: # CID has AWS managed buckets for deploy. Region must support QuickSight ( curl https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonQuickSight/current/region_index.json -s | jq '.regions | keys' ) + ap-northeast-1: {BucketName: aws-managed-cost-intelligence-dashboards-ap-northeast-1} + ap-northeast-2: {BucketName: aws-managed-cost-intelligence-dashboards-ap-northeast-2} + ap-south-1: {BucketName: aws-managed-cost-intelligence-dashboards-ap-south-1} + ap-southeast-1: {BucketName: aws-managed-cost-intelligence-dashboards-ap-southeast-1} + ap-southeast-2: {BucketName: aws-managed-cost-intelligence-dashboards-ap-southeast-2} + ca-central-1: {BucketName: aws-managed-cost-intelligence-dashboards-ca-central-1} + eu-central-1: {BucketName: aws-managed-cost-intelligence-dashboards-eu-central-1} + eu-north-1: {BucketName: aws-managed-cost-intelligence-dashboards-eu-north-1} + eu-west-1: {BucketName: aws-managed-cost-intelligence-dashboards-eu-west-1} + eu-west-2: {BucketName: aws-managed-cost-intelligence-dashboards-eu-west-2} + eu-west-3: {BucketName: aws-managed-cost-intelligence-dashboards-eu-west-3} + sa-east-1: {BucketName: aws-managed-cost-intelligence-dashboards-sa-east-1} + us-east-1: {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1} + us-east-2: {BucketName: aws-managed-cost-intelligence-dashboards-us-east-2} + us-west-1: {BucketName: aws-managed-cost-intelligence-dashboards-us-west-1} + us-west-2: {BucketName: aws-managed-cost-intelligence-dashboards-us-west-2} + #todo: add af-south-1 + #todo: add ap-southeast-3 + #todo: add eu-south-1 + #todo: add eu-central-2 + Resources: SpiceRefreshExecutionRole: #Role needed to schedule spice ingestion for the datasets not used by default @@ -306,7 +342,7 @@ Resources: - lambda.amazonaws.com Action: - sts:AssumeRole - PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref AWS::NoValue] + PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref 'AWS::NoValue'] Policies: - PolicyName: !Sub 'CidSpiceRefreshExecutionRole${Suffix}' PolicyDocument: @@ -562,6 +598,7 @@ Resources: WORKGROUP = os.environ['WORKGROUP'] CRAWLER = os.environ['CRAWLER'] QUICKSIGHT_USER = os.environ['QUICKSIGHT_USER'] + QUICKSIGHT_ROLE = os.environ.get('QUICKSIGHT_ROLE') def lambda_handler(event, context): print(event) @@ -653,6 +690,22 @@ Resources: log.append(f'ERROR: WorkGroup {WORKGROUP} - {errcode}') except Exception as exc: log.append(f'ERROR: WorkGroup {WORKGROUP} Error: {exc}') + + if QUICKSIGHT_ROLE: + try: + role_name = QUICKSIGHT_ROLE.split('/')[-1] + iam = boto3.client('iam') + attached_policies = iam.list_attached_role_policies(RoleName=role_name)['AttachedPolicies'] + for policy in attached_policies: + iam.detach_role_policy( + RoleName=role_name, + PolicyArn=policy['PolicyArn'] + ) + print(f"DEBUG: Detached managed policy: {policy['PolicyName']}") + log.append(f"DEBUG: Detached managed policy: {policy['PolicyName']}") + except Exception as exc: + log.append(f'ERROR: managed policy Error: {exc}') + print('\n'.join(log)) return (True, '\n'.join(log)) Layers: @@ -663,6 +716,7 @@ Resources: WORKGROUP: !If [NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, ''] CRAWLER: !If [NeedCURTable, !Ref MyGlueCURCrawler, ''] QUICKSIGHT_USER: !Ref QuickSightUser + QUICKSIGHT_ROLE: !If [ NeedQuickSightDataSourceRole, !Ref QuickSightDataSourceRole, !Ref 'AWS::NoValue' ] Metadata: cfn_nag: rules_to_suppress: @@ -699,7 +753,16 @@ Resources: - logs:PutLogEvents Resource: - !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/Cid*' - PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref AWS::NoValue] + - !If + - NeedQuickSightDataSourceRole + - Effect: Allow + Action: + - iam:GetRole + - iam:ListAttachedRolePolicies + - iam:DetachRolePolicy + Resource: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}" + - !Ref 'AWS::NoValue' + PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref 'AWS::NoValue'] InitLambdaExecutionRoleWorkGroupPolicy: Type: AWS::IAM::Policy @@ -785,7 +848,7 @@ Resources: Action: - sts:AssumeRole Path: !Ref RolePath - PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref AWS::NoValue] + PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref 'AWS::NoValue'] Policies: - PolicyName: AWSCURCrawlerComponentFunction PolicyDocument: @@ -896,7 +959,7 @@ Resources: CURPath: Type: Custom::CustomResourceProcessPath - Condition: NeedCUR + Condition: NeedLegacyCUR Properties: ServiceToken: !GetAtt CustomResourceProcessPath.Arn s3path: !Ref CURBucketPath @@ -1067,7 +1130,7 @@ Resources: Action: - 'sts:AssumeRole' Path: !Ref RolePath - PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref AWS::NoValue] + PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref 'AWS::NoValue'] Policies: - PolicyName: AWSCURCrawlerComponentFunction PolicyDocument: @@ -1151,7 +1214,7 @@ Resources: - 'sts:AssumeRole' Path: !Ref RolePath RoleName: !Sub '${QuickSightDataSourceRoleName}${Suffix}' - PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref AWS::NoValue] + PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref 'AWS::NoValue'] Policies: - PolicyName: AthenaAccess PolicyDocument: @@ -1177,6 +1240,21 @@ Resources: - glue:GetTables Resource: - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog' + - Fn::If: + - UseCUR2 + - !Join + - '/' + - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database + - !ImportValue cid-DataExports-Database + - !Ref 'AWS::NoValue' + - Fn::If: + - UseCUR2 + - !Join + - '/' + - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table + - !ImportValue cid-DataExports-Database + - "*" + - !Ref 'AWS::NoValue' - Fn::If: - NeedDatabase - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${CidDatabase} @@ -1209,6 +1287,13 @@ Resources: - NeedAthenaWorkgroup - !Sub 'arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${MyAthenaWorkGroup}' - !Sub 'arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${AthenaWorkgroup}' + - Fn::If: + - UseCUR2 + - !Join + - '/' + - - !Sub arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:database + - !ImportValue cid-DataExports-Database + - !Ref 'AWS::NoValue' - Effect: Allow Action: - s3:GetBucketLocation @@ -1234,9 +1319,10 @@ Resources: - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-data-exports # prefix for data-exports hardcoded here - !Sub arn:${AWS::Partition}:s3:::${ODCPath.Bucket} - !If - - NeedQuickSightDataSourceRoleAndCUR + - NeedQuickSightDataSourceRoleAndLegacyCUR - !Sub arn:${AWS::Partition}:s3:::${CURPath.Bucket} - !Ref "AWS::NoValue" + # FOR CUR2 there will be attached policy no need to add it here - Sid: AllowReadBucket Effect: Allow Action: @@ -1246,9 +1332,10 @@ Resources: - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-data-exports/* # prefix for data-exports hardcoded here - !Sub arn:${AWS::Partition}:s3:::${ODCPath.Bucket}/* - !If - - NeedQuickSightDataSourceRoleAndCUR + - NeedQuickSightDataSourceRoleAndLegacyCUR - !Sub arn:${AWS::Partition}:s3:::${CURPath.Bucket}/* - !Ref "AWS::NoValue" + # FOR CUR2 there will be attached policy no need to add it here - !If - NeedQuickSightDataSourceKMS - Sid: AllowKmsDecrypt @@ -1268,14 +1355,14 @@ Resources: CidAthenaDataSource: Type: AWS::QuickSight::DataSource Properties: - AwsAccountId: !Sub '${AWS::AccountId}' + AwsAccountId: !Ref 'AWS::AccountId' Type: ATHENA DataSourceId: !Sub 'CID-Athena${Suffix}' Name: !Sub 'CID-Athena${Suffix}' DataSourceParameters: AthenaParameters: WorkGroup: !If [ NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, !Ref AthenaWorkgroup ] - RoleArn: !If [ UseQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRoleName}", !Ref AWS::NoValue ] + RoleArn: !If [ UseQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRoleName}", !Ref 'AWS::NoValue'] Permissions: - Actions: - 'quicksight:DescribeDataSource' @@ -1300,7 +1387,8 @@ Resources: - lambda.amazonaws.com Action: - sts:AssumeRole - PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref AWS::NoValue] + PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref 'AWS::NoValue'] + ManagedPolicyArns: !If [UseCUR2, [ !ImportValue cid-DataExports-ReadAccessPolicyARN ] , !Ref 'AWS::NoValue'] Policies: - PolicyName: CidExecPolicy PolicyDocument: @@ -1325,7 +1413,16 @@ Resources: - glue:GetTables - glue:GetPartitions - glue:CreateTable + - glue:GetCrawler Resource: "*" # This is needed to allow Autodetect in CID-CMD + - Effect: Allow + Action: + - glue:DeleteTable + Resource: + - Fn::If: + - NeedDatabase + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${CidDatabase}/* + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${DatabaseName}/* - Effect: Allow Action: - s3:CreateBucket @@ -1396,6 +1493,22 @@ Resources: - athena:ListWorkGroups - athena:GetDatabase Resource: '*' # This is needed to allow Autodetect in CID-CMD + - !If + - NeedQuickSightDataSourceRole + - Effect: Allow + Action: + - cloudformation:ListExports + Resource: '*' # Not possible to limit listExports + - !Ref 'AWS::NoValue' + - !If + - NeedQuickSightDataSourceRole + - Effect: Allow + Action: + - iam:GetRole + - iam:ListAttachedRolePolicies + - iam:AttachRolePolicy + Resource: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}" + - !Ref 'AWS::NoValue' - Effect: Allow Action: - logs:CreateLogGroup @@ -1411,6 +1524,8 @@ Resources: - id: 'W28' reason: "Need explicit name to give permissions" + + DataLakeSettingsCidExecRolePerm: Type: AWS::LakeFormation::Permissions Condition: NeedLakeFormationEnabled @@ -1624,8 +1739,11 @@ Resources: LayerName: !Sub 'CidLambdaLayer${Suffix}' Description: An AWS managed layer with a cid-cmd package installed Content: - S3Bucket: !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}' - S3Key: 'cid-resource-lambda-layer/cid-0.3.15.zip' #replace version here if needed + S3Bucket: !If + - LambdaLayerBucketPrefixIsManaged + - !FindInMap [RegionMap, !Ref 'AWS::Region', BucketName] + - !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}' # Region added for backward compatibility + S3Key: 'cid-resource-lambda-layer/cid-4.0.0.zip' #replace version here if needed CompatibleRuntimes: - python3.10 - python3.11 @@ -1646,7 +1764,8 @@ Resources: quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ] athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] glue-data-catalog: !Ref GlueDataCatalog - cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] + cur-table-name: !If [ UseCUR2, 'cur2', !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] ] + cur-database: !If [ UseCUR2, !ImportValue cid-DataExports-Database, !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] ] quicksight-user: !Ref QuickSightUser account-map-source: 'dummy' #initial share-with-account: !Ref ShareDashboard @@ -1666,7 +1785,8 @@ Resources: quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ] athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] glue-data-catalog: !Ref GlueDataCatalog - cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] + cur-table-name: !If [ UseCUR2, 'cur2', !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] ] + cur-database: !If [ UseCUR2, !ImportValue cid-DataExports-Database, !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] ] quicksight-user: !Ref QuickSightUser account-map-source: 'dummy' #initial share-with-account: !Ref ShareDashboard @@ -1688,7 +1808,8 @@ Resources: quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ] athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] glue-data-catalog: !Ref GlueDataCatalog - cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] + cur-table-name: !If [ UseCUR2, 'cur2', !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] ] + cur-database: !If [ UseCUR2, !ImportValue cid-DataExports-Database, !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] ] quicksight-user: !Ref QuickSightUser account-map-source: 'dummy' #initial share-with-account: !Ref ShareDashboard @@ -1711,7 +1832,8 @@ Resources: quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ] athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] glue-data-catalog: !Ref GlueDataCatalog - cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] + cur-table-name: !If [ UseCUR2, 'cur2', !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] ] + cur-database: !If [ UseCUR2, !ImportValue cid-DataExports-Database, !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] ] quicksight-user: !Ref QuickSightUser account-map-source: 'dummy' #initial share-with-account: !Ref ShareDashboard @@ -1736,7 +1858,8 @@ Resources: quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ] athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] glue-data-catalog: !Ref GlueDataCatalog - cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] + cur-table-name: !If [ UseCUR2, 'cur2', !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] ] + cur-database: !If [ UseCUR2, !ImportValue cid-DataExports-Database, !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] ] quicksight-user: !Ref QuickSightUser share-with-account: !Ref ShareDashboard view-ta-organizational-view-reports-s3FolderPath: !Sub '${OptimizationDataCollectionBucketPath}/trusted-advisor/trusted-advisor-data' @@ -1756,7 +1879,8 @@ Resources: quicksight-datasource-role-arn: !If [ NeedQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ] athena-database: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] glue-data-catalog: !Ref GlueDataCatalog - cur-table-name: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] + cur-table-name: !If [ UseCUR2, 'cur2', !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] ] + cur-database: !If [ UseCUR2, !ImportValue cid-DataExports-Database, !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] ] quicksight-user: !Ref QuickSightUser share-with-account: !Ref ShareDashboard view-compute-optimizer-lambda-lines-s3FolderPath: !Sub '${OptimizationDataCollectionBucketPath}/compute_optimizer/compute_optimizer_lambda' @@ -1791,3 +1915,36 @@ Outputs: Description: "URL of ComputeOptimizerDashboard" Condition: NeedComputeOptimizerDashboard Value: !GetAtt ComputeOptimizerDashboard.DashboardURL + CidExecArn: + Description: Technical Value - CidExecArn + Value: !GetAtt CidExec.Arn + Export: { Name: !Sub 'cid${Suffix}-CidExecArn'} + AthenaWorkgroup: + Description: Technical Value - AthenaWorkgroup + Value: !If [ NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, !Ref AthenaWorkgroup ] + Export: { Name: !Sub 'cid${Suffix}-AthenaWorkgroup'} + QuickSightDatasourceId: + Description: Technical Value - QuickSightDatasourceId + Value: !Select [ 1, !Split [ '/', !GetAtt CidAthenaDataSource.Arn]] + Export: { Name: !Sub 'cid${Suffix}-QuickSightDatasourceId'} + QuickSightDatasourceRoleArn: + Description: Technical Value - QuickSightDatasourceRoleArn + Value: !If [ NeedQuickSightDataSourceRole, !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${QuickSightDataSourceRole}", "" ] + Export: { Name: !Sub 'cid${Suffix}-QuickSightDatasourceRoleArn'} + AthenaDatabase: + Description: Technical Value - AthenaDatabase + Value: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ] + Export: { Name: !Sub 'cid${Suffix}-AthenaDatabase'} + GlueCatalog: + Description: Technical Value - GlueCatalog + Value: !Ref GlueDataCatalog + Export: { Name: !Sub 'cid${Suffix}-GlueCatalog'} + CurTableName: + Description: Technical Value - CurTableName + Value: !If [ NeedCURTable, !Ref MyCURTable, !Ref CURTableName ] + Export: { Name: !Sub 'cid${Suffix}-CurTableName'} + QuickSightUser: + Description: Technical Value - QuickSightUser + Value: !Ref QuickSightUser + Export: { Name: !Sub 'cid${Suffix}-QuickSightUser'} + diff --git a/cfn-templates/cid-plugin.yml b/cfn-templates/cid-plugin.yml new file mode 100644 index 00000000..35b85783 --- /dev/null +++ b/cfn-templates/cid-plugin.yml @@ -0,0 +1,55 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: A Plugin for Cloud Intelligence Dashboards v0.0.1 + +Parameters: + DashboardId: + Type: String + Description: 'Dashboard id' + Default: '' + AllowedPattern: "^[a-zA-Z0-9_-]*$" + ResourcesUrl: + Type: String + Description: 'Resources File URL. Keep it empty if not sure.' + Default: '' + RequiresDataCollection: + Type: String + Description: 'Is DataCollectionRequired' + AllowedValues: ["yes", "no"] + Default: 'no' + RequiresDataExports: + Type: String + Description: 'Is DataCollectionRequired' + AllowedValues: ["yes", "no"] + Default: 'no' + +Conditions: + ResourcesUrlIsEmpty: !Equals [!Ref ResourcesUrl, ''] + RequiresDataCollection: !Equals [!Ref RequiresDataCollection, 'yes'] + RequiresDataExports: !Equals [!Ref RequiresDataExports, 'yes'] + +Resources: + Dashboard: + Type: Custom::CidDashboard + Properties: + Name: !Ref DashboardId + ServiceToken: {Fn::ImportValue: "cid-CidExecArn"} + Dashboard: + dashboard-id: !Ref DashboardId + account-map-source: 'dummy' + data-collection-database-name: 'optimization_data' + athena-workgroup: {Fn::ImportValue: "cid-AthenaWorkgroup"} + quicksight-datasource-id: {Fn::ImportValue: "cid-QuickSightDatasourceId"} + quicksight-datasource-role-arn: {Fn::ImportValue: "cid-QuickSightDatasourceRoleArn"} + athena-database: {Fn::ImportValue: "cid-AthenaDatabase"} + glue-data-catalog: {Fn::ImportValue: "cid-GlueCatalog"} + cur-table-name: {Fn::ImportValue: "cid-CurTableName"} + quicksight-user: {Fn::ImportValue: "cid-QuickSightUser"} + share-with-account: 'yes' + resources: !If [ResourcesUrlIsEmpty, !Ref 'AWS::NoValue', !Ref ResourcesUrl] + data_exports_database_name: !If [RequiresDataExports, {Fn::ImportValue: "cid-DataExports-Database"}, !Ref 'AWS::NoValue'] + data_collection_database_name: !If [RequiresDataCollection, {Fn::ImportValue: "cid-DataCollection-Database"}, !Ref 'AWS::NoValue'] + +Outputs: + DashboardURL: + Description: "URL of Dashboard" + Value: !GetAtt Dashboard.DashboardURL diff --git a/cfn-templates/data-exports-aggregation.yaml b/cfn-templates/data-exports-aggregation.yaml index 2d345f89..3631f08c 100644 --- a/cfn-templates/data-exports-aggregation.yaml +++ b/cfn-templates/data-exports-aggregation.yaml @@ -1447,7 +1447,7 @@ Resources: reason: "Need an explicit name for reference" Outputs: - DestinationBucketName: + AggregateBucketName: Description: Bucket with aggregate Data Exports Value: !Sub ${ResourcePrefix}-${DestinationAccountId}-data-exports Export: { Name: 'cid-DataExports-Bucket'} diff --git a/cfn-templates/tests/test_deploy_with_permissions.py b/cfn-templates/tests/test_deploy_with_permissions.py index 92372358..e07d0f3e 100644 --- a/cfn-templates/tests/test_deploy_with_permissions.py +++ b/cfn-templates/tests/test_deploy_with_permissions.py @@ -12,7 +12,7 @@ 3. Verify dashboard exists 3. Delete all in reverse order -This must be executed with admin privileges. +This must be executed with admin privileges. And takes around 15mins to complete. """ import os import json @@ -23,6 +23,13 @@ import boto3 import click +# Configure the logger +logging.basicConfig( + format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=logging.INFO +) + logger = logging.getLogger(__name__) # Console Colors @@ -218,8 +225,9 @@ def create_finops_role(update): {"ParameterKey": 'QuickSightManagement', "ParameterValue":'no'}, {"ParameterKey": 'QuickSightAdmin', "ParameterValue":'no'}, {"ParameterKey": 'CloudIntelligenceDashboardsCFNManagement', "ParameterValue":'yes'}, + {"ParameterKey": 'CloudIntelligenceDashboardsManagement', "ParameterValue":'yes'}, {"ParameterKey": 'CURDestination', "ParameterValue":'yes'}, - {"ParameterKey": 'CURReplication', "ParameterValue":'no'}, + {"ParameterKey": 'CURReplication', "ParameterValue":'yes'}, ], Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'], ) @@ -252,16 +260,17 @@ def create_cid_as_finops(update): logger.info('As Finops Creating CUR') finops_cfn = finops_session.client('cloudformation') + #finops_cfn = admin_cfn #FIXME !!! params = dict( StackName="CID-CUR-Destination", - TemplateURL=upload_to_s3('cfn-templates/cur-aggregation.yaml'), + TemplateURL=upload_to_s3('cfn-templates/data-exports-aggregation.yaml'), Parameters=[ {"ParameterKey": 'DestinationAccountId', "ParameterValue": account_id}, {"ParameterKey": 'ResourcePrefix', "ParameterValue": 'cid'}, - {"ParameterKey": 'CreateCUR', "ParameterValue": 'True'}, - {"ParameterKey": 'SourceAccountIds', "ParameterValue": ''}, + {"ParameterKey": 'ManageCUR2', "ParameterValue": 'yes'}, + {"ParameterKey": 'SourceAccountIds', "ParameterValue": account_id}, ], - Capabilities=['CAPABILITY_IAM'], + Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'], ) try: finops_cfn.create_stack(**params) @@ -282,7 +291,7 @@ def create_cid_as_finops(update): {"ParameterKey": 'PrerequisitesQuickSight', "ParameterValue": 'yes'}, {"ParameterKey": 'PrerequisitesQuickSightPermissions', "ParameterValue": 'yes'}, {"ParameterKey": 'QuickSightUser', "ParameterValue": get_qs_user()}, - {"ParameterKey": 'DeployCUDOSDashboard', "ParameterValue": 'yes'}, + {"ParameterKey": 'CURVersion', "ParameterValue": '2.0'}, {"ParameterKey": 'DeployCUDOSv5', "ParameterValue": 'yes'}, {"ParameterKey": 'LambdaLayerBucketPrefix', "ParameterValue": TMP_BUCKET_PREFIX}, ], @@ -303,34 +312,55 @@ def test_dashboard_exists(): """check that dashboard exists""" dash = boto3.client('quicksight').describe_dashboard( AwsAccountId=account_id, - DashboardId='cudos' + DashboardId='cudos-v5' )['Dashboard'] logger.info("Dashboard exists with status = %s", dash['Version']['Status']) def test_crawler_results(): - glue = boto3.client('glue') - logger.info('Waiting For Crawler to finish') + """ + Waits for the specified Glue crawlers to complete successfully. + + Raises: + AssertionError: If any of the crawlers fail or the timeout is reached. + """ + crawler_names = [ + 'cid-DataExportCUR2Crawler', + #'cid-DataExportFOCUSCrawler', + ] timeout_seconds = 300 + + glue = boto3.client('glue') start_time = time.time() + completed_crawlers = set() + + ## Start glue crawlers + for crawler_name in crawler_names: + glue.start_crawler(Name=crawler_name) + while time.time() - start_time < timeout_seconds: - try: - time.sleep(3) - crawler = glue.get_crawler(Name='CidCrawler')['Crawler'] - logger.debug('Crawler state = ' + crawler['State']) - if crawler['State'] != 'READY': + for crawler_name in crawler_names: + try: + crawler_status = glue.get_crawler(Name=crawler_name)['Crawler'] + logger.info(f"{crawler_name} crawler state = {crawler_status['State']}") + + if crawler_status['State'] != 'READY': + continue + last_crawl = crawler_status.get('LastCrawl') + if last_crawl: + if last_crawl.get('Status') != 'SUCCEEDED' or last_crawl.get('ErrorMessage'): + raise AssertionError(f'Something wrong with crawler {last_crawl}') + else: + completed_crawlers.add(crawler_name) #pylint: disable=superfluous-parens + logger.info(f"Crawler '{crawler_name}' has completed successfully.") + if len(completed_crawlers) == len(crawler_names): + logger.info("All crawlers have completed successfully.") + return + except glue.exceptions.EntityNotFoundException: + logger.debug('Crawler is not there yet ') continue - last_crawl = crawler.get('LastCrawl') - if last_crawl: - logger.debug(last_crawl) - if last_crawl.get('Status') != 'SUCCEEDED' or last_crawl.get('ErrorMessage'): - raise AssertionError(f'Something wrong with crawler {last_crawl}') - logger.info('Crawler SUCCEEDED') - break - except glue.exceptions.EntityNotFoundException: - logger.debug('Crawler is not there yet ') - continue - else: - raise AssertionError('Timeout while waiting for crawler') + time.sleep(10) + logger.error(f"Timeout reached while waiting for crawlers to complete.") + raise Exception(f"Timeout reached while waiting for crawlers to complete.") def test_dataset_scheduled(): @@ -397,14 +427,19 @@ def teardown(): logger.info('Finops Session created') finops_cfn = finops_session.client('cloudformation') + #finops_cfn = admin_cfn #FIXME: for wip only logger.info("Deleting bucket") + delete_bucket(f'cid-{account_id}-data-exports') # Cannot be done by CFN + delete_bucket(f'cid-{account_id}-data-local') # Cannot be done by CFN delete_bucket(f'cid-{account_id}-shared') # Cannot be done by CFN logger.info("Deleting Dashboards stack") try: finops_cfn.delete_stack(StackName="Cloud-Intelligence-Dashboards") except Exception as exc: # pylint: disable=broad-exception-caught logger.info(exc) + ## wait for deletion of previous stack + watch_stacks(finops_cfn, ["Cloud-Intelligence-Dashboards"]) logger.info("Deleting CUR stack") try: finops_cfn.delete_stack(StackName="CID-CUR-Destination") diff --git a/cid/_version.py b/cid/_version.py index 529e69fd..92a9b01a 100644 --- a/cid/_version.py +++ b/cid/_version.py @@ -1 +1,2 @@ -__version__ = '0.3.15' +__version__ = '4.0.0' + diff --git a/cid/builtin/core/data/datasets/shared/customer_all.json b/cid/builtin/core/data/datasets/shared/customer_all.json index 0b69a0cc..62c3f0b2 100644 --- a/cid/builtin/core/data/datasets/shared/customer_all.json +++ b/cid/builtin/core/data/datasets/shared/customer_all.json @@ -22,8 +22,8 @@ "b163ad4b-f459-4b01-b2f3-54a168088640": { "RelationalTable": { "DataSourceArn": "${athena_datasource_arn}", - "Schema": "${athena_database_name}", - "Name": "${cur_table_name}", + "Schema": "${cur1_database}", + "Name": "${cur1_table_name}", "InputColumns": [ { "Name": "product_updates", @@ -837,7 +837,7 @@ } }, "2f69dc70-9487-4f34-9120-e5f40660b257": { - "Alias": "${cur_table_name}", + "Alias": "${cur1_table_name}", "Source": { "PhysicalTableId": "b163ad4b-f459-4b01-b2f3-54a168088640" } diff --git a/cid/builtin/core/data/queries/cid/compute_savings_plan_eligible_spend.sql b/cid/builtin/core/data/queries/cid/compute_savings_plan_eligible_spend.sql index e9e9ab88..2a5e8e8b 100644 --- a/cid/builtin/core/data/queries/cid/compute_savings_plan_eligible_spend.sql +++ b/cid/builtin/core/data/queries/cid/compute_savings_plan_eligible_spend.sql @@ -1,7 +1,7 @@ CREATE OR REPLACE VIEW "compute_savings_plan_eligible_spend" AS SELECT DISTINCT - "year" - , "month" + split_part("billing_period", '-', 1) "year" + , split_part("billing_period", '-', 2) "month" , "bill_payer_account_id" "payer_account_id" , "line_item_usage_account_id" "linked_account_id" , "bill_billing_period_start_date" "billing_period" @@ -16,9 +16,9 @@ WHEN ("line_item_usage_type" LIKE '%Fargate%') THEN "line_item_unblended_cost" ELSE 0 END - ELSE 0 + ELSE 0 END) "unblended_cost" FROM - "${cur_table_name}" + "${cur2_database}"."${cur2_table_name}" WHERE (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '1' MONTH)) AND ("line_item_usage_start_date" < ("date_trunc"('day', current_timestamp) - INTERVAL '1' DAY)) AND ((("line_item_product_code" = 'AmazonEC2') AND ("line_item_operation" LIKE '%RunInstances%')) OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-Provisioned-GB-Second%')) OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-GB-Second%')) OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-Provisioned-Concurrency%')) OR ("line_item_usage_type" LIKE '%Fargate%')) AND ("line_item_line_item_type" = 'Usage') AND ("line_item_usage_type" NOT LIKE '%Spot%') AND ("product_servicecode" <> 'AWSDataTransfer') AND ("line_item_usage_type" NOT LIKE '%DataXfer%')) GROUP BY 1, 2, 3, 4,5,6 diff --git a/cid/builtin/core/data/queries/cid/ec2_running_cost.sql b/cid/builtin/core/data/queries/cid/ec2_running_cost.sql index 1689b7bb..a567b04d 100644 --- a/cid/builtin/core/data/queries/cid/ec2_running_cost.sql +++ b/cid/builtin/core/data/queries/cid/ec2_running_cost.sql @@ -1,30 +1,24 @@ - CREATE OR REPLACE VIEW "ec2_running_cost" AS + CREATE OR REPLACE VIEW "ec2_running_cost" AS SELECT DISTINCT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" + split_part("billing_period", '-', 1) "year" + , split_part("billing_period", '-', 2) "month" + , "bill_billing_period_start_date" "billing_period" , "date_trunc"('hour', "line_item_usage_start_date") "usage_date" , "bill_payer_account_id" "payer_account_id" , "line_item_usage_account_id" "linked_account_id" - , (CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' - WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' - ELSE 'OnDemand' END) "purchase_option" + , (CASE + WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' + WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' + WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' + ELSE 'OnDemand' + END) "purchase_option" , "sum"(CASE - -- WHEN "line_item_line_item_type" = 'SavingsPlanCoveredUsage' THEN "savings_plan_savings_plan_effective_cost" - -- WHEN "line_item_line_item_type" = 'DiscountedUsage' THEN "reservation_effective_cost" + WHEN "line_item_line_item_type" = 'SavingsPlanCoveredUsage' THEN "savings_plan_savings_plan_effective_cost" + WHEN "line_item_line_item_type" = 'DiscountedUsage' THEN "reservation_effective_cost" WHEN "line_item_line_item_type" = 'Usage' THEN "line_item_unblended_cost" ELSE 0 END) "amortized_cost" , "round"("sum"("line_item_usage_amount"), 2) "usage_quantity" FROM - "${cur_table_name}" - WHERE ( - ("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '1' MONTH)) AND ("line_item_product_code" = 'AmazonEC2') AND ("product_servicecode" <> 'AWSDataTransfer') AND ("line_item_operation" LIKE '%RunInstances%') AND ("line_item_usage_type" NOT LIKE '%DataXfer%') AND - (("line_item_line_item_type" = 'Usage') - -- OR - -- ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') - -- OR - --("line_item_line_item_type" = 'DiscountedUsage') - )) + "${cur2_database}"."${cur2_table_name}" + WHERE (((((("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '1' MONTH)) AND ("line_item_product_code" = 'AmazonEC2')) AND ("product_servicecode" <> 'AWSDataTransfer')) AND ("line_item_operation" LIKE '%RunInstances%')) AND ("line_item_usage_type" NOT LIKE '%DataXfer%')) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) GROUP BY 1, 2, 3, 4,5,6,7 diff --git a/cid/builtin/core/data/queries/cid/ec2_running_cost_ri.sql b/cid/builtin/core/data/queries/cid/ec2_running_cost_ri.sql deleted file mode 100644 index 4003b531..00000000 --- a/cid/builtin/core/data/queries/cid/ec2_running_cost_ri.sql +++ /dev/null @@ -1,29 +0,0 @@ - CREATE OR REPLACE VIEW "ec2_running_cost" AS - SELECT DISTINCT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , "date_trunc"('hour', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , (CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' - WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' - WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' - ELSE 'OnDemand' END) "purchase_option" - , "sum"(CASE - -- WHEN "line_item_line_item_type" = 'SavingsPlanCoveredUsage' THEN "savings_plan_savings_plan_effective_cost" - WHEN "line_item_line_item_type" = 'DiscountedUsage' THEN "reservation_effective_cost" - WHEN "line_item_line_item_type" = 'Usage' THEN "line_item_unblended_cost" - ELSE 0 END) "amortized_cost" - , "round"("sum"("line_item_usage_amount"), 2) "usage_quantity" - FROM - "${cur_table_name}" - WHERE ( - ("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '1' MONTH)) AND ("line_item_product_code" = 'AmazonEC2') AND ("product_servicecode" <> 'AWSDataTransfer') AND ("line_item_operation" LIKE '%RunInstances%') AND ("line_item_usage_type" NOT LIKE '%DataXfer%') AND - (("line_item_line_item_type" = 'Usage') - -- OR - -- ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') - OR - ("line_item_line_item_type" = 'DiscountedUsage'))) - GROUP BY 1, 2, 3, 4,5,6,7 diff --git a/cid/builtin/core/data/queries/cid/ec2_running_cost_sp.sql b/cid/builtin/core/data/queries/cid/ec2_running_cost_sp.sql deleted file mode 100644 index c666cfca..00000000 --- a/cid/builtin/core/data/queries/cid/ec2_running_cost_sp.sql +++ /dev/null @@ -1,30 +0,0 @@ - CREATE OR REPLACE VIEW "ec2_running_cost" AS - SELECT DISTINCT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , "date_trunc"('hour', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , (CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' - WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' - ELSE 'OnDemand' END) "purchase_option" - , "sum"(CASE - WHEN "line_item_line_item_type" = 'SavingsPlanCoveredUsage' THEN "savings_plan_savings_plan_effective_cost" - -- WHEN "line_item_line_item_type" = 'DiscountedUsage' THEN "reservation_effective_cost" - WHEN "line_item_line_item_type" = 'Usage' THEN "line_item_unblended_cost" - ELSE 0 END) "amortized_cost" - , "round"("sum"("line_item_usage_amount"), 2) "usage_quantity" - FROM - "${cur_table_name}" - WHERE ( - ("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '1' MONTH)) AND ("line_item_product_code" = 'AmazonEC2') AND ("product_servicecode" <> 'AWSDataTransfer') AND ("line_item_operation" LIKE '%RunInstances%') AND ("line_item_usage_type" NOT LIKE '%DataXfer%') AND - (("line_item_line_item_type" = 'Usage') - OR - ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') - -- OR - -- ("line_item_line_item_type" = 'DiscountedUsage') - )) - GROUP BY 1, 2, 3, 4,5,6,7 diff --git a/cid/builtin/core/data/queries/cid/ec2_running_cost_sp_ri.sql b/cid/builtin/core/data/queries/cid/ec2_running_cost_sp_ri.sql deleted file mode 100644 index 35312588..00000000 --- a/cid/builtin/core/data/queries/cid/ec2_running_cost_sp_ri.sql +++ /dev/null @@ -1,19 +0,0 @@ - CREATE OR REPLACE VIEW "ec2_running_cost" AS - SELECT DISTINCT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , "date_trunc"('hour', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' ELSE 'OnDemand' END) "purchase_option" - , "sum"(CASE - WHEN "line_item_line_item_type" = 'SavingsPlanCoveredUsage' THEN "savings_plan_savings_plan_effective_cost" - WHEN "line_item_line_item_type" = 'DiscountedUsage' THEN "reservation_effective_cost" - WHEN "line_item_line_item_type" = 'Usage' THEN "line_item_unblended_cost" - ELSE 0 END) "amortized_cost" - , "round"("sum"("line_item_usage_amount"), 2) "usage_quantity" - FROM - "${cur_table_name}" - WHERE (((((("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '1' MONTH)) AND ("line_item_product_code" = 'AmazonEC2')) AND ("product_servicecode" <> 'AWSDataTransfer')) AND ("line_item_operation" LIKE '%RunInstances%')) AND ("line_item_usage_type" NOT LIKE '%DataXfer%')) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))) - GROUP BY 1, 2, 3, 4,5,6,7 diff --git a/cid/builtin/core/data/queries/cid/ri_sp_mapping.sql b/cid/builtin/core/data/queries/cid/ri_sp_mapping.sql index 8c7cb559..3b13e561 100644 --- a/cid/builtin/core/data/queries/cid/ri_sp_mapping.sql +++ b/cid/builtin/core/data/queries/cid/ri_sp_mapping.sql @@ -1,71 +1,35 @@ - CREATE OR REPLACE VIEW "ri_sp_mapping" AS - SELECT DISTINCT - "a"."billing_period_mapping" - , "a"."payer_account_id_mapping" - , "a"."ri_sp_arn_mapping" - , "a"."ri_sp_end_date" - , "b"."ri_sp_term" - , "b"."ri_sp_offering" - , "b"."ri_sp_payment" - - FROM - (( - SELECT DISTINCT - "bill_billing_period_start_date" "billing_period_mapping" - , "bill_payer_account_id" "payer_account_id_mapping" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" - WHEN ("line_item_line_item_type" = 'Usage') THEN ' ' - ELSE '' END "ri_sp_arn_mapping" - , CAST(CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN CAST(from_iso8601_timestamp("savings_plan_end_time") AS timestamp) - -- WHEN ("reservation_reservation_a_r_n" <> '' AND "reservation_end_time" <> '') THEN CAST(from_iso8601_timestamp("reservation_end_time") AS timestamp) - WHEN ("line_item_line_item_type" = 'Usage') THEN ' ' - ELSE NULL END AS timestamp) "ri_sp_end_date" - FROM - "${cur_table_name}" - WHERE ( - ("line_item_line_item_type" <> 'Usage') - -- OR - -- ("line_item_line_item_type" = 'RIFee') - -- OR - -- ("line_item_line_item_type" = 'SavingsPlanRecurringFee') - ) - - ) a - LEFT JOIN ( - SELECT DISTINCT - "bill_billing_period_start_date" "billing_period_mapping" - , "bill_payer_account_id" "payer_account_id_mapping" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" - WHEN ("line_item_line_item_type" = 'Usage') THEN ' ' - ELSE '' END "ri_sp_arn_mapping" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_purchase_term" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_lease_contract_length" - WHEN ("line_item_line_item_type" = 'Usage') THEN ' ' - ELSE '' END "ri_sp_term" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_offering_type" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_offering_class" - WHEN ("line_item_line_item_type" = 'Usage') THEN ' ' - ELSE '' END "ri_sp_offering" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_payment_option" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_purchase_option" - WHEN ("line_item_line_item_type" = 'Usage') THEN ' ' - ELSE '' END "ri_sp_Payment" - FROM - "${cur_table_name}" - WHERE ( - ("line_item_line_item_type" <> 'Usage') - -- OR - -- ("line_item_line_item_type" = 'DiscountedUsage') - -- OR - -- ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') - ) - - ) b ON (("a"."ri_sp_arn_mapping" = "b"."ri_sp_arn_mapping") AND ("a"."billing_period_mapping" = "b"."billing_period_mapping") AND ("a"."payer_account_id_mapping" = "b"."payer_account_id_mapping"))) +CREATE OR REPLACE VIEW "ri_sp_mapping" AS +SELECT DISTINCT + "a"."billing_period_mapping" +, "a"."payer_account_id_mapping" +, "a"."ri_sp_arn_mapping" +, "a"."ri_sp_end_date" +, COALESCE("b"."ri_sp_term", "a"."ri_sp_term") "ri_sp_term" +, COALESCE("b"."ri_sp_offering", "a"."ri_sp_offering") "ri_sp_offering" +, COALESCE("b"."ri_sp_payment", "a"."ri_sp_payment") "ri_sp_payment" +FROM + (( + SELECT DISTINCT + "bill_billing_period_start_date" "billing_period_mapping" + , "bill_payer_account_id" "payer_account_id_mapping" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" ELSE '' END) "ri_sp_arn_mapping" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN CAST(CAST("from_iso8601_timestamp"("savings_plan_end_time") AS date) AS timestamp) WHEN (("reservation_reservation_a_r_n" <> '') AND ("reservation_end_time" <> '')) THEN CAST(CAST("from_iso8601_timestamp"("reservation_end_time") AS date) AS timestamp) ELSE null END) "ri_sp_end_date" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_purchase_term" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_lease_contract_length" ELSE '' END) "ri_sp_term" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_offering_type" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_offering_class" ELSE '' END) "ri_sp_offering" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_payment_option" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_purchase_option" ELSE '' END) "ri_sp_payment" + FROM + "${cur2_database}"."${cur2_table_name}" + WHERE (("line_item_line_item_type" = 'RIFee') OR ("line_item_line_item_type" = 'SavingsPlanRecurringFee')) +) a +LEFT JOIN ( + SELECT DISTINCT + "bill_billing_period_start_date" "billing_period_mapping" + , "bill_payer_account_id" "payer_account_id_mapping" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" ELSE '' END) "ri_sp_arn_mapping" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_purchase_term" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_lease_contract_length" ELSE '' END) "ri_sp_term" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_offering_type" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_offering_class" ELSE '' END) "ri_sp_offering" + , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_payment_option" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_purchase_option" ELSE '' END) "ri_sp_payment" + FROM + "${cur2_database}"."${cur2_table_name}" + WHERE (("line_item_line_item_type" = 'DiscountedUsage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) +) b ON ((("a"."ri_sp_arn_mapping" = "b"."ri_sp_arn_mapping") AND ("a"."billing_period_mapping" = "b"."billing_period_mapping")) AND ("a"."payer_account_id_mapping" = "b"."payer_account_id_mapping"))) diff --git a/cid/builtin/core/data/queries/cid/ri_sp_mapping_ri.sql b/cid/builtin/core/data/queries/cid/ri_sp_mapping_ri.sql deleted file mode 100644 index 1fded526..00000000 --- a/cid/builtin/core/data/queries/cid/ri_sp_mapping_ri.sql +++ /dev/null @@ -1,61 +0,0 @@ - CREATE OR REPLACE VIEW "ri_sp_mapping" AS - SELECT DISTINCT - "a"."billing_period_mapping" - , "a"."payer_account_id_mapping" - , "a"."ri_sp_arn_mapping" - , "a"."ri_sp_end_date" - , "b"."ri_sp_term" - , "b"."ri_sp_offering" - , "b"."ri_sp_payment" - - FROM - (( - SELECT DISTINCT - "bill_billing_period_start_date" "billing_period_mapping" - , "bill_payer_account_id" "payer_account_id_mapping" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" - ELSE '' END "ri_sp_arn_mapping" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN CAST(CAST(from_iso8601_timestamp("savings_plan_end_time") AS date) AS timestamp) - WHEN ("reservation_reservation_a_r_n" <> '' AND "reservation_end_time" <> '') THEN CAST(CAST(from_iso8601_timestamp("reservation_end_time") AS date) AS timestamp) - ELSE NULL END "ri_sp_end_date" - FROM - "${cur_table_name}" - WHERE ( - ("line_item_line_item_type" = 'RIFee') - -- OR - -- ("line_item_line_item_type" = 'SavingsPlanRecurringFee') - ) - - ) a - LEFT JOIN ( - SELECT DISTINCT - "bill_billing_period_start_date" "billing_period_mapping" - , "bill_payer_account_id" "payer_account_id_mapping" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" - ELSE '' END "ri_sp_arn_mapping" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_purchase_term" - WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_lease_contract_length" - ELSE '' END "ri_sp_term" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_offering_type" - WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_offering_class" - ELSE '' END "ri_sp_offering" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_payment_option" - WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_purchase_option" - ELSE '' END "ri_sp_Payment" - FROM - "${cur_table_name}" - WHERE ( - ("line_item_line_item_type" = 'DiscountedUsage') - -- OR - -- ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') - ) - - ) b ON (("a"."ri_sp_arn_mapping" = "b"."ri_sp_arn_mapping") AND ("a"."billing_period_mapping" = "b"."billing_period_mapping") AND ("a"."payer_account_id_mapping" = "b"."payer_account_id_mapping"))) diff --git a/cid/builtin/core/data/queries/cid/ri_sp_mapping_sp.sql b/cid/builtin/core/data/queries/cid/ri_sp_mapping_sp.sql deleted file mode 100644 index cc7f9a60..00000000 --- a/cid/builtin/core/data/queries/cid/ri_sp_mapping_sp.sql +++ /dev/null @@ -1,61 +0,0 @@ - CREATE OR REPLACE VIEW "ri_sp_mapping" AS - SELECT DISTINCT - "a"."billing_period_mapping" - , "a"."payer_account_id_mapping" - , "a"."ri_sp_arn_mapping" - , "a"."ri_sp_end_date" - , "b"."ri_sp_term" - , "b"."ri_sp_offering" - , "b"."ri_sp_payment" - - FROM - (( - SELECT DISTINCT - "bill_billing_period_start_date" "billing_period_mapping" - , "bill_payer_account_id" "payer_account_id_mapping" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" - ELSE '' END "ri_sp_arn_mapping" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN CAST(CAST(from_iso8601_timestamp("savings_plan_end_time") AS date) AS timestamp) - -- WHEN ("reservation_reservation_a_r_n" <> '' AND "reservation_end_time" <> '') THEN CAST(CAST(from_iso8601_timestamp("reservation_end_time") AS date) AS timestamp - ELSE NULL END "ri_sp_end_date" - FROM - "${cur_table_name}" - WHERE ( - -- ("line_item_line_item_type" = 'RIFee') - -- OR - ("line_item_line_item_type" = 'SavingsPlanRecurringFee') - ) - - ) a - LEFT JOIN ( - SELECT DISTINCT - "bill_billing_period_start_date" "billing_period_mapping" - , "bill_payer_account_id" "payer_account_id_mapping" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" - ELSE '' END "ri_sp_arn_mapping" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_purchase_term" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_lease_contract_length" - ELSE '' END "ri_sp_term" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_offering_type" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_offering_class" - ELSE '' END "ri_sp_offering" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_payment_option" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_purchase_option" - ELSE '' END "ri_sp_Payment" - FROM - "${cur_table_name}" - WHERE ( - -- ("line_item_line_item_type" = 'DiscountedUsage') - -- OR - ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') - ) - - ) b ON (("a"."ri_sp_arn_mapping" = "b"."ri_sp_arn_mapping") AND ("a"."billing_period_mapping" = "b"."billing_period_mapping") AND ("a"."payer_account_id_mapping" = "b"."payer_account_id_mapping"))) diff --git a/cid/builtin/core/data/queries/cid/ri_sp_mapping_sp_ri.sql b/cid/builtin/core/data/queries/cid/ri_sp_mapping_sp_ri.sql deleted file mode 100644 index 3472ba31..00000000 --- a/cid/builtin/core/data/queries/cid/ri_sp_mapping_sp_ri.sql +++ /dev/null @@ -1,35 +0,0 @@ -CREATE OR REPLACE VIEW "ri_sp_mapping" AS -SELECT DISTINCT - "a"."billing_period_mapping" -, "a"."payer_account_id_mapping" -, "a"."ri_sp_arn_mapping" -, "a"."ri_sp_end_date" -, COALESCE("b"."ri_sp_term", "a"."ri_sp_term") "ri_sp_term" -, COALESCE("b"."ri_sp_offering", "a"."ri_sp_offering") "ri_sp_offering" -, COALESCE("b"."ri_sp_payment", "a"."ri_sp_payment") "ri_sp_payment" -FROM - (( - SELECT DISTINCT - "bill_billing_period_start_date" "billing_period_mapping" - , "bill_payer_account_id" "payer_account_id_mapping" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" ELSE '' END) "ri_sp_arn_mapping" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN CAST(CAST("from_iso8601_timestamp"("savings_plan_end_time") AS date) AS timestamp) WHEN (("reservation_reservation_a_r_n" <> '') AND ("reservation_end_time" <> '')) THEN CAST(CAST("from_iso8601_timestamp"("reservation_end_time") AS date) AS timestamp) ELSE null END) "ri_sp_end_date" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_purchase_term" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_lease_contract_length" ELSE '' END) "ri_sp_term" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_offering_type" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_offering_class" ELSE '' END) "ri_sp_offering" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_payment_option" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_purchase_option" ELSE '' END) "ri_sp_Payment" - FROM - "${cur_table_name}" - WHERE (("line_item_line_item_type" = 'RIFee') OR ("line_item_line_item_type" = 'SavingsPlanRecurringFee')) -) a -LEFT JOIN ( - SELECT DISTINCT - "bill_billing_period_start_date" "billing_period_mapping" - , "bill_payer_account_id" "payer_account_id_mapping" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" ELSE '' END) "ri_sp_arn_mapping" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_purchase_term" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_lease_contract_length" ELSE '' END) "ri_sp_term" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_offering_type" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_offering_class" ELSE '' END) "ri_sp_offering" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_payment_option" WHEN ("reservation_reservation_a_r_n" <> '') THEN "pricing_purchase_option" ELSE '' END) "ri_sp_Payment" - FROM - "${cur_table_name}" - WHERE (("line_item_line_item_type" = 'DiscountedUsage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) -) b ON ((("a"."ri_sp_arn_mapping" = "b"."ri_sp_arn_mapping") AND ("a"."billing_period_mapping" = "b"."billing_period_mapping")) AND ("a"."payer_account_id_mapping" = "b"."payer_account_id_mapping"))) diff --git a/cid/builtin/core/data/queries/cid/s3.sql b/cid/builtin/core/data/queries/cid/s3.sql index 36fe8512..d51bf5ef 100644 --- a/cid/builtin/core/data/queries/cid/s3.sql +++ b/cid/builtin/core/data/queries/cid/s3.sql @@ -1,6 +1,7 @@ CREATE OR REPLACE VIEW "s3_view" AS -SELECT DISTINCT "year", - "month", +SELECT DISTINCT + split_part("billing_period", '-', 1) "year", + split_part("billing_period", '-', 2) "month", "bill_billing_period_start_date" "billing_period", "date_trunc"('day', "line_item_usage_start_date") "usage_date", "bill_payer_account_id" "payer_account_id", @@ -8,7 +9,7 @@ SELECT DISTINCT "year", "line_item_resource_id" "resource_id", "line_item_product_code" "product_code", "line_item_operation" "operation", - "product_region" "region", + product['region'] "region", "line_item_line_item_type" "charge_type", "pricing_unit" "pricing_unit", "sum"( @@ -19,7 +20,7 @@ SELECT DISTINCT "year", ) "usage_quantity", "sum"("line_item_unblended_cost") "unblended_cost", "sum"("pricing_public_on_demand_cost") "public_cost" -FROM "${cur_table_name}" +FROM "${cur2_database}"."${cur2_table_name}" WHERE ( ( ( diff --git a/cid/builtin/core/data/queries/cid/summary_view.sql b/cid/builtin/core/data/queries/cid/summary_view.sql index 621a2054..37f02b69 100644 --- a/cid/builtin/core/data/queries/cid/summary_view.sql +++ b/cid/builtin/core/data/queries/cid/summary_view.sql @@ -1,90 +1,96 @@ - CREATE OR REPLACE VIEW summary_view AS - SELECT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , CASE - WHEN ("date_trunc"('month',"line_item_usage_start_date")) >= ("date_trunc"('month', current_timestamp) - INTERVAL '3' MONTH) THEN "date_trunc"('day', "line_item_usage_start_date") ELSE "date_trunc"('month', "line_item_usage_start_date") END "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "bill_invoice_id" "invoice_id" - , "line_item_line_item_type" "charge_type" - , CASE - -- WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN 'Running_Usage' - -- WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN 'Running_Usage' - WHEN ("line_item_line_item_type" = 'Usage') THEN 'Running_Usage' - ELSE 'non_usage' END "charge_category" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' - WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' - ELSE 'OnDemand' END "purchase_option" - ,CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" - WHEN ("line_item_line_item_type" = 'Usage') THEN CAST('' AS varchar) - ELSE CAST('' AS varchar) END "ri_sp_arn" - , "line_item_product_code" "product_code" - , "product_product_name" "product_name" - , CASE - WHEN ("bill_billing_entity" = 'AWS Marketplace' AND "line_item_line_item_type" NOT LIKE '%Discount%') THEN "Product_Product_Name" - WHEN ("product_servicecode" = '') THEN "line_item_product_code" ELSE "product_servicecode" END "service" - , "product_product_family" "product_family" - , "line_item_usage_type" "usage_type" - , "line_item_operation" "operation" - , "line_item_line_item_description" "item_description" - , "line_item_availability_zone" "availability_zone" - , "product_region" "region" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", '.', 1) ELSE "product_instance_type_family" END "instance_type_family" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", ' ', 1) ELSE "product_instance_type" END "instance_type" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("split_part"("line_item_line_item_description", ' ', 2), '/', 1) ELSE "product_operating_system" END "platform" - , "product_tenancy" "tenancy" - , "product_physical_processor" "processor" - , "product_processor_features" "processor_features" - , "product_database_engine" "database_engine" - , "product_group" "product_group" - , "product_from_location" "product_from_location" - , "product_to_location" "product_to_location" - , "product_current_generation" "current_generation" - , "line_item_legal_entity" "legal_entity" - , "bill_billing_entity" "billing_entity" - , "pricing_unit" "pricing_unit" - , "approx_distinct"("Line_item_resource_id") "resource_id_count" - , sum(CASE - -- WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" - -- WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" - WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" - ELSE 0 END) "usage_quantity" - , sum ("line_item_unblended_cost") "unblended_cost" - , sum(CASE - WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_unblended_cost" - -- WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "savings_plan_savings_plan_effective_cost" - -- WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") - -- WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 - -- WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - -- WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "reservation_effective_cost" - -- WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") - -- WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 - ELSE "line_item_unblended_cost" END) "amortized_cost" - , CAST(sum(CASE - WHEN (line_item_line_item_type = 'Usage') THEN 0 - -- WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN (-"savings_plan_amortized_upfront_commitment_for_billing_period") - -- WHEN ("line_item_line_item_type" = 'RIFee') THEN (-"reservation_amortized_upfront_fee_for_billing_period") - ELSE 0 END) AS double) "ri_sp_trueup" - , CAST(sum(CASE - WHEN ("line_item_line_item_type" = 'Usage') THEN 0 - -- WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN "line_item_unblended_cost" - -- WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN "line_item_unblended_cost" - ELSE 0 END) AS double) "ri_sp_upfront_fees" - , sum(CASE - WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost" - FROM - "${cur_table_name}" - WHERE - (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '7' MONTH)) - AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '7' MONTH)) - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,33,34 + CREATE OR REPLACE VIEW summary_view AS + SELECT + split_part("billing_period", '-', 1) "year" + , split_part("billing_period", '-', 2) "month" + , "bill_billing_period_start_date" "billing_period" + , CASE + WHEN ("date_trunc"('month',"line_item_usage_start_date")) >= ("date_trunc"('month', current_timestamp) - INTERVAL '3' MONTH) + THEN "date_trunc"('day', "line_item_usage_start_date") + ELSE "date_trunc"('month', "line_item_usage_start_date") + END "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "bill_invoice_id" "invoice_id" + , "line_item_line_item_type" "charge_type" + , CASE + WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN 'Running_Usage' + WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN 'Running_Usage' + WHEN ("line_item_line_item_type" = 'Usage') THEN 'Running_Usage' + ELSE 'non_usage' + END "charge_category" + , CASE + WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' + WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' + WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' + ELSE 'OnDemand' + END "purchase_option" + , CASE + WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" + WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" + ELSE CAST('' AS varchar) + END "ri_sp_arn" + , "line_item_product_code" "product_code" + , product['product_name'] "product_name" + , CASE + WHEN ("bill_billing_entity" = 'AWS Marketplace' AND "line_item_line_item_type" NOT LIKE '%Discount%') THEN product['product_name'] + WHEN ("product_servicecode" = '') THEN "line_item_product_code" + ELSE "product_servicecode" + END "service" + , "product_product_family" "product_family" + , "line_item_usage_type" "usage_type" + , "line_item_operation" "operation" + , "line_item_line_item_description" "item_description" + , "line_item_availability_zone" "availability_zone" + , product['region'] "region" + , CASE + WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", '.', 1) ELSE product['instance_type_family'] END "instance_type_family" + , CASE + WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", ' ', 1) ELSE "product_instance_type" END "instance_type" + , CASE + WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("split_part"("line_item_line_item_description", ' ', 2), '/', 1) ELSE product['operating_system'] END "platform" + , product['tenancy'] "tenancy" + , product['physical_processor'] "processor" + , product['processor_features'] "processor_features" + , product['database_engine'] "database_engine" + , product['group'] "product_group" + , "product_from_location" "product_from_location" + , "product_to_location" "product_to_location" + , product['current_generation'] "current_generation" + , "line_item_legal_entity" "legal_entity" + , "bill_billing_entity" "billing_entity" + , "pricing_unit" "pricing_unit" + , approx_distinct("line_item_resource_id") "resource_id_count" + , sum(CASE + WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" + WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" + WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" ELSE 0 END) "usage_quantity" + , sum ("line_item_unblended_cost") "unblended_cost" + , sum(CASE + WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "savings_plan_savings_plan_effective_cost" + WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") + WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 + WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 + WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "reservation_effective_cost" + WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") + WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 + ELSE "line_item_unblended_cost" + END) "amortized_cost" + , sum(CASE + WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN (-"savings_plan_amortized_upfront_commitment_for_billing_period") + WHEN ("line_item_line_item_type" = 'RIFee') THEN (-"reservation_amortized_upfront_fee_for_billing_period") + ELSE 0 + END) "ri_sp_trueup" + , sum(CASE + WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN "line_item_unblended_cost" + WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN "line_item_unblended_cost" + ELSE 0 + END) "ri_sp_upfront_fees" + , sum(CASE + WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost" + FROM + "${cur2_database}"."${cur2_table_name}" + WHERE + (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '7' MONTH)) + AND (CAST("concat"("billing_period", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '7' MONTH)) + AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34 diff --git a/cid/builtin/core/data/queries/cid/summary_view_ri.sql b/cid/builtin/core/data/queries/cid/summary_view_ri.sql deleted file mode 100644 index 8ed01342..00000000 --- a/cid/builtin/core/data/queries/cid/summary_view_ri.sql +++ /dev/null @@ -1,80 +0,0 @@ - CREATE OR REPLACE VIEW summary_view AS - SELECT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , CASE - WHEN ("date_trunc"('month',"line_item_usage_start_date")) >= ("date_trunc"('month', current_timestamp) - INTERVAL '3' MONTH) THEN "date_trunc"('day', "line_item_usage_start_date") ELSE "date_trunc"('month', "line_item_usage_start_date") END "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "bill_invoice_id" "invoice_id" - , "line_item_line_item_type" "charge_type" - , CASE - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN 'Running_Usage' - -- WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN 'Running_Usage' - WHEN ("line_item_line_item_type" = 'Usage') THEN 'Running_Usage' ELSE 'non_usage' END "charge_category" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' - WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' - WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' - ELSE 'OnDemand' END "purchase_option" - , CASE - -- WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" ELSE CAST('' AS varchar) END "ri_sp_arn" - , "line_item_product_code" "product_code" - , "product_product_name" "product_name" - , CASE - WHEN ("bill_billing_entity" = 'AWS Marketplace' AND "line_item_line_item_type" NOT LIKE '%Discount%') THEN "Product_Product_Name" - WHEN ("product_servicecode" = '') THEN "line_item_product_code" ELSE "product_servicecode" END "service" - , "product_product_family" "product_family" - , "line_item_usage_type" "usage_type" - , "line_item_operation" "operation" - , "line_item_line_item_description" "item_description" - , "line_item_availability_zone" "availability_zone" - , "product_region" "region" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", '.', 1) ELSE "product_instance_type_family" END "instance_type_family" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", ' ', 1) ELSE "product_instance_type" END "instance_type" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("split_part"("line_item_line_item_description", ' ', 2), '/', 1) ELSE "product_operating_system" END "platform" - , "product_tenancy" "tenancy" - , "product_physical_processor" "processor" - , "product_processor_features" "processor_features" - , "product_database_engine" "database_engine" - , "product_group" "product_group" - , "product_from_location" "product_from_location" - , "product_to_location" "product_to_location" - , "product_current_generation" "current_generation" - , "line_item_legal_entity" "legal_entity" - , "bill_billing_entity" "billing_entity" - , "pricing_unit" "pricing_unit" - , "approx_distinct"("Line_item_resource_id") "resource_id_count" - , sum(CASE - -- WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" - WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" ELSE 0 END) "usage_quantity" - , sum ("line_item_unblended_cost") "unblended_cost" - , sum(CASE - -- WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "savings_plan_savings_plan_effective_cost" - -- WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") - -- WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 - -- WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "reservation_effective_cost" - WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") - WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE "line_item_unblended_cost" END) "amortized_cost" - , sum(CASE - -- WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN (-"savings_plan_amortized_upfront_commitment_for_billing_period") - WHEN ("line_item_line_item_type" = 'RIFee') THEN (-"reservation_amortized_upfront_fee_for_billing_period") ELSE 0 END) "ri_sp_trueup" - , sum(CASE - -- WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN "line_item_unblended_cost" - WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN "line_item_unblended_cost" ELSE 0 END) "ri_sp_upfront_fees" - , sum(CASE - WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost" - FROM - "${cur_table_name}" - WHERE - (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '7' MONTH)) - AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '7' MONTH)) - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,33,34 diff --git a/cid/builtin/core/data/queries/cid/summary_view_sp.sql b/cid/builtin/core/data/queries/cid/summary_view_sp.sql deleted file mode 100644 index fb5b1460..00000000 --- a/cid/builtin/core/data/queries/cid/summary_view_sp.sql +++ /dev/null @@ -1,84 +0,0 @@ - CREATE OR REPLACE VIEW summary_view AS - SELECT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , CASE - WHEN ("date_trunc"('month',"line_item_usage_start_date")) >= ("date_trunc"('month', current_timestamp) - INTERVAL '3' MONTH) THEN "date_trunc"('day', "line_item_usage_start_date") ELSE "date_trunc"('month', "line_item_usage_start_date") END "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "bill_invoice_id" "invoice_id" - , "line_item_line_item_type" "charge_type" - , CASE - -- WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN 'Running_Usage' - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN 'Running_Usage' - WHEN ("line_item_line_item_type" = 'Usage') THEN 'Running_Usage' ELSE 'non_usage' END "charge_category" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' - WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' - ELSE 'OnDemand' END "purchase_option" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - -- WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" - ELSE CAST('' AS varchar) END "ri_sp_arn" - , "line_item_product_code" "product_code" - , "product_product_name" "product_name" - , CASE - WHEN ("bill_billing_entity" = 'AWS Marketplace' AND "line_item_line_item_type" NOT LIKE '%Discount%') THEN "Product_Product_Name" - WHEN ("product_servicecode" = '') THEN "line_item_product_code" ELSE "product_servicecode" END "service" - , "product_product_family" "product_family" - , "line_item_usage_type" "usage_type" - , "line_item_operation" "operation" - , "line_item_line_item_description" "item_description" - , "line_item_availability_zone" "availability_zone" - , "product_region" "region" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", '.', 1) ELSE "product_instance_type_family" END "instance_type_family" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", ' ', 1) ELSE "product_instance_type" END "instance_type" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("split_part"("line_item_line_item_description", ' ', 2), '/', 1) ELSE "product_operating_system" END "platform" - , "product_tenancy" "tenancy" - , "product_physical_processor" "processor" - , "product_processor_features" "processor_features" - , "product_database_engine" "database_engine" - , "product_group" "product_group" - , "product_from_location" "product_from_location" - , "product_to_location" "product_to_location" - , "product_current_generation" "current_generation" - , "line_item_legal_entity" "legal_entity" - , "bill_billing_entity" "billing_entity" - , "pricing_unit" "pricing_unit" - , "approx_distinct"("Line_item_resource_id") "resource_id_count" - , sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" - -- WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" - WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" ELSE 0 END) "usage_quantity" - , sum ("line_item_unblended_cost") "unblended_cost" - , sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "savings_plan_savings_plan_effective_cost" - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") - WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 - WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - -- WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "reservation_effective_cost" - -- WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") - -- WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 - ELSE "line_item_unblended_cost" END) "amortized_cost" - , sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN (-"savings_plan_amortized_upfront_commitment_for_billing_period") - -- WHEN ("line_item_line_item_type" = 'RIFee') THEN (-"reservation_amortized_upfront_fee_for_billing_period") - ELSE 0 END) "ri_sp_trueup" - , sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN "line_item_unblended_cost" - -- WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN "line_item_unblended_cost" - ELSE 0 END) "ri_sp_upfront_fees" - , sum(CASE - WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost" - FROM - "${cur_table_name}" - WHERE - (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '7' MONTH)) - AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '7' MONTH)) - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,33,34 diff --git a/cid/builtin/core/data/queries/cid/summary_view_sp_ri.sql b/cid/builtin/core/data/queries/cid/summary_view_sp_ri.sql deleted file mode 100644 index a357288f..00000000 --- a/cid/builtin/core/data/queries/cid/summary_view_sp_ri.sql +++ /dev/null @@ -1,80 +0,0 @@ - CREATE OR REPLACE VIEW summary_view AS - SELECT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , CASE - WHEN ("date_trunc"('month',"line_item_usage_start_date")) >= ("date_trunc"('month', current_timestamp) - INTERVAL '3' MONTH) THEN "date_trunc"('day', "line_item_usage_start_date") ELSE "date_trunc"('month', "line_item_usage_start_date") END "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "bill_invoice_id" "invoice_id" - , "line_item_line_item_type" "charge_type" - , CASE - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN 'Running_Usage' - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN 'Running_Usage' - WHEN ("line_item_line_item_type" = 'Usage') THEN 'Running_Usage' ELSE 'non_usage' END "charge_category" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' - WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' - WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' - ELSE 'OnDemand' END "purchase_option" - , CASE - WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN "savings_plan_savings_plan_a_r_n" - WHEN ("reservation_reservation_a_r_n" <> '') THEN "reservation_reservation_a_r_n" ELSE CAST('' AS varchar) END "ri_sp_arn" - , "line_item_product_code" "product_code" - , "product_product_name" "product_name" - , CASE - WHEN ("bill_billing_entity" = 'AWS Marketplace' AND "line_item_line_item_type" NOT LIKE '%Discount%') THEN "Product_Product_Name" - WHEN ("product_servicecode" = '') THEN "line_item_product_code" ELSE "product_servicecode" END "service" - , "product_product_family" "product_family" - , "line_item_usage_type" "usage_type" - , "line_item_operation" "operation" - , "line_item_line_item_description" "item_description" - , "line_item_availability_zone" "availability_zone" - , "product_region" "region" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", '.', 1) ELSE "product_instance_type_family" END "instance_type_family" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("line_item_line_item_description", ' ', 1) ELSE "product_instance_type" END "instance_type" - , CASE - WHEN (("line_item_usage_type" LIKE '%Spot%') AND ("line_item_product_code" = 'AmazonEC2') AND ("line_item_line_item_type" = 'Usage')) THEN "split_part"("split_part"("line_item_line_item_description", ' ', 2), '/', 1) ELSE "product_operating_system" END "platform" - , "product_tenancy" "tenancy" - , "product_physical_processor" "processor" - , "product_processor_features" "processor_features" - , "product_database_engine" "database_engine" - , "product_group" "product_group" - , "product_from_location" "product_from_location" - , "product_to_location" "product_to_location" - , "product_current_generation" "current_generation" - , "line_item_legal_entity" "legal_entity" - , "bill_billing_entity" "billing_entity" - , "pricing_unit" "pricing_unit" - , approx_distinct("Line_item_resource_id") "resource_id_count" - , sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" - WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" ELSE 0 END) "usage_quantity" - , sum ("line_item_unblended_cost") "unblended_cost" - , sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "savings_plan_savings_plan_effective_cost" - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") - WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 - WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "reservation_effective_cost" - WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") - WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE "line_item_unblended_cost" END) "amortized_cost" - , sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN (-"savings_plan_amortized_upfront_commitment_for_billing_period") - WHEN ("line_item_line_item_type" = 'RIFee') THEN (-"reservation_amortized_upfront_fee_for_billing_period") ELSE 0 END) "ri_sp_trueup" - , sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN "line_item_unblended_cost" - WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN "line_item_unblended_cost"ELSE 0 END) "ri_sp_upfront_fees" - , sum(CASE - WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost" - FROM - "${cur_table_name}" - WHERE - (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '7' MONTH)) - AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '7' MONTH)) - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34 diff --git a/cid/builtin/core/data/queries/cudos/hourly_view.sql b/cid/builtin/core/data/queries/cudos/hourly_view.sql index 91a31c46..1ced7c8b 100644 --- a/cid/builtin/core/data/queries/cudos/hourly_view.sql +++ b/cid/builtin/core/data/queries/cudos/hourly_view.sql @@ -1,28 +1,28 @@ CREATE OR REPLACE VIEW hourly_view AS - SELECT DISTINCT - "line_item_product_code" "product_code" - , "product_servicecode" "service" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "line_item_line_item_description" "item_description" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "pricing_term" "pricing_term" - , "bill_billing_period_start_date" "billing_period" - , "line_item_usage_start_date" "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , CAST('' AS varchar) "savings_plan_a_r_n" - , CAST('' AS varchar) "reservation_a_r_n" - , "sum"("line_item_unblended_cost") "unblended_cost" - , CAST(0 AS double) "reservation_effective_cost" - , CAST(0 AS double) "savings_plan_effective_cost" - , "sum"("line_item_usage_amount") "usage_quantity" - FROM - "${cur_table_name}" - WHERE - (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) - AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')) - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + SELECT DISTINCT + "line_item_product_code" "product_code" + , "product_servicecode" "service" + , "line_item_operation" "operation" + , "line_item_line_item_type" "charge_type" + , "line_item_usage_type" "usage_type" + , "line_item_line_item_description" "item_description" + , "pricing_unit" "pricing_unit" + , product['region'] "region" + , "pricing_term" "pricing_term" + , "bill_billing_period_start_date" "billing_period" + , "line_item_usage_start_date" "usage_date" + , "bill_payer_account_id" "payer_account_id" + , "line_item_usage_account_id" "linked_account_id" + , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" + , "reservation_reservation_a_r_n" "reservation_a_r_n" + , "sum"("line_item_unblended_cost") "unblended_cost" + , "sum"("reservation_effective_cost") "reservation_effective_cost" + , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" + , "sum"("line_item_usage_amount") "usage_quantity" + FROM + "${cur2_database}"."${cur2_table_name}" + WHERE + (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) + AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')) + AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) + GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 diff --git a/cid/builtin/core/data/queries/cudos/hourly_view_ri.sql b/cid/builtin/core/data/queries/cudos/hourly_view_ri.sql deleted file mode 100644 index a966e72d..00000000 --- a/cid/builtin/core/data/queries/cudos/hourly_view_ri.sql +++ /dev/null @@ -1,28 +0,0 @@ -CREATE OR REPLACE VIEW hourly_view AS - SELECT DISTINCT - "line_item_product_code" "product_code" - , "product_servicecode" "service" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "line_item_line_item_description" "item_description" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "pricing_term" "pricing_term" - , "bill_billing_period_start_date" "billing_period" - , "line_item_usage_start_date" "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , CAST('' AS varchar) "savings_plan_a_r_n" - , "reservation_reservation_a_r_n" "reservation_a_r_n" - , "sum"("line_item_unblended_cost") "unblended_cost" - , "sum"("reservation_effective_cost") "reservation_effective_cost" - , CAST(0 AS double) "savings_plan_effective_cost" - , "sum"("line_item_usage_amount") "usage_quantity" - FROM - "${cur_table_name}" - WHERE - (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) - AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')) - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 diff --git a/cid/builtin/core/data/queries/cudos/hourly_view_sp.sql b/cid/builtin/core/data/queries/cudos/hourly_view_sp.sql deleted file mode 100644 index ec6cb3f5..00000000 --- a/cid/builtin/core/data/queries/cudos/hourly_view_sp.sql +++ /dev/null @@ -1,28 +0,0 @@ -CREATE OR REPLACE VIEW hourly_view AS - SELECT DISTINCT - "line_item_product_code" "product_code" - , "product_servicecode" "service" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "line_item_line_item_description" "item_description" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "pricing_term" "pricing_term" - , "bill_billing_period_start_date" "billing_period" - , "line_item_usage_start_date" "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" - , CAST('' AS varchar) "reservation_a_r_n" - , "sum"("line_item_unblended_cost") "unblended_cost" - , CAST(0 AS double) "reservation_effective_cost" - , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" - , "sum"("line_item_usage_amount") "usage_quantity" - FROM - "${cur_table_name}" - WHERE - (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) - AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')) - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 diff --git a/cid/builtin/core/data/queries/cudos/hourly_view_sp_ri.sql b/cid/builtin/core/data/queries/cudos/hourly_view_sp_ri.sql deleted file mode 100644 index 4619d09e..00000000 --- a/cid/builtin/core/data/queries/cudos/hourly_view_sp_ri.sql +++ /dev/null @@ -1,28 +0,0 @@ -CREATE OR REPLACE VIEW hourly_view AS - SELECT DISTINCT - "line_item_product_code" "product_code" - , "product_servicecode" "service" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "line_item_line_item_description" "item_description" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "pricing_term" "pricing_term" - , "bill_billing_period_start_date" "billing_period" - , "line_item_usage_start_date" "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" - , "reservation_reservation_a_r_n" "reservation_a_r_n" - , "sum"("line_item_unblended_cost") "unblended_cost" - , "sum"("reservation_effective_cost") "reservation_effective_cost" - , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" - , "sum"("line_item_usage_amount") "usage_quantity" - FROM - "${cur_table_name}" - WHERE - (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) - AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')) - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 diff --git a/cid/builtin/core/data/queries/cudos/resource_view.sql b/cid/builtin/core/data/queries/cudos/resource_view.sql index 8f30847c..19f13551 100644 --- a/cid/builtin/core/data/queries/cudos/resource_view.sql +++ b/cid/builtin/core/data/queries/cudos/resource_view.sql @@ -4,39 +4,39 @@ CREATE OR REPLACE VIEW resource_view AS , "bill_payer_account_id" "payer_account_id" , "line_item_usage_account_id" "linked_account_id" , "bill_billing_entity" "billing_entity" - , "product_product_name" "product_name" + , product['product_name'] "product_name" , "line_item_resource_id" "resource_id" , "line_item_product_code" "product_code" , "line_item_operation" "operation" , "line_item_line_item_type" "charge_type" , "line_item_usage_type" "usage_type" , "pricing_unit" "pricing_unit" - , "product_region" "region" + , product['region'] "region" , "line_item_line_item_description" "item_description" , "line_item_legal_entity" "legal_entity" , "pricing_term" "pricing_term" - , "product_database_engine" "database_engine" - , "product_deployment_option" "product_deployment_option" + , product['database_engine'] "database_engine" + , product['deployment_option'] "product_deployment_option" , "product_from_location" "product_from_location" - , "product_group" "product_group" + , product['group'] "product_group" , "product_instance_type" "instance_type" - , "product_instance_type_family" "instance_type_family" - , "product_operating_system" "platform" + , product['instance_type_family'] "instance_type_family" + , product['operating_system'] "platform" , "product_product_family" "product_family" , "product_servicecode" "service" - , "product_storage" "product_storage" + , product['storage'] "product_storage" , "product_to_location" "product_to_location" - , "product_volume_api_name" "product_volume_api_name" - , CAST('' AS varchar) "reservation_a_r_n" - , CAST('' AS varchar) "savings_plan_a_r_n" - , CAST(0 AS double) savings_plan_effective_cost - , CAST(0 AS double) reservation_effective_cost + , product['volume_api_name'] "product_volume_api_name" + , "reservation_reservation_a_r_n" "reservation_a_r_n" + , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" + , "sum"("savings_plan_savings_plan_effective_cost") "savings_plan_effective_cost" + , "sum"("reservation_effective_cost") "reservation_effective_cost" , "sum"("line_item_usage_amount") "usage_quantity" - , "sum"("line_item_unblended_cost") unblended_cost + , "sum"("line_item_unblended_cost") "unblended_cost" FROM - "${cur_table_name}" - WHERE - (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) - AND (line_item_resource_id <> '') - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) + "${cur2_database}"."${cur2_table_name}" + WHERE + (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) + AND (line_item_resource_id <> '') + AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 diff --git a/cid/builtin/core/data/queries/cudos/resource_view_ri.sql b/cid/builtin/core/data/queries/cudos/resource_view_ri.sql deleted file mode 100644 index a41d938d..00000000 --- a/cid/builtin/core/data/queries/cudos/resource_view_ri.sql +++ /dev/null @@ -1,42 +0,0 @@ -CREATE OR REPLACE VIEW resource_view AS - SELECT DISTINCT - "date_trunc"('day', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "bill_billing_entity" "billing_entity" - , "product_product_name" "product_name" - , "line_item_resource_id" "resource_id" - , "line_item_product_code" "product_code" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "line_item_line_item_description" "item_description" - , "line_item_legal_entity" "legal_entity" - , "pricing_term" "pricing_term" - , "product_database_engine" "database_engine" - , "product_deployment_option" "product_deployment_option" - , "product_from_location" "product_from_location" - , "product_group" "product_group" - , "product_instance_type" "instance_type" - , "product_instance_type_family" "instance_type_family" - , "product_operating_system" "platform" - , "product_product_family" "product_family" - , "product_servicecode" "service" - , "product_storage" "product_storage" - , "product_to_location" "product_to_location" - , "product_volume_api_name" "product_volume_api_name" - , "reservation_reservation_a_r_n" "reservation_a_r_n" - , CAST('' AS varchar) "savings_plan_a_r_n" - , CAST(0 AS double) savings_plan_effective_cost - , "sum"("reservation_effective_cost") reservation_effective_cost - , "sum"("line_item_usage_amount") "usage_quantity" - , "sum"("line_item_unblended_cost") unblended_cost - FROM - "${cur_table_name}" - WHERE - (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) - AND (line_item_resource_id <> '') - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 diff --git a/cid/builtin/core/data/queries/cudos/resource_view_sp.sql b/cid/builtin/core/data/queries/cudos/resource_view_sp.sql deleted file mode 100644 index 9e99ec6b..00000000 --- a/cid/builtin/core/data/queries/cudos/resource_view_sp.sql +++ /dev/null @@ -1,42 +0,0 @@ -CREATE OR REPLACE VIEW resource_view AS - SELECT DISTINCT - "date_trunc"('day', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "bill_billing_entity" "billing_entity" - , "product_product_name" "product_name" - , "line_item_resource_id" "resource_id" - , "line_item_product_code" "product_code" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "line_item_line_item_description" "item_description" - , "line_item_legal_entity" "legal_entity" - , "pricing_term" "pricing_term" - , "product_database_engine" "database_engine" - , "product_deployment_option" "product_deployment_option" - , "product_from_location" "product_from_location" - , "product_group" "product_group" - , "product_instance_type" "instance_type" - , "product_instance_type_family" "instance_type_family" - , "product_operating_system" "platform" - , "product_product_family" "product_family" - , "product_servicecode" "service" - , "product_storage" "product_storage" - , "product_to_location" "product_to_location" - , "product_volume_api_name" "product_volume_api_name" - , CAST('' AS varchar) "reservation_a_r_n" - , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" - , "sum"("savings_plan_savings_plan_effective_cost") savings_plan_effective_cost - , CAST(0 AS double) reservation_effective_cost - , "sum"("line_item_usage_amount") "usage_quantity" - , "sum"("line_item_unblended_cost") unblended_cost - FROM - "${cur_table_name}" - WHERE - (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) - AND (line_item_resource_id <> '') - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 diff --git a/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql b/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql deleted file mode 100644 index d7b7fc7a..00000000 --- a/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql +++ /dev/null @@ -1,42 +0,0 @@ -CREATE OR REPLACE VIEW resource_view AS - SELECT DISTINCT - "date_trunc"('day', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "bill_billing_entity" "billing_entity" - , "product_product_name" "product_name" - , "line_item_resource_id" "resource_id" - , "line_item_product_code" "product_code" - , "line_item_operation" "operation" - , "line_item_line_item_type" "charge_type" - , "line_item_usage_type" "usage_type" - , "pricing_unit" "pricing_unit" - , "product_region" "region" - , "line_item_line_item_description" "item_description" - , "line_item_legal_entity" "legal_entity" - , "pricing_term" "pricing_term" - , "product_database_engine" "database_engine" - , "product_deployment_option" "product_deployment_option" - , "product_from_location" "product_from_location" - , "product_group" "product_group" - , "product_instance_type" "instance_type" - , "product_instance_type_family" "instance_type_family" - , "product_operating_system" "platform" - , "product_product_family" "product_family" - , "product_servicecode" "service" - , "product_storage" "product_storage" - , "product_to_location" "product_to_location" - , "product_volume_api_name" "product_volume_api_name" - , "reservation_reservation_a_r_n" "reservation_a_r_n" - , "savings_plan_savings_plan_a_r_n" "savings_plan_a_r_n" - , "sum"("savings_plan_savings_plan_effective_cost") savings_plan_effective_cost - , "sum"("reservation_effective_cost") reservation_effective_cost - , "sum"("line_item_usage_amount") "usage_quantity" - , "sum"("line_item_unblended_cost") unblended_cost - FROM - "${cur_table_name}" - WHERE - (((current_date - INTERVAL '30' DAY) <= line_item_usage_start_date) - AND (line_item_resource_id <> '') - AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 diff --git a/cid/builtin/core/data/queries/kpi/kpi_ebs_snap_view.sql b/cid/builtin/core/data/queries/kpi/kpi_ebs_snap_view.sql index d4635829..72bcd85b 100644 --- a/cid/builtin/core/data/queries/kpi/kpi_ebs_snap_view.sql +++ b/cid/builtin/core/data/queries/kpi/kpi_ebs_snap_view.sql @@ -1,6 +1,6 @@ /*Replace customer_all in row 20 with your CUR table name */ -CREATE OR REPLACE VIEW kpi_ebs_snap AS - WITH +CREATE OR REPLACE VIEW kpi_ebs_snap AS + WITH -- Step 1: Add mapping view map AS(SELECT * FROM account_map), @@ -8,8 +8,8 @@ CREATE OR REPLACE VIEW kpi_ebs_snap AS -- Step 2: Filter CUR to return all ebs ec2 snapshot usage data snapshot_usage_all_time AS ( SELECT - year - , month + split_part(billing_period, '/', 1) year + , split_part(billing_period, '/', 2) month , bill_billing_period_start_date billing_period , line_item_usage_start_date usage_start_date , bill_payer_account_id payer_account_id @@ -20,7 +20,7 @@ CREATE OR REPLACE VIEW kpi_ebs_snap AS , line_item_unblended_cost , pricing_public_on_demand_cost FROM - "${cur_table_name}" + "${cur2_database}"."${cur2_table_name}" WHERE (((((bill_payer_account_id <> '') AND (line_item_resource_id <> '')) AND (line_item_line_item_type LIKE '%Usage%')) AND (line_item_product_code = 'AmazonEC2')) AND (line_item_usage_type LIKE '%EBS:Snapshot%')) ), diff --git a/cid/builtin/core/data/queries/kpi/kpi_ebs_storage_view.sql b/cid/builtin/core/data/queries/kpi/kpi_ebs_storage_view.sql index bd761f91..42400648 100644 --- a/cid/builtin/core/data/queries/kpi/kpi_ebs_storage_view.sql +++ b/cid/builtin/core/data/queries/kpi/kpi_ebs_storage_view.sql @@ -1,6 +1,6 @@ /*Replace customer_all in row 22 with your CUR table name */ -CREATE OR REPLACE VIEW kpi_ebs_storage_all AS -WITH +CREATE OR REPLACE VIEW kpi_ebs_storage_all AS +WITH -- Step 1: Add mapping view map AS( SELECT * @@ -14,24 +14,24 @@ ebs_all AS ( , line_item_usage_start_date , bill_payer_account_id , line_item_usage_account_id - , line_item_resource_id - , product_volume_api_name + , line_item_resource_id + , product['volume_api_name'] product_volume_api_name , line_item_usage_type , pricing_unit , line_item_unblended_cost , line_item_usage_amount FROM - "${cur_table_name}" - WHERE (line_item_product_code = 'AmazonEC2') AND (line_item_line_item_type = 'Usage') + "${cur2_database}"."${cur2_table_name}" + WHERE (line_item_product_code = 'AmazonEC2') AND (line_item_line_item_type = 'Usage') AND bill_payer_account_id <> '' - AND line_item_usage_account_id <> '' - AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '3' MONTH)) - AND product_volume_api_name <> '' + AND line_item_usage_account_id <> '' + AND (CAST("concat"("billing_period", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '3' MONTH)) + AND product['volume_api_name'] <> '' AND line_item_usage_type NOT LIKE '%Snap%' - AND line_item_usage_type LIKE '%EBS%' + AND line_item_usage_type LIKE '%EBS%' ), --- Step 3: Pivot table so storage types cost and usage into separate columns +-- Step 3: Pivot table so storage types cost and usage into separate columns ebs_spend AS ( SELECT DISTINCT bill_billing_period_start_date AS billing_period @@ -56,13 +56,13 @@ ebs_spend AS ( ), ebs_spend_with_unit_cost AS ( - SELECT + SELECT * , cost_storage_gb_mo/usage_storage_gb_mo AS "current_unit_cost" , CASE WHEN usage_storage_gb_mo <= 150 THEN 'under 150GB-Mo' - WHEN usage_storage_gb_mo > 150 AND usage_storage_gb_mo <= 1000 THEN 'between 150-1000GB-Mo' - ELSE 'over 1000GB-Mo' + WHEN usage_storage_gb_mo > 150 AND usage_storage_gb_mo <= 1000 THEN 'between 150-1000GB-Mo' + ELSE 'over 1000GB-Mo' END AS storage_summary , CASE WHEN volume_api_name <> 'gp2' THEN 0 @@ -72,22 +72,22 @@ ebs_spend_with_unit_cost AS ( END AS gp2_usage_added_iops_mo , CASE WHEN volume_api_name <> 'gp2' THEN 0 - WHEN usage_storage_gb_mo <= 150 THEN 0 + WHEN usage_storage_gb_mo <= 150 THEN 0 ELSE 125 END AS gp2_usage_added_throughput_gibps_mo , cost_storage_gb_mo + cost_iops_mo + cost_throughput_gibps_mo AS ebs_all_cost , CASE WHEN volume_api_name = 'sc1' THEN (cost_iops_mo + cost_throughput_gibps_mo + cost_storage_gb_mo) ELSE 0 - END "ebs_sc1_cost" + END "ebs_sc1_cost" , CASE WHEN volume_api_name = 'st1' THEN (cost_iops_mo + cost_throughput_gibps_mo + cost_storage_gb_mo) ELSE 0 - END "ebs_st1_cost" + END "ebs_st1_cost" , CASE WHEN volume_api_name = 'standard' THEN (cost_iops_mo + cost_throughput_gibps_mo + cost_storage_gb_mo) ELSE 0 - END "ebs_standard_cost" + END "ebs_standard_cost" , CASE WHEN volume_api_name = 'io1' THEN (cost_iops_mo + cost_throughput_gibps_mo + cost_storage_gb_mo) ELSE 0 @@ -95,7 +95,7 @@ ebs_spend_with_unit_cost AS ( , CASE WHEN volume_api_name = 'io2' THEN (cost_iops_mo + cost_throughput_gibps_mo + cost_storage_gb_mo) ELSE 0 - END "ebs_io2_cost" + END "ebs_io2_cost" , CASE WHEN volume_api_name = 'gp2' THEN (cost_iops_mo + cost_throughput_gibps_mo + cost_storage_gb_mo) ELSE 0 @@ -111,7 +111,7 @@ ebs_spend_with_unit_cost AS ( FROM ebs_spend ), -ebs_before_map AS ( +ebs_before_map AS ( SELECT DISTINCT billing_period , payer_account_id @@ -137,16 +137,16 @@ ebs_before_map AS ( - Storage always 20% cheaper - Additional iops per iops-mo is 6% of the cost of 1 gp3 GB-mo - Additional throughput per gibps-mo is 50% of the cost of 1 gp3 GB-mo */ -, sum(CASE +, sum(CASE /*ignore non gp2' */ WHEN volume_api_name = 'gp2' THEN ebs_gp2_cost - - (cost_storage_gb_mo*0.8 + - (cost_storage_gb_mo*0.8 + estimated_gp3_unit_cost * 0.5 * gp2_usage_added_throughput_gibps_mo + estimated_gp3_unit_cost * 0.06 * gp2_usage_added_iops_mo) ELSE 0 END) AS ebs_gp3_potential_savings -FROM - ebs_spend_with_unit_cost +FROM + ebs_spend_with_unit_cost GROUP BY 1, 2, 3, 4, 5, 6) SELECT DISTINCT billing_period @@ -163,13 +163,13 @@ SELECT DISTINCT , gp2_usage_added_throughput_gibps_mo , ebs_all_cost , ebs_sc1_cost - , ebs_st1_cost + , ebs_st1_cost , ebs_standard_cost , ebs_io1_cost , ebs_io2_cost , ebs_gp2_cost , ebs_gp3_cost , ebs_gp3_potential_savings -FROM +FROM ebs_before_map - LEFT JOIN map ON map.account_id = linked_account_id + LEFT JOIN map ON map.account_id = linked_account_id diff --git a/cid/builtin/core/data/queries/kpi/kpi_instance_all_view.sql b/cid/builtin/core/data/queries/kpi/kpi_instance_all_view.sql index 89f16fb8..d5d9f0f1 100644 --- a/cid/builtin/core/data/queries/kpi/kpi_instance_all_view.sql +++ b/cid/builtin/core/data/queries/kpi/kpi_instance_all_view.sql @@ -1,19 +1,19 @@ /*Replace customer_all in row 71 with your CUR table name */ - CREATE OR REPLACE VIEW kpi_instance_all AS - WITH + CREATE OR REPLACE VIEW kpi_instance_all AS + WITH -- Step 1: Add mapping view map AS(SELECT * FROM account_map), - - -- Step 2: Add instance mapping data + + -- Step 2: Add instance mapping data instance_map AS (SELECT * FROM kpi_instance_mapping), - - -- Step 3: Filter CUR to return all usage data + + -- Step 3: Filter CUR to return all usage data cur_all AS (SELECT DISTINCT - "year" - , "month" + split_part("billing_period", '-', 1) "year" + , split_part("billing_period", '-', 2) "month" , "bill_billing_period_start_date" "billing_period" , "date_trunc"('month', "line_item_usage_start_date") "usage_date" , "bill_payer_account_id" "payer_account_id" @@ -22,7 +22,7 @@ , "line_item_line_item_type" "charge_type" , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' ELSE 'OnDemand' END) "purchase_option" , "line_item_product_code" "product_code" - , CASE + , CASE WHEN ("line_item_product_code" in ('AmazonSageMaker','MachineLearningSavingsPlans')) THEN 'Machine Learning' WHEN ("line_item_product_code" in ('AmazonEC2','AmazonECS','AmazonEKS','AWSLambda','ComputeSavingsPlans')) THEN 'Compute' WHEN (("line_item_product_code" = 'AmazonElastiCache')) THEN 'ElastiCache' @@ -30,48 +30,48 @@ WHEN (("line_item_product_code" = 'AmazonRDS')) THEN 'RDS' WHEN (("line_item_product_code" = 'AmazonRedshift')) THEN 'Redshift' WHEN (("line_item_product_code" = 'AmazonDynamoDB') AND (line_item_operation = 'CommittedThroughput')) THEN 'DynamoDB' - ELSE 'Other' END "commit_service_group" - , savings_plan_offering_type "savings_plan_offering_type" - , product_region "region" + ELSE 'Other' END "commit_service_group" + , savings_plan_offering_type "savings_plan_offering_type" + , product['region'] "region" , line_item_operation "operation" , line_item_usage_type "usage_type" , CASE WHEN ("line_item_product_code" in ('AmazonRDS','AmazonElastiCache')) THEN "lower"("split_part"("product_instance_type", '.', 2)) ELSE "lower"("split_part"("product_instance_type", '.', 1)) END "instance_type_family" , "product_instance_type" "instance_type" - , "product_operating_system" "platform" - , "product_tenancy" "tenancy" - , "product_physical_processor" "processor" - , (CASE WHEN (("line_item_line_item_type" LIKE '%Usage%') AND ("product_physical_processor" LIKE '%Graviton%')) THEN 'Graviton' WHEN (("line_item_line_item_type" LIKE '%Usage%') AND ("product_physical_processor" LIKE '%AMD%')) THEN 'AMD' + , product['operating_system'] "platform" + , product['tenancy'] "tenancy" + , product['physical_processor'] "processor" + , (CASE WHEN (("line_item_line_item_type" LIKE '%Usage%') AND (product['physical_processor'] LIKE '%Graviton%')) THEN 'Graviton' WHEN (("line_item_line_item_type" LIKE '%Usage%') AND (product['physical_processor'] LIKE '%AMD%')) THEN 'AMD' WHEN line_item_product_code IN ('AmazonES','AmazonElastiCache') AND (product_instance_type LIKE '%6g%' OR product_instance_type LIKE '%7g%' OR product_instance_type LIKE '%4g%') THEN 'Graviton' - WHEN line_item_product_code IN ('AWSLambda') AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' + WHEN line_item_product_code IN ('AWSLambda') AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' WHEN line_item_usage_type LIKE '%Fargate%' AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' ELSE 'Other' END) "adjusted_processor" - , product_database_engine "database_engine" - , product_deployment_option "deployment_option" - , product_license_model "license_model" - , product_cache_engine "cache_engine" + , product['database_engine'] "database_engine" + , product['deployment_option'] "deployment_option" + , product['license_model'] "license_model" + , product['cache_engine'] "cache_engine" , "sum"("line_item_usage_amount") "usage_quantity" - , "sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN ("savings_plan_savings_plan_effective_cost") - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN (("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment")) + , "sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN ("savings_plan_savings_plan_effective_cost") + WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN (("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment")) WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN ("reservation_effective_cost") + WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN ("reservation_effective_cost") WHEN ("line_item_line_item_type" = 'RIFee') THEN (("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee")) WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE ("line_item_unblended_cost" ) END)) "amortized_cost" - , "sum"((CASE - WHEN ("line_item_usage_type" LIKE '%Spot%' AND "pricing_public_on_demand_cost" > 0) THEN "pricing_public_on_demand_cost" - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN ("pricing_public_on_demand_cost") - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") + , "sum"((CASE + WHEN ("line_item_usage_type" LIKE '%Spot%' AND "pricing_public_on_demand_cost" > 0) THEN "pricing_public_on_demand_cost" + WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN ("pricing_public_on_demand_cost") + WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN ("pricing_public_on_demand_cost") + WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN ("pricing_public_on_demand_cost") WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE ("line_item_unblended_cost" ) END)) "adjusted_amortized_cost" , "sum"("pricing_public_on_demand_cost") "public_cost" - from "${cur_table_name}" - WHERE - (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '3' MONTH) - AND ("bill_payer_account_id" <>'') - AND ("line_item_resource_id" <>'') + FROM "${cur2_database}"."${cur2_table_name}" + WHERE + (CAST("concat"("billing_period", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '3' MONTH) + AND ("bill_payer_account_id" <>'') + AND ("line_item_resource_id" <>'') AND ("product_servicecode" <> 'AWSDataTransfer') AND ("line_item_usage_type" NOT LIKE '%DataXfer%') AND (("line_item_line_item_type" LIKE '%Usage%') OR ("line_item_line_item_type" = 'RIFee') OR ("line_item_line_item_type" = 'SavingsPlanRecurringFee')) @@ -168,7 +168,7 @@ WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '') AND adjusted_processor = 'AMD') THEN (amortized_cost * 1E-1) ELSE 0 END "ec2_graviton_potential_savings" /*Uses 20% savings estimate for intel and 10% for AMD*/ , CASE WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_amd <> '') AND adjusted_processor <> 'AMD') THEN (amortized_cost * 1E-1) ELSE 0 END "ec2_amd_potential_savings" /*Uses 10% savings estimate for intel and 0% for Graviton*/ -/*RDS*/ +/*RDS*/ , CASE WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '')) THEN adjusted_amortized_cost ELSE 0 END "rds_all_cost" , CASE diff --git a/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noRI.sql b/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noRI.sql deleted file mode 100644 index 8fb52719..00000000 --- a/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noRI.sql +++ /dev/null @@ -1,269 +0,0 @@ -/*Replace customer_all in row 67 with your CUR table name */ - CREATE OR REPLACE VIEW kpi_instance_all AS - WITH - -- Step 1: Add mapping view - map AS(SELECT * - FROM account_map), - - -- Step 2: Add instance mapping data - instance_map AS (SELECT * - FROM - kpi_instance_mapping), - - -- Step 3: Filter CUR to return all usage data - cur_all AS (SELECT DISTINCT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , "date_trunc"('month', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "line_item_resource_id" "resource_id" - , "line_item_line_item_type" "charge_type" - , (CASE WHEN ("savings_plan_savings_plan_a_r_n" <> '') THEN 'SavingsPlan' WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' ELSE 'OnDemand' END) "purchase_option" - , "line_item_product_code" "product_code" - , CASE - WHEN ("line_item_product_code" in ('AmazonSageMaker','MachineLearningSavingsPlans')) THEN 'Machine Learning' - WHEN ("line_item_product_code" in ('AmazonEC2','AmazonECS','AmazonEKS','AWSLambda','ComputeSavingsPlans')) THEN 'Compute' - WHEN (("line_item_product_code" = 'AmazonElastiCache')) THEN 'ElastiCache' - WHEN (("line_item_product_code" = 'AmazonES')) THEN 'OpenSearch' - WHEN (("line_item_product_code" = 'AmazonRDS')) THEN 'RDS' - WHEN (("line_item_product_code" = 'AmazonRedshift')) THEN 'Redshift' - WHEN (("line_item_product_code" = 'AmazonDynamoDB') AND (line_item_operation = 'CommittedThroughput')) THEN 'DynamoDB' - ELSE 'Other' END "commit_service_group" - , savings_plan_offering_type "savings_plan_offering_type" - , product_region "region" - , line_item_operation "operation" - , line_item_usage_type "usage_type" - , CASE WHEN ("line_item_product_code" in ('AmazonRDS','AmazonElastiCache')) THEN "lower"("split_part"("product_instance_type", '.', 2)) ELSE "lower"("split_part"("product_instance_type", '.', 1)) END "instance_type_family" - , "product_instance_type" "instance_type" - , "product_operating_system" "platform" - , "product_tenancy" "tenancy" - , "product_physical_processor" "processor" - , (CASE WHEN (("line_item_line_item_type" LIKE '%Usage%') AND ("product_physical_processor" LIKE '%Graviton%')) THEN 'Graviton' WHEN (("line_item_line_item_type" LIKE '%Usage%') AND ("product_physical_processor" LIKE '%AMD%')) THEN 'AMD' - WHEN line_item_product_code IN ('AmazonES','AmazonElastiCache') AND (product_instance_type LIKE '%6g%' OR product_instance_type LIKE '%7g%' OR product_instance_type LIKE '%4g%') THEN 'Graviton' - WHEN line_item_product_code IN ('AWSLambda') AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' - WHEN line_item_usage_type LIKE '%Fargate%' AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' - ELSE 'Other' END) "adjusted_processor" - , product_database_engine "database_engine" - , product_deployment_option "deployment_option" - , product_license_model "license_model" - , product_cache_engine "cache_engine" - , "sum"("line_item_usage_amount") "usage_quantity" - , "sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN ("savings_plan_savings_plan_effective_cost") - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN (("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment")) - WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 - WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - ELSE ("line_item_unblended_cost" ) END)) "amortized_cost" - , "sum"((CASE - WHEN ("line_item_usage_type" LIKE '%Spot%' AND "pricing_public_on_demand_cost" > 0) THEN "pricing_public_on_demand_cost" - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN ("pricing_public_on_demand_cost") - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") - WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 - WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - ELSE ("line_item_unblended_cost" ) END)) "adjusted_amortized_cost" - , "sum"("pricing_public_on_demand_cost") "public_cost" - from "${cur_table_name}" - WHERE - (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '3' MONTH) - AND ("bill_payer_account_id" <>'') - AND ("line_item_resource_id" <>'') - AND ("product_servicecode" <> 'AWSDataTransfer') - AND ("line_item_usage_type" NOT LIKE '%DataXfer%') - AND (("line_item_line_item_type" LIKE '%Usage%') OR ("line_item_line_item_type" = 'RIFee') OR ("line_item_line_item_type" = 'SavingsPlanRecurringFee')) - AND ( - (("line_item_product_code" = 'AmazonEC2') AND ("product_instance_type" <> '') AND ("line_item_operation" LIKE '%RunInstances%')) - OR(("line_item_product_code" = 'AmazonElastiCache') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonES') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonRDS') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonRedshift') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonDynamoDB') AND ("line_item_operation" in ('CommittedThroughput','PayPerRequestThroughput')) AND (("line_item_usage_type" LIKE '%ReadCapacityUnit-Hrs%') or ("line_item_usage_type" LIKE '%WriteCapacityUnit-Hrs%')) AND ("line_item_usage_type" NOT LIKE '%Repl%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-Provisioned-GB-Second%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-GB-Second%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-Provisioned-Concurrency%')) - OR ("line_item_usage_type" LIKE '%Fargate%') - OR (("line_item_product_code" = 'AmazonSageMaker') AND ("product_instance_type" <> '')) - OR ("line_item_product_code" = 'ComputeSavingsPlans') - OR ("line_item_product_code" = 'MachineLearningSavingsPlans') - )) - - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,17,18,19,20,21,22,23,24,25 - ) - SELECT - cur_all.* - , CASE - WHEN (product_code = 'AmazonEC2' AND lower(platform) NOT LIKE '%window%') THEN latest_graviton - WHEN (product_code = 'AmazonRDS' AND database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) THEN latest_graviton - WHEN (product_code = 'AmazonES') THEN latest_graviton - WHEN (product_code = 'AmazonElastiCache') THEN latest_graviton - END "latest_graviton" - , latest_amd - , latest_intel - , generation - , instance_processor - -/*map*/ - , map.* - -/*SageMaker - Should we change sagemaker to machine learning*/ - , CASE - WHEN ("commit_service_group" = 'Machine Learning') THEN "adjusted_amortized_cost" ELSE 0 END "sagemaker_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "sagemaker_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "sagemaker_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'Machine Learning')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "sagemaker_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '') AND ("purchase_option" = 'OnDemand')) THEN ("amortized_cost" * 2E-1) ELSE 0 END "sagemaker_commit_potential_savings" /*Uses 20% savings estimate*/ -/*Compute SavingsPlan*/ - , CASE - WHEN ("commit_service_group" = 'Compute') THEN "adjusted_amortized_cost" ELSE 0 END "compute_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute')) THEN adjusted_amortized_cost ELSE 0 END "compute_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute') AND (purchase_option = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "compute_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'Compute')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "compute_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute') AND ("purchase_option" = 'OnDemand')) THEN ("amortized_cost" * 2E-1) ELSE 0 END "compute_commit_potential_savings" /*Uses 20% savings estimate*/ - -/*EC2*/ - , CASE - WHEN ("product_code" = 'AmazonEC2') THEN adjusted_amortized_cost ELSE 0 END ec2_all_cost - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%')) THEN amortized_cost ELSE 0 END ec2_usage_cost - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option = 'Spot')) THEN adjusted_amortized_cost ELSE 0 END "ec2_spot_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (generation IN ('Previous')) AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN amortized_cost ELSE 0 END "ec2_previous_generation_cost" - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') - AND ((adjusted_processor = 'Graviton') - OR (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> ''))) - THEN amortized_cost ELSE 0 END "ec2_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "ec2_graviton_cost" - , CASE - WHEN adjusted_processor = 'Graviton' THEN 0 - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') - AND ((adjusted_processor = 'AMD') - OR (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'AMD') AND (latest_amd <> ''))) - THEN amortized_cost ELSE 0 END "ec2_amd_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (instance_processor = 'AMD')) THEN amortized_cost ELSE 0 END "ec2_amd_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN (adjusted_amortized_cost * 5.5E-1) ELSE 0 END "ec2_spot_potential_savings" /*Uses 55% savings estimate*/ - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option = 'Spot')) THEN (adjusted_amortized_cost -amortized_cost) ELSE 0 END "ec2_spot_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (generation IN ('Previous')) AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN (amortized_cost * 5E-2) ELSE 0 END "ec2_previous_generation_potential_savings" /*Uses 5% savings estimate*/ - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '') AND adjusted_processor <> 'AMD') THEN (amortized_cost * 2E-1) - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '') AND adjusted_processor = 'AMD') THEN (amortized_cost * 1E-1) ELSE 0 END "ec2_graviton_potential_savings" /*Uses 20% savings estimate for intel and 10% for AMD*/ - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_amd <> '') AND adjusted_processor <> 'AMD') THEN (amortized_cost * 1E-1) ELSE 0 END "ec2_amd_potential_savings" /*Uses 10% savings estimate for intel and 0% for Graviton*/ -/*RDS*/ - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '')) THEN adjusted_amortized_cost ELSE 0 END "rds_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "rds_ondemand_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "rds_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "rds_graviton_cost" - , CASE - WHEN ("charge_type" NOT LIKE '%Usage%') THEN 0 - WHEN ("product_code" <> 'AmazonRDS') THEN 0 - WHEN (adjusted_processor = 'Graviton') THEN 0 - WHEN (latest_graviton = '') THEN 0 - WHEN ((latest_graviton <> '') AND purchase_option = 'OnDemand' AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL'))) THEN (amortized_cost * 1E-1) ELSE 0 END "rds_graviton_potential_savings" /*Uses 10% savings estimate*/ - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonRDS')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "rds_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "rds_commit_potential_savings" /*Uses 20% savings estimate*/ - , (CASE WHEN (((("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS')) AND ("instance_type" <> '')) AND (database_engine IN ('Oracle'))) THEN adjusted_amortized_cost ELSE 0 END) "rds_oracle_cost" - , (CASE WHEN (((("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS')) AND ("instance_type" <> '')) AND (database_engine IN ('SQL Server'))) THEN adjusted_amortized_cost ELSE 0 END) "rds_sql_server_cost" - -/*ElastiCache*/ - , CASE - WHEN ("product_code" = 'AmazonElastiCache') THEN adjusted_amortized_cost ELSE 0 END "elasticache_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "elasticache_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "elasticache_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonElastiCache')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "elasticache_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "elasticache_commit_potential_savings" /*Uses 20% savings estimate*/ - , CASE - WHEN (("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "elasticache_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (instance_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "elasticache_graviton_cost" - , CASE - WHEN (adjusted_processor = 'Graviton') THEN 0 - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN (amortized_cost * 5E-2) ELSE 0 END "elasticache_graviton_potential_savings" /*Uses 5% savings estimate*/ -/*opensearch*/ - , CASE - WHEN ("product_code" = 'AmazonES') THEN adjusted_amortized_cost ELSE 0 END "opensearch_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "opensearch_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "opensearch_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonES')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "opensearch_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "opensearch_commit_potential_savings" /*Uses 20% savings estimate*/ - , CASE - WHEN (("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "opensearch_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "opensearch_graviton_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN 0 - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN (amortized_cost * 5E-2) - ELSE 0 END "opensearch_graviton_potential_savings" /*Uses 5% savings estimate*/ -/*Redshift*/ - , CASE - WHEN ("product_code" = 'AmazonRedshift') THEN adjusted_amortized_cost ELSE 0 END "redshift_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "redshift_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "redshift_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonRedshift')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "redshift_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "redshift_commit_potential_savings" /*Uses 20% savings estimate*/ -/*DynamoDB*/ - , CASE - WHEN ("product_code" = 'AmazonDynamoDB') THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_all_cost" - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_committed_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonDynamoDB')) THEN amortized_cost ELSE 0 END "dynamodb_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') AND ("purchase_option" = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'DynamoDB')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "dynamodb_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "dynamodb_commit_potential_savings" /*Uses 20% savings estimate*/ -/*Lambda*/ - , CASE - WHEN ("product_code" = 'AWSLambda') THEN "adjusted_amortized_cost" ELSE 0 END "lambda_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda')) THEN amortized_cost ELSE 0 END "lambda_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND("product_code" = 'AWSLambda') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda')) THEN amortized_cost ELSE 0 END "lambda_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "lambda_graviton_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda') AND (adjusted_processor <> 'Graviton')) THEN amortized_cost*.2 ELSE 0 END "lambda_graviton_potential_savings" /*Uses 20% savings estimate*/ - - FROM - cur_all cur_all - LEFT JOIN instance_map ON (instance_map.product = product_code AND instance_map.family = instance_type_family) - LEFT JOIN map ON map.account_id= linked_account_id - diff --git a/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noRISP.sql b/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noRISP.sql deleted file mode 100644 index 80a3af9a..00000000 --- a/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noRISP.sql +++ /dev/null @@ -1,262 +0,0 @@ -/*Replace customer_all in row 59 with your CUR table name */ - CREATE OR REPLACE VIEW kpi_instance_all AS - WITH - -- Step 1: Add mapping view - map AS(SELECT * - FROM account_map), - - -- Step 2: Add instance mapping data - instance_map AS (SELECT * - FROM - kpi_instance_mapping), - - -- Step 3: Filter CUR to return all usage data - cur_all AS (SELECT DISTINCT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , "date_trunc"('month', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "line_item_resource_id" "resource_id" - , "line_item_line_item_type" "charge_type" - , (CASE WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' ELSE 'OnDemand' END) "purchase_option" - , "line_item_product_code" "product_code" - , CASE - WHEN ("line_item_product_code" in ('AmazonSageMaker','MachineLearningSavingsPlans')) THEN 'Machine Learning' - WHEN ("line_item_product_code" in ('AmazonEC2','AmazonECS','AmazonEKS','AWSLambda','ComputeSavingsPlans')) THEN 'Compute' - WHEN (("line_item_product_code" = 'AmazonElastiCache')) THEN 'ElastiCache' - WHEN (("line_item_product_code" = 'AmazonES')) THEN 'OpenSearch' - WHEN (("line_item_product_code" = 'AmazonRDS')) THEN 'RDS' - WHEN (("line_item_product_code" = 'AmazonRedshift')) THEN 'Redshift' - WHEN (("line_item_product_code" = 'AmazonDynamoDB') AND (line_item_operation = 'CommittedThroughput')) THEN 'DynamoDB' - ELSE 'Other' END "commit_service_group" - , ' ' "savings_plan_offering_type" - , product_region "region" - , line_item_operation "operation" - , line_item_usage_type "usage_type" - , CASE WHEN ("line_item_product_code" in ('AmazonRDS','AmazonElastiCache')) THEN "lower"("split_part"("product_instance_type", '.', 2)) ELSE "lower"("split_part"("product_instance_type", '.', 1)) END "instance_type_family" - , "product_instance_type" "instance_type" - , "product_operating_system" "platform" - , "product_tenancy" "tenancy" - , "product_physical_processor" "processor" - , (CASE WHEN (("line_item_line_item_type" LIKE '%Usage%') AND ("product_physical_processor" LIKE '%Graviton%')) THEN 'Graviton' WHEN (("line_item_line_item_type" LIKE '%Usage%') AND ("product_physical_processor" LIKE '%AMD%')) THEN 'AMD' - WHEN line_item_product_code IN ('AmazonES','AmazonElastiCache') AND (product_instance_type LIKE '%6g%' OR product_instance_type LIKE '%7g%' OR product_instance_type LIKE '%4g%') THEN 'Graviton' - WHEN line_item_product_code IN ('AWSLambda') AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' - WHEN line_item_usage_type LIKE '%Fargate%' AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' - ELSE 'Other' END) "adjusted_processor" - , product_database_engine "database_engine" - , product_deployment_option "deployment_option" - , product_license_model "license_model" - , product_cache_engine "cache_engine" - , "sum"("line_item_usage_amount") "usage_quantity" - , "sum"("line_item_unblended_cost") "amortized_cost" - , "sum"((CASE - WHEN ("line_item_usage_type" LIKE '%Spot%' AND "pricing_public_on_demand_cost" > 0) THEN "pricing_public_on_demand_cost" - ELSE ("line_item_unblended_cost" ) END)) "adjusted_amortized_cost" - , "sum"("pricing_public_on_demand_cost") "public_cost" - from "${cur_table_name}" - WHERE - (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '3' MONTH) - AND ("bill_payer_account_id" <>'') - AND ("line_item_resource_id" <>'') - AND ("product_servicecode" <> 'AWSDataTransfer') - AND ("line_item_usage_type" NOT LIKE '%DataXfer%') - AND (("line_item_line_item_type" LIKE '%Usage%')) - AND ( - (("line_item_product_code" = 'AmazonEC2') AND ("product_instance_type" <> '') AND ("line_item_operation" LIKE '%RunInstances%')) - OR(("line_item_product_code" = 'AmazonElastiCache') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonES') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonRDS') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonRedshift') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonDynamoDB') AND ("line_item_operation" in ('CommittedThroughput','PayPerRequestThroughput')) AND (("line_item_usage_type" LIKE '%ReadCapacityUnit-Hrs%') or ("line_item_usage_type" LIKE '%WriteCapacityUnit-Hrs%')) AND ("line_item_usage_type" NOT LIKE '%Repl%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-Provisioned-GB-Second%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-GB-Second%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-Provisioned-Concurrency%')) - OR ("line_item_usage_type" LIKE '%Fargate%') - OR (("line_item_product_code" = 'AmazonSageMaker') AND ("product_instance_type" <> '')) - OR ("line_item_product_code" = 'ComputeSavingsPlans') - OR ("line_item_product_code" = 'MachineLearningSavingsPlans') - )) - - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,17,18,19,20,21,22,23,24,25 - ) - SELECT - cur_all.* - , CASE - WHEN (product_code = 'AmazonEC2' AND lower(platform) NOT LIKE '%window%') THEN latest_graviton - WHEN (product_code = 'AmazonRDS' AND database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) THEN latest_graviton - WHEN (product_code = 'AmazonES') THEN latest_graviton - WHEN (product_code = 'AmazonElastiCache') THEN latest_graviton - END "latest_graviton" - , latest_amd - , latest_intel - , generation - , instance_processor - -/*map*/ - , map.* - -/*SageMaker - Should we change sagemaker to machine learning*/ - , CASE - WHEN ("commit_service_group" = 'Machine Learning') THEN "adjusted_amortized_cost" ELSE 0 END "sagemaker_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "sagemaker_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "sagemaker_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'Machine Learning')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "sagemaker_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '') AND ("purchase_option" = 'OnDemand')) THEN ("amortized_cost" * 2E-1) ELSE 0 END "sagemaker_commit_potential_savings" /*Uses 20% savings estimate*/ -/*Compute SavingsPlan*/ - , CASE - WHEN ("commit_service_group" = 'Compute') THEN "adjusted_amortized_cost" ELSE 0 END "compute_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute')) THEN adjusted_amortized_cost ELSE 0 END "compute_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute') AND (purchase_option = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "compute_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'Compute')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "compute_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute') AND ("purchase_option" = 'OnDemand')) THEN ("amortized_cost" * 2E-1) ELSE 0 END "compute_commit_potential_savings" /*Uses 20% savings estimate*/ - -/*EC2*/ - , CASE - WHEN ("product_code" = 'AmazonEC2') THEN adjusted_amortized_cost ELSE 0 END ec2_all_cost - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%')) THEN amortized_cost ELSE 0 END ec2_usage_cost - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option = 'Spot')) THEN adjusted_amortized_cost ELSE 0 END "ec2_spot_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (generation IN ('Previous')) AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN amortized_cost ELSE 0 END "ec2_previous_generation_cost" - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') - AND ((adjusted_processor = 'Graviton') - OR (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> ''))) - THEN amortized_cost ELSE 0 END "ec2_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "ec2_graviton_cost" - , CASE - WHEN adjusted_processor = 'Graviton' THEN 0 - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') - AND ((adjusted_processor = 'AMD') - OR (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'AMD') AND (latest_amd <> ''))) - THEN amortized_cost ELSE 0 END "ec2_amd_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (instance_processor = 'AMD')) THEN amortized_cost ELSE 0 END "ec2_amd_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN (adjusted_amortized_cost * 5.5E-1) ELSE 0 END "ec2_spot_potential_savings" /*Uses 55% savings estimate*/ - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option = 'Spot')) THEN (adjusted_amortized_cost -amortized_cost) ELSE 0 END "ec2_spot_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (generation IN ('Previous')) AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN (amortized_cost * 5E-2) ELSE 0 END "ec2_previous_generation_potential_savings" /*Uses 5% savings estimate*/ - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '') AND adjusted_processor <> 'AMD') THEN (amortized_cost * 2E-1) - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '') AND adjusted_processor = 'AMD') THEN (amortized_cost * 1E-1) ELSE 0 END "ec2_graviton_potential_savings" /*Uses 20% savings estimate for intel and 10% for AMD*/ - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_amd <> '') AND adjusted_processor <> 'AMD') THEN (amortized_cost * 1E-1) ELSE 0 END "ec2_amd_potential_savings" /*Uses 10% savings estimate for intel and 0% for Graviton*/ -/*RDS*/ - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '')) THEN adjusted_amortized_cost ELSE 0 END "rds_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "rds_ondemand_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "rds_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "rds_graviton_cost" - , CASE - WHEN ("charge_type" NOT LIKE '%Usage%') THEN 0 - WHEN ("product_code" <> 'AmazonRDS') THEN 0 - WHEN (adjusted_processor = 'Graviton') THEN 0 - WHEN (latest_graviton = '') THEN 0 - WHEN ((latest_graviton <> '') AND purchase_option = 'OnDemand' AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL'))) THEN (amortized_cost * 1E-1) ELSE 0 END "rds_graviton_potential_savings" /*Uses 10% savings estimate*/ - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonRDS')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "rds_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "rds_commit_potential_savings" /*Uses 20% savings estimate*/ - , (CASE WHEN (((("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS')) AND ("instance_type" <> '')) AND (database_engine IN ('Oracle'))) THEN adjusted_amortized_cost ELSE 0 END) "rds_oracle_cost" - , (CASE WHEN (((("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS')) AND ("instance_type" <> '')) AND (database_engine IN ('SQL Server'))) THEN adjusted_amortized_cost ELSE 0 END) "rds_sql_server_cost" - - -/*ElastiCache*/ - , CASE - WHEN ("product_code" = 'AmazonElastiCache') THEN adjusted_amortized_cost ELSE 0 END "elasticache_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "elasticache_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "elasticache_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonElastiCache')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "elasticache_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "elasticache_commit_potential_savings" /*Uses 20% savings estimate*/ - , CASE - WHEN (("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "elasticache_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (instance_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "elasticache_graviton_cost" - , CASE - WHEN (adjusted_processor = 'Graviton') THEN 0 - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN (amortized_cost * 5E-2) ELSE 0 END "elasticache_graviton_potential_savings" /*Uses 5% savings estimate*/ -/*opensearch*/ - , CASE - WHEN ("product_code" = 'AmazonES') THEN adjusted_amortized_cost ELSE 0 END "opensearch_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "opensearch_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "opensearch_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonES')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "opensearch_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "opensearch_commit_potential_savings" /*Uses 20% savings estimate*/ - , CASE - WHEN (("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "opensearch_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "opensearch_graviton_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN 0 - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN (amortized_cost * 5E-2) - ELSE 0 END "opensearch_graviton_potential_savings" /*Uses 5% savings estimate*/ -/*Redshift*/ - , CASE - WHEN ("product_code" = 'AmazonRedshift') THEN adjusted_amortized_cost ELSE 0 END "redshift_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "redshift_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "redshift_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonRedshift')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "redshift_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "redshift_commit_potential_savings" /*Uses 20% savings estimate*/ -/*DynamoDB*/ - , CASE - WHEN ("product_code" = 'AmazonDynamoDB') THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_all_cost" - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_committed_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonDynamoDB')) THEN amortized_cost ELSE 0 END "dynamodb_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') AND ("purchase_option" = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'DynamoDB')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "dynamodb_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "dynamodb_commit_potential_savings" /*Uses 20% savings estimate*/ -/*Lambda*/ - , CASE - WHEN ("product_code" = 'AWSLambda') THEN "adjusted_amortized_cost" ELSE 0 END "lambda_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda')) THEN amortized_cost ELSE 0 END "lambda_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND("product_code" = 'AWSLambda') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda')) THEN amortized_cost ELSE 0 END "lambda_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "lambda_graviton_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda') AND (adjusted_processor <> 'Graviton')) THEN amortized_cost*.2 ELSE 0 END "lambda_graviton_potential_savings" /*Uses 20% savings estimate*/ - - FROM - cur_all cur_all - LEFT JOIN instance_map ON (instance_map.product = product_code AND instance_map.family = instance_type_family) - LEFT JOIN map ON map.account_id= linked_account_id - diff --git a/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noSP.sql b/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noSP.sql deleted file mode 100644 index 6ffdd3c8..00000000 --- a/cid/builtin/core/data/queries/kpi/kpi_instance_all_view_noSP.sql +++ /dev/null @@ -1,265 +0,0 @@ -/*Replace customer_all in row 62 with your CUR table name */ - CREATE OR REPLACE VIEW kpi_instance_all AS - WITH - -- Step 1: Add mapping view - map AS(SELECT * - FROM account_map), - - -- Step 2: Add instance mapping data - instance_map AS (SELECT * - FROM - kpi_instance_mapping), - - -- Step 3: Filter CUR to return all usage data - cur_all AS (SELECT DISTINCT - "year" - , "month" - , "bill_billing_period_start_date" "billing_period" - , "date_trunc"('month', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "line_item_resource_id" "resource_id" - , "line_item_line_item_type" "charge_type" - , (CASE WHEN ("reservation_reservation_a_r_n" <> '') THEN 'Reserved' WHEN ("line_item_usage_type" LIKE '%Spot%') THEN 'Spot' ELSE 'OnDemand' END) "purchase_option" - , "line_item_product_code" "product_code" - , CASE - WHEN ("line_item_product_code" in ('AmazonSageMaker','MachineLearningSavingsPlans')) THEN 'Machine Learning' - WHEN ("line_item_product_code" in ('AmazonEC2','AmazonECS','AmazonEKS','AWSLambda','ComputeSavingsPlans')) THEN 'Compute' - WHEN (("line_item_product_code" = 'AmazonElastiCache')) THEN 'ElastiCache' - WHEN (("line_item_product_code" = 'AmazonES')) THEN 'OpenSearch' - WHEN (("line_item_product_code" = 'AmazonRDS')) THEN 'RDS' - WHEN (("line_item_product_code" = 'AmazonRedshift')) THEN 'Redshift' - WHEN (("line_item_product_code" = 'AmazonDynamoDB') AND (line_item_operation = 'CommittedThroughput')) THEN 'DynamoDB' - ELSE 'Other' END "commit_service_group" - , ' ' "savings_plan_offering_type" - , product_region "region" - , line_item_operation "operation" - , line_item_usage_type "usage_type" - , CASE WHEN ("line_item_product_code" in ('AmazonRDS','AmazonElastiCache')) THEN "lower"("split_part"("product_instance_type", '.', 2)) ELSE "lower"("split_part"("product_instance_type", '.', 1)) END "instance_type_family" - , "product_instance_type" "instance_type" - , "product_operating_system" "platform" - , "product_tenancy" "tenancy" - , "product_physical_processor" "processor" - , (CASE WHEN (("line_item_line_item_type" LIKE '%Usage%') AND ("product_physical_processor" LIKE '%Graviton%')) THEN 'Graviton' WHEN (("line_item_line_item_type" LIKE '%Usage%') AND ("product_physical_processor" LIKE '%AMD%')) THEN 'AMD' - WHEN line_item_product_code IN ('AmazonES','AmazonElastiCache') AND (product_instance_type LIKE '%6g%' OR product_instance_type LIKE '%7g%' OR product_instance_type LIKE '%4g%') THEN 'Graviton' - WHEN line_item_product_code IN ('AWSLambda') AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' - WHEN line_item_usage_type LIKE '%Fargate%' AND line_item_usage_type LIKE '%ARM%' THEN 'Graviton' - ELSE 'Other' END) "adjusted_processor" - , product_database_engine "database_engine" - , product_deployment_option "deployment_option" - , product_license_model "license_model" - , product_cache_engine "cache_engine" - , "sum"("line_item_usage_amount") "usage_quantity" - , "sum"((CASE WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN ("reservation_effective_cost") - WHEN ("line_item_line_item_type" = 'RIFee') THEN (("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee")) - WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE ("line_item_unblended_cost" ) END)) "amortized_cost" - , "sum"((CASE WHEN ("line_item_usage_type" LIKE '%Spot%' AND "pricing_public_on_demand_cost" > 0) THEN "pricing_public_on_demand_cost" - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN ("pricing_public_on_demand_cost") - WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") - WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE ("line_item_unblended_cost" ) END)) "adjusted_amortized_cost" - , "sum"("pricing_public_on_demand_cost") "public_cost" - from "${cur_table_name}" - WHERE - (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '3' MONTH) - AND ("bill_payer_account_id" <>'') - AND ("line_item_resource_id" <>'') - AND ("product_servicecode" <> 'AWSDataTransfer') - AND ("line_item_usage_type" NOT LIKE '%DataXfer%') - AND (("line_item_line_item_type" LIKE '%Usage%') OR ("line_item_line_item_type" = 'RIFee') OR ("line_item_line_item_type" = 'SavingsPlanRecurringFee')) - AND ( - (("line_item_product_code" = 'AmazonEC2') AND ("product_instance_type" <> '') AND ("line_item_operation" LIKE '%RunInstances%')) - OR(("line_item_product_code" = 'AmazonElastiCache') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonES') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonRDS') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonRedshift') AND ("product_instance_type" <> '')) - OR (("line_item_product_code" = 'AmazonDynamoDB') AND ("line_item_operation" in ('CommittedThroughput','PayPerRequestThroughput')) AND (("line_item_usage_type" LIKE '%ReadCapacityUnit-Hrs%') or ("line_item_usage_type" LIKE '%WriteCapacityUnit-Hrs%')) AND ("line_item_usage_type" NOT LIKE '%Repl%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-Provisioned-GB-Second%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-GB-Second%')) - OR (("line_item_product_code" = 'AWSLambda') AND ("line_item_usage_type" LIKE '%Lambda-Provisioned-Concurrency%')) - OR ("line_item_usage_type" LIKE '%Fargate%') - OR (("line_item_product_code" = 'AmazonSageMaker') AND ("product_instance_type" <> '')) - OR ("line_item_product_code" = 'ComputeSavingsPlans') - OR ("line_item_product_code" = 'MachineLearningSavingsPlans') - )) - - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,17,18,19,20,21,22,23,24,25 - ) - SELECT - cur_all.* - , CASE - WHEN (product_code = 'AmazonEC2' AND lower(platform) NOT LIKE '%window%') THEN latest_graviton - WHEN (product_code = 'AmazonRDS' AND database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) THEN latest_graviton - WHEN (product_code = 'AmazonES') THEN latest_graviton - WHEN (product_code = 'AmazonElastiCache') THEN latest_graviton - END "latest_graviton" - , latest_amd - , latest_intel - , generation - , instance_processor - -/*map*/ - , map.* - -/*SageMaker - Should we change sagemaker to machine learning*/ - , CASE - WHEN ("commit_service_group" = 'Machine Learning') THEN "adjusted_amortized_cost" ELSE 0 END "sagemaker_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "sagemaker_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "sagemaker_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'Machine Learning')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "sagemaker_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Machine Learning') AND ("instance_type" <> '') AND ("purchase_option" = 'OnDemand')) THEN ("amortized_cost" * 2E-1) ELSE 0 END "sagemaker_commit_potential_savings" /*Uses 20% savings estimate*/ -/*Compute SavingsPlan*/ - , CASE - WHEN ("commit_service_group" = 'Compute') THEN "adjusted_amortized_cost" ELSE 0 END "compute_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute')) THEN adjusted_amortized_cost ELSE 0 END "compute_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute') AND (purchase_option = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "compute_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'Compute')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "compute_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'Compute') AND ("purchase_option" = 'OnDemand')) THEN ("amortized_cost" * 2E-1) ELSE 0 END "compute_commit_potential_savings" /*Uses 20% savings estimate*/ - -/*EC2*/ - , CASE - WHEN ("product_code" = 'AmazonEC2') THEN adjusted_amortized_cost ELSE 0 END ec2_all_cost - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%')) THEN amortized_cost ELSE 0 END ec2_usage_cost - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option = 'Spot')) THEN adjusted_amortized_cost ELSE 0 END "ec2_spot_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (generation IN ('Previous')) AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN amortized_cost ELSE 0 END "ec2_previous_generation_cost" - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') - AND ((adjusted_processor = 'Graviton') - OR (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> ''))) - THEN amortized_cost ELSE 0 END "ec2_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "ec2_graviton_cost" - , CASE - WHEN adjusted_processor = 'Graviton' THEN 0 - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') - AND ((adjusted_processor = 'AMD') - OR (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'AMD') AND (latest_amd <> ''))) - THEN amortized_cost ELSE 0 END "ec2_amd_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (instance_processor = 'AMD')) THEN amortized_cost ELSE 0 END "ec2_amd_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN (adjusted_amortized_cost * 5.5E-1) ELSE 0 END "ec2_spot_potential_savings" /*Uses 55% savings estimate*/ - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (purchase_option = 'Spot')) THEN (adjusted_amortized_cost -amortized_cost) ELSE 0 END "ec2_spot_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (generation IN ('Previous')) AND (purchase_option <> 'Spot') AND (purchase_option <> 'Reserved') AND (savings_plan_offering_type NOT LIKE '%EC2%')) THEN (amortized_cost * 5E-2) ELSE 0 END "ec2_previous_generation_potential_savings" /*Uses 5% savings estimate*/ - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '') AND adjusted_processor <> 'AMD') THEN (amortized_cost * 2E-1) - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (lower(platform) NOT LIKE '%window%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '') AND adjusted_processor = 'AMD') THEN (amortized_cost * 1E-1) ELSE 0 END "ec2_graviton_potential_savings" /*Uses 20% savings estimate for intel and 10% for AMD*/ - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonEC2') AND ("instance_type" <> '') AND ("operation" LIKE '%RunInstances%') AND (((purchase_option = 'OnDemand') OR (savings_plan_offering_type = 'ComputeSavingsPlans')) AND (adjusted_processor <> 'Graviton') AND (latest_amd <> '') AND adjusted_processor <> 'AMD') THEN (amortized_cost * 1E-1) ELSE 0 END "ec2_amd_potential_savings" /*Uses 10% savings estimate for intel and 0% for Graviton*/ -/*RDS*/ - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '')) THEN adjusted_amortized_cost ELSE 0 END "rds_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "rds_ondemand_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) AND (adjusted_processor <> 'Graviton') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "rds_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL')) AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "rds_graviton_cost" - , CASE - WHEN ("charge_type" NOT LIKE '%Usage%') THEN 0 - WHEN ("product_code" <> 'AmazonRDS') THEN 0 - WHEN (adjusted_processor = 'Graviton') THEN 0 - WHEN (latest_graviton = '') THEN 0 - WHEN ((latest_graviton <> '') AND purchase_option = 'OnDemand' AND (database_engine in ('Aurora MySQL','Aurora PostgreSQL','MariaDB','PostgreSQL','MySQL'))) THEN (amortized_cost * 1E-1) ELSE 0 END "rds_graviton_potential_savings" /*Uses 10% savings estimate*/ - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonRDS')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "rds_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "rds_commit_potential_savings" /*Uses 20% savings estimate*/ - , (CASE WHEN (((("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS')) AND ("instance_type" <> '')) AND (database_engine IN ('Oracle'))) THEN adjusted_amortized_cost ELSE 0 END) "rds_oracle_cost" - , (CASE WHEN (((("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRDS')) AND ("instance_type" <> '')) AND (database_engine IN ('SQL Server'))) THEN adjusted_amortized_cost ELSE 0 END) "rds_sql_server_cost" - - -/*ElastiCache*/ - , CASE - WHEN ("product_code" = 'AmazonElastiCache') THEN adjusted_amortized_cost ELSE 0 END "elasticache_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "elasticache_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "elasticache_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonElastiCache')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "elasticache_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "elasticache_commit_potential_savings" /*Uses 20% savings estimate*/ - , CASE - WHEN (("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "elasticache_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (instance_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "elasticache_graviton_cost" - , CASE - WHEN (adjusted_processor = 'Graviton') THEN 0 - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonElastiCache') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN (amortized_cost * 5E-2) ELSE 0 END "elasticache_graviton_potential_savings" /*Uses 5% savings estimate*/ -/*opensearch*/ - , CASE - WHEN ("product_code" = 'AmazonES') THEN adjusted_amortized_cost ELSE 0 END "opensearch_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "opensearch_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "opensearch_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonES')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "opensearch_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "opensearch_commit_potential_savings" /*Uses 20% savings estimate*/ - , CASE - WHEN (("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN amortized_cost ELSE 0 END "opensearch_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "opensearch_graviton_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (adjusted_processor = 'Graviton')) THEN 0 - WHEN (("charge_type" = 'Usage') AND ("product_code" = 'AmazonES') AND ("instance_type" <> '') AND (latest_graviton <> '')) THEN (amortized_cost * 5E-2) - ELSE 0 END "opensearch_graviton_potential_savings" /*Uses 5% savings estimate*/ -/*Redshift*/ - , CASE - WHEN ("product_code" = 'AmazonRedshift') THEN adjusted_amortized_cost ELSE 0 END "redshift_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '')) THEN amortized_cost ELSE 0 END "redshift_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN adjusted_amortized_cost ELSE 0 END "redshift_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("product_code" = 'AmazonRedshift')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "redshift_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonRedshift') AND ("instance_type" <> '') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "redshift_commit_potential_savings" /*Uses 20% savings estimate*/ -/*DynamoDB*/ - , CASE - WHEN ("product_code" = 'AmazonDynamoDB') THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_all_cost" - , CASE - WHEN ("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_committed_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AmazonDynamoDB')) THEN amortized_cost ELSE 0 END "dynamodb_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') AND ("purchase_option" = 'OnDemand')) THEN "adjusted_amortized_cost" ELSE 0 END "dynamodb_ondemand_cost" - , CASE - WHEN (("purchase_option" in ('Reserved','SavingsPlan')) AND ("commit_service_group" = 'DynamoDB')) THEN ("adjusted_amortized_cost" - "amortized_cost") ELSE 0 END "dynamodb_commit_savings" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("commit_service_group" = 'DynamoDB') AND (purchase_option = 'OnDemand')) THEN (amortized_cost * 2E-1) ELSE 0 END "dynamodb_commit_potential_savings" /*Uses 20% savings estimate*/ -/*Lambda*/ - , CASE - WHEN ("product_code" = 'AWSLambda') THEN "adjusted_amortized_cost" ELSE 0 END "lambda_all_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda')) THEN amortized_cost ELSE 0 END "lambda_usage_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND("product_code" = 'AWSLambda') AND (adjusted_processor = 'Graviton')) THEN amortized_cost - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda')) THEN amortized_cost ELSE 0 END "lambda_graviton_eligible_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda') AND (adjusted_processor = 'Graviton')) THEN amortized_cost ELSE 0 END "lambda_graviton_cost" - , CASE - WHEN (("charge_type" LIKE '%Usage%') AND ("product_code" = 'AWSLambda') AND (adjusted_processor <> 'Graviton')) THEN amortized_cost*.2 ELSE 0 END "lambda_graviton_potential_savings" /*Uses 20% savings estimate*/ - - FROM - cur_all cur_all - LEFT JOIN instance_map ON (instance_map.product = product_code AND instance_map.family = instance_type_family) - LEFT JOIN map ON map.account_id= linked_account_id - diff --git a/cid/builtin/core/data/queries/kpi/kpi_s3_storage_view.sql b/cid/builtin/core/data/queries/kpi/kpi_s3_storage_view.sql index 3e51b257..f304faa7 100644 --- a/cid/builtin/core/data/queries/kpi/kpi_s3_storage_view.sql +++ b/cid/builtin/core/data/queries/kpi/kpi_s3_storage_view.sql @@ -1,19 +1,23 @@ /*Replace customer_all in row 42 with your CUR table name */ - CREATE OR REPLACE VIEW kpi_s3_storage_all AS - -- Step 1: Enter S3 standard savings savings assumption. Default is set to 0.3 for 30% savings + CREATE OR REPLACE VIEW kpi_s3_storage_all AS + -- Step 1: Enter S3 standard savings savings assumption. Default is set to 0.3 for 30% savings WITH inputs AS ( - SELECT * FROM (VALUES (0.3)) t(s3_standard_savings)), - + SELECT * FROM ( + VALUES (0.3) + ) t(s3_standard_savings) + ), + -- Step: 2 Add mapping view - map AS(SELECT - * - FROM account_map), + map AS( + SELECT * + FROM account_map + ), -- Step 3: Filter CUR to return all storage usage data s3_usage_all_time AS ( SELECT - year - , month + split_part("billing_period", '-', 1) "year" + , split_part("billing_period", '-', 2) "month" , bill_billing_period_start_date AS billing_period , line_item_usage_start_date AS usage_start_date , bill_payer_account_id AS payer_account_id @@ -22,25 +26,23 @@ , s3_standard_savings , line_item_operation AS operation , line_item_usage_type AS usage_type - , CASE - WHEN line_item_usage_type LIKE '%EarlyDelete%' THEN 'EarlyDelete' ELSE line_item_operation END "early_delete_adjusted_operation" + , CASE + WHEN line_item_usage_type LIKE '%EarlyDelete%' THEN 'EarlyDelete' ELSE line_item_operation END "early_delete_adjusted_operation" , CASE WHEN line_item_product_code = 'AmazonGlacier' AND line_item_operation = 'Storage' THEN 'Amazon Glacier' - - WHEN line_item_product_code = 'AmazonS3' AND product_volume_type LIKE '%Intelligent%' AND line_item_operation LIKE '%IntelligentTiering%' THEN 'Intelligent-Tiering' - ELSE product_volume_type + WHEN line_item_product_code = 'AmazonS3' AND product['volume_type'] LIKE '%Intelligent%' AND line_item_operation LIKE '%IntelligentTiering%' THEN 'Intelligent-Tiering' + ELSE product['volume_type'] END AS storage_class_type - , pricing_unit + , pricing_unit , sum(line_item_usage_amount) AS usage_quantity , sum(line_item_unblended_cost) unblended_cost , sum(CASE - WHEN (pricing_unit = 'GB-Mo' AND line_item_operation like '%Storage%' AND product_volume_type LIKE '%Glacier Deep Archive%') THEN line_item_unblended_cost + WHEN (pricing_unit = 'GB-Mo' AND line_item_operation like '%Storage%' AND product['volume_type'] LIKE '%Glacier Deep Archive%') THEN line_item_unblended_cost WHEN (pricing_unit = 'GB-Mo' AND line_item_operation like '%Storage%') THEN line_item_unblended_cost ELSE 0 END) AS s3_all_storage_cost , sum(CASE WHEN (pricing_unit = 'GB-Mo' AND line_item_operation like '%Storage%') THEN line_item_usage_amount ELSE 0 END) AS s3_all_storage_usage_quantity - FROM - "${cur_table_name}" + FROM "${cur2_database}"."${cur2_table_name}" , inputs WHERE bill_payer_account_id <> '' AND line_item_resource_id <> '' diff --git a/cid/builtin/core/data/queries/kpi/last_kpi_tracker_view.sql b/cid/builtin/core/data/queries/kpi/last_kpi_tracker_view.sql index cfbbe7b7..25fb84e4 100644 --- a/cid/builtin/core/data/queries/kpi/last_kpi_tracker_view.sql +++ b/cid/builtin/core/data/queries/kpi/last_kpi_tracker_view.sql @@ -1,10 +1,10 @@ CREATE OR REPLACE VIEW kpi_tracker AS SELECT DISTINCT -spend_all.billing_period + spend_all.billing_period , spend_all.payer_account_id , spend_all.linked_account_id +, spend_all.spend_all_cost , account_map.* -, spend_all.spend_all_cost , instance_all.ec2_all_cost , instance_all.ec2_spot_cost , instance_all.ec2_spot_potential_savings diff --git a/cid/builtin/core/data/queries/shared/account_map.sql b/cid/builtin/core/data/queries/shared/account_map.sql index 2d056b50..0b840285 100644 --- a/cid/builtin/core/data/queries/shared/account_map.sql +++ b/cid/builtin/core/data/queries/shared/account_map.sql @@ -1,10 +1,10 @@ CREATE OR REPLACE VIEW account_map AS -SELECT +SELECT ${account_id} AS account_id, - MAX_BY(${account_name}, concat(${account_name}, ': ', ${account_id})) AS account_name -FROM - ${metadata_table_name} -WHERE + MAX_BY(${account_name}, ${account_id}) AS account_name +FROM + "${metadata_database_name}"."${metadata_table_name}" +WHERE ${account_name} <> '' -GROUP BY +GROUP BY ${account_id} diff --git a/cid/builtin/core/data/queries/shared/account_map_cur2.sql b/cid/builtin/core/data/queries/shared/account_map_cur2.sql new file mode 100644 index 00000000..22102652 --- /dev/null +++ b/cid/builtin/core/data/queries/shared/account_map_cur2.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE VIEW ${athena_view_name} AS +SELECT DISTINCT + line_item_usage_account_id account_id, + MAX_BY(line_item_usage_account_name, line_item_usage_start_date) account_name, + MAX_BY(bill_payer_account_id, line_item_usage_start_date) parent_account_id, + MAX_BY(bill_payer_account_name, line_item_usage_start_date) parent_account_name +FROM + "${cur_database}"."${cur_table_name}" +GROUP BY + line_item_usage_account_id \ No newline at end of file diff --git a/cid/builtin/core/data/queries/shared/account_map_dummy.sql b/cid/builtin/core/data/queries/shared/account_map_dummy.sql index 54e62769..356f0101 100644 --- a/cid/builtin/core/data/queries/shared/account_map_dummy.sql +++ b/cid/builtin/core/data/queries/shared/account_map_dummy.sql @@ -5,6 +5,6 @@ SELECT DISTINCT MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_name, MAX_BY(line_item_usage_account_id, line_item_usage_start_date) account_email_id FROM - "${cur_table_name}" + "${cur_database}"."${cur_table_name}" GROUP BY line_item_usage_account_id diff --git a/cid/builtin/core/data/queries/shared/aws_accounts.sql b/cid/builtin/core/data/queries/shared/aws_accounts.sql index 0511c7c3..6a8a4e29 100644 --- a/cid/builtin/core/data/queries/shared/aws_accounts.sql +++ b/cid/builtin/core/data/queries/shared/aws_accounts.sql @@ -1,16 +1,21 @@ -CREATE OR REPLACE VIEW aws_accounts AS WITH +CREATE OR REPLACE VIEW aws_accounts AS +WITH m AS ( - SELECT ${account_id} as account_id, + SELECT + ${account_id} as account_id, ${account_name} as account_name, email account_email_id - FROM ${metadata_table_name} + FROM + "${metadata_database_name}"."${metadata_table_name}" ), cur AS ( - SELECT DISTINCT line_item_usage_account_id, + SELECT DISTINCT + line_item_usage_account_id, bill_payer_account_id parent_account_id - FROM "${cur_table_name}" + FROM "${cur_database}"."${cur_table_name}" ) -SELECT m.account_id, +SELECT + m.account_id, m.account_name, cur.parent_account_id, m.account_email_id, diff --git a/cid/builtin/core/data/queries/shared/customer_all_unlimited.sql b/cid/builtin/core/data/queries/shared/customer_all_unlimited.sql new file mode 100644 index 00000000..72bef27e --- /dev/null +++ b/cid/builtin/core/data/queries/shared/customer_all_unlimited.sql @@ -0,0 +1,204 @@ +CREATE OR REPLACE VIEW "customer_all_unlimited" AS +SELECT + split_part("billing_period", '-', 1) "year" + , split_part("billing_period", '-', 2) "month" + , "product_updates" + , "bill_billing_period_end_date" + , product['enhanced_networking_supported'] "product_enhanced_networking_supported" + , "product_maximum_storage_volume" + , product['operating_system'] "product_operating_system" + , product['storage'] "product_storage" + , "product_content_type" + , product['finding_storage'] "product_finding_storage" + , "line_item_line_item_description" + , "reservation_amortized_upfront_fee_for_billing_period" + , "line_item_normalization_factor" + , "reservation_start_time" + , "product_maximum_extended_storage" + , "savings_plan_used_commitment" + , "line_item_blended_cost" + , "product_free_trial" + , "product_input_mode" + , "product_gpu" + , product['physical_processor'] "product_physical_processor" + , "product_directory_type_description" + , "product_from_location" + , product['product_name'] "product_product_name" + , "product_describes" + , product['memory'] "product_memory" + , product['capacitystatus'] "product_capacitystatus" + , "product_bundle_group" + , "reservation_recurring_fee_for_usage" + , product['normalization_size_factor'] "product_normalization_size_factor" + , "pricing_public_on_demand_rate" + , "line_item_resource_id" + , "pricing_lease_contract_length" + , "product_origin" + , "line_item_legal_entity" + , "product_directory_type" + , product['deployment_option'] "product_deployment_option" + , "product_resource_type" + , product['category'] "product_category" + , product['edition'] "product_edition" + , "product_directory_size" + , "line_item_tax_type" + , "product_instance_name" + , product['storage_class'] "product_storage_class" + , "product_standard_storage_retention_included" + , "product_dominantnondominant" + , "product_datatransferout" + , "product_compute_type" + , "product_physical_cpu" + , "product_ops_items" + , "reservation_effective_cost" + , product['volume_api_name'] "product_volume_api_name" + , "product_fee_description" + , product['request_type'] "product_request_type" + , product['database_engine'] "product_database_engine" + , "savings_plan_savings_plan_a_r_n" + , product['queue_type'] "product_queue_type" + , "product_software_included" + , "product_request_description" + , "line_item_availability_zone" + , "bill_billing_entity" + , "product_bundle_description" + , "line_item_usage_account_id" + , product['processor_features'] "product_processor_features" + , product['vcpu'] "product_vcpu" + , "product_enhanced_networking_support" + , "product_component" + , product['region'] "product_region" + , product['finding_source'] "product_finding_source" + , "line_item_usage_type" + , product['intel_avx_available'] "product_intel_avx_available" + , product['dedicated_ebs_throughput'] "product_dedicated_ebs_throughput" + , product['standard_storage'] "product_standard_storage" + , "product_sku" + , "product_free_usage_included" + , product['max_iopsvolume'] "product_max_iopsvolume" + , "identity_line_item_id" + , "product_routing_target" + , "line_item_line_item_type" + , "reservation_total_reserved_units" + , product['group'] "product_group" + , "line_item_currency_code" + , product['servicename'] "product_servicename" + , "pricing_term" + , "savings_plan_amortized_upfront_commitment_for_billing_period" + , "savings_plan_savings_plan_effective_cost" + , "reservation_end_time" + , product['type'] "product_type" + , "product_client_location" + , product['max_throughputvolume'] "product_max_throughputvolume" + , product['memory_gib'] "product_memory_gib" + , "product_to_location" + , "reservation_unused_quantity" + , "product_granularity" + , "product_ratetype" + , product['clock_speed'] "product_clock_speed" + , "pricing_public_on_demand_cost" + , "reservation_normalized_units_per_reservation" + , "product_instances" + , "product_broker_engine" + , "product_product_family" + , product['availability_zone'] "product_availability_zone" + , product['description'] "product_description" + , product['processor_architecture'] "product_processor_architecture" + , "line_item_blended_rate" + , "product_rootvolume" + , "product_endpoint_type" + , "product_routing_type" + , "pricing_rate_id" + , product['max_iops_burst_performance'] "product_max_iops_burst_performance" + , product['transfer_type'] "product_transfer_type" + , "product_fee_code" + , "reservation_upfront_value" + , product['gpu_memory'] "product_gpu_memory" + , "line_item_operation" + , "line_item_unblended_cost" + , "reservation_number_of_reservations" + , "savings_plan_savings_plan_rate" + , "bill_payer_account_id" + , product['memorytype'] "product_memorytype" + , product['logs_destination'] "product_logs_destination" + , "reservation_unused_amortized_upfront_fee_for_billing_period" + , "product_pricing_unit" + , product['network_performance'] "product_network_performance" + , "product_servicecode" + , product['group_description'] "product_group_description" + , "product_steps" + , "reservation_modification_status" + , "reservation_total_reserved_normalized_units" + , "product_compute_family" + , product['message_delivery_frequency'] "product_message_delivery_frequency" + , product['intel_turbo_available'] "product_intel_turbo_available" + , "reservation_unused_normalized_unit_quantity" + , "reservation_unused_recurring_fee" + , "product_traffic_direction" + , product['availability'] "product_availability" + , product['ecu'] "product_ecu" + , "product_insightstype" + , "product_gets" + , product['intel_avx2_available'] "product_intel_avx2_available" + , "reservation_units_per_reservation" + , "pricing_purchase_option" + , "reservation_amortized_upfront_cost_for_usage" + , "line_item_usage_amount" + , "product_instance_type" + , product['standard_group'] "product_standard_group" + , product['event_type'] "product_event_type" + , "line_item_usage_end_date" + , product['pre_installed_sw'] "product_pre_installed_sw" + , product['alarm_type'] "product_alarm_type" + , "product_location" + , product['current_generation'] "product_current_generation" + , "savings_plan_total_commitment_to_date" + , "bill_bill_type" + , "product_output_mode" + , "bill_billing_period_start_date" + , "product_operation" + , product['max_volume_size'] "product_max_volume_size" + , "product_running_mode" + , "line_item_normalized_usage_amount" + , "product_license" + , "savings_plan_recurring_commitment_for_billing_period" + , "line_item_usage_start_date" + , "product_physical_gpu" + , "product_instance_family" + , "product_instance" + , product['engine_code'] "product_engine_code" + , "bill_invoice_id" + , "product_storage_type" + , "product_recipient" + , product['storage_media'] "product_storage_media" + , "identity_time_interval" + , product['cache_engine'] "product_cache_engine" + , product['volume_type'] "product_volume_type" + , "pricing_offering_class" + , "product_provisioned" + , product['license_model'] "product_license_model" + , product['instance_type_family'] "product_instance_type_family" + , "line_item_product_code" + , "product_location_type" + , "product_usagetype" + , product['subscription_type'] "product_subscription_type" + , "line_item_unblended_rate" + , "product_bundle" + , product['message_delivery_order'] "product_message_delivery_order" + , "reservation_subscription_id" + , "product_cputype" + , product['durability'] "product_durability" + , "reservation_reservation_a_r_n" + , "product_uservolume" + , "product_from_location_type" + , "product_to_location_type" + , product['version'] "product_version" + , "pricing_currency" + , "product_minimum_storage_volume" + , product['tenancy'] "product_tenancy" + , product['finding_group'] "product_finding_group" + , product['min_volume_size'] "product_min_volume_size" + , product['free_query_types'] "product_free_query_types" + , "pricing_unit" +FROM + "${cur2_database}"."${cur2_table_name}" diff --git a/cid/builtin/core/data/queries/shared/payer_account_name_map.sql b/cid/builtin/core/data/queries/shared/payer_account_name_map.sql index 687f955d..6e5132eb 100644 --- a/cid/builtin/core/data/queries/shared/payer_account_name_map.sql +++ b/cid/builtin/core/data/queries/shared/payer_account_name_map.sql @@ -4,8 +4,4 @@ SELECT , "account_name" "payer_account_name" FROM aws_accounts -WHERE ("account_id" IN (SELECT "parent_account_id" -FROM - aws_accounts -WHERE ("parent_account_id" = "account_id") -)) \ No newline at end of file +WHERE ("parent_account_id" = "account_id") \ No newline at end of file diff --git a/cid/builtin/core/data/queries/trends/daily_anomaly_detection.sql b/cid/builtin/core/data/queries/trends/daily_anomaly_detection.sql index cfe8435d..79587278 100644 --- a/cid/builtin/core/data/queries/trends/daily_anomaly_detection.sql +++ b/cid/builtin/core/data/queries/trends/daily_anomaly_detection.sql @@ -3,11 +3,11 @@ SELECT "line_item_usage_start_date" , "line_item_usage_account_id" , "account_name" -, (CASE WHEN ("product_product_name" LIKE '') THEN "line_item_product_code" ELSE "product_product_name" END) "product_product_name" +, (CASE WHEN (product['product_name'] LIKE '') THEN "line_item_product_code" ELSE product['product_name'] END) "product_product_name" , "round"("sum"("line_item_unblended_cost"), 2) "unblended_cost" , "round"("sum"("line_item_usage_amount"), 2) "line_item_usage_amount" FROM - ("${cur_table_name}" + ("${cur2_database}"."${cur2_table_name}" LEFT JOIN aws_accounts ON ("line_item_usage_account_id" = "account_id")) WHERE ("date_diff"('day', "date"("line_item_usage_start_date"), "date"("now"())) <= 110) GROUP BY "line_item_usage_start_date", "line_item_usage_account_id", "account_name", 4 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/trends/monthly_anomaly_detection.sql b/cid/builtin/core/data/queries/trends/monthly_anomaly_detection.sql index 6c7388fb..d115ef78 100644 --- a/cid/builtin/core/data/queries/trends/monthly_anomaly_detection.sql +++ b/cid/builtin/core/data/queries/trends/monthly_anomaly_detection.sql @@ -3,11 +3,14 @@ SELECT "bill_billing_period_start_date" , "line_item_usage_account_id" , "account_name" -, (CASE WHEN ("product_product_name" LIKE '') THEN "line_item_product_code" ELSE "product_product_name" END) "product_product_name" +, (CASE + WHEN (product['product_name'] LIKE '') THEN "line_item_product_code" + ELSE product['product_name'] + END) "product_product_name" , "round"("sum"("line_item_unblended_cost"), 2) "unblended_cost" , "round"("sum"("line_item_usage_amount"), 2) "line_item_usage_amount" FROM - ("${cur_table_name}" + ("${cur2_database}"."${cur2_table_name}" LEFT JOIN aws_accounts ON ("line_item_usage_account_id" = "account_id")) WHERE ("date_diff"('month', "date"("bill_billing_period_start_date"), "date"("now"())) <= 20) GROUP BY "bill_billing_period_start_date", "line_item_usage_account_id", "account_name", 4 \ No newline at end of file diff --git a/cid/builtin/core/data/queries/trends/monthly_bill_by_account.sql b/cid/builtin/core/data/queries/trends/monthly_bill_by_account.sql index 09c6655b..6500c0f3 100644 --- a/cid/builtin/core/data/queries/trends/monthly_bill_by_account.sql +++ b/cid/builtin/core/data/queries/trends/monthly_bill_by_account.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE VIEW monthly_bill_by_account AS +CREATE OR REPLACE VIEW monthly_bill_by_account AS WITH t1 AS ( SELECT @@ -6,13 +6,13 @@ WITH , "bill_payer_account_id" , "line_item_usage_account_id" , "line_item_line_item_type" "charge_type" - , (CASE WHEN ("product_product_name" LIKE '') THEN "line_item_product_code" ELSE "product_product_name" END) "product_product_name" - , "product_region" + , (CASE WHEN (product['product_name'] LIKE '') THEN "line_item_product_code" ELSE product['product_name'] END) "product_product_name" + , product['region'] "product_region" , "line_item_product_code" , "round"("sum"("line_item_unblended_cost"), 2) "unblended_cost" - , "round"(sum("line_item_unblended_cost") , 2) "amortized_cost" + , "round"("sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "savings_plan_savings_plan_effective_cost" WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "reservation_effective_cost" WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE "line_item_unblended_cost" END)), 2) "amortized_cost" FROM - "${cur_table_name}" + "${cur2_database}"."${cur2_table_name}" GROUP BY 1, 2, 3, 4, 5, 6, 7 ) , t2 AS ( @@ -53,7 +53,7 @@ SELECT , (CASE WHEN (("t5"."aws_service_category" IS NULL) AND ("t1"."product_product_name" IS NOT NULL)) THEN "t1"."product_product_name" WHEN (("aws_service_category" IS NULL) AND ("t1"."product_product_name" IS NULL)) THEN 'Other' ELSE "t5"."aws_service_category" END) "aws_service_category" FROM ((((t1 -LEFT JOIN t2 ON (CAST("t1"."line_item_usage_account_id" AS char(12)) = CAST("t2"."account_id" AS char(12)))) -LEFT JOIN t3 ON (CAST("t1"."bill_payer_account_id" AS char(12)) = CAST("t3"."account_id" AS char(12)))) +LEFT JOIN t2 ON ("t1"."line_item_usage_account_id" = "t2"."account_id")) +LEFT JOIN t3 ON ("t1"."bill_payer_account_id" = "t3"."account_id")) LEFT JOIN t4 ON ("t1"."product_region" = "t4"."region_name")) LEFT JOIN t5 ON ("t1"."line_item_product_code" = "t5"."line_item_product_code")) \ No newline at end of file diff --git a/cid/builtin/core/data/queries/trends/monthly_bill_by_account_ri.sql b/cid/builtin/core/data/queries/trends/monthly_bill_by_account_ri.sql deleted file mode 100644 index 2c8562d4..00000000 --- a/cid/builtin/core/data/queries/trends/monthly_bill_by_account_ri.sql +++ /dev/null @@ -1,62 +0,0 @@ -CREATE OR REPLACE VIEW monthly_bill_by_account AS -WITH - t1 AS ( - SELECT - "bill_billing_period_start_date" - , "bill_payer_account_id" - , "line_item_usage_account_id" - , "line_item_line_item_type" "charge_type" - , (CASE WHEN ("product_product_name" LIKE '') THEN "line_item_product_code" ELSE "product_product_name" END) "product_product_name" - , "product_region" - , "line_item_product_code" - , "round"("sum"("line_item_unblended_cost"), 2) "unblended_cost" - , "round"(sum(CASE - WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "reservation_effective_cost" - WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") - WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE "line_item_unblended_cost" END) , 2) "amortized_cost" - FROM - "${cur_table_name}" - GROUP BY 1, 2, 3, 4, 5, 6, 7 -) -, t2 AS ( - SELECT - "account_name" - , "account_id" - FROM - aws_accounts -) -, t3 AS ( - SELECT - "payer_account_name" - , "account_id" - FROM - payer_account_name_map -) -, t4 AS ( - SELECT - "region_latitude" - , "region_longitude" - , "region_name" - FROM - aws_regions -) -, t5 AS ( - SELECT - "aws_service_category" - , "line_item_product_code" - FROM - aws_service_category_map -) -SELECT - t1.* -, "t2"."account_name" -, "t3"."payer_account_name" -, TRY_CAST("t4"."region_latitude" AS decimal) "region_latitude" -, TRY_CAST("t4"."region_longitude" AS decimal) "region_longitude" -, (CASE WHEN (("t5"."aws_service_category" IS NULL) AND ("t1"."product_product_name" IS NOT NULL)) THEN "t1"."product_product_name" WHEN (("aws_service_category" IS NULL) AND ("t1"."product_product_name" IS NULL)) THEN 'Other' ELSE "t5"."aws_service_category" END) "aws_service_category" -FROM - ((((t1 -LEFT JOIN t2 ON (CAST("t1"."line_item_usage_account_id" AS char(12)) = CAST("t2"."account_id" AS char(12)))) -LEFT JOIN t3 ON (CAST("t1"."bill_payer_account_id" AS char(12)) = CAST("t3"."account_id" AS char(12)))) -LEFT JOIN t4 ON ("t1"."product_region" = "t4"."region_name")) -LEFT JOIN t5 ON ("t1"."line_item_product_code" = "t5"."line_item_product_code")) \ No newline at end of file diff --git a/cid/builtin/core/data/queries/trends/monthly_bill_by_account_sp.sql b/cid/builtin/core/data/queries/trends/monthly_bill_by_account_sp.sql deleted file mode 100644 index 96e5e513..00000000 --- a/cid/builtin/core/data/queries/trends/monthly_bill_by_account_sp.sql +++ /dev/null @@ -1,64 +0,0 @@ -CREATE OR REPLACE VIEW monthly_bill_by_account AS -WITH - t1 AS ( - SELECT - "bill_billing_period_start_date" - , "bill_payer_account_id" - , "line_item_usage_account_id" - , "line_item_line_item_type" "charge_type" - , (CASE WHEN ("product_product_name" LIKE '') THEN "line_item_product_code" ELSE "product_product_name" END) "product_product_name" - , "product_region" - , "line_item_product_code" - , "round"("sum"("line_item_unblended_cost"), 2) "unblended_cost" - , "round"(sum(CASE - WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "savings_plan_savings_plan_effective_cost" - WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") - WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 - WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 - ELSE "line_item_unblended_cost" END), 2) "amortized_cost" - FROM - "${cur_table_name}" - GROUP BY 1, 2, 3, 4, 5, 6, 7 -) -, t2 AS ( - SELECT - "account_name" - , "account_id" - FROM - aws_accounts -) -, t3 AS ( - SELECT - "payer_account_name" - , "account_id" - FROM - payer_account_name_map -) -, t4 AS ( - SELECT - "region_latitude" - , "region_longitude" - , "region_name" - FROM - aws_regions -) -, t5 AS ( - SELECT - "aws_service_category" - , "line_item_product_code" - FROM - aws_service_category_map -) -SELECT - t1.* -, "t2"."account_name" -, "t3"."payer_account_name" -, TRY_CAST("t4"."region_latitude" AS decimal) "region_latitude" -, TRY_CAST("t4"."region_longitude" AS decimal) "region_longitude" -, (CASE WHEN (("t5"."aws_service_category" IS NULL) AND ("t1"."product_product_name" IS NOT NULL)) THEN "t1"."product_product_name" WHEN (("aws_service_category" IS NULL) AND ("t1"."product_product_name" IS NULL)) THEN 'Other' ELSE "t5"."aws_service_category" END) "aws_service_category" -FROM - ((((t1 -LEFT JOIN t2 ON (CAST("t1"."line_item_usage_account_id" AS char(12)) = CAST("t2"."account_id" AS char(12)))) -LEFT JOIN t3 ON (CAST("t1"."bill_payer_account_id" AS char(12)) = CAST("t3"."account_id" AS char(12)))) -LEFT JOIN t4 ON ("t1"."product_region" = "t4"."region_name")) -LEFT JOIN t5 ON ("t1"."line_item_product_code" = "t5"."line_item_product_code")) \ No newline at end of file diff --git a/cid/builtin/core/data/queries/trends/monthly_bill_by_account_sp_ri.sql b/cid/builtin/core/data/queries/trends/monthly_bill_by_account_sp_ri.sql deleted file mode 100644 index 72e5d3fe..00000000 --- a/cid/builtin/core/data/queries/trends/monthly_bill_by_account_sp_ri.sql +++ /dev/null @@ -1,59 +0,0 @@ -CREATE OR REPLACE VIEW monthly_bill_by_account AS -WITH - t1 AS ( - SELECT - "bill_billing_period_start_date" - , "bill_payer_account_id" - , "line_item_usage_account_id" - , "line_item_line_item_type" "charge_type" - , (CASE WHEN ("product_product_name" LIKE '') THEN "line_item_product_code" ELSE "product_product_name" END) "product_product_name" - , "product_region" - , "line_item_product_code" - , "round"("sum"("line_item_unblended_cost"), 2) "unblended_cost" - , "round"("sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "savings_plan_savings_plan_effective_cost" WHEN ("line_item_line_item_type" = 'SavingsPlanRecurringFee') THEN ("savings_plan_total_commitment_to_date" - "savings_plan_used_commitment") WHEN ("line_item_line_item_type" = 'SavingsPlanNegation') THEN 0 WHEN ("line_item_line_item_type" = 'SavingsPlanUpfrontFee') THEN 0 WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "reservation_effective_cost" WHEN ("line_item_line_item_type" = 'RIFee') THEN ("reservation_unused_amortized_upfront_fee_for_billing_period" + "reservation_unused_recurring_fee") WHEN (("line_item_line_item_type" = 'Fee') AND ("reservation_reservation_a_r_n" <> '')) THEN 0 ELSE "line_item_unblended_cost" END)), 2) "amortized_cost" - FROM - "${cur_table_name}" - GROUP BY 1, 2, 3, 4, 5, 6, 7 -) -, t2 AS ( - SELECT - "account_name" - , "account_id" - FROM - aws_accounts -) -, t3 AS ( - SELECT - "payer_account_name" - , "account_id" - FROM - payer_account_name_map -) -, t4 AS ( - SELECT - "region_latitude" - , "region_longitude" - , "region_name" - FROM - aws_regions -) -, t5 AS ( - SELECT - "aws_service_category" - , "line_item_product_code" - FROM - aws_service_category_map -) -SELECT - t1.* -, "t2"."account_name" -, "t3"."payer_account_name" -, TRY_CAST("t4"."region_latitude" AS decimal) "region_latitude" -, TRY_CAST("t4"."region_longitude" AS decimal) "region_longitude" -, (CASE WHEN (("t5"."aws_service_category" IS NULL) AND ("t1"."product_product_name" IS NOT NULL)) THEN "t1"."product_product_name" WHEN (("aws_service_category" IS NULL) AND ("t1"."product_product_name" IS NULL)) THEN 'Other' ELSE "t5"."aws_service_category" END) "aws_service_category" -FROM - ((((t1 -LEFT JOIN t2 ON ("t1"."line_item_usage_account_id" = "t2"."account_id")) -LEFT JOIN t3 ON ("t1"."bill_payer_account_id" = "t3"."account_id")) -LEFT JOIN t4 ON ("t1"."product_region" = "t4"."region_name")) -LEFT JOIN t5 ON ("t1"."line_item_product_code" = "t5"."line_item_product_code")) \ No newline at end of file diff --git a/cid/builtin/core/data/resources.yaml b/cid/builtin/core/data/resources.yaml index 5fb17a3e..11732a6a 100644 --- a/cid/builtin/core/data/resources.yaml +++ b/cid/builtin/core/data/resources.yaml @@ -7,7 +7,6 @@ dashboards: templateId: cudos_dashboard_v3 dashboardId: cudos deprecationNotice: "This version of the CUDOS deprecated. Deploy CUDOS Dashboard v5 instead" - localConfigs: ["update-dashboard.json"] dependsOn: datasets: - summary_view @@ -112,6 +111,8 @@ datasets: summary_view: File: cid/summary_view.json dependsOn: + dataProviders: + - DataExports views: - account_map - summary_view @@ -122,6 +123,8 @@ datasets: ec2_running_cost: File: cid/ec2_running_cost.json dependsOn: + dataProviders: + - DataExports views: - ec2_running_cost schedules: @@ -130,6 +133,8 @@ datasets: compute_savings_plan_eligible_spend: File: cid/compute.json dependsOn: + dataProviders: + - DataExports views: - compute_savings_plan_eligible_spend schedules: @@ -138,6 +143,8 @@ datasets: s3_view: File: cid/s3_view.json dependsOn: + dataProviders: + - DataExports views: - s3_view - account_map @@ -148,13 +155,17 @@ datasets: customer_all: File: shared/customer_all.json dependsOn: - cur: true + dataProviders: + - DataExports + cur2: true views: - - ${cur_table_name} + - customer_all # CUDOS v5 datasets hourly_view: File: cudos/hourly_view.json dependsOn: + dataProviders: + - DataExports views: - hourly_view - account_map @@ -163,6 +174,8 @@ datasets: resource_view: File: cudos/resource_view.json dependsOn: + dataProviders: + - DataExports views: - resource_view - account_map @@ -172,6 +185,8 @@ datasets: kpi_ebs_snap: File: kpi/kpi_ebs_snap.json dependsOn: + dataProviders: + - DataExports views: - kpi_ebs_snap schedules: @@ -179,6 +194,8 @@ datasets: kpi_ebs_storage_all: File: kpi/kpi_ebs_storage_all.json dependsOn: + dataProviders: + - DataExports views: - kpi_ebs_storage_all schedules: @@ -186,6 +203,8 @@ datasets: kpi_instance_all: File: kpi/kpi_instance_all.json dependsOn: + dataProviders: + - DataExports views: - kpi_instance_all schedules: @@ -193,6 +212,8 @@ datasets: kpi_s3_storage_all: File: kpi/kpi_s3_storage_all.json dependsOn: + dataProviders: + - DataExports views: - kpi_s3_storage_all schedules: @@ -200,8 +221,11 @@ datasets: kpi_tracker: File: kpi/kpi_tracker.json dependsOn: + dataProviders: + - DataExports views: - kpi_tracker + - summary_view schedules: - default @@ -210,6 +234,8 @@ datasets: ta-organizational-view: File: tao/dataset.json dependsOn: + dataProviders: + - DataCollection views: - ta_org_view schedules: @@ -219,6 +245,8 @@ datasets: daily-anomaly-detection: File: trends/daily_anomaly_detection.json dependsOn: + dataProviders: + - DataExports views: - daily_anomaly_detection schedules: @@ -226,25 +254,28 @@ datasets: monthly-anomaly-detection: File: trends/monthly_anomaly_detection.json dependsOn: + dataProviders: + - DataExports views: - monthly_anomaly_detection schedules: - default monthly-bill-by-account: - spriFile: trends/monthly_bill_by_account_sp_ri.json - spFile: trends/monthly_bill_by_account_sp.json - riFile: trends/monthly_bill_by_account_ri.json File: trends/monthly_bill_by_account.json dependsOn: + dataProviders: + - DataExports views: - monthly_bill_by_account schedules: - default - # Compute Optimiser (CO) + # Compute Optimizer (CO) compute_optimizer_all_options: File: co/dataset.json dependsOn: + dataProviders: + - DataCollection views: - compute_optimizer_all_options - business_units_map @@ -261,13 +292,213 @@ datasets: # Athena views definitions views: # CID Shared Views + customer_all: + File: shared/customer_all_unlimited.sql + dependsOn: + cur2: + - "bill_bill_type" + - "bill_billing_entity" + - "bill_billing_period_end_date" + - "bill_billing_period_start_date" + - "bill_invoice_id" + - "bill_payer_account_id" + - "identity_line_item_id" + - "identity_time_interval" + - "line_item_availability_zone" + - "line_item_blended_cost" + - "line_item_blended_rate" + - "line_item_currency_code" + - "line_item_legal_entity" + - "line_item_line_item_description" + - "line_item_line_item_type" + - "line_item_normalization_factor" + - "line_item_normalized_usage_amount" + - "line_item_operation" + - "line_item_product_code" + - "line_item_resource_id" + - "line_item_tax_type" + - "line_item_unblended_cost" + - "line_item_unblended_rate" + - "line_item_usage_account_id" + - "line_item_usage_amount" + - "line_item_usage_end_date" + - "line_item_usage_start_date" + - "line_item_usage_type" + - "pricing_currency" + - "pricing_lease_contract_length" + - "pricing_offering_class" + - "pricing_public_on_demand_cost" + - "pricing_public_on_demand_rate" + - "pricing_purchase_option" + - "pricing_rate_id" + - "pricing_term" + - "pricing_unit" + - "product_broker_engine" + - "product_bundle_description" + - "product_bundle_group" + - "product_bundle" + - "product_client_location" + - "product_component" + - "product_compute_family" + - "product_compute_type" + - "product_content_type" + - "product_cputype" + - "product_datatransferout" + - "product_describes" + - "product_directory_size" + - "product_directory_type_description" + - "product_directory_type" + - "product_dominantnondominant" + - "product_endpoint_type" + - "product_enhanced_networking_support" + - "product_fee_code" + - "product_fee_description" + - "product_free_trial" + - "product_free_usage_included" + - "product_from_location_type" + - "product_from_location" + - "product_gets" + - "product_gpu" + - "product_granularity" + - "product_input_mode" + - "product_insightstype" + - "product_instance_family" + - "product_instance_name" + - "product_instance_type" + - "product_instance" + - "product_instances" + - "product_license" + - "product_location_type" + - "product_location" + - "product_maximum_extended_storage" + - "product_maximum_storage_volume" + - "product_minimum_storage_volume" + - "product_operation" + - "product_ops_items" + - "product_origin" + - "product_output_mode" + - "product_physical_cpu" + - "product_physical_gpu" + - "product_pricing_unit" + - "product_product_family" + - "product_provisioned" + - "product_ratetype" + - "product_recipient" + - "product_request_description" + - "product_resource_type" + - "product_rootvolume" + - "product_routing_target" + - "product_routing_type" + - "product_running_mode" + - "product_servicecode" + - "product_sku" + - "product_software_included" + - "product_standard_storage_retention_included" + - "product_steps" + - "product_storage_type" + - "product_to_location_type" + - "product_to_location" + - "product_traffic_direction" + - "product_updates" + - "product_usagetype" + - "product_uservolume" + - "reservation_amortized_upfront_cost_for_usage" + - "reservation_amortized_upfront_fee_for_billing_period" + - "reservation_effective_cost" + - "reservation_end_time" + - "reservation_modification_status" + - "reservation_normalized_units_per_reservation" + - "reservation_number_of_reservations" + - "reservation_recurring_fee_for_usage" + - "reservation_reservation_a_r_n" + - "reservation_start_time" + - "reservation_subscription_id" + - "reservation_total_reserved_normalized_units" + - "reservation_total_reserved_units" + - "reservation_units_per_reservation" + - "reservation_unused_amortized_upfront_fee_for_billing_period" + - "reservation_unused_normalized_unit_quantity" + - "reservation_unused_quantity" + - "reservation_unused_recurring_fee" + - "reservation_upfront_value" + - "savings_plan_amortized_upfront_commitment_for_billing_period" + - "savings_plan_recurring_commitment_for_billing_period" + - "savings_plan_savings_plan_a_r_n" + - "savings_plan_savings_plan_effective_cost" + - "savings_plan_savings_plan_rate" + - "savings_plan_total_commitment_to_date" + - "savings_plan_used_commitment" + - product['alarm_type'] + - product['availability_zone'] + - product['availability'] + - product['cache_engine'] + - product['capacitystatus'] + - product['category'] + - product['clock_speed'] + - product['current_generation'] + - product['database_engine'] + - product['dedicated_ebs_throughput'] + - product['deployment_option'] + - product['description'] + - product['durability'] + - product['ecu'] + - product['edition'] + - product['engine_code'] + - product['enhanced_networking_supported'] + - product['event_type'] + - product['finding_group'] + - product['finding_source'] + - product['finding_storage'] + - product['free_query_types'] + - product['gpu_memory'] + - product['group_description'] + - product['group'] + - product['instance_type_family'] + - product['intel_avx_available'] + - product['intel_avx2_available'] + - product['intel_turbo_available'] + - product['license_model'] + - product['logs_destination'] + - product['max_iops_burst_performance'] + - product['max_iopsvolume'] + - product['max_throughputvolume'] + - product['max_volume_size'] + - product['memory_gib'] + - product['memory'] + - product['memorytype'] + - product['message_delivery_frequency'] + - product['message_delivery_order'] + - product['min_volume_size'] + - product['network_performance'] + - product['normalization_size_factor'] + - product['operating_system'] + - product['physical_processor'] + - product['pre_installed_sw'] + - product['processor_architecture'] + - product['processor_features'] + - product['product_name'] + - product['queue_type'] + - product['region'] + - product['request_type'] + - product['servicename'] + - product['standard_group'] + - product['standard_storage'] + - product['storage_class'] + - product['storage_media'] + - product['storage'] + - product['subscription_type'] + - product['tenancy'] + - product['transfer_type'] + - product['type'] + - product['vcpu'] + - product['version'] + - product['volume_api_name'] + - product['volume_type'] + summary_view: - spriFile: cid/summary_view_sp_ri.sql - spFile: cid/summary_view_sp.sql - riFile: cid/summary_view_ri.sql File: cid/summary_view.sql dependsOn: - cur: + cur2: - bill_billing_entity - bill_billing_period_start_date - bill_invoice_id @@ -286,18 +517,18 @@ views: - line_item_usage_type - pricing_public_on_demand_cost - pricing_unit - - product_current_generation - - product_database_engine + - product['current_generation'] + - product['database_engine'] - product_from_location - - product_group - - product_operating_system - - product_physical_processor - - product_processor_features + - product['group'] + - product['operating_system'] + - product['physical_processor'] + - product['processor_features'] - product_product_family - - product_product_name - - product_region + - product['product_name'] + - product['region'] - product_servicecode - - product_tenancy + - product['tenancy'] - product_to_location - reservation_effective_cost - reservation_reservation_a_r_n @@ -310,12 +541,9 @@ views: - savings_plan_used_commitment ec2_running_cost: - spriFile: cid/ec2_running_cost_sp_ri.sql - spFile: cid/ec2_running_cost_sp.sql - riFile: cid/ec2_running_cost_ri.sql File: cid/ec2_running_cost.sql dependsOn: - cur: + cur2: - bill_billing_period_start_date - bill_payer_account_id - line_item_line_item_type @@ -332,7 +560,7 @@ views: compute_savings_plan_eligible_spend: File: cid/compute_savings_plan_eligible_spend.sql dependsOn: - cur: true + cur2: - bill_billing_period_start_date - bill_payer_account_id - line_item_line_item_type @@ -347,7 +575,7 @@ views: s3_view: File: cid/s3.sql dependsOn: - cur: + cur2: - bill_billing_period_start_date - bill_payer_account_id - line_item_line_item_type @@ -360,15 +588,12 @@ views: - line_item_usage_start_date - pricing_public_on_demand_cost - pricing_unit - - product_region + - product['region'] ri_sp_mapping: - spriFile: cid/ri_sp_mapping_sp_ri.sql - spFile: cid/ri_sp_mapping_sp.sql - riFile: cid/ri_sp_mapping_ri.sql File: cid/ri_sp_mapping.sql dependsOn: - cur: + cur2: - bill_billing_period_start_date - bill_payer_account_id - line_item_line_item_type @@ -378,14 +603,19 @@ views: - reservation_reservation_a_r_n - savings_plan_end_time - savings_plan_savings_plan_a_r_n + - reservation_effective_cost + - reservation_start_time + - reservation_end_time + - savings_plan_savings_plan_effective_cost + - savings_plan_start_time + - savings_plan_purchase_term + - savings_plan_offering_type + - savings_plan_payment_option hourly_view: - spriFile: cudos/hourly_view_sp_ri.sql - spFile: cudos/hourly_view_sp.sql - riFile: cudos/hourly_view_ri.sql File: cudos/hourly_view.sql dependsOn: - cur: + cur2: - bill_billing_period_start_date - bill_payer_account_id - line_item_line_item_description @@ -399,7 +629,7 @@ views: - line_item_usage_type - pricing_term - pricing_unit - - product_region + - product['region'] - product_servicecode - reservation_effective_cost - reservation_reservation_a_r_n @@ -407,12 +637,9 @@ views: - savings_plan_savings_plan_effective_cost resource_view: - spriFile: cudos/resource_view_sp_ri.sql - spFile: cudos/resource_view_sp.sql - riFile: cudos/resource_view_ri.sql File: cudos/resource_view.sql dependsOn: - cur: + cur2: - bill_billing_entity - bill_payer_account_id - line_item_legal_entity @@ -428,20 +655,20 @@ views: - line_item_usage_type - pricing_term - pricing_unit - - product_database_engine - - product_deployment_option + - product['database_engine'] + - product['deployment_option'] - product_from_location - - product_group + - product['group'] - product_instance_type - - product_instance_type_family - - product_operating_system + - product['instance_type_family'] + - product['operating_system'] - product_product_family - - product_product_name - - product_region + - product['product_name'] + - product['region'] - product_servicecode - - product_storage + - product['storage'] - product_to_location - - product_volume_api_name + - product['volume_api_name'] - reservation_effective_cost - reservation_reservation_a_r_n - savings_plan_savings_plan_a_r_n @@ -451,22 +678,19 @@ views: daily_anomaly_detection: File: trends/daily_anomaly_detection.sql dependsOn: - cur: true + cur2: true views: - aws_accounts monthly_anomaly_detection: File: trends/monthly_anomaly_detection.sql dependsOn: - cur: true + cur2: true views: - aws_accounts monthly_bill_by_account: - riFile: trends/monthly_bill_by_account_ri.sql - spriFile: trends/monthly_bill_by_account_sp_ri.sql - spFile: trends/monthly_bill_by_account_sp.sql File: trends/monthly_bill_by_account.sql dependsOn: - cur: true + cur2: true views: - aws_accounts - aws_regions @@ -480,24 +704,21 @@ views: kpi_ebs_snap: File: kpi/kpi_ebs_snap_view.sql dependsOn: - cur: true + cur2: true views: - account_map kpi_ebs_storage_all: File: kpi/kpi_ebs_storage_view.sql dependsOn: - cur: true + cur2: true views: - account_map kpi_instance_all: - spriFile: kpi/kpi_instance_all_view.sql - spFile: kpi/kpi_instance_all_view_noRI.sql - riFile: kpi/kpi_instance_all_view_noSP.sql - File: kpi/kpi_instance_all_view_noRISP.sql + File: kpi/kpi_instance_all_view.sql dependsOn: - cur: true + cur2: true views: - account_map - kpi_instance_mapping @@ -505,7 +726,7 @@ views: kpi_s3_storage_all: File: kpi/kpi_s3_storage_view.sql dependsOn: - cur: true + cur2: true views: - account_map @@ -513,7 +734,7 @@ views: kpi_tracker: File: kpi/last_kpi_tracker_view.sql dependsOn: - cur: true + cur2: true views: - account_map - kpi_ebs_snap @@ -536,14 +757,14 @@ views: views: - ta_organizational_view_reports - # Compute Optimiser (CO) + # Compute Optimizer (CO) compute_optimizer_ec2_instance_lines: type: Glue_Table File: co/ec2_instance.json parameters: s3FolderPath: default: 's3://cid-data-{account_id}/compute_optimizer/compute_optimizer_ec2_instance' - description: Compute Optimiser EC2 report S3 path + description: Compute Optimizer EC2 report S3 path compute_optimizer_rds_instance_lines: type: Glue_Table @@ -559,7 +780,7 @@ views: parameters: s3FolderPath: default: 's3://cid-data-{account_id}/compute_optimizer/compute_optimizer_auto_scale' - description: Compute Optimiser auto_scale report S3 path + description: Compute Optimizer auto_scale report S3 path compute_optimizer_ebs_volume_lines: type: Glue_Table @@ -567,7 +788,7 @@ views: parameters: s3FolderPath: default: 's3://cid-data-{account_id}/compute_optimizer/compute_optimizer_ebs_volume' - description: Compute Optimiser EBS report S3 path + description: Compute Optimizer EBS report S3 path compute_optimizer_lambda_lines: type: Glue_Table @@ -575,7 +796,7 @@ views: parameters: s3FolderPath: default: 's3://cid-data-{account_id}/compute_optimizer/compute_optimizer_lambda' - description: Compute Optimiser Lambda report S3 path + description: Compute Optimizer Lambda report S3 path compute_optimizer_ec2_instance_options: File: co/ec2_instance_options.sql @@ -633,7 +854,7 @@ views: aws_accounts: File: shared/aws_accounts.sql dependsOn: - cur: true + cur2: true aws_regions: File: shared/aws_regions.sql aws_service_category_map: diff --git a/cid/cli.py b/cid/cli.py index acbe267f..7ab5a761 100644 --- a/cid/cli.py +++ b/cid/cli.py @@ -9,7 +9,7 @@ from cid._version import __version__ from cid.exceptions import CidCritical, CidError -logger = logging.getLogger(__name__) +logger = logging.getLogger('cid') version = __version__ latest_version = get_latest_tool_version() prog_name="CLOUD INTELLIGENCE DASHBOARDS (CID) CLI" @@ -20,6 +20,15 @@ def cid_command(func): def wrapper(ctx, **kwargs): + + def get_command_line(): + params = get_parameters() + return ('cid-cmd ' + ctx.info_name + + ''.join([f" --{k.replace('_','-')}" for k, v in ctx.params.items() if isinstance(v, bool) and v]) + + ''.join([f" --{k.replace('_','-')} '{v}'" for k, v in ctx.params.items() if not isinstance(v, bool) and v is not None]) + + ''.join([f" --{k} '{v}' " for k, v in params.items() if not isinstance(v, bool) and v is not None]) + ) + # Complete kwargs with other parameters if len(ctx.args) % 2 != 0: print(f"Unknown extra argument, or an option without value {ctx.args}") @@ -33,19 +42,13 @@ def wrapper(ctx, **kwargs): res = None try: res = func(ctx, **kwargs) - except CidCritical as exc: - logger.debug(exc, exc_info=True) - logger.critical(exc) - except CidError as exc: + except (CidCritical, CidError) as exc: logger.debug(exc, exc_info=True) + logger.debug(f'When running {get_command_line()}') logger.error(exc) params = get_parameters() logger.info('Next time you can use following command:') - logger.info(' cid-cmd ' + ctx.info_name - + ''.join([f" --{k.replace('_','-')}" for k, v in ctx.params.items() if isinstance(v, bool) and v]) - + ''.join([f" --{k.replace('_','-')} '{v}'" for k, v in ctx.params.items() if not isinstance(v, bool) and v is not None]) - + ''.join([f" --{k} '{v}' " for k, v in params.items() if not isinstance(v, bool) and v is not None]) - ) + logger.info(get_command_line()) return res wrapper.__doc__ = func.__doc__ wrapper.__name__ = func.__name__ @@ -113,7 +116,7 @@ def csv2view(ctx, **kwargs): @cid_command def deploy(ctx, **kwargs): """Deploy Dashboard - + \b Command options: --category TEXT The dashboards category to choose from. Not needed if dashboard-id provided directly @@ -146,7 +149,7 @@ def deploy(ctx, **kwargs): @cid_command def export(ctx, **kwargs): """Export Dashboard - + \b Command options: --analysis-name Analysis you want to share (not needed if analysis-id is provided). @@ -200,7 +203,6 @@ def update(ctx, dashboard_id, force, recursive, **kwargs): """Update Dashboard \b - --on-drift (show|override) Action if a drift of view and dataset is discovered. 'override' = override drift(will destroy customization) or 'show' (default) = show a diff. In Unattended mode (without terminal on-drift will have allways override behaviour) --theme TEXT A QuickSight Theme (CLASSIC|MIDNIGHT|SEASIDE|RAINIER) @@ -264,6 +266,24 @@ def create_cur_table(ctx, **kwargs): ctx.obj.create_cur_table(**kwargs) +@click.option('-v', '--verbose', count=True) +@click.option('--cur-version', help='Cur Version (1 or 2)') +@click.option('--fields', help='CUR fields', default='') +@cid_command +def create_cur_proxy(ctx, cur_version, fields, **kwargs): + """Create CUR proxy - an Athena view that transforms cur1 to cur2 or cur2 > cur1 + + \b + --cur-version (1|2) The target version of CUR + --fields Comma Separated list of additional CUR fields + --cur-table-name TEXT CUR table name + --cur-database TEXT Athena database of CUR + --athena-database TEXT Athena database to create proxy + """ + + ctx.obj.create_cur_proxy(cur_version=cur_version, fields=fields, **kwargs) + + @click.option('-v', '--verbose', count=True) @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False) @cid_command @@ -271,8 +291,7 @@ def teardown(ctx, **kwargs): """Delete all CID assets \b - - THIS IS VERY DANGEROUS. DO NOT USE IT. + THIS IS VERY DANGEROUS. DO NOT USE THIS COMMAND. """ ctx.obj.teardown(**kwargs) diff --git a/cid/common.py b/cid/common.py index 5ff9ddc8..5e41585e 100644 --- a/cid/common.py +++ b/cid/common.py @@ -1,10 +1,8 @@ import os -import sys import json import urllib import logging import functools -from pathlib import Path from string import Template from typing import Dict from pkg_resources import resource_string @@ -22,7 +20,7 @@ from cid.plugin import Plugin from cid.utils import get_parameter, get_parameters, set_parameters, unset_parameter, get_yesno_parameter, cid_print, isatty, merge_objects, IsolatedParameters from cid.helpers.account_map import AccountMap -from cid.helpers import Athena, S3, IAM, CUR, Glue, QuickSight, Dashboard, Dataset, Datasource, csv2view, Organizations +from cid.helpers import Athena, S3, IAM, CUR, ProxyCUR, Glue, QuickSight, Dashboard, Dataset, Datasource, csv2view, Organizations, CFN from cid.helpers.quicksight.template import Template as CidQsTemplate from cid._version import __version__ from cid.export import export_analysis @@ -105,6 +103,10 @@ def glue(self) -> Glue: def iam(self) -> IAM: return IAM(self.base.session) + @cached_property + def cfn(self) -> CFN: + return CFN(self.base.session) + @cached_property def organizations(self) -> Organizations: return Organizations(self.base.session) @@ -113,8 +115,26 @@ def organizations(self) -> Organizations: def s3(self) -> S3: return S3(self.base.session) + @cached_property + def cur1(self): + """ get/create a cur1 """ + return self.get_cur('1') + + @cached_property + def cur2(self): + """ get/create a cur2 """ + return self.get_cur('2') + + def get_cur(self, target_cur_version): + """ get a cur """ + cur_version = self.cur.version + if cur_version != target_cur_version or get_parameters().get('use-cur-proxy'): + return ProxyCUR(self.cur, target_cur_version=target_cur_version) + return self.cur + @property def cur(self) -> CUR: + '''can return any CUR (1 or 2) that customer provides''' if not self._clients.get('cur'): while True: try: @@ -140,17 +160,13 @@ def cur(self) -> CUR: self.create_cur_table() return self._clients['cur'] - @property - def accountMap(self) -> AccountMap: - if not self._clients.get('accountMap'): - _account_map = AccountMap(self.base.session) - _account_map.athena = self.athena - _account_map.cur = self.cur - - self._clients.update({ - 'accountMap': _account_map - }) - return self._clients.get('accountMap') + def create_or_update_account_map(self, name): + account_map = AccountMap( + self.base.session, + self.athena, + self.cur, # can be any CUR. But it is only needed for trends and dummy + ) + return account_map.create_or_update(name) def command(func): ''' a decorator that ensure that we logged in to AWS acc, and loaded additional resource files @@ -1348,7 +1364,8 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non # Read dataset definition from template data = self.get_data_from_definition('dataset', dataset_definition) template = Template(json.dumps(data)) - cur_required = dataset_definition.get('dependsOn', dict()).get('cur') + cur1_required = dataset_definition.get('dependsOn', dict()).get('cur') or dataset_definition.get('dependsOn', dict()).get('cur') + cur2_required = dataset_definition.get('dependsOn', dict()).get('cur2') athena_datasource = None # Manage datasource @@ -1438,9 +1455,17 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non else: logger.debug('Athena_datasource is not defined. Will only create views') + # attach roles + if isinstance(athena_datasource, Datasource) and athena_datasource.role_name: + data_providers = dataset_definition.get('dependsOn', {}).get('dataProviders', []) + policies_arns = [self.cfn.get_read_access_policy_for_module(provider) for provider in data_providers] + policies_arns = [policies_arn for policies_arn in policies_arns if policies_arn] # filter out nones + if policies_arns: + self.iam.ensure_managed_policies_attached(role_name=athena_datasource.role_name, policies_arns=','.join(policies_arns)) + # Check for required views _views = dataset_definition.get('dependsOn', {}).get('views', []) - required_views = [(self.cur.table_name if cur_required and name =='${cur_table_name}' else name) for name in _views] + required_views = _views self.athena.discover_views(required_views) found_views = utils.intersection(required_views, self.athena._metadata.keys()) @@ -1449,9 +1474,9 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non if recursive: print(f"Detected views: {', '.join(found_views)}") for view_name in found_views: - if cur_required and view_name == self.cur.table_name: - logger.debug(f'Dependency view {view_name} is a CUR. Skip.') - continue + #if cur_required and view_name == self.cur.table_name: + # logger.debug(f'Dependency view {view_name} is a CUR. Skip.') + # continue if view_name == 'account_map': logger.debug(f'Dependency view is {view_name}. Skip.') continue @@ -1470,7 +1495,12 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non columns_tpl = { 'athena_datasource_arn': athena_datasource.arn, 'athena_database_name': self.athena.DatabaseName, - 'cur_table_name': self.cur.table_name if cur_required else None + 'cur_database': self.cur1.database if cur1_required else None, # for backward compatibly + 'cur_table_name': self.cur1.table_name if cur1_required else None, # for backward compatibly + 'cur1_database': self.cur1.database if cur1_required else None, + 'cur1_table_name': self.cur1.table_name if cur1_required else None, + 'cur2_database': self.cur2.database if cur2_required else None, + 'cur2_table_name': self.cur2.table_name if cur2_required else None, } logger.debug(f'dataset_id={dataset_id}') @@ -1486,7 +1516,7 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non try: compiled_dataset = json.loads(compiled_dataset_text) except json.JSONDecodeError as exc: - logger.error('The json of dataset is not correct. Please check parameters of the dasbhoard.') + logger.error('The json of dataset is not correct. Please check parameters of the dashboard.') logger.debug(compiled_dataset_text) raise if dataset_id: @@ -1500,7 +1530,7 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non elif found_dataset.name != compiled_dataset.get('Name'): print(f"Dataset found with name {found_dataset.name}, but {compiled_dataset.get('Name')} expected. Updating.") update_dataset = True - if update_dataset and get_parameters().get('on-drift', 'show').lower() != 'override' and isatty() and not cur_required: + if update_dataset and get_parameters().get('on-drift', 'show').lower() != 'override' and isatty() and not cur1_required and not cur2_required: while True: diff = self.qs.dataset_diff(found_dataset.raw, compiled_dataset) if diff and diff['diff']: @@ -1555,17 +1585,20 @@ def create_or_update_dataset(self, dataset_definition: dict, dataset_id: str=Non def create_or_update_view(self, view_name: str, recursive: bool=True, update: bool=False) -> None: # Avoid checking a views multiple times in one cid session + update = update or get_parameters().get('update') + logger.trace(f'create_or_update_view({view_name}, recursive={recursive}, update={update})') if view_name in self._visited_views: + logger.trace(f'{view_name} is in _visited_views.skipping') return self._visited_views.append(view_name) logger.info(f'Processing view: {view_name}') # For account mappings create a view using a special helper if view_name in ['account_map', 'aws_accounts']: - if view_name in self.athena._metadata.keys(): + if view_name in self.athena._metadata.keys() and (not update and not recursive): print(f'Account map {view_name} exists. Skipping.') else: - self.accountMap.create(view_name) #FIXME: add or_update + self.create_or_update_account_map(view_name) return # Create a view @@ -1581,17 +1614,17 @@ def create_or_update_view(self, view_name: str, recursive: bool=True, update: bo dependencies = view_definition.get('dependsOn', {}) # Process CUR columns - if isinstance(dependencies.get('cur'), list): - for column in dependencies.get('cur'): - self.cur.ensure_column(column) - elif isinstance(dependencies.get('cur'), dict): - for column, column_type in dependencies.get('cur').items(): - self.cur.ensure_column(column, column_type) + if dependencies.get('cur'): + self.cur1.ensure_columns(dependencies.get('cur')) + if dependencies.get('cur2'): + self.cur2.ensure_columns(dependencies.get('cur2')) if recursive: dependency_views = dependencies.get('views', []) if 'cur' in dependency_views: dependency_views.remove('cur') + if 'cur2' in dependency_views: + dependency_views.remove('cur2') # Discover dependency views (may not be discovered earlier) self.athena.discover_views(dependency_views) logger.info(f"Dependency views: {', '.join(dependency_views)}" if dependency_views else 'No dependency views') @@ -1612,44 +1645,8 @@ def create_or_update_view(self, view_name: str, recursive: bool=True, update: bo else: if 'CREATE EXTERNAL TABLE' in view_query.upper(): logger.warning('Cannot recreate table {view_name}') - elif 'CREATE OR REPLACE' in view_query.upper(): - update_view = False - while get_parameters().get('on-drift', 'show').lower() != 'override' and isatty(): - cid_print(f'Analyzing view {view_name}') - diff = self.athena.get_view_diff(view_name, view_query) - if diff and diff['diff']: - cid_print(f'Found a difference between existing view {view_name} and the one we want to deploy. ') - cid_print(diff['printable']) - choice = get_parameter( - param_name='view-' + view_name + '-override', - message=f'The existing view is different. Override?', - choices=['retry diff', 'proceed and override', 'keep existing', 'exit'], - default='retry diff' - ) - if choice == 'retry diff': - unset_parameter('view-' + view_name + '-override') - continue - elif choice == 'proceed and override': - update_view = True - break - elif choice == 'keep existing': - update_view = False - break - else: - raise CidCritical(f'User choice is not to update {view_name}.') - elif not diff: - if not get_yesno_parameter( - param_name='view-' + view_name + '-override', - message=f'Cannot get sql diff for {view_name}. Continue?', - default='yes' - ): - raise CidCritical(f'User choice is not to update {view_name}.') - update_view = True - break - if update_view: - print(f'Updating view: "{view_name}"') - self.athena.execute_query(view_query) + self.athena.create_or_update_view(view_name=view_name, view_query=view_query) else: print(f'View "{view_name}" is not compatible with update. Skipping.') if 'CREATE OR REPLACE VIEW' in view_query.upper() or 'CREATE VIEW' in view_query.upper(): @@ -1719,18 +1716,19 @@ def get_view_query(self, view_name: str) -> str: """ Returns a fully compiled AHQ """ # View path view_definition = self.get_definition("view", name=view_name) - cur_required = view_definition.get('dependsOn', dict()).get('cur') - if cur_required and self.cur.has_savings_plans and self.cur.has_reservations and view_definition.get('spriFile'): - view_definition['File'] = view_definition.get('spriFile') - elif cur_required and self.cur.has_savings_plans and view_definition.get('spFile'): - view_definition['File'] = view_definition.get('spFile') - elif cur_required and self.cur.has_reservations and view_definition.get('riFile'): - view_definition['File'] = view_definition.get('riFile') - elif view_definition.get('File') or view_definition.get('Data') or view_definition.get('data'): - pass - else: - logger.critical(f'\nCannot find view {view_name}. View information is incorrect, please check resources.yaml') - raise Exception(f'\nCannot find view {view_name}') + cur1_required = view_definition.get('dependsOn', dict()).get('cur') or view_definition.get('dependsOn', dict()).get('cur1') + cur2_required = view_definition.get('dependsOn', dict()).get('cur2') + #if cur_required and self.cur.has_savings_plans and self.cur.has_reservations and view_definition.get('spriFile'): + # view_definition['File'] = view_definition.get('spriFile') + #elif cur_required and self.cur.has_savings_plans and view_definition.get('spFile'): + # view_definition['File'] = view_definition.get('spFile') + #elif cur_required and self.cur.has_reservations and view_definition.get('riFile'): + # view_definition['File'] = view_definition.get('riFile') + #if view_definition.get('File') or view_definition.get('Data') or view_definition.get('data'): + # pass + #else: + # logger.critical(f'\nCannot find view {view_name}. View information is incorrect, please check resources.yaml') + # raise Exception(f'\nCannot find view {view_name}') # Load TPL file data = self.get_data_from_definition('view', view_definition) @@ -1741,9 +1739,14 @@ def get_view_query(self, view_name: str) -> str: # Prepare template parameters columns_tpl = { - 'cur_table_name': self.cur.table_name if cur_required else None, - 'athenaTableName': view_name, + #'athena_datasource_arn': athena_datasource.arn, 'athena_database_name': self.athena.DatabaseName, + 'cur_database': self.cur1.database if cur1_required else None, # for backward compatibly + 'cur_table_name': self.cur1.table_name if cur1_required else None, # for backward compatibly + 'cur1_database': self.cur1.database if cur1_required else None, + 'cur1_table_name': self.cur1.table_name if cur1_required else None, + 'cur2_database': self.cur2.database if cur2_required else None, + 'cur2_table_name': self.cur2.table_name if cur2_required else None, } columns_tpl = self.get_template_parameters( @@ -1770,7 +1773,7 @@ def csv2view(self, **kwargs): def map(self, **kwargs): """Create account mapping Athena views""" for v in ['account_map', 'aws_accounts']: - self.accountMap.create(v) + self.create_or_update_account_map(v) @command def teardown(self, **kwargs): @@ -1786,6 +1789,24 @@ def init_qs(self, **kwargs): """ Initialize QuickSight resources for deployment """ return InitQsCommand(cid=self, **kwargs).execute() + @command + def create_cur_proxy(self, cur_version=None, fields=None, **kwargs): + cid_print(f'Using CUR {self.cur.table_name}') # need to call self.cur + cur_version = cur_version or get_parameter( + 'cur-version', + message='Enter a version of CUR you want to create or update', + choices=['1', '2'], + ) + if cur_version.startswith('1'): + cur_proxy = self.cur1 + if cur_version.startswith('2'): + cur_proxy = self.cur2 + fields = get_parameters().get('fields', []) + cur_proxy.metadata + cur_proxy.proxy.fields_to_expose += (fields.split(',') if fields else []) + cur_proxy.proxy.create_or_update_view() + print('done') + @command def create_cur_table(self, **kwargs): """ Initialize CUR """ diff --git a/cid/export.py b/cid/export.py index cc8935da..06aea2a0 100644 --- a/cid/export.py +++ b/cid/export.py @@ -183,18 +183,23 @@ def export_analysis(qs, athena, glue): #FIXME add value['Source']['DataSetArn'] to the list of dataset_arn raise CidCritical(f"DataSet {dataset.raw['Name']} contains unsupported join. Please replace join of {value.get('Alias')} from DataSet to DataSource") - dep_cur = False + cur_version = False for dep_view in dependency_views[:]: - if cur_helper.table_is_cur(name=dep_view): + version = cur_helper.table_is_cur(name=dep_view) + if version: dependency_views.remove(dep_view) - dep_cur = True + cur_version = True datasets[dataset_name] = { 'data': dataset_data, 'dependsOn': {'views': dependency_views}, 'schedules': ['default'], #FIXME: need to read a real schedule } - if dep_cur: + # FIXME: add a list of all columns used in the view + if cur_version == '1': datasets[dataset_name]['dependsOn']['cur'] = True + elif cur_version == '2': + datasets[dataset_name]['dependsOn']['cur2'] = True + all_views = [view_and_database[0] for view_and_database in all_views_and_databases] all_databases = [view_and_database[1] for view_and_database in all_views_and_databases] if len(all_databases) > 1: @@ -213,7 +218,7 @@ def export_analysis(qs, athena, glue): cur_tables = [] for key, view_data in all_views_data.items(): if all_databases and isinstance(view_data.get('data'), str): - view_data['data'] = view_data['data'].replace(f'{all_databases[0]}.', '${athena_database_name}.') + view_data['data'] = view_data['data'].replace(f'{all_databases[0]}.', '"${athena_database_name}".') if isinstance(view_data.get('data'), str): view_data['data'] = view_data['data'].replace('CREATE VIEW ', 'CREATE OR REPLACE VIEW ') @@ -224,12 +229,17 @@ def export_analysis(qs, athena, glue): for dep_view in deps.get('views', []): dep_view_name = dep_view.split('.')[-1] if dep_view_name in cur_tables or cur_helper.table_is_cur(name=dep_view_name): + cur_version = cur_helper.table_is_cur(name=dep_view_name) logger.debug(f'{dep_view_name} is cur') view_data['dependsOn']['cur'] = True # replace cur table name with a variable if isinstance(view_data.get('data'), str): # cur tables treated separately as we don't manage CUR table here - view_data['data'] = view_data['data'].replace(f'{dep_view_name}', '${cur_table_name}') + if dep_view_name != 'cur': + backslash = "\\" # workaround f-string limitation + view_data['data'] = view_data['data'].replace(f'{dep_view_name}', f'"${backslash}cur{cur_version}_database{backslash}"."${backslash}cur{cur_version}_table_name{backslash}"') + else: + pass # FIXME: this replace is too dangerous as cur can be a part of other words. Need to find some other way cur_tables.append(dep_view_name) else: logger.debug(f'{dep_view_name} is not cur') diff --git a/cid/helpers/__init__.py b/cid/helpers/__init__.py index f83a3117..17d48aba 100644 --- a/cid/helpers/__init__.py +++ b/cid/helpers/__init__.py @@ -2,17 +2,20 @@ from cid.helpers.s3 import S3 from cid.helpers.athena import Athena from cid.helpers.iam import IAM -from cid.helpers.cur import CUR +from cid.helpers.cur import CUR, ProxyCUR from cid.helpers.diff import diff from cid.helpers.quicksight import QuickSight, Dashboard, Dataset, Datasource, Template from cid.helpers.csv2view import csv2view from cid.helpers.organizations import Organizations +from cid.helpers.cur_proxy import ProxyView +from cid.helpers.cloudformation import CFN __all__ = [ "Athena", "S3", "IAM", "CUR", + "ProxyCUR", "Glue", "QuickSight", "Dashboard", @@ -22,4 +25,6 @@ "diff", "csv2view", "Organizations", + "ProxyView", + "CFN", ] diff --git a/cid/helpers/account_map.py b/cid/helpers/account_map.py index dd44176e..2681f56e 100644 --- a/cid/helpers/account_map.py +++ b/cid/helpers/account_map.py @@ -1,209 +1,186 @@ -""" Create an Account Map +""" Account Map + +Account Map is an athena view that typically contains 'account_id' 'account_name' and other attributes of account (Business Unit, Scope, Cost Center or other). + +There are several Inputs and several Outputs that we need to support. + +# Output +There are 2 kinds of account map supported as output: + 1) account_map (used in most dashboards) + Columns: + account_id, + account_name, + plus additional fields can be added by user + + 2) aws_accounts (used only in Trends) + Columns: + account_id, + account_name, + parent_account_id, + account_email_id, + account_status + +# Input +this module supports several input +1) Pull from account metadata table (table containing accounts and data like name, tags, orgs etc). Several tables can be used to source information about Accounts. One is the source from organization_data provided by CID data collection lab +2) Dummy - a simple mapping from CUR. +3) CSV file in a format compatible with AWS Organization CSV output #FIXME add link +4) AWS Organization (rare and not recommended case when we install in Management Account or Delegated account that has access to the AWS Org) + +If user do not specify the choice explicitly, we will try to autodetect. If metadata table is found we will use that if not - we will propose a choice to the user. If unattended mode will use dummy. -Account Map is needed for mapping account id to account name and other attributes (Business Unit, Scope, Cost Center or other) """ + import os import logging from pathlib import Path from functools import lru_cache from string import Template +from boto3.session import Session from pkg_resources import resource_string from cid.base import CidBase from cid.helpers import Athena, CUR -from cid.utils import get_parameter, get_parameters, cid_print, unset_parameter +from cid.utils import get_parameter, get_parameters, cid_print, unset_parameter, set_parameters from cid.exceptions import CidCritical, CidError from cid.helpers.csv2view import read_csv logger = logging.getLogger(__name__) class AccountMap(CidBase): + """ Create an account_map view or aws_accounts """ - Create an account_map view or aws_accounts view with following mandatory fields: - account_id - account_name - parent_account_id (only used in trends) - account_status (only used in trends) - account_email (only used in trends) - - aws_accounts is only used in trends. - """ - defaults = { - 'MetadataTableNames': ['acc_metadata_details', 'organisation_data'] - } - _clients = {} - _accounts = [] - _metadata_source: str = None - _athena_table_name: str = None + # different metadata tables have different keys for the information we need. So here is a mapping showing for each + # account map we support, what are the keys depending on the source table: + # mappings[target_map][source][key] = key_in_source table mappings = { 'account_map': { 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name'}, - 'organisation_data': {'account_id': 'id', 'account_name': 'name', 'email': 'email'} + 'organization_data': {'account_id': 'id', 'account_name': 'name', 'email': 'email'} }, 'aws_accounts': { 'acc_metadata_details': {'account_id': 'account_id', 'account_name': 'account_name', 'email': 'email'}, - 'organisation_data': { 'account_id': 'id', 'account_name': 'name', 'email': 'email', 'status': 'status'}, - 'cur_fields': ['bill_payer_account_id'] + 'organization_data': { 'account_id': 'id', 'account_name': 'name', 'email': 'email', 'status': 'status'}, } } - - @property - def athena(self) -> Athena: - """ Getter for Athena """ - if not self._clients.get('athena'): - self._clients.update({ - 'athena': Athena(self.session) - }) - return self._clients.get('athena') - - @athena.setter - def athena(self, client) -> Athena: - """ Setter for Athena """ - if not self._clients.get('athena'): - self._clients.update({ - 'athena': client - }) - return self.athena - - @property - def cur(self) -> CUR: - """ Getter for CUR """ - return self._clients.get('cur') - - @cur.setter - def cur(self, client) -> CUR: - """ Setter for CUR """ - if not self._clients.get('cur'): - self._clients.update({ - 'cur': client - }) - return self.cur - - @property - def accounts(self) -> dict: - """ Get Accounts with the right mapping """ - if self._accounts: - # Required keys mapping, used in renaming below - key_mapping = { - 'accountid': 'account_id', - 'name': 'account_name' - } - for account in self._accounts: - # Rename required keys according to expected names - for old_key, new_key in key_mapping.items(): - if not account.get(new_key) and account.get(old_key): - account[new_key] = account[old_key] - del account[old_key] - # Fill optional keys - account.update({ - 'parent_account_id': account.get('parent_account_id', '0'), - 'account_status': account.get('account_status', 'unknown'), - 'account_email': account.get('account_email', 'unknown') - }) - return self._accounts + def __init__(self, session: Session, athena: Athena, cur=None) -> None: + self.cur = cur # Only for trends and dummy + self.athena = athena + super().__init__(session) @lru_cache(1000) - def detect_metadata_table(self, name): - """ detect meta table with the list of accounts """ - cid_print('Autodiscovering metadata table') - - # FIXME: This will only work for current Athena Database. We might want to check optimization_data base as well - tables = self.athena.list_table_metadata() - tables = [t for t in tables if t.get('TableType') == 'EXTERNAL_TABLE'] #filter only tables - tables = [t for t in tables if t.get('Name') in self.defaults.get('MetadataTableNames')] #filter only with support names - if not tables: - cid_print('Account metadata not detected') - return None - for table in tables: - table_name = table.get('Name') - logger.debug(f"Detected table {table_name}") - field_found = [col.get('Name') for col in table.get('Columns')] - field_required = list(self.mappings.get(name).get(table_name).values()) - # Check if we have all the required fields - if all(field in field_found for field in field_required): - logger.info(f'All required fields found in {table_name} ') - return table.get('Name') - logger.info('Missing required fields') - logger.info(f"Detected fields: {field_found}") - logger.info(f"Required fields: {field_required}") - return None - - - def create(self, name) -> bool: - """Create account map""" + def detect_metadata_table(self, target_name): + """ Detect a table with the list of accounts + + This function scans all tables in the current database and few others, searching for a table + with one of given names and a set of field. The found table most likely contains the list of + accounts and their metadata. + + :target view: a target view (output) + :returns: a table object + """ + cid_print('Autodiscover metadata table') + databases_to_look_for = set([ + self.athena.DatabaseName, # current database + 'organization_data', # from data collection + 'organisation_data', # from older versions of data collection + # FIXME probably also can be customizable via command line parameter + ]) + found_metadata_table = None + for database_name in databases_to_look_for: + try: + athena = Athena(session=self.athena.session, database_name=database_name) + tables = athena.list_table_metadata() + except Exception as exc: + logger.debug(f'{type(exc)}: {exc}') + continue + tables = [t for t in tables if t.get('TableType') == 'EXTERNAL_TABLE'] #filter only tables + tables = [t for t in tables if t.get('Name') in self.mappings[target_name]] #filter only with supported names + for table in tables: + table_name = table.get('Name') + logger.debug(f"Detected table {table_name}") + field_found = [col.get('Name') for col in table.get('Columns')] + field_required = list(self.mappings[target_name][table_name].values()) + # Check if we have all the required fields + if all(field in field_found for field in field_required): + logger.debug(f'All required fields found in {database_name}.{table_name} ') + table['Database'] = database_name + found_metadata_table = table # return the first found. FIXME: probably need a choice if more then one + break + if found_metadata_table: + break + return found_metadata_table + + def create_sql_from_metadata(self, name, metadata_table): + ''' create account map from the found metadata table ''' + view_definition = self.athena._resources.get('views').get(name, {}) + if view_definition.get('File'): + view_file = view_definition.get('File') + template = Template( + resource_string( + view_definition.get('providedBy'), + f'data/queries/{view_file}' + ).decode('utf-8') + ) + elif view_definition.get('data'): + template = Template(str(view_definition.get('data'))) + else: + raise CidError(f'{name} definition does not contain File or data: {view_definition}') + + # Fill template variables + vars = { + 'metadata_table_name': metadata_table['Name'], + 'metadata_database_name': metadata_table['Database'], + 'cur_database': self.cur.database, # only for trends in metadata + 'cur_table_name': self.cur.table_name, # only for trends in metadata + } + for key, val in self.mappings.get(name).get(metadata_table['Name']).items(): + logger.debug(f'Mapping field {key} to {val}') + vars[key] = val + compiled_query = template.safe_substitute(vars) + return compiled_query + def create_or_update(self, name) -> bool: + """Create account map""" cid_print(f'Creating account mapping {name}') - try: - if self.accounts: - logger.info('Account information found, skipping autodiscovery') - raise CidError('Account information found, skipping autodiscovery') - if get_parameters().get('account-map-source'): - raise CidError('Skipping autodiscovery') - - self._athena_table_name = self.detect_metadata_table(name) - if not self._athena_table_name: - raise CidError('Metadata table not found') - - # Query path - view_definition = self.athena._resources.get('views').get(name, {}) - if view_definition.get('File'): - view_file = view_definition.get('File') - template = Template(resource_string(view_definition.get('providedBy'), f'data/queries/{view_file}').decode('utf-8')) - elif view_definition.get('data'): - template = Template(str(view_definition.get('data'))) - else: - raise CidError(f'{name} definition does not contain File or data: {view_definition}') - - # Fill in TPLs - columns_tpl = { - 'metadata_table_name': self._athena_table_name, - 'cur_table_name': self.cur.table_name # only for trends - } - for key, val in self.mappings.get(name).get(self._athena_table_name).items(): - logger.debug(f'Mapping field {key} to {val}') - columns_tpl[key] = val - compiled_query = template.safe_substitute(columns_tpl) - logger.debug('compiled view.') - - except CidError as exc: - logger.info(exc) - compiled_query = self.create_account_mapping_sql(name) - - # Execute query - cid_print('Creating Athena view') - self.athena.query(compiled_query) - cid_print(f'Created account mapping {name}') + compiled_query = self.create_account_mapping_sql(name) + self.athena.create_or_update_view(name, compiled_query) def get_dummy_account_mapping_sql(self, name) -> list: """Create dummy account mapping""" logger.info(f'Creating dummy account mapping for {name}') + if self.cur.version.startswith('2'): + sql_file = 'data/queries/shared/account_map_cur2.sql' + else: + sql_file = 'data/queries/shared/account_map_dummy.sql' template = Template(resource_string( package_or_requirement='cid.builtin.core', - resource_name='data/queries/shared/account_map_dummy.sql', + resource_name=sql_file, ).decode('utf-8')) columns_tpl = { 'athena_view_name': name, - 'cur_table_name': self.cur.table_name + 'cur_table_name': self.cur.table_name, + 'cur_database': self.cur.database, } compiled_query = template.safe_substitute(columns_tpl) return compiled_query def get_organization_accounts(self) -> list: """ Retrieve AWS Organization account """ - # Init clients + cid_print('\nCollecting account info from AWS Organizations') accounts = [] orgs = self.session.client('organizations') try: - for page in orgs.get_paginator('list_accounts').paginate(): - for account in page['Accounts']: - accounts.append({ - 'account_id': account.get('Id'), - 'account_name': account.get('Name'), - 'account_status': account.get('Status'), #Only for Trends - 'account_email': account.get('Email'), #Only for Trends - }) + for account in orgs.get_paginator('list_accounts').paginate().search('Accounts'): + accounts.append({ + 'account_id': account.get('Id'), + 'account_name': account.get('Name'), + 'account_status': account.get('Status'), #Only for Trends + 'account_email': account.get('Email'), #Only for Trends + }) + cid_print(f'{len(accounts)} accounts collected') except orgs.exceptions.AWSOrganizationsNotInUseException: cid_print('AWS Organization is not enabled') except orgs.exceptions.AccessDeniedException: @@ -217,73 +194,54 @@ def check_file_exists(self, file_path) -> bool: """ Checks if the given file exists """ # Set base paths abs_path = Path().absolute() - return Path.is_file(abs_path / file_path) - def get_csv_accounts(self, file_path) -> list: + def get_csv_accounts(self) -> list: """ Retrieve accounts from CSV file """ + # get csv file from user + finished = False + while not finished: + mapping_file = get_parameter( + param_name='account-map-file', + message="Enter file path", + ) + mapping_file = os.path.expanduser(mapping_file) + finished = self.check_file_exists(mapping_file) + if not finished: + cid_print(f'File not found: {mapping_file}') + unset_parameter('account-map-file') + cid_print(f'\nReading {mapping_file}') + accounts = [ {str(k).lower().replace(" ", "_"): str(v) for k, v in row.items()} - for row in read_csv(file_path) + for row in read_csv(mapping_file) ] - + cid_print(f'{len(accounts)} accounts collected') return accounts - def select_metadata_collection_method(self) -> str: - """ Selects the method to collect metadata """ - logger.info('Metadata source selection') - # Ask user which method to use to retrieve account list - account_map_sources = { - 'Dummy (CUR account data, no names)': 'dummy', - 'AWS Organizations (one time account listing)': 'organization', - 'CSV file (relative path required)': 'csv', - } - selected_source = get_parameter( - param_name='account-map-source', - message="Please select account metadata collection method", - choices=account_map_sources, - ) - logger.info(f'Selected {selected_source}') - self._metadata_source = selected_source - - # Collect account list from different sources of user choice - if self._metadata_source == 'csv': - finished = False - while not finished: - mapping_file = get_parameter( - param_name='account-map-file', - message="Enter file path", - ) - mapping_file = os.path.expanduser(mapping_file) - finished = self.check_file_exists(mapping_file) - if not finished: - cid_print(f'File not found: {mapping_file}') - unset_parameter('account-map-file') - cid_print(f'\nReading {mapping_file}') - self._accounts = self.get_csv_accounts(mapping_file) - cid_print(f'{len(self.accounts)} accounts collected') - elif self._metadata_source == 'organization': - cid_print('\nCollecting account info from AWS Organizations') - self._accounts = self.get_organization_accounts() - cid_print(f'{len(self.accounts)} accounts collected') - elif self._metadata_source == 'dummy': - cid_print('Notice: Dummy account mapping will be created.') - else: - cid_print('Unsupported selection') - - def create_account_mapping_sql(self, name) -> str: - """ Returns account mapping Athena query """ - for attempt in range(3): - if self.accounts or self._metadata_source == 'dummy': - break - self.select_metadata_collection_method() - logger.info(f'Attempt {attempt + 2}' ) - else: - raise CidCritical('Failed to create account map') + def get_sql_for_accounts(self, name, accounts) -> str: + ''' return a sql when given accounts''' - if self._metadata_source == 'dummy': - return self.get_dummy_account_mapping_sql(name) + # Different tables have different keys, so we need to align keys from different source tables + accounts = accounts.copy() + # Required keys mapping, used in renaming below + key_mapping = { + 'accountid': 'account_id', + 'name': 'account_name' + } + for account in accounts: + # Rename required keys according to expected names + for old_key, new_key in key_mapping.items(): + if not account.get(new_key) and account.get(old_key): + account[new_key] = account[old_key] + del account[old_key] + # Fill optional keys with defaults + account.update({ + 'parent_account_id': account.get('parent_account_id', '0'), + 'account_status': account.get('account_status', 'unknown'), + 'account_email': account.get('account_email', 'unknown') + }) template_str = '''CREATE OR REPLACE VIEW ${athena_view_name} AS SELECT @@ -295,15 +253,60 @@ def create_account_mapping_sql(self, name) -> str: template = Template(template_str) accounts_sql = [] row_template = """ROW ('{account_id}', '{account_name}:{account_id}', '{parent_account_id}', '{account_status}', '{account_email}')""" - for account in self.accounts: + for account in accounts: acc = account.copy() - account_name = acc.pop('account_name').replace("'", "''") - accounts_sql.append(row_template.format(account_name=account_name, **acc)) - # Fill in TPLs + acc['account_name'] = acc['account_name'].replace("'", "''") #escape SQL characters + accounts_sql.append(row_template.format(**acc)) + # Fill template columns_tpl = { 'athena_view_name': name, 'rows': ','.join(accounts_sql) } compiled_query = template.safe_substitute(columns_tpl) - return compiled_query + + + def create_account_mapping_sql(self, name) -> str: + """ Returns Athena query for account mapping, based on user's choice and reasonable defaults """ + + if not get_parameters().get('account-map-source'): + # try to find a table with data about accounts if no explicit choice + try: + metadata_table = self.detect_metadata_table(name) + if metadata_table: + return self.create_sql_from_metadata(name=name, metadata_table=metadata_table) + except CidError as exc: + logger.debug(exc) + + if self.cur.version.startswith('2'): + cid_print('Looks like CUR2 is used. Will use it for account map.') + set_parameters({'account-map-source': 'dummy'}) + + for attempt in range(3): + try: + metadata_source = get_parameter( + param_name='account-map-source', + message="Please select account metadata collection method", + choices={ + 'Dummy (CUR account data, no names for CUR1)': 'dummy', + 'AWS Organizations (one time account listing)': 'organization', + 'CSV file (relative path required)': 'csv', + }, + ) + logger.info(f'Attempt {attempt + 2}' ) + # Collect account list from different sources of user choice + if metadata_source == 'csv': + accounts = self.get_csv_accounts() + compiled_query = self.get_sql_for_accounts(name=name, accounts=accounts) + elif metadata_source == 'organization': + accounts = self.get_organization_accounts() + compiled_query = self.get_sql_for_accounts(name=name, accounts=accounts) + elif metadata_source == 'dummy': + compiled_query = self.get_dummy_account_mapping_sql(name) + else: + cid_print('Unsupported selection') + return compiled_query + except Exception as exc: + logger.debug(exc) + else: + raise CidCritical('Failed to create account map') \ No newline at end of file diff --git a/cid/helpers/athena.py b/cid/helpers/athena.py index 22b4ae26..87396cc3 100644 --- a/cid/helpers/athena.py +++ b/cid/helpers/athena.py @@ -5,7 +5,7 @@ from cid.base import CidBase from cid.helpers import S3 -from cid.utils import get_parameter, get_parameters, cid_print +from cid.utils import get_parameter, get_parameters, cid_print, isatty, unset_parameter, get_yesno_parameter from cid.helpers.diff import diff from cid.exceptions import CidCritical, CidError @@ -25,9 +25,11 @@ class Athena(CidBase): _resources = dict() _client = None - def __init__(self, session, resources: dict=None) -> None: + def __init__(self, session, resources: dict=None, database_name: str=None) -> None: super().__init__(session) self._resources = resources + if database_name: + self.DatabaseName = database_name # this can raise AccessDenied or CidError if database not found @property def client(self): @@ -39,19 +41,17 @@ def client(self): def CatalogName(self) -> str: """ Check if AWS DataCatalog and Athena database exist """ if not self._CatalogName: - # Get AWS Glue DataCatalogs - glue_data_catalogs = [d for d in self.list_data_catalogs() if d['Type'] == 'GLUE'] - if not len(glue_data_catalogs): - logger.error('AWS DataCatalog of type GLUE not found!') - self._status = 'AWS DataCatalog of type GLUE not found' + glue_data_catalogs = self.list_data_catalogs() + if not glue_data_catalogs: + raise CidCritical('AWS DataCatalog of type GLUE not found!') if len(glue_data_catalogs) == 1: - self._CatalogName = glue_data_catalogs.pop().get('CatalogName') + self._CatalogName = glue_data_catalogs[0] elif len(glue_data_catalogs) > 1: # Ask user self._CatalogName = get_parameter( param_name='glue-data-catalog', message="Select AWS DataCatalog to use", - choices=[catalog.get('CatalogName') for catalog in glue_data_catalogs], + choices=glue_data_catalogs, ) logger.info(f'Using DataCatalog: {self._CatalogName}') return self._CatalogName @@ -63,7 +63,8 @@ def set_catalog_name(self, catalog): @property def DatabaseName(self) -> str: """ Check if Athena database exist """ - if self._DatabaseName: return self._DatabaseName + if self._DatabaseName: + return self._DatabaseName if get_parameters().get('athena-database'): self._DatabaseName = get_parameters().get('athena-database') @@ -80,17 +81,18 @@ def DatabaseName(self) -> str: athena_databases = self.list_databases() # check if we have a default database - default_databases = [database for database in athena_databases if database['Name'] == self.defaults.get('DatabaseName')] + print(athena_databases) + default_databases = [database for database in athena_databases if database == self.defaults.get('DatabaseName')] # Ask user - choices = [d['Name'] for d in athena_databases] + choices = list(athena_databases) if self.defaults.get('DatabaseName') not in choices: choices.append(self.defaults.get('DatabaseName') + ' (CREATE NEW)') self._DatabaseName = get_parameter( param_name='athena-database', message="Select AWS Athena database to use", choices=choices, - default=default_databases[0]['Name'] if default_databases else None, + default=default_databases[0] if default_databases else None, ) if self._DatabaseName.endswith( ' (CREATE NEW)'): self._DatabaseName = self.defaults.get('DatabaseName') @@ -99,8 +101,12 @@ def DatabaseName(self) -> str: return self._DatabaseName @DatabaseName.setter - def DatabaseName(self, database): - self._DatabaseName = database + def DatabaseName(self, database_name): + database = self.get_database(database_name) # this can raise AccessDenied error + if database: + self._DatabaseName = database_name + else: + raise CidError(f'Database {database_name} not found') @property def WorkGroup(self) -> str: @@ -162,7 +168,7 @@ def _ensure_workgroup(self, name: str) -> str: if name == 'primary': # QuickSight manages primary wg differently, relying exclusively on bucket with a predefined name bucket_name = f'{self.partition}-athena-query-results-{self.region}-{self.account_id}' else: - bucket_name = f'{self.partition}-athena-query-results-cid-{self.account_id}-{self.region}' + bucket_name = f'{self.partition}-athena-query-results-cidcmd-{self.account_id}-{self.region}' try: workgroup = self.client.get_work_group(WorkGroup=name) @@ -222,11 +228,16 @@ def _ensure_workgroup(self, name: str) -> str: logger.exception(exc) raise CidCritical(f'Failed to create Athena work group ({exc})') from exc - def list_data_catalogs(self) -> list: - return self.client.list_data_catalogs().get('DataCatalogsSummary') + def list_data_catalogs(self, type_: str='GLUE', workgroup: str=None) -> list: + """get data catalogs""" + params = {} + if workgroup: + params['WorkGroup'] = workgroup + return list(self.client.get_paginator('list_data_catalogs').paginate(**params).search(f"DataCatalogsSummary[?Type == '{type_}'].CatalogName")) - def list_databases(self) -> list: - return self.client.list_databases(CatalogName=self.CatalogName).get('DatabaseList') + def list_databases(self, catalog_name: str=None) -> list: + """get database""" + return list(self.client.get_paginator('list_databases').paginate(CatalogName=catalog_name or self.CatalogName).search('DatabaseList[].Name')) def get_database(self, DatabaseName: str=None) -> bool: """ Check if AWS DataCatalog and Athena database exist """ @@ -240,10 +251,10 @@ def get_database(self, DatabaseName: str=None) -> bool: logger.debug(exc, exc_info=True) return None - def list_table_metadata(self, DatabaseName: str=None, max_items: int=None) -> dict: + def list_table_metadata(self, database_name: str=None, max_items: int=None) -> dict: params = { 'CatalogName': self.CatalogName, - 'DatabaseName': DatabaseName or self.DatabaseName, + 'DatabaseName': database_name or self.DatabaseName, 'PaginationConfig':{ 'MaxItems': max_items, }, @@ -255,9 +266,9 @@ def list_table_metadata(self, DatabaseName: str=None, max_items: int=None) -> di for page in response_iterator: table_metadata.extend(page.get('TableMetadataList')) logger.debug(f'Table metadata: {table_metadata}') - logger.info(f'Found {len(table_metadata)} tables in {DatabaseName if DatabaseName else self.DatabaseName}') + logger.info(f'Found {len(table_metadata)} tables in {database_name or self.DatabaseName}') except Exception as e: - logger.error(f'Failed to list tables in {DatabaseName if DatabaseName else self.DatabaseName}') + logger.error(f'Failed to list tables in {database_name or self.DatabaseName}') logger.error(e) return table_metadata @@ -268,16 +279,16 @@ def list_work_groups(self) -> list: logger.debug(f'WorkGroups: {result.get("WorkGroups")}') return result.get('WorkGroups') - def get_table_metadata(self, TableName: str) -> dict: - table_metadata = self._metadata.get(TableName) + def get_table_metadata(self, table_name: str, database_name: str=None) -> dict: + table_metadata = self._metadata.get(table_name) params = { 'CatalogName': self.CatalogName, - 'DatabaseName': self.DatabaseName, - 'TableName': TableName + 'DatabaseName': database_name or self.DatabaseName, + 'TableName': table_name } if not table_metadata: table_metadata = self.client.get_table_metadata(**params).get('TableMetadata') - self._metadata.update({TableName: table_metadata}) + self._metadata.update({table_name: table_metadata}) return table_metadata @@ -311,7 +322,7 @@ def execute_query(self, sql_query, sleep_duration=1, database: str=None, catalog raise CidCritical(f'InvalidRequestException: {exc}') from exc except Exception as exc: logger.debug(f'Full query: {sql_query}') - raise CidCritical(f'Athena query failed: {exc}') from exc + raise CidCritical(f'Query:\n{sql_query}\n\nAthena query failed: {exc}') from exc current_status = query_status['QueryExecution']['Status']['State'] @@ -329,7 +340,7 @@ def execute_query(self, sql_query, sleep_duration=1, database: str=None, catalog logger.info(f'Athena query failed: {failure_reason}') logger.debug(f'Full query: {sql_query}') if fail: - raise CidCritical(f'Athena query failed: {failure_reason}') + raise CidCritical(f'Query:\n{sql_query}\n\nAthena query status failed : {failure_reason}') return False def get_query_results(self, query_id): @@ -359,7 +370,7 @@ def discover_views(self, views: dict={}) -> None: """ Discover views from a given list of view names and cache them. """ for view_name in views: try: - self.get_table_metadata(TableName=view_name) + self.get_table_metadata(table_name=view_name) except self.client.exceptions.MetadataException: pass @@ -512,4 +523,87 @@ def _recursively_process_view(view): for view in views: _recursively_process_view(view) - return all_views \ No newline at end of file + return all_views + + def create_or_update_view(self, view_name, view_query): + """ update view while asking user + """ + update_view = None + # first understand if view exists + self.discover_views([view_name]) + logger.trace(str(list(self._metadata.keys()))) + if view_name not in self._metadata: + update_view = True + else: # view exists + while get_parameters().get('on-drift', 'show').lower() != 'override' and isatty(): + cid_print(f'Analyzing view {view_name}') + diff = self.get_view_diff(view_name, view_query) + if diff and diff['diff']: + cid_print(f'Found a difference between existing view {view_name} and the one we want to deploy. ') + cid_print(diff['printable']) + choice = get_parameter( + param_name='view-' + view_name + '-override', + message=f'The existing view is different. Override?', + choices=['retry diff', 'proceed and override', 'keep existing', 'exit'], + default='retry diff' + ) + if choice == 'retry diff': + unset_parameter('view-' + view_name + '-override') + continue + elif choice == 'proceed and override': + update_view = True + break + elif choice == 'keep existing': + update_view = False + break + else: + raise CidCritical(f'User choice is not to update {view_name}.') + elif not diff: + if not get_yesno_parameter( + param_name='view-' + view_name + '-override', + message=f'Cannot get sql diff for {view_name}. Continue?', + default='yes' + ): + raise CidCritical(f'User choice is not to update {view_name}.') + update_view = True + elif diff and not diff['diff']: + cid_print(f'No need to update {view_name}. Skipping.') + break + if update_view: + cid_print(f'Updating view: "{view_name}"') + self.execute_query(view_query) + + + def find_tables_with_columns_in_information_schema(self, columns): + """ find table and database that contain given set of columns + this function takes 1+ min to fetch information schema. For better performance consider find_tables_with_columns. + """ + columns = set(columns) + return self.execute_query(f''' + SELECT + table_schema, + table_name + FROM + information_schema.columns + WHERE + column_name IN ({[','.join(["'"+col+"'" for col in columns])]}) + GROUP BY + table_schema, + table_name + HAVING + COUNT(DISTINCT column_name) = {len(columns)} + ''') + + def find_tables_with_columns(self, columns: list, database_name: str=None, catalog_name: str=None, max_items: int=10000): + """ This function searches a table with a given set of columns + """ + iterator = self.client.get_paginator('list_table_metadata').paginate( + DatabaseName=database_name or self.DatabaseName, + CatalogName=catalog_name or self.CatalogName, + PaginationConfig= { + 'MaxItems': max_items, # sometimes customers can have 1'000s of tables (due to a crawler going crazy for example) + }, + ) + return iterator.search(f""" + TableMetadataList[?{' && '.join(["contains(Columns[].Name, '"+col+"' )" for col in columns])}].Name + """) diff --git a/cid/helpers/cloudformation.py b/cid/helpers/cloudformation.py new file mode 100644 index 00000000..26f1ea23 --- /dev/null +++ b/cid/helpers/cloudformation.py @@ -0,0 +1,58 @@ +""" CloudFormation Helper +""" +import logging +from functools import lru_cache + +from cid.base import CidBase + +logger = logging.getLogger(__name__) + + +class CFN(CidBase): + + def __init__(self, session): + super().__init__(session) + self.client = self.session.client('cloudformation', region_name=self.region) + + @lru_cache(1000) + def get_read_access_policies(self): + ''' returns a dict of read access policies provided by other stacks + + data collection stack can add CFN export ReadAccessPolicyARN + + return example: + { + "cid-DataCollection-ReadAccessPolicyARN": "arn:aws:iam::xxx:policy/CID-DC-DataCollectionReadAccess", + "cid-DataExports-ReadAccessPolicyARN": "arn:aws:iam::xxx:policy/cidDataExportsReadAccess", + "cid-SecurityHub-ReadAccessPolicyARN": "arn:aws:iam::xxx:policy/cid-SecurityHubReadAccess" + } + ''' + try: + res = dict( + self.client + .get_paginator('list_exports') + .paginate() + .search("Exports[? starts_with(Name, 'cid-') && ends_with(Name,'ReadAccessPolicyARN')][Name, Value]" + ) + ) + except self.client.exceptions.ClientError as exc: + if 'AccessDenied' in str(exc): + logger.warning('AccessDenied on cloudformation:ListExports. Most likely not critical.') + res = {} + else: + raise + return res + + @lru_cache(1000) + def get_read_access_policy(self, key): + ''' return a list of read access policies provided by other stacks + key: cfn export name + ''' + return self.get_read_access_policies().get(key, None) + + @lru_cache(1000) + def get_read_access_policy_for_module(self, module): + ''' return a list of read access policies provided by other stacks + module: module name (DataCollection DataExports or SecurityHub) + ''' + return self.get_read_access_policy(f'cid-{module}-ReadAccessPolicyARN') diff --git a/cid/helpers/cur.py b/cid/helpers/cur.py index f19b5657..e8c368d4 100644 --- a/cid/helpers/cur.py +++ b/cid/helpers/cur.py @@ -2,17 +2,22 @@ """ import json import logging +from abc import abstractmethod from cid.base import CidBase -from cid.utils import get_parameter, get_parameters, cid_print +from cid.utils import get_parameter, get_parameters, set_parameters, cid_print from cid.exceptions import CidCritical +from cid.helpers.cur_proxy import ProxyView + +from tqdm import tqdm logger = logging.getLogger(__name__) -class CUR(CidBase): +class AbstractCUR(CidBase): """ Manage AWS CUR """ + # Must be both CUR1 and 2 compatible cur_minimal_required_columns = [ 'bill_bill_type', 'bill_billing_entity', @@ -55,15 +60,24 @@ class CUR(CidBase): 'savings_plan_payment_option' ] _metadata = None + _database = None def __init__(self, athena, glue): self.athena = athena self.glue = glue + self.proxy = None + + @property + def database(self) -> str: + """get Athena database of CUR """ + if self.metadata is None: # Need explicit call of metadata. Do not remove this line + raise CidCritical('Error: Cannot detect any CUR table. Hint: Check if AWS Lake Formation is activated on your account, verify that the LakeFormationEnabled parameter is set to yes on the deployment stack') + return self._database @property def table_name(self) -> str: """ Get Athena table name """ - if self.metadata is None: + if self.metadata is None: # Need explicit call of metadata. Do not remove this line raise CidCritical('Error: Cannot detect any CUR table. Hint: Check if AWS Lake Formation is activated on your account, verify that the LakeFormationEnabled parameter is set to yes on the deployment stack') return self.metadata.get('Name') @@ -82,7 +96,12 @@ def has_savings_plans(self) -> bool: """ Return True if CUR has savings plan """ return all(col in self.fields for col in self.sp_required_columns) - def get_type_of_column(self, column: str): + @property + def version(self) -> str: + """ Return version of CUR """ + return '2' if 'bill_payer_account_name' in self.fields else '1' + + def get_type_of_column(self, column: str, version=None): """ Return an Athena type of a given non existent CUR column """ if column.startswith('cost_category_') or column.startswith('resource_tags_'): return 'STRING' @@ -91,7 +110,13 @@ def get_type_of_column(self, column: str): return 'DOUBLE' if column.endswith('_date') and not column.endswith('_to_date'): return 'TIMESTAMP' + if column.endswith('_time') and (version or self.version) == '2': + return 'TIMESTAMP' special_cases = { + "cost_category": "MAP", + "discount": "MAP", + "product": "MAP", + "resource_tags": "MAP", "reservation_amortized_upfront_cost_for_usage": "DOUBLE", "reservation_amortized_upfront_fee_for_billing_period": "DOUBLE", "reservation_recurring_fee_for_usage": "DOUBLE", @@ -111,12 +136,134 @@ def get_type_of_column(self, column: str): } return special_cases.get(column, 'STRING') + def column_exists(self, column: str): + return column.lower() in [col.lower() for col in self.fields] + + def ensure_columns(self, columns): + if not isinstance(columns, list): + logger.debug(f'{columns} must be list') + return + for column in columns: + self.ensure_column(column) + + @abstractmethod + def ensure_column(self, column: str, column_type: str=None): + """ Ensure column is in the cur. If it is not there - add column """ + pass + + def table_is_cur(self, table: dict=None, name: str=None, return_reason: bool=False) -> bool: + """ return cur version if table metadata fits CUR definition. """ + try: + table = table or self.athena.get_table_metadata(name) + except Exception as exc: #pylint: disable=broad-exception-caught + logger.warning(exc) + return False if not return_reason else (False, f'cannot get table {name}. {exc}.') + + table_name = table.get('Name') + columns = [col.get('Name') for col in table.get('Columns')] + missing_columns = [col for col in self.cur_minimal_required_columns if col not in columns] + logger.debug(f'missing_columns={missing_columns}') + if missing_columns: + return False if not return_reason else (False, f"Table {table_name} does not contain columns: {','.join(missing_columns)}. You can try ALTER TABLE {table_name} ADD COLUMNS (missing_column string).") + + version = '1' + if 'bill_payer_account_name' in columns: + version = '2' + return version if not return_reason else (True, 'all good') + + @property + def fields(self) -> list: + """get CUR fields """ + return [col.get('Name') for col in (self.metadata.get('Columns', []) + self.metadata.get('PartitionKeys', []))] + + @property + def tag_and_cost_category_fields(self) -> list: + """ Returns all tags and cost category fields. """ + if self.version == '1': + return [field for field in self.fields if field.startswith('resource_tags_user_') or field.startswith('cost_category_')] + elif self.version == '2': + raise NotImplemented('Need to run a query to get all fields of resource_tags') + else: + raise NotImplemented('cur version not known') + + +class CUR(AbstractCUR): + """This Class represents CUR table (1 or 2 versions)""" + + def __init__(self, athena, glue): + super().__init__(athena, glue) + + @property + def metadata(self) -> dict: + """get Athena metadata for the table of CUR """ + if self._metadata: + return self._metadata + self._database, self._metadata = self.find_cur() + # good place to set a database for athena + return self._metadata + + def find_cur(self): + """Choose CUR""" + metadata = None + cur_database = get_parameters().get('cur-database') + if get_parameters().get('cur-table-name'): + table_name = get_parameters().get('cur-table-name') + try: + metadata = self.athena.get_table_metadata(table_name, cur_database) + except self.athena.client.exceptions.MetadataException as exc: + raise CidCritical(f'Provided cur-table-name "{table_name}" in database "{cur_database or self.athena.DatabaseName}" is not found. Please make sure the table exists.') from exc + res, message = self.table_is_cur(table=metadata, return_reason=True) + if not res: + raise CidCritical(f'Table {table_name} does not look like CUR. {message}') + return cur_database or self.athena.DatabaseName, metadata + + all_cur_tables = [] + if cur_database: + cur_databases = [cur_database] + else: + cur_databases = list(self.athena.list_databases()) # user have not provided the cur database + for database in tqdm(cur_databases, desc='Fining CUR in Athena Databases'): + try: + tables = self.athena.find_tables_with_columns( + columns=self.cur_minimal_required_columns, + database_name=database, + ) + all_cur_tables += [(database, table) for table in tables] + except self.athena.client.exceptions.AccessDenied: + logger.info(f'Cannot read from athena database {database}') + + if not all_cur_tables: + # FIXME : distinguish a case where we have NONE tables in any database. This might be because + raise CidCritical( + f'CUR table not found in {self.athena.region}. We need a least one table with these columns: {self.cur_minimal_required_columns}.' + ' Please make sure you created cur and created Athena table, preferably with CID Cloud Formation stack.' + ' Also if you have AWS Lake Formation, the user running the tool might need additional permissions' + ) + databases = set([database for database, _ in all_cur_tables]) + if len(databases) > 1: + choices = dict(sorted([(f'{database}.{tab}', (database, tab)) for database, tab in all_cur_tables], reverse=True)) + else: + choices = dict(sorted([(f'{tab}', (database, tab)) for database, tab in all_cur_tables], reverse=True)) + if not choices: + choices[''] = '' + answer = get_parameter( + param_name='cur-table-name-and-db', + message="Please select CUR table", + choices=choices, + ) + if answer == '': + raise CidCritical('CUR creation was requested') # to be captured in common.py + database, table = answer + set_parameters({'cur-table-name': table,'cur-database': database, }) + return database, self.athena.get_table_metadata(table, database_name=database) + def ensure_column(self, column: str, column_type: str=None): """ Ensure column is in the cur. If it is not there - add column """ column = column.lower() - if column in [col.get('Name', '').lower() for col in self.metadata.get('Columns', [])]: + column = column.split('[')[0] + if self.column_exists(column): return - + logger.trace(f'trying to add to CUR following column: {column}') table_can_be_updated = False # Check Crawler Behavior - if it has a Configuration/CrawlerOutput/TablesAddOrUpdateBehavior != MergeNewColumns, it will override columns crawler_name = self.metadata.get('Parameters', {}).get('UPDATED_BY_CRAWLER') @@ -128,11 +275,11 @@ def ensure_column(self, column: str, column_type: str=None): if add_or_update == 'MergeNewColumns': table_can_be_updated = True except self.glue.client.exceptions.ClientError as exc: - logger.debug('Failed to get crawler info') + logger.debug(f'Failed to get crawler info: {exc}') if table_can_be_updated: column_type = column_type or self.get_type_of_column(column) try: - self.athena.query(f'ALTER TABLE {self.table_name} ADD COLUMNS ({column} {column_type})') + self.athena.query(f'ALTER TABLE {self.table_name} ADD COLUMNS ({column} {column_type})', database=self.database) except (self.athena.client.exceptions.ClientError, CidCritical) as exc: raise CidCritical(f'Column {column} is not found in CUR and we were unable to add it. Please check FAQ.') from exc # table takes time to update so just adding column to cached data @@ -146,73 +293,60 @@ def ensure_column(self, column: str, column_type: str=None): return # if a required column is not there and not ri/sp -> stop - logger.critical(f"Column '{column}' is not in ({self.table_name}). Cannot continue.") + logger.warning(f"Column '{column}' is not in CUR ({self.table_name}).") - def table_is_cur(self, table: dict=None, name: str=None, return_reason: bool=False) -> bool: - """ return True if table metadata fits CUR definition. """ - try: - table = table or self.athena.get_table_metadata(name) - except Exception as exc: #pylint: disable=broad-exception-caught - logger.debug(exc) - return False if not return_reason else (False, f'cannot get table {name}. {exc}.') - table_name = table.get('Name') - columns = [col.get('Name') for col in table.get('Columns')] - missing_columns = [col for col in self.cur_minimal_required_columns if col not in columns] - if missing_columns: - return False if not return_reason else (False, f"Table {table_name} does not contain columns: {','.join(missing_columns)}. You can try ALTER TABLE {table_name} ADD COLUMNS (missing_column string).") +class ProxyCUR(AbstractCUR): + """ Cur Proxy behaves as CUR (1 or 2) but actually is a proxy view in Athena + """ + def __init__(self, cur, target_cur_version): + self.proxy = [] + self.cur = cur + self.target_cur_version = target_cur_version + super().__init__(cur.athena, cur.glue) - return True if not return_reason else (True, 'all good') + @property + def database(self) -> str: + return self.athena.DatabaseName # use CID database for proxy view @property def metadata(self) -> dict: - """get Athena metadata for the table of CUR """ + """get Athena metadata for the view of CUR """ if self._metadata: return self._metadata + self.cur = CUR(self.athena, self.glue) + if self.cur.version != self.target_cur_version: + cid_print(f'CUR{self.cur.version} ({self.athena.DatabaseName}.{self.cur.table_name}) provided but {self.target_cur_version} is needed. CUR Proxy view will be used.') + self.proxy = ProxyView(cur=self.cur, target_cur_version=self.target_cur_version) + try: + self._metadata = self.athena.get_table_metadata(self.proxy.name) + except self.athena.client.exceptions.MetadataException: # if no table - create it + self.proxy.create_or_update_view() + self._metadata = self.athena.get_table_metadata(self.proxy.name) - if get_parameters().get('cur-table-name'): - table_name = get_parameters().get('cur-table-name') - try: - self._metadata = self.athena.get_table_metadata(table_name) - except self.athena.client.exceptions.ResourceNotFoundException as exc: - raise CidCritical(f'Provided cur-table-name "{table_name}" is not found. Please make sure the table exists.') from exc - except self.athena.client.exceptions.MetadataException as exc: - raise CidCritical(f'Provided cur-table-name "{table_name}" is not found in database "{self.athena.DatabaseName}". Please make sure the table exists. Also if you use LakeFormation, make sure that the user is granted with read rights.') from exc - res, message = self.table_is_cur(table=self._metadata, return_reason=True) - if not res: - raise CidCritical(f'Table {table_name} does not look like CUR. {message}') - else: - # Look all tables and filter ones with CUR fields - all_tables = self.athena.list_table_metadata() - if not all_tables: - logger.warning( - f'No tables found in Athena Database {self.athena.DatabaseName} in {self.athena.region}.' - f' (Hint: If you see tables in this Database, please check AWS Lake Formation permissions)' - ) - cur_tables = [tab for tab in all_tables if self.table_is_cur(table=tab)] - - if not cur_tables: - raise CidCritical(f'CUR table not found. (scanned {len(all_tables)} tables in Athena Database {self.athena.DatabaseName} in {self.athena.region}). But none has required fields: {self.cur_minimal_required_columns}.') - - choices = sorted([tab.get('Name') for tab in cur_tables], reverse=True) - - answer = get_parameter( - param_name='cur-table-name', - message="Please select CUR", - choices=choices + [''], - ) - if answer == '': - raise CidCritical('CUR creation was requested') # to be captured in common.py - self._metadata = self.athena.get_table_metadata(answer) return self._metadata - @property - def fields(self) -> list: - """get CUR fields """ - return [col.get('Name') for col in self.metadata.get('Columns', [])] - - @property - def tag_and_cost_category_fields(self) -> list: - """ Returns all tags and cost category fields. """ - return [field for field in self.fields if field.startswith('resource_tags_user_') or field.startswith('cost_category_')] + def ensure_columns(self, columns): + if not self.metadata:# To make sure metadata exists + raise RuntimeError('No metadata') + if isinstance(columns, list): + if all(self.proxy.column_surely_exist(col) for col in columns): + return + if self.cur.metadata.get('TableType') == 'EXTERNAL_TABLE': + try: + equivalent_columns = sum([self.proxy.source_column_equivalents(col) for col in columns], []) + logger.trace(f'equivalent_columns = {dict(zip(columns,equivalent_columns))}') + self.cur.ensure_columns(list(set(equivalent_columns))) + # add field from underlying cur to the proxy + for item in self.cur._metadata.get('Columns', []): + for existing_item in self._metadata.get('Columns', []): + if item['Name'] == existing_item['Name']: + break + else: # if not found + self._metadata.get('Columns', []).append(item) + except Exception as exc: + logger.exception(exc) + for column in columns: + self.proxy.fields_to_expose.append(column) + self.proxy.create_or_update_view() diff --git a/cid/helpers/cur_proxy.py b/cid/helpers/cur_proxy.py new file mode 100644 index 00000000..78e2f5e8 --- /dev/null +++ b/cid/helpers/cur_proxy.py @@ -0,0 +1,612 @@ +import re +import logging + +try: + from itertools import batched #python3.12+ +except ImportError: + from itertools import islice + def batched(iterable, n): + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): + yield batch + +logger = logging.getLogger(__name__) + +cur1to2_mapping = { + '''concat("year", '-' , "month")''': "billing_period", + 'year': """split_part("billing_period", '-', 1)""", + 'month': """split_part("billing_period", '-', 2)""", + 'identity_line_item_id': 'identity_line_item_id', + 'identity_time_interval': 'identity_time_interval', + 'pricing_lease_contract_length': 'pricing_lease_contract_length', + 'bill_invoice_id': 'bill_invoice_id', + 'bill_invoicing_entity': 'bill_invoicing_entity', + 'bill_billing_entity': 'bill_billing_entity', + 'pricing_offering_class': 'pricing_offering_class', + 'bill_bill_type': 'bill_bill_type', + "savings_plan_start_time": "savings_plan_start_time", + "savings_plan_end_time": "savings_plan_end_time", + "reservation_start_time": "reservation_start_time", + "reservation_end_time": "reservation_end_time", + 'bill_payer_account_id': 'bill_payer_account_id', + 'bill_billing_period_start_date': 'bill_billing_period_start_date', + 'bill_billing_period_end_date': 'bill_billing_period_end_date', + 'line_item_usage_account_id': 'line_item_usage_account_id', + 'line_item_line_item_type': 'line_item_line_item_type', + 'line_item_usage_start_date': 'line_item_usage_start_date', + 'line_item_usage_end_date': 'line_item_usage_end_date', + 'line_item_product_code': 'line_item_product_code', + 'line_item_usage_type': 'line_item_usage_type', + 'line_item_operation': 'line_item_operation', + 'line_item_availability_zone': 'line_item_availability_zone', + 'line_item_resource_id': 'line_item_resource_id', + 'line_item_usage_amount': 'line_item_usage_amount', + 'line_item_normalization_factor': 'line_item_normalization_factor', + 'line_item_normalized_usage_amount': 'line_item_normalized_usage_amount', + 'line_item_currency_code': 'line_item_currency_code', + 'line_item_unblended_rate': 'line_item_unblended_rate', + 'line_item_unblended_cost': 'line_item_unblended_cost', + 'line_item_blended_rate': 'line_item_blended_rate', + 'line_item_blended_cost': 'line_item_blended_cost', + 'line_item_line_item_description': 'line_item_line_item_description', + 'line_item_tax_type': 'line_item_tax_type', + 'line_item_legal_entity': 'line_item_legal_entity', + 'product_fee_code': 'product_fee_code', + 'product_fee_description': 'product_fee_description', + 'product_from_location': 'product_from_location', + 'product_from_location_type': 'product_from_location_type', + 'product_from_region_code': 'product_from_region_code', + 'product_instance_family': 'product_instance_family', + 'product_instance_type': 'product_instance_type', + 'product_location': 'product_location', + 'product_location_type': 'product_location_type', + 'product_operation': 'product_operation', + 'product_product_family': 'product_product_family', + 'product_region_code': 'product_region_code', + 'product_servicecode': 'product_servicecode', + 'product_sku': 'product_sku', + 'product_to_location': 'product_to_location', + 'product_to_location_type': 'product_to_location_type', + 'product_to_region_code': 'product_to_region_code', + 'product_usagetype': 'product_usagetype', + 'pricing_rate_code': 'pricing_rate_code', + 'pricing_rate_id': 'pricing_rate_id', + 'pricing_currency': 'pricing_currency', + 'pricing_public_on_demand_cost': 'pricing_public_on_demand_cost', + 'pricing_public_on_demand_rate': 'pricing_public_on_demand_rate', + 'pricing_term': 'pricing_term', + 'pricing_unit': 'pricing_unit', + 'pricing_purchase_option': 'pricing_purchase_option', + 'reservation_amortized_upfront_cost_for_usage': 'reservation_amortized_upfront_cost_for_usage', + 'reservation_amortized_upfront_fee_for_billing_period': 'reservation_amortized_upfront_fee_for_billing_period', + 'reservation_effective_cost': 'reservation_effective_cost', + 'reservation_modification_status': 'reservation_modification_status', + 'reservation_normalized_units_per_reservation': 'reservation_normalized_units_per_reservation', + 'reservation_number_of_reservations': 'reservation_number_of_reservations', + 'reservation_recurring_fee_for_usage': 'reservation_recurring_fee_for_usage', + 'reservation_subscription_id': 'reservation_subscription_id', + 'reservation_total_reserved_normalized_units': 'reservation_total_reserved_normalized_units', + 'reservation_total_reserved_units': 'reservation_total_reserved_units', + 'reservation_units_per_reservation': 'reservation_units_per_reservation', + 'reservation_unused_amortized_upfront_fee_for_billing_period': 'reservation_unused_amortized_upfront_fee_for_billing_period', + 'reservation_unused_normalized_unit_quantity': 'reservation_unused_normalized_unit_quantity', + 'reservation_unused_quantity': 'reservation_unused_quantity', + 'reservation_unused_recurring_fee': 'reservation_unused_recurring_fee', + 'reservation_upfront_value': 'reservation_upfront_value', + 'savings_plan_total_commitment_to_date': 'savings_plan_total_commitment_to_date', + 'savings_plan_savings_plan_a_r_n': 'savings_plan_savings_plan_a_r_n', + 'savings_plan_savings_plan_rate': 'savings_plan_savings_plan_rate', + 'savings_plan_used_commitment': 'savings_plan_used_commitment', + 'savings_plan_savings_plan_effective_cost': 'savings_plan_savings_plan_effective_cost', + 'savings_plan_amortized_upfront_commitment_for_billing_period': 'savings_plan_amortized_upfront_commitment_for_billing_period', + 'savings_plan_recurring_commitment_for_billing_period': 'savings_plan_recurring_commitment_for_billing_period', + 'savings_plan_offering_type': 'savings_plan_offering_type', + 'savings_plan_payment_option': 'savings_plan_payment_option', + 'savings_plan_purchase_term': 'savings_plan_purchase_term', + 'product_product_name': "product['product_name']", + 'product_alarm_type': "product['alarm_type']", + 'product_attachment_type': "product['attachment_type']", + 'product_availability': "product['availability']", + 'product_availability_zone': "product['availability_zone']", + 'product_capacitystatus': "product['capacitystatus']", + 'product_category': "product['category']", + 'product_ci_type': "product['ci_type']", + 'product_classicnetworkingsupport': "product['classicnetworkingsupport']", + 'product_clock_speed': "product['clock_speed']", + 'product_current_generation': "product['current_generation']", + 'product_database_engine': "product['database_engine']", + 'product_dedicated_ebs_throughput': "product['dedicated_ebs_throughput']", + 'product_deployment_option': "product['deployment_option']", + 'product_description': "product['description']", + 'product_durability': "product['durability']", + 'product_ecu': "product['ecu']", + 'product_engine_code': "product['engine_code']", + 'product_enhanced_networking_supported': "product['enhanced_networking_supported']", + 'product_event_type': "product['event_type']", + 'product_free_query_types': "product['free_query_types']", + 'product_group': "product['group']", + 'product_group_description': "product['group_description']", + 'product_instance_type_family': "product['instance_type_family']", + 'product_intel_avx2_available': "product['intel_avx2_available']", + 'product_intel_avx_available': "product['intel_avx_available']", + 'product_intel_turbo_available': "product['intel_turbo_available']", + 'product_license_model': "product['license_model']", + 'product_logs_destination': "product['logs_destination']", + 'product_marketoption': "product['marketoption']", + 'product_max_iops_burst_performance': "product['max_iops_burst_performance']", + 'product_max_iopsvolume': "product['max_iopsvolume']", + 'product_max_throughputvolume': "product['max_throughputvolume']", + 'product_max_volume_size': "product['max_volume_size']", + 'product_memory': "product['memory']", + 'product_message_delivery_frequency': "product['message_delivery_frequency']", + 'product_message_delivery_order': "product['message_delivery_order']", + 'product_min_volume_size': "product['min_volume_size']", + 'product_network_performance': "product['network_performance']", + 'product_normalization_size_factor': "product['normalization_size_factor']", + 'product_operating_system': "product['operating_system']", + 'product_physical_processor': "product['physical_processor']", + 'product_platopricingtype': "product['platopricingtype']", + 'product_platovolumetype': "product['platovolumetype']", + 'product_pre_installed_sw': "product['pre_installed_sw']", + 'product_processor_architecture': "product['processor_architecture']", + 'product_processor_features': "product['processor_features']", + 'product_queue_type': "product['queue_type']", + 'product_region': "product['region']", + 'product_request_type': "product['request_type']", + 'product_servicename': "product['servicename']", + 'product_storage': "product['storage']", + 'product_storage_class': "product['storage_class']", + 'product_storage_media': "product['storage_media']", + 'product_tenancy': "product['tenancy']", + 'reservation_reservation_a_r_n': 'reservation_reservation_a_r_n', + 'product_transfer_type': "product['transfer_type']", + 'product_vcpu': "product['vcpu']", + 'product_version': "product['version']", + 'product_volume_api_name': "product['volume_api_name']", + 'product_volume_type': "product['volume_type']", + 'product_vpcnetworkingsupport': "product['vpcnetworkingsupport']", + 'product_edition': "product['edition']", + 'product_gpu_memory': "product['gpu_memory']", + 'product_pack_size': "product['pack_size']", + 'product_q_present': "product['q_present']", + 'product_subscription_type': "product['subscription_type']", + 'product_usage_group': "product['usage_group']", + 'product_cache_engine': "product['cache_engine']", + 'product_invocation': "product['invocation']", + 'product_memory_gib': "product['memory_gib']", + 'product_time_window': "product['time_window']", + 'product_finding_group': "product['finding_group']", + 'product_finding_source': "product['finding_source']", + 'product_finding_storage': "product['finding_storage']", + 'product_standard_group': "product['standard_group']", + 'product_standard_storage': "product['standard_storage']", + 'product_product_type': "product['product_type']", + 'product_pricingplan': "product['pricingplan']", + 'product_provider': "product['provider']", + 'product_subservice': "product['subservice']", + 'product_type': "product['type']", + 'product_tickettype': "product['tickettype']", + 'product_memorytype': "product['memorytype']", + 'product_platousagetype': "product['platousagetype']", + 'product_with_active_users': "product['with_active_users']", + 'product_abd_instance_class': "product['abd_instance_class']", + 'product_size_flex': "product['size_flex']", + 'product_engine_major_version': "product['engine_major_version']", + 'product_extended_support_pricing_year': "product['extended_support_pricing_year']", + '''concat('name', "bill_payer_account_id")''': 'bill_payer_account_name', + '''concat('name', "line_item_usage_account_id")''': 'line_item_usage_account_name', +} + +# Default columns are used as a basic set of columns when CUR is created. More can be added by views. For CUR1 and 2 these are different ,,,,,,,, +default_columns = { + '1': [ + "year", + "month", + "bill_bill_type", + "bill_billing_entity", + "bill_billing_period_end_date", + "bill_billing_period_start_date", + "bill_invoice_id", + "bill_payer_account_id", + "identity_line_item_id", + "identity_time_interval", + "line_item_availability_zone", + "line_item_legal_entity", + "line_item_line_item_description", + "line_item_line_item_type", + "line_item_operation", + "line_item_product_code", + "line_item_resource_id", + "line_item_unblended_cost", + "line_item_usage_account_id", + "line_item_usage_amount", + "line_item_usage_end_date", + "line_item_usage_start_date", + "line_item_usage_type", + "pricing_lease_contract_length", + "pricing_offering_class", + "pricing_public_on_demand_cost", + "pricing_purchase_option", + "pricing_term", + "pricing_unit", + "product_cache_engine", + "product_current_generation", + "product_database_engine", + "product_deployment_option", + "product_from_location", + "product_group", + "product_instance_type", + "product_instance_type_family", + "product_license_model", + "product_operating_system", + "product_physical_processor", + "product_processor_features", + "product_product_family", + "product_product_name", + "product_region", + "product_servicecode", + "product_storage", + "product_tenancy", + "product_to_location", + "product_volume_api_name", + "product_volume_type", + "reservation_amortized_upfront_fee_for_billing_period", + "reservation_effective_cost", + "reservation_end_time", + "reservation_reservation_a_r_n", + "reservation_start_time", + "reservation_unused_amortized_upfront_fee_for_billing_period", + "reservation_unused_recurring_fee", + "savings_plan_amortized_upfront_commitment_for_billing_period", + "savings_plan_end_time", + "savings_plan_offering_type", + "savings_plan_payment_option", + "savings_plan_purchase_term", + "savings_plan_savings_plan_a_r_n", + "savings_plan_savings_plan_effective_cost", + "savings_plan_start_time", + "savings_plan_total_commitment_to_date", + "savings_plan_used_commitment", + ], + '2': [ + "billing_period", + "bill_bill_type", + "bill_billing_entity", + "bill_billing_period_end_date", + "bill_billing_period_start_date", + "bill_invoice_id", + "bill_payer_account_id", + "bill_payer_account_name", + "identity_line_item_id", + "identity_time_interval", + "line_item_availability_zone", + "line_item_legal_entity", + "line_item_line_item_description", + "line_item_line_item_type", + "line_item_operation", + "line_item_product_code", + "line_item_resource_id", + "line_item_unblended_cost", + "line_item_usage_account_id", + "line_item_usage_account_name", + "line_item_usage_amount", + "line_item_usage_end_date", + "line_item_usage_start_date", + "line_item_usage_type", + "pricing_lease_contract_length", + "pricing_offering_class", + "pricing_public_on_demand_cost", + "pricing_purchase_option", + "pricing_term", + "pricing_unit", + "product['cache_engine']", + "product['current_generation']", + "product['database_engine']", + "product['deployment_option']", + "product_from_location", + "product['group']", + "product_instance_type", + "product['instance_type_family']", + "product['license_model']", + "product['operating_system']", + "product['physical_processor']", + "product['processor_features']", + "product_product_family", + "product['product_name']", + "product['region']", + "product_servicecode", + "product['storage']", + "product['tenancy']", + "product_to_location", + "product['volume_api_name']", + "product['volume_type']", + "reservation_amortized_upfront_fee_for_billing_period", + "reservation_effective_cost", + "reservation_end_time", + "reservation_reservation_a_r_n", + "reservation_start_time", + "reservation_unused_amortized_upfront_fee_for_billing_period", + "reservation_unused_recurring_fee", + "savings_plan_amortized_upfront_commitment_for_billing_period", + "savings_plan_end_time", + "savings_plan_offering_type", + "savings_plan_payment_option", + "savings_plan_purchase_term", + "savings_plan_savings_plan_a_r_n", + "savings_plan_savings_plan_effective_cost", + "savings_plan_start_time", + "savings_plan_total_commitment_to_date", + "savings_plan_used_commitment", + ], +} + +# various types require various empty +empty = { + 'string': 'cast(null as varchar)', + 'varchar': 'cast(null as varchar)', + 'double': 'cast(null as double)', + 'timestamp': 'cast(null as timestamp)', +} + +cur2_maps = { + 'product', + 'resource_tags', + 'cost_category', + 'discount', +} + +class ProxyView(): + """ Proxy for CUR + + creates a proxy view for CUR + """ + def __init__(self, cur, target_cur_version): + self.cur = cur + self.target_cur_version = target_cur_version + self.current_cur_version = self.cur.version + logger.debug(f'CUR proxy from {self.current_cur_version } to {self.target_cur_version }') + self.fields_to_expose = list(set(default_columns[self.target_cur_version])) + # add tags in cur2 proxy + if cur.version.startswith('1') and target_cur_version.startswith('2') : + for field in cur.tag_and_cost_category_fields: + self.fields_to_expose.append(re.sub("(resource_tags|cost_category)_(.+)", r"\1['\2']", field)) + self.athena = self.cur.athena + self.name = f'cur{self.target_cur_version}_proxy' + self.exposed_fields = [] + self.exposed_maps = {} + self.fields_to_expose_in_maps = {} + self.fields_with_missing_requirements = [] # keep the list to show warning just once + self.updated_once = False + + def read_from_athena(self): + """ read the current state from athena. read all fields and their types from existing SQL view and also for each MAP read existing keys + """ + # Read fields from the current view + if not self.exposed_fields: + exposed_fields_and_types = dict(self.athena.query(f''' + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = '{self.athena.DatabaseName}' + AND table_name = '{self.name}'; + ''')) + self.exposed_fields = list(exposed_fields_and_types.keys()) + logger.debug(f'exposed fields: {self.exposed_fields}') + if not self.exposed_fields: + return + # If we have existing fields, we need to read also all map definitions from the SQL of the existing view + def _parse_mapping_string(mapping_string): + ''' read keys and values from definition + ex: + MAP(ARRAY['key1', 'key2'], ARRAY[value1, value2]) + ''' + arrays = re.findall(r'ARRAY\[(.*?)\]', mapping_string) # Extract arrays from the string + keys = arrays[0].strip('\'').split(',') # Split arrays into elements + values = arrays[1].strip('\'').split(',') + return {key.strip(): value.strip() for key, value in zip(keys, values)} + + _current_sql = '\n'.join([line[0] for line in self.athena.query(f'SHOW CREATE VIEW {self.name}')]) + + # transform: concat(ARRAY[a,b,c], ARRAY[d,e,f], ARRAY[e,g,h]) >> ARRAY[a,b,c,d,e,f,e,g,h] + _current_sql = re.sub(r'concat\((.+?)\)', lambda match: match.group(1).replace('], ARRAY[', ','), _current_sql) + for field in self.exposed_fields: + if field in cur2_maps: + if field not in self.exposed_maps: + self.exposed_maps[field] = set() + maps = re.findall(fr'MAP\(ARRAY\[.+?\], ARRAY\[.+?\]\) {field}', _current_sql) + if not maps: + logger.warning(f'Cannot find map {field} definition in the view {self.name}. It must be defined as MAP.') + continue + current_map = _parse_mapping_string(maps[0]) + logger.debug(f'current definition of {field} of {current_map}') + self.exposed_maps[field].update([key[0] for key in current_map]) + logger.debug(f'{self.exposed_maps}') + + def source_column_equivalents(self, field): + """ Given a field of cur return an equivalent field in the target cur system. This one does not care if field actually exists + field: target CUR field + returns: source CUR SQL + + there can be more then one field + """ + logger.trace(f'source_column_equivalents {field}') + if field in self.cur.fields and not field.endswith('_time'): # Same field name is more then common case so try it first + return [field] + + if self.current_cur_version.startswith('2') and self.target_cur_version.startswith('2'): # field from CUR2 to CUR2 + return [field] # all the same + if self.current_cur_version.startswith('1') and self.target_cur_version.startswith('1'): # field from CUR1 to CUR1 + return [field] # all the same + if self.current_cur_version.startswith('2') and self.target_cur_version.startswith('1'): # field from CUR1 to CUR2 + if field not in cur1to2_mapping: #check if this field is in mapping + for cur2map in cur2_maps: + if field.startswith(cur2map + '_'): + return [cur2map] + logger.warning(f"{field} not known field of CUR2. needs to be added in code. Please create a github issue") + res = cur1to2_mapping.get(field, field) + return self.get_fields_from_sql(res) + if self.current_cur_version.startswith('1') and self.target_cur_version.startswith('2'): # field from CUR2 to CUR1 + if field.lower() in cur2_maps: + return [] # field mapping itself without indication to concrete component + matches = re.findall(r"(\w+)\['(\w+)'\]", field) + if matches: + map_field, key = matches[0] + if map_field not in self.fields_to_expose_in_maps: + self.fields_to_expose_in_maps[map_field] = set() + self.fields_to_expose_in_maps[map_field].add(key) + logger.trace(f"ret {map_field}_{key}") + return [f'{map_field}_{key}'] + + #if field.endswith('_time'): # can be + # return self.get_fields_from_sql(field) + + cur2to1_mapping = {value: key for key, value in cur1to2_mapping.items()} + if field not in cur2to1_mapping and field not in cur2_maps: + logger.warning(f"The field '{field}' is not known field of CUR1. needs to be added in code. Please create a github issue") + ret = self.get_fields_from_sql(cur2to1_mapping.get(field, field)) + logger.trace(f"ret {ret}") + return ret + + def get_fields_from_sql(self, field): + """ get a sql field from an expression + ex: + '''concat('name', "bill_payer_account_id")''' => "bill_payer_account_id" + parameters['aaa'] => parameters + plane_fields => plane_fields + """ + if '"' in field: + matches = re.findall(r'"(.+_time)"', field) + return matches + if '[' in field: + return [field.split('[')[0]] + return [field] + + def get_sql_expression(self, field, field_type): + """ Given a field of cur return an SQL representation of the field in the target cur system. Takes into account existence of fields in the current cur. + field: target CUR field + returns: CUR field SQL representation. Can be NULL + """ + equivalents = self.source_column_equivalents(field) # Do not remove this. needed to populate self.fields_to_expose_in_maps + missing_requirements = [] + for requirement in equivalents: + if not self.cur.column_exists(requirement): + missing_requirements.append(requirement) + + if missing_requirements: + logger.trace(f'field {field} cannot be present as prereqs are missing in source cur: {missing_requirements}') + return empty[field_type.lower()] + + if field in self.cur.fields and not field.endswith('_time'): # Same field name is more then common case so try it first _date have different fields + return field + if self.current_cur_version.startswith('2') and self.target_cur_version.startswith('2'): # field from CUR2 to CUR2 + return field.split('[')[0] + if self.current_cur_version.startswith('1') and self.target_cur_version.startswith('1'): # field from CUR1 to CUR2 + return field + if self.current_cur_version.startswith('1') and self.target_cur_version.startswith('2'): # field from CUR1 to CUR2 + if field_type.lower().startswith('map'): + map_field = field.split('[')[0] + map_mapping = {} + keys_set = set(self.exposed_maps.get(field, set())) + keys_set.update(self.fields_to_expose_in_maps.get(map_field, set())) + for key in keys_set: + if f'{map_field}_{key}' in self.cur.fields: + map_mapping[key] = f'{map_field}_{key}' + else: + map_mapping[key] = 'CAST(NULL as VARCHAR)' + if not map_mapping: + return 'cast(NULL AS MAP)' + map_mapping = dict(sorted(map_mapping.items())) # ordered dict + + chunk_size = 254 # https://github.com/prestodb/presto/issues/9073 + key_arrays = [f"""ARRAY[{', '.join(["'" + key + "'" for key in chunk])}]""" for chunk in batched(map_mapping.keys(), chunk_size)] + value_arrays = [f"""ARRAY[{', '.join(chunk )}]""" for chunk in batched(map_mapping.values(), chunk_size)] + return f''' + MAP( + {' || '.join(key_arrays)}, + {' || '.join(value_arrays)} + ) + ''' + + cur2to1_mapping = {value: key for key, value in cur1to2_mapping.items()} + if field in cur2to1_mapping: + return cur2to1_mapping[field] + else: + raise NotImplementedError(f'CUR1 field {field} has no known equivalent') + if self.current_cur_version.startswith('2') and self.target_cur_version.startswith('1'): + if field.startswith('resource_tags_'): + return f"resource_tags['{field[len('resource_tags_'):]}']" + if field.startswith('cost_category_'): + return f"cost_category['{field[len('cost_category_'):]}']" + return cur1to2_mapping.get(field, field) + + + def column_surely_exist(self, field_to_expose): + if not self.updated_once: + return False + target_field = self.get_fields_from_sql(field_to_expose)[0] + if target_field not in self.exposed_fields: + return False + if '[' in field_to_expose: + key = field_to_expose.split('[')[1].split(']')[0].replace("'",'') + if key not in self.exposed_maps.get(target_field.lower(),{}): + return False + return True + + def create_or_update_view(self): + """ Create or update view + """ + if all([self.column_surely_exist(field_to_expose) for field_to_expose in self.fields_to_expose]): + logger.debug('no need for proxy change. skip.') + return + + self.read_from_athena() + all_target_fields = sorted(list(set(self.exposed_fields + self.fields_to_expose))) + logger.trace(f'all_target_fields = {all_target_fields}') + lines = {} + for field in all_target_fields: + target_field = self.get_fields_from_sql(field)[0] + if target_field not in self.exposed_fields: + something_changed = True + target_field_type = self.cur.get_type_of_column(target_field, self.target_cur_version) + logger.trace(f'get_sql_expression {field} {target_field_type}') + mapped_expression = self.get_sql_expression(field, target_field_type) # + logger.trace(f'cur_proxy: mapped_expression({field}) = {mapped_expression}') + requirements = self.source_column_equivalents(target_field) + missing_requirements = [r for r in requirements if not self.cur.column_exists(r)] + if not missing_requirements: + expression = mapped_expression # then take resulting expression as is + else: + if field not in self.fields_with_missing_requirements: + self.fields_with_missing_requirements.append(field) + logger.warning(f"Missing requirement for field {field}: {', '.join(missing_requirements)}. Setting as empty.") + if target_field_type.lower() not in empty: + raise RuntimeError(f'{target_field_type} not in empty list for field {field}. Raise a github issue.') + expression = empty.get(target_field_type.lower(), 'null') + lines[target_field] = expression # for map we will take the latest + select_block = '\n ,'.join([f'{expression} {field}' for field, expression in sorted(lines.items())]) + query = (f''' + CREATE OR REPLACE VIEW "{self.name}" AS + SELECT + {select_block} + FROM + "{self.cur.database}"."{self.cur.table_name}" + ''') + res = self.athena.create_or_update_view(self.name, query) # this cost 3 athena calls + logging.debug(res) + + self.exposed_fields = all_target_fields + + def get_table_metadata(self): + return self.athena.get_table_metadata(self.name) + + +def _experiment(): + ''' if you want to play with Proxy CUR you can run following code + ''' + import boto3 + from cid.helpers.cur import ProxyCUR + from cid.helpers import Athena,Glue + from cid.helpers.cur import CUR + cur = CUR(athena=Athena(session=boto3.session.Session()),glue=Glue(session=boto3.session.Session())) + proxy = ProxyCUR(cur=cur, target_cur_version='2') + + \ No newline at end of file diff --git a/cid/helpers/iam.py b/cid/helpers/iam.py index 1737e685..5b2d806a 100644 --- a/cid/helpers/iam.py +++ b/cid/helpers/iam.py @@ -4,6 +4,7 @@ import copy import time import logging +from functools import lru_cache import boto3 from tqdm import tqdm @@ -38,6 +39,43 @@ def __init__(self, session): super().__init__(session) self.client = self.session.client('iam') + @lru_cache(1000) + def list_attached_policies(self, role_name) -> list: + """ List attached policies for the role + """ + return list(self.client.get_paginator("list_attached_role_policies").paginate(RoleName=role_name).search('AttachedPolicies.PolicyArn')) + + @lru_cache(1000) + def ensure_managed_policies_attached(self, role_name, policies_arns='') -> None: + """ Make sure policies are attached to the role + """ + policies_arns = [arn for arn in policies_arns.split(',') if arn] + if not policies_arns: + logger.debug('No need to attach roles') + return + + if role_name in ['aws-quicksight-service-role-v0']: + logger.warning(f'{role_name} is a managed role. Please manage it in QuickSight UI. Make sure that equivalent of those polices is configured: {policies_arns}') + return + + try: + attached_policies = self.list_attached_policies(role_name) + except self.client.exceptions.ClientError as exc: + logger.info(f'Unable to list attached policies {role_name}: {exc}') + return + + for policy_arn in policies_arns: + if policy_arn not in attached_policies: + try: + self.client.attach_role_policy( + RoleName=role_name, + PolicyArn=policy_arn, + ) + logger.info('Attached {policy_arn} to the role {role_name}') + except self.client.exceptions.ClientError as exc: + logger.warning(f'Unable to attach policy {policy_arn} to {role_name}: {exc}') + + def ensure_role_for_crawler(self, s3bucket, database, table, role_name: str='CidCmdCrawlerRole', policy_name: str="CidCmdGlueCrawlerPolicy") -> None: """Ensure Role for Crawler exists""" diff --git a/cid/helpers/quicksight/datasource.py b/cid/helpers/quicksight/datasource.py index cc90c7d8..14451bab 100644 --- a/cid/helpers/quicksight/datasource.py +++ b/cid/helpers/quicksight/datasource.py @@ -10,6 +10,13 @@ class Datasource(CidQsResource): def AthenaParameters(self) -> dict: return self.parameters.get('AthenaParameters', {}) + @property + def role_name(self) -> dict: + role_arn = self.parameters.get('AthenaParameters', {}).get('RoleArn') + if not role_arn: + return None + return role_arn.split('/')[-1] + @property def id(self) -> str: return self.get_property('DataSourceId') diff --git a/cid/logger.py b/cid/logger.py index a3ee2922..8dd89e3e 100644 --- a/cid/logger.py +++ b/cid/logger.py @@ -37,22 +37,29 @@ def set_cid_logger(verbosity=2, log_filename=None): # File handler logs everything down to DEBUG level if log_filename and not os.environ.get('AWS_EXECUTION_ENV', '').startswith('AWS_Lambda'): fh = logging.FileHandler(log_filename) - fh.setLevel(logging.TRACE) + #fh.setLevel(logging.TRACE) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(name)s:%(funcName)s:%(lineno)d - %(message)s') fh.setFormatter(formatter) logger.addHandler(fh) # Console handler logs everything down to WARNING level ch = logging.StreamHandler() - ch.setLevel(logging.WARNING) + #ch.setLevel(logging.WARNING) formatter = logging.Formatter('%(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) if verbosity: # Limit Logging level to DEBUG, base level is WARNING - verbosity = min(verbosity, 2) - logger.setLevel(logger.getEffectiveLevel()-10 * verbosity) + verbosity = min(verbosity, 3) + level_map = { + 1: logging.ERROR, + 2: logging.WARNING, + 3: logging.INFO, + 4: logging.DEBUG, + 5: logging.TRACE, + } + logger.setLevel(level_map.get(2 + verbosity, logging.INFO)) # Logging application start here due to logging configuration print(f'Logging level set to: {logging.getLevelName(logger.getEffectiveLevel())}') diff --git a/cid/test/bats/10-deploy-update-delete/cudos.bats b/cid/test/bats/10-deploy-update-delete/cudos.bats index 29bb8d6b..89ffa057 100644 --- a/cid/test/bats/10-deploy-update-delete/cudos.bats +++ b/cid/test/bats/10-deploy-update-delete/cudos.bats @@ -4,7 +4,7 @@ account_id=$(aws sts get-caller-identity --query "Account" --output text ) database_name="${database_name:-athenacurcfn_cur1}" # If variable not set or null, use default quicksight_user="${quicksight_user:-cicd-staging}" # If variable not set or null, use default -quicksight_datasource_id="${quicksight_datasource_id:-31c87a3c-8494-4f42-a590-c6930602e8e7}" # If variable not set or null, use default +quicksight_datasource_id="${quicksight_datasource_id:-CID-CMD-Athena}" # If variable not set or null, use default cur_table="${cur_table:-cur1}" # If variable not set or null, use default. FIXME can be autodetected! @@ -14,10 +14,13 @@ cur_table="${cur_table:-cur1}" # If variable not set or null, use default. FIXME --athena-database $database_name\ --account-map-source dummy \ --cur-table-name $cur_table \ + --athena-workgroup primary\ --quicksight-user $quicksight_user \ --share-with-account \ + --timezone 'Europe/Paris' \ --quicksight-datasource-id $quicksight_datasource_id + [ "$status" -eq 0 ] } @@ -26,7 +29,7 @@ cur_table="${cur_table:-cur1}" # If variable not set or null, use default. FIXME --catalog-name 'AwsDataCatalog'\ --database-name $database_name \ --table-name 'summary_view' \ - + # FIXME: add # compute_savings_plan_eligible_spend # summary_view @@ -57,6 +60,7 @@ cur_table="${cur_table:-cur1}" # If variable not set or null, use default. FIXME run cid-cmd -vv --yes update --force --recursive \ --dashboard-id cudos-v5 \ --cur-table-name $cur_table \ + --athena-workgroup primary\ --quicksight-user $quicksight_user \ [ "$status" -eq 0 ] @@ -65,6 +69,7 @@ cur_table="${cur_table:-cur1}" # If variable not set or null, use default. FIXME @test "Delete runs" { run cid-cmd -vv --yes delete \ + --athena-workgroup primary\ --dashboard-id cudos-v5 [ "$status" -eq 0 ] diff --git a/dashboards/aws-marketplace/aws-marketplace-spg.yaml b/dashboards/aws-marketplace/aws-marketplace-spg.yaml index 6ebd4008..d8082d34 100644 --- a/dashboards/aws-marketplace/aws-marketplace-spg.yaml +++ b/dashboards/aws-marketplace/aws-marketplace-spg.yaml @@ -290,36 +290,37 @@ datasets: views: marketplace_view: dependsOn: - cur: true + cur2: true data: |- CREATE OR REPLACE VIEW "${athena_database_name}".marketplace_view AS SELECT - "year" "year" - , "month" "month" - , "bill_billing_period_start_date" "billing_period" - , "date_trunc"('day', "line_item_usage_start_date") "usage_date" - , "bill_payer_account_id" "payer_account_id" - , "line_item_usage_account_id" "linked_account_id" - , "bill_invoice_id" "invoice_id" - , "line_item_line_item_type" "charge_type" - , (CASE WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN 'Running_Usage' WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN 'Running_Usage' WHEN ("line_item_line_item_type" = 'Usage') THEN 'Running_Usage' ELSE 'non_usage' END) "charge_category" - , "line_item_product_code" "product_code" - , "product_product_name" "product_name" - , (CASE WHEN (("bill_billing_entity" = 'AWS Marketplace') AND (NOT ("line_item_line_item_type" LIKE '%Discount%'))) THEN "Product_Product_Name" WHEN ("product_servicecode" = '') THEN "line_item_product_code" ELSE "product_servicecode" END) "service" - , "line_item_usage_type" "usage_type" - , CAST(from_iso8601_timestamp(split_part(identity_time_interval, '/', 1)) AS timestamp) "subscription_start_date" - , CAST(from_iso8601_timestamp(split_part(identity_time_interval, '/', 2)) AS timestamp) "subscription_end_date" - , "line_item_line_item_description" "item_description" - , "line_item_availability_zone" "availability_zone" - , "product_region" "region" - , "line_item_legal_entity" "legal_entity" - , "bill_billing_entity" "billing_entity" - , "pricing_unit" "pricing_unit" - , "sum"((CASE WHEN ("line_item_line_item_type" = 'SavingsPlanCoveredUsage') THEN "line_item_usage_amount" WHEN ("line_item_line_item_type" = 'DiscountedUsage') THEN "line_item_usage_amount" WHEN ("line_item_line_item_type" = 'Usage') THEN "line_item_usage_amount" ELSE 0 END)) "usage_quantity" - , "sum"("line_item_unblended_cost") "unblended_cost" - FROM - "${cur_table_name}" - WHERE ((("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL '36' MONTH)) AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL '36' MONTH))) AND ("bill_billing_entity" = 'AWS Marketplace')) + split_part("billing_period", '-', 1) AS year + , split_part("billing_period", '-', 2) AS month + , bill_billing_period_start_date AS billing_period + , date_trunc('day', line_item_usage_start_date) AS usage_date + , bill_payer_account_id AS payer_account_id + , line_item_usage_account_id AS linked_account_id + , bill_invoice_id AS invoice_id + , line_item_line_item_type AS charge_type + , (CASE WHEN (line_item_line_item_type = 'DiscountedUsage') THEN 'Running_Usage' WHEN (line_item_line_item_type = 'SavingsPlanCoveredUsage') THEN 'Running_Usage' WHEN (line_item_line_item_type = 'Usage') THEN 'Running_Usage' ELSE 'non_usage' END) AS charge_category + , line_item_product_code AS product_code + , product['product_name'] AS product_name + , (CASE WHEN ((bill_billing_entity = 'AWS Marketplace') AND (NOT (line_item_line_item_type LIKE '%Discount%'))) THEN product['product_name'] WHEN (product_servicecode = '') THEN line_item_product_code ELSE product_servicecode END) AS service + , line_item_usage_type AS usage_type + , CAST(from_iso8601_timestamp(split_part(identity_time_interval, '/', 1)) AS timestamp) AS subscription_start_date + , CAST(from_iso8601_timestamp(split_part(identity_time_interval, '/', 2)) AS timestamp) AS subscription_end_date + , line_item_line_item_description AS item_description + , line_item_availability_zone AS availability_zone + , product['region'] AS region + , line_item_legal_entity AS legal_entity + , bill_billing_entity AS billing_entity + , pricing_unit AS pricing_unit + , sum((CASE WHEN (line_item_line_item_type = 'SavingsPlanCoveredUsage') THEN line_item_usage_amount WHEN (line_item_line_item_type = 'DiscountedUsage') THEN line_item_usage_amount WHEN (line_item_line_item_type = 'Usage') THEN line_item_usage_amount ELSE 0 END)) AS usage_quantity + , sum(line_item_unblended_cost) AS unblended_cost + FROM "${cur2_database}"."${cur2_table_name}" + WHERE (((bill_billing_period_start_date >= (date_trunc('month', current_timestamp) - INTERVAL '36' MONTH)) + AND (CAST(concat("billing_period", '-01') AS date) >= (date_trunc('month', current_date) - INTERVAL '36' MONTH))) + AND (bill_billing_entity = 'AWS Marketplace')) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21 marketplace_licenses_grants_view: data: |-