From 5585162725474b735d1724c19a90a5f3a77b77f6 Mon Sep 17 00:00:00 2001 From: thomas-serre-sonarsource Date: Fri, 31 Jan 2025 12:52:56 +0000 Subject: [PATCH 1/3] Create rule S7196 --- rules/S7196/metadata.json | 2 ++ rules/S7196/python/metadata.json | 25 ++++++++++++++++++ rules/S7196/python/rule.adoc | 44 ++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 rules/S7196/metadata.json create mode 100644 rules/S7196/python/metadata.json create mode 100644 rules/S7196/python/rule.adoc diff --git a/rules/S7196/metadata.json b/rules/S7196/metadata.json new file mode 100644 index 00000000000..2c63c085104 --- /dev/null +++ b/rules/S7196/metadata.json @@ -0,0 +1,2 @@ +{ +} diff --git a/rules/S7196/python/metadata.json b/rules/S7196/python/metadata.json new file mode 100644 index 00000000000..396c5c590d3 --- /dev/null +++ b/rules/S7196/python/metadata.json @@ -0,0 +1,25 @@ +{ + "title": "FIXME", + "type": "CODE_SMELL", + "status": "ready", + "remediation": { + "func": "Constant\/Issue", + "constantCost": "5min" + }, + "tags": [ + ], + "defaultSeverity": "Major", + "ruleSpecification": "RSPEC-7196", + "sqKey": "S7196", + "scope": "All", + "defaultQualityProfiles": ["Sonar way"], + "quickfix": "unknown", + "code": { + "impacts": { + "MAINTAINABILITY": "HIGH", + "RELIABILITY": "MEDIUM", + "SECURITY": "LOW" + }, + "attribute": "CONVENTIONAL" + } +} diff --git a/rules/S7196/python/rule.adoc b/rules/S7196/python/rule.adoc new file mode 100644 index 00000000000..caae0d69054 --- /dev/null +++ b/rules/S7196/python/rule.adoc @@ -0,0 +1,44 @@ +FIXME: add a description + +// If you want to factorize the description uncomment the following line and create the file. +//include::../description.adoc[] + +== Why is this an issue? + +FIXME: remove the unused optional headers (that are commented out) + +//=== What is the potential impact? + +== How to fix it +//== How to fix it in FRAMEWORK NAME + +=== Code examples + +==== Noncompliant code example + +[source,python,diff-id=1,diff-type=noncompliant] +---- +FIXME +---- + +==== Compliant solution + +[source,python,diff-id=1,diff-type=compliant] +---- +FIXME +---- + +//=== How does this work? + +//=== Pitfalls + +//=== Going the extra mile + + +//== Resources +//=== Documentation +//=== Articles & blog posts +//=== Conference presentations +//=== Standards +//=== External coding guidelines +//=== Benchmarks From 2df357b0cc5940604bb3d261d11df76c01ca8f80 Mon Sep 17 00:00:00 2001 From: Thomas Serre Date: Fri, 31 Jan 2025 15:05:01 +0100 Subject: [PATCH 2/3] Create rule S7196: Complex logic provided to PySpark withColumn method should be refactored into a separate expression --- rules/S7196/python/metadata.json | 11 ++++---- rules/S7196/python/rule.adoc | 44 +++++++++++++++----------------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/rules/S7196/python/metadata.json b/rules/S7196/python/metadata.json index 396c5c590d3..c63822b14bc 100644 --- a/rules/S7196/python/metadata.json +++ b/rules/S7196/python/metadata.json @@ -1,5 +1,5 @@ { - "title": "FIXME", + "title": "Complex logic provided to PySpark withColumn method should be refactored into a separate expression", "type": "CODE_SMELL", "status": "ready", "remediation": { @@ -7,8 +7,10 @@ "constantCost": "5min" }, "tags": [ + "data-science", + "pyspark" ], - "defaultSeverity": "Major", + "defaultSeverity": "Medium", "ruleSpecification": "RSPEC-7196", "sqKey": "S7196", "scope": "All", @@ -16,10 +18,9 @@ "quickfix": "unknown", "code": { "impacts": { - "MAINTAINABILITY": "HIGH", + "MAINTAINABILITY": "LOW", "RELIABILITY": "MEDIUM", - "SECURITY": "LOW" }, - "attribute": "CONVENTIONAL" + "attribute": "FOCUSED" } } diff --git a/rules/S7196/python/rule.adoc b/rules/S7196/python/rule.adoc index caae0d69054..b606ce1b950 100644 --- a/rules/S7196/python/rule.adoc +++ b/rules/S7196/python/rule.adoc @@ -1,16 +1,12 @@ -FIXME: add a description - -// If you want to factorize the description uncomment the following line and create the file. -//include::../description.adoc[] +This rule raises an issue when complex functions or expressions are directly passed to withColumn == Why is this an issue? -FIXME: remove the unused optional headers (that are commented out) - -//=== What is the potential impact? +`withColumn` method is commonly used to add or modify columns in a DataFrame. When complex functions or expressions are directly passed to withColumn, it can lead to code that is difficult to read, understand, and maintain. Also, it will become easier to write unit tests for these functions, ensuring that the logic is correct and behaves as expected. This leads to more robust and reliable code. == How to fix it -//== How to fix it in FRAMEWORK NAME + +To fix this issue, complex logic within `withColumn` logic should be refactored into separate functions or variables before being passed to `withColumn` to improve code clarity and maintainability, === Code examples @@ -18,27 +14,29 @@ FIXME: remove the unused optional headers (that are commented out) [source,python,diff-id=1,diff-type=noncompliant] ---- -FIXME +from pyspark.sql.functions import * +df = df.withColumn('Revenue', col('fare_amount').substr(0, 10).cast("float") + col('extra').substr(0, 5).cast("float") + col('tax').substr(0, 3).cast("float")) ---- ==== Compliant solution [source,python,diff-id=1,diff-type=compliant] ---- -FIXME +from pyspark.sql.functions import * +def convert_to_float(col_str): + return col_str.substr(0, 10).cast("float") + +def compute_revenue(): + fare_amount = col('fare_amount').substr(0, 10).cast("float") + extra = col('extra').substr(0, 5).cast("float") + tax = col('tax').substr(0, 3).cast("float") + + return fare_amount + extra + tax + +df = df.withColumn("Revenue", compute_revenue()) # Compliant ---- -//=== How does this work? - -//=== Pitfalls - -//=== Going the extra mile - +== Resources +=== Documentation -//== Resources -//=== Documentation -//=== Articles & blog posts -//=== Conference presentations -//=== Standards -//=== External coding guidelines -//=== Benchmarks + * PySpark withColumn Documentation - https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html[pyspark.sql.DataFrame.withColumn] From 2ea372b716aca6c216130810a255c8d0ccef55b2 Mon Sep 17 00:00:00 2001 From: Thomas Serre Date: Mon, 3 Feb 2025 14:44:25 +0100 Subject: [PATCH 3/3] Fix after review --- rules/S7196/python/metadata.json | 6 +++--- rules/S7196/python/rule.adoc | 33 +++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/rules/S7196/python/metadata.json b/rules/S7196/python/metadata.json index c63822b14bc..ab3d7952a7a 100644 --- a/rules/S7196/python/metadata.json +++ b/rules/S7196/python/metadata.json @@ -1,5 +1,5 @@ { - "title": "Complex logic provided to PySpark withColumn method should be refactored into a separate expression", + "title": "Complex logic provided to PySpark \"withColumn\", \"filter\" and \"when\" methods should be refactored into separate expressions", "type": "CODE_SMELL", "status": "ready", "remediation": { @@ -10,7 +10,7 @@ "data-science", "pyspark" ], - "defaultSeverity": "Medium", + "defaultSeverity": "Major", "ruleSpecification": "RSPEC-7196", "sqKey": "S7196", "scope": "All", @@ -19,7 +19,7 @@ "code": { "impacts": { "MAINTAINABILITY": "LOW", - "RELIABILITY": "MEDIUM", + "RELIABILITY": "MEDIUM" }, "attribute": "FOCUSED" } diff --git a/rules/S7196/python/rule.adoc b/rules/S7196/python/rule.adoc index b606ce1b950..ace09e394e5 100644 --- a/rules/S7196/python/rule.adoc +++ b/rules/S7196/python/rule.adoc @@ -1,12 +1,12 @@ -This rule raises an issue when complex functions or expressions are directly passed to withColumn +This rule raises an issue when complex expressions are directly passed to `withColumn`, `filter` or `when` functions. == Why is this an issue? -`withColumn` method is commonly used to add or modify columns in a DataFrame. When complex functions or expressions are directly passed to withColumn, it can lead to code that is difficult to read, understand, and maintain. Also, it will become easier to write unit tests for these functions, ensuring that the logic is correct and behaves as expected. This leads to more robust and reliable code. +`withColumn`, `filter` and `when` methods are commonly used to add, modify, or filter columns in a DataFrame. When long or complex expressions are directly passed to those functions, it can lead to code that is difficult to read, understand, and maintain. Refactoring such expressions into functions or variables will help with readability. Also, it will become easier to write unit tests for these functions, ensuring that the logic is correct and behaves as expected. This leads to more robust and reliable code. == How to fix it -To fix this issue, complex logic within `withColumn` logic should be refactored into separate functions or variables before being passed to `withColumn` to improve code clarity and maintainability, +To fix this issue, complex logic within `withColumn`, `filter` and `when` calls should be refactored into separate functions or variables to improve code clarity and maintainability, === Code examples @@ -15,7 +15,9 @@ To fix this issue, complex logic within `withColumn` logic should be refactored [source,python,diff-id=1,diff-type=noncompliant] ---- from pyspark.sql.functions import * -df = df.withColumn('Revenue', col('fare_amount').substr(0, 10).cast("float") + col('extra').substr(0, 5).cast("float") + col('tax').substr(0, 3).cast("float")) +df = df.withColumn('Revenue', col('fare_amount').substr(0, 10).cast("float") + col('extra').substr(0, 5).cast("float") + col('tax').substr(0, 3).cast("float")) # Noncompliant +df = df.withColumn('High revenue', when(col('fare_amount').substr(0, 10).cast("float") > 100. and col('extra').substr(0, 5).cast("float") > 100. and col('tax').substr(0, 3).cast("float") < 50.)) # Noncompliant +df = df.filter(col('fare_amount').substr(0, 10).cast("float") > 100. and col('extra').substr(0, 5).cast("float") > 100. and col('tax').substr(0, 3).cast("float") < 50.) # Noncompliant ---- ==== Compliant solution @@ -23,20 +25,29 @@ df = df.withColumn('Revenue', col('fare_amount').substr(0, 10).cast("float") + c [source,python,diff-id=1,diff-type=compliant] ---- from pyspark.sql.functions import * -def convert_to_float(col_str): - return col_str.substr(0, 10).cast("float") - -def compute_revenue(): - fare_amount = col('fare_amount').substr(0, 10).cast("float") + +def get_revenue_inputs(): + fare_amount = col('fare_amount').substr(0, 15).cast("float") extra = col('extra').substr(0, 5).cast("float") tax = col('tax').substr(0, 3).cast("float") - + return fare_amount, extra, tax + +def compute_revenue(): + fare_amount, extra, tax = get_revenue_inputs() return fare_amount + extra + tax + +def is_high_revenue(): + fare_amount, extra, tax = get_revenue_inputs() + return when( (fare_amount > 100.) & (extra > 100.) & (tax < 50), True).otherwise(False) df = df.withColumn("Revenue", compute_revenue()) # Compliant +df = df.withColumn('High revenue', is_high_revenue()) # Compliant +df = df.filter( is_high_revenue() ) # Compliant ---- == Resources === Documentation - * PySpark withColumn Documentation - https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html[pyspark.sql.DataFrame.withColumn] + * PySpark withColumn documentation - https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.withColumn.html[pyspark.sql.DataFrame.withColumn] + * PySpark filter documentation - https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.filter.html[pyspark.sql.DataFrame.filter] + * PySpark when documentation - https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.when.html[pyspark.sql.functions.when]