Skip to content

Commit

Permalink
#2195: S3A and S3N path are not correctly handled
Browse files Browse the repository at this point in the history
* changed to preserve the protocol
* added UT
  • Loading branch information
benedeki committed Oct 18, 2023
1 parent 1635d8c commit 2aaabb4
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ object FileSystemUtils {
path.toSimpleS3Location match {

case Some(s3Location) => // s3 over hadoop fs api
val s3BucketUri: String = s"s3://${s3Location.bucketName}" // s3://<bucket>
val s3BucketUri: String = s"${s3Location.protocol}://${s3Location.bucketName}" // s3://<bucket>
val s3uri: URI = new URI(s3BucketUri)
FileSystem.get(s3uri, hadoopConf)

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright 2018 ABSA Group Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package za.co.absa.enceladus.utils.fs

import org.apache.hadoop.conf.Configuration
import org.scalatest.funsuite.{AnyFunSuite, AnyFunSuiteLike}
import za.co.absa.enceladus.utils.testUtils.SparkTestBase

class FileSystemUtilsSpec extends AnyFunSuiteLike with SparkTestBase {
implicit val hadoopConf: Configuration = spark.sparkContext.hadoopConfiguration

test("hdfs protocol default") {
val fs = FileSystemUtils.getFileSystemFromPath("hdfs://my/path")
assert(fs.getUri.toString == "hdfs://")
}

test("s3 protocol recognition and bucket set") {
val fs = FileSystemUtils.getFileSystemFromPath("s3://my_bucket/my/path")
assert(fs.getUri.toString == "s3a://my_bucket")
}

test("s3a protocol recognition and bucket set") {
val fs = FileSystemUtils.getFileSystemFromPath("s3a://my_bucket/my/path")
assert(fs.getUri.toString == "s3a://my_bucket")
}

}

0 comments on commit 2aaabb4

Please sign in to comment.