diff --git a/clients/spark/README.md b/clients/spark/README.md index d8ab8c16187..60d55d01bc5 100644 --- a/clients/spark/README.md +++ b/clients/spark/README.md @@ -26,7 +26,7 @@ io.lakefs:lakefs-spark-client_2.12:${CLIENT_VERSION} Replace `` below with the latest version available. See [available versions](https://mvnrepository.com/artifact/io.lakefs/lakefs-spark-client_2.12). ``` -CLIENT_VERSION=0.11.0 +CLIENT_VERSION=0.14.0 spark-submit --conf spark.hadoop.lakefs.api.url=https://lakefs.example.com/api/v1 \ --conf spark.hadoop.fs.s3a.access.key= \ --conf spark.hadoop.fs.s3a.secret.key= \ @@ -42,7 +42,7 @@ spark-submit --conf spark.hadoop.lakefs.api.url=https://lakefs.example.com/api/v Replace `` below with the latest version available. See [available versions](https://mvnrepository.com/artifact/io.lakefs/lakefs-spark-client_2.12). ``` -CLIENT_VERSION=0.11.0 +CLIENT_VERSION=0.14.0 spark-submit --conf spark.hadoop.lakefs.api.url=https://lakefs.example.com/api/v1 \ --conf spark.hadoop.fs.s3a.access.key= \ --conf spark.hadoop.fs.s3a.secret.key= \ diff --git a/docs/howto/export.md b/docs/howto/export.md index 44b51cf3adf..9e9b8a61576 100644 --- a/docs/howto/export.md +++ b/docs/howto/export.md @@ -55,7 +55,7 @@ The complete `spark-submit` command would look as follows: spark-submit --conf spark.hadoop.lakefs.api.url=https:///api/v1 \ --conf spark.hadoop.lakefs.api.access_key= \ --conf spark.hadoop.lakefs.api.secret_key= \ - --packages io.lakefs:lakefs-spark-client_2.12:0.11.0 \ + --packages io.lakefs:lakefs-spark-client_2.12:0.14.0 \ --class io.treeverse.clients.Main export-app example-repo s3://example-bucket/prefix \ --branch=example-branch ``` diff --git a/docs/howto/garbage-collection/gc.md b/docs/howto/garbage-collection/gc.md index 11d3a7e4a2c..ab9d96ef9b1 100644 --- a/docs/howto/garbage-collection/gc.md +++ b/docs/howto/garbage-collection/gc.md @@ -121,7 +121,7 @@ spark-submit --class io.treeverse.gc.GarbageCollection \ -c spark.hadoop.lakefs.api.secret_key= \ -c spark.hadoop.fs.s3a.access.key= \ -c spark.hadoop.fs.s3a.secret.key= \ - http://treeverse-clients-us-east.s3-website-us-east-1.amazonaws.com/lakefs-spark-client/0.11.0/lakefs-spark-client-assembly-0.11.0.jar \ + http://treeverse-clients-us-east.s3-website-us-east-1.amazonaws.com/lakefs-spark-client/0.14.0/lakefs-spark-client-assembly-0.14.0.jar \ example-repo us-east-1 ``` @@ -136,7 +136,7 @@ spark-submit --class io.treeverse.gc.GarbageCollection \ -c spark.hadoop.lakefs.api.access_key= \ -c spark.hadoop.lakefs.api.secret_key= \ -c spark.hadoop.fs.azure.account.key..dfs.core.windows.net= \ - http://treeverse-clients-us-east.s3-website-us-east-1.amazonaws.com/lakefs-spark-client/0.11.0/lakefs-spark-client-assembly-0.11.0.jar \ + http://treeverse-clients-us-east.s3-website-us-east-1.amazonaws.com/lakefs-spark-client/0.14.0/lakefs-spark-client-assembly-0.14.0.jar \ example-repo ``` @@ -153,7 +153,7 @@ spark-submit --class io.treeverse.gc.GarbageCollection \ -c spark.hadoop.fs.azure.account.oauth2.client.id..dfs.core.windows.net= \ -c spark.hadoop.fs.azure.account.oauth2.client.secret..dfs.core.windows.net= \ -c spark.hadoop.fs.azure.account.oauth2.client.endpoint..dfs.core.windows.net=https://login.microsoftonline.com//oauth2/token \ - http://treeverse-clients-us-east.s3-website-us-east-1.amazonaws.com/lakefs-spark-client/0.11.0/lakefs-spark-client-assembly-0.11.0.jar \ + http://treeverse-clients-us-east.s3-website-us-east-1.amazonaws.com/lakefs-spark-client/0.14.0/lakefs-spark-client-assembly-0.14.0.jar \ example-repo ``` @@ -181,7 +181,7 @@ spark-submit --class io.treeverse.gc.GarbageCollection \ -c spark.hadoop.fs.gs.impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem \ -c spark.hadoop.fs.AbstractFileSystem.gs.impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS \ -c spark.hadoop.lakefs.gc.do_sweep=false \ - http://treeverse-clients-us-east.s3-website-us-east-1.amazonaws.com/lakefs-spark-client/0.11.0/lakefs-spark-client-assembly-0.11.0.jar \ + http://treeverse-clients-us-east.s3-website-us-east-1.amazonaws.com/lakefs-spark-client/0.14.0/lakefs-spark-client-assembly-0.14.0.jar \ example-repo ``` diff --git a/docs/reference/spark-client.md b/docs/reference/spark-client.md index 6420e460c1d..db2eb282b68 100644 --- a/docs/reference/spark-client.md +++ b/docs/reference/spark-client.md @@ -31,18 +31,18 @@ can work for other Spark versions and higher Hadoop versions. Start Spark Shell / PySpark with the `--packages` flag, for instance: ```bash -spark-shell --packages io.lakefs:lakefs-spark-client_2.12:0.13.0 +spark-shell --packages io.lakefs:lakefs-spark-client_2.12:0.14.0 ``` Alternatively use the assembled jar (an "Überjar") on S3, from -`s3://treeverse-clients-us-east/lakefs-spark-client/0.13.0/lakefs-spark-client-assembly-0.13.0.jar` +`s3://treeverse-clients-us-east/lakefs-spark-client/0.14.0/lakefs-spark-client-assembly-0.14.0.jar` by passing its path to `--jars`. The assembled jar is larger but shades several common libraries. Use it if Spark complains about bad classes or missing methods.
Include this assembled jar (an "Überjar") from S3, from -`s3://treeverse-clients-us-east/lakefs-spark-client/0.13.0/lakefs-spark-client-assembly-0.13.0.jar`. +`s3://treeverse-clients-us-east/lakefs-spark-client/0.14.0/lakefs-spark-client-assembly-0.14.0.jar`.