From cbae7fa439a9f0cfc8247d86cf09da390ba2c540 Mon Sep 17 00:00:00 2001 From: bugfix4u Date: Thu, 22 Aug 2024 08:49:47 -0500 Subject: [PATCH 1/4] Updated AWS S3 packages to Version 2, Version 1 is now in maintenance mode. Added a versioned to the docker build image to match the go.mod go version. Pulling the latest image failed to build the worker. --- Dockerfile | 2 +- go.mod | 24 ++++++- go.sum | 51 +++++++++++-- worker/main.go | 10 +-- worker/s3.go | 191 +++++++++++++++++++++++++++++-------------------- 5 files changed, 184 insertions(+), 94 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb721a7..e11df00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:alpine as builder +FROM golang:1.20.14-alpine3.19 as builder ARG TYPE ENV GOBIN=/go/bin ENV GOPATH=/go/src diff --git a/go.mod b/go.mod index edbce11..f46dc96 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 - github.com/aws/aws-sdk-go v1.49.5 + github.com/aws/aws-sdk-go-v2/credentials v1.17.28 github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 github.com/sirupsen/logrus v1.9.3 @@ -13,12 +13,32 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.4 // indirect + github.com/aws/smithy-go v1.20.4 // indirect +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.30.4 + github.com/aws/aws-sdk-go-v2/config v1.27.28 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.60.0 github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect diff --git a/go.sum b/go.sum index 54e61d3..8cf6cea 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,44 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aws/aws-sdk-go v1.49.5 h1:y2yfBlwjPDi3/sBVKeznYEdDy6wIhjA2L5NCBMLUIYA= -github.com/aws/aws-sdk-go v1.49.5/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= +github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw= +github.com/aws/aws-sdk-go-v2/config v1.27.28 h1:OTxWGW/91C61QlneCtnD62NLb4W616/NM1jA8LhJqbg= +github.com/aws/aws-sdk-go-v2/config v1.27.28/go.mod h1:uzVRVtJSU5EFv6Fu82AoVFKozJi2ZCY6WRCXj06rbvs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.28 h1:m8+AHY/ND8CMHJnPoH7PJIRakWGa4gbfbxuY9TGTUXM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.28/go.mod h1:6TF7dSc78ehD1SL6KpRIPKMA1GyyWflIkjqg+qmf4+c= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.12 h1:i7cJ1izNlox4ka6cvbHPTztYGtbpW4Je/jyQIKOIU4A= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.12/go.mod h1:lHnam/4CTEVHaANZD54IrpE80VLK+lUU84WEeJ1FJ8M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.60.0 h1:2QXGJvG19QwqXUvgcdoCOZPyLuvZf8LiXPCN4P53TdI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.60.0/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.4 h1:iAckBT2OeEK/kBDyN/jDtpEExhjeeA/Im2q4X0rJZT8= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.4/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -124,6 +160,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -141,10 +178,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -160,9 +193,11 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= @@ -214,6 +249,7 @@ github.com/prometheus/statsd_exporter v0.26.0 h1:SQl3M6suC6NWQYEzOvIv+EF6dAMYEqI github.com/prometheus/statsd_exporter v0.26.0/go.mod h1:GXFLADOmBTVDrHc7b04nX8ooq3azG61pnECNqT7O5DM= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -386,6 +422,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -518,12 +555,12 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/worker/main.go b/worker/main.go index 0214c44..a43129e 100644 --- a/worker/main.go +++ b/worker/main.go @@ -12,7 +12,7 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/mulbc/gosbench/common" log "github.com/sirupsen/logrus" ) @@ -245,14 +245,14 @@ func fillWorkqueue(testConfig *common.TestCaseConfiguration, Workqueue *Workqueu if err != nil { log.WithError(err).WithField("bucket", bucketName).Error("Error when creating bucket") } - var preExistingObjects *s3.ListObjectsV2Output + var preExistingObjects []types.Object var preExistingObjectCount uint64 if testConfig.ExistingReadWeight > 0 { preExistingObjects, err = listObjects(housekeepingSvc, "", bucketName) if err != nil { log.WithError(err).Fatalf("Problems when listing contents of bucket %s", bucketName) } - preExistingObjectCount = uint64(len(preExistingObjects.Contents)) + preExistingObjectCount = uint64(len(preExistingObjects)) log.Debugf("Found %d objects in bucket %s", preExistingObjectCount, bucketName) if preExistingObjectCount <= 0 { @@ -287,8 +287,8 @@ func fillWorkqueue(testConfig *common.TestCaseConfiguration, Workqueue *Workqueu new := &ReadOperation{ TestName: testConfig.Name, Bucket: bucketName, - ObjectName: *preExistingObjects.Contents[object%preExistingObjectCount].Key, - ObjectSize: uint64(*preExistingObjects.Contents[object%preExistingObjectCount].Size), + ObjectName: *preExistingObjects[object%preExistingObjectCount].Key, + ObjectSize: uint64(*preExistingObjects[object%preExistingObjectCount].Size), WorksOnPreexistingObject: true, } *Workqueue.Queue = append(*Workqueue.Queue, new) diff --git a/worker/s3.go b/worker/s3.go index 25b6e09..862b59c 100755 --- a/worker/s3.go +++ b/worker/s3.go @@ -3,17 +3,17 @@ package main import ( "context" "crypto/tls" + "errors" "fmt" "io" "net/http" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" + s3config "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" log "github.com/sirupsen/logrus" "github.com/mulbc/gosbench/common" @@ -21,7 +21,7 @@ import ( "go.opencensus.io/stats/view" ) -var svc, housekeepingSvc *s3.S3 +var svc, housekeepingSvc *s3.Client var ctx context.Context var hc *http.Client @@ -63,61 +63,67 @@ func InitS3(config common.S3Configuration) { Transport: tr2, } - sess := session.Must(session.NewSession(&aws.Config{ - HTTPClient: hc, - // TODO Also set the remaining S3 connection details... - Region: &config.Region, - Credentials: credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""), - Endpoint: &config.Endpoint, - S3ForcePathStyle: aws.Bool(true), - })) + // TODO Create a context with a timeout - we already use this context in all S3 calls + // Usually this shouldn't be a problem ;) + ctx = context.Background() + + cfg, err := s3config.LoadDefaultConfig(ctx, + s3config.WithHTTPClient(hc), + s3config.WithRegion(config.Region), + s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.AccessKey, config.SecretKey, "")), + ) + if err != nil { + log.WithError(err).Fatal("Unable to build S3 config") + } // Use this Session to do things that are hidden from the performance monitoring - housekeepingSess := session.Must(session.NewSession(&aws.Config{ - HTTPClient: &http.Client{Transport: tr}, - // TODO Also set the remaining S3 connection details... - Region: &config.Region, - Credentials: credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""), - Endpoint: &config.Endpoint, - S3ForcePathStyle: aws.Bool(true), - })) + // Setting up the housekeeping S3 client + hkhc := &http.Client{ + Transport: tr, + } + + hkCfg, err := s3config.LoadDefaultConfig(ctx, + s3config.WithHTTPClient(hkhc), + s3config.WithRegion(config.Region), + s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.AccessKey, config.SecretKey, "")), + ) + if err != nil { + log.WithError(err).Fatal("Unable to build S3 housekeeping config") + } // Create a new instance of the service's client with a Session. // Optional aws.Config values can also be provided as variadic arguments // to the New function. This option allows you to provide service // specific configuration. - svc = s3.New(sess) + svc = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(config.Endpoint) + }) // Use this service to do things that are hidden from the performance monitoring - housekeepingSvc = s3.New(housekeepingSess) + housekeepingSvc = s3.NewFromConfig(hkCfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(config.Endpoint) + }) - // TODO Create a context with a timeout - we already use this context in all S3 calls - // Usually this shouldn't be a problem ;) - ctx = context.Background() log.Debug("S3 Init done") } -func putObject(service *s3.S3, objectName string, objectContent io.ReadSeeker, bucket string) error { +func putObject(service *s3.Client, objectName string, objectContent io.ReadSeeker, bucket string) error { // Create an uploader with S3 client and custom options - uploader := s3manager.NewUploaderWithClient(service) + uploader := manager.NewUploader(service, func(d *manager.Uploader) { + d.MaxUploadParts = 1 + }) - _, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + _, err := uploader.Upload(ctx, &s3.PutObjectInput{ Bucket: &bucket, Key: &objectName, Body: objectContent, - }, func(d *s3manager.Uploader) { - d.MaxUploadParts = 1 }) + if err != nil { - if aerr, ok := err.(awserr.Error); ok && aerr.Code() == request.CanceledErrorCode { - // If the SDK can determine the request or retry delay was canceled - // by a context the CanceledErrorCode error code will be returned. - log.WithError(aerr).Errorf("Upload canceled due to timeout") - } else { - log.WithError(err).WithField("object", objectName).WithField("bucket", bucket).Errorf("Failed to upload object,") - } + log.WithError(err).WithField("object", objectName).WithField("bucket", bucket).Errorf("Failed to upload object,") return err } log.WithField("bucket", bucket).WithField("key", objectName).Tracef("Upload successful") + return err } @@ -144,24 +150,26 @@ func putObject(service *s3.S3, objectName string, objectContent io.ReadSeeker, b // log.Debugf("Object Properties:\n%+v", result) // } -func listObjects(service *s3.S3, prefix string, bucket string) (*s3.ListObjectsV2Output, error) { - var result *s3.ListObjectsV2Output - err := service.ListObjectsV2Pages(&s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String(prefix)}, - func(page *s3.ListObjectsV2Output, lastPage bool) bool { - if result == nil { - result = page - } else { - result.Contents = append(result.Contents, page.Contents...) - } - return !(*page.IsTruncated) - }) - - return result, err +func listObjects(service *s3.Client, prefix string, bucket string) ([]types.Object, error) { + var bucketContents []types.Object + p := s3.NewListObjectsV2Paginator(service, &s3.ListObjectsV2Input{Bucket: aws.String(bucket), Prefix: aws.String(prefix)}) + for p.HasMorePages() { + // Next Page takes a new context for each page retrieval. This is where + // you could add timeouts or deadlines. + page, err := p.NextPage(ctx) + if err != nil { + log.WithError(err).WithField("prefix", prefix).WithField("bucket", bucket).Errorf("Failed to list objects") + return nil, err + } + bucketContents = append(bucketContents, page.Contents...) + } + + return bucketContents, nil } -func getObject(service *s3.S3, objectName string, bucket string, objectSize uint64) error { +func getObject(service *s3.Client, objectName string, bucket string, objectSize uint64) error { // Remove the allocation of buffer - result, err := svc.GetObjectWithContext(ctx, &s3.GetObjectInput{ + result, err := service.GetObject(ctx, &s3.GetObjectInput{ Bucket: &bucket, Key: &objectName, }) @@ -178,50 +186,75 @@ func getObject(service *s3.S3, objectName string, bucket string, objectSize uint return nil } -func deleteObject(service *s3.S3, objectName string, bucket string) error { - _, err := service.DeleteObjectsWithContext(ctx, &s3.DeleteObjectsInput{ +func deleteObject(service *s3.Client, objectName string, bucket string) error { + _, err := service.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &bucket, - Delete: &s3.Delete{ - Objects: []*s3.ObjectIdentifier{{Key: &objectName}}, - }, + Key: &objectName, }) if err != nil { - // Cast err to awserr.Error to handle specific error codes. - aerr, ok := err.(awserr.Error) - if ok && aerr.Code() == s3.ErrCodeNoSuchKey { - log.WithError(aerr).Errorf("Could not find object %s in bucket %s for deletion", objectName, bucket) - } + log.WithError(err).Errorf("Could not find object %s in bucket %s for deletion", objectName, bucket) } return err } -func createBucket(service *s3.S3, bucket string) error { - // TODO do not err when the bucket is already there... - _, err := service.CreateBucket(&s3.CreateBucketInput{ +func createBucket(service *s3.Client, bucket string) error { + // Do not err when the bucket is already there... + _, err := service.CreateBucket(ctx, &s3.CreateBucketInput{ Bucket: &bucket, }) if err != nil { - aerr, _ := err.(awserr.Error) + var bne *types.BucketAlreadyExists // Ignore error if bucket already exists - if aerr.Code() == s3.ErrCodeBucketAlreadyExists { + if errors.As(err, &bne) { return nil } - log.WithField("Message", aerr.Message()).WithField("Code", aerr.Code()).Info("Issues when creating bucket") + log.WithError(err).Errorf("Issues when creating bucket %s", bucket) } return err } -func deleteBucket(service *s3.S3, bucket string) error { +func deleteBucket(service *s3.Client, bucket string) error { // First delete all objects in the bucket - iter := s3manager.NewDeleteListIterator(service, &s3.ListObjectsInput{ - Bucket: &bucket, - }) + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + } - if err := s3manager.NewBatchDeleteWithClient(service).Delete(aws.BackgroundContext(), iter); err != nil { - return err + var bucketContents []types.Object + isTruncated := true + for isTruncated { + result, err := service.ListObjectsV2(ctx, input) + if err != nil { + return err + } + bucketContents = append(bucketContents, result.Contents...) + input.ContinuationToken = result.NextContinuationToken + isTruncated = *result.IsTruncated } + + if len(bucketContents) > 0 { + var objectsToDelete []types.ObjectIdentifier + for _, item := range bucketContents { + objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{ + Key: item.Key, + }) + } + + deleteObjectsInput := &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{ + Objects: objectsToDelete, + Quiet: aws.Bool(true), + }, + } + + _, err := svc.DeleteObjects(ctx, deleteObjectsInput) + if err != nil { + return err + } + } + // Then delete the (now empty) bucket itself - _, err := service.DeleteBucket(&s3.DeleteBucketInput{ + _, err := service.DeleteBucket(ctx, &s3.DeleteBucketInput{ Bucket: &bucket, }) return err From bffca9dbac96fabbb1b1b2af603c14dd14bac92f Mon Sep 17 00:00:00 2001 From: Chris Blum Date: Mon, 26 Aug 2024 14:59:39 +0200 Subject: [PATCH 2/4] Revert container tag change --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e11df00..bb721a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.14-alpine3.19 as builder +FROM golang:alpine as builder ARG TYPE ENV GOBIN=/go/bin ENV GOPATH=/go/src From 4f0c20731d84f4d98ebc8a66e8b85d90898aa3c7 Mon Sep 17 00:00:00 2001 From: Chris Blum Date: Mon, 26 Aug 2024 15:00:09 +0200 Subject: [PATCH 3/4] Update go.mod and fix old golang container image issue --- Makefile | 1 + go.mod | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 53dc7b2..3d40bc0 100755 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ UNIX_DATE := $(shell date -u +"%s") VCS_REF := $(shell git rev-parse HEAD) build: + docker pull golang:alpine docker build --tag quay.io/mulbc/gosbench-server:$(VCS_REF) --build-arg "TYPE=server" --build-arg "BUILD_DATE=$(BUILD_DATE)" --build-arg "VCS_REF=$(VCS_REF)" . docker build --tag quay.io/mulbc/gosbench-worker:$(VCS_REF) --build-arg "TYPE=worker" --build-arg "BUILD_DATE=$(BUILD_DATE)" --build-arg "VCS_REF=$(VCS_REF)" . debug-server: diff --git a/go.mod b/go.mod index f46dc96..4f10450 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/mulbc/gosbench -go 1.20 +go 1.22 + +toolchain go1.22.6 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 From 21cf18fb86824ada6c586755f3d8bed1e94d2a86 Mon Sep 17 00:00:00 2001 From: bugfix4u Date: Thu, 19 Sep 2024 14:16:03 -0500 Subject: [PATCH 4/4] Added an UsePathStyle field to the S3 structure. This allows you to enable the client to use path-style addressing. --- common/configFile.go | 1 + common/configFile_test.go | 1 + examples/example_config.json | 6 +++--- examples/example_config.yaml | 3 +++ k8s/gosbench.yaml | 1 + k8s/gosbench_template.yaml.j2 | 1 + worker/s3.go | 2 ++ 7 files changed, 12 insertions(+), 3 deletions(-) diff --git a/common/configFile.go b/common/configFile.go index 60b56c4..41316ed 100644 --- a/common/configFile.go +++ b/common/configFile.go @@ -32,6 +32,7 @@ type S3Configuration struct { Endpoint string `yaml:"endpoint" json:"endpoint"` Timeout time.Duration `yaml:"timeout" json:"timeout"` SkipSSLVerify bool `yaml:"skipSSLverify" json:"skipSSLverify"` + UsePathStyle bool `yaml:"usePathStyle" json:"usePathStyle"` } // GrafanaConfiguration contains all information necessary to add annotations diff --git a/common/configFile_test.go b/common/configFile_test.go index fab6dcd..c6812e5 100644 --- a/common/configFile_test.go +++ b/common/configFile_test.go @@ -389,6 +389,7 @@ tests: SecretKey: "secretSecret", Region: "us-east-1", SkipSSLVerify: true, + UsePathStyle: false, }, }, Tests: []*TestCaseConfiguration{ diff --git a/examples/example_config.json b/examples/example_config.json index 7cd15f6..e9c9fd3 100644 --- a/examples/example_config.json +++ b/examples/example_config.json @@ -2,15 +2,15 @@ "s3_config": [ { "access_key": "abc", "secret_key": "as", "region": "eu-central-1", "endpoint": "https://my.rgw.endpoint:8080", - "skipSSLverify": false + "skipSSLverify": false, "usePathStyle": false }, { "access_key": "def", "secret_key": "as", "region": "eu-central-2", "endpoint": "https://my.rgw.endpoint:8080", - "skipSSLverify": false + "skipSSLverify": false, "usePathStyle": false }, { "access_key": "ghi", "secret_key": "as", "region": "eu-central-3", "endpoint": "https://my.rgw.endpoint:8080", - "skipSSLverify": false + "skipSSLverify": false, "usePathStyle": false } ], "grafana_config": { "endpoint": "http://grafana", "username": "admin", "password": "grafana" }, diff --git a/examples/example_config.yaml b/examples/example_config.yaml index 48242de..f7b8710 100644 --- a/examples/example_config.yaml +++ b/examples/example_config.yaml @@ -6,16 +6,19 @@ s3_config: region: eu-central-1 endpoint: https://my.rgw.endpoint:8080 skipSSLverify: false + usePathStyle: false - access_key: def secret_key: as region: eu-central-2 endpoint: https://my.rgw.endpoint:8080 skipSSLverify: false + usePathStyle: false - access_key: ghi secret_key: as region: eu-central-3 endpoint: https://my.rgw.endpoint:8080 skipSSLverify: false + usePathStyle: false # For generating annotations when we start/stop testcases # https://grafana.com/docs/http_api/annotations/#create-annotation diff --git a/k8s/gosbench.yaml b/k8s/gosbench.yaml index 43999bb..242044a 100644 --- a/k8s/gosbench.yaml +++ b/k8s/gosbench.yaml @@ -9,6 +9,7 @@ data: region: eu-central-1 endpoint: https://172.30.196.58:443 skipSSLverify: true + usePathStyle: false # For generating annotations when we start/stop testcases # https://grafana.com/docs/http_api/annotations/#create-annotation diff --git a/k8s/gosbench_template.yaml.j2 b/k8s/gosbench_template.yaml.j2 index 46ebdd2..888b07e 100644 --- a/k8s/gosbench_template.yaml.j2 +++ b/k8s/gosbench_template.yaml.j2 @@ -12,6 +12,7 @@ data: region: eu-central-1 endpoint: https://172.30.196.58:443 skipSSLverify: true + usePathStyle: false # For generating annotations when we start/stop testcases # https://grafana.com/docs/http_api/annotations/#create-annotation diff --git a/worker/s3.go b/worker/s3.go index 862b59c..68b6f98 100755 --- a/worker/s3.go +++ b/worker/s3.go @@ -96,10 +96,12 @@ func InitS3(config common.S3Configuration) { // specific configuration. svc = s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(config.Endpoint) + o.UsePathStyle = config.UsePathStyle }) // Use this service to do things that are hidden from the performance monitoring housekeepingSvc = s3.NewFromConfig(hkCfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(config.Endpoint) + o.UsePathStyle = config.UsePathStyle }) log.Debug("S3 Init done")