From 834b366bb280fc932f4ca63697b6794025a68098 Mon Sep 17 00:00:00 2001 From: xelat09 Date: Fri, 11 Oct 2024 13:09:47 +0200 Subject: [PATCH 1/6] #1135 migration to current azure sdk --- go.mod | 10 +- go.sum | 50 +++----- pkg/azurestore/azureservice.go | 198 +++++++++++++++--------------- pkg/azurestore/azurestore_test.go | 4 +- 4 files changed, 125 insertions(+), 137 deletions(-) diff --git a/go.mod b/go.mod index b278b7760..07115375c 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,15 @@ module github.com/tus/tusd/v2 // Specify the Go version needed for the Heroku deployment // See https://github.com/heroku/heroku-buildpack-go#go-module-specifics // +heroku goVersion go1.22 -go 1.21.0 +go 1.22.1 + toolchain go1.22.7 require ( cloud.google.com/go/storage v1.43.0 github.com/Acconut/go-httptest-recorder v1.0.0 - github.com/Azure/azure-storage-blob-go v0.14.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 github.com/Shopify/toxiproxy/v2 v2.9.0 github.com/aws/aws-sdk-go-v2 v1.31.0 github.com/aws/aws-sdk-go-v2/config v1.27.39 @@ -43,7 +45,7 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.2 // indirect cloud.google.com/go/iam v1.2.0 // indirect - github.com/Azure/azure-pipeline-go v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.37 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect @@ -77,7 +79,6 @@ require ( github.com/hashicorp/yamux v0.1.1 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -86,7 +87,6 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/xid v1.5.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect diff --git a/go.sum b/go.sum index 23bf23fcf..4b2f1293f 100644 --- a/go.sum +++ b/go.sum @@ -15,21 +15,18 @@ cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyX cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb6KZsFFFTGhoJg= github.com/Acconut/go-httptest-recorder v1.0.0/go.mod h1:CwQyhTH1kq/gLyWiRieo7c0uokpu3PXeyF/nZjUNtmM= -github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= -github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= -github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Shopify/toxiproxy/v2 v2.9.0 h1:DIaDZG2/r/kv3Em6UxYBUVnnWl1mHlYTGFv+sTPV7VI= github.com/Shopify/toxiproxy/v2 v2.9.0/go.mod h1:2uPRyxR46fsx2yUr9i8zcejzdkWfK7p6G23jV/X6YNs= @@ -101,8 +98,6 @@ github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -118,6 +113,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d h1:lBXNCxVENCipq4D1Is42JVOP4eQjlB8TQ6H69Yx5J9Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -152,7 +149,6 @@ github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0Z github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= @@ -181,12 +177,12 @@ github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= @@ -195,8 +191,6 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= -github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -213,8 +207,9 @@ github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -227,8 +222,8 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= @@ -278,7 +273,6 @@ go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -297,7 +291,6 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -319,10 +312,8 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -339,7 +330,6 @@ golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= diff --git a/pkg/azurestore/azureservice.go b/pkg/azurestore/azureservice.go index 031d07159..aa89cb963 100644 --- a/pkg/azurestore/azureservice.go +++ b/pkg/azurestore/azureservice.go @@ -20,24 +20,29 @@ import ( "encoding/binary" "fmt" "io" - "net/url" "sort" "strings" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/tus/tusd/v2/pkg/handler" ) const ( InfoBlobSuffix string = ".info" - MaxBlockBlobSize int64 = azblob.BlockBlobMaxBlocks * azblob.BlockBlobMaxStageBlockBytes - MaxBlockBlobChunkSize int64 = azblob.BlockBlobMaxStageBlockBytes + MaxBlockBlobSize int64 = blockblob.MaxBlocks * blockblob.MaxStageBlockBytes + MaxBlockBlobChunkSize int64 = blockblob.MaxStageBlockBytes ) type azService struct { - BlobAccessTier azblob.AccessTierType - ContainerURL *azblob.ContainerURL - ContainerName string + ContainerClient *container.Client + ContainerName string + BlobAccessTier *blob.AccessTier } type AzService interface { @@ -67,87 +72,92 @@ type AzBlob interface { } type BlockBlob struct { - Blob *azblob.BlockBlobURL - AccessTier azblob.AccessTierType - Indexes []int + Blob *blockblob.Client + Indexes []int + BlobAccessTier *blob.AccessTier } type InfoBlob struct { - Blob *azblob.BlockBlobURL + Blob *blockblob.Client } // New Azure service for communication to Azure BlockBlob Storage API func NewAzureService(config *AzConfig) (AzService, error) { // struct to store your credentials. - credential, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey) + cred, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey) if err != nil { return nil, err } - // Might be limited by the storage account - // "" or default inherits the access type from the Storage Account - var containerAccessType azblob.PublicAccessType + serviceURL := fmt.Sprintf("%s/%s", config.Endpoint, config.ContainerName) + retryOpts := policy.RetryOptions{ + MaxRetries: 20, // Max 20 retries + RetryDelay: 100, // Retry after 100ms initially + MaxRetryDelay: 5000, // Max retry delay 5 seconds + } + containerClient, err := container.NewClientWithSharedKeyCredential(serviceURL, cred, &container.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Retry: retryOpts, + }, + }) + if err != nil { + return nil, err + } + + var containerAccessType container.PublicAccessType switch config.ContainerAccessType { case "container": - containerAccessType = azblob.PublicAccessContainer + containerAccessType = container.PublicAccessTypeContainer case "blob": - containerAccessType = azblob.PublicAccessBlob - case "": + containerAccessType = container.PublicAccessTypeBlob default: - containerAccessType = azblob.PublicAccessNone + containerAccessType = "" + } + + // Do not care about response since it will fail if container exists and create if it does not. + _, err = containerClient.Create(context.Background(), &container.CreateOptions{ + Access: to.Ptr(containerAccessType), + }) + if err != nil && !strings.Contains(err.Error(), "ContainerAlreadyExists") { + return nil, err } - // Does not support the premium access tiers - var blobAccessTierType azblob.AccessTierType + var blobAccessTier *blob.AccessTier switch config.BlobAccessTier { case "archive": - blobAccessTierType = azblob.AccessTierArchive + blobAccessTier = to.Ptr(blob.AccessTierArchive) case "cool": - blobAccessTierType = azblob.AccessTierCool + blobAccessTier = to.Ptr(blob.AccessTierCool) case "hot": - blobAccessTierType = azblob.AccessTierHot + blobAccessTier = to.Ptr(blob.AccessTierHot) case "": default: - blobAccessTierType = azblob.DefaultAccessTier + blobAccessTier = nil } - // The pipeline specifies things like retry policies, logging, deserialization of HTTP response payloads, and more. - p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) - cURL, _ := url.Parse(fmt.Sprintf("%s/%s", config.Endpoint, config.ContainerName)) - - // Get the ContainerURL URL - containerURL := azblob.NewContainerURL(*cURL, p) - // Do not care about response since it will fail if container exists and create if it does not. - _, _ = containerURL.Create(context.Background(), azblob.Metadata{}, containerAccessType) - return &azService{ - BlobAccessTier: blobAccessTierType, - ContainerURL: &containerURL, - ContainerName: config.ContainerName, + ContainerClient: containerClient, + ContainerName: config.ContainerName, + BlobAccessTier: blobAccessTier, }, nil } // Determine if we return a InfoBlob or BlockBlob, based on the name func (service *azService) NewBlob(ctx context.Context, name string) (AzBlob, error) { - var fileBlob AzBlob - bb := service.ContainerURL.NewBlockBlobURL(name) + blobClient := service.ContainerClient.NewBlockBlobClient(name) if strings.HasSuffix(name, InfoBlobSuffix) { - fileBlob = &InfoBlob{ - Blob: &bb, - } - } else { - fileBlob = &BlockBlob{ - Blob: &bb, - Indexes: []int{}, - AccessTier: service.BlobAccessTier, - } + return &InfoBlob{Blob: blobClient}, nil } - return fileBlob, nil + return &BlockBlob{ + Blob: blobClient, + Indexes: []int{}, + BlobAccessTier: service.BlobAccessTier, + }, nil } // Delete the blockBlob from Azure Blob Storage func (blockBlob *BlockBlob) Delete(ctx context.Context) error { - _, err := blockBlob.Blob.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) + _, err := blockBlob.Blob.Delete(ctx, nil) return err } @@ -161,31 +171,22 @@ func (blockBlob *BlockBlob) Upload(ctx context.Context, body io.ReadSeeker) erro index = blockBlob.Indexes[len(blockBlob.Indexes)-1] + 1 } blockBlob.Indexes = append(blockBlob.Indexes, index) - - _, err := blockBlob.Blob.StageBlock(ctx, blockIDIntToBase64(index), body, azblob.LeaseAccessConditions{}, nil, azblob.ClientProvidedKeyOptions{}) - if err != nil { - return err - } - return nil + blockID := blockIDIntToBase64(index) + readSeekCloserBody := readSeekCloser{body} + _, err := blockBlob.Blob.StageBlock(ctx, blockID, readSeekCloserBody, nil) + return err } // Download the blockBlob from Azure Blob Storage func (blockBlob *BlockBlob) Download(ctx context.Context) (io.ReadCloser, error) { - downloadResponse, err := blockBlob.Blob.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) - - // If the file does not exist, it will not return an error, but a 404 status and body - if downloadResponse != nil && downloadResponse.StatusCode() == 404 { - return nil, handler.ErrNotFound - } + resp, err := blockBlob.Blob.DownloadStream(ctx, nil) if err != nil { - // This might occur when the blob is being uploaded, but a block list has not been committed yet - if isAzureError(err, "BlobNotFound") { - err = handler.ErrNotFound + if strings.Contains(err.Error(), "404") { // Check for not found error. + return nil, handler.ErrNotFound } return nil, err } - - return downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20}), nil + return resp.Body, nil } func (blockBlob *BlockBlob) GetOffset(ctx context.Context) (int64, error) { @@ -194,24 +195,22 @@ func (blockBlob *BlockBlob) GetOffset(ctx context.Context) (int64, error) { var indexes []int var offset int64 - getBlock, err := blockBlob.Blob.GetBlockList(ctx, azblob.BlockListAll, azblob.LeaseAccessConditions{}) + resp, err := blockBlob.Blob.GetBlockList(ctx, blockblob.BlockListTypeAll, nil) if err != nil { - if isAzureError(err, "BlobNotFound") { - err = handler.ErrNotFound + if strings.Contains(err.Error(), "BlobNotFound") { + return 0, handler.ErrNotFound } - return 0, err } // Need committed blocks to be added to offset to know how big the file really is - for _, block := range getBlock.CommittedBlocks { - offset += int64(block.Size) + for _, block := range resp.CommittedBlocks { + offset += *block.Size indexes = append(indexes, blockIDBase64ToInt(block.Name)) } - // Need to get the uncommitted blocks so that we can commit them - for _, block := range getBlock.UncommittedBlocks { - offset += int64(block.Size) + for _, block := range resp.UncommittedBlocks { + offset += *block.Size indexes = append(indexes, blockIDBase64ToInt(block.Name)) } @@ -226,17 +225,18 @@ func (blockBlob *BlockBlob) GetOffset(ctx context.Context) (int64, error) { // After all the blocks have been uploaded, we commit the unstaged blocks by sending a Block List func (blockBlob *BlockBlob) Commit(ctx context.Context) error { base64BlockIDs := make([]string, len(blockBlob.Indexes)) - for index, id := range blockBlob.Indexes { - base64BlockIDs[index] = blockIDIntToBase64(id) + for i, id := range blockBlob.Indexes { + base64BlockIDs[i] = blockIDIntToBase64(id) } - - _, err := blockBlob.Blob.CommitBlockList(ctx, base64BlockIDs, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, blockBlob.AccessTier, nil, azblob.ClientProvidedKeyOptions{}) + _, err := blockBlob.Blob.CommitBlockList(ctx, base64BlockIDs, &blockblob.CommitBlockListOptions{ + Tier: blockBlob.BlobAccessTier, + }) return err } // Delete the infoBlob from Azure Blob Storage func (infoBlob *InfoBlob) Delete(ctx context.Context) error { - _, err := infoBlob.Blob.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) + _, err := infoBlob.Blob.Delete(ctx, nil) return err } @@ -244,28 +244,23 @@ func (infoBlob *InfoBlob) Delete(ctx context.Context) error { // Because the info file is presumed to be smaller than azblob.BlockBlobMaxUploadBlobBytes (256MiB), we can upload it all in one go // New uploaded data will create a new, or overwrite the existing block blob func (infoBlob *InfoBlob) Upload(ctx context.Context, body io.ReadSeeker) error { - _, err := infoBlob.Blob.Upload(ctx, body, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, azblob.DefaultAccessTier, nil, azblob.ClientProvidedKeyOptions{}) + _, err := infoBlob.Blob.UploadStream(ctx, body, nil) return err } // Download the infoBlob from Azure Blob Storage func (infoBlob *InfoBlob) Download(ctx context.Context) (io.ReadCloser, error) { - downloadResponse, err := infoBlob.Blob.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) - - // If the file does not exist, it will not return an error, but a 404 status and body - if downloadResponse != nil && downloadResponse.StatusCode() == 404 { - return nil, fmt.Errorf("file %s does not exist", infoBlob.Blob.ToBlockBlobURL()) - } + resp, err := infoBlob.Blob.DownloadStream(ctx, nil) if err != nil { - if isAzureError(err, "BlobNotFound") { - err = handler.ErrNotFound + if strings.Contains(err.Error(), "404") { // Check for not found error. + return nil, handler.ErrNotFound } return nil, err } - - return downloadResponse.Body(azblob.RetryReaderOptions{MaxRetryRequests: 20}), nil + return resp.Body, nil } + // infoBlob does not utilise offset, so just return 0, nil func (infoBlob *InfoBlob) GetOffset(ctx context.Context) (int64, error) { return 0, nil @@ -283,8 +278,8 @@ func blockIDBinaryToBase64(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) } -func blockIDBase64ToBinary(blockID string) []byte { - binary, _ := base64.StdEncoding.DecodeString(blockID) +func blockIDBase64ToBinary(blockID *string) []byte { + binary, _ := base64.StdEncoding.DecodeString(*blockID) return binary } @@ -295,14 +290,17 @@ func blockIDIntToBase64(blockID int) string { return blockIDBinaryToBase64(binaryBlockID) } -func blockIDBase64ToInt(blockID string) int { +func blockIDBase64ToInt(blockID *string) int { blockIDBase64ToBinary(blockID) return int(binary.LittleEndian.Uint32(blockIDBase64ToBinary(blockID))) } -func isAzureError(err error, code string) bool { - if err, ok := err.(azblob.StorageError); ok && string(err.ServiceCode()) == code { - return true - } - return false +// readSeekCloser is a wrapper that adds a no-op Close method to an io.ReadSeeker. +type readSeekCloser struct { + io.ReadSeeker +} + +// Close implements io.Closer for readSeekCloser. +func (rsc readSeekCloser) Close() error { + return nil } diff --git a/pkg/azurestore/azurestore_test.go b/pkg/azurestore/azurestore_test.go index 023d59801..5b64e7290 100644 --- a/pkg/azurestore/azurestore_test.go +++ b/pkg/azurestore/azurestore_test.go @@ -8,7 +8,7 @@ import ( "io" "testing" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/tus/tusd/v2/pkg/azurestore" @@ -216,7 +216,7 @@ func TestGetUploadNotFound(t *testing.T) { ctx := context.Background() gomock.InOrder( service.EXPECT().NewBlob(ctx, mockID+".info").Return(infoBlob, nil).Times(1), - infoBlob.EXPECT().Download(ctx).Return(nil, errors.New(string(azblob.StorageErrorCodeBlobNotFound))).Times(1), + infoBlob.EXPECT().Download(ctx).Return(nil, errors.New(string(bloberror.BlobNotFound))).Times(1), ) _, err := store.GetUpload(context.Background(), mockID) From 74f7c657f73d1615a9898e49d7f9a023558bfa4b Mon Sep 17 00:00:00 2001 From: xelat09 Date: Wed, 23 Oct 2024 15:33:47 +0200 Subject: [PATCH 2/6] fix findings --- pkg/azurestore/azureservice.go | 76 ++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/pkg/azurestore/azureservice.go b/pkg/azurestore/azureservice.go index aa89cb963..6d925ed0d 100644 --- a/pkg/azurestore/azureservice.go +++ b/pkg/azurestore/azureservice.go @@ -18,6 +18,7 @@ import ( "context" "encoding/base64" "encoding/binary" + "errors" "fmt" "io" "sort" @@ -28,6 +29,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/tus/tusd/v2/pkg/handler" @@ -72,13 +74,13 @@ type AzBlob interface { } type BlockBlob struct { - Blob *blockblob.Client + BlobClient *blockblob.Client Indexes []int BlobAccessTier *blob.AccessTier } type InfoBlob struct { - Blob *blockblob.Client + BlobClient *blockblob.Client } // New Azure service for communication to Azure BlockBlob Storage API @@ -114,10 +116,16 @@ func NewAzureService(config *AzConfig) (AzService, error) { containerAccessType = "" } + // default is private + var containerCreateOptions *container.CreateOptions = nil + if containerAccessType != "" { + containerCreateOptions = &container.CreateOptions{ + Access: to.Ptr(containerAccessType), + } + } + // Do not care about response since it will fail if container exists and create if it does not. - _, err = containerClient.Create(context.Background(), &container.CreateOptions{ - Access: to.Ptr(containerAccessType), - }) + _, err = containerClient.Create(context.Background(), containerCreateOptions) if err != nil && !strings.Contains(err.Error(), "ContainerAlreadyExists") { return nil, err } @@ -146,10 +154,10 @@ func NewAzureService(config *AzConfig) (AzService, error) { func (service *azService) NewBlob(ctx context.Context, name string) (AzBlob, error) { blobClient := service.ContainerClient.NewBlockBlobClient(name) if strings.HasSuffix(name, InfoBlobSuffix) { - return &InfoBlob{Blob: blobClient}, nil + return &InfoBlob{BlobClient: blobClient}, nil } return &BlockBlob{ - Blob: blobClient, + BlobClient: blobClient, Indexes: []int{}, BlobAccessTier: service.BlobAccessTier, }, nil @@ -157,7 +165,7 @@ func (service *azService) NewBlob(ctx context.Context, name string) (AzBlob, err // Delete the blockBlob from Azure Blob Storage func (blockBlob *BlockBlob) Delete(ctx context.Context) error { - _, err := blockBlob.Blob.Delete(ctx, nil) + _, err := blockBlob.BlobClient.Delete(ctx, nil) return err } @@ -173,18 +181,15 @@ func (blockBlob *BlockBlob) Upload(ctx context.Context, body io.ReadSeeker) erro blockBlob.Indexes = append(blockBlob.Indexes, index) blockID := blockIDIntToBase64(index) readSeekCloserBody := readSeekCloser{body} - _, err := blockBlob.Blob.StageBlock(ctx, blockID, readSeekCloserBody, nil) + _, err := blockBlob.BlobClient.StageBlock(ctx, blockID, readSeekCloserBody, nil) return err } // Download the blockBlob from Azure Blob Storage func (blockBlob *BlockBlob) Download(ctx context.Context) (io.ReadCloser, error) { - resp, err := blockBlob.Blob.DownloadStream(ctx, nil) + resp, err := blockBlob.BlobClient.DownloadStream(ctx, nil) if err != nil { - if strings.Contains(err.Error(), "404") { // Check for not found error. - return nil, handler.ErrNotFound - } - return nil, err + return nil, handleError(err) } return resp.Body, nil } @@ -195,12 +200,9 @@ func (blockBlob *BlockBlob) GetOffset(ctx context.Context) (int64, error) { var indexes []int var offset int64 - resp, err := blockBlob.Blob.GetBlockList(ctx, blockblob.BlockListTypeAll, nil) + resp, err := blockBlob.BlobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil) if err != nil { - if strings.Contains(err.Error(), "BlobNotFound") { - return 0, handler.ErrNotFound - } - return 0, err + return 0, handleError(err) } // Need committed blocks to be added to offset to know how big the file really is @@ -228,15 +230,22 @@ func (blockBlob *BlockBlob) Commit(ctx context.Context) error { for i, id := range blockBlob.Indexes { base64BlockIDs[i] = blockIDIntToBase64(id) } - _, err := blockBlob.Blob.CommitBlockList(ctx, base64BlockIDs, &blockblob.CommitBlockListOptions{ - Tier: blockBlob.BlobAccessTier, - }) + + // default is set on container + var commitBlockListOptions *blockblob.CommitBlockListOptions + if blockBlob.BlobAccessTier != nil { + commitBlockListOptions = &blockblob.CommitBlockListOptions{ + Tier: blockBlob.BlobAccessTier, + } + } + + _, err := blockBlob.BlobClient.CommitBlockList(ctx, base64BlockIDs, commitBlockListOptions) return err } // Delete the infoBlob from Azure Blob Storage func (infoBlob *InfoBlob) Delete(ctx context.Context) error { - _, err := infoBlob.Blob.Delete(ctx, nil) + _, err := infoBlob.BlobClient.Delete(ctx, nil) return err } @@ -244,23 +253,19 @@ func (infoBlob *InfoBlob) Delete(ctx context.Context) error { // Because the info file is presumed to be smaller than azblob.BlockBlobMaxUploadBlobBytes (256MiB), we can upload it all in one go // New uploaded data will create a new, or overwrite the existing block blob func (infoBlob *InfoBlob) Upload(ctx context.Context, body io.ReadSeeker) error { - _, err := infoBlob.Blob.UploadStream(ctx, body, nil) + _, err := infoBlob.BlobClient.UploadStream(ctx, body, nil) return err } // Download the infoBlob from Azure Blob Storage func (infoBlob *InfoBlob) Download(ctx context.Context) (io.ReadCloser, error) { - resp, err := infoBlob.Blob.DownloadStream(ctx, nil) + resp, err := infoBlob.BlobClient.DownloadStream(ctx, nil) if err != nil { - if strings.Contains(err.Error(), "404") { // Check for not found error. - return nil, handler.ErrNotFound - } - return nil, err + return nil, handleError(err) } return resp.Body, nil } - // infoBlob does not utilise offset, so just return 0, nil func (infoBlob *InfoBlob) GetOffset(ctx context.Context) (int64, error) { return 0, nil @@ -304,3 +309,14 @@ type readSeekCloser struct { func (rsc readSeekCloser) Close() error { return nil } + +func handleError(err error) error { + var azureError *azcore.ResponseError + if errors.As(err, &azureError) { + code := bloberror.Code(azureError.ErrorCode) + if code == bloberror.BlobNotFound || azureError.StatusCode == 404 { + return handler.ErrNotFound + } + } + return err +} From 05005ae48d6f452ffbee1ff0768c63016b70a46d Mon Sep 17 00:00:00 2001 From: xelat09 Date: Wed, 23 Oct 2024 15:55:13 +0200 Subject: [PATCH 3/6] make sure snapshots are deleted too --- pkg/azurestore/azureservice.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/azurestore/azureservice.go b/pkg/azurestore/azureservice.go index 6d925ed0d..63b5907d0 100644 --- a/pkg/azurestore/azureservice.go +++ b/pkg/azurestore/azureservice.go @@ -165,7 +165,11 @@ func (service *azService) NewBlob(ctx context.Context, name string) (AzBlob, err // Delete the blockBlob from Azure Blob Storage func (blockBlob *BlockBlob) Delete(ctx context.Context) error { - _, err := blockBlob.BlobClient.Delete(ctx, nil) + // Specify that you want to delete both the blob and its snapshots + deleteOptions := &azblob.DeleteBlobOptions{ + DeleteSnapshots: to.Ptr(azblob.DeleteSnapshotsOptionTypeInclude), + } + _, err := blockBlob.BlobClient.Delete(ctx, deleteOptions) return err } From 6273ddf6cf6d4a2966264b8805d95717218f5089 Mon Sep 17 00:00:00 2001 From: Marius Kleidl Date: Tue, 26 Nov 2024 14:57:23 +0100 Subject: [PATCH 4/6] Minor touches --- pkg/azurestore/azureservice.go | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/pkg/azurestore/azureservice.go b/pkg/azurestore/azureservice.go index 63b5907d0..f0b0856c3 100644 --- a/pkg/azurestore/azureservice.go +++ b/pkg/azurestore/azureservice.go @@ -106,22 +106,15 @@ func NewAzureService(config *AzConfig) (AzService, error) { return nil, err } - var containerAccessType container.PublicAccessType + // default is private + containerCreateOptions := &container.CreateOptions{} switch config.ContainerAccessType { case "container": - containerAccessType = container.PublicAccessTypeContainer + containerCreateOptions.Access = to.Ptr(container.PublicAccessTypeContainer) case "blob": - containerAccessType = container.PublicAccessTypeBlob + containerCreateOptions.Access = to.Ptr(container.PublicAccessTypeBlob) default: - containerAccessType = "" - } - - // default is private - var containerCreateOptions *container.CreateOptions = nil - if containerAccessType != "" { - containerCreateOptions = &container.CreateOptions{ - Access: to.Ptr(containerAccessType), - } + // Leaving Access nil will default to private access } // Do not care about response since it will fail if container exists and create if it does not. @@ -138,9 +131,6 @@ func NewAzureService(config *AzConfig) (AzService, error) { blobAccessTier = to.Ptr(blob.AccessTierCool) case "hot": blobAccessTier = to.Ptr(blob.AccessTierHot) - case "": - default: - blobAccessTier = nil } return &azService{ @@ -193,7 +183,7 @@ func (blockBlob *BlockBlob) Upload(ctx context.Context, body io.ReadSeeker) erro func (blockBlob *BlockBlob) Download(ctx context.Context) (io.ReadCloser, error) { resp, err := blockBlob.BlobClient.DownloadStream(ctx, nil) if err != nil { - return nil, handleError(err) + return nil, checkForNotFoundError(err) } return resp.Body, nil } @@ -206,7 +196,7 @@ func (blockBlob *BlockBlob) GetOffset(ctx context.Context) (int64, error) { resp, err := blockBlob.BlobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil) if err != nil { - return 0, handleError(err) + return 0, checkForNotFoundError(err) } // Need committed blocks to be added to offset to know how big the file really is @@ -265,7 +255,7 @@ func (infoBlob *InfoBlob) Upload(ctx context.Context, body io.ReadSeeker) error func (infoBlob *InfoBlob) Download(ctx context.Context) (io.ReadCloser, error) { resp, err := infoBlob.BlobClient.DownloadStream(ctx, nil) if err != nil { - return nil, handleError(err) + return nil, checkForNotFoundError(err) } return resp.Body, nil } @@ -314,7 +304,9 @@ func (rsc readSeekCloser) Close() error { return nil } -func handleError(err error) error { +// checkForNotFoundError checks if the error indicates that a resource was not found. +// If so, we return the corresponding tusd error. +func checkForNotFoundError(err error) error { var azureError *azcore.ResponseError if errors.As(err, &azureError) { code := bloberror.Code(azureError.ErrorCode) From b230a2475060e968125f25e0752940795b8dbe00 Mon Sep 17 00:00:00 2001 From: Marius Kleidl Date: Tue, 26 Nov 2024 15:06:52 +0100 Subject: [PATCH 5/6] More touches --- pkg/azurestore/azureservice.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/pkg/azurestore/azureservice.go b/pkg/azurestore/azureservice.go index f0b0856c3..4bac8d0af 100644 --- a/pkg/azurestore/azureservice.go +++ b/pkg/azurestore/azureservice.go @@ -93,7 +93,7 @@ func NewAzureService(config *AzConfig) (AzService, error) { serviceURL := fmt.Sprintf("%s/%s", config.Endpoint, config.ContainerName) retryOpts := policy.RetryOptions{ - MaxRetries: 20, // Max 20 retries + MaxRetries: 5, RetryDelay: 100, // Retry after 100ms initially MaxRetryDelay: 5000, // Max retry delay 5 seconds } @@ -106,7 +106,6 @@ func NewAzureService(config *AzConfig) (AzService, error) { return nil, err } - // default is private containerCreateOptions := &container.CreateOptions{} switch config.ContainerAccessType { case "container": @@ -117,12 +116,12 @@ func NewAzureService(config *AzConfig) (AzService, error) { // Leaving Access nil will default to private access } - // Do not care about response since it will fail if container exists and create if it does not. _, err = containerClient.Create(context.Background(), containerCreateOptions) if err != nil && !strings.Contains(err.Error(), "ContainerAlreadyExists") { return nil, err } + // Does not support the premium access tiers yet. var blobAccessTier *blob.AccessTier switch config.BlobAccessTier { case "archive": @@ -225,15 +224,9 @@ func (blockBlob *BlockBlob) Commit(ctx context.Context) error { base64BlockIDs[i] = blockIDIntToBase64(id) } - // default is set on container - var commitBlockListOptions *blockblob.CommitBlockListOptions - if blockBlob.BlobAccessTier != nil { - commitBlockListOptions = &blockblob.CommitBlockListOptions{ - Tier: blockBlob.BlobAccessTier, - } - } - - _, err := blockBlob.BlobClient.CommitBlockList(ctx, base64BlockIDs, commitBlockListOptions) + _, err := blockBlob.BlobClient.CommitBlockList(ctx, base64BlockIDs, &blockblob.CommitBlockListOptions{ + Tier: blockBlob.BlobAccessTier, + }) return err } From 67aeec9aedfa1ce05316f4e5bb2b86adcae99a56 Mon Sep 17 00:00:00 2001 From: Marius Kleidl Date: Tue, 26 Nov 2024 15:12:21 +0100 Subject: [PATCH 6/6] Undo change in go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4c6a1b63b..ceee40ab6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/tus/tusd/v2 // Specify the Go version needed for the Heroku deployment // See https://github.com/heroku/heroku-buildpack-go#go-module-specifics // +heroku goVersion go1.22 -go 1.22.1 +go 1.21.0 toolchain go1.22.7