fix: Retry gRPC UNAUTHENTICATED errors like 401s. (#2781)

HTTP 401s are retried transparently by GCSFuse, because we may have a
delay between when we fetch an OAuth token and when it is validated in
GCS. This applies equivalently to the gRPC error space.

internal/storage/storageutil/custom_retry.go

@@ -18,6 +18,8 @@ import (
 	"cloud.google.com/go/storage"
 	"github.com/googlecloudplatform/gcsfuse/v2/internal/logger"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func ShouldRetry(err error) (b bool) {
@@ -41,5 +43,16 @@ func ShouldRetry(err error) (b bool) {
 			return
 		}
 	}
+
+	// This is the same case as above, but for gRPC UNAUTHENTICATED errors. See
+	// https://github.com/golang/oauth2/issues/623
+	// TODO: Please incorporate the correct fix post resolution of the above issue.
+	if status, ok := status.FromError(err); ok {
+		if status.Code() == codes.Unauthenticated {
+			b = true
+			logger.Infof("Retrying for UNAUTHENTICATED error: %v", err)
+			return
+		}
+	}
 	return
 }

internal/storage/storageutil/custom_retry_test.go

@@ -23,6 +23,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func TestShouldRetryReturnsTrueWithGoogleApiError(t *testing.T) {
@@ -118,3 +120,29 @@ func TestShouldRetryReturnsTrueForConnectionRefusedAndResetErrors(t *testing.T)
 		})
 	}
 }
+
+func TestShouldRetryReturnsTrueForUnauthenticatedGrpcErrors(t *testing.T) {
+	testCases := []struct {
+		name           string
+		err            error
+		expectedResult bool
+	}{
+		{
+			name:           "UNAUTHENTICATED",
+			err:            status.Error(codes.Unauthenticated, "Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project."),
+			expectedResult: true,
+		},
+		{
+			name:           "PERMISSION_DENIED",
+			err:            status.Error(codes.PermissionDenied, "unauthorized"),
+			expectedResult: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actualResult := ShouldRetry(tc.err)
+			assert.Equal(t, tc.expectedResult, actualResult)
+		})
+	}
+}