Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cancel S3 CRT requests if should not continue #3231

Merged
merged 1 commit into from
Jan 2, 2025
Merged

Cancel S3 CRT requests if should not continue #3231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions generated/src/aws-cpp-sdk-s3-crt/include/aws/s3-crt/S3CrtClient.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6976,14 +6976,12 @@ namespace Aws
std::shared_ptr<Aws::Utils::Threading::Semaphore> clientShutdownSem;
};

static void CrtClientShutdownCallback(void *data) {
auto *wrappedData = static_cast<CrtClientShutdownCallbackDataWrapper*>(data);
if (wrappedData->fn)
{
wrappedData->fn(wrappedData->data);
}
wrappedData->clientShutdownSem->Release();
}
static void CrtClientShutdownCallback(void *data);
void CancelCrtRequestAsync(aws_s3_meta_request *meta_request) const;
static int S3CrtRequestHeadersCallback(aws_s3_meta_request *meta_request, const struct aws_http_headers *headers, int response_status, void *user_data);
static int S3CrtRequestGetBodyCallback(struct aws_s3_meta_request *meta_request, const struct aws_byte_cursor *body, uint64_t range_start, void *user_data);
static void S3CrtRequestProgressCallback(struct aws_s3_meta_request *meta_request, const struct aws_s3_meta_request_progress *progress, void *user_data);
static void S3CrtRequestFinishCallback(struct aws_s3_meta_request *meta_request, const struct aws_s3_meta_request_result *meta_request_result, void *user_data);

void InitCrtEndpointFromUri(aws_uri &endpoint_uri, const Aws::Http::URI &uri) const;

Expand Down
154 changes: 135 additions & 19 deletions generated/src/aws-cpp-sdk-s3-crt/source/S3CrtClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -517,28 +517,59 @@ void S3CrtClient::OverrideEndpoint(const Aws::String& endpoint)
m_endpointProvider->OverrideEndpoint(endpoint);
}

void S3CrtClient::CrtClientShutdownCallback(void *data)
{
AWS_CHECK_PTR(SERVICE_NAME, data);
auto *wrappedData = static_cast<CrtClientShutdownCallbackDataWrapper*>(data);
if (wrappedData->fn)
{
wrappedData->fn(wrappedData->data);
}
AWS_CHECK_PTR(SERVICE_NAME, wrappedData->clientShutdownSem);
wrappedData->clientShutdownSem->Release();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

null ptr check

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I moved this method from .h to .cpp, but it is added now, thank you.

}

void S3CrtClient::CancelCrtRequestAsync(aws_s3_meta_request *meta_request) const {
assert(meta_request);
m_clientConfiguration.executor->Submit([meta_request]() {
aws_s3_meta_request_cancel(meta_request);
});
}

static int S3CrtRequestHeadersCallback(struct aws_s3_meta_request *meta_request, const struct aws_http_headers *headers,
int S3CrtClient::S3CrtRequestHeadersCallback(struct aws_s3_meta_request *meta_request, const struct aws_http_headers *headers,
int response_status, void *user_data)
{
AWS_UNREFERENCED_PARAM(meta_request);
auto *userData = static_cast<S3CrtClient::CrtRequestCallbackUserData*>(user_data);
if (!userData || !userData->response || !userData->originalRequest) {
return AWS_OP_ERR;
}

size_t headersCount = aws_http_headers_count(headers);
for (size_t i = 0; i < headersCount; i++)
{
for (size_t i = 0; i < headersCount; i++) {
struct aws_http_header header;
aws_http_headers_get_index(headers, i, &header);
userData->response->AddHeader(StringUtils::FromByteCursor(header.name), StringUtils::FromByteCursor(header.value));
}
userData->response->SetResponseCode(static_cast<HttpResponseCode>(response_status));

auto& shouldContinueFn = userData->originalRequest->GetContinueRequestHandler();
const HttpRequest* httpRequest = userData->request ? userData->request.get() : nullptr;
if (shouldContinueFn && !shouldContinueFn(httpRequest)) {
userData->s3CrtClient->CancelCrtRequestAsync(meta_request);
}

return AWS_OP_SUCCESS;
}

static int S3CrtRequestGetBodyCallback(struct aws_s3_meta_request *meta_request, const struct aws_byte_cursor *body, uint64_t range_start, void *user_data)
int S3CrtClient::S3CrtRequestGetBodyCallback(struct aws_s3_meta_request *meta_request, const struct aws_byte_cursor *body, uint64_t range_start, void *user_data)
{
AWS_UNREFERENCED_PARAM(range_start);

auto *userData = static_cast<S3CrtClient::CrtRequestCallbackUserData*>(user_data);
if (!userData || !userData->response || !userData->request) {
return AWS_OP_ERR;
}
auto& bodyStream = userData->response->GetResponseBody();

bodyStream.write(reinterpret_cast<char*>(body->ptr), static_cast<std::streamsize>(body->len));
Expand All @@ -557,26 +588,90 @@ static int S3CrtRequestGetBodyCallback(struct aws_s3_meta_request *meta_request,
receivedHandler(userData->request.get(), userData->response.get(), static_cast<long long>(body->len));
}
AWS_LOGSTREAM_TRACE(ALLOCATION_TAG, body->len << " bytes written to response.");
auto& shouldContinueFn = userData->originalRequest->GetContinueRequestHandler();
const HttpRequest* httpRequest = userData->request ? userData->request.get() : nullptr;
if (shouldContinueFn && !shouldContinueFn(httpRequest)) {
userData->s3CrtClient->CancelCrtRequestAsync(meta_request);
}

return AWS_OP_SUCCESS;
}

static void S3CrtRequestProgressCallback(struct aws_s3_meta_request *meta_request, const struct aws_s3_meta_request_progress *progress, void *user_data)
void S3CrtClient::S3CrtRequestProgressCallback(struct aws_s3_meta_request *meta_request, const struct aws_s3_meta_request_progress *progress, void *user_data)
{
AWS_UNREFERENCED_PARAM(meta_request);
AWS_CHECK_PTR(SERVICE_NAME, user_data);
auto *userData = static_cast<S3CrtClient::CrtRequestCallbackUserData*>(user_data);

AWS_CHECK_PTR(SERVICE_NAME, userData->request);
auto& progressHandler = userData->request->GetDataSentEventHandler();
if (progressHandler)
{
progressHandler(userData->request.get(), static_cast<long long>(progress->bytes_transferred));
if (progressHandler) {
progressHandler(userData->request.get(), static_cast<long long>(progress->bytes_transferred));
}
AWS_LOGSTREAM_TRACE(ALLOCATION_TAG, progress->bytes_transferred << " bytes transferred.");
AWS_CHECK_PTR(SERVICE_NAME, userData->originalRequest);
auto& shouldContinueFn = userData->originalRequest->GetContinueRequestHandler();
const HttpRequest* httpRequest = userData->request ? userData->request.get() : nullptr;
if (shouldContinueFn && !shouldContinueFn(httpRequest)) {
userData->s3CrtClient->CancelCrtRequestAsync(meta_request);
}

return;
}

static void S3CrtRequestFinishCallback(struct aws_s3_meta_request *meta_request,
CoreErrors MapCrtError(const int crtErrorCode) {
switch (crtErrorCode) {
case aws_s3_errors::AWS_ERROR_S3_REQUEST_HAS_COMPLETED:
return CoreErrors::OK;
case aws_s3_errors::AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER:
case aws_s3_errors::AWS_ERROR_S3_MISSING_CONTENT_LENGTH_HEADER:
case aws_s3_errors::AWS_ERROR_S3_MISSING_ETAG:
case aws_s3_errors::AWS_ERROR_S3_MISSING_UPLOAD_ID:
return CoreErrors::MISSING_PARAMETER;
case aws_s3_errors::AWS_ERROR_S3_INVALID_CONTENT_RANGE_HEADER:
case aws_s3_errors::AWS_ERROR_S3_INVALID_CONTENT_LENGTH_HEADER:
case aws_s3_errors::AWS_ERROR_S3_INVALID_RANGE_HEADER:
case aws_s3_errors::AWS_ERROR_S3_MULTIRANGE_HEADER_UNSUPPORTED:
case aws_s3_errors::AWS_ERROR_S3_INCORRECT_CONTENT_LENGTH:
case aws_s3_errors::AWS_ERROR_S3_INVALID_MEMORY_LIMIT_CONFIG:
return CoreErrors::INVALID_PARAMETER_VALUE;
case aws_s3_errors::AWS_ERROR_S3_INTERNAL_ERROR:
case aws_s3_errors::AWS_ERROR_S3_PROXY_PARSE_FAILED:
case aws_s3_errors::AWS_ERROR_S3_UNSUPPORTED_PROXY_SCHEME:
case aws_s3_errors::AWS_ERROR_S3_NON_RECOVERABLE_ASYNC_ERROR:
case aws_s3_errors::AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE:
case aws_s3_errors::AWS_ERROR_S3_EXCEEDS_MEMORY_LIMIT:
return CoreErrors::INTERNAL_FAILURE;
case aws_s3_errors::AWS_ERROR_S3_SLOW_DOWN:
return CoreErrors::SLOW_DOWN;
case aws_s3_errors::AWS_ERROR_S3_INVALID_RESPONSE_STATUS:
case aws_s3_errors::AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH:
case aws_s3_errors::AWS_ERROR_S3_CHECKSUM_CALCULATION_FAILED:
case aws_s3_errors::AWS_ERROR_S3_LIST_PARTS_PARSE_FAILED:
case aws_s3_errors::AWS_ERROR_S3_RESUMED_PART_CHECKSUM_MISMATCH:
case aws_s3_errors::AWS_ERROR_S3_FILE_MODIFIED:
case aws_s3_errors::AWS_ERROR_S3_INTERNAL_PART_SIZE_MISMATCH_RETRYING_WITH_RANGE:
case aws_s3_errors::AWS_ERROR_S3_RECV_FILE_ALREADY_EXISTS:
case aws_s3_errors::AWS_ERROR_S3_RECV_FILE_NOT_FOUND:
return CoreErrors::VALIDATION;
case aws_s3_errors::AWS_ERROR_S3_CANCELED:
return CoreErrors::USER_CANCELLED;
case aws_s3_errors::AWS_ERROR_S3_PAUSED:
case aws_s3_errors::AWS_ERROR_S3_RESUME_FAILED:
case aws_s3_errors::AWS_ERROR_S3_OBJECT_MODIFIED:
return CoreErrors::UNKNOWN;
case aws_s3_errors::AWS_ERROR_S3_REQUEST_TIME_TOO_SKEWED:
return CoreErrors::REQUEST_TIME_TOO_SKEWED;
case aws_s3_errors::AWS_ERROR_S3EXPRESS_CREATE_SESSION_FAILED:
return CoreErrors::CLIENT_SIGNING_FAILURE;
case aws_s3_errors::AWS_ERROR_S3_REQUEST_TIMEOUT:
return CoreErrors::REQUEST_TIMEOUT;
default:
return CoreErrors::INTERNAL_FAILURE;
}
}

void S3CrtClient::S3CrtRequestFinishCallback(struct aws_s3_meta_request *meta_request,
const struct aws_s3_meta_request_result *meta_request_result, void *user_data)
{
AWS_UNREFERENCED_PARAM(meta_request);
Expand Down Expand Up @@ -623,7 +718,7 @@ static void S3CrtRequestFinishCallback(struct aws_s3_meta_request *meta_request,
<< " (" << aws_error_lib_name(meta_request_result->error_code) << ": " << aws_error_name(meta_request_result->error_code) << ")";

userData->response->SetClientErrorMessage(ss.str());
userData->response->SetClientErrorType(CoreErrors::INTERNAL_FAILURE);
userData->response->SetClientErrorType(MapCrtError(meta_request_result->error_code));
}

aws_s3_meta_request_release(meta_request);
Expand Down Expand Up @@ -876,9 +971,16 @@ void S3CrtClient::CopyObjectAsync(const CopyObjectRequest& request, const CopyOb
options.message= crtHttpRequest->GetUnderlyingMessage();
userData->crtHttpRequest = crtHttpRequest;

if (aws_s3_client_make_meta_request(m_s3CrtClient, &options) == nullptr)
{
return handler(this, request, CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
aws_s3_meta_request* meta_request = aws_s3_client_make_meta_request(m_s3CrtClient, &options);
if (meta_request == nullptr) {
return handler(this, request,
CopyObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE",
"Unable to create s3 meta request", false)), handlerContext);
}
auto& shouldContinueFn = request.GetContinueRequestHandler();
const HttpRequest* httpRequest = userData->request ? userData->request.get() : nullptr;
if (shouldContinueFn && !shouldContinueFn(httpRequest)) {
aws_s3_meta_request_cancel(meta_request);
}
}

Expand Down Expand Up @@ -1021,9 +1123,16 @@ void S3CrtClient::GetObjectAsync(const GetObjectRequest& request, const GetObjec
options.message= crtHttpRequest->GetUnderlyingMessage();
userData->crtHttpRequest = crtHttpRequest;

if (aws_s3_client_make_meta_request(m_s3CrtClient, &options) == nullptr)
{
return handler(this, request, GetObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
aws_s3_meta_request* meta_request = aws_s3_client_make_meta_request(m_s3CrtClient, &options);
if (meta_request == nullptr) {
return handler(this, request,
GetObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE",
"Unable to create s3 meta request", false)), handlerContext);
}
auto& shouldContinueFn = request.GetContinueRequestHandler();
const HttpRequest* httpRequest = userData->request ? userData->request.get() : nullptr;
if (shouldContinueFn && !shouldContinueFn(httpRequest)) {
aws_s3_meta_request_cancel(meta_request);
}
}

Expand Down Expand Up @@ -1201,9 +1310,16 @@ void S3CrtClient::PutObjectAsync(const PutObjectRequest& request, const PutObjec
options.message= crtHttpRequest->GetUnderlyingMessage();
userData->crtHttpRequest = crtHttpRequest;

if (aws_s3_client_make_meta_request(m_s3CrtClient, &options) == nullptr)
{
return handler(this, request, PutObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false)), handlerContext);
aws_s3_meta_request* meta_request = aws_s3_client_make_meta_request(m_s3CrtClient, &options);
if (meta_request == nullptr) {
return handler(this, request,
PutObjectOutcome(Aws::Client::AWSError<S3CrtErrors>(S3CrtErrors::INTERNAL_FAILURE, "INTERNAL_FAILURE",
"Unable to create s3 meta request", false)), handlerContext);
}
auto& shouldContinueFn = request.GetContinueRequestHandler();
const HttpRequest* httpRequest = userData->request ? userData->request.get() : nullptr;
if (shouldContinueFn && !shouldContinueFn(httpRequest)) {
aws_s3_meta_request_cancel(meta_request);
}
}

Expand Down
1 change: 1 addition & 0 deletions tests/aws-cpp-sdk-s3-crt-integration-tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ foreach(TEST IN LISTS TEST_LIST)
"${TEST_MAIN_FILE}"
"BucketAndObjectOperationTest.cpp"
"S3ExpressTest.cpp"
"CancelCrtRequestTest.cpp"
)

file(GLOB AWS_S3_CRT_INTEGRATION_TESTS_SRC
Expand Down
Loading
Loading