Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Execute websocket failure on secure tunnel event loop #104

Closed
wants to merge 10 commits into from
Closed

Execute websocket failure on secure tunnel event loop #104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
da3be14
Execute websocket failure on secure tunnel event loop
sfodagain Dec 28, 2023
d01b79c
Fix format
sfodagain Dec 28, 2023
99e7afe
Add test for ws setup failure
sfodagain Dec 29, 2023
d615464
Fix format
sfodagain Dec 29, 2023
8e44180
Use managed thread in tests
sfodagain Dec 29, 2023
3fd1dad
Add comments
sfodagain Dec 29, 2023
9ee6ebf
Revert "Use managed thread in tests"
sfodagain Dec 29, 2023
9ed79b6
Merge branch 'main' into fix-data-race-on-ws-fail
sfodagain Jan 2, 2024
1782f07
Fix naming
sfodagain Jan 3, 2024
57e9f20
Remove excess checks in tests
sfodagain Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 63 additions & 5 deletions source/secure_tunneling.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1080,6 +1080,68 @@ static void s_on_websocket_shutdown(struct aws_websocket *websocket, int error_c
}
}

static void s_secure_tunnel_websocket_shutdown(
struct aws_websocket *websocket,
int error_code,
struct aws_secure_tunnel *secure_tunnel) {

AWS_FATAL_ASSERT(aws_event_loop_thread_is_callers_thread(secure_tunnel->loop));

/* Report a failed WebSocket Upgrade attempt */
if (secure_tunnel->config->on_connection_complete) {
secure_tunnel->config->on_connection_complete(NULL, error_code, secure_tunnel->config->user_data);
}
s_on_websocket_shutdown(websocket, error_code, secure_tunnel);
}

struct aws_secure_tunnel_websocket_shutdown_task {
struct aws_task task;
struct aws_allocator *allocator;
struct aws_secure_tunnel *secure_tunnel;
struct aws_websocket *websocket;
int error_code;
};

void s_secure_tunnel_websocket_shutdown_task_fn(struct aws_task *task, void *arg, enum aws_task_status status) {
(void)task;

struct aws_secure_tunnel_websocket_shutdown_task *shutdown_task = arg;

if (status != AWS_TASK_STATUS_RUN_READY) {
goto done;
}

s_secure_tunnel_websocket_shutdown(
shutdown_task->websocket, shutdown_task->error_code, shutdown_task->secure_tunnel);

done:
aws_mem_release(shutdown_task->allocator, shutdown_task);
}

/* Called on failed WebSocket setup attempt. Ensures that the actual failed setup logic is executed on the secure
* tunnel's event loop. */
static void s_on_websocket_setup_failed(
struct aws_websocket *websocket,
int error_code,
struct aws_secure_tunnel *secure_tunnel) {

if (aws_event_loop_thread_is_callers_thread(secure_tunnel->loop)) {
s_secure_tunnel_websocket_shutdown(websocket, error_code, secure_tunnel);
return;
}

struct aws_secure_tunnel_websocket_shutdown_task *shutdown_task =
aws_mem_calloc(secure_tunnel->allocator, 1, sizeof(struct aws_secure_tunnel_websocket_shutdown_task));

aws_task_init(&shutdown_task->task, s_secure_tunnel_websocket_shutdown_task_fn, shutdown_task, "ShutdownTask");
shutdown_task->allocator = secure_tunnel->allocator;
shutdown_task->secure_tunnel = secure_tunnel;
shutdown_task->websocket = websocket;
shutdown_task->error_code = error_code;

aws_event_loop_schedule_task_now(secure_tunnel->loop, &shutdown_task->task);
}

/* Called on successful or failed websocket setup attempt */
static void s_on_websocket_setup(const struct aws_websocket_on_connection_setup_data *setup, void *user_data) {
struct aws_secure_tunnel *secure_tunnel = user_data;
Expand All @@ -1091,12 +1153,8 @@ static void s_on_websocket_setup(const struct aws_websocket_on_connection_setup_
secure_tunnel->websocket = setup->websocket;

if (setup->error_code != AWS_OP_SUCCESS) {
/* Report a failed WebSocket Upgrade attempt */
if (secure_tunnel->config->on_connection_complete) {
secure_tunnel->config->on_connection_complete(NULL, setup->error_code, secure_tunnel->config->user_data);
}
/* Failed/Successful websocket creation and associated errors logged by "websocket-setup" */
s_on_websocket_shutdown(secure_tunnel->websocket, setup->error_code, secure_tunnel);
s_on_websocket_setup_failed(secure_tunnel->websocket, setup->error_code, secure_tunnel);
return;
}

Expand Down
1 change: 1 addition & 0 deletions tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ endif()
add_net_test_case(secure_tunneling_functionality_connect_test)
add_net_test_case(secure_tunneling_functionality_client_token_test)
add_net_test_case(secure_tunneling_fail_and_retry_connection_test)
add_net_test_case(secure_tunneling_fail_ws_in_another_thread_test)
add_net_test_case(secure_tunneling_store_service_ids_test)
add_net_test_case(secure_tunneling_receive_stream_start_test)
add_net_test_case(secure_tunneling_rejected_service_id_stream_start_test)
Expand Down
107 changes: 93 additions & 14 deletions tests/secure_tunnel_tests.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,9 @@ struct aws_secure_tunnel_mock_test_fixture {
enum aws_secure_tunnel_message_type type;
int error_code;
} on_send_message_complete_result;

struct aws_thread host_resolver_thread;
bool host_resolver_thread_executed;
};

static bool s_secure_tunnel_check_active_stream_id(
Expand Down Expand Up @@ -383,6 +386,18 @@ static void s_wait_for_connected_successfully(struct aws_secure_tunnel_mock_test
aws_mutex_unlock(&test_fixture->lock);
}

static bool s_has_secure_tunnel_connection_failed(void *arg) {
struct aws_secure_tunnel_mock_test_fixture *test_fixture = arg;
return test_fixture->secure_tunnel_connection_failed;
}

static void s_wait_for_connection_failed(struct aws_secure_tunnel_mock_test_fixture *test_fixture) {
aws_mutex_lock(&test_fixture->lock);
aws_condition_variable_wait_pred(
&test_fixture->signal, &test_fixture->lock, s_has_secure_tunnel_connection_failed, test_fixture);
aws_mutex_unlock(&test_fixture->lock);
}

static bool s_has_secure_tunnel_connection_shutdown(void *arg) {
struct aws_secure_tunnel_mock_test_fixture *test_fixture = arg;
return test_fixture->secure_tunnel_connection_shutdown;
Expand Down Expand Up @@ -602,13 +617,6 @@ int aws_websocket_client_connect_mock_fn(const struct aws_websocket_client_conne
struct aws_secure_tunnel *secure_tunnel = options->user_data;
struct aws_secure_tunnel_mock_test_fixture *test_fixture = secure_tunnel->config->user_data;

if (!options->handshake_request) {
AWS_LOGF_ERROR(
AWS_LS_HTTP_WEBSOCKET_SETUP,
"id=static: Invalid connection options, missing required request for websocket client handshake.");
return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
}

const struct aws_http_headers *request_headers = aws_http_message_get_headers(options->handshake_request);
if (test_fixture->header_check) {
ASSERT_SUCCESS(test_fixture->header_check(request_headers, test_fixture));
Expand Down Expand Up @@ -987,13 +995,6 @@ int aws_websocket_client_connect_fail_once_fn(const struct aws_websocket_client_
aws_mutex_unlock(&test_fixture->lock);

if (is_connection_failed_once) {
if (!options->handshake_request) {
AWS_LOGF_ERROR(
AWS_LS_HTTP_WEBSOCKET_SETUP,
"id=static: Invalid connection options, missing required request for websocket client handshake.");
return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
}

const struct aws_http_headers *request_headers = aws_http_message_get_headers(options->handshake_request);
if (test_fixture->header_check) {
ASSERT_SUCCESS(test_fixture->header_check(request_headers, test_fixture));
Expand Down Expand Up @@ -1030,6 +1031,49 @@ int aws_websocket_client_connect_fail_once_fn(const struct aws_websocket_client_
}
}

static void s_host_resolver_thread(void *arg) {
struct aws_secure_tunnel_mock_test_fixture *test_fixture = arg;
struct aws_websocket_on_connection_setup_data websocket_setup = {.error_code = AWS_ERROR_HTTP_UNKNOWN};
(test_fixture->websocket_function_table->on_connection_setup_fn)(&websocket_setup, test_fixture->secure_tunnel);
test_fixture->host_resolver_thread_executed = true;
}

int aws_websocket_client_connect_fail_in_another_thread_fn(
const struct aws_websocket_client_connection_options *options) {

struct aws_secure_tunnel *secure_tunnel = options->user_data;
struct aws_secure_tunnel_mock_test_fixture *test_fixture = secure_tunnel->config->user_data;

test_fixture->host_resolver_thread_executed = false;

const struct aws_http_headers *request_headers = aws_http_message_get_headers(options->handshake_request);
if (test_fixture->header_check) {
ASSERT_SUCCESS(test_fixture->header_check(request_headers, test_fixture));
}

test_fixture->websocket_function_table->on_connection_setup_fn = options->on_connection_setup;
test_fixture->websocket_function_table->on_connection_shutdown_fn = options->on_connection_shutdown;
test_fixture->websocket_function_table->on_incoming_frame_begin_fn = options->on_incoming_frame_begin;
test_fixture->websocket_function_table->on_incoming_frame_payload_fn = options->on_incoming_frame_payload;
test_fixture->websocket_function_table->on_incoming_frame_complete_fn = options->on_incoming_frame_complete;

if (aws_thread_init(&test_fixture->host_resolver_thread, test_fixture->allocator)) {
AWS_LOGF_ERROR(AWS_LS_HTTP_WEBSOCKET_SETUP, "id=static: Failed to initialize thread.");
return aws_raise_error(AWS_ERROR_HTTP_UNKNOWN);
}

struct aws_thread_options thread_options = *aws_default_thread_options();
thread_options.name = aws_byte_cursor_from_c_str("HostResolver");

if (aws_thread_launch(&test_fixture->host_resolver_thread, s_host_resolver_thread, test_fixture, &thread_options) !=
AWS_OP_SUCCESS) {
AWS_LOGF_ERROR(AWS_LS_HTTP_WEBSOCKET_SETUP, "id=static: Failed to launch thread.");
return aws_raise_error(AWS_ERROR_HTTP_UNKNOWN);
}

return AWS_OP_SUCCESS;
}

static int s_secure_tunneling_fail_and_retry_connection_test_fn(struct aws_allocator *allocator, void *ctx) {
(void)ctx;
struct secure_tunnel_test_options test_options;
Expand Down Expand Up @@ -1057,6 +1101,41 @@ static int s_secure_tunneling_fail_and_retry_connection_test_fn(struct aws_alloc

AWS_TEST_CASE(secure_tunneling_fail_and_retry_connection_test, s_secure_tunneling_fail_and_retry_connection_test_fn)

/* Check the case when WebSocket connection fails and a WebSocket setup callback is called from another thread. This
* situation can happen on host resolution failure.
* NOTE: This test is supposed to be verified by thread sanitizer. */
static int s_secure_tunneling_fail_ws_in_another_thread_test_fn(struct aws_allocator *allocator, void *ctx) {
(void)ctx;
struct secure_tunnel_test_options test_options;
struct aws_secure_tunnel_mock_test_fixture test_fixture;
aws_secure_tunnel_mock_test_init(allocator, &test_options, &test_fixture, AWS_SECURE_TUNNELING_DESTINATION_MODE);
struct aws_secure_tunnel *secure_tunnel = test_fixture.secure_tunnel;

test_fixture.secure_tunnel_vtable = *aws_secure_tunnel_get_default_vtable();
test_fixture.secure_tunnel_vtable.aws_websocket_client_connect_fn =
aws_websocket_client_connect_fail_in_another_thread_fn;
test_fixture.secure_tunnel_vtable.aws_websocket_send_frame_fn = aws_websocket_send_frame_mock_fn;
test_fixture.secure_tunnel_vtable.aws_websocket_release_fn = aws_websocket_release_mock_fn;
test_fixture.secure_tunnel_vtable.aws_websocket_close_fn = aws_websocket_close_mock_fn;
test_fixture.secure_tunnel_vtable.vtable_user_data = &test_fixture;

test_fixture.secure_tunnel_connection_failed = false;
ASSERT_SUCCESS(aws_secure_tunnel_start(secure_tunnel));
s_wait_for_connection_failed(&test_fixture);
ASSERT_SUCCESS(aws_secure_tunnel_stop(secure_tunnel));

ASSERT_SUCCESS(aws_thread_join(&test_fixture.host_resolver_thread));
aws_thread_clean_up(&test_fixture.host_resolver_thread);

ASSERT_TRUE(test_fixture.host_resolver_thread_executed);

aws_secure_tunnel_mock_test_clean_up(&test_fixture);

return AWS_OP_SUCCESS;
}

AWS_TEST_CASE(secure_tunneling_fail_ws_in_another_thread_test, s_secure_tunneling_fail_ws_in_another_thread_test_fn)

static int s_secure_tunneling_store_service_ids_test_fn(struct aws_allocator *allocator, void *ctx) {
(void)ctx;
struct secure_tunnel_test_options test_options;
Expand Down