diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b24b6cd7..6bd58e84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,7 +161,7 @@ jobs: python3 builder.pyz build -p aws-c-http --cmake-extra=-DENABLE_LOCALHOST_INTEGRATION_TESTS=ON --config Debug localhost-test-mac: - runs-on: macos-11 # latest + runs-on: macos-13 # latest steps: - name: Checkout uses: actions/checkout@v3 diff --git a/include/aws/http/connection_manager.h b/include/aws/http/connection_manager.h index 70b1e77b..c33ebbd9 100644 --- a/include/aws/http/connection_manager.h +++ b/include/aws/http/connection_manager.h @@ -124,6 +124,19 @@ struct aws_http_connection_manager_options { * timeout will be closed automatically. */ uint64_t max_connection_idle_in_milliseconds; + + /** + * (Optional) + * An array of network interface names. The manager will distribute the + * connections across network interface names provided in this array. If any interface name is invalid, goes down, + * or has any issues like network access, you will see connection failures. If + * `socket_options.network_interface_name` is also set, an `AWS_ERROR_INVALID_ARGUMENT` error will be raised. + * + * This option is only supported on Linux, MacOS, and platforms that have either SO_BINDTODEVICE or IP_BOUND_IF. It + * is not supported on Windows. `AWS_ERROR_PLATFORM_NOT_SUPPORTED` will be raised on unsupported platforms. + */ + const struct aws_byte_cursor *network_interface_names_array; + size_t num_network_interface_names; }; AWS_EXTERN_C_BEGIN diff --git a/source/connection_manager.c b/source/connection_manager.c index 195a7d6c..5fb3199e 100644 --- a/source/connection_manager.c +++ b/source/connection_manager.c @@ -292,6 +292,17 @@ struct aws_http_connection_manager { */ struct aws_task *cull_task; struct aws_event_loop *cull_event_loop; + + /* + * An aws_array_list of network interface names to distribute the connections using the + * round-robin algorithm. We picked round-robin because it is trivial to implement and good enough. We can later + * update to a more complex distribution algorithm if required. + */ + struct aws_array_list network_interface_names; + /* + * Current index in the network_interface_names array_list. + */ + size_t network_interface_names_index; }; struct aws_http_connection_manager_snapshot { @@ -703,6 +714,13 @@ static void s_aws_http_connection_manager_finish_destroy(struct aws_http_connect aws_http_proxy_config_destroy(manager->proxy_config); } + for (size_t i = 0; i < aws_array_list_length(&manager->network_interface_names); i++) { + struct aws_string *interface_name = NULL; + aws_array_list_get_at(&manager->network_interface_names, &interface_name, i); + aws_string_destroy(interface_name); + } + aws_array_list_clean_up(&manager->network_interface_names); + /* * If this task exists then we are actually in the corresponding event loop running the final destruction task. * In that case, we've already cancelled this task and when you cancel, it runs synchronously. So in that @@ -819,6 +837,15 @@ struct aws_http_connection_manager *aws_http_connection_manager_new( return NULL; } + if (options->socket_options->network_interface_name[0] != '\0' && options->num_network_interface_names > 0) { + AWS_LOGF_ERROR( + AWS_LS_HTTP_CONNECTION_MANAGER, + "Invalid options - socket_options.network_interface_name and network_interface_names_array cannot be both " + "set."); + aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + return NULL; + } + struct aws_http_connection_manager *manager = aws_mem_calloc(allocator, 1, sizeof(struct aws_http_connection_manager)); if (manager == NULL) { @@ -896,6 +923,20 @@ struct aws_http_connection_manager *aws_http_connection_manager_new( manager->max_closed_streams = options->max_closed_streams; manager->http2_conn_manual_window_management = options->http2_conn_manual_window_management; + manager->network_interface_names_index = 0; + if (options->num_network_interface_names > 0) { + aws_array_list_init_dynamic( + &manager->network_interface_names, + allocator, + options->num_network_interface_names, + sizeof(struct aws_string *)); + for (size_t i = 0; i < options->num_network_interface_names; i++) { + struct aws_byte_cursor interface_name = options->network_interface_names_array[i]; + struct aws_string *interface_name_str = aws_string_new_from_cursor(allocator, &interface_name); + aws_array_list_push_back(&manager->network_interface_names, &interface_name_str); + } + } + /* NOTHING can fail after here */ s_schedule_connection_culling(manager); @@ -990,7 +1031,26 @@ static int s_aws_http_connection_manager_new_connection(struct aws_http_connecti options.host_name = aws_byte_cursor_from_string(manager->host); options.port = manager->port; options.initial_window_size = manager->initial_window_size; - options.socket_options = &manager->socket_options; + struct aws_socket_options socket_options = manager->socket_options; + if (aws_array_list_length(&manager->network_interface_names)) { + struct aws_string *interface_name = NULL; + aws_array_list_get_at( + &manager->network_interface_names, &interface_name, manager->network_interface_names_index); + manager->network_interface_names_index = + (manager->network_interface_names_index + 1) % aws_array_list_length(&manager->network_interface_names); +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4996) /* allow strncpy() */ +#endif + /* If the interface_name is too long or not null terminated, it will be caught in the `aws_socket_init` function + * so we don't need to worry about that here.*/ + strncpy( + socket_options.network_interface_name, aws_string_c_str(interface_name), AWS_NETWORK_INTERFACE_NAME_MAX); +#if defined(_MSC_VER) +# pragma warning(pop) +#endif + } + options.socket_options = &socket_options; options.on_setup = s_aws_http_connection_manager_on_connection_setup; options.on_shutdown = s_aws_http_connection_manager_on_connection_shutdown; options.manual_window_management = manager->enable_read_back_pressure; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9422fc80..6a0a4de1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -525,6 +525,7 @@ add_net_test_case(test_connection_manager_idle_culling_single) add_net_test_case(test_connection_manager_idle_culling_many) add_net_test_case(test_connection_manager_idle_culling_mixture) add_net_test_case(test_connection_manager_idle_culling_refcount) +add_net_test_case(test_connection_manager_with_network_interface_list) # tests where we establish real connections add_net_test_case(test_connection_manager_single_connection) diff --git a/tests/test_connection_manager.c b/tests/test_connection_manager.c index 616939e9..5e394b9d 100644 --- a/tests/test_connection_manager.c +++ b/tests/test_connection_manager.c @@ -58,6 +58,8 @@ struct cm_tester_options { struct aws_http2_setting *initial_settings_array; size_t num_initial_settings; bool self_lib_init; + const struct aws_byte_cursor *verify_network_interface_names_array; + size_t num_network_interface_names; }; struct cm_tester { @@ -73,6 +75,8 @@ struct cm_tester { struct aws_tls_ctx_options tls_ctx_options; struct aws_tls_connection_options tls_connection_options; struct aws_http_proxy_options *verify_proxy_options; + const struct aws_byte_cursor *verify_network_interface_names_array; + size_t num_network_interface_names; struct aws_mutex lock; struct aws_condition_variable signal; @@ -219,6 +223,8 @@ static int s_cm_tester_init(struct cm_tester_options *options) { .http2_prior_knowledge = !options->use_tls && options->http2, .initial_settings_array = options->initial_settings_array, .num_initial_settings = options->num_initial_settings, + .network_interface_names_array = options->verify_network_interface_names_array, + .num_network_interface_names = options->num_network_interface_names, }; if (options->mock_table) { @@ -234,6 +240,8 @@ static int s_cm_tester_init(struct cm_tester_options *options) { } tester->mock_table = options->mock_table; + tester->verify_network_interface_names_array = options->verify_network_interface_names_array; + tester->num_network_interface_names = options->num_network_interface_names; aws_atomic_store_int(&tester->next_connection_id, 0); @@ -729,6 +737,13 @@ static int s_aws_http_connection_manager_create_connection_sync_mock( const struct aws_http_client_connection_options *options) { struct cm_tester *tester = &s_tester; + if (tester->num_network_interface_names) { + struct aws_byte_cursor interface_name = + tester->verify_network_interface_names_array + [aws_atomic_load_int(&tester->next_connection_id) % tester->num_network_interface_names]; + ASSERT_TRUE(aws_byte_cursor_eq_c_str(&interface_name, options->socket_options->network_interface_name)); + } + size_t next_connection_id = aws_atomic_fetch_add(&tester->next_connection_id, 1); ASSERT_SUCCESS(aws_mutex_lock(&tester->lock)); @@ -819,6 +834,39 @@ static struct aws_http_connection_manager_system_vtable s_synchronous_mocks = { .aws_http_connection_get_version = s_aws_http_connection_manager_connection_get_version_sync_mock, }; +static int s_test_connection_manager_with_network_interface_list(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_byte_cursor *interface_names_array = aws_mem_calloc(allocator, 3, sizeof(struct aws_byte_cursor)); + interface_names_array[0] = aws_byte_cursor_from_c_str("ens32"); + interface_names_array[1] = aws_byte_cursor_from_c_str("ens64"); + interface_names_array[2] = aws_byte_cursor_from_c_str("ens96"); + + struct cm_tester_options options = { + .allocator = allocator, + .max_connections = 20, + .mock_table = &s_synchronous_mocks, + .verify_network_interface_names_array = interface_names_array, + }; + + ASSERT_SUCCESS(s_cm_tester_init(&options)); + size_t num_connections = 6; + for (size_t i = 0; i < num_connections; ++i) { + s_add_mock_connections(1, AWS_NCRT_SUCCESS, i % 1 == 0); + } + s_acquire_connections(num_connections); + + ASSERT_SUCCESS(s_wait_on_connection_reply_count(num_connections)); + ASSERT_SUCCESS(s_release_connections(num_connections, false)); + ASSERT_UINT_EQUALS(0, s_tester.connection_errors); + + ASSERT_SUCCESS(s_cm_tester_clean_up()); + aws_mem_release(allocator, interface_names_array); + return AWS_OP_SUCCESS; +} +AWS_TEST_CASE( + test_connection_manager_with_network_interface_list, + s_test_connection_manager_with_network_interface_list); + static int s_test_connection_manager_acquire_release_mix_synchronous(struct aws_allocator *allocator, void *ctx) { (void)ctx;