Adopt optimized isolated path for SelectableEL

IntegrationTests/tests_04_performance/Thresholds/5.10.json

@@ -20,7 +20,7 @@
     "1000_udpbootstraps": 2050,
     "1000_udpconnections": 75050,
     "1_reqs_1000_conn": 384050,
-    "assume_isolated_scheduling_10000_executions": 20100,
+    "assume_isolated_scheduling_10000_executions": 89,
     "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
     "decode_1000_ws_frames": 2050,
@@ -34,7 +34,7 @@
     "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
     "flat_schedule_10000_tasks": 130100,
-    "flat_schedule_assume_isolated_10000_tasks": 110100,
+    "flat_schedule_assume_isolated_10000_tasks": 100100,
     "future_assume_isolated_lots_of_callbacks": 74050,
     "future_erase_result": 4050,
     "future_lots_of_callbacks": 74050,
@@ -48,12 +48,12 @@
     "read_10000_chunks_from_file": 110050,
     "schedule_10000_tasks": 40100,
     "schedule_and_run_10000_tasks": 50050,
-    "schedule_assume_isolated_10000_tasks": 50100,
+    "schedule_assume_isolated_10000_tasks": 40100,
     "schedule_with_deadline_10000_tasks": 40100,
-    "schedule_with_deadline_assume_isolated_10000_tasks": 50100,
+    "schedule_with_deadline_assume_isolated_10000_tasks": 40100,
     "scheduling_10000_executions": 89,
     "submit_10000_tasks": 20100,
-    "submit_assume_isolated_10000_tasks": 30100,
+    "submit_assume_isolated_10000_tasks": 20100,
     "udp_1000_reqs_1_conn": 6200,
     "udp_1_reqs_1000_conn": 162050
 }

IntegrationTests/tests_04_performance/Thresholds/5.9.json

@@ -20,7 +20,7 @@
     "1000_udpbootstraps": 2050,
     "1000_udpconnections": 75050,
     "1_reqs_1000_conn": 393050,
-    "assume_isolated_scheduling_10000_executions": 20100,
+    "assume_isolated_scheduling_10000_executions": 89,
     "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
     "decode_1000_ws_frames": 2050,
@@ -34,7 +34,7 @@
     "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
     "flat_schedule_10000_tasks": 130100,
-    "flat_schedule_assume_isolated_10000_tasks": 110100,
+    "flat_schedule_assume_isolated_10000_tasks": 100100,
     "future_assume_isolated_lots_of_callbacks": 74050,
     "future_erase_result": 4050,
     "future_lots_of_callbacks": 74050,
@@ -48,12 +48,12 @@
     "read_10000_chunks_from_file": 110050,
     "schedule_10000_tasks": 40100,
     "schedule_and_run_10000_tasks": 50050,
-    "schedule_assume_isolated_10000_tasks": 50100,
+    "schedule_assume_isolated_10000_tasks": 40100,
     "schedule_with_deadline_10000_tasks": 40100,
-    "schedule_with_deadline_assume_isolated_10000_tasks": 50100,
+    "schedule_with_deadline_assume_isolated_10000_tasks": 40100,
     "scheduling_10000_executions": 89,
     "submit_10000_tasks": 20100,
-    "submit_assume_isolated_10000_tasks": 30100,
+    "submit_assume_isolated_10000_tasks": 20100,
     "udp_1000_reqs_1_conn": 6200,
     "udp_1_reqs_1000_conn": 162050
 }

IntegrationTests/tests_04_performance/Thresholds/6.0.json

@@ -20,7 +20,7 @@
     "1000_udpbootstraps": 2050,
     "1000_udpconnections": 75050,
     "1_reqs_1000_conn": 384050,
-    "assume_isolated_scheduling_10000_executions": 20100,
+    "assume_isolated_scheduling_10000_executions": 89,
     "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
     "decode_1000_ws_frames": 2050,
@@ -34,7 +34,7 @@
     "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
     "flat_schedule_10000_tasks": 130100,
-    "flat_schedule_assume_isolated_10000_tasks": 110100,
+    "flat_schedule_assume_isolated_10000_tasks": 100100,
     "future_assume_isolated_lots_of_callbacks": 74050,
     "future_erase_result": 4050,
     "future_lots_of_callbacks": 74050,
@@ -48,12 +48,12 @@
     "read_10000_chunks_from_file": 110050,
     "schedule_10000_tasks": 40100,
     "schedule_and_run_10000_tasks": 50050,
-    "schedule_assume_isolated_10000_tasks": 50100,
+    "schedule_assume_isolated_10000_tasks": 40100,
     "schedule_with_deadline_10000_tasks": 40100,
-    "schedule_with_deadline_assume_isolated_10000_tasks": 50100,
+    "schedule_with_deadline_assume_isolated_10000_tasks": 40100,
     "scheduling_10000_executions": 89,
     "submit_10000_tasks": 20100,
-    "submit_assume_isolated_10000_tasks": 30100,
+    "submit_assume_isolated_10000_tasks": 20100,
     "udp_1000_reqs_1_conn": 6200,
     "udp_1_reqs_1000_conn": 162050
 }

IntegrationTests/tests_04_performance/Thresholds/nightly-6.0.json

@@ -20,7 +20,7 @@
     "1000_udpbootstraps": 2050,
     "1000_udpconnections": 75050,
     "1_reqs_1000_conn": 384050,
-    "assume_isolated_scheduling_10000_executions": 20100,
+    "assume_isolated_scheduling_10000_executions": 89,
     "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
     "decode_1000_ws_frames": 2050,
@@ -34,7 +34,7 @@
     "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
     "flat_schedule_10000_tasks": 130100,
-    "flat_schedule_assume_isolated_10000_tasks": 110100,
+    "flat_schedule_assume_isolated_10000_tasks": 100100,
     "future_assume_isolated_lots_of_callbacks": 74050,
     "future_erase_result": 4050,
     "future_lots_of_callbacks": 74050,
@@ -48,12 +48,12 @@
     "read_10000_chunks_from_file": 110050,
     "schedule_10000_tasks": 40100,
     "schedule_and_run_10000_tasks": 50050,
-    "schedule_assume_isolated_10000_tasks": 50100,
+    "schedule_assume_isolated_10000_tasks": 40100,
     "schedule_with_deadline_10000_tasks": 40100,
-    "schedule_with_deadline_assume_isolated_10000_tasks": 50100,
+    "schedule_with_deadline_assume_isolated_10000_tasks": 40100,
     "scheduling_10000_executions": 89,
     "submit_10000_tasks": 20100,
-    "submit_assume_isolated_10000_tasks": 30100,
+    "submit_assume_isolated_10000_tasks": 20100,
     "udp_1000_reqs_1_conn": 6200,
     "udp_1_reqs_1000_conn": 162050
 }

IntegrationTests/tests_04_performance/Thresholds/nightly-main.json

@@ -20,7 +20,7 @@
     "1000_udpbootstraps": 2050,
     "1000_udpconnections": 75050,
     "1_reqs_1000_conn": 384050,
-    "assume_isolated_scheduling_10000_executions": 20100,
+    "assume_isolated_scheduling_10000_executions": 89,
     "bytebuffer_lots_of_rw": 2050,
     "creating_10000_headers": 0,
     "decode_1000_ws_frames": 2050,
@@ -34,7 +34,7 @@
     "encode_1000_ws_frames_new_buffer_with_space_with_mask": 5050,
     "execute_hop_10000_tasks": 0,
     "flat_schedule_10000_tasks": 130100,
-    "flat_schedule_assume_isolated_10000_tasks": 110100,
+    "flat_schedule_assume_isolated_10000_tasks": 100100,
     "future_assume_isolated_lots_of_callbacks": 74050,
     "future_erase_result": 4050,
     "future_lots_of_callbacks": 74050,
@@ -48,12 +48,12 @@
     "read_10000_chunks_from_file": 110050,
     "schedule_10000_tasks": 40100,
     "schedule_and_run_10000_tasks": 50050,
-    "schedule_assume_isolated_10000_tasks": 50100,
+    "schedule_assume_isolated_10000_tasks": 40100,
     "schedule_with_deadline_10000_tasks": 40100,
-    "schedule_with_deadline_assume_isolated_10000_tasks": 50100,
+    "schedule_with_deadline_assume_isolated_10000_tasks": 40100,
     "scheduling_10000_executions": 89,
     "submit_10000_tasks": 20100,
-    "submit_assume_isolated_10000_tasks": 30100,
+    "submit_assume_isolated_10000_tasks": 20100,
     "udp_1000_reqs_1_conn": 6200,
     "udp_1_reqs_1000_conn": 162050
 }

Sources/NIOPosix/SelectableEventLoop.swift

@@ -308,6 +308,29 @@ internal final class SelectableEventLoop: EventLoop, @unchecked Sendable {
     @inlinable
     internal func scheduleTask<T>(deadline: NIODeadline, _ task: @escaping () throws -> T) -> Scheduled<T> {
         let promise: EventLoopPromise<T> = self.makePromise()
+        let (task, scheduled) = self._prepareToSchedule(deadline: deadline, promise: promise, task: task)
+
+        do {
+            try self._schedule0(.scheduled(task))
+        } catch {
+            promise.fail(error)
+        }
+
+        return scheduled
+    }
+
+    /// - see: `EventLoop.scheduleTask(in:_:)`
+    @inlinable
+    internal func scheduleTask<T>(in: TimeAmount, _ task: @escaping () throws -> T) -> Scheduled<T> {
+        scheduleTask(deadline: .now() + `in`, task)
+    }
+
+    @inlinable
+    func _prepareToSchedule<T>(
+        deadline: NIODeadline,
+        promise: EventLoopPromise<T>,
+        task: @escaping () throws -> T
+    ) -> (ScheduledTask, Scheduled<T>) {
         let task = ScheduledTask(
             id: self.scheduledTaskCounter.loadThenWrappingIncrement(ordering: .relaxed),
             {
@@ -337,20 +360,56 @@ internal final class SelectableEventLoop: EventLoop, @unchecked Sendable {
                 // one wakeup.
             }
         )
+        return (task, scheduled)
+    }
+
+    @inlinable
+    func _executeIsolatedUnsafeUnchecked(_ task: @escaping () -> Void) {
+        // nothing we can do if we fail enqueuing here.
+        try? self._scheduleIsolated0(.immediate(.function(task)))
+    }
+
+    @inlinable
+    func _submitIsolatedUnsafeUnchecked<T>(_ task: @escaping () throws -> T) -> EventLoopFuture<T> {
+        let promise = self.makePromise(of: T.self)
+
+        self._executeIsolatedUnsafeUnchecked {
+            do {
+                // UnsafeUnchecked is allowed here because we know we are on the EL.
+                promise.assumeIsolatedUnsafeUnchecked().succeed(try task())
+            } catch let err {
+                promise.fail(err)
+            }
+        }
+
+        return promise.futureResult
+    }
+
+    @inlinable
+    @discardableResult
+    func _scheduleTaskIsolatedUnsafeUnchecked<T>(
+        deadline: NIODeadline,
+        _ task: @escaping () throws -> T
+    ) -> Scheduled<T> {
+        let promise: EventLoopPromise<T> = self.makePromise()
+        let (task, scheduled) = self._prepareToSchedule(deadline: deadline, promise: promise, task: task)
 
         do {
-            try self._schedule0(.scheduled(task))
+            try self._scheduleIsolated0(.scheduled(task))
         } catch {
             promise.fail(error)
         }
 
         return scheduled
     }
 
-    /// - see: `EventLoop.scheduleTask(in:_:)`
     @inlinable
-    internal func scheduleTask<T>(in: TimeAmount, _ task: @escaping () throws -> T) -> Scheduled<T> {
-        scheduleTask(deadline: .now() + `in`, task)
+    @discardableResult
+    func _scheduleTaskIsolatedUnsafeUnchecked<T>(
+        in delay: TimeAmount,
+        _ task: @escaping () throws -> T
+    ) -> Scheduled<T> {
+        self._scheduleTaskIsolatedUnsafeUnchecked(deadline: .now() + delay, task)
     }
 
     // - see: `EventLoop.execute`
@@ -372,19 +431,7 @@ internal final class SelectableEventLoop: EventLoop, @unchecked Sendable {
     @usableFromInline
     internal func _schedule0(_ task: LoopTask) throws {
         if self.inEventLoop {
-            precondition(
-                self._validInternalStateToScheduleTasks,
-                "BUG IN NIO (please report): EventLoop is shutdown, yet we're on the EventLoop."
-            )
-
-            self._tasksLock.withLock { () -> Void in
-                switch task {
-                case .scheduled(let task):
-                    self._scheduledTasks.push(task)
-                case .immediate(let task):
-                    self._immediateTasks.append(task)
-                }
-            }
+            try self._scheduleIsolated0(task)
         } else {
             let shouldWakeSelector: Bool = self.externalStateLock.withLock {
                 guard self.validExternalStateToScheduleTasks else {
@@ -434,6 +481,25 @@ internal final class SelectableEventLoop: EventLoop, @unchecked Sendable {
         }
     }
 
+    /// Add the `ScheduledTask` to be executed.
+    @usableFromInline
+    internal func _scheduleIsolated0(_ task: LoopTask) throws {
+        self.assertInEventLoop()
+        precondition(
+            self._validInternalStateToScheduleTasks,
+            "BUG IN NIO (please report): EventLoop is shutdown, yet we're on the EventLoop."
+        )
+
+        self._tasksLock.withLock { () -> Void in
+            switch task {
+            case .scheduled(let task):
+                self._scheduledTasks.push(task)
+            case .immediate(let task):
+                self._immediateTasks.append(task)
+            }
+        }
+    }
+
     /// Wake the `Selector` which means `Selector.whenReady(...)` will unblock.
     @usableFromInline
     internal func _wakeupSelector() throws {