From d066c0cd110a1cb1ad2a68fa6d91e3e4e8e294ba Mon Sep 17 00:00:00 2001 From: Matej Knopp Date: Wed, 25 Sep 2024 22:05:56 +0200 Subject: [PATCH] feat(macOS): preliminary support for writing tools (#441) --- super_context_menu/lib/src/desktop.dart | 13 + super_context_menu/lib/src/menu.dart | 18 ++ super_native_extensions/lib/src/menu.dart | 22 ++ .../lib/src/native/menu.dart | 6 + super_native_extensions/rust/Cargo.toml | 1 + super_native_extensions/rust/src/api_model.rs | 15 ++ .../rust/src/darwin/macos/drag.rs | 13 +- .../rust/src/darwin/macos/menu.rs | 241 ++++++++++++++++-- .../rust/src/menu_manager.rs | 19 +- 9 files changed, 318 insertions(+), 30 deletions(-) diff --git a/super_context_menu/lib/src/desktop.dart b/super_context_menu/lib/src/desktop.dart index a414f8d8..ddd9b424 100644 --- a/super_context_menu/lib/src/desktop.dart +++ b/super_context_menu/lib/src/desktop.dart @@ -143,6 +143,7 @@ class DesktopContextMenuWidget extends StatelessWidget { required this.menuProvider, required this.contextMenuIsAllowed, required this.menuWidgetBuilder, + this.writingToolsConfigurationProvider, this.iconTheme, }); @@ -156,6 +157,8 @@ class DesktopContextMenuWidget extends StatelessWidget { /// on platform. final IconThemeData? iconTheme; + final WritingToolsConfiguration? Function()? + writingToolsConfigurationProvider; @override Widget build(BuildContext context) { return _ContextMenuDetector( @@ -229,10 +232,20 @@ class DesktopContextMenuWidget extends StatelessWidget { } onMenuResolved(true); onShowMenu.notify(); + final writingToolsConfiguration = + writingToolsConfigurationProvider?.call(); + raw.writingToolsSuggestionCallback = + writingToolsConfiguration?.onSuggestion; + final request = raw.DesktopContextMenuRequest( iconTheme: serializationOptions.iconTheme, position: globalPosition, menu: handle, + writingToolsConfiguration: switch (writingToolsConfiguration) { + (WritingToolsConfiguration c) => + raw.WritingToolsConfiguration(rect: c.rect, text: c.text), + _ => null, + }, fallback: () { final completer = Completer(); ContextMenuSession( diff --git a/super_context_menu/lib/src/menu.dart b/super_context_menu/lib/src/menu.dart index 0027e3c7..21f9745c 100644 --- a/super_context_menu/lib/src/menu.dart +++ b/super_context_menu/lib/src/menu.dart @@ -62,6 +62,7 @@ class ContextMenuWidget extends StatelessWidget { this.contextMenuIsAllowed = _defaultContextMenuIsAllowed, MobileMenuWidgetBuilder? mobileMenuWidgetBuilder, DesktopMenuWidgetBuilder? desktopMenuWidgetBuilder, + this.writingToolsConfigurationProvider, }) : assert(previewBuilder == null || deferredPreviewBuilder == null, 'Cannot use both previewBuilder and deferredPreviewBuilder'), mobileMenuWidgetBuilder = @@ -81,6 +82,9 @@ class ContextMenuWidget extends StatelessWidget { final MobileMenuWidgetBuilder mobileMenuWidgetBuilder; final DesktopMenuWidgetBuilder desktopMenuWidgetBuilder; + final WritingToolsConfiguration? Function()? + writingToolsConfigurationProvider; + /// Base icon theme for menu icons. The size will be overridden depending /// on platform. final IconThemeData? iconTheme; @@ -111,6 +115,8 @@ class ContextMenuWidget extends StatelessWidget { contextMenuIsAllowed: contextMenuIsAllowed, iconTheme: iconTheme, menuWidgetBuilder: desktopMenuWidgetBuilder, + writingToolsConfigurationProvider: + writingToolsConfigurationProvider, child: child!, ); } @@ -120,3 +126,15 @@ class ContextMenuWidget extends StatelessWidget { } bool _defaultContextMenuIsAllowed(Offset location) => true; + +class WritingToolsConfiguration { + WritingToolsConfiguration({ + required this.text, + required this.rect, + required this.onSuggestion, + }); + + final String text; + final Rect rect; + final ValueChanged onSuggestion; +} diff --git a/super_native_extensions/lib/src/menu.dart b/super_native_extensions/lib/src/menu.dart index 996cf9d7..b434c4e5 100644 --- a/super_native_extensions/lib/src/menu.dart +++ b/super_native_extensions/lib/src/menu.dart @@ -10,6 +10,7 @@ import 'mutex.dart'; import 'native/menu.dart' if (dart.library.js_interop) 'web/menu.dart'; import 'menu_flutter.dart'; import 'gesture/pointer_device_kind.dart'; +import 'util.dart'; import 'widget_snapshot/widget_snapshot.dart'; abstract class MobileMenuDelegate { @@ -17,6 +18,8 @@ abstract class MobileMenuDelegate { void hideMenu({required bool itemSelected}); } +void Function(String)? writingToolsSuggestionCallback; + typedef MobileMenuWidgetFactory = Widget Function( BuildContext context, Menu rootMenu, @@ -96,17 +99,36 @@ class MenuSerializationOptions { final double devicePixelRatio; } +class WritingToolsConfiguration { + WritingToolsConfiguration({ + required this.rect, + required this.text, + }); + + final Rect rect; + final String text; + + Map serialize() { + return { + 'rect': rect.serialize(), + 'text': text, + }; + } +} + class DesktopContextMenuRequest { DesktopContextMenuRequest({ required this.menu, required this.position, required this.iconTheme, required this.fallback, + required this.writingToolsConfiguration, }); final MenuHandle menu; final Offset position; final IconThemeData iconTheme; + final WritingToolsConfiguration? writingToolsConfiguration; // Passed to delegate when requesting Flutter desktop menu implementation. final Future Function() fallback; diff --git a/super_native_extensions/lib/src/native/menu.dart b/super_native_extensions/lib/src/native/menu.dart index f4b8bbd0..9f9d574c 100644 --- a/super_native_extensions/lib/src/native/menu.dart +++ b/super_native_extensions/lib/src/native/menu.dart @@ -208,6 +208,9 @@ class MenuContextImpl extends MenuContext { final res = await _channel.invokeMethod('showContextMenu', { 'menuHandle': (request.menu as NativeMenuHandle).handle, 'location': request.position.serialize(), + if (request.writingToolsConfiguration != null) + 'writingToolsConfiguration': + request.writingToolsConfiguration!.serialize(), }) as Map; return MenuResult( itemSelected: res['itemSelected'], @@ -317,6 +320,9 @@ class MenuContextImpl extends MenuContext { } return {'elements': res}; }, () => {'elements': []}); + } else if (call.method == 'sendWritingToolsReplacement') { + final text = call.arguments['text'] as String; + writingToolsSuggestionCallback?.call(text); } else { return null; } diff --git a/super_native_extensions/rust/Cargo.toml b/super_native_extensions/rust/Cargo.toml index 5743c226..f378b79a 100644 --- a/super_native_extensions/rust/Cargo.toml +++ b/super_native_extensions/rust/Cargo.toml @@ -79,6 +79,7 @@ objc2-app-kit = { version = "0.2.2", features = [ "NSEvent", "NSFilePromiseProvider", "NSFilePromiseReceiver", + "NSGraphics", "NSGraphicsContext", "NSImage", "NSImage", diff --git a/super_native_extensions/rust/src/api_model.rs b/super_native_extensions/rust/src/api_model.rs index f274ba18..7d6e7d04 100644 --- a/super_native_extensions/rust/src/api_model.rs +++ b/super_native_extensions/rust/src/api_model.rs @@ -232,15 +232,30 @@ pub struct MenuConfiguration { pub menu: Option>, } +/// macOS only +#[derive(TryFromValue)] +#[irondash(rename_all = "camelCase")] +pub struct WritingToolsConfiguration { + pub rect: Rect, + pub text: String, +} + #[derive(TryFromValue)] #[irondash(rename_all = "camelCase")] pub struct ShowContextMenuRequest { pub menu_handle: i64, pub location: Point, + pub writing_tools_configuration: Option, #[irondash(skip)] pub menu: Option>, } +#[derive(IntoValue)] +#[irondash(rename_all = "camelCase")] +pub struct WritingToolsReplacementRequest { + pub text: String, +} + #[derive(IntoValue)] #[irondash(rename_all = "camelCase")] pub struct ShowContextMenuResponse { diff --git a/super_native_extensions/rust/src/darwin/macos/drag.rs b/super_native_extensions/rust/src/darwin/macos/drag.rs index e775294b..650b90aa 100644 --- a/super_native_extensions/rust/src/darwin/macos/drag.rs +++ b/super_native_extensions/rust/src/darwin/macos/drag.rs @@ -131,18 +131,18 @@ impl PlatformDragContext { } } - pub unsafe fn synthesize_mouse_up_event(&self) { + pub unsafe fn synthesize_mouse_up_event(&self) -> Option> { self.finish_scroll_events(); - if let Some(event) = self.last_mouse_down_event.borrow().as_ref().cloned() { + if let Some(original_event) = self.last_mouse_down_event.borrow().as_ref().cloned() { #[allow(non_upper_case_globals)] - let opposite = match event.r#type() { + let opposite = match original_event.r#type() { NSEventType::LeftMouseDown => CGEventType::LeftMouseUp, NSEventType::RightMouseDown => CGEventType::RightMouseUp, - _ => return, + _ => return None, }; - let event = event.CGEvent(); + let event = original_event.CGEvent(); let event = CGEventCreateCopy(event); CGEventSetType(event, opposite); @@ -153,6 +153,9 @@ impl PlatformDragContext { if let Some(window) = window { window.sendEvent(&synthesized); } + Some(original_event.clone()) + } else { + None } } diff --git a/super_native_extensions/rust/src/darwin/macos/menu.rs b/super_native_extensions/rust/src/darwin/macos/menu.rs index 21f24ddb..3293e213 100644 --- a/super_native_extensions/rust/src/darwin/macos/menu.rs +++ b/super_native_extensions/rust/src/darwin/macos/menu.rs @@ -1,37 +1,49 @@ -use std::rc::{Rc, Weak}; +use std::{ + cell::{Cell, RefCell}, + rc::{Rc, Weak}, +}; use block2::{Block, RcBlock}; use irondash_engine_context::EngineContext; -use irondash_message_channel::IsolateId; +use irondash_message_channel::{IsolateId, Late}; use irondash_run_loop::{spawn, util::FutureCompleter, RunLoop}; use objc2::{ - extern_class, extern_methods, - mutability::MainThreadOnly, + declare_class, extern_class, extern_methods, msg_send_id, + mutability::{self, MainThreadOnly}, rc::{Allocated, Id}, - ClassType, + ClassType, DeclaredClass, +}; +use objc2_app_kit::{ + NSEvent, NSEventModifierFlags, NSEventType, NSMenu, NSMenuItem, NSPasteboard, NSPasteboardType, + NSPasteboardTypeString, NSServicesMenuRequestor, NSView, +}; +use objc2_foundation::{ + ns_string, MainThreadMarker, NSArray, NSObjectProtocol, NSPoint, NSRect, NSString, NSUInteger, }; -use objc2_app_kit::{NSEvent, NSEventModifierFlags, NSEventType, NSMenu, NSMenuItem, NSView}; -use objc2_foundation::{ns_string, MainThreadMarker, NSPoint, NSString, NSUInteger}; use crate::{ api_model::{ Activator, ImageData, Menu, MenuElement, MenuImage, ShowContextMenuRequest, - ShowContextMenuResponse, + ShowContextMenuResponse, WritingToolsReplacementRequest, }, error::NativeExtensionsResult, log::OkLog, menu_manager::{PlatformMenuContextDelegate, PlatformMenuContextId, PlatformMenuDelegate}, }; -use super::util::{flip_position, ns_image_for_menu_item}; +use super::util::{flip_position, flip_rect, ns_image_for_menu_item}; pub struct PlatformMenuContext { delegate: Weak, + context_id: PlatformMenuContextId, view: Id, + context_menu_view: Late>, + writing_tool_text: RefCell>, } pub struct PlatformMenu { menu: Id, + selected: Rc>, } impl std::fmt::Debug for PlatformMenu { @@ -103,6 +115,7 @@ impl PlatformMenu { isolate: IsolateId, delegate: Weak, main_thread_marker: MainThreadMarker, + selected: Rc>, ) -> Id { let title = menu.title.as_deref().unwrap_or_default(); let res = SNEMenu::initWithTitle( @@ -110,8 +123,13 @@ impl PlatformMenu { &NSString::from_str(title), ); for child in &menu.children { - let child = - Self::translate_element(child, isolate, delegate.clone(), main_thread_marker); + let child = Self::translate_element( + child, + isolate, + delegate.clone(), + main_thread_marker, + selected.clone(), + ); res.addItem(&child); } Id::into_super(res) @@ -123,6 +141,7 @@ impl PlatformMenu { isolate: IsolateId, weak_delegate: Weak, main_thread_marker: MainThreadMarker, + selected: Rc>, ) { if let Some(delegate) = weak_delegate.upgrade() { let parent_menu = item.menu(); @@ -137,6 +156,7 @@ impl PlatformMenu { isolate, weak_delegate.clone(), main_thread_marker, + selected.clone(), ); let index = parent_menu.indexOfItem(item); parent_menu.insertItem_atIndex(&element, index); @@ -150,6 +170,7 @@ impl PlatformMenu { isolate: IsolateId, delegate: Weak, main_thread_marker: MainThreadMarker, + selected: Rc>, ) -> Id { match element { MenuElement::Action(menu_action) => { @@ -165,6 +186,7 @@ impl PlatformMenu { } else { let action = menu_action.unique_id; let action = move |_item: *mut NSMenuItem| { + selected.set(true); if let Some(delegate) = delegate.upgrade() { delegate.on_action(isolate, action); } @@ -217,8 +239,13 @@ impl PlatformMenu { let image = ns_image_for_menu_item(data.clone()); item.setImage(Some(&image)); } - let submenu = - Self::translate_menu(menu, isolate, delegate.clone(), main_thread_marker); + let submenu = Self::translate_menu( + menu, + isolate, + delegate.clone(), + main_thread_marker, + selected.clone(), + ); item.setSubmenu(Some(&submenu)); item } @@ -227,6 +254,7 @@ impl PlatformMenu { let action = move |item: *mut NSMenuItem| { let item = unsafe { &*item }; let delegate = delegate.clone(); + let selected = selected.clone(); let item = item.retain(); spawn(async move { Self::load_deferred_menu_item( @@ -235,6 +263,7 @@ impl PlatformMenu { isolate, delegate.clone(), main_thread_marker, + selected, ) .await; }); @@ -257,21 +286,33 @@ impl PlatformMenu { menu: Menu, ) -> NativeExtensionsResult> { let main_thread_marker = MainThreadMarker::new().unwrap(); - let menu = unsafe { Self::translate_menu(&menu, isolate, delegate, main_thread_marker) }; - Ok(Rc::new(Self { menu })) + let selected = Rc::new(Cell::new(false)); + let menu = unsafe { + Self::translate_menu( + &menu, + isolate, + delegate, + main_thread_marker, + selected.clone(), + ) + }; + Ok(Rc::new(Self { menu, selected })) } } impl PlatformMenuContext { pub fn new( - _id: PlatformMenuContextId, + id: PlatformMenuContextId, engine_handle: i64, delegate: Weak, ) -> NativeExtensionsResult { let view = EngineContext::get()?.get_flutter_view(engine_handle)?; Ok(Self { delegate, + context_id: id, view: unsafe { Id::cast(view) }, + context_menu_view: Late::new(), + writing_tool_text: RefCell::new(None), }) } @@ -283,19 +324,28 @@ impl PlatformMenuContext { Ok(()) } - pub fn assign_weak_self(&self, _weak_self: Weak) {} + pub fn assign_weak_self(&self, weak_self: Weak) { + let context_menu_inner = SNEContextMenuViewInner { + context: weak_self.clone(), + }; + self.context_menu_view.set(SNEContextMenuView::new( + context_menu_inner, + MainThreadMarker::new().unwrap(), + )); + } - fn synthesize_mouse_up_event(&self) { + fn synthesize_mouse_up_event(&self) -> Option> { if let Some(delegate) = self.delegate.upgrade() { let drag_contexts = delegate.get_platform_drag_contexts(); for context in drag_contexts { if *context.view == *self.view { unsafe { - context.synthesize_mouse_up_event(); + return context.synthesize_mouse_up_event(); } } } } + None } pub async fn show_context_menu( @@ -307,17 +357,39 @@ impl PlatformMenuContext { let (future, completer) = FutureCompleter::new(); - let menu = request.menu.unwrap().menu.clone(); + let ns_menu = request.menu.as_ref().unwrap().menu.clone(); let view = self.view.clone(); + let context_menu_view = self.context_menu_view.clone(); + unsafe { self.view.addSubview(&context_menu_view) }; + + let writing_tools_configuration = &request.writing_tools_configuration; + if let Some(writing_tools_configuration) = writing_tools_configuration { + let mut rect: NSRect = writing_tools_configuration.rect.clone().into(); + flip_rect(&view, &mut rect); + unsafe { context_menu_view.setFrame(rect) }; + self.writing_tool_text + .replace(Some(writing_tools_configuration.text.clone())); + } else { + self.writing_tool_text.replace(None); + } + // remember the modifier flags before showing the popup menu let flags_before = unsafe { NSEvent::modifierFlags_class() }; - self.synthesize_mouse_up_event(); + let event = self.synthesize_mouse_up_event(); + let selected = request.menu.as_ref().unwrap().selected.clone(); let cb = move || { - let item_selected = unsafe { - menu.popUpMenuPositioningItem_atLocation_inView(None, position, Some(&view)) + let first_responder = view.window().unwrap().firstResponder(); + let window = view.window().unwrap(); + window.makeFirstResponder(Some(&context_menu_view)); + unsafe { + NSMenu::popUpContextMenu_withEvent_forView( + &ns_menu, + &event.unwrap(), + &context_menu_view, + ) }; // If the the popup menu was shown because of control + click and the // control is no longe pressed after menu is closed we need to let Flutter @@ -337,7 +409,12 @@ impl PlatformMenuContext { } } } - completer.complete(Ok(ShowContextMenuResponse { item_selected })); + + window.makeFirstResponder(first_responder.as_deref()); + + completer.complete(Ok(ShowContextMenuResponse { + item_selected: selected.get(), + })); }; // this method might possibly be invoked from dispatch_async. @@ -406,3 +483,119 @@ extern_methods!( ) -> Id; } ); + +struct SNEContextMenuViewInner { + context: Weak, +} + +impl SNEContextMenuViewInner { + fn is_valid_requestor( + &self, + send_type: Option<&NSPasteboardType>, + _return_type: Option<&NSPasteboardType>, + ) -> bool { + if let (Some(send_type), Some(context)) = (send_type, self.context.upgrade()) { + context.writing_tool_text.borrow().is_some() + && send_type == unsafe { NSPasteboardTypeString } + } else { + false + } + } + + fn write_selection_to_pasteboard( + &self, + pboard: &NSPasteboard, + types: &NSArray, + ) -> bool { + if unsafe { types.containsObject(NSPasteboardTypeString) } { + if let Some(text) = self + .context + .upgrade() + .and_then(|f| f.writing_tool_text.borrow().clone()) + { + let text = NSString::from_str(&text); + unsafe { pboard.setString_forType(&text, NSPasteboardTypeString) }; + return true; + } + } + false + } + + fn read_selection_from_pasteboard(&self, pboard: &NSPasteboard) -> bool { + let string = unsafe { pboard.stringForType(NSPasteboardTypeString) }; + if let (Some(string), Some(context), Some(delegate)) = ( + string, + self.context.upgrade(), + self.context.upgrade().and_then(|c| c.delegate.upgrade()), + ) { + delegate.send_writing_tools_replacement( + context.context_id, + WritingToolsReplacementRequest { + text: string.to_string(), + }, + ); + return true; + } + false + } +} + +declare_class!( + struct SNEContextMenuView; + + unsafe impl ClassType for SNEContextMenuView { + type Super = NSView; + type Mutability = mutability::MainThreadOnly; + const NAME: &'static str = "SNEContextMenuView"; + } + + impl DeclaredClass for SNEContextMenuView { + type Ivars = SNEContextMenuViewInner; + } + + unsafe impl NSObjectProtocol for SNEContextMenuView {} + + #[allow(non_snake_case)] + unsafe impl NSServicesMenuRequestor for SNEContextMenuView { + #[method(writeSelectionToPasteboard:types:)] + unsafe fn writeSelectionToPasteboard_types( + &self, + pboard: &NSPasteboard, + types: &NSArray, + ) -> bool { + self.ivars().write_selection_to_pasteboard(pboard, types) + } + + #[method(readSelectionFromPasteboard:)] + unsafe fn readSelectionFromPasteboard(&self, pboard: &NSPasteboard) -> bool { + self.ivars().read_selection_from_pasteboard(pboard) + } + } + + #[allow(non_snake_case)] + unsafe impl SNEContextMenuView { + #[method_id(validRequestorForSendType:returnType:)] + unsafe fn validRequestorForSendType_returnType( + &self, + send_type: Option<&NSPasteboardType>, + return_type: Option<&NSPasteboardType>, + ) -> Option> { + if self.ivars().is_valid_requestor(send_type, return_type) { + Some(self.retain()) + } else { + unsafe { msg_send_id![super(self), validRequestorForSendType:send_type returnType:return_type ] } + } + } + } +); + +impl SNEContextMenuView { + fn new(inner: SNEContextMenuViewInner, mtm: MainThreadMarker) -> Id { + unsafe { + let this = mtm.alloc::(); + let this = this.set_ivars(inner); + let this: Id = msg_send_id![super(this), init]; + this + } + } +} diff --git a/super_native_extensions/rust/src/menu_manager.rs b/super_native_extensions/rust/src/menu_manager.rs index 41062739..5b09efe2 100644 --- a/super_native_extensions/rust/src/menu_manager.rs +++ b/super_native_extensions/rust/src/menu_manager.rs @@ -16,7 +16,7 @@ use log::warn; use crate::{ api_model::{ DeferredMenuResponse, ImageData, MenuConfiguration, MenuElement, Point, - ShowContextMenuRequest, ShowContextMenuResponse, + ShowContextMenuRequest, ShowContextMenuResponse, WritingToolsReplacementRequest, }, context::Context, drag_manager::GetDragManager, @@ -49,6 +49,12 @@ pub trait PlatformMenuContextDelegate { context_id: PlatformMenuContextId, location: Point, ) -> Arc>>; + + fn send_writing_tools_replacement( + &self, + context_id: PlatformMenuContextId, + request: WritingToolsReplacementRequest, + ); } #[async_trait(?Send)] @@ -327,6 +333,17 @@ impl PlatformMenuContextDelegate for MenuManager { }); res } + + fn send_writing_tools_replacement( + &self, + context_id: PlatformMenuContextId, + request: WritingToolsReplacementRequest, + ) { + self.invoker + .call_method_sync(context_id, "sendWritingToolsReplacement", request, |r| { + r.ok_log(); + }); + } } #[async_trait(?Send)]