diff --git a/Examples/NativeExamples/MovExample/GameViewController.swift b/Examples/NativeExamples/MovExample/GameViewController.swift index 8995d74..05aae44 100644 --- a/Examples/NativeExamples/MovExample/GameViewController.swift +++ b/Examples/NativeExamples/MovExample/GameViewController.swift @@ -54,9 +54,9 @@ class GameViewController: UIViewController { link?.add(to: RunLoop.main, forMode: .common) link?.isPaused = true - audioEngine.onBufferPublisher.sink { [weak self] (buffer: AVAudioPCMBuffer, timeSec: Double) in - self?.write(buffer: buffer, timeSec: timeSec) - }.store(in: &cancellables) +// audioEngine.onBufferPublisher.sink { [weak self] (buffer: AVAudioPCMBuffer, timeSec: Double) in +// self?.write(buffer: buffer, timeSec: timeSec) +// }.store(in: &cancellables) let tmpDir = FileManager.default.temporaryDirectory.appendingPathComponent("tmpDri") try! FileManager.default.createDirectory(at: tmpDir, withIntermediateDirectories: true, attributes: nil) @@ -67,24 +67,37 @@ class GameViewController: UIViewController { var recording: Bool = false var tmpUrl: NSString? var sentFirstFrame: Bool = false - let audioEngine = AudioEngineService() +// let audioEngine = AudioEngineService() + + var uuid: NSString = "" @IBAction func onTapButton(_ sender: Any) { recording.toggle() if recording { print("start recording") - UnityMediaCreator_initAsMovWithAudio(tmpUrl?.utf8String, - "h264", - Int64(view.frame.width), - Int64(view.frame.height), 1, Float(AppConfig.fs)) + uuid = UUID().uuidString as NSString +// UnityMediaCreator_initAsMovWithAudio(tmpUrl?.utf8String, +// "h264", Int64(view.frame.width), Int64(view.frame.height), 1, Float(AppConfig.fs), +// uuid.utf8String) + let tex = mtkView.currentDrawable!.texture + print("init \(tex.width) x \(tex.height)") + UnityMediaCreator_initAsMovWithNoAudio(tmpUrl?.utf8String, "h264", + Int64(tex.width), Int64(tex.height), +// "") + uuid.utf8String) sentFirstFrame = false link!.isPaused = false - audioEngine.start() +// audioEngine.start() } else { link!.isPaused = true - audioEngine.stop() +// audioEngine.stop() UnityMediaCreator_finishSync() - UnityMediaSaver_saveVideo(tmpUrl?.utf8String) +// UnityMediaSaver_saveVideo(tmpUrl?.utf8String) + + let tex = mtkView.currentDrawable!.texture + UnityMediaSaver_saveLivePhotos(Unmanaged.passUnretained(tex).toOpaque(), + uuid.utf8String, + tmpUrl?.utf8String) print("finish recording") } } diff --git a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Headers/UnityVideoCreator-Swift.h b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Headers/UnityVideoCreator-Swift.h index b5aa462..216c725 100644 --- a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Headers/UnityVideoCreator-Swift.h +++ b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Headers/UnityVideoCreator-Swift.h @@ -207,13 +207,14 @@ typedef unsigned int swift_uint4 __attribute__((__ext_vector_type__(4))); #endif void UnityMediaCreator_finishSync(void); -void UnityMediaCreator_initAsMovWithAudio(char const * _Nullable url, char const * _Nullable codec, int64_t width, int64_t height, int64_t channel, float samplingRate); -void UnityMediaCreator_initAsMovWithNoAudio(char const * _Nullable url, char const * _Nullable codec, int64_t width, int64_t height); +void UnityMediaCreator_initAsMovWithAudio(char const * _Nullable url, char const * _Nullable codec, int64_t width, int64_t height, int64_t channel, float samplingRate, char const * _Nullable contentIdentifier); +void UnityMediaCreator_initAsMovWithNoAudio(char const * _Nullable url, char const * _Nullable codec, int64_t width, int64_t height, char const * _Nullable contentIdentifier); void UnityMediaCreator_initAsWav(char const * _Nullable url, int64_t channel, float samplingRate, NSInteger bitDepth); BOOL UnityMediaCreator_isRecording(void) SWIFT_WARN_UNUSED_RESULT; void UnityMediaCreator_start(int64_t microSec); void UnityMediaCreator_writeAudio(float const * _Nonnull pcm, int64_t frame, int64_t microSec); void UnityMediaCreator_writeVideo(void const * _Nullable texturePtr, int64_t microSec); +void UnityMediaSaver_saveLivePhotos(void const * _Nullable texturePtr, char const * _Nullable contentIdentifier, char const * _Nullable url); void UnityMediaSaver_saveVideo(char const * _Nullable url); SWIFT_CLASS("_TtC17UnityVideoCreator17VideoCreatorUnity") diff --git a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/Project/arm64-apple-ios.swiftsourceinfo b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/Project/arm64-apple-ios.swiftsourceinfo index 01df230..714332a 100644 Binary files a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/Project/arm64-apple-ios.swiftsourceinfo and b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/Project/arm64-apple-ios.swiftsourceinfo differ diff --git a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/Project/arm64.swiftsourceinfo b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/Project/arm64.swiftsourceinfo index 01df230..714332a 100644 Binary files a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/Project/arm64.swiftsourceinfo and b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/Project/arm64.swiftsourceinfo differ diff --git a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/arm64-apple-ios.swiftmodule b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/arm64-apple-ios.swiftmodule index cb14ac4..639cb0a 100644 Binary files a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/arm64-apple-ios.swiftmodule and b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/arm64-apple-ios.swiftmodule differ diff --git a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/arm64.swiftmodule b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/arm64.swiftmodule index cb14ac4..639cb0a 100644 Binary files a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/arm64.swiftmodule and b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/Modules/UnityVideoCreator.swiftmodule/arm64.swiftmodule differ diff --git a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/UnityVideoCreator b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/UnityVideoCreator index bb301bb..c5b2d35 100755 Binary files a/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/UnityVideoCreator and b/Examples/UnityExample/Assets/Plugin/VideoCreator/Plugins/iOS/UnityVideoCreator.framework/UnityVideoCreator differ diff --git a/Examples/UnityExample/Assets/Plugin/VideoCreator/Scripts/MediaCreator/MediaCreator.cs b/Examples/UnityExample/Assets/Plugin/VideoCreator/Scripts/MediaCreator/MediaCreator.cs index 2eeb6d7..0841072 100644 --- a/Examples/UnityExample/Assets/Plugin/VideoCreator/Scripts/MediaCreator/MediaCreator.cs +++ b/Examples/UnityExample/Assets/Plugin/VideoCreator/Scripts/MediaCreator/MediaCreator.cs @@ -10,10 +10,18 @@ public class MediaSaver [DllImport("__Internal")] private static extern void UnityMediaSaver_saveVideo(string url); + [DllImport("__Internal")] + private static extern void UnityMediaSaver_saveLivePhotos(IntPtr texturePtr, string contentIdentifier, string url); + public static void SaveVideo(string url) { UnityMediaSaver_saveVideo(url); } + + public static void SaveLivePhotos(Texture texture, string contentIdentifier, string url) + { + UnityMediaSaver_saveLivePhotos(texture.GetNativeTexturePtr(), contentIdentifier, url); + } #endif } @@ -21,10 +29,10 @@ public class MediaCreator { #if UNITY_IOS [DllImport("__Internal")] - private static extern void UnityMediaCreator_initAsMovWithNoAudio(string url, string codec, long width, long height); + private static extern void UnityMediaCreator_initAsMovWithNoAudio(string url, string codec, long width, long height, string contentIdentifier); [DllImport("__Internal")] - private static extern void UnityMediaCreator_initAsMovWithAudio(string url, string codec, long width, long height, long channel, float samplingRate); + private static extern void UnityMediaCreator_initAsMovWithAudio(string url, string codec, long width, long height, long channel, float samplingRate, string contentIdentifier); [DllImport("__Internal")] private static extern void UnityMediaCreator_initAsWav(string url, long channel, float samplingRate, long bitDepth); @@ -45,17 +53,17 @@ public class MediaCreator private static extern void UnityMediaCreator_writeAudio(float[] pcm, long frame, long microSec); #endif - public static void InitAsMovWithNoAudio(string url, string codec, long width, long height) + public static void InitAsMovWithNoAudio(string url, string codec, long width, long height, string contentIdentifier = "") { #if UNITY_IOS - UnityMediaCreator_initAsMovWithNoAudio(url, codec, width, height); + UnityMediaCreator_initAsMovWithNoAudio(url, codec, width, height, contentIdentifier); #endif } - public static void InitAsMovWithAudio(string url, string codec, long width, long height, long channel, float samplingRate) + public static void InitAsMovWithAudio(string url, string codec, long width, long height, long channel, float samplingRate, string contentIdentifier = "") { #if UNITY_IOS - UnityMediaCreator_initAsMovWithAudio(url, codec, width, height, channel, samplingRate); + UnityMediaCreator_initAsMovWithAudio(url, codec, width, height, channel, samplingRate, contentIdentifier); #endif } diff --git a/Examples/UnityExample/Assets/Scenes/SampleScene.unity b/Examples/UnityExample/Assets/Scenes/SampleScene.unity index 83b0de8..787d605 100644 --- a/Examples/UnityExample/Assets/Scenes/SampleScene.unity +++ b/Examples/UnityExample/Assets/Scenes/SampleScene.unity @@ -221,6 +221,7 @@ RectTransform: - {fileID: 1091966527} - {fileID: 1927427053} - {fileID: 1563167747} + - {fileID: 1511568283} m_Father: {fileID: 0} m_RootOrder: 5 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} @@ -1747,6 +1748,218 @@ CanvasRenderer: m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 1320822199} m_CullTransparentMesh: 0 +--- !u!1 &1347798594 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1347798595} + - component: {fileID: 1347798597} + - component: {fileID: 1347798596} + m_Layer: 5 + m_Name: Text + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &1347798595 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1347798594} + m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: [] + m_Father: {fileID: 1511568283} + m_RootOrder: 0 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 1, y: 1} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &1347798596 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1347798594} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 5f7201a12d95ffc409449d95f23cf332, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_FontData: + m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0} + m_FontSize: 32 + m_FontStyle: 0 + m_BestFit: 0 + m_MinSize: 3 + m_MaxSize: 40 + m_Alignment: 4 + m_AlignByGeometry: 0 + m_RichText: 1 + m_HorizontalOverflow: 0 + m_VerticalOverflow: 0 + m_LineSpacing: 1 + m_Text: rec live photos +--- !u!222 &1347798597 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1347798594} + m_CullTransparentMesh: 1 +--- !u!1 &1511568282 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1511568283} + - component: {fileID: 1511568286} + - component: {fileID: 1511568285} + - component: {fileID: 1511568284} + m_Layer: 5 + m_Name: rec live phots + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &1511568283 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1511568282} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_Children: + - {fileID: 1347798595} + m_Father: {fileID: 14364497} + m_RootOrder: 6 + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0.5, y: 0.5} + m_AnchorMax: {x: 0.5, y: 0.5} + m_AnchoredPosition: {x: -400, y: 0} + m_SizeDelta: {x: 480, y: 64} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &1511568284 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1511568282} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 4e29b1a8efbd4b44bb3f3716e73f07ff, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Navigation: + m_Mode: 3 + m_WrapAround: 0 + m_SelectOnUp: {fileID: 0} + m_SelectOnDown: {fileID: 0} + m_SelectOnLeft: {fileID: 0} + m_SelectOnRight: {fileID: 0} + m_Transition: 1 + m_Colors: + m_NormalColor: {r: 1, g: 1, b: 1, a: 1} + m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1} + m_PressedColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 1} + m_SelectedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1} + m_DisabledColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 0.5019608} + m_ColorMultiplier: 1 + m_FadeDuration: 0.1 + m_SpriteState: + m_HighlightedSprite: {fileID: 0} + m_PressedSprite: {fileID: 0} + m_SelectedSprite: {fileID: 0} + m_DisabledSprite: {fileID: 0} + m_AnimationTriggers: + m_NormalTrigger: Normal + m_HighlightedTrigger: Highlighted + m_PressedTrigger: Pressed + m_SelectedTrigger: Selected + m_DisabledTrigger: Disabled + m_Interactable: 1 + m_TargetGraphic: {fileID: 1511568285} + m_OnClick: + m_PersistentCalls: + m_Calls: + - m_Target: {fileID: 668232124} + m_TargetAssemblyTypeName: RecordingController, Assembly-CSharp + m_MethodName: RecLivePhotos + m_Mode: 1 + m_Arguments: + m_ObjectArgument: {fileID: 0} + m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine + m_IntArgument: 0 + m_FloatArgument: 0 + m_StringArgument: + m_BoolArgument: 0 + m_CallState: 2 +--- !u!114 &1511568285 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1511568282} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Sprite: {fileID: 10905, guid: 0000000000000000f000000000000000, type: 0} + m_Type: 1 + m_PreserveAspect: 0 + m_FillCenter: 1 + m_FillMethod: 4 + m_FillAmount: 1 + m_FillClockwise: 1 + m_FillOrigin: 0 + m_UseSpriteMesh: 0 + m_PixelsPerUnitMultiplier: 1 +--- !u!222 &1511568286 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1511568282} + m_CullTransparentMesh: 1 --- !u!1 &1563167746 GameObject: m_ObjectHideFlags: 0 diff --git a/Examples/UnityExample/Assets/Scripts/RecordingController.cs b/Examples/UnityExample/Assets/Scripts/RecordingController.cs index 9167b11..0221884 100644 --- a/Examples/UnityExample/Assets/Scripts/RecordingController.cs +++ b/Examples/UnityExample/Assets/Scripts/RecordingController.cs @@ -13,7 +13,9 @@ public class RecordingController : MonoBehaviour private bool recordTexture = false; private bool recordAudio = false; private bool saveAfterFinish = false; + private int livePhotosRecording = -1; + private string uuid = ""; private string cachePath = ""; private long amountFrame = 0; @@ -42,6 +44,32 @@ void Update() Debug.Log($"write texture: {time}"); MediaCreator.WriteVideo(texture, time); + + if (livePhotosRecording < 0) return; + livePhotosRecording += 1; + if (livePhotosRecording > 60) FinishRec(); + } + + public void RecLivePhotos() + { + if (isRecording) return; + text.text = "start rec live photo!"; + + cachePath = "file://" + Application.temporaryCachePath + "/tmp.mov"; + Debug.Log($"cachePath: {cachePath}, {texture.width}x{texture.height}"); + + uuid = System.Guid.NewGuid().ToString(); + MediaCreator.InitAsMovWithNoAudio(cachePath, "h264", texture.width, texture.height, uuid); + MediaCreator.Start(startTimeOffset); + + startTime = Time.time; + + isRecording = true; + recordTexture = true; + recordAudio = false; + saveAfterFinish = false; + amountFrame = 0; + livePhotosRecording = 1; } public void StartRecMovWithAudio() @@ -63,11 +91,13 @@ public void StartRecMovWithAudio() startTime = Time.time; + uuid = ""; isRecording = true; recordTexture = true; recordAudio = true; saveAfterFinish = true; amountFrame = 0; + livePhotosRecording = -1; source.Play(); } @@ -85,11 +115,13 @@ public void StartRecMovWithNoAudio() startTime = Time.time; + uuid = ""; isRecording = true; recordTexture = true; recordAudio = false; saveAfterFinish = true; amountFrame = 0; + livePhotosRecording = -1; } public void StartRecWav() @@ -111,11 +143,13 @@ public void StartRecWav() startTime = Time.time; + uuid = ""; isRecording = true; recordTexture = false; recordAudio = true; saveAfterFinish = false; amountFrame = 0; + livePhotosRecording = -1; source.Play(); } @@ -134,10 +168,16 @@ public void FinishRec() MediaSaver.SaveVideo(cachePath); } + if (livePhotosRecording > 0) + { + MediaSaver.SaveLivePhotos(texture, uuid, cachePath); + } + isRecording = false; recordTexture = false; recordAudio = false; saveAfterFinish = false; + livePhotosRecording = -1; } void OnAudioFilterRead(float[] data, int channels) diff --git a/Package.swift b/Package.swift index 430049b..2641297 100644 --- a/Package.swift +++ b/Package.swift @@ -5,7 +5,10 @@ import PackageDescription let package = Package( name: "UnityVideoCreator", - platforms: [.iOS(.v13), .macOS(SupportedPlatform.MacOSVersion.v10_15)], + platforms: [ + .iOS(.v13), + .macOS(.v10_15), + ], products: [ // Products define the executables and libraries a package produces, and make them visible to other packages. .library( diff --git a/README.md b/README.md index 876d6f6..3df7772 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ * [x] float array * Container * [x] mov + * [x] Live Photos * [ ] mp4 @@ -63,6 +64,15 @@ string cachePath = "file://" + Application.temporaryCachePath + "/tmp.mov"; MediaCreator.InitAsMovWithAudio(cachePath, "h264", texture.width, texture.height, 1, 48_000); ``` +### Setup MediaCreator for Live Photos +* In addition to the usual mov, set Content Identifier. + +```c# +string uuid = System.Guid.NewGuid().ToString(); +string cachePath = "file://" + Application.temporaryCachePath + "/tmp.mov"; +MediaCreator.InitAsMovWithAudio(cachePath, "h264", width, height, uuid); +``` + ### Setup MediaCreator for wav file * In addition to the number of audio channels and sampling rate, set the Bit Depth. @@ -112,7 +122,7 @@ MediaCreator.WriteAudio(pcm, time); MediaCreator.FinishSync(); ``` -## Save to album app (optional) +## Save mov to album app (optional) * If you want to save your recorded videos to an album, you can use MediaSaver to do so. @@ -120,6 +130,15 @@ MediaCreator.FinishSync(); MediaSaver.SaveVideo(cachePath); ``` +## Save Live Photos to album app (optional) + +* If you want to save your recorded Live Photos to an album, you can use MediaSaver to do so. +* Set the thumbnail and the same Content Identifier as the video. + +```c# +MediaSaver.SaveLivePhotos(texture, uuid, cachePath); +``` + # Examples ## UnityExample * Example for Unity diff --git a/Sources/UnityVideoCreator/MediaCreator/Core/MediaCreator.swift b/Sources/UnityVideoCreator/MediaCreator/Core/MediaCreator.swift index 6a8314f..15266f1 100644 --- a/Sources/UnityVideoCreator/MediaCreator/Core/MediaCreator.swift +++ b/Sources/UnityVideoCreator/MediaCreator/Core/MediaCreator.swift @@ -29,7 +29,10 @@ class DefaultMediaCreator: MediaCreator { private let audioFactory: SampleBufferAudioFactory = SampleBufferAudioFactory() init(config: MediaWriterConfig) throws { - writer = try MediaWriter(url: config.url, fileType: config.fileType, inputConfigs: config.inputConfigs) + writer = try MediaWriter(url: config.url, + fileType: config.fileType, + inputConfigs: config.inputConfigs, + contentIdentifier: config.contentIdentifier) if let config = config as? MovMediaWriterConfig { videoFactory = SampleBufferVideoFactory(width: config.video.width, height: config.video.height) diff --git a/Sources/UnityVideoCreator/MediaCreator/Core/MediaWriter/MediaWriter.swift b/Sources/UnityVideoCreator/MediaCreator/Core/MediaWriter/MediaWriter.swift index 25ff280..e3f6119 100644 --- a/Sources/UnityVideoCreator/MediaCreator/Core/MediaWriter/MediaWriter.swift +++ b/Sources/UnityVideoCreator/MediaCreator/Core/MediaWriter/MediaWriter.swift @@ -23,10 +23,14 @@ class MediaWriter { } private let assetWriter: AVAssetWriter + private var metadataAdaptor: AVAssetWriterInputMetadataAdaptor? + private var startTime: CMTime? + private var latestTime: CMTime? init(url: URL, fileType: AVFileType, - inputConfigs: [InputConfig]) throws { + inputConfigs: [InputConfig], + contentIdentifier: String) throws { let assetWriter = try AVAssetWriter(outputURL: url, fileType: fileType) self.assetWriter = assetWriter @@ -36,6 +40,11 @@ class MediaWriter { input.expectsMediaDataInRealTime = config.expectsMediaDataInRealTime assetWriter.add(input) } + + if contentIdentifier.count == 0 { return } + assetWriter.metadata = [makeContentIdentifierMetadataItem(identifier: contentIdentifier)] + self.metadataAdaptor = makeStillImageTimeMetadataAdaptor() + assetWriter.add(metadataAdaptor!.assetWriterInput) } public func start(time: CMTime) throws { @@ -44,9 +53,18 @@ class MediaWriter { throw assetWriter.error ?? MediaWriterError.unknown } assetWriter.startSession(atSourceTime: time) + + self.startTime = time } public func finish(completionHandler: @escaping () -> Void){ + if let metadataAdaptor = self.metadataAdaptor, + let startTime = self.startTime, + let latestTime = self.latestTime { + let timeRange = CMTimeRange(start: startTime, end: latestTime) + metadataAdaptor.append(AVTimedMetadataGroup(items: [makeStillImageTimeMetadataItem()], + timeRange: timeRange)) + } assetWriter.finishWriting(completionHandler: completionHandler) } @@ -65,5 +83,40 @@ class MediaWriter { os_log(.error, log: .default, "failed append %@", [sample]) throw assetWriter.error ?? MediaWriterError.unknown } + self.latestTime = CMSampleBufferGetPresentationTimeStamp(sample) + } + + private func makeContentIdentifierMetadataItem(identifier: String) -> AVMetadataItem { + let item = AVMutableMetadataItem() + item.key = AVMetadataKey.quickTimeMetadataKeyContentIdentifier as NSString + item.keySpace = AVMetadataKeySpace.quickTimeMetadata + item.value = identifier as NSString + item.dataType = kCMMetadataBaseDataType_UTF8 as String + return item + } + + private func makeStillImageTimeMetadataItem() -> AVMetadataItem { + let item = AVMutableMetadataItem() + item.key = "com.apple.quicktime.still-image-time" as NSString + item.keySpace = AVMetadataKeySpace.quickTimeMetadata + item.value = 0 as NSNumber + item.dataType = kCMMetadataBaseDataType_SInt8 as String + return item + } + + private func makeStillImageTimeMetadataAdaptor() -> AVAssetWriterInputMetadataAdaptor { + let spec : NSDictionary = [ + kCMMetadataFormatDescriptionMetadataSpecificationKey_Identifier: "mdta/com.apple.quicktime.still-image-time", + kCMMetadataFormatDescriptionMetadataSpecificationKey_DataType: kCMMetadataBaseDataType_SInt8 + ] + var desc : CMFormatDescription? = nil + CMMetadataFormatDescriptionCreateWithMetadataSpecifications(allocator: kCFAllocatorDefault, + metadataType: kCMMetadataFormatType_Boxed, + metadataSpecifications: [spec] as CFArray, + formatDescriptionOut: &desc) + let input = AVAssetWriterInput(mediaType: .metadata, + outputSettings: nil, + sourceFormatHint: desc) + return AVAssetWriterInputMetadataAdaptor(assetWriterInput: input) } } diff --git a/Sources/UnityVideoCreator/MediaCreator/Core/MediaWriter/MediaWriterConfig/MediaWriterConfig.swift b/Sources/UnityVideoCreator/MediaCreator/Core/MediaWriter/MediaWriterConfig/MediaWriterConfig.swift index ce01f23..08c0110 100644 --- a/Sources/UnityVideoCreator/MediaCreator/Core/MediaWriter/MediaWriterConfig/MediaWriterConfig.swift +++ b/Sources/UnityVideoCreator/MediaCreator/Core/MediaWriter/MediaWriterConfig/MediaWriterConfig.swift @@ -12,6 +12,7 @@ protocol MediaWriterConfig { var url: URL { get } var fileType: AVFileType { get } var inputConfigs: [MediaWriter.InputConfig] { get } + var contentIdentifier: String { get } } struct MovMediaWriterConfig: MediaWriterConfig { @@ -22,6 +23,7 @@ struct MovMediaWriterConfig: MediaWriterConfig { var inputConfigs: [MediaWriter.InputConfig] { return [video.config, audio?.config].compactMap { $0 } } + let contentIdentifier: String } struct WavMediaWriterConfig: MediaWriterConfig { @@ -31,4 +33,5 @@ struct WavMediaWriterConfig: MediaWriterConfig { var inputConfigs: [MediaWriter.InputConfig] { return [audio.config] } + let contentIdentifier: String = "" } diff --git a/Sources/UnityVideoCreator/MediaCreator/Unity/CdeclUnityMediaCreator.swift b/Sources/UnityVideoCreator/MediaCreator/Unity/CdeclUnityMediaCreator.swift index 65493fd..8466cdc 100644 --- a/Sources/UnityVideoCreator/MediaCreator/Unity/CdeclUnityMediaCreator.swift +++ b/Sources/UnityVideoCreator/MediaCreator/Unity/CdeclUnityMediaCreator.swift @@ -12,13 +12,16 @@ import Metal public func UnityMediaCreator_initAsMovWithNoAudio(_ url: UnsafePointer?, _ codec: UnsafePointer?, _ width: Int64, - _ height: Int64) { + _ height: Int64, + _ contentIdentifier: UnsafePointer?) { let url = String(cString: url!) let codec = String(cString: codec!) + let contentIdentifier = String(cString: contentIdentifier!) UnityMediaCreator.shared.initAsMovWithNoAudio(url: url, codec: codec, width: Int(width), - height: Int(height)) + height: Int(height), + contentIdentifier: contentIdentifier) } @_cdecl("UnityMediaCreator_initAsMovWithAudio") @@ -27,11 +30,14 @@ public func UnityMediaCreator_initAsMovWithAudio(_ url: UnsafePointer?, _ width: Int64, _ height: Int64, _ channel: Int64, - _ samplingRate: Float) { + _ samplingRate: Float, + _ contentIdentifier: UnsafePointer?) { let url = String(cString: url!) let codec = String(cString: codec!) + let contentIdentifier = String(cString: contentIdentifier!) UnityMediaCreator.shared.initAsMovWithAudio(url: url, codec: codec, width: Int(width), height: Int(height), - channel: Int(channel), samplingRate: samplingRate) + channel: Int(channel), samplingRate: samplingRate, + contentIdentifier: contentIdentifier) } @_cdecl("UnityMediaCreator_initAsWav") diff --git a/Sources/UnityVideoCreator/MediaCreator/Unity/UnityMediaCreator.swift b/Sources/UnityVideoCreator/MediaCreator/Unity/UnityMediaCreator.swift index 4d96f29..7d05776 100644 --- a/Sources/UnityVideoCreator/MediaCreator/Unity/UnityMediaCreator.swift +++ b/Sources/UnityVideoCreator/MediaCreator/Unity/UnityMediaCreator.swift @@ -32,7 +32,10 @@ class UnityMediaCreator { private var channel: Int? = nil private var creator: MediaCreator? = nil - public func initAsMovWithAudio(url: String, codec: String, width: Int, height: Int, channel: Int, samplingRate: Float) { + public func initAsMovWithAudio(url: String, + codec: String, width: Int, height: Int, + channel: Int, samplingRate: Float, + contentIdentifier: String) { finishSync() let url = URL(string: url)! clean(url: url) @@ -44,13 +47,13 @@ class UnityMediaCreator { samplingRate: samplingRate, bitRate: 128000, expectsMediaDataInRealTime: true) - let config = MovMediaWriterConfig(url: url, video: video, audio: audio) + let config = MovMediaWriterConfig(url: url, video: video, audio: audio, contentIdentifier: contentIdentifier) self.samplingRate = samplingRate self.channel = channel self.creator = try! provider.make(config: config) } - public func initAsMovWithNoAudio(url: String, codec: String, width: Int, height: Int) { + public func initAsMovWithNoAudio(url: String, codec: String, width: Int, height: Int, contentIdentifier: String) { finishSync() let url = URL(string: url)! clean(url: url) @@ -58,7 +61,7 @@ class UnityMediaCreator { width: width, height: height, expectsMediaDataInRealTime: true) - let config = MovMediaWriterConfig(url: url, video: video, audio: nil) + let config = MovMediaWriterConfig(url: url, video: video, audio: nil, contentIdentifier: contentIdentifier) self.creator = try! provider.make(config: config) } diff --git a/Sources/UnityVideoCreator/Utils/Extensions/CMSampleBuffer+Extension.swift b/Sources/UnityVideoCreator/Utils/Extensions/CMSampleBuffer+Extension.swift index f5138cf..54f1668 100644 --- a/Sources/UnityVideoCreator/Utils/Extensions/CMSampleBuffer+Extension.swift +++ b/Sources/UnityVideoCreator/Utils/Extensions/CMSampleBuffer+Extension.swift @@ -7,6 +7,7 @@ // import AVFoundation +import CoreImage class SampleBufferVideoFactory { let context = CIContext() diff --git a/Sources/UnityVideoCreator/Utils/MediaSaver/MediaSaver.swift b/Sources/UnityVideoCreator/Utils/MediaSaver/MediaSaver.swift index 3649113..4cf3308 100644 --- a/Sources/UnityVideoCreator/Utils/MediaSaver/MediaSaver.swift +++ b/Sources/UnityVideoCreator/Utils/MediaSaver/MediaSaver.swift @@ -5,6 +5,7 @@ // Created by fuziki on 2021/06/19. // +import os import Photos @_cdecl("UnityMediaSaver_saveVideo") @@ -13,8 +14,37 @@ public func UnityMediaSaver_saveVideo(_ url: UnsafePointer?) { let url = URL(string: urlStr)! try! PHPhotoLibrary.shared().performChangesAndWait { let options = PHAssetResourceCreationOptions() - options.shouldMoveFile = false + options.shouldMoveFile = true let request = PHAssetCreationRequest.forAsset() request.addResource(with: .video, fileURL: url, options: options) } } + +@_cdecl("UnityMediaSaver_saveLivePhotos") +public func UnityMediaSaver_saveLivePhotos(_ texturePtr: UnsafeRawPointer?, + _ contentIdentifier: UnsafePointer?, + _ url: UnsafePointer?) { + let brideged: MTLTexture = __bridge(texturePtr!) + let srgb = brideged.makeTextureView(pixelFormat: .rgba8Unorm_srgb)! + let ci = CIImage(mtlTexture: srgb, options: [:])! + let context = CIContext() + let contentIdentifier = String(cString: contentIdentifier!) + let properties: [CFString : Any] = [kCGImagePropertyMakerAppleDictionary: ["17": contentIdentifier]] + let propertiedCi = ci.settingProperties(properties) + let jpegData = context.jpegRepresentation(of: propertiedCi, colorSpace: CGColorSpace(name: CGColorSpace.sRGB)!, options: [:])! + + let urlStr = String(cString: url!) + let url = URL(string: urlStr)! + do { + try PHPhotoLibrary.shared().performChangesAndWait { + let request = PHAssetCreationRequest.forAsset() + request.addResource(with: .photo, data: jpegData, options: nil) + let videoOptions = PHAssetResourceCreationOptions() + videoOptions.shouldMoveFile = true + request.addResource(with: .pairedVideo, fileURL: url, options: videoOptions) + } + } catch let error { + os_log(.error, log: .default, "failed save as livephotos. error: %@", [error]) + } +} + diff --git a/Tests/UnityVideoCreatorTests/Feature/MediaCreator/Core/MediaWriter/InitMediaWriterTest.swift b/Tests/UnityVideoCreatorTests/Feature/MediaCreator/Core/MediaWriter/InitMediaWriterTest.swift index bf41c04..06b5ac1 100644 --- a/Tests/UnityVideoCreatorTests/Feature/MediaCreator/Core/MediaWriter/InitMediaWriterTest.swift +++ b/Tests/UnityVideoCreatorTests/Feature/MediaCreator/Core/MediaWriter/InitMediaWriterTest.swift @@ -20,15 +20,15 @@ final class InitMediaWriterTest: XCTestCase { func testCorrectConfig() { let configs: [MediaWriterConfig] = [ - MovMediaWriterConfig(url: url, video: h264, audio: nil), - MovMediaWriterConfig(url: url, video: h265, audio: nil), - MovMediaWriterConfig(url: url, video: h264, audio: aac), - MovMediaWriterConfig(url: url, video: h265, audio: aac), + MovMediaWriterConfig(url: url, video: h264, audio: nil, contentIdentifier: ""), + MovMediaWriterConfig(url: url, video: h265, audio: nil, contentIdentifier: ""), + MovMediaWriterConfig(url: url, video: h264, audio: aac, contentIdentifier: ""), + MovMediaWriterConfig(url: url, video: h265, audio: aac, contentIdentifier: ""), WavMediaWriterConfig(url: url, audio: liner) ] for config in configs { - XCTAssertNoThrow(try MediaWriter(url: config.url, fileType: config.fileType, inputConfigs: config.inputConfigs)) + XCTAssertNoThrow(try MediaWriter(url: config.url, fileType: config.fileType, inputConfigs: config.inputConfigs, contentIdentifier: config.contentIdentifier)) } } diff --git a/Tests/UnityVideoCreatorTests/Feature/MediaCreator/Unity/UnityMediaCreatorTest.swift b/Tests/UnityVideoCreatorTests/Feature/MediaCreator/Unity/UnityMediaCreatorTest.swift index 679da7d..6a8c9ab 100644 --- a/Tests/UnityVideoCreatorTests/Feature/MediaCreator/Unity/UnityMediaCreatorTest.swift +++ b/Tests/UnityVideoCreatorTests/Feature/MediaCreator/Unity/UnityMediaCreatorTest.swift @@ -22,7 +22,8 @@ final class UnityMediaCreatorTest: XCTestCase { let testCreator = UnityMediaCreator(provider: provider) testCreator.initAsMovWithAudio(url: url.absoluteString, codec: "h264", width: 1920, height: 1080, - channel: 1, samplingRate: 48_000) + channel: 1, samplingRate: 48_000, + contentIdentifier: "") let pcm = [Float](repeating: 0, count: 1024) testCreator.write(pcm: pcm, frame: pcm.count, microSec: 0) @@ -39,7 +40,7 @@ final class UnityMediaCreatorTest: XCTestCase { let testCreator = UnityMediaCreator(provider: provider) - testCreator.initAsMovWithNoAudio(url: url.absoluteString, codec: "h264", width: 1920, height: 1080) + testCreator.initAsMovWithNoAudio(url: url.absoluteString, codec: "h264", width: 1920, height: 1080, contentIdentifier: "") let pcm = [Float](repeating: 0, count: 1024) testCreator.write(pcm: pcm, frame: pcm.count, microSec: 0) @@ -94,7 +95,8 @@ final class UnityMediaCreatorTest: XCTestCase { let testCreator = UnityMediaCreator(provider: provider) testCreator.initAsMovWithAudio(url: url.absoluteString, codec: "h264", width: 1920, height: 1080, - channel: 1, samplingRate: 48_000) + channel: 1, samplingRate: 48_000, + contentIdentifier: "") XCTAssertEqual(deinitCallCount, 0)