Feature/uni 169 add ios noise cancellation

.gitattributes

@@ -0,0 +1,2 @@
+Packages/StreamVideo/Runtime/Libs/io.stream.unity.webrtc/Runtime/Plugins/iOS/webrtc.framework.dSYM/Contents/Resources/DWARF/webrtc filter=lfs diff=lfs merge=lfs -text
+Packages/StreamVideo/Runtime/Libs/io.stream.unity.webrtc/Runtime/Plugins/iOS/webrtc.framework.dSYM/Contents/Resources/Relocations/**/*.yml filter=lfs diff=lfs merge=lfs -text

Assets/Samples/Stream Video & Audio Chat SDK/0.9.0/Video & Audio Chat Example Project/Scripts/StreamVideoManager.cs

@@ -216,7 +216,7 @@ protected void OnApplicationPause(bool pauseStatus)
             if (pauseStatus)
             {
                 // App is going to background
-                Client.PauseAndroidAudioPlayback();
+                Client.PauseMobileAudioPlayback();
                 _wasAudioPublishEnabledOnPause = Client.AudioDeviceManager.IsEnabled;
                 _wasVideoPublishEnabledOnPause = Client.VideoDeviceManager.IsEnabled;
 
@@ -226,7 +226,7 @@ protected void OnApplicationPause(bool pauseStatus)
             else
             {
                 // App is coming to foreground
-                Client.ResumeAndroidAudioPlayback();
+                Client.ResumeMobileAudioPlayback();
 
                 if (_wasAudioPublishEnabledOnPause)
                 {

Packages/StreamVideo/Runtime/Core/DeviceManagers/StreamAudioDeviceManager.cs

@@ -31,6 +31,7 @@ public override IEnumerable<MicrophoneDeviceInfo> EnumerateDevices()
             {
             }
 
+#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
             if (RtcSession.UseNativeAudioBindings)
             {
                 NativeAudioDeviceManager.GetAudioInputDevices(ref _inputDevicesBuffer);
@@ -43,13 +44,13 @@ public override IEnumerable<MicrophoneDeviceInfo> EnumerateDevices()
 
                     yield return new MicrophoneDeviceInfo(device.Id, device.Name);
                 }
+                yield break;
             }
-            else
+#endif
+
+            foreach (var deviceName in Microphone.devices)
             {
-                foreach (var deviceName in Microphone.devices)
-                {
-                    yield return new MicrophoneDeviceInfo(deviceName);
-                }
+                yield return new MicrophoneDeviceInfo(deviceName);
             }
         }
 
@@ -129,10 +130,12 @@ public void SelectDevice(MicrophoneDeviceInfo device, bool enable)
 
             IsEnabled = enable;
 
+#if UNITY_ANDROID && !UNITY_EDITOR
             if (RtcSession.UseNativeAudioBindings)
             {
                 SetAudioRoutingAsync((NativeAudioDeviceManager.AudioRouting)SelectedDevice.IntId.Value).LogIfFailed();
             }
+#endif
         }
 
         //StreamTodo: https://docs.unity3d.com/ScriptReference/AudioSource-ignoreListenerPause.html perhaps this should be enabled so that AudioListener doesn't affect recorded audio

Packages/StreamVideo/Runtime/Core/IStreamVideoClient.cs

@@ -31,6 +31,11 @@ public interface IStreamVideoClient : IStreamVideoClientEventsListener, IDisposa
         /// </summary>
         event CallHandler CallStarted;
 
+        /// <summary>
+        /// Event fired when a call is about to be left. You can still access full call data because the leaving process is just starting.
+        /// </summary>
+        event CallHandler CallLeaving;
+
         /// <summary>
         /// Event fired when a call ended
         /// </summary>
@@ -129,16 +134,17 @@ Task<IStreamCall> GetCallAsync(StreamCallType callType, string callId,
 
 
         /// <summary>
-        /// Temporary method (can be removed in the future) to pause audio playback on Android.
-        /// This will completely suspend playback of any audio coming from the StreamVideo SDK on the Android platform.
+        /// Mutes native SDK audio playback (silences remote participants on the device's
+        /// speakers). Call from app lifecycle events such as <c>OnApplicationPause(true)</c>.
+        /// The native audio device stays open so resume is instant. No-op on platforms
+        /// that don't use the native audio pipeline (Editor, Standalone).
         /// </summary>
-        void PauseAndroidAudioPlayback();
+        void PauseMobileAudioPlayback();
 
         /// <summary>
-        /// Temporary method (can be removed in the future) to resume audio playback on Android.
-        /// Call this resume audio playback if it was previously paused using <see cref="PauseAndroidAudioPlayback"/>.
+        /// Resumes playback previously muted via <see cref="PauseMobileAudioPlayback"/>.
         /// </summary>
-        void ResumeAndroidAudioPlayback();
+        void ResumeMobileAudioPlayback();
 
         /// <summary>
         /// Set Android Audio usage mode

Packages/StreamVideo/Runtime/Core/LowLevelClient/PeerConnectionBase.cs

@@ -1,6 +1,6 @@
 //StreamTodo: duplicated declaration of STREAM_NATIVE_AUDIO (also in RtcSession.cs) easy to get out of sync.
 
-#if UNITY_ANDROID && !UNITY_EDITOR
+#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
 #define STREAM_NATIVE_AUDIO //Defined in multiple files
 #endif
 using System;

Packages/StreamVideo/Runtime/Core/LowLevelClient/PublisherPeerConnection.cs

@@ -1,4 +1,4 @@
-#if UNITY_ANDROID && !UNITY_EDITOR
+#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
 #define STREAM_NATIVE_AUDIO //Defined in multiple files
 #endif
 using System;

Packages/StreamVideo/Runtime/Core/LowLevelClient/RtcSession.cs

@@ -1,4 +1,4 @@
-#if UNITY_ANDROID && !UNITY_EDITOR
+#if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR
 #define STREAM_NATIVE_AUDIO //Defined in multiple files
 #endif
 using System;
@@ -67,7 +67,6 @@ internal class RtcSession : IMediaInputProvider, ISfuClient, IDisposable
 
         // Some sources claim the 48kHz is the most optimal sample rate for WebRTC, other cause internal resampling
         public const int AudioOutputSampleRate = 48_000;
-        public const int AudioOutputChannels = 2;
 
 #if STREAM_NATIVE_AUDIO
         public const bool UseNativeAudioBindings = true;
@@ -588,7 +587,16 @@ public async Task DoJoin(JoinCallData joinCallData, CancellationToken cancellati
 
                     //StreamTODO: if we try to rejoin a call with no other participants we'll get error from SFU not call FOUND
                     // What should we do then?
-
+
+                    if (ActiveCall == null)
+                    {
+                        throw new Exception("ActiveCall should never be null here.");
+                    }
+
+                    if (joinResponse == null)
+                    {
+                        throw new Exception("joinResponse was null");
+                    }
 
                     ActiveCall.UpdateFromSfu(joinResponse);
                     _logs.WarningIfDebug($"{nameof(DoJoin)} - SFU Sending join response received. startNewPeerConnections: {startNewPeerConnections}");
@@ -660,7 +668,10 @@ public async Task DoJoin(JoinCallData joinCallData, CancellationToken cancellati
                 {
                     //StreamTODO: Either use UseNativeAudioBindings const or STREAM_NATIVE_AUDIO flag but not both. Once we replace the webRTC package we could remove STREAM_NATIVE_AUDIO
 #if STREAM_NATIVE_AUDIO
-                    WebRTC.StartAudioPlayback(AudioOutputSampleRate, AudioOutputChannels);
+                    // iOS VPIO requires PlayAndRecord before the audio unit opens (no automatic retry on '!rec').
+                    EnsureIOSAudioSessionReadyForVPIO($"{nameof(DoJoin)} StartAudioPlayback");
+
+                    WebRTC.StartAudioPlayback(AudioOutputSampleRate);
 #endif
                 }
 
@@ -675,6 +686,15 @@ public async Task DoJoin(JoinCallData joinCallData, CancellationToken cancellati
                 {
                     CallState = prevCallState;
                 }
+
+                try
+                {
+                    await ClearSessionAsync();
+                }
+                catch (Exception cleanupEx)
+                {
+                    _logs.Warning($"{nameof(DoJoin)} session cleanup after failure encountered an error: {cleanupEx.Message}");
+                }
 
                 throw;
             }
@@ -731,7 +751,18 @@ public Task StopAsync(string reason = "")
             if (UseNativeAudioBindings)
             {
 #if STREAM_NATIVE_AUDIO
+                // iOS order: stop playback + capture (closes the duplex VPIO unit),
+                // THEN deactivate AVAudioSession. Reverse order returns IsBusy.
                 WebRTC.StopAudioPlayback();
+
+#if UNITY_IOS && !UNITY_EDITOR
+                if (Publisher?.PublisherAudioTrack != null)
+                {
+                    Publisher.PublisherAudioTrack.StopLocalAudioCapture();
+                }
+
+                Libs.iOSAudioManagers.IOSAudioManager.DeconfigureAudioSession();
+#endif
 #endif
             }
 
@@ -882,37 +913,30 @@ public void SetAudioRecordingDevice(MicrophoneDeviceInfo device)
 
         public void TryRestartAudioPlayback()
         {
-            if (!UseNativeAudioBindings)
+            if (UseNativeAudioBindings)
             {
-                return;
-            }
 #if STREAM_NATIVE_AUDIO
-            WebRTC.StopAudioPlayback();
-            WebRTC.StartAudioPlayback(AudioOutputSampleRate, AudioOutputChannels);
+                WebRTC.StopAudioPlayback();
+                EnsureIOSAudioSessionReadyForVPIO(nameof(TryRestartAudioPlayback));
+                WebRTC.StartAudioPlayback(AudioOutputSampleRate);
 #endif
+            }
         }
 
-        //StreamTODO: temp solution to allow stopping the audio when app is minimized. User tried disabling the AudioSource but the audio is handled natively so it has no effect
-        public void PauseAndroidAudioPlayback()
+        // Mutes the AudioMixer so the playback callback writes silence to the speakers.
+        // The native audio device keeps running; on iOS this also keeps the VPIO audio
+        // unit alive so unmuting is instant and does not need a session reconfigure.
+        public void PauseMobileAudioPlayback()
         {
 #if STREAM_NATIVE_AUDIO
-            WebRTC.MuteAndroidAudioPlayback();
-            _logs.Warning("Audio Playback is paused. This stops all audio coming from StreamVideo SDK on Android platform.");
-#else
-            throw new NotSupportedException(
-                $"{nameof(PauseAndroidAudioPlayback)} is only supported on Android platform.");
+            WebRTC.MuteAudioPlayback();
 #endif
         }
 
-        //StreamTODO: temp solution to allow stopping the audio when app is minimized. User tried disabling the AudioSource but the audio is handled natively so it has no effect
-        public void ResumeAndroidAudioPlayback()
+        public void ResumeMobileAudioPlayback()
         {
 #if STREAM_NATIVE_AUDIO
-            WebRTC.UnmuteAndroidAudioPlayback();
-            _logs.Warning("Audio Playback is resumed. This resumes audio coming from StreamVideo SDK on Android platform.");
-#else
-            throw new NotSupportedException(
-                $"{nameof(ResumeAndroidAudioPlayback)} is only supported on Android platform.");
+            WebRTC.UnmuteAudioPlayback();
 #endif
         }
 
@@ -1416,13 +1440,53 @@ private void UpdateAudioRecording()
                 //StreamTODO: implement proper passing deviceID -> for Android and IOS we're skipping the deviceID
                 //because they operate on audio routing instead of actual devices. The underlying native implementation for Android let's OS pick the preferred device
 
+                // Configure session BEFORE capture so VPIO opens with HW AEC/NS/AGC from sample 0.
+                EnsureIOSAudioSessionReadyForVPIO($"{nameof(UpdateAudioRecording)} StartLocalAudioCapture");
+
                 _logs.WarningIfDebug("RtcSession.UpdateAudioRecording -> START local audio capture");
                 Publisher.PublisherAudioTrack.StartLocalAudioCapture(-1, AudioInputSampleRate);
+
+#if UNITY_IOS && !UNITY_EDITOR
+                if (!Libs.iOSAudioManagers.IOSAudioManager.IsHardwareNoiseCancellationActive)
+                {
+                    _logs.Warning(
+                        "RtcSession.UpdateAudioRecording -> iOS hardware noise cancellation (VoiceProcessingIO) NOT active. "
+                        + "AEC/NS/AGC will not be applied. Check that no other plugin overrode AVAudioSession mode.");
+                }
+#endif
             }
             else
             {
                 _logs.WarningIfDebug("RtcSession.UpdateAudioRecording -> STOP local audio capture");
                 Publisher.PublisherAudioTrack.StopLocalAudioCapture();
+
+                // Do NOT deactivate the AVAudioSession on mute: the duplex VPIO unit is
+                // shared with playback and would also stop, killing remote audio. The
+                // session is deactivated in StopAsync after both directions are torn down.
+            }
+#endif
+        }
+
+        /// <summary>
+        /// Configure the iOS AVAudioSession for VPIO (PlayAndRecord + VideoChat) before
+        /// opening the duplex audio unit. Verifies the result and retries once to catch
+        /// races with other components reasserting the session. No-op on non-iOS targets.
+        /// </summary>
+        private void EnsureIOSAudioSessionReadyForVPIO(string callerContext)
+        {
+#if UNITY_IOS && !UNITY_EDITOR
+            var configureOk = Libs.iOSAudioManagers.IOSAudioManager.ConfigureForWebRTC();
+            if (configureOk && Libs.iOSAudioManagers.IOSAudioManager.IsHardwareNoiseCancellationActive)
+            {
+                return;
+            }
+
+            configureOk = Libs.iOSAudioManagers.IOSAudioManager.ConfigureForWebRTC();
+            if (!configureOk || !Libs.iOSAudioManagers.IOSAudioManager.IsHardwareNoiseCancellationActive)
+            {
+                _logs.Error(
+                    $"RtcSession.{nameof(EnsureIOSAudioSessionReadyForVPIO)} ({callerContext}): session not VPIO-ready after retry. "
+                    + "Next StartAudio call may fail with '!rec'. See [StreamVideo iOS Audio] device log for NSError details.");
             }
 #endif
         }