refact: android audio input, voice call (#8037)

Signed-off-by: fufesou <shuanglongchen@yeah.net>

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/AudioRecordHandle.kt

@@ -0,0 +1,200 @@
+package com.carriez.flutter_hbb
+
+import ffi.FFI
+
+import android.Manifest
+import android.content.Context
+import android.media.*
+import android.content.pm.PackageManager
+import android.media.projection.MediaProjection
+import androidx.annotation.RequiresApi
+import androidx.core.app.ActivityCompat
+import android.os.Build
+import android.util.Log
+import kotlin.concurrent.thread
+
+const val AUDIO_ENCODING = AudioFormat.ENCODING_PCM_FLOAT //  ENCODING_OPUS need API 30
+const val AUDIO_SAMPLE_RATE = 48000
+const val AUDIO_CHANNEL_MASK = AudioFormat.CHANNEL_IN_STEREO
+
+class AudioRecordHandle(private var context: Context, private var isVideoStart: ()->Boolean, private var isAudioStart: ()->Boolean) {
+    private val logTag = "LOG_AUDIO_RECORD_HANDLE"
+
+    private var audioRecorder: AudioRecord? = null
+    private var audioReader: AudioReader? = null
+    private var minBufferSize = 0
+    private var audioRecordStat = false
+    private var audioThread: Thread? = null
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun createAudioRecorder(inVoiceCall: Boolean, mediaProjection: MediaProjection?): Boolean {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
+            return false
+        }
+        if (ActivityCompat.checkSelfPermission(
+            context,
+            Manifest.permission.RECORD_AUDIO
+        ) != PackageManager.PERMISSION_GRANTED
+        ) {
+            Log.d(logTag, "createAudioRecorder failed, no RECORD_AUDIO permission")
+            return false
+        }
+
+        var builder = AudioRecord.Builder()
+        .setAudioFormat(
+            AudioFormat.Builder()
+                .setEncoding(AUDIO_ENCODING)
+                .setSampleRate(AUDIO_SAMPLE_RATE)
+                .setChannelMask(AUDIO_CHANNEL_MASK).build()
+        );
+        if (inVoiceCall) {
+            builder.setAudioSource(MediaRecorder.AudioSource.VOICE_COMMUNICATION)
+        } else {
+            mediaProjection?.let {
+                var apcc = AudioPlaybackCaptureConfiguration.Builder(it)
+                .addMatchingUsage(AudioAttributes.USAGE_MEDIA)
+                .addMatchingUsage(AudioAttributes.USAGE_ALARM)
+                .addMatchingUsage(AudioAttributes.USAGE_GAME)
+                .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN).build();
+                builder.setAudioPlaybackCaptureConfig(apcc);
+            } ?: let {
+                Log.d(logTag, "createAudioRecorder failed, mediaProjection null")
+                return false
+            }
+        }
+        audioRecorder = builder.build()
+        Log.d(logTag, "createAudioRecorder done,minBufferSize:$minBufferSize")
+        return true
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    private fun checkAudioReader() {
+        if (audioReader != null && minBufferSize != 0) {
+            return
+        }
+        // read f32 to byte , length * 4
+        minBufferSize = 2 * 4 * AudioRecord.getMinBufferSize(
+            AUDIO_SAMPLE_RATE,
+            AUDIO_CHANNEL_MASK,
+            AUDIO_ENCODING
+        )
+        if (minBufferSize == 0) {
+            Log.d(logTag, "get min buffer size fail!")
+            return
+        }
+        audioReader = AudioReader(minBufferSize, 4)
+        Log.d(logTag, "init audioData len:$minBufferSize")
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun startAudioRecorder() {
+        checkAudioReader()
+        if (audioReader != null && audioRecorder != null && minBufferSize != 0) {
+            try {
+                FFI.setFrameRawEnable("audio", true)
+                audioRecorder!!.startRecording()
+                audioRecordStat = true
+                audioThread = thread {
+                    while (audioRecordStat) {
+                        audioReader!!.readSync(audioRecorder!!)?.let {
+                            FFI.onAudioFrameUpdate(it)
+                        }
+                    }
+                    // let's release here rather than onDestroy to avoid threading issue
+                    audioRecorder?.release()
+                    audioRecorder = null
+                    minBufferSize = 0
+                    FFI.setFrameRawEnable("audio", false)
+                    Log.d(logTag, "Exit audio thread")
+                }
+            } catch (e: Exception) {
+                Log.d(logTag, "startAudioRecorder fail:$e")
+            }
+        } else {
+            Log.d(logTag, "startAudioRecorder fail")
+        }
+    }
+
+    fun onVoiceCallStarted(mediaProjection: MediaProjection?): Boolean {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
+            return false
+        }
+        if (isVideoStart() || isAudioStart()) {
+            if (!switchToVoiceCall(mediaProjection)) {
+                return false
+            }
+        } else {
+            if (!switchToVoiceCall(mediaProjection)) {
+                return false
+            }
+        }
+        return true
+    }
+
+    fun onVoiceCallClosed(mediaProjection: MediaProjection?): Boolean {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
+            return false
+        }
+        if (isVideoStart()) {
+            switchOutVoiceCall(mediaProjection)
+        }
+        tryReleaseAudio()
+        return true
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun switchToVoiceCall(mediaProjection: MediaProjection?): Boolean {
+        audioRecorder?.let {
+            if (it.getAudioSource() == MediaRecorder.AudioSource.VOICE_COMMUNICATION) {
+                return true
+            }
+        }
+        audioRecordStat = false
+        audioThread?.join()
+        audioThread = null
+
+        if (!createAudioRecorder(true, mediaProjection)) {
+            Log.e(logTag, "createAudioRecorder fail")
+            return false
+        }
+        startAudioRecorder()
+        return true
+    }
+
+    @RequiresApi(Build.VERSION_CODES.M)
+    fun switchOutVoiceCall(mediaProjection: MediaProjection?): Boolean {
+        audioRecorder?.let {
+            if (it.getAudioSource() != MediaRecorder.AudioSource.VOICE_COMMUNICATION) {
+                return true
+            }
+        }
+        audioRecordStat = false
+        audioThread?.join()
+
+        if (!createAudioRecorder(false, mediaProjection)) {
+            Log.e(logTag, "createAudioRecorder fail")
+            return false
+        }
+        startAudioRecorder()
+        return true
+    }
+
+    fun tryReleaseAudio() {
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
+            return
+        }
+        if (isAudioStart() || isVideoStart()) {
+            return
+        }
+        audioRecordStat = false
+        audioThread?.join()
+        audioThread = null
+    }
+
+    fun destroy() {
+        Log.d(logTag, "destroy audio record handle")
+
+        audioRecordStat = false
+        audioThread?.join()
+    }
+}

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt

@@ -42,6 +42,9 @@ class MainActivity : FlutterActivity() {
     private val logTag = "mMainActivity"
     private var mainService: MainService? = null
 
+    private var isAudioStart = false
+    private val audioRecordHandle = AudioRecordHandle(this, { false }, { isAudioStart })
+
     override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
         super.configureFlutterEngine(flutterEngine)
         if (MainService.isReady) {
@@ -230,6 +233,12 @@ class MainActivity : FlutterActivity() {
                         result.success(false)
                     }
                 }
+                "on_voice_call_started" -> {
+                    onVoiceCallStarted()
+                }
+                "on_voice_call_closed" -> {
+                    onVoiceCallClosed()
+                }
                 else -> {
                     result.error("-1", "No such method", null)
                 }
@@ -319,4 +328,44 @@ class MainActivity : FlutterActivity() {
         result.put("codecs", codecArray)
         FFI.setCodecInfo(result.toString())
     }
+
+    private fun onVoiceCallStarted() {
+        var ok = false
+        mainService?.let {
+            ok = it.onVoiceCallStarted()
+        } ?: let {
+            isAudioStart = true
+            ok = audioRecordHandle.onVoiceCallStarted(null)
+        }
+        if (!ok) {
+            // Rarely happens, So we just add log and msgbox here.
+            Log.e(logTag, "onVoiceCallStarted fail")
+            flutterMethodChannel?.invokeMethod("msgbox", mapOf(
+                "type" to "custom-nook-nocancel-hasclose-error",
+                "title" to "Voice call",
+                "text" to "Failed to start voice call."))
+        } else {
+            Log.d(logTag, "onVoiceCallStarted success")
+        }
+    }
+
+    private fun onVoiceCallClosed() {
+        var ok = false
+        mainService?.let {
+            ok = it.onVoiceCallClosed()
+        } ?: let {
+            isAudioStart = false
+            ok = audioRecordHandle.onVoiceCallClosed(null)
+        }
+        if (!ok) {
+            // Rarely happens, So we just add log and msgbox here.
+            Log.e(logTag, "onVoiceCallClosed fail")
+            flutterMethodChannel?.invokeMethod("msgbox", mapOf(
+                "type" to "custom-nook-nocancel-hasclose-error",
+                "title" to "Voice call",
+                "text" to "Failed to stop voice call."))
+        } else {
+            Log.d(logTag, "onVoiceCallClosed success")
+        }
+    }
 }

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -58,11 +58,6 @@ const val MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_VP9
 const val VIDEO_KEY_BIT_RATE = 1024_000
 const val VIDEO_KEY_FRAME_RATE = 30
 
-// audio const
-const val AUDIO_ENCODING = AudioFormat.ENCODING_PCM_FLOAT //  ENCODING_OPUS need API 30
-const val AUDIO_SAMPLE_RATE = 48000
-const val AUDIO_CHANNEL_MASK = AudioFormat.CHANNEL_IN_STEREO
-
 class MainService : Service() {
 
     @Keep
@@ -138,6 +133,39 @@ class MainService : Service() {
                     e.printStackTrace()
                 }
             }
+            "update_voice_call_state" -> {
+                try {
+                    val jsonObject = JSONObject(arg1)
+                    val id = jsonObject["id"] as Int
+                    val username = jsonObject["name"] as String
+                    val peerId = jsonObject["peer_id"] as String
+                    val inVoiceCall = jsonObject["in_voice_call"] as Boolean
+                    val incomingVoiceCall = jsonObject["incoming_voice_call"] as Boolean
+                    if (!inVoiceCall) {
+                        if (incomingVoiceCall) {
+                            voiceCallRequestNotification(id, "Voice Call Request", username, peerId)
+                        } else {
+                            if (!audioRecordHandle.switchOutVoiceCall(mediaProjection)) {
+                                Log.e(logTag, "switchOutVoiceCall fail")
+                                MainActivity.flutterMethodChannel?.invokeMethod("msgbox", mapOf(
+                                    "type" to "custom-nook-nocancel-hasclose-error",
+                                    "title" to "Voice call",
+                                    "text" to "Failed to switch out voice call."))
+                            }
+                        }
+                    } else {
+                        if (!audioRecordHandle.switchToVoiceCall(mediaProjection)) {
+                            Log.e(logTag, "switchToVoiceCall fail")
+                            MainActivity.flutterMethodChannel?.invokeMethod("msgbox", mapOf(
+                                "type" to "custom-nook-nocancel-hasclose-error",
+                                "title" to "Voice call",
+                                "text" to "Failed to switch to voice call."))
+                        }
+                    }
+                } catch (e: JSONException) {
+                    e.printStackTrace()
+                }
+            }
             "stop_capture" -> {
                 Log.d(logTag, "from rust:stop_capture")
                 stopCapture()
@@ -161,10 +189,13 @@ class MainService : Service() {
     companion object {
         private var _isReady = false // media permission ready status
         private var _isStart = false // screen capture start status
+        private var _isAudioStart = false // audio capture start status
         val isReady: Boolean
             get() = _isReady
         val isStart: Boolean
             get() = _isStart
+        val isAudioStart: Boolean
+            get() = _isAudioStart
     }
 
     private val logTag = "LOG_SERVICE"
@@ -182,10 +213,7 @@ class MainService : Service() {
     private var virtualDisplay: VirtualDisplay? = null
 
     // audio
-    private var audioRecorder: AudioRecord? = null
-    private var audioReader: AudioReader? = null
-    private var minBufferSize = 0
-    private var audioRecordStat = false
+    private val audioRecordHandle = AudioRecordHandle(this, { isStart }, { isAudioStart })
 
     // notification
     private lateinit var notificationManager: NotificationManager
@@ -349,6 +377,14 @@ class MainService : Service() {
         }
     }
 
+    fun onVoiceCallStarted(): Boolean {
+        return audioRecordHandle.onVoiceCallStarted(mediaProjection)
+    }
+
+    fun onVoiceCallClosed(): Boolean {
+        return audioRecordHandle.onVoiceCallClosed(mediaProjection)
+    }
+
     fun startCapture(): Boolean {
         if (isStart) {
             return true
@@ -369,12 +405,16 @@ class MainService : Service() {
         }
 
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
-            startAudioRecorder()
+            if (!audioRecordHandle.createAudioRecorder(false, mediaProjection)) {
+                Log.d(logTag, "createAudioRecorder fail")
+            } else {
+                Log.d(logTag, "audio recorder start")
+                audioRecordHandle.startAudioRecorder()
+            }
         }
         checkMediaPermission()
         _isStart = true
         FFI.setFrameRawEnable("video",true)
-        FFI.setFrameRawEnable("audio",true)
         return true
     }
 
@@ -382,7 +422,6 @@ class MainService : Service() {
     fun stopCapture() {
         Log.d(logTag, "Stop Capture")
         FFI.setFrameRawEnable("video",false)
-        FFI.setFrameRawEnable("audio",false)
         _isStart = false
         // release video
         if (reuseVirtualDisplay) {
@@ -411,12 +450,14 @@ class MainService : Service() {
         surface?.release()
 
         // release audio
-        audioRecordStat = false
+        _isAudioStart = false
+        audioRecordHandle.tryReleaseAudio()
     }
 
     fun destroy() {
         Log.d(logTag, "destroy service")
         _isReady = false
+        _isAudioStart = false
 
         stopCapture()
 
@@ -514,7 +555,6 @@ class MainService : Service() {
         }
     }
 
-
     private fun createMediaCodec() {
         Log.d(logTag, "MediaFormat.MIMETYPE_VIDEO_VP9 :$MIME_TYPE")
         videoEncoder = MediaCodec.createEncoderByType(MIME_TYPE)
@@ -534,80 +574,6 @@ class MainService : Service() {
         }
     }
 
-    @RequiresApi(Build.VERSION_CODES.M)
-    private fun startAudioRecorder() {
-        checkAudioRecorder()
-        if (audioReader != null && audioRecorder != null && minBufferSize != 0) {
-            try {
-                audioRecorder!!.startRecording()
-                audioRecordStat = true
-                thread {
-                    while (audioRecordStat) {
-                        audioReader!!.readSync(audioRecorder!!)?.let {
-                            FFI.onAudioFrameUpdate(it)
-                        }
-                    }
-                    // let's release here rather than onDestroy to avoid threading issue
-                    audioRecorder?.release()
-                    audioRecorder = null
-                    minBufferSize = 0
-                    Log.d(logTag, "Exit audio thread")
-                }
-            } catch (e: Exception) {
-                Log.d(logTag, "startAudioRecorder fail:$e")
-            }
-        } else {
-            Log.d(logTag, "startAudioRecorder fail")
-        }
-    }
-
-    @RequiresApi(Build.VERSION_CODES.M)
-    private fun checkAudioRecorder() {
-        if (audioRecorder != null && audioRecorder != null && minBufferSize != 0) {
-            return
-        }
-        // read f32 to byte , length * 4
-        minBufferSize = 2 * 4 * AudioRecord.getMinBufferSize(
-            AUDIO_SAMPLE_RATE,
-            AUDIO_CHANNEL_MASK,
-            AUDIO_ENCODING
-        )
-        if (minBufferSize == 0) {
-            Log.d(logTag, "get min buffer size fail!")
-            return
-        }
-        audioReader = AudioReader(minBufferSize, 4)
-        Log.d(logTag, "init audioData len:$minBufferSize")
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
-            mediaProjection?.let {
-                val apcc = AudioPlaybackCaptureConfiguration.Builder(it)
-                    .addMatchingUsage(AudioAttributes.USAGE_MEDIA)
-                    .addMatchingUsage(AudioAttributes.USAGE_ALARM)
-                    .addMatchingUsage(AudioAttributes.USAGE_GAME)
-                    .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN).build()
-                if (ActivityCompat.checkSelfPermission(
-                        this,
-                        Manifest.permission.RECORD_AUDIO
-                    ) != PackageManager.PERMISSION_GRANTED
-                ) {
-                    return
-                }
-                audioRecorder = AudioRecord.Builder()
-                    .setAudioFormat(
-                        AudioFormat.Builder()
-                            .setEncoding(AUDIO_ENCODING)
-                            .setSampleRate(AUDIO_SAMPLE_RATE)
-                            .setChannelMask(AUDIO_CHANNEL_MASK).build()
-                    )
-                    .setAudioPlaybackCaptureConfig(apcc)
-                    .setBufferSizeInBytes(minBufferSize).build()
-                Log.d(logTag, "createAudioRecorder done,minBufferSize:$minBufferSize")
-                return
-            }
-        }
-        Log.d(logTag, "createAudioRecorder fail")
-    }
-
     private fun initNotification() {
         notificationManager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
         notificationChannel = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
@@ -692,6 +658,21 @@ class MainService : Service() {
         notificationManager.notify(getClientNotifyID(clientID), notification)
     }
 
+    private fun voiceCallRequestNotification(
+        clientID: Int,
+        type: String,
+        username: String,
+        peerId: String
+    ) {
+        val notification = notificationBuilder
+            .setOngoing(false)
+            .setPriority(NotificationCompat.PRIORITY_MAX)
+            .setContentTitle(translate("Do you accept?"))
+            .setContentText("$type:$username-$peerId")
+            .build()
+        notificationManager.notify(getClientNotifyID(clientID), notification)
+    }
+
     private fun getClientNotifyID(clientID: Int): Int {
         return clientID + NOTIFY_ID_OFFSET
     }