video service 0.1

2025-10-29 17:00:05 +00:00 · 2022-01-20 15:57:54 +08:00
parent eeb30aa0d1
commit 668b34c228
8 changed files with 542 additions and 16 deletions
--- a/android/app/build.gradle
+++ b/android/app/build.gradle
@@ -33,7 +33,7 @@ apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"

 android {
    compileSdkVersion 30
-
+    // ndkVersion '22.1.7171670' // * 仅个人使用 存在多版本NDK无法自动选择 需要使用此配置指定NDK版本 [CSF]
    sourceSets {
        main.java.srcDirs += 'src/main/kotlin'
    }
--- a/android/app/src/main/AndroidManifest.xml
+++ b/android/app/src/main/AndroidManifest.xml
@@ -28,17 +28,24 @@
              android:resource="@drawable/launch_background"
              />
            <intent-filter>
-                <action android:name="android.intent.action.MAIN"/>
-                <category android:name="android.intent.category.LAUNCHER"/>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>
        </activity>
+        <service
+            android:name=".MainService"
+            android:enabled="true"
+            android:foregroundServiceType="mediaProjection"/>
        <!-- Don't delete the meta-data below.
             This is used by the Flutter tool to generate GeneratedPluginRegistrant.java -->
        <meta-data
            android:name="flutterEmbedding"
            android:value="2" />
-      </application>
-      <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
-      <uses-permission android:name="android.permission.INTERNET" />
-      <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+    </application>
+
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
 </manifest>
--- a/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt
+++ b/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt
@@ -1,6 +1,106 @@
 package com.carriez.flutter_hbb

+import android.app.Activity
+import android.content.Context
+import android.content.Intent
+import android.media.projection.MediaProjectionManager
+import android.os.Build
+import android.os.Bundle
+import android.os.PersistableBundle
+import android.util.Log
+import androidx.annotation.RequiresApi
 import io.flutter.embedding.android.FlutterActivity
+import io.flutter.embedding.engine.FlutterEngine
+import io.flutter.plugin.common.MethodCall
+import io.flutter.plugin.common.MethodChannel
+import java.nio.ByteBuffer
+import kotlin.concurrent.thread

-class MainActivity: FlutterActivity() {
+
+class MainActivity : FlutterActivity() {
+    private val channelTag = "mChannel"
+    private var mediaProjectionResultIntent: Intent? = null
+    private val requestCode = 1
+    private val buf = ByteBuffer.allocate(16)
+
+    init {
+        System.loadLibrary("rustdesk")
+    }
+
+    override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
+        super.configureFlutterEngine(flutterEngine) // 必要 否则无法正确初始化flutter
+
+        MethodChannel(
+            flutterEngine.dartExecutor.binaryMessenger,
+            channelTag
+        ).setMethodCallHandler { call, result ->
+            when (call.method) {
+                "getPer" -> {
+                    Log.d(channelTag, "event from flutter,getPer")
+                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+                        getMediaProjection()
+                    }
+                    result.success(true)
+                }
+                "startSer" -> {
+                    mStarService()
+                    result.success(true)
+                }
+                "stopSer" -> {
+                    mStopService()
+                    result.success(true)
+                }
+                else -> {}
+            }
+        }
+    }
+
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    private fun getMediaProjection() {
+        val mMediaProjectionManager =
+            getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
+        val mIntent = mMediaProjectionManager.createScreenCaptureIntent()
+        startActivityForResult(mIntent, requestCode)
+    }
+
+    private fun mStarService() {
+        if (mediaProjectionResultIntent == null) {
+            Log.w(channelTag, "mediaProjectionResultIntent is null")
+            return
+        }
+        Log.d(channelTag, "Start a service")
+        val serviceIntent = Intent(this, MainService::class.java)
+        serviceIntent.action = START_SERVICE
+        serviceIntent.putExtra(EXTRA_MP_DATA, mediaProjectionResultIntent)
+
+        // TEST api < O
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+            startForegroundService(serviceIntent)
+        } else {
+            startService(serviceIntent)
+        }
+    }
+
+    private fun mStopService() {
+        Log.d(channelTag, "Stop service")
+        val serviceIntent = Intent(this, MainService::class.java)
+
+        serviceIntent.action = STOP_SERVICE
+
+        // TEST api < O
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+            startForegroundService(serviceIntent)
+        } else {
+            startService(serviceIntent)
+        }
+    }
+
+    override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
+        super.onActivityResult(requestCode, resultCode, data)
+        if (resultCode == Activity.RESULT_OK && data != null) {
+            Log.d(channelTag, "got mediaProjectionResultIntent ok")
+            mediaProjectionResultIntent = data
+        }
+    }
 }
--- a/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt
+++ b/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt
@@ -0,0 +1,248 @@
+package com.carriez.flutter_hbb
+
+import android.annotation.SuppressLint
+import android.app.*
+import android.content.Context
+import android.content.Intent
+import android.graphics.Color
+import android.graphics.PixelFormat
+import android.hardware.display.DisplayManager.VIRTUAL_DISPLAY_FLAG_PUBLIC
+import android.media.*
+import android.media.projection.MediaProjection
+import android.media.projection.MediaProjectionManager
+import android.os.Build
+import android.os.IBinder
+import android.util.Log
+import android.view.Surface
+import android.widget.Toast
+import androidx.annotation.RequiresApi
+import androidx.core.app.NotificationCompat
+import androidx.core.app.NotificationCompat.PRIORITY_MIN
+import java.nio.ByteBuffer
+import java.util.concurrent.Executors
+
+const val EXTRA_MP_DATA = "mp_intent"
+const val START_SERVICE = "start_service"
+const val STOP_SERVICE = "stop_service"
+const val MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_VP9
+
+// 获取手机尺寸 建立连接时发送尺寸和基础信息
+const val FIXED_WIDTH = 500 // 编码器有上限
+const val FIXED_HEIGHT = 1000
+const val M_KEY_BIT_RATE = 1024_000
+const val M_KEY_FRAME_RATE = 30
+
+class MainService : Service() {
+
+    fun rustGetRaw():ByteArray{
+        return rawByteArray!!
+    }
+
+    external fun init(ctx:Context)
+
+    init {
+        System.loadLibrary("rustdesk")
+    }
+
+    private val logTag = "LOG_SERVICE"
+    private var mMediaProjection: MediaProjection? = null
+    private var surface: Surface? = null
+    private val singleThread = Executors.newSingleThreadExecutor()
+    private var mEncoder: MediaCodec? = null
+    private var rawByteArray :ByteArray? = null
+
+    override fun onBind(intent: Intent): IBinder? {
+        return null
+    }
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
+        Log.d("whichService", "this service:${Thread.currentThread()}")
+        init(this) // 注册到rust
+        if (intent?.action == START_SERVICE) {
+            Log.d(logTag, "service starting:${startId}:${Thread.currentThread()}")
+            createNotification()
+            val mMediaProjectionManager =
+                getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
+            mMediaProjection = intent.getParcelableExtra<Intent>(EXTRA_MP_DATA)?.let {
+                mMediaProjectionManager.getMediaProjection(Activity.RESULT_OK, it)
+            }
+            Log.d(logTag, "获取mMediaProjection成功$mMediaProjection")
+            if (testSupport()) {
+                startRecorder()
+            } else {
+                Toast.makeText(this, "此设备不支持:$MIME_TYPE", Toast.LENGTH_SHORT).show()
+                stopSelf(startId)
+            }
+        } else if (intent?.action == STOP_SERVICE) {
+            mEncoder?.let {
+                try {
+                    Log.d(logTag, "正在释放encoder")
+                    it.signalEndOfInputStream()
+                    it.stop()
+                    it.release()
+                } catch (e: Exception) {
+                    null
+                }
+            }
+            stopSelf()
+        }
+        return super.onStartCommand(intent, flags, startId)
+    }
+
+    lateinit var mImageReader:ImageReader // * 注意 这里要成为成员变量，防止被回收 https://www.cnblogs.com/yongdaimi/p/11004560.html
+    @SuppressLint("WrongConstant")
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    private fun startRecorder() {
+        Log.d(logTag, "startRecorder")
+        mMediaProjection?.let { mp ->
+            // 使用原始数据
+            mImageReader =
+                ImageReader.newInstance(FIXED_WIDTH, FIXED_HEIGHT, PixelFormat.RGBA_8888, 2) // 至少是2
+            mImageReader.setOnImageAvailableListener({ imageReader: ImageReader ->
+                Log.d(logTag, "on image")
+                    try {
+                        imageReader.acquireLatestImage().use { image ->
+                            if (image == null) return@setOnImageAvailableListener
+                            val planes = image.planes
+                            val buffer = planes[0].buffer
+                            buffer.rewind()
+                            // 这里注意 处理不当会引发OOM
+                            if (rawByteArray == null){
+                                rawByteArray = ByteArray(buffer.capacity())
+                                buffer.get(rawByteArray!!)
+                            }else{
+                                buffer.get(rawByteArray!!)
+                            }
+                        }
+                    } catch (ignored: java.lang.Exception) {
+                    }
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+                    imageReader.discardFreeBuffers()
+                }
+            }, null)
+            mp.createVirtualDisplay(
+                "rustdesk test",
+                FIXED_WIDTH, FIXED_HEIGHT, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
+                mImageReader.surface, null, null
+            )
+
+
+            // 使用内置编码器
+//            createMediaCodec()
+//            mEncoder?.let {
+//                surface = it.createInputSurface()
+//                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+//                    surface!!.setFrameRate(1F, FRAME_RATE_COMPATIBILITY_DEFAULT)
+//                }
+//                it.setCallback(cb)
+//                it.start()
+//                mp.createVirtualDisplay(
+//                    "rustdesk test",
+//                    FIXED_WIDTH, FIXED_HEIGHT, 200, VIRTUAL_DISPLAY_FLAG_PUBLIC,
+//                    surface, null, null
+//                )
+//            }
+        } ?: let {
+            Log.d(logTag, "startRecorder fail,mMediaProjection is null")
+        }
+    }
+
+    private val cb: MediaCodec.Callback = @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    object : MediaCodec.Callback() {
+        override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {}
+        override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {}
+
+        override fun onOutputBufferAvailable(
+            codec: MediaCodec,
+            index: Int,
+            info: MediaCodec.BufferInfo
+        ) {
+            codec.getOutputBuffer(index)?.let { buf ->
+                singleThread.execute {
+                    // TODO 优化内存使用方式
+                    val byteArray = ByteArray(buf.limit())
+                    buf.get(byteArray)
+                    sendVp9(byteArray)
+                    codec.releaseOutputBuffer(index, false)
+                }
+            }
+        }
+
+        override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
+            Log.e(logTag, "MediaCodec.Callback error:$e")
+        }
+    }
+
+    external fun sendRaw(buf: ByteBuffer)
+    external fun sendVp9(data: ByteArray)
+
+    @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
+    private fun testSupport(): Boolean {
+        val res = MediaCodecList(MediaCodecList.ALL_CODECS)
+            .findEncoderForFormat(
+                MediaFormat.createVideoFormat(
+                    MediaFormat.MIMETYPE_VIDEO_VP9,
+                    FIXED_WIDTH,
+                    FIXED_HEIGHT
+                )
+            )
+        return res?.let {
+            true
+        } ?: let {
+            false
+        }
+    }
+
+    private fun createMediaCodec() {
+        Log.d(logTag, "MediaFormat.MIMETYPE_VIDEO_VP9 :$MIME_TYPE")
+        mEncoder = MediaCodec.createEncoderByType(MIME_TYPE)
+        val mFormat = MediaFormat.createVideoFormat(MIME_TYPE, FIXED_WIDTH, FIXED_HEIGHT)
+        mFormat.setInteger(MediaFormat.KEY_BIT_RATE, M_KEY_BIT_RATE)
+        mFormat.setInteger(MediaFormat.KEY_FRAME_RATE, M_KEY_FRAME_RATE) // codec的帧率设置无效
+        mFormat.setInteger(
+            MediaFormat.KEY_COLOR_FORMAT,
+            MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible
+        )
+        mFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5)
+        try {
+            mEncoder!!.configure(mFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
+        } catch (e: Exception) {
+            Log.e(logTag, "mEncoder.configure fail!")
+        }
+    }
+
+    private fun createNotification() {
+        val channelId =
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                createNotificationChannel("my_service", "My Background Service")
+            } else {
+                ""
+            }
+        val notification: Notification = NotificationCompat.Builder(this, channelId)
+            .setOngoing(true)
+            .setContentTitle("Hello")
+            .setPriority(PRIORITY_MIN)
+            .setContentText("TEST TEST")
+            .build()
+        startForeground(11, notification)
+    }
+
+    @RequiresApi(Build.VERSION_CODES.O)
+    private fun createNotificationChannel(channelId: String, channelName: String): String {
+        val chan = NotificationChannel(
+            channelId,
+            channelName, NotificationManager.IMPORTANCE_NONE
+        )
+        chan.lightColor = Color.BLUE
+        chan.lockscreenVisibility = Notification.VISIBILITY_PRIVATE
+        val service = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
+        service.createNotificationChannel(chan)
+        return channelId
+    }
+
+    override fun onDestroy() {
+        Log.d(logTag, "service stop:${Thread.currentThread()}")
+        Toast.makeText(this, "service done", Toast.LENGTH_SHORT).show()
+    }
+}