diff --git a/src/client/helper.rs b/src/client/helper.rs
index 005b2df72..248cf5928 100644
--- a/src/client/helper.rs
+++ b/src/client/helper.rs
@@ -18,7 +18,7 @@ pub struct LatencyController {
     last_video_remote_ts: i64, // generated on remote device
     update_time: Instant,
     allow_audio: bool,
-    enabled: bool
+    audio_only: bool
 }
 
 impl Default for LatencyController {
@@ -27,7 +27,7 @@ impl Default for LatencyController {
             last_video_remote_ts: Default::default(),
             update_time: Instant::now(),
             allow_audio: Default::default(),
-            enabled: true
+            audio_only: true
         }
     }
 }
@@ -38,9 +38,9 @@ impl LatencyController {
         Arc::new(Mutex::new(LatencyController::default()))
     }
 
-    /// Set whether this [LatencyController] should be enabled.
-    pub fn set_enabled(&mut self, enable: bool) {
-        self.enabled = enable;
+    /// Set whether this [LatencyController] should be working in audio only mode.
+    pub fn set_audio_only(&mut self, only: bool) {
+        self.audio_only = only;
     }
 
     /// Update the latency controller with the latest video timestamp.
@@ -51,10 +51,6 @@ impl LatencyController {
 
     /// Check if the audio should be played based on the current latency.
     pub fn check_audio(&mut self, timestamp: i64) -> bool {
-        if !self.enabled {
-            self.allow_audio = true;
-            return self.allow_audio;
-        }
         // Compute audio latency.
         let expected = self.update_time.elapsed().as_millis() as i64 + self.last_video_remote_ts;
         let latency = expected - timestamp;
@@ -70,6 +66,11 @@ impl LatencyController {
                 self.allow_audio = true;
             }
         }
+        // No video frame here, which means the update time is not triggered.
+        // We manually update the time here.
+        if self.audio_only {
+            self.update_time = Instant::now();
+        }
         self.allow_audio
     }
 }
diff --git a/src/server/connection.rs b/src/server/connection.rs
index 1924cfca0..d5c2103b1 100644
--- a/src/server/connection.rs
+++ b/src/server/connection.rs
@@ -172,7 +172,7 @@ impl Connection {
         // Start a audio thread to play the audio sent by peer.
         let latency_controller = LatencyController::new();
         // No video frame will be sent here, so we need to disable latency controller, or audio check may fail.
-        latency_controller.lock().unwrap().set_enabled(false);
+        latency_controller.lock().unwrap().set_audio_only(true);
         let audio_sender = start_audio_thread(Some(latency_controller));
         let mut conn = Self {
             inner: ConnInner {