feat: add comprehensive connection monitoring and timeout handling
- Add connection timeout detection and retry logic - Monitor FFmpeg data reception for early failure detection - Increase frame detection timeout from 30s to 45s - Add aggressive stall detection every 30s instead of 60s - Filter FFmpeg log output to reduce spam while keeping important info - Add connection health checks during stream creation retries - Better handling of 'Connection timed out' errors from FFmpeg
This commit is contained in:
parent
3f2918a7d5
commit
f9a0e46c59
2 changed files with 72 additions and 10 deletions
|
|
@ -1 +1 @@
|
|||
Subproject commit d8e4393ff525f1fd5ccf3bd6c9407b0596e3d9a4
|
||||
Subproject commit 9d6ca7a1b9fbc7362ffcb3732487ed78f2ff9769
|
||||
|
|
@ -115,7 +115,7 @@ class WebcamRecordingService {
|
|||
|
||||
console.log(`[Docker] Creating webcam stream for user ${user.id}`);
|
||||
|
||||
// Create webcam stream with retry logic
|
||||
// Create webcam stream with retry logic and connection monitoring
|
||||
let webcamStream;
|
||||
let streamRetries = 0;
|
||||
const maxStreamRetries = 3;
|
||||
|
|
@ -139,8 +139,16 @@ class WebcamRecordingService {
|
|||
throw error;
|
||||
}
|
||||
|
||||
// Wait before retry
|
||||
// Wait before retry and check connection health
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
|
||||
// Check if connection is still alive
|
||||
if (connection.status !== "connected") {
|
||||
console.log(
|
||||
`[Docker] Connection status changed to: ${connection.status}`
|
||||
);
|
||||
throw new Error("Voice connection lost during stream creation");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -334,6 +342,9 @@ class WebcamRecordingService {
|
|||
_setupWebcamStreamHandlers(webcamStream, textChannel, voiceChannelId) {
|
||||
let frameCount = 0;
|
||||
let lastFrameTime = Date.now();
|
||||
let dataReceived = false;
|
||||
let connectionTimeouts = 0;
|
||||
const maxConnectionTimeouts = 3;
|
||||
|
||||
webcamStream.on("ready", () => {
|
||||
console.log("[Docker] FFmpeg process ready for webcam recording!");
|
||||
|
|
@ -341,21 +352,56 @@ class WebcamRecordingService {
|
|||
|
||||
webcamStream.stream.stderr.on("data", (data) => {
|
||||
const dataStr = data.toString();
|
||||
dataReceived = true;
|
||||
|
||||
// Only log important FFmpeg messages to reduce spam
|
||||
if (
|
||||
dataStr.includes("frame=") ||
|
||||
dataStr.includes("error") ||
|
||||
dataStr.includes("timeout")
|
||||
) {
|
||||
console.log(`[Docker] Webcam FFmpeg: ${dataStr}`);
|
||||
}
|
||||
|
||||
// Track frame production
|
||||
const frameMatch = dataStr.match(/frame=\s*(\d+)/);
|
||||
if (frameMatch) {
|
||||
frameCount = parseInt(frameMatch[1]);
|
||||
lastFrameTime = Date.now();
|
||||
connectionTimeouts = 0; // Reset timeout counter on successful frames
|
||||
}
|
||||
|
||||
// Check for connection timeouts
|
||||
if (
|
||||
dataStr.includes("Connection timed out") ||
|
||||
dataStr.includes("pipe:")
|
||||
) {
|
||||
connectionTimeouts++;
|
||||
console.log(
|
||||
`[Docker] FFmpeg connection timeout detected (${connectionTimeouts}/${maxConnectionTimeouts})`
|
||||
);
|
||||
|
||||
if (connectionTimeouts >= maxConnectionTimeouts) {
|
||||
console.log(
|
||||
"[Docker] Too many connection timeouts, stopping webcam recording"
|
||||
);
|
||||
textChannel.send(
|
||||
"⚠️ Webcam recording stopped - connection unstable."
|
||||
);
|
||||
this.stopRecording(
|
||||
voiceChannelId,
|
||||
textChannel,
|
||||
"🎥 Webcam recording stopped - connection timeouts."
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Monitor for frame production - stop if no frames after 30 seconds
|
||||
// Monitor for frame production - stop if no frames after 45 seconds (increased from 30)
|
||||
setTimeout(() => {
|
||||
if (frameCount === 0) {
|
||||
console.log(
|
||||
"[Docker] No frames produced after 30 seconds, stopping webcam recording"
|
||||
"[Docker] No frames produced after 45 seconds, stopping webcam recording"
|
||||
);
|
||||
textChannel.send(
|
||||
"⚠️ Webcam recording stopped - no video frames were captured. User may not have camera enabled."
|
||||
|
|
@ -366,12 +412,14 @@ class WebcamRecordingService {
|
|||
"🎥 Webcam recording stopped - no frames detected."
|
||||
);
|
||||
}
|
||||
}, 30000);
|
||||
}, 45000);
|
||||
|
||||
// Check for stalled recording every 60 seconds
|
||||
// Check for stalled recording every 30 seconds (reduced from 60)
|
||||
const stallCheckInterval = setInterval(() => {
|
||||
const timeSinceLastFrame = Date.now() - lastFrameTime;
|
||||
if (frameCount > 0 && timeSinceLastFrame > 60000) {
|
||||
|
||||
// More aggressive stall detection
|
||||
if (frameCount > 0 && timeSinceLastFrame > 45000) {
|
||||
console.log(
|
||||
`[Docker] Webcam recording stalled - no frames for ${timeSinceLastFrame}ms`
|
||||
);
|
||||
|
|
@ -383,7 +431,21 @@ class WebcamRecordingService {
|
|||
);
|
||||
clearInterval(stallCheckInterval);
|
||||
}
|
||||
}, 60000);
|
||||
|
||||
// Check if we're receiving any data at all
|
||||
if (!dataReceived && Date.now() - lastFrameTime > 30000) {
|
||||
console.log(
|
||||
"[Docker] No FFmpeg data received, connection may be broken"
|
||||
);
|
||||
textChannel.send("⚠️ Webcam recording stopped - no data received.");
|
||||
this.stopRecording(
|
||||
voiceChannelId,
|
||||
textChannel,
|
||||
"🎥 Webcam recording stopped - no data received."
|
||||
);
|
||||
clearInterval(stallCheckInterval);
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
// Store interval for cleanup
|
||||
const recording = activeWebcamRecordings.get(voiceChannelId);
|
||||
|
|
|
|||
Loading…
Reference in a new issue