feat: add comprehensive connection monitoring and timeout handling

- Add connection timeout detection and retry logic
- Monitor FFmpeg data reception for early failure detection
- Increase frame detection timeout from 30s to 45s
- Add aggressive stall detection every 30s instead of 60s
- Filter FFmpeg log output to reduce spam while keeping important info
- Add connection health checks during stream creation retries
- Better handling of 'Connection timed out' errors from FFmpeg
This commit is contained in:
Mikolaj Wojciech Gorski 2025-07-26 17:01:59 +02:00
parent 3f2918a7d5
commit f9a0e46c59
2 changed files with 72 additions and 10 deletions

@ -1 +1 @@
Subproject commit d8e4393ff525f1fd5ccf3bd6c9407b0596e3d9a4
Subproject commit 9d6ca7a1b9fbc7362ffcb3732487ed78f2ff9769

View file

@ -115,7 +115,7 @@ class WebcamRecordingService {
console.log(`[Docker] Creating webcam stream for user ${user.id}`);
// Create webcam stream with retry logic
// Create webcam stream with retry logic and connection monitoring
let webcamStream;
let streamRetries = 0;
const maxStreamRetries = 3;
@ -139,8 +139,16 @@ class WebcamRecordingService {
throw error;
}
// Wait before retry
// Wait before retry and check connection health
await new Promise((resolve) => setTimeout(resolve, 2000));
// Check if connection is still alive
if (connection.status !== "connected") {
console.log(
`[Docker] Connection status changed to: ${connection.status}`
);
throw new Error("Voice connection lost during stream creation");
}
}
}
@ -334,6 +342,9 @@ class WebcamRecordingService {
_setupWebcamStreamHandlers(webcamStream, textChannel, voiceChannelId) {
let frameCount = 0;
let lastFrameTime = Date.now();
let dataReceived = false;
let connectionTimeouts = 0;
const maxConnectionTimeouts = 3;
webcamStream.on("ready", () => {
console.log("[Docker] FFmpeg process ready for webcam recording!");
@ -341,21 +352,56 @@ class WebcamRecordingService {
webcamStream.stream.stderr.on("data", (data) => {
const dataStr = data.toString();
dataReceived = true;
// Only log important FFmpeg messages to reduce spam
if (
dataStr.includes("frame=") ||
dataStr.includes("error") ||
dataStr.includes("timeout")
) {
console.log(`[Docker] Webcam FFmpeg: ${dataStr}`);
}
// Track frame production
const frameMatch = dataStr.match(/frame=\s*(\d+)/);
if (frameMatch) {
frameCount = parseInt(frameMatch[1]);
lastFrameTime = Date.now();
connectionTimeouts = 0; // Reset timeout counter on successful frames
}
// Check for connection timeouts
if (
dataStr.includes("Connection timed out") ||
dataStr.includes("pipe:")
) {
connectionTimeouts++;
console.log(
`[Docker] FFmpeg connection timeout detected (${connectionTimeouts}/${maxConnectionTimeouts})`
);
if (connectionTimeouts >= maxConnectionTimeouts) {
console.log(
"[Docker] Too many connection timeouts, stopping webcam recording"
);
textChannel.send(
"⚠️ Webcam recording stopped - connection unstable."
);
this.stopRecording(
voiceChannelId,
textChannel,
"🎥 Webcam recording stopped - connection timeouts."
);
}
}
});
// Monitor for frame production - stop if no frames after 30 seconds
// Monitor for frame production - stop if no frames after 45 seconds (increased from 30)
setTimeout(() => {
if (frameCount === 0) {
console.log(
"[Docker] No frames produced after 30 seconds, stopping webcam recording"
"[Docker] No frames produced after 45 seconds, stopping webcam recording"
);
textChannel.send(
"⚠️ Webcam recording stopped - no video frames were captured. User may not have camera enabled."
@ -366,12 +412,14 @@ class WebcamRecordingService {
"🎥 Webcam recording stopped - no frames detected."
);
}
}, 30000);
}, 45000);
// Check for stalled recording every 60 seconds
// Check for stalled recording every 30 seconds (reduced from 60)
const stallCheckInterval = setInterval(() => {
const timeSinceLastFrame = Date.now() - lastFrameTime;
if (frameCount > 0 && timeSinceLastFrame > 60000) {
// More aggressive stall detection
if (frameCount > 0 && timeSinceLastFrame > 45000) {
console.log(
`[Docker] Webcam recording stalled - no frames for ${timeSinceLastFrame}ms`
);
@ -383,7 +431,21 @@ class WebcamRecordingService {
);
clearInterval(stallCheckInterval);
}
}, 60000);
// Check if we're receiving any data at all
if (!dataReceived && Date.now() - lastFrameTime > 30000) {
console.log(
"[Docker] No FFmpeg data received, connection may be broken"
);
textChannel.send("⚠️ Webcam recording stopped - no data received.");
this.stopRecording(
voiceChannelId,
textChannel,
"🎥 Webcam recording stopped - no data received."
);
clearInterval(stallCheckInterval);
}
}, 30000);
// Store interval for cleanup
const recording = activeWebcamRecordings.get(voiceChannelId);