From d0f57b04c282a0b237a9a5634ba6fbeb471b2f01 Mon Sep 17 00:00:00 2001 From: tornikegomareli Date: Tue, 8 Sep 2026 14:26:37 +0400 Subject: [PATCH 1/2] Recover the microphone when the audio route changes A Bluetooth headset switches to its 16 kHz hands-free profile the moment something opens the input. AVAudioEngine answers a hardware change by stopping itself and posting AVAudioEngineConfigurationChange. Nothing observed it, so a session that began on that switch listened to an engine that was no longer running: no levels, the dead-microphone amber after 600ms, no words, and no error. Reproduced against the device with the old setup, which delivered zero buffers, and confirmed the recovery delivers audio at the new rate. Three things the reproduction settled. Restarting the same engine fails with -10868, so the engine is replaced. Installing a tap with a format captured moments earlier throws an Objective-C exception once the hardware has moved on, which Swift cannot catch, so the tap now takes nil and uses the bus's live format. And the converter is rebuilt when the buffers change rate, since the session no longer starts and ends in one format. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012Abfbei4sXuCngAWEH8Z2k --- .oss | 0 Talkify/Dictation/MicrophoneInput.swift | 175 ++++++++++++++++++++---- TalkifyTests/MicrophoneInputTests.swift | 33 +++++ 3 files changed, 181 insertions(+), 27 deletions(-) create mode 100644 .oss diff --git a/.oss b/.oss new file mode 100644 index 0000000..e69de29 diff --git a/Talkify/Dictation/MicrophoneInput.swift b/Talkify/Dictation/MicrophoneInput.swift index 3dc51f1..1170b54 100644 --- a/Talkify/Dictation/MicrophoneInput.swift +++ b/Talkify/Dictation/MicrophoneInput.swift @@ -23,14 +23,32 @@ final class MicrophoneInput: @unchecked Sendable { } } - private final class ConverterBox: @unchecked Sendable { - let converter: AVAudioConverter + /// The converter into the analyzer's format, rebuilt whenever the tap + /// starts delivering a different one. + /// + /// The input format is not fixed for the life of a session: a Bluetooth + /// headset switches from A2DP to its 16 kHz hands-free profile the moment + /// something opens the microphone, so the first buffers after a route + /// change arrive in a format the session did not start with. One box per + /// tap, so the audio thread is the only thread that touches it. + final class ConverterBox: @unchecked Sendable { let outputFormat: AVAudioFormat + private var converter: AVAudioConverter? + private var inputFormat: AVAudioFormat? - init(converter: AVAudioConverter, outputFormat: AVAudioFormat) { - self.converter = converter + init(outputFormat: AVAudioFormat) { self.outputFormat = outputFormat } + + func converter(for format: AVAudioFormat) throws -> AVAudioConverter { + if let converter, inputFormat == format { return converter } + guard let made = AVAudioConverter(from: format, to: outputFormat) else { + throw InputError.converterCreationFailed + } + converter = made + inputFormat = format + return made + } } private final class InputProvider: @unchecked Sendable { @@ -56,7 +74,19 @@ final class MicrophoneInput: @unchecked Sendable { } } - private let audioEngine = AVAudioEngine() + /// Replaced rather than restarted when the audio route changes: the old + /// engine's input chain cannot re-initialise across a Bluetooth profile + /// switch, which fails with -10868. + private var audioEngine = AVAudioEngine() + /// Serialises route recovery off the notification thread. + private let recoveryQueue = DispatchQueue(label: "com.tgomareli.Talkify.mic-recovery") + private var configurationObserver: (any NSObjectProtocol)? + private var analyzerFormat: AVAudioFormat? + private var recovering = false + /// A route change that arrived while recovering. Dropping it would leave a + /// dead engine behind, and a headset settling its profile can post more + /// than one. + private var recoveryPending = false private let analyzerContinuation: AsyncStream.Continuation private let failureHandler: @Sendable (InputError) -> Void /// Normalized microphone level (0–1) per tap buffer, for the HUD's @@ -78,42 +108,130 @@ final class MicrophoneInput: @unchecked Sendable { } func start(outputFormat: AVAudioFormat) throws { - let inputNode = audioEngine.inputNode - let hardwareFormat = inputNode.inputFormat(forBus: 0) + let hardwareFormat = audioEngine.inputNode.inputFormat(forBus: 0) guard Self.hasUsableHardwareInput(hardwareFormat) else { throw InputError.unavailable } - let inputFormat = inputNode.outputFormat(forBus: 0) - guard inputFormat.channelCount > 0, inputFormat.sampleRate > 0 else { - throw InputError.unavailable + stateLock.withLock { analyzerFormat = outputFormat } + observeConfigurationChanges(of: audioEngine) + do { + try startCapturing(on: audioEngine, into: outputFormat) + } catch { + stopObservingConfigurationChanges() + throw error } + stateLock.withLock { running = true } + } - guard let converter = AVAudioConverter(from: inputFormat, to: outputFormat) else { - throw InputError.converterCreationFailed - } + /// Installs the tap and starts the engine. + /// + /// The tap takes no format. Handing it one captured moments earlier throws + /// an Objective-C exception the moment the hardware has moved on ("Failed + /// to create tap due to format mismatch"), which Swift cannot catch, so the + /// app would die rather than recover. Nil means the bus's live format. + private func startCapturing(on engine: AVAudioEngine, into outputFormat: AVAudioFormat) throws { + let inputNode = engine.inputNode + inputNode.removeTap(onBus: 0) - let converterBox = ConverterBox(converter: converter, outputFormat: outputFormat) - inputNode.installTap( - onBus: 0, - bufferSize: 1_024, - format: inputFormat - ) { [weak self] buffer, _ in + let converterBox = ConverterBox(outputFormat: outputFormat) + inputNode.installTap(onBus: 0, bufferSize: 1_024, format: nil) { [weak self] buffer, _ in self?.receive(buffer, converterBox: converterBox) } - audioEngine.prepare() + engine.prepare() do { - try audioEngine.start() - stateLock.withLock { - running = true - } + try engine.start() } catch { inputNode.removeTap(onBus: 0) throw error } } + private func observeConfigurationChanges(of engine: AVAudioEngine) { + stopObservingConfigurationChanges() + let observer = NotificationCenter.default.addObserver( + forName: .AVAudioEngineConfigurationChange, + object: engine, + queue: nil + ) { [weak self] _ in + self?.recoverFromRouteChange() + } + stateLock.withLock { configurationObserver = observer } + } + + private func stopObservingConfigurationChanges() { + let observer = stateLock.withLock { () -> (any NSObjectProtocol)? in + defer { configurationObserver = nil } + return configurationObserver + } + if let observer { NotificationCenter.default.removeObserver(observer) } + } + + /// AVAudioEngine stops itself when the audio hardware changes underneath it + /// and says so through this notification. Nothing else restarts it, so + /// without this a session that began just as a headset switched profile + /// listens to an engine that is no longer running: no levels, no words, and + /// no error to show for it. + private func recoverFromRouteChange() { + let shouldRecover = stateLock.withLock { () -> Bool in + guard running else { return false } + guard !recovering else { + recoveryPending = true + return false + } + recovering = true + return true + } + guard shouldRecover else { return } + scheduleRecovery() + } + + private func scheduleRecovery() { + // Off the notification thread, and after the hardware has settled: asked + // any sooner, the new engine fails to initialise its input chain. + recoveryQueue.asyncAfter(deadline: .now() + 0.25) { [weak self] in + guard let self else { return } + rebuildEngine() + + // A change that arrived mid-rebuild describes hardware the new engine + // never saw, so it gets its own pass rather than being dropped. + let again = stateLock.withLock { () -> Bool in + guard running, recoveryPending else { + recovering = false + recoveryPending = false + return false + } + recoveryPending = false + return true + } + if again { scheduleRecovery() } + } + } + + private func rebuildEngine() { + let outputFormat = stateLock.withLock { () -> AVAudioFormat? in + guard running else { return nil } + return analyzerFormat + } + guard let outputFormat else { return } + + let previous = stateLock.withLock { audioEngine } + previous.stop() + previous.inputNode.removeTap(onBus: 0) + + let engine = AVAudioEngine() + stateLock.withLock { audioEngine = engine } + observeConfigurationChanges(of: engine) + do { + try startCapturing(on: engine, into: outputFormat) + } catch { + // Nothing else is coming. Ending the session with a reason beats + // leaving the shape up in front of a microphone that is not running. + reportFailure(.unavailable) + } + } + static func hasUsableHardwareInput(_ format: AVAudioFormat) -> Bool { format.channelCount > 0 && format.sampleRate > 0 } @@ -126,8 +244,10 @@ final class MicrophoneInput: @unchecked Sendable { } guard shouldStop else { return } - audioEngine.stop() - audioEngine.inputNode.removeTap(onBus: 0) + stopObservingConfigurationChanges() + let engine = stateLock.withLock { audioEngine } + engine.stop() + engine.inputNode.removeTap(onBus: 0) } private func receive(_ buffer: AVAudioPCMBuffer, converterBox: ConverterBox) { @@ -160,9 +280,10 @@ final class MicrophoneInput: @unchecked Sendable { throw InputError.conversionFailed("Unable to allocate an audio buffer.") } + let converter = try converterBox.converter(for: inputBuffer.format) let inputProvider = InputProvider(buffer: inputBuffer) var conversionError: NSError? - let status = converterBox.converter.convert( + let status = converter.convert( to: outputBuffer, error: &conversionError ) { _, inputStatus in diff --git a/TalkifyTests/MicrophoneInputTests.swift b/TalkifyTests/MicrophoneInputTests.swift index b446bf9..9a828eb 100644 --- a/TalkifyTests/MicrophoneInputTests.swift +++ b/TalkifyTests/MicrophoneInputTests.swift @@ -22,4 +22,37 @@ struct MicrophoneInputTests { #expect(MicrophoneInput.hasUsableHardwareInput(format)) } + + /// A Bluetooth headset switches to its 16 kHz hands-free profile the moment + /// the microphone opens, so buffers after a route change arrive in a format + /// the session did not start with. A converter kept from the old format + /// would go on resampling from a rate nothing is sending. + @Test func theConverterFollowsAChangeOfInputFormat() throws { + let analyzerFormat = try #require(AVAudioFormat( + standardFormatWithSampleRate: 16_000, + channels: 1 + )) + let a2dp = try #require(AVAudioFormat(standardFormatWithSampleRate: 44_100, channels: 1)) + let handsFree = try #require(AVAudioFormat(standardFormatWithSampleRate: 16_000, channels: 1)) + + let box = MicrophoneInput.ConverterBox(outputFormat: analyzerFormat) + let first = try box.converter(for: a2dp) + #expect(first.inputFormat.sampleRate == 44_100) + + let afterSwitch = try box.converter(for: handsFree) + #expect(afterSwitch.inputFormat.sampleRate == 16_000) + #expect(afterSwitch !== first, "the converter must be rebuilt, not reused") + } + + /// Rebuilding on every buffer would allocate a converter at the tap rate. + @Test func theConverterIsKeptWhileTheFormatHolds() throws { + let analyzerFormat = try #require(AVAudioFormat( + standardFormatWithSampleRate: 16_000, + channels: 1 + )) + let input = try #require(AVAudioFormat(standardFormatWithSampleRate: 44_100, channels: 1)) + + let box = MicrophoneInput.ConverterBox(outputFormat: analyzerFormat) + #expect(try box.converter(for: input) === (try box.converter(for: input))) + } } From 979b2ab712ff8e3f27f4e1b57fb434b6d117a7e1 Mon Sep 17 00:00:00 2001 From: tornikegomareli Date: Tue, 8 Sep 2026 14:35:24 +0400 Subject: [PATCH 2/2] Do not call a microphone dead while it is still opening Measured against the headset: 670ms for the first engine.start, 270ms until the route change, then the rebuild, and the first buffer at about 1.4 seconds. The watchdog calls a microphone dead after 600ms of no levels, so every Bluetooth session went amber and looked broken for the second before it started working. That budget is for a microphone that stopped mid-sentence. A session that has never had a buffer is a different case and now gets 2.5s, after which amber still means what it always did. Also drops the settle delay before the rebuild, worth 250ms of the 1.4. Measurement showed a fresh engine starts cleanly with no delay at all: the -10868 that looked like it needed one came from restarting the old engine, not from being early. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012Abfbei4sXuCngAWEH8Z2k --- .../HUD/DictationHUDController.swift | 23 ++++++++++++++++++- Talkify/Dictation/MicrophoneInput.swift | 7 +++--- TalkifyTests/HUDPlaceholderTests.swift | 19 +++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/Talkify/Dictation/HUD/DictationHUDController.swift b/Talkify/Dictation/HUD/DictationHUDController.swift index 8f990df..d233f63 100644 --- a/Talkify/Dictation/HUD/DictationHUDController.swift +++ b/Talkify/Dictation/HUD/DictationHUDController.swift @@ -12,6 +12,15 @@ final class DictationHUDController { /// Levels stopping for this long while listening means the microphone is /// dead, which must look different from silence (CONTEXT.md). private static let deadMicrophoneAfter = Duration.milliseconds(600) + /// How long a session waits for its first buffer before calling the + /// microphone dead. + /// + /// Opening the input makes a Bluetooth headset switch to its hands-free + /// profile, which stops the audio engine and costs about 1.4 seconds before + /// the first buffer arrives. Judging that by the 600ms that catches a + /// microphone which stopped mid-sentence reported every Bluetooth session + /// as broken for the second before it started working. + private static let firstAudioAfter = Duration.milliseconds(2_500) private let stage: HUDStage private var sessionSettings: DictationSessionSettings @@ -23,6 +32,9 @@ final class DictationHUDController { /// the shaping caption is the case that found it. private var hasStoppedListening = false private var lastLevelAt = ContinuousClock.now + /// Whether this session has had a single buffer yet. Until it has, the + /// input may simply still be opening rather than broken. + private var hasHeardAudio = false private var micWatchdogTask: Task? private var hasPlayedBeginSound = false /// Mirrors hasPlayedBeginSound: a shaping phase plays End when speech @@ -46,6 +58,7 @@ final class DictationHUDController { hasPlayedBeginSound = true stage.sounds.playBegin(using: sessionSettings.sounds) } + hasHeardAudio = true content.audioLevel = max(Double(level), content.audioLevel * 0.88) content.levelHistory.removeFirst() // Light EMA against the previous bar calms per-tick jitter without @@ -205,19 +218,27 @@ final class DictationHUDController { content.levelHistory = [Float](repeating: 0, count: HUDWaveformView.barCount) content.isAudioAlive = true lastLevelAt = ContinuousClock.now + hasHeardAudio = false micWatchdogTask?.cancel() micWatchdogTask = Task { [weak self] in while !Task.isCancelled { try? await Task.sleep(for: .milliseconds(300)) guard let self, !Task.isCancelled else { return } - if lastLevelAt.duration(to: .now) > Self.deadMicrophoneAfter { + if lastLevelAt.duration(to: .now) > Self.silenceBudget(hasHeardAudio: hasHeardAudio) { content.isAudioAlive = false } } } } + /// How long the visual waits before calling the microphone dead: the short + /// budget once audio has been heard, the long one while the input is still + /// opening. + static func silenceBudget(hasHeardAudio: Bool) -> Duration { + hasHeardAudio ? deadMicrophoneAfter : firstAudioAfter + } + private func stopVoiceVisual() { stopWatchdog() content.showsVoiceVisual = false diff --git a/Talkify/Dictation/MicrophoneInput.swift b/Talkify/Dictation/MicrophoneInput.swift index 1170b54..e76e679 100644 --- a/Talkify/Dictation/MicrophoneInput.swift +++ b/Talkify/Dictation/MicrophoneInput.swift @@ -188,9 +188,10 @@ final class MicrophoneInput: @unchecked Sendable { } private func scheduleRecovery() { - // Off the notification thread, and after the hardware has settled: asked - // any sooner, the new engine fails to initialise its input chain. - recoveryQueue.asyncAfter(deadline: .now() + 0.25) { [weak self] in + // Off the notification thread. No settle delay: a fresh engine starts + // cleanly straight away, and the -10868 that looked like it needed one + // came from restarting the old engine rather than from being early. + recoveryQueue.async { [weak self] in guard let self else { return } rebuildEngine() diff --git a/TalkifyTests/HUDPlaceholderTests.swift b/TalkifyTests/HUDPlaceholderTests.swift index 65ba822..830c37e 100644 --- a/TalkifyTests/HUDPlaceholderTests.swift +++ b/TalkifyTests/HUDPlaceholderTests.swift @@ -128,6 +128,25 @@ struct HUDPlaceholderTests { #expect(hud.textForTesting == "Listening…") } + /// A Bluetooth headset costs about 1.4 seconds before its first buffer, + /// because opening the input switches its profile and restarts the engine. + /// The budget that catches a microphone dying mid-sentence is 600ms, and + /// judging the opening by it called every Bluetooth session broken. + @Test func theMicrophoneIsNotCalledDeadWhileTheInputIsStillOpening() { + #expect( + DictationHUDController.silenceBudget(hasHeardAudio: false) > .milliseconds(1_400), + "a cold Bluetooth session would be reported as a dead microphone" + ) + #expect( + DictationHUDController.silenceBudget(hasHeardAudio: true) == .milliseconds(600), + "a microphone that stops mid-sentence still has to show up quickly" + ) + #expect( + DictationHUDController.silenceBudget(hasHeardAudio: true) + < DictationHUDController.silenceBudget(hasHeardAudio: false) + ) + } + /// The words being rewritten are not a placeholder, so the shaping phase /// leaves them where they are. @Test func theShapingPhaseKeepsARealDraft() {