mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-17 18:10:51 +08:00
重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
|
|
@ -0,0 +1,178 @@
|
|||
import Foundation
|
||||
import MLXAudioTTS
|
||||
import OSLog
|
||||
|
||||
// swiftformat:disable wrap wrapMultilineStatementBraces trailingCommas redundantSelf extensionAccessControl
|
||||
/// Runtime access stays serialized through `TalkModeRuntime` actor helper methods.
|
||||
final class TalkMLXSpeechSynthesizer {
|
||||
enum SynthesizeError: Error {
|
||||
case canceled
|
||||
case modelLoadFailed(String)
|
||||
case audioGenerationFailed
|
||||
case audioPlaybackFailed
|
||||
case timedOut
|
||||
}
|
||||
|
||||
static let shared = TalkMLXSpeechSynthesizer()
|
||||
static let defaultModelRepo = "mlx-community/Soprano-80M-bf16"
|
||||
|
||||
private let logger = Logger(subsystem: "ai.openclaw", category: "talk.mlx")
|
||||
private var currentToken = UUID()
|
||||
private var modelRepo: String?
|
||||
private var model: (any SpeechGenerationModel)?
|
||||
|
||||
private init() {}
|
||||
|
||||
func stop() {
|
||||
self.currentToken = UUID()
|
||||
}
|
||||
|
||||
func synthesize(
|
||||
text: String,
|
||||
modelRepo: String?,
|
||||
language: String?,
|
||||
voicePreset: String?) async throws -> Data {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return Data() }
|
||||
|
||||
self.stop()
|
||||
let token = UUID()
|
||||
self.currentToken = token
|
||||
|
||||
let resolvedRepo = Self.resolvedModelRepo(modelRepo)
|
||||
let rawModel = try await self.loadModel(
|
||||
modelRepo: resolvedRepo,
|
||||
token: token)
|
||||
let model = UncheckedSpeechModel(raw: rawModel)
|
||||
guard self.currentToken == token else {
|
||||
throw SynthesizeError.canceled
|
||||
}
|
||||
|
||||
let audioData: Data
|
||||
do {
|
||||
let audio = try await model.generateAudio(
|
||||
text: trimmed,
|
||||
voice: voicePreset,
|
||||
language: language)
|
||||
audioData = Self.makeWavData(
|
||||
samples: audio,
|
||||
sampleRate: Double(model.sampleRateValue()))
|
||||
} catch {
|
||||
self.logger.error(
|
||||
"talk mlx generation failed: \(error.localizedDescription, privacy: .public)")
|
||||
throw SynthesizeError.audioGenerationFailed
|
||||
}
|
||||
|
||||
guard self.currentToken == token else {
|
||||
throw SynthesizeError.canceled
|
||||
}
|
||||
return audioData
|
||||
}
|
||||
|
||||
private func loadModel(
|
||||
modelRepo: String,
|
||||
token: UUID) async throws -> any SpeechGenerationModel {
|
||||
if let model = self.model, self.modelRepo == modelRepo {
|
||||
return model
|
||||
}
|
||||
|
||||
self.logger.info("talk mlx loading modelRepo=\(modelRepo, privacy: .public)")
|
||||
do {
|
||||
let model = try await TTS.loadModel(modelRepo: modelRepo)
|
||||
guard self.currentToken == token else {
|
||||
throw SynthesizeError.canceled
|
||||
}
|
||||
self.model = model
|
||||
self.modelRepo = modelRepo
|
||||
return model
|
||||
} catch is CancellationError {
|
||||
throw SynthesizeError.canceled
|
||||
} catch {
|
||||
self.logger.error(
|
||||
"talk mlx load failed: \(error.localizedDescription, privacy: .public)")
|
||||
throw SynthesizeError.modelLoadFailed(modelRepo)
|
||||
}
|
||||
}
|
||||
|
||||
private static func resolvedModelRepo(_ modelRepo: String?) -> String {
|
||||
let trimmed = modelRepo?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
||||
return trimmed.isEmpty ? Self.defaultModelRepo : trimmed
|
||||
}
|
||||
|
||||
private static func makeWavData(samples: [Float], sampleRate: Double) -> Data {
|
||||
let channels: UInt16 = 1
|
||||
let bitsPerSample: UInt16 = 16
|
||||
let blockAlign = channels * (bitsPerSample / 8)
|
||||
let sampleRateInt = UInt32(sampleRate.rounded())
|
||||
let byteRate = sampleRateInt * UInt32(blockAlign)
|
||||
let dataSize = UInt32(samples.count) * UInt32(blockAlign)
|
||||
|
||||
var data = Data(capacity: Int(44 + dataSize))
|
||||
data.append(contentsOf: [0x52, 0x49, 0x46, 0x46]) // RIFF
|
||||
data.appendLEUInt32(36 + dataSize)
|
||||
data.append(contentsOf: [0x57, 0x41, 0x56, 0x45]) // WAVE
|
||||
|
||||
data.append(contentsOf: [0x66, 0x6D, 0x74, 0x20]) // fmt
|
||||
data.appendLEUInt32(16)
|
||||
data.appendLEUInt16(1)
|
||||
data.appendLEUInt16(channels)
|
||||
data.appendLEUInt32(sampleRateInt)
|
||||
data.appendLEUInt32(byteRate)
|
||||
data.appendLEUInt16(blockAlign)
|
||||
data.appendLEUInt16(bitsPerSample)
|
||||
|
||||
data.append(contentsOf: [0x64, 0x61, 0x74, 0x61]) // data
|
||||
data.appendLEUInt32(dataSize)
|
||||
|
||||
for sample in samples {
|
||||
let clamped = max(-1.0, min(1.0, sample))
|
||||
let scaled = Int16((clamped * Float(Int16.max)).rounded())
|
||||
data.appendLEInt16(scaled)
|
||||
}
|
||||
return data
|
||||
}
|
||||
}
|
||||
|
||||
extension TalkMLXSpeechSynthesizer: @unchecked Sendable {}
|
||||
|
||||
private struct UncheckedSpeechModel {
|
||||
let raw: any SpeechGenerationModel
|
||||
|
||||
func sampleRateValue() -> Int {
|
||||
raw.sampleRate
|
||||
}
|
||||
|
||||
func generateAudio(
|
||||
text: String,
|
||||
voice: String?,
|
||||
language: String?) async throws -> [Float] {
|
||||
let generatedAudio = try await raw.generate(
|
||||
text: text,
|
||||
voice: voice,
|
||||
refAudio: nil,
|
||||
refText: nil,
|
||||
language: language)
|
||||
return generatedAudio.asArray(Float.self)
|
||||
}
|
||||
}
|
||||
|
||||
extension UncheckedSpeechModel: @unchecked Sendable {}
|
||||
|
||||
extension Data {
|
||||
fileprivate mutating func appendLEUInt16(_ value: UInt16) {
|
||||
var littleEndian = value.littleEndian
|
||||
Swift.withUnsafeBytes(of: &littleEndian) { append(contentsOf: $0) }
|
||||
}
|
||||
|
||||
fileprivate mutating func appendLEUInt32(_ value: UInt32) {
|
||||
var littleEndian = value.littleEndian
|
||||
Swift.withUnsafeBytes(of: &littleEndian) { append(contentsOf: $0) }
|
||||
}
|
||||
|
||||
fileprivate mutating func appendLEInt16(_ value: Int16) {
|
||||
var littleEndian = value.littleEndian
|
||||
Swift.withUnsafeBytes(of: &littleEndian) { append(contentsOf: $0) }
|
||||
}
|
||||
}
|
||||
|
||||
// swiftformat:enable wrap wrapMultilineStatementBraces trailingCommas redundantSelf extensionAccessControl
|
||||
Loading…
Add table
Add a link
Reference in a new issue