iOS & iPadOS

Expression 2

// excerpt: inside your app. samples is the clip as [Float], 16 kHz mono;
// show(_:_:_:) draws B, G, R bytes; the URLs point at the files above.
import Expression2
import AVFoundation

Expression2Credential.set(ProcessInfo.processInfo.environment["BITHUMAN_API_SECRET"] ?? "")
let engine = try Expression2Engine.create(
    avatarContainer: avatarURL,              // wise-pup.imx
    sharedEngineContainer: sharedEngineURL,  // mac-arm64-1.0.0.engine
    stagingDir: stagingURL)                  // any writable directory; keep it between launches

let audio = AVAudioEngine(), player = AVAudioPlayerNode()         // your app's audio output
let format = AVAudioFormat(standardFormatWithSampleRate: 16000, channels: 1)!
audio.attach(player); audio.connect(player, to: audio.mainMixerNode, format: format); try audio.start()
let reply = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(samples.count))!
reply.frameLength = reply.frameCapacity
samples.withUnsafeBufferPointer { reply.floatChannelData![0].update(from: $0.baseAddress!, count: samples.count) }

// seconds of the current reply your player has played (nil before it starts); frames() reads it
// off the main actor, so the player goes in a Sendable box
final class PlayedSeconds: @unchecked Sendable {
    let node: AVAudioPlayerNode
    init(_ node: AVAudioPlayerNode) { self.node = node }
    func callAsFunction() -> Double? {
        guard let t = node.lastRenderTime, let pt = node.playerTime(forNodeTime: t) else { return nil }
        return Double(pt.sampleTime) / pt.sampleRate
    }
}
let played = PlayedSeconds(player)

engine.feed(samples)   // [Float], 16 kHz mono
engine.flushTail()     // end of the reply
for await frame in engine.frames(audioClock: { played() }) {
    show(frame.bgr, frame.width, frame.height)                        // B, G, R bytes
    if frame.audioTime == 0 { player.scheduleBuffer(reply, completionHandler: nil); player.play() }   // the reply's first frame: start its audio
    if frame.endsReply { break }                                      // the reply is over; idle frames follow
}