-
Notifications
You must be signed in to change notification settings - Fork 25
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add opus custom support #18
base: main
Are you sure you want to change the base?
Changes from 35 commits
b96e0ad
144063f
fb5c31c
04fa515
db32719
d1be9a6
80f7e0f
dc5f841
4e452fd
6104386
7273180
40be05d
593766d
605b9ec
fefebd7
3204c1a
b0c9e3b
c898e20
61b6a4b
9fa606f
21164e1
bada8cf
ed39aac
2c72a72
3ff24bf
566f056
b41d414
40fb047
e3788e3
8b656b3
e7cb649
a7f8272
a5b29aa
712b0ff
33c68f4
033520f
aa3db6f
42d044f
f5f8735
00d42cf
a48104d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,4 @@ | ||
.build | ||
.DS_Store | ||
DerivedData | ||
.swiftpm |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
#ifndef __OPUS_VARIADIC_WRAPPER_H__ | ||
#define __OPUS_VARIADIC_WRAPPER_H__ | ||
|
||
//#include "" | ||
#include <opus_defines.h> | ||
#include <opus_custom.h> | ||
|
||
int opus_custom_encoder_ctl_wrapper(OpusCustomEncoder *OPUS_RESTRICT st, int request, opus_int32 val); | ||
int opus_custom_decoder_ctl_wrapper(OpusCustomDecoder *OPUS_RESTRICT st, int request, opus_int32 val); | ||
|
||
#endif |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
#include "variadic-wrapper.h" | ||
|
||
int opus_custom_encoder_ctl_wrapper(OpusCustomEncoder *OPUS_RESTRICT st, int request, opus_int32 val) | ||
{ | ||
return opus_custom_encoder_ctl(st, request, val); | ||
} | ||
|
||
int opus_custom_decoder_ctl_wrapper(OpusCustomDecoder *OPUS_RESTRICT st, int request, opus_int32 val) | ||
{ | ||
return opus_custom_decoder_ctl(st, request, val); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,95 @@ | ||
import AVFoundation | ||
import Copus | ||
import Copuswrapper | ||
import Foundation | ||
|
||
public extension Opus { | ||
/// | ||
/// Implements a custom opus encoder / decoder. | ||
/// Custom implementations can have non-standard frame sizes | ||
/// | ||
final class Custom { | ||
private let opusCustomMode: OpaquePointer | ||
let encoder: Opus.Encoder | ||
let decoder: Opus.Decoder | ||
private let format: AVAudioFormat | ||
public let frameSize: Int32 | ||
|
||
public init(format: AVAudioFormat, | ||
application _: Application = .audio, | ||
frameSize: UInt32 = 128) throws | ||
{ | ||
if !format.isValidOpusPCMFormat { | ||
throw Opus.Error.badArgument | ||
} | ||
self.format = format | ||
self.frameSize = Int32(frameSize) | ||
|
||
var error: Opus.Error = .ok | ||
|
||
// Create custom parameters | ||
guard let customMode = opus_custom_mode_create( | ||
Int32(format.sampleRate), | ||
Int32(frameSize), | ||
&error.rawValue | ||
) else { throw error } | ||
opusCustomMode = customMode | ||
|
||
// Create custom encoder | ||
encoder = try Opus.Encoder(customOpus: opusCustomMode, | ||
format: format, | ||
frameSize: frameSize) | ||
// Create custom decoder | ||
decoder = try Opus.Decoder(customOpus: opusCustomMode, | ||
format: format, | ||
frameSize: frameSize) | ||
} | ||
|
||
deinit { | ||
opus_custom_mode_destroy(opusCustomMode) | ||
} | ||
|
||
/// | ||
/// Wrapper onto the opus_custom_encoder_ctl function | ||
/// https://www.opus-codec.org/docs/opus_api-1.3.1/group__opus__encoderctls.html | ||
/// - Parameter request The Opus CTL to change | ||
/// - Parameter value The value to set it to | ||
/// | ||
/// - Returns Opus.Error code | ||
public func encoderCtl(request: Int32, value: Int32) -> Opus.Error { | ||
Opus.Error( | ||
rawValue: opus_custom_encoder_ctl_wrapper(encoder.encoder, request, value) | ||
) | ||
} | ||
|
||
/// | ||
/// Encode a PCM buffer to data using the custom mode configuration and max size | ||
/// - parameter avData Audio data to compress | ||
/// - parameter compressedSize Opus packet size to compress to | ||
/// - Returns Data containing the Opus packet | ||
public func encode(_ avData: AVAudioPCMBuffer, | ||
compressedSize: Int) throws -> Data | ||
{ | ||
try encoder.encode(avData, compressedSize: compressedSize) | ||
} | ||
|
||
/// | ||
/// Decode an opus packet | ||
/// If the data is empty, it is treated as a dropped packet | ||
/// - Parameter data Compressed data | ||
/// - Parameter compressedPacketSize Number of bytes of data | ||
/// - Parameter sampleMultiplier Frame size multiplier if greater than one | ||
/// - Returns Uncompressed audio buffer | ||
public func decode(_ data: Data, | ||
compressedPacketSize: Int32, | ||
sampleMultiplier: Int32 = 1) throws -> AVAudioPCMBuffer | ||
{ | ||
guard data.isEmpty || data.count == compressedPacketSize else { | ||
throw Opus.Error.bufferTooSmall | ||
} | ||
return try decoder.decode(data, | ||
compressedPacketSize: compressedPacketSize, | ||
sampleMultiplier: sampleMultiplier) | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,9 +5,10 @@ public extension Opus { | |
class Decoder { | ||
let format: AVAudioFormat | ||
let decoder: OpaquePointer | ||
let customFrameSize: Int32? | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This can probably be a non-optional int that defaults to We’ll also need to add a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Actually no, this is a way to avoid having a duplicate un-needed pointer. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There’s no function in the Opus library to extract the frame size used to initialize an There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't follow your point here; can you clarify? |
||
|
||
// TODO: throw an error if format is unsupported | ||
public init(format: AVAudioFormat, application _: Application = .audio) throws { | ||
customFrameSize = nil | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If this is made non-optional, this can be There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. And then you need 2 new vars; customMode and customFrameSize. |
||
if !format.isValidOpusPCMFormat { | ||
throw Opus.Error.badArgument | ||
} | ||
|
@@ -22,6 +23,27 @@ public extension Opus { | |
} | ||
} | ||
|
||
public init(customOpus: OpaquePointer, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Given the encoder and decoder typically exist on different machines, I think It can be freed in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I disagree - custom encoders and decoders need the custom object; you need to have a custom object in order to use either. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The constructor for an The sample rate can be derived from the It’s probably useful to have common helper code to aid in the creation of a valid There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
That is the Opus.Custom.swift file. |
||
format: AVAudioFormat, | ||
frameSize: UInt32) throws | ||
emlynmac marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{ | ||
customFrameSize = Int32(frameSize) | ||
if !format.isValidOpusPCMFormat { | ||
throw Opus.Error.badArgument | ||
} | ||
|
||
self.format = format | ||
|
||
// Initialize Opus decoder | ||
var error: Opus.Error = .ok | ||
decoder = opus_custom_decoder_create(customOpus, | ||
Int32(format.channelCount), | ||
&error.rawValue) | ||
if error != .ok { | ||
throw error | ||
} | ||
} | ||
|
||
deinit { | ||
opus_decoder_destroy(decoder) | ||
} | ||
|
@@ -38,28 +60,42 @@ public extension Opus { | |
// MARK: Public decode methods | ||
|
||
public extension Opus.Decoder { | ||
func decode(_ input: Data) throws -> AVAudioPCMBuffer { | ||
func decode(_ input: Data, | ||
compressedPacketSize: Int32? = nil, | ||
sampleMultiplier: Int32 = 1) throws -> AVAudioPCMBuffer | ||
{ | ||
try input.withUnsafeBytes { | ||
var output: AVAudioPCMBuffer | ||
let input = $0.bindMemory(to: UInt8.self) | ||
let sampleCount = opus_decoder_get_nb_samples(decoder, input.baseAddress!, Int32($0.count)) | ||
if sampleCount < 0 { | ||
throw Opus.Error(sampleCount) | ||
if compressedPacketSize != nil, let frameSize = customFrameSize { | ||
output = AVAudioPCMBuffer( | ||
pcmFormat: format, | ||
frameCapacity: AVAudioFrameCount(frameSize * sampleMultiplier) | ||
)! | ||
} else { | ||
let sampleCount = opus_decoder_get_nb_samples(decoder, input.baseAddress!, Int32($0.count)) | ||
if sampleCount < 0 { | ||
throw Opus.Error(sampleCount) | ||
} | ||
output = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(sampleCount))! | ||
} | ||
let output = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(sampleCount))! | ||
try decode(input, to: output) | ||
|
||
try decode(input, to: output, packetSize: compressedPacketSize) | ||
return output | ||
} | ||
} | ||
|
||
func decode(_ input: UnsafeBufferPointer<UInt8>, to output: AVAudioPCMBuffer) throws { | ||
func decode(_ input: UnsafeBufferPointer<UInt8>, to output: AVAudioPCMBuffer, | ||
packetSize: Int32? = nil) throws | ||
{ | ||
let decodedCount: Int | ||
switch output.format.commonFormat { | ||
case .pcmFormatInt16: | ||
let output = UnsafeMutableBufferPointer(start: output.int16ChannelData![0], count: Int(output.frameCapacity)) | ||
decodedCount = try decode(input, to: output) | ||
decodedCount = try decode(input, to: output, packetSize: packetSize) | ||
case .pcmFormatFloat32: | ||
let output = UnsafeMutableBufferPointer(start: output.floatChannelData![0], count: Int(output.frameCapacity)) | ||
decodedCount = try decode(input, to: output) | ||
decodedCount = try decode(input, to: output, packetSize: packetSize) | ||
default: | ||
throw Opus.Error.badArgument | ||
} | ||
|
@@ -73,30 +109,58 @@ public extension Opus.Decoder { | |
// MARK: Private decode methods | ||
|
||
extension Opus.Decoder { | ||
private func decode(_ input: UnsafeBufferPointer<UInt8>, to output: UnsafeMutableBufferPointer<Int16>) throws -> Int { | ||
let decodedCount = opus_decode( | ||
decoder, | ||
input.baseAddress!, | ||
Int32(input.count), | ||
output.baseAddress!, | ||
Int32(output.count), | ||
0 | ||
) | ||
private func decode(_ input: UnsafeBufferPointer<UInt8>, | ||
to output: UnsafeMutableBufferPointer<Int16>, | ||
packetSize: Int32?) throws -> Int | ||
{ | ||
var decodedCount: Int32 = 0 | ||
if let size = packetSize { | ||
decodedCount = opus_custom_decode( | ||
decoder, | ||
input.isEmpty ? nil : input.baseAddress, | ||
size, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why is There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. packet size is needed for the custom decoder to know how much data it has. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. See comment at the top-level of this PR regarding dropped packets. If the decoder was initialized with a custom frame size, then either use that, or have There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Custom frame size and compressed packet size are not the same thing. Custom encode / decode need to know both in order to work. |
||
output.baseAddress!, | ||
Int32(output.count) | ||
) | ||
} else { | ||
decodedCount = opus_decode( | ||
decoder, | ||
input.isEmpty ? nil : input.baseAddress, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This change means that if you pass an empty data packet, opus decoder gets the nil it needs to signify a dropped packet. |
||
Int32(input.count), | ||
output.baseAddress!, | ||
Int32(output.count), | ||
0 | ||
) | ||
} | ||
if decodedCount < 0 { | ||
throw Opus.Error(decodedCount) | ||
} | ||
return Int(decodedCount) | ||
} | ||
|
||
private func decode(_ input: UnsafeBufferPointer<UInt8>, to output: UnsafeMutableBufferPointer<Float32>) throws -> Int { | ||
let decodedCount = opus_decode_float( | ||
decoder, | ||
input.baseAddress!, | ||
Int32(input.count), | ||
output.baseAddress!, | ||
Int32(output.count), | ||
0 | ||
) | ||
private func decode(_ input: UnsafeBufferPointer<UInt8>, | ||
to output: UnsafeMutableBufferPointer<Float32>, | ||
packetSize: Int32?) throws -> Int | ||
{ | ||
var decodedCount: Int32 = 0 | ||
if let size = packetSize { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If possible, this should branch on the presence of a non-nil There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've implemented this a different way; using the optional presence of a custom frame size. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When decoding a stream of packets, is the frame size consistent, or does it vary? If it’s consistent, then this parameter shouldn’t exist. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The size can be multiples of the underlying frame size There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why does it need the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. OK, now I've had my coffee... Compressed packet size != frame size. Both are needed. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. OK, great. Then can that be taken from the |
||
decodedCount = opus_custom_decode_float( | ||
decoder, | ||
input.isEmpty ? nil : input.baseAddress, | ||
size, | ||
output.baseAddress!, | ||
Int32(output.count) | ||
) | ||
} else { | ||
decodedCount = opus_decode_float( | ||
decoder, | ||
input.isEmpty ? nil : input.baseAddress, | ||
Int32(input.count), | ||
output.baseAddress!, | ||
Int32(output.count), | ||
0 | ||
) | ||
} | ||
if decodedCount < 0 { | ||
throw Opus.Error(decodedCount) | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What’s the purpose of this file? Can it be deleted?
(I think it should be, if
Opus.Encoder
andOpus.Decoder
can support nonstandard frame sizes.