TeaSpeak/TeaSpeak-Client
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Switched to libsoundio

Browse Source
This commit is contained in:
WolverinDEV 2020-02-08 16:50:48 +01:00
parent 7007e8f6aa
commit 411778f932
31 changed files with 1725 additions and 738 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
modules/renderer/audio/AudioPlayer.ts
View File

@ -3,6 +3,8 @@ window["require_setup"](module);
import {audio as naudio} from "teaclient_connection"; import {audio as naudio} from "teaclient_connection";
namespace audio.player { namespace audio.player {
//FIXME: Native audio initialize handle!
export interface Device { export interface Device {
device_id: string; device_id: string;
name: string; name: string;
@ -15,6 +17,7 @@ namespace audio.player {
let _initialized_callbacks: (() => any)[] = []; let _initialized_callbacks: (() => any)[] = [];
export let _initialized = false; export let _initialized = false;
export let _initializing = false;
export let _audioContext: AudioContext; export let _audioContext: AudioContext;
export let _processor: ScriptProcessorNode; export let _processor: ScriptProcessorNode;
export let _output_stream: naudio.playback.OwnedAudioOutputStream; export let _output_stream: naudio.playback.OwnedAudioOutputStream;
@ -44,6 +47,10 @@ namespace audio.player {
} }
export function initialize() { export function initialize() {
if(_initializing) return;
_initializing = true;
naudio.initialize(() => {
_output_stream = naudio.playback.create_stream(); _output_stream = naudio.playback.create_stream();
_output_stream.set_buffer_max_latency(0.4); _output_stream.set_buffer_max_latency(0.4);
_output_stream.set_buffer_latency(0.02); _output_stream.set_buffer_latency(0.02);
@ -77,26 +84,28 @@ namespace audio.player {
for(const callback of _initialized_callbacks) for(const callback of _initialized_callbacks)
callback(); callback();
_initialized_callbacks = []; _initialized_callbacks = [];
});
return true; return true;
} }
export async function available_devices() : Promise<Device[]> { export async function available_devices() : Promise<Device[]> {
return naudio.available_devices().filter(e => e.output_supported || e.output_default).map(e => { return naudio.available_devices().filter(e => e.output_supported || e.output_default);
return {
device_id: e.device_id,
name: e.name
}
});
} }
export async function set_device(device_id?: string) : Promise<void> { export async function set_device(device_id?: string) : Promise<void> {
const dev = naudio.available_devices().filter(e => e.device_id == device_id); const dev = naudio.available_devices().filter(e => e.device_id == device_id);
if(dev.length == 0) { if(dev.length == 0) {
console.warn("Missing audio device with is %s", device_id) console.warn("Missing audio device with is %s", device_id);
throw "invalid device id"; throw "invalid device id";
} }
await naudio.playback.set_device(dev[0].device_index); try {
naudio.playback.set_device(dev[0].device_id);
} catch(error) {
if(error instanceof Error)
throw error.message;
throw error;
}
_current_device = dev[0]; _current_device = dev[0];
} }

41
modules/renderer/audio/AudioRecorder.ts
View File

@ -1,8 +1,8 @@
window["require_setup"](module); window["require_setup"](module);
import {audio as naudio} from "teaclient_connection"; import {audio as naudio} from "teaclient_connection";
//import {audio, tr} from "../imports/imports_shared"; // <reference types="../imports/import_shared.d.ts" />
/// <reference types="./imports/import_shared.d.ts" /> /// <reference types="../../modules/renderer/imports/imports_shared.d.ts" />
export namespace _audio.recorder { export namespace _audio.recorder {
import InputDevice = audio.recorder.InputDevice; import InputDevice = audio.recorder.InputDevice;
@ -14,6 +14,9 @@ export namespace _audio.recorder {
let _device_cache: NativeDevice[] = undefined; let _device_cache: NativeDevice[] = undefined;
export function devices() : InputDevice[] { export function devices() : InputDevice[] {
//TODO: Handle device updates!
if(!naudio.initialized()) return [];
return _device_cache || (_device_cache = naudio.available_devices().filter(e => e.input_supported || e.input_default).map(e => { return _device_cache || (_device_cache = naudio.available_devices().filter(e => e.input_supported || e.input_default).map(e => {
return { return {
unique_id: e.device_id, unique_id: e.device_id,
@ -22,8 +25,7 @@ export namespace _audio.recorder {
supported: e.input_supported, supported: e.input_supported,
name: e.name, name: e.name,
driver: e.driver, driver: e.driver,
sample_rate: 44100, /* TODO! */ sample_rate: 48000, /* TODO! */
device_index: e.device_index,
} as NativeDevice } as NativeDevice
})); }));
} }
@ -293,22 +295,13 @@ export namespace _audio.recorder {
const device = _device as NativeDevice; /* TODO: test for? */ const device = _device as NativeDevice; /* TODO: test for? */
this._current_device = _device; this._current_device = _device;
try { try {
await new Promise((resolve, reject) => { await new Promise(resolve => this.handle.set_device(this._current_device.unique_id, resolve));
this.handle.set_device(device ? device.device_index : -1, flag => {
if(typeof(flag) === "boolean" && flag)
resolve();
else
reject("failed to set device" + (typeof(flag) === "string" ? (": " + flag) : ""));
});
});
if(!device) return;
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
this.handle.start(flag => { this.handle.start(flag => {
if(flag) if(typeof flag === "boolean" && flag)
resolve(); resolve();
else else
reject("start failed"); reject(typeof flag === "string" ? flag : "failed to start");
}); });
}); });
} catch(error) { } catch(error) {
@ -453,20 +446,13 @@ export namespace _audio.recorder {
this._filter.set_attack_smooth(.75); this._filter.set_attack_smooth(.75);
this._filter.set_release_smooth(.75); this._filter.set_release_smooth(.75);
await new Promise((resolve, reject) => { await new Promise(resolve => this._recorder.set_device(this._device.unique_id, resolve));
this._recorder.set_device(this._device.device_index, flag => {
if(typeof(flag) === "boolean" && flag)
resolve();
else
reject("initialize failed" + (typeof(flag) === "string" ? (": " + flag) : ""));
});
});
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
this._recorder.start(flag => { this._recorder.start(flag => {
if(flag) if(typeof flag === "boolean" && flag)
resolve(); resolve();
else else
reject("start failed"); reject(typeof flag === "string" ? flag : "failed to start");
}); });
}); });
} catch(error) { } catch(error) {
@ -490,7 +476,7 @@ export namespace _audio.recorder {
if(this._consumer) if(this._consumer)
this._recorder.delete_consumer(this._consumer); this._recorder.delete_consumer(this._consumer);
this._recorder.stop(); this._recorder.stop();
this._recorder.set_device(-1, () => {}); /* -1 := No device */ this._recorder.set_device(undefined, () => {}); /* -1 := No device */
this._recorder = undefined; this._recorder = undefined;
this._consumer = undefined; this._consumer = undefined;
this._filter = undefined; this._filter = undefined;
@ -507,4 +493,3 @@ export namespace _audio.recorder {
} }
Object.assign(window["audio"] || (window["audio"] = {} as any), _audio); Object.assign(window["audio"] || (window["audio"] = {} as any), _audio);
_audio.recorder.devices(); /* query devices */

9
native/dist/ext_nan/include/NanEventCallback.h vendored
View File

@ -84,7 +84,7 @@ namespace Nan {
struct callback_wrap { struct callback_wrap {
std::shared_ptr<callback_scoped<Args...>> handle; std::shared_ptr<callback_scoped<Args...>> handle;
void call_cpy(Args... args, bool no_throw = false) { void call_cpy(Args... args, bool no_throw = false) const {
if(!this->handle) { if(!this->handle) {
if(no_throw) if(no_throw)
return; return;
@ -93,7 +93,7 @@ namespace Nan {
handle->callback(std::forward<Args>(args)...); handle->callback(std::forward<Args>(args)...);
} }
void call(Args&&... args, bool no_throw = false) { void call(Args&&... args, bool no_throw = false) const {
if(!this->handle) { if(!this->handle) {
if(no_throw) if(no_throw)
return; return;
@ -102,11 +102,6 @@ namespace Nan {
handle->callback(std::forward<Args>(args)...); handle->callback(std::forward<Args>(args)...);
} }
void operator()(Args&&... args) {
if(!this->handle)
throw std::bad_function_call();
handle->callback(std::forward<Args>(args)...);
}
void operator()(Args&&... args) const { void operator()(Args&&... args) const {
if(!this->handle) if(!this->handle)
throw std::bad_function_call(); throw std::bad_function_call();

18
native/serverconnection/CMakeLists.txt
View File

@ -5,12 +5,12 @@ set(SOURCE_FILES
src/logger.cpp src/logger.cpp
src/EventLoop.cpp src/EventLoop.cpp
src/hwuid.cpp src/hwuid.cpp
src/ring_buffer.cpp
src/connection/ft/FileTransferManager.cpp src/connection/ft/FileTransferManager.cpp
src/connection/ft/FileTransferObject.cpp src/connection/ft/FileTransferObject.cpp
src/audio/AudioSamples.cpp src/audio/AudioSamples.cpp
src/audio/AudioDevice.cpp
src/audio/AudioMerger.cpp src/audio/AudioMerger.cpp
src/audio/AudioOutput.cpp src/audio/AudioOutput.cpp
src/audio/AudioInput.cpp src/audio/AudioInput.cpp
@ -23,6 +23,11 @@ set(SOURCE_FILES
src/audio/codec/Converter.cpp src/audio/codec/Converter.cpp
src/audio/codec/OpusConverter.cpp src/audio/codec/OpusConverter.cpp
src/audio/driver/AudioDriver.cpp
src/audio/driver/SoundIO.cpp
src/audio/driver/SoundIOPlayback.cpp
src/audio/driver/SoundIORecord.cpp
) )
set(NODEJS_SOURCE_FILES set(NODEJS_SOURCE_FILES
@ -83,6 +88,8 @@ include_directories(${StringVariable_INCLUDE_DIR})
find_package(Ed25519 REQUIRED) find_package(Ed25519 REQUIRED)
include_directories(${ed25519_INCLUDE_DIR}) include_directories(${ed25519_INCLUDE_DIR})
find_package(soundio REQUIRED)
find_package(ThreadPool REQUIRED) find_package(ThreadPool REQUIRED)
include_directories(${ThreadPool_INCLUDE_DIR}) include_directories(${ThreadPool_INCLUDE_DIR})
if (WIN32) if (WIN32)
@ -95,9 +102,6 @@ endif ()
find_package(Soxr REQUIRED) find_package(Soxr REQUIRED)
include_directories(${soxr_INCLUDE_DIR}) include_directories(${soxr_INCLUDE_DIR})
find_package(PortAudio REQUIRED)
include_directories(${PortAudio_INCLUDE_DIR})
find_package(fvad REQUIRED) find_package(fvad REQUIRED)
include_directories(${fvad_INCLUDE_DIR}) include_directories(${fvad_INCLUDE_DIR})
@ -118,11 +122,11 @@ set(REQUIRED_LIBRARIES
${DataPipes_LIBRARIES_STATIC} #Needs to be static because something causes ca bad function call when loaded in electron ${DataPipes_LIBRARIES_STATIC} #Needs to be static because something causes ca bad function call when loaded in electron
${ThreadPool_LIBRARIES_STATIC} ${ThreadPool_LIBRARIES_STATIC}
${soxr_LIBRARIES_STATIC} ${soxr_LIBRARIES_STATIC}
${PortAudio_LIBRARIES_STATIC}
${fvad_LIBRARIES_STATIC} ${fvad_LIBRARIES_STATIC}
${opus_LIBRARIES_STATIC} ${opus_LIBRARIES_STATIC}
${ed25519_LIBRARIES_STATIC} ${ed25519_LIBRARIES_STATIC}
soundio::static
spdlog::spdlog_header_only spdlog::spdlog_header_only
Nan::Helpers Nan::Helpers
@ -144,10 +148,10 @@ target_link_libraries(${MODULE_NAME} ${REQUIRED_LIBRARIES})
target_compile_definitions(${MODULE_NAME} PUBLIC -DNODEJS_API) target_compile_definitions(${MODULE_NAME} PUBLIC -DNODEJS_API)
add_executable(Audio-Test ${SOURCE_FILES} test/audio/main.cpp) add_executable(Audio-Test ${SOURCE_FILES} test/audio/main.cpp)
target_link_libraries(Audio-Test ${REQUIRED_LIBRARIES}) target_link_libraries(Audio-Test ${REQUIRED_LIBRARIES} soundio.a)
add_executable(Audio-Test-2 ${SOURCE_FILES} test/audio/sio.cpp) add_executable(Audio-Test-2 ${SOURCE_FILES} test/audio/sio.cpp)
target_link_libraries(Audio-Test-2 ${REQUIRED_LIBRARIES} soundio.a pulse) target_link_libraries(Audio-Test-2 ${REQUIRED_LIBRARIES} soundio.a)
add_executable(HW-UID-Test src/hwuid.cpp) add_executable(HW-UID-Test src/hwuid.cpp)
target_link_libraries(HW-UID-Test target_link_libraries(HW-UID-Test

15
native/serverconnection/exports/exports.d.ts vendored
View File

@ -144,8 +144,6 @@ declare module "teaclient_connection" {
input_default: boolean; input_default: boolean;
output_default: boolean; output_default: boolean;
device_index: number;
} }
export namespace playback { export namespace playback {
@ -174,8 +172,8 @@ declare module "teaclient_connection" {
delete(); delete();
} }
export function set_device(device: number); export function set_device(device_id: string);
export function current_device() : number; export function current_device() : string;
export function create_stream() : OwnedAudioOutputStream; export function create_stream() : OwnedAudioOutputStream;
@ -234,10 +232,10 @@ declare module "teaclient_connection" {
} }
export interface AudioRecorder { export interface AudioRecorder {
get_device() : number; get_device() : string;
set_device(device: number, callback: (flag: boolean | string) => void); /* Recorder needs to be started afterwards */ set_device(device_id: string, callback: () => void); /* Recorder needs to be started afterwards */
start(callback: (flag: boolean) => void); start(callback: (result: boolean | string) => void);
started() : boolean; started() : boolean;
stop(); stop();
@ -252,7 +250,8 @@ declare module "teaclient_connection" {
export function create_recorder() : AudioRecorder; export function create_recorder() : AudioRecorder;
} }
export function initialize(); export function initialize(callback: () => any);
export function initialized() : boolean;
export function available_devices() : AudioDevice[]; export function available_devices() : AudioDevice[];
} }
} }

102
native/serverconnection/src/audio/AudioDevice.cpp
View File

@ -1,102 +0,0 @@
#include "AudioDevice.h"
#include "../logger.h"
using namespace std;
using namespace tc;
using namespace tc::audio;
extern bool devices_cached(); /* if the result is false then the call to devices() may take a while */
extern std::deque<std::shared_ptr<AudioDevice>> devices();
bool _devices_cached = false;
std::mutex _audio_devices_lock;
std::deque<std::shared_ptr<AudioDevice>> _audio_devices{};
bool audio::devices_cached() {
return _devices_cached;
}
void audio::clear_device_cache() {
std::lock_guard lock(_audio_devices_lock);
_audio_devices.clear();
_devices_cached = false;
}
deque<shared_ptr<AudioDevice>> audio::devices() {
std::lock_guard lock(_audio_devices_lock);
if(_devices_cached)
return _audio_devices;
/* query devices */
auto device_count = Pa_GetDeviceCount();
if(device_count < 0) {
log_error(category::audio, tr("Pa_GetDeviceCount() returned {}"), device_count);
return {};
}
auto default_input_device = Pa_GetDefaultInputDevice();
auto default_output_device = Pa_GetDefaultOutputDevice();
for(PaDeviceIndex device_index = 0; device_index < device_count; device_index++) {
auto device_info = Pa_GetDeviceInfo(device_index);
if(!device_info) {
log_warn(category::audio, tr("Pa_GetDeviceInfo(...) failed for device {}"), device_index);
continue;
}
auto device_host_info = Pa_GetHostApiInfo(device_info->hostApi);
if(!device_host_info) {
log_warn(category::audio, tr("Pa_GetHostApiInfo(...) failed for device {} with host api {}"), device_index, device_info->hostApi);
continue;
}
auto info = make_shared<AudioDevice>();
info->device_id = device_index;
info->name = device_info->name;
info->max_inputs = device_info->maxInputChannels;
info->input_supported = device_info->maxInputChannels > 0;
info->max_outputs = device_info->maxOutputChannels;
info->output_supported = device_info->maxOutputChannels > 0;
info->is_default_input = device_index == default_input_device;
info->is_default_output = device_index == default_output_device;
info->driver = device_host_info->name;
info->is_default_driver_input = device_index == device_host_info->defaultInputDevice;
info->is_default_driver_output = device_index == device_host_info->defaultOutputDevice;
PaStreamParameters test_parameters{};
test_parameters.device = device_index;
test_parameters.sampleFormat = paFloat32;
test_parameters.suggestedLatency = 0; /* ignored by Pa_IsFormatSupported() */
test_parameters.hostApiSpecificStreamInfo = nullptr;
/*
if(info->input_supported) {
test_parameters.channelCount = device_info->maxInputChannels;
for(size_t index = 0; standard_sample_rates[index] > 0; index++) {
auto rate = standard_sample_rates[index];
auto err = Pa_IsFormatSupported(&test_parameters, nullptr, rate);
if(err == paFormatIsSupported)
info->supported_input_rates.push_back(rate);
}
}
if(info->output_supported) {
test_parameters.channelCount = device_info->maxOutputChannels;
for(size_t index = 0; standard_sample_rates[index] > 0; index++) {
auto rate = standard_sample_rates[index];
auto err = Pa_IsFormatSupported(nullptr, &test_parameters, rate);
if(err == paFormatIsSupported)
info->supported_output_rates.push_back(rate);
}
}
*/
_audio_devices.push_back(info);
}
_devices_cached = true;
return _audio_devices;
}

44
native/serverconnection/src/audio/AudioDevice.h
View File

@ -1,44 +0,0 @@
#pragma once
#include <mutex>
#include <deque>
#include <memory>
#include <iostream>
#include <functional>
#include <portaudio.h>
namespace tc {
namespace audio {
static constexpr double standard_sample_rates[] = {
8000.0, 9600.0, 11025.0, 12000.0, 16000.0, 22050.0, 24000.0, 32000.0,
44100.0, 48000.0, 88200.0, 96000.0, 192000.0, -1 /* negative terminated list */
};
struct AudioDevice {
PaDeviceIndex device_id;
bool is_default_output;
bool is_default_input;
bool is_default_driver_output;
bool is_default_driver_input;
bool input_supported;
bool output_supported;
std::string name;
std::string driver;
/*
std::vector<double> supported_input_rates;
std::vector<double> supported_output_rates;
*/
int max_inputs;
int max_outputs;
};
extern void clear_device_cache();
extern bool devices_cached(); /* if the result is false then the call to devices() may take a while */
extern std::deque<std::shared_ptr<AudioDevice>> devices();
}
}

65
native/serverconnection/src/audio/AudioInput.cpp
View File

@ -9,6 +9,7 @@ using namespace std;
using namespace tc; using namespace tc;
using namespace tc::audio; using namespace tc::audio;
#if false
class AudioInputSource { class AudioInputSource {
public: public:
constexpr static auto kChannelCount{2}; constexpr static auto kChannelCount{2};
@ -121,7 +122,7 @@ class AudioInputSource {
std::lock_guard lock{input_source->registered_inputs_lock}; std::lock_guard lock{input_source->registered_inputs_lock};
for(auto& client : input_source->registered_inputs) for(auto& client : input_source->registered_inputs)
client->audio_callback(input, frameCount, timeInfo, statusFlags); client->consume(input, frameCount, 2);
return 0; return 0;
} }
@ -152,6 +153,7 @@ std::shared_ptr<AudioInputSource> get_input_source(PaDeviceIndex device_index, b
input_sources.push_back(std::make_shared<AudioInputSource>(device_index)); input_sources.push_back(std::make_shared<AudioInputSource>(device_index));
return input; return input;
} }
#endif
AudioConsumer::AudioConsumer(tc::audio::AudioInput *handle, size_t channel_count, size_t sample_rate, size_t frame_size) : AudioConsumer::AudioConsumer(tc::audio::AudioInput *handle, size_t channel_count, size_t sample_rate, size_t frame_size) :
handle(handle), handle(handle),
@ -189,50 +191,57 @@ AudioInput::~AudioInput() {
consumer->handle = nullptr; consumer->handle = nullptr;
} }
PaDeviceIndex AudioInput::current_device() { void AudioInput::set_device(const std::shared_ptr<AudioDevice> &device) {
lock_guard lock(this->input_source_lock); lock_guard lock(this->input_source_lock);
return this->input_source ? this->input_source->device_index : paNoDevice; if(device == this->input_device) return;
}
bool AudioInput::open_device(std::string& error, PaDeviceIndex index) {
lock_guard lock(this->input_source_lock);
if(index == (this->input_source ? this->input_source->device_index : paNoDevice))
return true;
this->close_device(); this->close_device();
if(index == paNoDevice) this->input_device = device;
return true;
this->input_source = get_input_source(index, true);
this->input_source->register_consumer(this);
return this->input_source->begin_recording(error);
} }
void AudioInput::close_device() { void AudioInput::close_device() {
lock_guard lock(this->input_source_lock); lock_guard lock(this->input_source_lock);
if(this->input_source) { if(this->input_recorder) {
this->input_source->remove_consumer(this); this->input_recorder->remove_consumer(this);
this->input_source->stop_recording_if_possible(); this->input_recorder->stop_if_possible();
this->input_source.reset(); this->input_recorder.reset();
} }
this->input_recording = false; this->input_device = nullptr;
} }
bool AudioInput::record() { bool AudioInput::record(std::string& error) {
lock_guard lock(this->input_source_lock); lock_guard lock(this->input_source_lock);
if(!this->input_source) return false; if(!this->input_device) {
error = "no device";
return false;
}
if(this->input_recorder) return true;
this->input_recording = true; this->input_recorder = this->input_device->record();
if(!this->input_recorder) {
error = "failed to get recorder";
return false;
}
this->input_recorder->register_consumer(this);
if(!this->input_recorder->start(error)) {
this->input_recorder->remove_consumer(this);
this->input_recorder.reset();
return false;
}
return true; return true;
} }
bool AudioInput::recording() { bool AudioInput::recording() {
return this->input_recording; return !!this->input_recorder;
} }
void AudioInput::stop() { void AudioInput::stop() {
this->input_recording = false; if(!this->input_recorder) return;
this->input_recorder->remove_consumer(this);
this->input_recorder->stop_if_possible();
this->input_recorder.reset();
} }
std::shared_ptr<AudioConsumer> AudioInput::create_consumer(size_t frame_length) { std::shared_ptr<AudioConsumer> AudioInput::create_consumer(size_t frame_length) {
@ -255,9 +264,7 @@ void AudioInput::delete_consumer(const std::shared_ptr<AudioConsumer> &source) {
source->handle = nullptr; source->handle = nullptr;
} }
void AudioInput::audio_callback(const void *input, unsigned long frameCount, const PaStreamCallbackTimeInfo* timeInfo, PaStreamCallbackFlags statusFlags) { void AudioInput::consume(const void *input, unsigned long frameCount, size_t /* channels */) {
if(!this->input_recording) return;
if(this->_volume != 1 && false) { if(this->_volume != 1 && false) {
auto ptr = (float*) input; auto ptr = (float*) input;
auto left = frameCount * this->_channel_count; auto left = frameCount * this->_channel_count;

21
native/serverconnection/src/audio/AudioInput.h
View File

@ -5,9 +5,9 @@
#include <memory> #include <memory>
#include <iostream> #include <iostream>
#include <functional> #include <functional>
#include <portaudio.h>
#include <misc/spin_lock.h> #include <misc/spin_lock.h>
#include "AudioSamples.h" #include "AudioSamples.h"
#include "driver/AudioDriver.h"
class AudioInputSource; class AudioInputSource;
namespace tc { namespace tc {
@ -31,25 +31,22 @@ namespace tc {
AudioConsumer(AudioInput* handle, size_t channel_count, size_t sample_rate, size_t frame_size); AudioConsumer(AudioInput* handle, size_t channel_count, size_t sample_rate, size_t frame_size);
std::unique_ptr<Reframer> reframer; std::unique_ptr<Reframer> reframer;
std::mutex buffer_lock;
size_t buffered_samples = 0;
std::deque<std::shared_ptr<SampleBuffer>> sample_buffers;
void process_data(const void* /* buffer */, size_t /* samples */); void process_data(const void* /* buffer */, size_t /* samples */);
void handle_framed_data(const void* /* buffer */, size_t /* samples */); void handle_framed_data(const void* /* buffer */, size_t /* samples */);
}; };
class AudioInput { class AudioInput : public AudioDeviceRecord::Consumer {
friend class ::AudioInputSource; friend class ::AudioInputSource;
public: public:
AudioInput(size_t /* channels */, size_t /* rate */); AudioInput(size_t /* channels */, size_t /* rate */);
virtual ~AudioInput(); virtual ~AudioInput();
[[nodiscard]] bool open_device(std::string& /* error */, PaDeviceIndex); void set_device(const std::shared_ptr<AudioDevice>& /* device */);
[[nodiscard]] PaDeviceIndex current_device(); [[nodiscard]] std::shared_ptr<AudioDevice> current_device() const { return this->input_device; }
void close_device(); void close_device();
[[nodiscard]] bool record(); [[nodiscard]] bool record(std::string& /* error */);
[[nodiscard]] bool recording(); [[nodiscard]] bool recording();
void stop(); void stop();
@ -67,7 +64,7 @@ namespace tc {
inline float volume() { return this->_volume; } inline float volume() { return this->_volume; }
inline void set_volume(float value) { this->_volume = value; } inline void set_volume(float value) { this->_volume = value; }
private: private:
void audio_callback(const void *, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags); void consume(const void *, unsigned long, size_t) override;
size_t const _channel_count; size_t const _channel_count;
size_t const _sample_rate; size_t const _sample_rate;
@ -76,10 +73,10 @@ namespace tc {
std::deque<std::shared_ptr<AudioConsumer>> _consumers; std::deque<std::shared_ptr<AudioConsumer>> _consumers;
std::recursive_mutex input_source_lock; std::recursive_mutex input_source_lock;
bool input_recording{false}; std::shared_ptr<AudioDevice> input_device{};
std::shared_ptr<::AudioInputSource> input_source{};
float _volume = 1.f; float _volume = 1.f;
std::shared_ptr<AudioDeviceRecord> input_recorder{};
}; };
} }
} }

280
native/serverconnection/src/audio/AudioOutput.cpp
View File

@ -10,110 +10,127 @@ using namespace tc;
using namespace tc::audio; using namespace tc::audio;
void AudioOutputSource::clear() { void AudioOutputSource::clear() {
lock_guard lock(this->buffer_lock); this->buffer.clear();
this->sample_buffers.clear();
this->buffered_samples = 0;
}
ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) {
auto sample_count = samples;
_retest:
{
lock_guard lock(this->buffer_lock);
if(this->buffering) {
if(this->buffered_samples > this->min_buffer) {
this->buffering = false;
} else {
return 0;
}
}
while(sample_count > 0 && !this->sample_buffers.empty()) {
auto buf = this->sample_buffers[0];
auto sc = min((size_t) (buf->sample_size - buf->sample_index), (size_t) sample_count);
if(sc > 0 && buffer) { /* just to ensure */
memcpy(buffer, (char *) buf->sample_data + this->channel_count * buf->sample_index * 4, sc * this->channel_count * 4);
} else {
#ifndef WIN32
/* for my debugger */
__asm__("nop");
#endif
}
sample_count -= sc;
buf->sample_index += (uint16_t) sc;
if(buf->sample_index == buf->sample_size)
this->sample_buffers.pop_front();
if(buffer)
buffer = (char*) buffer + sc * this->channel_count * 4;
}
this->buffered_samples -= samples - sample_count;
}
if(sample_count > 0) {
if(this->on_underflow) {
if(this->on_underflow()) {
goto _retest;
}
}
this->buffering = true; this->buffering = true;
} }
ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) {
size_t written{0}, written_bytes{0};
load_buffer:
auto available_bytes = this->buffer.fill_count();
if(available_bytes < sizeof(float) * this->channel_count) return written;
auto available_samples = available_bytes / sizeof(float) / this->channel_count;
//log_trace(category::audio, tr("Min: {}, Max: {}, Current: {}, Buffering: {}"), this->min_buffered_samples, this->max_buffered_samples, available_samples, this->buffering);
if(this->buffering && available_samples < this->min_buffered_samples) return -2;
this->buffering = false;
if(available_samples >= samples - written) {
const auto byte_length = (samples - written) * sizeof(float) * this->channel_count;
memcpy((char*) buffer + written_bytes, this->buffer.read_ptr(), byte_length);
this->buffer.advance_read_ptr(byte_length);
return samples;
} else {
const auto byte_length = available_samples * sizeof(float) * this->channel_count;
memcpy((char*) buffer + written_bytes, this->buffer.read_ptr(), byte_length);
this->buffer.advance_read_ptr(byte_length);
written += available_samples;
written_bytes += byte_length;
}
if(auto fn = this->on_underflow; fn)
if(fn())
goto load_buffer;
this->buffering = true;
if(this->on_read) if(this->on_read)
this->on_read(); this->on_read();
return samples - sample_count; /* return the written samples */ return written; /* return the written samples */
} }
ssize_t AudioOutputSource::enqueue_samples(const void *buffer, size_t samples) { ssize_t AudioOutputSource::enqueue_samples(const void *buffer, size_t samples) {
auto buf = SampleBuffer::allocate((uint8_t) this->channel_count, (uint16_t) samples); size_t enqueued{0};
if(!buf)
return -1;
buf->sample_index = 0; auto free_bytes = this->buffer.free_count();
memcpy(buf->sample_data, buffer, this->channel_count * samples * 4); auto free_samples = free_bytes / sizeof(float) / this->channel_count;
if(this->max_buffered_samples && free_samples > this->max_buffered_samples) free_samples = this->max_buffered_samples;
return this->enqueue_samples(buf); if(free_samples >= samples) {
const auto byte_length = samples * sizeof(float) * this->channel_count;
memcpy(this->buffer.write_ptr(), buffer, byte_length);
this->buffer.advance_write_ptr(byte_length);
return samples;
} else {
const auto byte_length = free_samples * sizeof(float) * this->channel_count;
memcpy(this->buffer.write_ptr(), buffer, byte_length);
this->buffer.advance_write_ptr(byte_length);
enqueued += free_samples;
} }
ssize_t AudioOutputSource::enqueue_samples(const std::shared_ptr<tc::audio::SampleBuffer> &buf) { if(auto fn = this->on_overflow; fn)
if(!buf) return 0; fn(samples - enqueued);
{
unique_lock lock(this->buffer_lock);
if(this->max_latency > 0 && this->buffered_samples + buf->sample_size > this->max_latency) {
/* overflow! */
auto overflow_length = this->buffered_samples + buf->sample_size - this->max_latency;
if(this->on_overflow) {
lock.unlock();
this->on_overflow(overflow_length);
lock.lock();
}
switch (this->overflow_strategy) { switch (this->overflow_strategy) {
case overflow_strategy::discard_input: case overflow_strategy::discard_input:
return -2; return -2;
case overflow_strategy::discard_buffer_all: case overflow_strategy::discard_buffer_all:
this->sample_buffers.clear(); this->buffer.clear();
break; break;
case overflow_strategy::discard_buffer_half: case overflow_strategy::discard_buffer_half:
this->sample_buffers.erase(this->sample_buffers.begin(), this->sample_buffers.begin() + (int) ceil(this->sample_buffers.size() / 2)); this->buffer.advance_read_ptr(this->buffer.fill_count() / 2);
break; break;
case overflow_strategy::ignore: case overflow_strategy::ignore:
break; break;
} }
return enqueued;
} }
this->sample_buffers.push_back(buf); ssize_t AudioOutputSource::enqueue_samples_no_interleave(const void *buffer, size_t samples) {
this->buffered_samples += buf->sample_size; auto free_bytes = this->buffer.free_count();
auto free_samples = free_bytes / sizeof(float) / this->channel_count;
if(this->max_buffered_samples && free_samples > this->max_buffered_samples) free_samples = this->max_buffered_samples;
auto samples_to_write{samples};
if(samples_to_write > free_samples) samples_to_write = free_samples;
const auto enqueued{samples_to_write};
{
auto src_buffer = (const float*) buffer;
auto target_buffer = (float*) this->buffer.write_ptr();
while (samples_to_write-- > 0) {
*target_buffer = *src_buffer;
*(target_buffer + 1) = *(src_buffer + samples);
target_buffer += 2;
src_buffer++;
}
} }
return buf->sample_size; if(auto fn = this->on_overflow; fn)
fn(samples - enqueued);
switch (this->overflow_strategy) {
case overflow_strategy::discard_input:
return -2;
case overflow_strategy::discard_buffer_all:
this->buffer.clear();
break;
case overflow_strategy::discard_buffer_half:
this->buffer.advance_read_ptr(this->buffer.fill_count() / 2);
break;
case overflow_strategy::ignore:
break;
}
return enqueued;
} }
AudioOutput::AudioOutput(size_t channels, size_t rate) : _channel_count(channels), _sample_rate(rate) { } AudioOutput::AudioOutput(size_t channels, size_t rate) : _channel_count(channels), _sample_rate(rate) { }
AudioOutput::~AudioOutput() { AudioOutput::~AudioOutput() {
this->close_device(); this->close_device();
this->cleanup_buffers(); this->cleanup_buffers();
@ -152,30 +169,32 @@ void AudioOutput::cleanup_buffers() {
this->source_buffer_length = 0; this->source_buffer_length = 0;
} }
int AudioOutput::_audio_callback(const void *a, void *b, unsigned long c, const PaStreamCallbackTimeInfo* d, PaStreamCallbackFlags e, void *_ptr_audio_output) { void AudioOutput::fill_buffer(void *output, unsigned long frameCount, size_t channels) {
return reinterpret_cast<AudioOutput*>(_ptr_audio_output)->audio_callback(a, b, c, d, e); lock_guard buffer_lock(this->buffer_lock);
if(this->_volume <= 0) {
for(auto& source : this->_sources)
source->pop_samples(nullptr, frameCount);
memset(output, 0, sizeof(frameCount) * channels * sizeof(float));
return;
} }
int AudioOutput::audio_callback(const void *input, void *output, unsigned long frameCount, const PaStreamCallbackTimeInfo* timeInfo, PaStreamCallbackFlags statusFlags) {
if(!output) /* hmmm.. suspicious */
return 0;
lock_guard buffer_lock(this->buffer_lock);
size_t buffer_length = frameCount * 4 * this->_channel_count; size_t buffer_length = frameCount * 4 * this->_channel_count;
size_t sources = 0; size_t sources = 0;
size_t actual_sources = 0; size_t actual_sources = 0;
auto volume = this->_volume;
{ {
lock_guard lock(this->sources_lock); lock_guard lock(this->sources_lock);
sources = this->_sources.size(); sources = this->_sources.size();
actual_sources = sources; actual_sources = sources;
if(sources > 0) { if(sources > 0) {
if(volume > 0) { /* allocate the required space */ /* allocate the required space */
auto source_buffer_length = buffer_length * sources; auto source_buffer_length = buffer_length * sources;
auto source_merge_buffer_length = sizeof(void*) * sources; auto source_merge_buffer_length = sizeof(void*) * sources;
//TODO: Move this out of the loop?
{
if(this->source_buffer_length < source_buffer_length || !this->source_buffer) { if(this->source_buffer_length < source_buffer_length || !this->source_buffer) {
if(this->source_buffer) if(this->source_buffer)
free(this->source_buffer); free(this->source_buffer);
@ -193,11 +212,10 @@ int AudioOutput::audio_callback(const void *input, void *output, unsigned long f
for(size_t index = 0; index < sources; index++) { for(size_t index = 0; index < sources; index++) {
auto& source = this->_sources[index]; auto& source = this->_sources[index];
if(volume > 0) {
this->source_merge_buffer[index] = (char*) this->source_buffer + (buffer_length * index); this->source_merge_buffer[index] = (char*) this->source_buffer + (buffer_length * index);
auto written_frames = this->_sources[index]->pop_samples(this->source_merge_buffer[index], frameCount); auto written_frames = this->_sources[index]->pop_samples(this->source_merge_buffer[index], frameCount);
if(written_frames != frameCount) { if(written_frames != frameCount) {
if(written_frames == 0) { if(written_frames <= 0) {
this->source_merge_buffer[index] = nullptr; this->source_merge_buffer[index] = nullptr;
actual_sources--; actual_sources--;
} else { } else {
@ -206,82 +224,60 @@ int AudioOutput::audio_callback(const void *input, void *output, unsigned long f
memset((char*) this->source_merge_buffer[index] + written, 0, (frameCount - written_frames) * this->_channel_count * 4); memset((char*) this->source_merge_buffer[index] + written, 0, (frameCount - written_frames) * this->_channel_count * 4);
} }
} }
} else {
this->_sources[index]->pop_samples(nullptr, frameCount);
}
} }
} }
} }
if(actual_sources > 0 && volume > 0) { if(actual_sources > 0) {
if(!merge::merge_n_sources(output, this->source_merge_buffer, sources, this->_channel_count, frameCount)) { if(!merge::merge_n_sources(output, this->source_merge_buffer, sources, this->_channel_count, frameCount))
log_warn(category::audio, tr("failed to merge buffers!")); log_warn(category::audio, tr("failed to merge buffers!"));
}
auto volume = this->_volume;
if(volume != 1) {
auto float_length = this->_channel_count * frameCount; auto float_length = this->_channel_count * frameCount;
auto data = (float*) output; auto data = (float*) output;
while(float_length-- > 0) while(float_length-- > 0)
*data++ *= volume; *data++ *= volume;
}
} else { } else {
memset(output, 0, this->_channel_count * 4 * frameCount); memset(output, 0, this->_channel_count * sizeof(float) * frameCount);
}
} }
return 0; void AudioOutput::set_device(const std::shared_ptr<AudioDevice> &device) {
} lock_guard lock(this->device_lock);
if(this->device == device) return;
bool AudioOutput::open_device(std::string& error, PaDeviceIndex index) {
lock_guard lock(this->output_stream_lock);
if(index == this->_current_device_index)
return true;
this->close_device(); this->close_device();
this->_current_device_index = index; this->device = device;
this->_current_device = Pa_GetDeviceInfo(index);
if(!this->_current_device) {
this->_current_device_index = paNoDevice;
error = "failed to get device info";
return false;
}
PaStreamParameters output_parameters{};
memset(&output_parameters, 0, sizeof(output_parameters));
output_parameters.channelCount = (int) this->_channel_count;
output_parameters.device = this->_current_device_index;
output_parameters.sampleFormat = paFloat32;
output_parameters.suggestedLatency = this->_current_device->defaultLowOutputLatency;
auto err = Pa_OpenStream(&output_stream, nullptr, &output_parameters, (double) this->_sample_rate, paFramesPerBufferUnspecified, paClipOff, &AudioOutput::_audio_callback, this);
if(err != paNoError) {
error = to_string(err) + "/" + Pa_GetErrorText(err);
return false;
}
return true;
}
bool AudioOutput::playback() {
lock_guard lock(this->output_stream_lock);
if(!this->output_stream)
return false;
auto err = Pa_StartStream(this->output_stream);
if(err != paNoError && err != paStreamIsNotStopped) {
log_error(category::audio, tr("Pa_StartStream returned {}"), err);
return false;
}
return true;
} }
void AudioOutput::close_device() { void AudioOutput::close_device() {
lock_guard lock(this->output_stream_lock); lock_guard lock(this->device_lock);
if(this->output_stream) { if(this->_playback) {
/* TODO: Test for errors */ this->_playback->remove_source(this);
Pa_StopStream(this->output_stream); this->_playback->stop_if_possible();
Pa_CloseStream(this->output_stream); this->_playback.reset();
}
this->output_stream = nullptr; this->device = nullptr;
} }
bool AudioOutput::playback(std::string& error) {
lock_guard lock(this->device_lock);
if(!this->device) {
error = "invalid device handle";
return false;
}
if(this->_playback) return true;
this->_playback = this->device->playback();
if(!this->_playback) {
error = "failed to allocate memory";
return false;
}
this->_playback->register_source(this);
return this->_playback->start(error);
} }

57
native/serverconnection/src/audio/AudioOutput.h
View File

@ -5,15 +5,15 @@
#include <memory> #include <memory>
#include <iostream> #include <iostream>
#include <functional> #include <functional>
#include <portaudio.h> #include "./AudioSamples.h"
#include "AudioSamples.h" #include "./driver/AudioDriver.h"
#include "../ring_buffer.h"
#ifdef WIN32 #ifdef WIN32
#define ssize_t int64_t #define ssize_t int64_t
#endif #endif
namespace tc { namespace tc::audio {
namespace audio {
class AudioOutput; class AudioOutput;
namespace overflow_strategy { namespace overflow_strategy {
@ -32,13 +32,21 @@ namespace tc {
size_t const channel_count = 0; size_t const channel_count = 0;
size_t const sample_rate = 0; size_t const sample_rate = 0;
bool buffering = true; [[nodiscard]] inline size_t max_supported_latency() const {
size_t min_buffer = 0; return this->buffer.capacity() / this->channel_count / sizeof(float);
}
[[nodiscard]] inline size_t max_latency() const {
const auto max_samples = this->max_supported_latency();
if(this->max_buffered_samples && this->max_buffered_samples <= max_samples) return this->max_buffered_samples;
return max_samples;
}
bool buffering{true};
size_t min_buffered_samples{0};
size_t max_buffered_samples{0};
/* For stream set it to the max latency in samples you want.
* Zero means unlimited
*/
size_t max_latency = 0;
overflow_strategy::value overflow_strategy = overflow_strategy::discard_buffer_half; overflow_strategy::value overflow_strategy = overflow_strategy::discard_buffer_half;
/* if it returns true then the it means that the buffer has been refilled, we have to test again */ /* if it returns true then the it means that the buffer has been refilled, we have to test again */
@ -49,24 +57,24 @@ namespace tc {
void clear(); void clear();
ssize_t pop_samples(void* /* output buffer */, size_t /* sample count */); ssize_t pop_samples(void* /* output buffer */, size_t /* sample count */);
ssize_t enqueue_samples(const void * /* input buffer */, size_t /* sample count */); ssize_t enqueue_samples(const void * /* input buffer */, size_t /* sample count */);
ssize_t enqueue_samples(const std::shared_ptr<SampleBuffer>& /* buffer */); ssize_t enqueue_samples_no_interleave(const void * /* input buffer */, size_t /* sample count */);
private: private:
AudioOutputSource(AudioOutput* handle, size_t channel_count, size_t sample_rate) : handle(handle), channel_count(channel_count), sample_rate(sample_rate) {} AudioOutputSource(AudioOutput* handle, size_t channel_count, size_t sample_rate) :
handle(handle), channel_count(channel_count), sample_rate(sample_rate), buffer{channel_count * sample_rate * sizeof(float)} {
}
std::mutex buffer_lock; tc::ring_buffer buffer;
size_t buffered_samples = 0;
std::deque<std::shared_ptr<SampleBuffer>> sample_buffers;
}; };
class AudioOutput { class AudioOutput : public AudioDevicePlayback::Source {
public: public:
AudioOutput(size_t /* channels */, size_t /* rate */); AudioOutput(size_t /* channels */, size_t /* rate */);
virtual ~AudioOutput(); virtual ~AudioOutput();
bool open_device(std::string& /* error */, PaDeviceIndex); void set_device(const std::shared_ptr<AudioDevice>& /* device */);
bool playback(); bool playback(std::string& /* error */);
void close_device(); void close_device();
PaDeviceIndex current_device() { return this->_current_device_index; } std::shared_ptr<AudioDevice> current_device() { return this->device; }
std::deque<std::shared_ptr<AudioOutputSource>> sources() { std::deque<std::shared_ptr<AudioOutputSource>> sources() {
std::lock_guard lock(this->sources_lock); std::lock_guard lock(this->sources_lock);
@ -82,8 +90,7 @@ namespace tc {
inline float volume() { return this->_volume; } inline float volume() { return this->_volume; }
inline void set_volume(float value) { this->_volume = value; } inline void set_volume(float value) { this->_volume = value; }
private: private:
static int _audio_callback(const void *, void *, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags, void*); void fill_buffer(void *, size_t frames, size_t channels) override ;
int audio_callback(const void *, void *, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags);
size_t const _channel_count; size_t const _channel_count;
size_t const _sample_rate; size_t const _sample_rate;
@ -91,10 +98,9 @@ namespace tc {
std::mutex sources_lock; std::mutex sources_lock;
std::deque<std::shared_ptr<AudioOutputSource>> _sources; std::deque<std::shared_ptr<AudioOutputSource>> _sources;
std::recursive_mutex output_stream_lock; std::recursive_mutex device_lock;
const PaDeviceInfo* _current_device = nullptr; std::shared_ptr<AudioDevice> device{nullptr};
PaDeviceIndex _current_device_index = paNoDevice; std::shared_ptr<AudioDevicePlayback> _playback{nullptr};
PaStream* output_stream = nullptr;
std::mutex buffer_lock; /* not required, but why not. Usually only used within audio_callback! */ std::mutex buffer_lock; /* not required, but why not. Usually only used within audio_callback! */
void* source_buffer = nullptr; void* source_buffer = nullptr;
@ -107,4 +113,3 @@ namespace tc {
float _volume = 1.f; float _volume = 1.f;
}; };
} }
}

219
native/serverconnection/src/audio/driver/AudioDriver.cpp Normal file
View File

@ -0,0 +1,219 @@
//
// Created by wolverindev on 07.02.20.
//
#include <thread>
#include <condition_variable>
#include <ThreadPool/ThreadHelper.h>
#include "AudioDriver.h"
#include "SoundIO.h"
#include "../../logger.h"
#include "../AudioMerger.h"
using namespace tc::audio;
namespace tc::audio {
std::deque<std::shared_ptr<AudioDevice>> devices() {
std::deque<std::shared_ptr<AudioDevice>> result{};
for(auto& backend : SoundIOBackendHandler::all_backends()) {
auto input_devices = backend->input_devices();
auto output_devices = backend->output_devices();
result.insert(result.end(), input_devices.begin(), input_devices.end());
result.insert(result.end(), output_devices.begin(), output_devices.end());
}
return result;
}
std::shared_ptr<AudioDevice> find_device_by_id(const std::string_view& id, bool input) {
for(auto& backend : SoundIOBackendHandler::all_backends()) {
for(auto& dev : input ? backend->input_devices() : backend->output_devices())
if(dev->id() == id)
return dev;
}
return nullptr;
}
std::mutex initialize_lock{};
std::deque<initialize_callback_t> initialize_callbacks{};
int initialize_state{0}; /* 0 := not initialized | 1 := initialized | 2 := initializing */
void _initialize() {
SoundIOBackendHandler::initialize_all();
SoundIOBackendHandler::connect_all();
}
void _finalize() {
SoundIOBackendHandler::shutdown_all();
}
void initialize(const initialize_callback_t& callback) {
{
std::unique_lock init_lock{initialize_lock};
if(initialize_state == 2) {
if(callback)
initialize_callbacks.push_back(callback);
return;
} else if(initialize_state == 1) {
init_lock.unlock();
callback();
return;
} else if(initialize_state != 0) {
init_lock.unlock();
callback();
log_warn(category::audio, tr("Invalid initialize state ({})"), initialize_state);
return;
}
initialize_state = 2;
}
std::thread init_thread([]{
_initialize();
std::unique_lock lock{initialize_lock};
auto callbacks = std::move(initialize_callbacks);
initialize_state = 1;
lock.unlock();
for(auto& callback : callbacks)
callback();
});
threads::name(init_thread, tr("audio init"));
init_thread.detach();
}
void await_initialized() {
std::condition_variable cv{};
std::mutex m{};
std::unique_lock init_lock{initialize_lock};
if(initialize_state != 2) return;
initialize_callbacks.emplace_back([&]{ cv.notify_all(); });
init_lock.unlock();
std::unique_lock m_lock{m};
cv.wait(m_lock);
}
bool initialized() {
std::unique_lock init_lock{initialize_lock};
return initialize_state == 1;
}
void finalize() {
await_initialized();
_finalize();
}
bool AudioDevicePlayback::start(std::string &error) {
std::lock_guard lock{this->state_lock};
if(this->running) return true;
if(!this->impl_start(error)) {
log_error(category::audio, tr("Failed to start playback: {}"), error);
return false;
}
this->running = true;
return true;
}
void AudioDevicePlayback::stop_if_possible() {
std::lock_guard lock{this->state_lock};
{
std::lock_guard s_lock{this->source_lock};
if(!this->_sources.empty()) return;
}
this->impl_stop();
this->running = false;
}
void AudioDevicePlayback::stop() {
std::lock_guard lock{this->state_lock};
if(this->running) return;
this->impl_stop();
this->running = false;
}
void AudioDevicePlayback::register_source(Source* source) {
std::lock_guard s_lock{this->source_lock};
this->_sources.push_back(source);
}
void AudioDevicePlayback::remove_source(Source* source) {
std::lock_guard s_lock{this->source_lock};
auto index = find(this->_sources.begin(), this->_sources.end(), source);
if(index == this->_sources.end()) return;
this->_sources.erase(index);
}
#define TMP_BUFFER_SIZE 8096
void AudioDevicePlayback::fill_buffer(void *buffer, size_t samples, size_t channels) {
std::lock_guard lock{this->source_lock};
const auto size = this->_sources.size();
if(size == 1) {
this->_sources.front()->fill_buffer(buffer, samples, channels);
} else if(size > 1) {
this->_sources.front()->fill_buffer(buffer, samples, channels);
uint8_t tmp_buffer[TMP_BUFFER_SIZE];
if(sizeof(float) * samples * channels > TMP_BUFFER_SIZE) {
log_warn(category::audio, tr("Dropping input source data because of too small merge buffer"));
return;
}
for(auto it = this->_sources.begin() + 1; it != this->_sources.end(); it++) {
(*it)->fill_buffer(tmp_buffer, samples, channels);
merge::merge_sources(buffer, buffer, tmp_buffer, channels, samples);
}
} else {
memset(buffer, 0, samples * channels * sizeof(float));
}
}
bool AudioDeviceRecord::start(std::string &error) {
std::lock_guard lock{this->state_lock};
if(this->running) return true;
if(!this->impl_start(error)) {
log_error(category::audio, tr("Failed to start record: {}"), error);
return false;
}
this->running = true;
return true;
}
void AudioDeviceRecord::stop_if_possible() {
std::lock_guard lock{this->state_lock};
{
std::lock_guard s_lock{this->consumer_lock};
if(!this->_consumers.empty()) return;
}
this->impl_stop();
this->running = false;
}
void AudioDeviceRecord::stop() {
std::lock_guard lock{this->state_lock};
if(this->running) return;
this->impl_stop();
this->running = false;
}
void AudioDeviceRecord::register_consumer(Consumer* source) {
std::lock_guard s_lock{this->consumer_lock};
this->_consumers.push_back(source);
}
void AudioDeviceRecord::remove_consumer(Consumer* source) {
std::lock_guard s_lock{this->consumer_lock};
auto index = find(this->_consumers.begin(), this->_consumers.end(), source);
if(index == this->_consumers.end()) return;
this->_consumers.erase(index);
}
}

96
native/serverconnection/src/audio/driver/AudioDriver.h Normal file
View File

@ -0,0 +1,96 @@
#pragma once
#include <mutex>
#include <deque>
#include <memory>
#include <iostream>
#include <functional>
namespace tc::audio {
class AudioDeviceRecord {
public:
class Consumer {
public:
virtual void consume(const void* /* buffer */, size_t /* samples */, size_t /* channel count */) = 0;
};
[[nodiscard]] bool start(std::string& /* error */);
void stop_if_possible();
void stop();
[[nodiscard]] inline std::vector<Consumer*> consumer() {
std::lock_guard lock{this->consumer_lock};
return this->_consumers;
}
void register_consumer(Consumer* /* source */);
void remove_consumer(Consumer* /* source */);
protected:
virtual bool impl_start(std::string& /* error */) = 0;
virtual void impl_stop() = 0;
std::mutex state_lock{};
bool running{false};
std::mutex consumer_lock{};
std::vector<Consumer*> _consumers{};
};
class AudioDevicePlayback {
public:
class Source {
public:
virtual void fill_buffer(void* /* target */, size_t /* samples */, size_t /* channel count */) = 0;
};
[[nodiscard]] bool start(std::string& /* error */);
void stop_if_possible();
void stop();
[[nodiscard]] inline std::vector<Source*> sources() {
std::lock_guard lock{this->source_lock};
return this->_sources;
}
void register_source(Source* /* source */);
void remove_source(Source* /* source */);
protected:
virtual bool impl_start(std::string& /* error */) = 0;
virtual void impl_stop() = 0;
void fill_buffer(void* /* target */, size_t /* samples */, size_t /* channel count */);
std::mutex state_lock{};
bool running{false};
std::mutex source_lock{};
std::vector<Source*> _sources{};
};
class AudioDevice {
public:
/* information */
[[nodiscard]] virtual std::string id() const = 0;
[[nodiscard]] virtual std::string name() const = 0;
[[nodiscard]] virtual std::string driver() const = 0;
[[nodiscard]] virtual bool is_input_supported() const = 0;
[[nodiscard]] virtual bool is_output_supported() const = 0;
[[nodiscard]] virtual bool is_input_default() const = 0;
[[nodiscard]] virtual bool is_output_default() const = 0;
[[nodiscard]] virtual std::shared_ptr<AudioDevicePlayback> playback() = 0;
[[nodiscard]] virtual std::shared_ptr<AudioDeviceRecord> record() = 0;
};
typedef std::function<void()> initialize_callback_t;
extern void finalize();
extern void initialize(const initialize_callback_t& /* callback */ = []{});
extern void await_initialized();
extern bool initialized();
extern std::deque<std::shared_ptr<AudioDevice>> devices();
extern std::shared_ptr<AudioDevice> find_device_by_id(const std::string_view& /* id */, bool /* input */);
}

260
native/serverconnection/src/audio/driver/SoundIO.cpp Normal file
View File

@ -0,0 +1,260 @@
//
// Created by wolverindev on 07.02.20.
//
#include "SoundIO.h"
#include <algorithm>
#include "../../logger.h"
using namespace tc::audio;
std::mutex SoundIOBackendHandler::backend_lock{};
std::vector<std::shared_ptr<SoundIOBackendHandler>> SoundIOBackendHandler::backends{};
std::shared_ptr<SoundIOBackendHandler> SoundIOBackendHandler::get_backend(SoundIoBackend backend_type) {
std::lock_guard lock{backend_lock};
for(auto& backend : SoundIOBackendHandler::backends)
if(backend->backend == backend_type)
return backend;
return nullptr;
}
void SoundIOBackendHandler::initialize_all() {
std::lock_guard lock{backend_lock};
for(const auto& backend : {
SoundIoBackendJack,
SoundIoBackendPulseAudio,
SoundIoBackendAlsa,
SoundIoBackendCoreAudio,
SoundIoBackendWasapi,
SoundIoBackendDummy
}) {
if(!soundio_have_backend(backend)) {
log_debug(category::audio, tr("Skipping audio backend {} because its not supported on this platform."), soundio_backend_name(backend));
continue;
}
auto handler = std::make_shared<SoundIOBackendHandler>(backend);
if(std::string error{}; !handler->initialize(error)) {
log_error(category::audio, tr("Failed to initialize sound backed {}: {}"), soundio_backend_name(backend), error);
continue;
}
backends.push_back(handler);
}
std::stable_sort(backends.begin(), backends.end(), [](const auto& a, const auto& b) { return a->priority() > b->priority(); });
}
void SoundIOBackendHandler::connect_all() {
std::string error{};
std::lock_guard lock{backend_lock};
for(const auto& backend : backends)
if(!backend->connect(error))
log_error(category::audio, tr("Failed to connect to audio backend {}: {}"), backend->name(), error);
}
void SoundIOBackendHandler::shutdown_all() {
std::lock_guard lock{backend_lock};
for(auto& entry : backends)
entry->shutdown();
backends.clear();
}
SoundIOBackendHandler::SoundIOBackendHandler(SoundIoBackend backed) : backend{backed} {}
SoundIOBackendHandler::~SoundIOBackendHandler() {
this->shutdown();
}
bool SoundIOBackendHandler::initialize(std::string &error) {
assert(!this->soundio_handle);
this->soundio_handle = soundio_create();
if(!this->soundio_handle) {
error = "out of memory";
return false;
}
this->soundio_handle->userdata = this;
this->soundio_handle->on_devices_change = [](auto handle){
reinterpret_cast<SoundIOBackendHandler*>(handle->userdata)->handle_device_change();
};
this->soundio_handle->on_backend_disconnect = [](auto handle, auto err){
reinterpret_cast<SoundIOBackendHandler*>(handle->userdata)->handle_backend_disconnect(err);
};
return true;
}
void SoundIOBackendHandler::shutdown() {
if(!this->soundio_handle) return;
soundio_destroy(this->soundio_handle);
this->soundio_handle = nullptr;
}
bool SoundIOBackendHandler::connect(std::string &error, bool enforce) {
if(!this->soundio_handle) {
error = "invalid handle";
return false;
}
if(this->_connected && !enforce) {
error = "already connected";
return false;
}
auto err = soundio_connect_backend(this->soundio_handle, this->backend);
if(err) {
error = soundio_strerror(err);
return false;
}
this->soundio_handle->app_name = "TeaClient";
this->_connected = true;
{
auto begin = std::chrono::system_clock::now();
soundio_flush_events(this->soundio_handle);
auto end = std::chrono::system_clock::now();
log_debug(category::audio, tr("Flushed connect events within {}ms for backend {}"),
std::chrono::ceil<std::chrono::milliseconds>(end - begin).count(),
this->name());
}
return true;
}
void SoundIOBackendHandler::disconnect() {
if(!this->soundio_handle || !this->_connected) return;
soundio_disconnect(this->soundio_handle);
}
void SoundIOBackendHandler::handle_device_change() {
log_debug(category::audio, tr("Device list changed for backend {}. Reindexing devices."), this->name());
std::lock_guard lock{this->device_lock};
this->_default_output_device.reset();
this->_default_input_device.reset();
this->cached_input_devices.clear();
this->cached_output_devices.clear();
if(!this->_connected || !this->soundio_handle) return;
size_t input_devices{0}, output_devices{0};
auto default_input_device{soundio_default_input_device_index(this->soundio_handle)};
for(int i = 0; i < soundio_input_device_count(this->soundio_handle); i++) {
auto dev = soundio_get_input_device(this->soundio_handle, i);
if(!dev) {
log_warn(category::audio, tr("Failed to get input device at index {} for backend {}."), i, this->name());
continue;
}
if(dev->probe_error) {
log_trace(category::audio, tr("Skipping input device {} ({}) for backend {} because of probe error: {}"), dev->id, dev->name, this->name(), soundio_strerror(dev->probe_error));
soundio_device_unref(dev);
continue;
}
auto device = std::make_shared<SoundIODevice>(dev, this->name(), i == default_input_device, true);
log_trace(category::audio, tr("Found input device {} ({})."), dev->id, dev->name);
this->cached_input_devices.push_back(device);
if(i == default_input_device)
this->_default_input_device = device;
input_devices++;
}
auto default_output_device{soundio_default_output_device_index(this->soundio_handle)};
for(int i = 0; i < soundio_output_device_count(this->soundio_handle); i++) {
auto dev = soundio_get_output_device(this->soundio_handle, i);
if(!dev) {
log_warn(category::audio, tr("Failed to get output device at index {} for backend {}."), i, this->name());
continue;
}
if(dev->probe_error) {
log_trace(category::audio, tr("Skipping output device {} ({}) for backend {} because of probe error: {}"), dev->id, dev->name, this->name(), soundio_strerror(dev->probe_error));
soundio_device_unref(dev);
continue;
}
auto device = std::make_shared<SoundIODevice>(dev, this->name(), i == default_output_device, true);
log_trace(category::audio, tr("Found output device {} ({})."), dev->id, dev->name);
this->cached_output_devices.push_back(device);
if(i == default_output_device)
this->_default_output_device = device;
output_devices++;
}
log_info(category::audio, tr("Queried devices for backend {}, resulting in {} input and {} output devices."),
this->name(),
input_devices,
output_devices
);
}
void SoundIOBackendHandler::handle_backend_disconnect(int error) {
log_info(category::audio, tr("Backend {} disconnected with error {}."), this->name(), soundio_strerror(error));
}
SoundIODevice::SoundIODevice(struct ::SoundIoDevice *dev, std::string driver, bool default_, bool owned) : device_handle{dev}, driver_name{std::move(driver)}, _default{default_} {
if(!owned) soundio_device_ref(dev);
}
SoundIODevice::~SoundIODevice() {
soundio_device_unref(this->device_handle);
}
std::string SoundIODevice::id() const {
return this->device_handle->id;
}
std::string SoundIODevice::name() const {
return this->device_handle->name;
}
std::string SoundIODevice::driver() const {
return this->driver_name; /* we do not use this->device_handle->soundio->current_backend because the soundio could be null */
}
bool SoundIODevice::is_input_supported() const {
return this->device_handle->aim == SoundIoDeviceAimInput;
}
bool SoundIODevice::is_output_supported() const {
return this->device_handle->aim == SoundIoDeviceAimOutput;
}
bool SoundIODevice::is_input_default() const {
return this->_default && this->is_input_supported();
}
bool SoundIODevice::is_output_default() const {
return this->_default && this->is_output_supported();
}
std::shared_ptr<AudioDevicePlayback> SoundIODevice::playback() {
if(!this->is_output_supported()) {
log_warn(category::audio, tr("Tried to create playback manager for device which does not supports it."));
return nullptr;
}
std::lock_guard lock{this->io_lock};
if(!this->_playback)
this->_playback = std::make_shared<SoundIOPlayback>(this->device_handle);
return this->_playback;
}
std::shared_ptr<AudioDeviceRecord> SoundIODevice::record() {
if(!this->is_input_supported()) {
log_warn(category::audio, tr("Tried to create record manager for device which does not supports it."));
return nullptr;
}
std::lock_guard lock{this->io_lock};
if(!this->_record)
this->_record = std::make_shared<SoundIORecord>(this->device_handle);
return this->_record;
}

158
native/serverconnection/src/audio/driver/SoundIO.h Normal file
View File

@ -0,0 +1,158 @@
#pragma once
#include <soundio/soundio.h>
#include <memory>
#include <vector>
#include <mutex>
#include "./AudioDriver.h"
namespace tc::audio {
struct BackendPriority {
static constexpr std::array<int, SoundIoBackendDummy + 1> mapping{
/* SoundIoBackendNone */ -100,
/* SoundIoBackendJack */ 100,
/* SoundIoBackendPulseAudio */ 90,
/* SoundIoBackendAlsa */ 50,
/* SoundIoBackendCoreAudio */ 100,
/* SoundIoBackendWasapi */ 100,
/* SoundIoBackendDummy */ 0
};
[[nodiscard]] static constexpr auto priority(SoundIoBackend backend) {
if(backend >= mapping.size())
return 0;
return mapping[backend];
}
};
class SoundIOPlayback : public AudioDevicePlayback {
public:
constexpr static auto kChunkSize{960};
explicit SoundIOPlayback(struct ::SoundIoDevice* /* handle */);
virtual ~SoundIOPlayback();
protected:
bool impl_start(std::string& /* error */) override;
void impl_stop() override;
private:
bool stream_invalid{false};
struct ::SoundIoDevice* device_handle{nullptr};
struct ::SoundIoOutStream* stream{nullptr};
struct ::SoundIoRingBuffer* buffer{nullptr};
void write_callback(int frame_count_min, int frame_count_max);
};
class SoundIORecord : public AudioDeviceRecord {
public:
constexpr static auto kChunkSize{960};
explicit SoundIORecord(struct ::SoundIoDevice* /* handle */);
virtual ~SoundIORecord();
protected:
bool impl_start(std::string& /* error */) override;
void impl_stop() override;
private:
bool stream_invalid{false};
struct ::SoundIoDevice* device_handle{nullptr};
struct ::SoundIoInStream* stream{nullptr};
struct ::SoundIoRingBuffer* buffer{nullptr};
void read_callback(int frame_count_min, int frame_count_max);
};
class SoundIODevice : public AudioDevice {
public:
explicit SoundIODevice(struct ::SoundIoDevice* /* handle */, std::string /* driver */, bool /* default */, bool /* owned */);
virtual ~SoundIODevice();
[[nodiscard]] std::string id() const override;
[[nodiscard]] std::string name() const override;
[[nodiscard]] std::string driver() const override;
[[nodiscard]] bool is_input_supported() const override;
[[nodiscard]] bool is_output_supported() const override;
[[nodiscard]] bool is_input_default() const override;
[[nodiscard]] bool is_output_default() const override;
[[nodiscard]] std::shared_ptr<AudioDevicePlayback> playback() override;
[[nodiscard]] std::shared_ptr<AudioDeviceRecord> record() override;
private:
std::string driver_name{};
struct ::SoundIoDevice* device_handle{nullptr};
bool _default{false};
std::mutex io_lock{};
std::shared_ptr<SoundIOPlayback> _playback;
std::shared_ptr<SoundIORecord> _record;
};
class SoundIOBackendHandler {
public:
/* its sorted by priority */
static std::vector<std::shared_ptr<SoundIOBackendHandler>> all_backends() {
std::lock_guard lock{backend_lock};
return backends;;
}
static std::shared_ptr<SoundIOBackendHandler> get_backend(SoundIoBackend backend);
static void initialize_all();
static void connect_all();
static void shutdown_all();
explicit SoundIOBackendHandler(SoundIoBackend backed);
virtual ~SoundIOBackendHandler();
bool initialize(std::string& error);
void shutdown();
bool connect(std::string& /* error */, bool /* enforce */ = false);
[[nodiscard]] inline bool connected() const { return this->_connected; }
void disconnect();
[[nodiscard]] inline int priority() const { return BackendPriority::priority(this->backend); }
[[nodiscard]] inline const char* name() const { return soundio_backend_name(this->backend); }
[[nodiscard]] inline std::vector<std::shared_ptr<SoundIODevice>> input_devices() const {
std::lock_guard lock{this->device_lock};
return this->cached_input_devices;
}
[[nodiscard]] inline std::vector<std::shared_ptr<SoundIODevice>> output_devices() const {
std::lock_guard lock{this->device_lock};
return this->cached_output_devices;
}
[[nodiscard]] inline std::shared_ptr<SoundIODevice> default_input_device() const {
return this->_default_input_device;
}
[[nodiscard]] inline std::shared_ptr<SoundIODevice> default_output_device() const {
return this->_default_output_device;
}
const SoundIoBackend backend;
private:
static std::mutex backend_lock;
static std::vector<std::shared_ptr<SoundIOBackendHandler>> backends;
void handle_backend_disconnect(int /* error */);
void handle_device_change();
bool _connected{false};
struct SoundIo* soundio_handle{nullptr};
mutable std::mutex device_lock{};
std::vector<std::shared_ptr<SoundIODevice>> cached_input_devices{};
std::vector<std::shared_ptr<SoundIODevice>> cached_output_devices{};
std::shared_ptr<SoundIODevice> _default_output_device{nullptr};
std::shared_ptr<SoundIODevice> _default_input_device{nullptr};
};
}

145
native/serverconnection/src/audio/driver/SoundIOPlayback.cpp Normal file
View File

@ -0,0 +1,145 @@
//
// Created by wolverindev on 07.02.20.
//
#include "SoundIO.h"
#include <algorithm>
#include "../../logger.h"
using namespace tc::audio;
SoundIOPlayback::SoundIOPlayback(struct ::SoundIoDevice *device) : device_handle{device} {
soundio_device_ref(device);
}
SoundIOPlayback::~SoundIOPlayback() {
soundio_device_unref(this->device_handle);
}
bool SoundIOPlayback::impl_start(std::string &error) {
assert(this->device_handle);
this->buffer = soundio_ring_buffer_create(nullptr, kChunkSize * sizeof(float) * 2); /* 2 channels */
if(!buffer) {
error = "failed to allocate the buffer";
return false;
}
soundio_ring_buffer_clear(this->buffer);
this->stream = soundio_outstream_create(this->device_handle);
if(!this->stream) {
error = "out of memory";
return false;
}
this->stream->userdata = this;
this->stream->format = SoundIoFormatFloat32LE;
this->stream->software_latency = 0.02;
this->stream->underflow_callback = [](auto str) {
auto handle = reinterpret_cast<SoundIOPlayback*>(str->userdata);
log_info(category::audio, tr("Having an underflow on {}"), handle->device_handle->id);
};
this->stream->error_callback = [](auto str, int err) {
auto handle = reinterpret_cast<SoundIOPlayback*>(str->userdata);
log_info(category::audio, tr("Having an error on {}: {}. Aborting playback."), handle->device_handle->id, soundio_strerror(err));
handle->stream_invalid = true;
};
this->stream->write_callback = [](struct SoundIoOutStream *str, int frame_count_min, int frame_count_max) {
auto handle = reinterpret_cast<SoundIOPlayback*>(str->userdata);
handle->write_callback(frame_count_min, frame_count_max);
};
if(auto err = soundio_outstream_open(this->stream); err) {
error = soundio_strerror(err) + std::string{" (open)"};
soundio_outstream_destroy(this->stream);
this->stream = nullptr;
return false;
}
if(false && this->stream->layout_error) {
error = std::string{"failed to set audio layout: "} + soundio_strerror(this->stream->layout_error);
soundio_outstream_destroy(this->stream);
this->stream = nullptr;
return false;
}
if(auto err = soundio_outstream_start(this->stream); err) {
error = soundio_strerror(err) + std::string{" (start)"};
soundio_outstream_destroy(this->stream);
this->stream = nullptr;
return false;
}
//TODO: Test for interleaved channel layout!
return true;
}
void SoundIOPlayback::impl_stop() {
if(!this->stream) return;
soundio_outstream_destroy(this->stream);
this->stream = nullptr;
soundio_ring_buffer_destroy(this->buffer);
this->buffer = nullptr;
}
void SoundIOPlayback::write_callback(int frame_count_min, int frame_count_max) {
const struct SoundIoChannelLayout *layout = &this->stream->layout;
struct SoundIoChannelArea *areas;
int frames_left{frame_count_min}, err;
if(frames_left < 120)
frames_left = 120;
if(frames_left > frame_count_max)
frames_left = frame_count_max;
while(frames_left > 0) {
int frame_count{frames_left};
auto buffered = soundio_ring_buffer_fill_count(this->buffer) / (sizeof(float) * layout->channel_count);
if(frame_count > buffered) {
if(buffered == 0) {
const auto length = sizeof(float) * frame_count * layout->channel_count;
this->fill_buffer(soundio_ring_buffer_write_ptr(this->buffer), frame_count, layout->channel_count);
soundio_ring_buffer_advance_write_ptr(this->buffer, length);
} else
frame_count = buffered;
}
if((err = soundio_outstream_begin_write(this->stream, &areas, &frame_count))) {
log_warn(category::audio, tr("Failed to begin a write to the soundio buffer: {}"), err);
return;
}
/* test for interleaved */
{
char* begin = areas[0].ptr - sizeof(float);
for(size_t channel{0}; channel < layout->channel_count; channel++) {
if((begin += sizeof(float)) != areas[channel].ptr) {
log_error(category::audio, tr("Expected interleaved buffer, which it isn't"));
return;
}
if(areas[channel].step != sizeof(float) * layout->channel_count) {
log_error(category::audio, tr("Invalid step size for channel {}"), channel);
return;
}
}
}
const auto length = sizeof(float) * frame_count * layout->channel_count;
memcpy(areas[0].ptr, soundio_ring_buffer_read_ptr(this->buffer), length);
soundio_ring_buffer_advance_read_ptr(this->buffer, length);
if((err = soundio_outstream_end_write(this->stream))) {
log_warn(category::audio, tr("Failed to end a write to the soundio buffer: {}"), err);
return;
}
frames_left -= frame_count;
}
}

145
native/serverconnection/src/audio/driver/SoundIORecord.cpp Normal file
View File

@ -0,0 +1,145 @@
//
// Created by wolverindev on 07.02.20.
//
#include "SoundIO.h"
#include <algorithm>
#include "../../logger.h"
using namespace tc::audio;
SoundIORecord::SoundIORecord(struct ::SoundIoDevice *device) : device_handle{device} {
soundio_device_ref(device);
}
SoundIORecord::~SoundIORecord() {
soundio_device_unref(this->device_handle);
}
bool SoundIORecord::impl_start(std::string &error) {
assert(this->device_handle);
this->buffer = soundio_ring_buffer_create(nullptr, kChunkSize * sizeof(float) * 2); //2 Channels
if(!buffer) {
error = "failed to allocate the buffer";
return false;
}
soundio_ring_buffer_clear(this->buffer);
this->stream = soundio_instream_create(this->device_handle);
if(!this->stream) {
error = "out of memory";
return false;
}
this->stream->userdata = this;
this->stream->format = SoundIoFormatFloat32LE;
this->stream->software_latency = 0.02;
this->stream->overflow_callback = [](auto str) {
auto handle = reinterpret_cast<SoundIORecord*>(str->userdata);
log_info(category::audio, tr("Having an overflow on {}"), handle->device_handle->id);
};
this->stream->error_callback = [](auto str, int err) {
auto handle = reinterpret_cast<SoundIORecord*>(str->userdata);
log_info(category::audio, tr("Having an error on {}: {}. Aborting recording."), handle->device_handle->id, soundio_strerror(err));
handle->stream_invalid = true;
};
this->stream->read_callback = [](struct SoundIoInStream *str, int frame_count_min, int frame_count_max) {
auto handle = reinterpret_cast<SoundIORecord*>(str->userdata);
handle->read_callback(frame_count_min, frame_count_max);
};
if(auto err = soundio_instream_open(this->stream); err) {
error = soundio_strerror(err) + std::string{" (open)"};
soundio_instream_destroy(this->stream);
this->stream = nullptr;
return false;
}
if(false && this->stream->layout_error) {
error = std::string{"failed to set audio layout: "} + soundio_strerror(this->stream->layout_error);
soundio_instream_destroy(this->stream);
this->stream = nullptr;
return false;
}
if(auto err = soundio_instream_start(this->stream); err) {
error = soundio_strerror(err) + std::string{" (start)"};
soundio_instream_destroy(this->stream);
this->stream = nullptr;
return false;
}
//TODO: Test for interleaved channel layout!
return true;
}
void SoundIORecord::impl_stop() {
if(!this->stream) return;
soundio_instream_destroy(this->stream);
this->stream = nullptr;
soundio_ring_buffer_destroy(this->buffer);
this->buffer = nullptr;
}
void SoundIORecord::read_callback(int frame_count_min, int frame_count_max) {
const struct SoundIoChannelLayout *layout = &this->stream->layout;
struct SoundIoChannelArea *areas;
int frames_left{frame_count_max};
int buffer_samples = soundio_ring_buffer_free_count(this->buffer) / (sizeof(float) * layout->channel_count);
while(frames_left > 0) {
int frame_count{frames_left};
if(frame_count > buffer_samples)
frame_count = buffer_samples;
if(auto err = soundio_instream_begin_read(this->stream, &areas, &frame_count); err) {
log_error(category::audio, tr("Failed to begin read from input stream buffer: {}"), soundio_strerror(err));
return;
}
/* test for interleaved */
{
char* begin = areas[0].ptr - sizeof(float);
for(size_t channel{0}; channel < layout->channel_count; channel++) {
if((begin += sizeof(float)) != areas[channel].ptr) {
log_error(category::audio, tr("Expected interleaved buffer, which it isn't"));
return;
}
if(areas[channel].step != sizeof(float) * layout->channel_count) {
log_error(category::audio, tr("Invalid step size for channel {}"), channel);
return;
}
}
}
const auto length = sizeof(float) * layout->channel_count * frame_count;
memcpy(soundio_ring_buffer_write_ptr(this->buffer), areas[0].ptr, length);
soundio_ring_buffer_advance_write_ptr(this->buffer, length);
buffer_samples -= frame_count;
frames_left -= frame_count;
if(buffer_samples == 0) {
std::lock_guard consumer{this->consumer_lock};
const auto byte_count = soundio_ring_buffer_fill_count(this->buffer);
const auto frame_count = byte_count / (sizeof(float) * layout->channel_count);
for(auto& consumer : this->_consumers)
consumer->consume(soundio_ring_buffer_read_ptr(this->buffer), frame_count, layout->channel_count);
soundio_ring_buffer_advance_read_ptr(this->buffer, byte_count);
buffer_samples = frame_count;
}
if(auto err = soundio_instream_end_read(this->stream); err) {
log_error(category::audio, tr("Failed to close input stream buffer: {}"), soundio_strerror(err));
return;
}
}
}

34
native/serverconnection/src/audio/js/AudioOutputStream.cpp
View File

@ -121,25 +121,10 @@ NAN_METHOD(AudioOutputStreamWrapper::_delete) {
} }
ssize_t AudioOutputStreamWrapper::write_data(const std::shared_ptr<AudioOutputSource>& handle, void *source, size_t samples, bool interleaved) { ssize_t AudioOutputStreamWrapper::write_data(const std::shared_ptr<AudioOutputSource>& handle, void *source, size_t samples, bool interleaved) {
ssize_t result = 0; if(interleaved)
if(interleaved) { return handle->enqueue_samples(source, samples);
result = handle->enqueue_samples(source, samples); else
} else { return handle->enqueue_samples_no_interleave(source, samples);
auto buffer = SampleBuffer::allocate(handle->channel_count, samples);
auto src_buffer = (float*) source;
auto target_buffer = (float*) buffer->sample_data;
while (samples-- > 0) {
*target_buffer = *src_buffer;
*(target_buffer + 1) = *(src_buffer + buffer->sample_size);
target_buffer += 2;
src_buffer++;
}
result = handle->enqueue_samples(buffer);
}
return result;
} }
NAN_METHOD(AudioOutputStreamWrapper::_write_data) { NAN_METHOD(AudioOutputStreamWrapper::_write_data) {
@ -198,6 +183,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_write_data_rated) {
return; return;
} }
//TODO: Use a tmp preallocated buffer here!
ssize_t target_samples = client->_resampler->estimated_output_size(samples); ssize_t target_samples = client->_resampler->estimated_output_size(samples);
auto buffer = SampleBuffer::allocate(handle->channel_count, max((size_t) samples, (size_t) target_samples)); auto buffer = SampleBuffer::allocate(handle->channel_count, max((size_t) samples, (size_t) target_samples));
auto source_buffer = js_buffer.Data(); auto source_buffer = js_buffer.Data();
@ -224,7 +210,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_write_data_rated) {
buffer->sample_index = 0; buffer->sample_index = 0;
buffer->sample_size = target_samples; buffer->sample_size = target_samples;
info.GetReturnValue().Set((int32_t) handle->enqueue_samples(buffer)); info.GetReturnValue().Set((int32_t) handle->enqueue_samples(buffer->sample_data, target_samples));
} }
} }
@ -237,7 +223,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_get_buffer_latency) {
return; return;
} }
info.GetReturnValue().Set((float) handle->min_buffer / (float) handle->sample_rate); info.GetReturnValue().Set((float) handle->min_buffered_samples / (float) handle->sample_rate);
} }
NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_latency) { NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_latency) {
@ -254,7 +240,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_latency) {
return; return;
} }
handle->min_buffer = (size_t) ceil(handle->sample_rate * info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0)); handle->min_buffered_samples = (size_t) ceil(handle->sample_rate * info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0));
} }
NAN_METHOD(AudioOutputStreamWrapper::_get_buffer_max_latency) { NAN_METHOD(AudioOutputStreamWrapper::_get_buffer_max_latency) {
@ -266,7 +252,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_get_buffer_max_latency) {
return; return;
} }
info.GetReturnValue().Set((float) handle->max_latency / (float) handle->sample_rate); info.GetReturnValue().Set((float) handle->max_latency() / (float) handle->sample_rate);
} }
NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_max_latency) { NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_max_latency) {
@ -283,7 +269,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_max_latency) {
return; return;
} }
handle->max_latency = (size_t) ceil(handle->sample_rate * info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0)); handle->max_buffered_samples = (size_t) ceil(handle->sample_rate * info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0));
} }
NAN_METHOD(AudioOutputStreamWrapper::_flush_buffer) { NAN_METHOD(AudioOutputStreamWrapper::_flush_buffer) {

88
native/serverconnection/src/audio/js/AudioPlayer.cpp
View File

@ -1,10 +1,9 @@
#include <misc/base64.h>
#include <misc/digest.h> #include <misc/digest.h>
#include <include/NanStrings.h> #include <include/NanStrings.h>
#include "AudioPlayer.h" #include "AudioPlayer.h"
#include "../AudioOutput.h" #include "../AudioOutput.h"
#include "../AudioDevice.h"
#include "AudioOutputStream.h" #include "AudioOutputStream.h"
#include "../../logger.h"
using namespace tc; using namespace tc;
using namespace tc::audio; using namespace tc::audio;
@ -22,6 +21,11 @@ NAN_MODULE_INIT(player::init_js) {
} }
NAN_METHOD(audio::available_devices) { NAN_METHOD(audio::available_devices) {
if(!audio::initialized()) {
Nan::ThrowError(tr("audio hasn't been initialized!"));
return;
}
auto devices = audio::devices(); auto devices = audio::devices();
auto result = Nan::New<v8::Array>(devices.size()); auto result = Nan::New<v8::Array>(devices.size());
@ -29,17 +33,15 @@ NAN_METHOD(audio::available_devices) {
auto device_info = Nan::New<v8::Object>(); auto device_info = Nan::New<v8::Object>();
auto device = devices[index]; auto device = devices[index];
Nan::Set(device_info, Nan::LocalString("name"), Nan::LocalStringUTF8(device->name)); Nan::Set(device_info, Nan::LocalString("name"), Nan::LocalStringUTF8(device->name()));
Nan::Set(device_info, Nan::New<v8::String>("driver").ToLocalChecked(), Nan::New<v8::String>(device->driver).ToLocalChecked()); Nan::Set(device_info, Nan::New<v8::String>("driver").ToLocalChecked(), Nan::New<v8::String>(device->driver()).ToLocalChecked());
Nan::Set(device_info, Nan::New<v8::String>("device_id").ToLocalChecked(), Nan::New<v8::String>(base64::encode(digest::sha1(device->name + device->driver))).ToLocalChecked()); Nan::Set(device_info, Nan::New<v8::String>("device_id").ToLocalChecked(), Nan::New<v8::String>(device->id()).ToLocalChecked());
Nan::Set(device_info, Nan::New<v8::String>("input_supported").ToLocalChecked(), Nan::New<v8::Boolean>(device->input_supported)); Nan::Set(device_info, Nan::New<v8::String>("input_supported").ToLocalChecked(), Nan::New<v8::Boolean>(device->is_input_supported()));
Nan::Set(device_info, Nan::New<v8::String>("output_supported").ToLocalChecked(), Nan::New<v8::Boolean>(device->output_supported)); Nan::Set(device_info, Nan::New<v8::String>("output_supported").ToLocalChecked(), Nan::New<v8::Boolean>(device->is_output_supported()));
Nan::Set(device_info, Nan::New<v8::String>("input_default").ToLocalChecked(), Nan::New<v8::Boolean>(device->is_default_input)); Nan::Set(device_info, Nan::New<v8::String>("input_default").ToLocalChecked(), Nan::New<v8::Boolean>(device->is_input_default()));
Nan::Set(device_info, Nan::New<v8::String>("output_default").ToLocalChecked(), Nan::New<v8::Boolean>(device->is_default_output)); Nan::Set(device_info, Nan::New<v8::String>("output_default").ToLocalChecked(), Nan::New<v8::Boolean>(device->is_output_default()));
Nan::Set(device_info, Nan::New<v8::String>("device_index").ToLocalChecked(), Nan::New<v8::Number>(device->device_id));
Nan::Set(result, index, device_info); Nan::Set(result, index, device_info);
} }
@ -47,13 +49,48 @@ NAN_METHOD(audio::available_devices) {
info.GetReturnValue().Set(result); info.GetReturnValue().Set(result);
} }
NAN_METHOD(player::current_playback_device) { NAN_METHOD(audio::await_initialized_js) {
if(!global_audio_output) { if(info.Length() != 1 || !info[0]->IsFunction()) {
info.GetReturnValue().Set(paNoDevice); Nan::ThrowError(tr("Invalid arguments"));
return; return;
} }
info.GetReturnValue().Set(global_audio_output->current_device()); if(audio::initialized()) {
(void) info[0].As<v8::Function>()->Call(Nan::GetCurrentContext(), Nan::Undefined(), 0, nullptr);
} else {
auto _callback = std::make_unique<Nan::Persistent<v8::Function>>(info[0].As<v8::Function>());
auto _async_callback = Nan::async_callback([call = std::move(_callback)] {
Nan::HandleScope scope;
auto callback_function = call->Get(Nan::GetCurrentContext()->GetIsolate());
(void) callback_function->Call(Nan::GetCurrentContext(), Nan::Undefined(), 0, nullptr);
call->Reset();
}).option_destroyed_execute(true);
audio::initialize([_async_callback] {
_async_callback.call();
});
}
}
NAN_METHOD(audio::initialized_js) {
info.GetReturnValue().Set(audio::initialized());
}
NAN_METHOD(player::current_playback_device) {
if(!global_audio_output) {
info.GetReturnValue().Set(Nan::Undefined());
return;
}
auto device = global_audio_output->current_device();
if(!device) {
info.GetReturnValue().Set(Nan::Undefined());
return;
}
info.GetReturnValue().Set(Nan::LocalString(device->id()));
} }
NAN_METHOD(player::set_playback_device) { NAN_METHOD(player::set_playback_device) {
@ -62,24 +99,33 @@ NAN_METHOD(player::set_playback_device) {
return; return;
} }
if(info.Length() != 1 || !info[0]->IsNumber()) { const auto null_device = info[0]->IsNullOrUndefined();
if(info.Length() != 1 || !(info[0]->IsString() || null_device)) {
Nan::ThrowError("invalid arguments"); Nan::ThrowError("invalid arguments");
return; return;
} }
std::string error; auto device = null_device ? nullptr : audio::find_device_by_id(*Nan::Utf8String(info[0]), false);
if(!global_audio_output->open_device(error, info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0))) { if(!device && !null_device) {
Nan::ThrowError(Nan::New<v8::String>("failed to open device (" + error + ")").ToLocalChecked()); Nan::ThrowError("invalid device id");
return; return;
} }
if(!global_audio_output->playback()) { std::string error;
Nan::ThrowError("failed to open playback stream");
global_audio_output->set_device(device);
if(!global_audio_output->playback(error)) {
Nan::ThrowError(error.c_str());
return; return;
} }
} }
NAN_METHOD(player::create_stream) { NAN_METHOD(player::create_stream) {
if(!audio::initialized()) {
Nan::ThrowError(tr("audio hasn't been initialized yet"));
return;
}
if(!global_audio_output) { if(!global_audio_output) {
Nan::ThrowError("Global audio output hasn't been yet initialized!"); Nan::ThrowError("Global audio output hasn't been yet initialized!");
return; return;

6
native/serverconnection/src/audio/js/AudioPlayer.h
View File

@ -2,9 +2,10 @@
#include <nan.h> #include <nan.h>
namespace tc { namespace tc::audio {
namespace audio {
extern NAN_METHOD(available_devices); extern NAN_METHOD(available_devices);
extern NAN_METHOD(await_initialized_js);
extern NAN_METHOD(initialized_js);
namespace player { namespace player {
extern NAN_MODULE_INIT(init_js); extern NAN_MODULE_INIT(init_js);
@ -30,4 +31,3 @@ namespace tc {
*/ */
} }
} }
}

47
native/serverconnection/src/audio/js/AudioRecorder.cpp
View File

@ -1,4 +1,5 @@
#include <utility> #include <utility>
#include <NanStrings.h>
#include "AudioRecorder.h" #include "AudioRecorder.h"
#include "AudioConsumer.h" #include "AudioConsumer.h"
@ -14,6 +15,10 @@ NAN_MODULE_INIT(recorder::init_js) {
} }
NAN_METHOD(recorder::create_recorder) { NAN_METHOD(recorder::create_recorder) {
if(!audio::initialized()) {
Nan::ThrowError(tr("audio hasn't been initialized yet"));
return;
}
auto input = make_shared<AudioInput>(2, 48000); auto input = make_shared<AudioInput>(2, 48000);
auto wrapper = new AudioRecorderWrapper(input); auto wrapper = new AudioRecorderWrapper(input);
auto js_object = Nan::NewInstance(Nan::New(AudioRecorderWrapper::constructor())).ToLocalChecked(); auto js_object = Nan::NewInstance(Nan::New(AudioRecorderWrapper::constructor())).ToLocalChecked();
@ -122,42 +127,50 @@ NAN_METHOD(AudioRecorderWrapper::_get_device) {
auto handle = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder()); auto handle = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder());
auto input = handle->_input; auto input = handle->_input;
info.GetReturnValue().Set(input->current_device()); auto device = input->current_device();
if(device)
info.GetReturnValue().Set(Nan::LocalString(device->id()));
else
info.GetReturnValue().Set(Nan::Undefined());
} }
NAN_METHOD(AudioRecorderWrapper::_set_device) { NAN_METHOD(AudioRecorderWrapper::_set_device) {
auto handle = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder()); auto handle = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder());
auto input = handle->_input; auto input = handle->_input;
if(info.Length() != 2 || !info[0]->IsNumber() || !info[1]->IsFunction()) { const auto is_null_device = info[0]->IsNullOrUndefined();
if(info.Length() != 2 || !(is_null_device || info[0]->IsString()) || !info[1]->IsFunction()) {
Nan::ThrowError("invalid arguments"); Nan::ThrowError("invalid arguments");
return; return;
} }
auto device_id = info[0]->Int32Value(Nan::GetCurrentContext()).FromMaybe(0); if(!audio::initialized()) {
Nan::ThrowError("audio hasn't been initialized yet");
return;
}
auto device = is_null_device ? nullptr : audio::find_device_by_id(*Nan::Utf8String(info[0]), true);
if(!device && !is_null_device) {
Nan::ThrowError("invalid device id");
return;
}
unique_ptr<Nan::Persistent<v8::Function>> _callback = make_unique<Nan::Persistent<v8::Function>>(info[1].As<v8::Function>()); unique_ptr<Nan::Persistent<v8::Function>> _callback = make_unique<Nan::Persistent<v8::Function>>(info[1].As<v8::Function>());
unique_ptr<Nan::Persistent<v8::Object>> _recorder = make_unique<Nan::Persistent<v8::Object>>(info.Holder()); unique_ptr<Nan::Persistent<v8::Object>> _recorder = make_unique<Nan::Persistent<v8::Object>>(info.Holder());
auto _async_callback = Nan::async_callback([call = std::move(_callback), recorder = move(_recorder)](bool result, std::string error) mutable { auto _async_callback = Nan::async_callback([call = std::move(_callback), recorder = move(_recorder)] {
Nan::HandleScope scope; Nan::HandleScope scope;
auto callback_function = call->Get(Nan::GetCurrentContext()->GetIsolate()); auto callback_function = call->Get(Nan::GetCurrentContext()->GetIsolate());
v8::Local<v8::Value> argv[1]; (void) callback_function->Call(Nan::GetCurrentContext(), Nan::Undefined(), 0, nullptr);
if(result)
argv[0] = v8::Boolean::New(Nan::GetCurrentContext()->GetIsolate(), result);
else
argv[0] = Nan::NewOneByteString((uint8_t*) error.data(), error.length()).ToLocalChecked();
callback_function->Call(Nan::GetCurrentContext(), Nan::Undefined(), 1, argv);
recorder->Reset(); recorder->Reset();
call->Reset(); call->Reset();
}).option_destroyed_execute(true); }).option_destroyed_execute(true);
std::thread([_async_callback, input, device_id]{ std::thread([_async_callback, input, device]{
string error; input->set_device(device);
auto flag = input->open_device(error, device_id); _async_callback();
_async_callback(std::forward<bool>(flag), std::forward<std::string>(error));
}).detach(); }).detach();
} }
@ -173,9 +186,13 @@ NAN_METHOD(AudioRecorderWrapper::_start) {
} }
auto input = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder())->_input; auto input = ObjectWrap::Unwrap<AudioRecorderWrapper>(info.Holder())->_input;
std::string error{};
v8::Local<v8::Value> argv[1]; v8::Local<v8::Value> argv[1];
argv[0] = v8::Boolean::New(Nan::GetCurrentContext()->GetIsolate(), input->record()); if(input->record(error))
argv[0] = Nan::New<v8::Boolean>(true);
else
argv[0] = Nan::LocalString(error);
(void) info[0].As<v8::Function>()->Call(Nan::GetCurrentContext(), Nan::Undefined(), 1, argv); (void) info[0].As<v8::Function>()->Call(Nan::GetCurrentContext(), Nan::Undefined(), 1, argv);
} }

37
native/serverconnection/src/bindings.cpp
View File

@ -5,6 +5,7 @@
#include <mutex> #include <mutex>
#include <event2/thread.h> #include <event2/thread.h>
#include <misc/digest.h> #include <misc/digest.h>
#include <NanStrings.h>
#include "logger.h" #include "logger.h"
#include "include/NanException.h" #include "include/NanException.h"
@ -15,7 +16,7 @@
#include "connection/ft/FileTransferManager.h" #include "connection/ft/FileTransferManager.h"
#include "connection/ft/FileTransferObject.h" #include "connection/ft/FileTransferObject.h"
#include "audio/AudioOutput.h" #include "audio/AudioOutput.h"
#include "audio/AudioDevice.h" #include "audio/driver/AudioDriver.h"
#include "audio/js/AudioOutputStream.h" #include "audio/js/AudioOutputStream.h"
#include "audio/js/AudioPlayer.h" #include "audio/js/AudioPlayer.h"
#include "audio/js/AudioRecorder.h" #include "audio/js/AudioRecorder.h"
@ -104,11 +105,7 @@ NAN_MODULE_INIT(init) {
} }
string error; string error;
//TODO here tc::audio::initialize(); //TODO: Notify JS when initialized?
//PaJack_SetClientName("TeaClient");
Pa_Initialize();
std::thread(audio::devices).detach(); /* cache the devices */
logger::info(category::general, "Loading crypt modules"); logger::info(category::general, "Loading crypt modules");
std::string descriptors = "LTGE"; std::string descriptors = "LTGE";
@ -141,19 +138,31 @@ NAN_MODULE_INIT(init) {
#endif #endif
tc::audio::init_event_loops(); tc::audio::init_event_loops();
/* TODO: Test error codes and make the audi playback device configurable */ tc::audio::initialize([]{
global_audio_output = new tc::audio::AudioOutput(2, 48000); //48000 44100 std::string error{};
if(!global_audio_output->open_device(error, Pa_GetDefaultOutputDevice())) {
logger::error(category::audio, "Failed to initialize default audio playback: {}", error); std::shared_ptr<tc::audio::AudioDevice> default_output{};
} else { for(auto& device : tc::audio::devices()) {
if(!global_audio_output->playback()) { if(device->is_output_default()) {
logger::error(category::audio, "Failed to start audio playback"); default_output = device;
break;
} }
} }
/* TODO: Test error codes and make the audi playback device configurable */
global_audio_output = new tc::audio::AudioOutput(2, 48000);
global_audio_output->set_device(default_output);
if(!global_audio_output->playback(error)) {
logger::error(category::audio, "Failed to start audio playback: {}", error);
}
});
{ {
auto namespace_audio = Nan::New<v8::Object>(); auto namespace_audio = Nan::New<v8::Object>();
Nan::Set(namespace_audio, Nan::New<v8::String>("available_devices").ToLocalChecked(), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(audio::available_devices)).ToLocalChecked()); Nan::Set(namespace_audio, Nan::LocalString("available_devices"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(audio::available_devices)).ToLocalChecked());
Nan::Set(namespace_audio, Nan::LocalString("initialize"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(audio::await_initialized_js)).ToLocalChecked());
Nan::Set(namespace_audio, Nan::LocalString("initialized"), Nan::GetFunction(Nan::New<v8::FunctionTemplate>(audio::initialized_js)).ToLocalChecked());
{ {
auto namespace_playback = Nan::New<v8::Object>(); auto namespace_playback = Nan::New<v8::Object>();
audio::player::init_js(namespace_playback); audio::player::init_js(namespace_playback);

7
native/serverconnection/src/connection/audio/VoiceClient.cpp
View File

@ -211,8 +211,8 @@ VoiceClientWrap::~VoiceClientWrap() {}
VoiceClient::VoiceClient(const std::shared_ptr<VoiceConnection>&, uint16_t client_id) : _client_id(client_id) { VoiceClient::VoiceClient(const std::shared_ptr<VoiceConnection>&, uint16_t client_id) : _client_id(client_id) {
this->output_source = global_audio_output->create_source(); this->output_source = global_audio_output->create_source();
this->output_source->overflow_strategy = audio::overflow_strategy::ignore; this->output_source->overflow_strategy = audio::overflow_strategy::ignore;
this->output_source->max_latency = (size_t) ceil(this->output_source->sample_rate * 0.5); this->output_source->max_buffered_samples = (size_t) ceil(this->output_source->sample_rate * 0.5);
this->output_source->min_buffer = (size_t) ceil(this->output_source->sample_rate * 0.04); this->output_source->min_buffered_samples = (size_t) ceil(this->output_source->sample_rate * 0.04);
this->output_source->on_underflow = [&]{ this->output_source->on_underflow = [&]{
if(this->_state == state::stopping) if(this->_state == state::stopping)
@ -533,8 +533,9 @@ void VoiceClient::event_execute(const std::chrono::system_clock::time_point &sch
if(replay_head->reset_decoder) if(replay_head->reset_decoder)
audio_codec.converter->reset_decoder(); audio_codec.converter->reset_decoder();
//TODO: Use statically allocated buffer?
auto decoded = this->decode_buffer(audio_codec.codec, replay_head->buffer); auto decoded = this->decode_buffer(audio_codec.codec, replay_head->buffer);
this->output_source->enqueue_samples(decoded); this->output_source->enqueue_samples(decoded->sample_data, decoded->sample_size);
this->set_state(state::playing); this->set_state(state::playing);
} }

2
native/serverconnection/src/connection/ft/FileTransferManager.cpp
View File

@ -1,6 +1,5 @@
#include "FileTransferManager.h" #include "FileTransferManager.h"
#include "FileTransferObject.h" #include "FileTransferObject.h"
#include <NanGet.h>
#include <misc/net.h> #include <misc/net.h>
#include <algorithm> #include <algorithm>
@ -593,6 +592,7 @@ void FileTransferManager::remove_transfer(tc::ft::Transfer *transfer) {
} }
#ifdef NODEJS_API #ifdef NODEJS_API
#include <NanGet.h>
NAN_MODULE_INIT(JSTransfer::Init) { NAN_MODULE_INIT(JSTransfer::Init) {
auto klass = Nan::New<v8::FunctionTemplate>(JSTransfer::NewInstance); auto klass = Nan::New<v8::FunctionTemplate>(JSTransfer::NewInstance);

3
native/serverconnection/src/logger.cpp
View File

@ -1,5 +1,4 @@
#include <iostream> #include <iostream>
#include <NanGet.h>
#include "logger.h" #include "logger.h"
/* Basic */ /* Basic */
@ -9,6 +8,7 @@ void force_log_raw(logger::category::value, spdlog::level::level_enum level, con
void force_log_node(logger::category::value, spdlog::level::level_enum, const std::string_view &); void force_log_node(logger::category::value, spdlog::level::level_enum, const std::string_view &);
#ifdef NODEJS_API #ifdef NODEJS_API
#include <NanGet.h>
#include <include/NanEventCallback.h> #include <include/NanEventCallback.h>
#include <include/NanStrings.h> #include <include/NanStrings.h>
@ -109,7 +109,6 @@ void force_log_node(logger::category::value category, spdlog::level::level_enum
} }
log_messages_callback(); log_messages_callback();
} }
#endif #endif
void logger::initialize_raw() { void logger::initialize_raw() {

49
native/serverconnection/src/ring_buffer.cpp Normal file
View File

@ -0,0 +1,49 @@
//
// Created by wolverindev on 07.02.20.
//
#include "ring_buffer.h"
#include <soundio/soundio.h>
using namespace tc;
ring_buffer::ring_buffer(size_t cap) {
this->handle = soundio_ring_buffer_create(nullptr, cap);
}
ring_buffer::~ring_buffer() {
soundio_ring_buffer_destroy((SoundIoRingBuffer*) this->handle);
}
size_t ring_buffer::capacity() const {
return soundio_ring_buffer_capacity((SoundIoRingBuffer*) this->handle);
}
size_t ring_buffer::free_count() const {
return soundio_ring_buffer_free_count((SoundIoRingBuffer*) this->handle);
}
size_t ring_buffer::fill_count() const {
return soundio_ring_buffer_fill_count((SoundIoRingBuffer*) this->handle);
}
char* ring_buffer::write_ptr() {
return soundio_ring_buffer_write_ptr((SoundIoRingBuffer*) this->handle);
}
void ring_buffer::advance_write_ptr(size_t bytes) {
soundio_ring_buffer_advance_write_ptr((SoundIoRingBuffer*) this->handle, bytes);
}
const void* ring_buffer::read_ptr() const {
return soundio_ring_buffer_read_ptr((SoundIoRingBuffer*) this->handle);
}
void ring_buffer::advance_read_ptr(size_t bytes) {
soundio_ring_buffer_advance_read_ptr((SoundIoRingBuffer*) this->handle, bytes);
}
void ring_buffer::clear() {
soundio_ring_buffer_clear((SoundIoRingBuffer*) this->handle);
}

28
native/serverconnection/src/ring_buffer.h Normal file
View File

@ -0,0 +1,28 @@
#pragma once
#include <cstddef>
namespace tc {
class ring_buffer {
public:
/* Attention: Actual size may be larger than given capacity */
explicit ring_buffer(size_t /* minimum capacity */);
~ring_buffer();
[[nodiscard]] size_t capacity() const;
[[nodiscard]] size_t fill_count() const;
[[nodiscard]] size_t free_count() const;
/* do not write more than the capacity! */
char* write_ptr();
void advance_write_ptr(size_t /* count */);
/* do not read more than the capacity! */
[[nodiscard]] const void* read_ptr() const;
void advance_read_ptr(size_t /* count */);
void clear();
private:
void* handle{nullptr};
};
}

71
native/serverconnection/test/audio/main.cpp
View File

@ -1,17 +1,14 @@
#include <stdio.h>
#include <math.h>
#include <cstring> #include <cstring>
#include <string> #include <string>
#include <iostream> #include <iostream>
#include <thread> #include <thread>
#include <chrono> #include <chrono>
#include <deque>
#include "../../src/audio/AudioOutput.h" #include "../../src/audio/AudioOutput.h"
#include "../../src/audio/AudioInput.h" #include "../../src/audio/AudioInput.h"
#include "../../src/audio/filter/FilterVad.h" #include "../../src/audio/filter/FilterVad.h"
#include "../../src/audio/filter/FilterThreshold.h" #include "../../src/audio/filter/FilterThreshold.h"
#include "../../src/audio/Audio.h" #include "../../src/logger.h"
#ifdef WIN32 #ifdef WIN32
#include <windows.h> #include <windows.h>
@ -21,34 +18,51 @@ using namespace std;
using namespace tc; using namespace tc;
int main() { int main() {
string error; std::string error{};
if(!tc::audio::initialize(error)) {
cerr << "Failed to initialize audio: " << error << endl; Pa_Initialize();
return 1;
logger::initialize_raw();
tc::audio::initialize();
tc::audio::await_initialized();
std::shared_ptr<tc::audio::AudioDevice> default_playback{nullptr}, default_record{nullptr};
for(auto& device : tc::audio::devices()) {
if(device->is_output_default())
default_playback = device;
if(device->is_input_default())
default_record = device;
} }
auto playback_manager = audio::AudioOutput(2, 48000); assert(default_record);
if(!playback_manager.open_device(error, Pa_GetDefaultOutputDevice())) { assert(default_playback);
for(auto& dev : tc::audio::devices()) {
if(!dev->is_input_supported()) continue;
auto playback_manager = std::make_unique<audio::AudioOutput>(2, 48000);
if(!playback_manager->set_device(error, default_playback)) {
cerr << "Failed to open output device (" << error << ")" << endl; cerr << "Failed to open output device (" << error << ")" << endl;
return 1; return 1;
} }
if(!playback_manager.playback()) { if(!playback_manager->playback()) {
cerr << "failed to start playback" << endl; cerr << "failed to start playback" << endl;
return 1; return 1;
} }
auto input = audio::AudioInput(2, 48000); auto input = std::make_unique<audio::AudioInput>(2, 48000);
if(!input.open_device(error, Pa_GetDefaultInputDevice())) { if(!input->set_device(error, dev)) {
cerr << "Failed to open input device (" << error << ")" << endl; cerr << "Failed to open input device (" << error << "): " << dev->id() << " (" << dev->name() << ")" << endl;
return 1; continue;
} }
if(!input.record()) { if(!input->record()) {
cerr << "failed to start record" << endl; cerr << "failed to start record for " << dev->id() << " (" << dev->name() << ")" << endl;
return 1; continue;
} }
{ {
auto consumer = input.create_consumer(960); auto target_stream = playback_manager->create_source();
auto target_stream = playback_manager.create_source();
auto consumer = input->create_consumer(960);
auto vad_handler = make_shared<audio::filter::VadFilter>(2, 48000, 960); auto vad_handler = make_shared<audio::filter::VadFilter>(2, 48000, 960);
if(!vad_handler->initialize(error, 3, 4)) { if(!vad_handler->initialize(error, 3, 4)) {
cerr << "failed to initialize vad handler (" << error << ")"; cerr << "failed to initialize vad handler (" << error << ")";
@ -61,21 +75,28 @@ int main() {
return 1; return 1;
} }
consumer->on_read = [target_stream, vad_handler, threshold_filter](const void* buffer, size_t samples) { consumer->on_read = [target_stream, vad_handler, threshold_filter](const void* buffer, size_t samples) { //Do not capture consumer!
target_stream->enqueue_samples(buffer, samples); target_stream->enqueue_samples(buffer, samples);
cout << "T: " << threshold_filter->analyze(buffer, 0) << endl; /*
const auto analize_value = threshold_filter->analyze(buffer, 0);
if(vad_handler->process(buffer)) { if(vad_handler->process(buffer)) {
cout << "Read " << samples << endl; cout << "Read " << samples << " (" << analize_value << ")" << endl;
target_stream->enqueue_samples(buffer, samples); target_stream->enqueue_samples(buffer, samples);
} else { } else {
cout << "Drop " << samples << endl; cout << "Drop " << samples << " (" << analize_value << ")" << endl;
} }
*/
}; };
input.release();
cout << "Read started" << endl; cout << "Read started" << endl;
} }
this_thread::sleep_for(chrono::seconds(60)); playback_manager.release(); //FIXME: Memory leak!
}
this_thread::sleep_for(chrono::seconds(360));
/* /*
while(true) { while(true) {

31
native/serverconnection/test/audio/sio.cpp
View File

@ -1,13 +1,16 @@
#include <soundio/soundio.h> #include <soundio/soundio.h>
#include <cmath>
#include <iostream> #include <iostream>
#include <stdio.h>
#include <stdlib.h>
#include <string> #include <string>
#include <math.h>
#include <chrono> #include <chrono>
#include <mutex>
#include "../../src/logger.h"
#include "../../src/audio/driver/SoundIO.h"
using namespace std; using namespace std;
using namespace tc::audio;
static const float PI = 3.1415926535f; static const float PI = 3.1415926535f;
static float seconds_offset = 0.0f; static float seconds_offset = 0.0f;
@ -63,8 +66,27 @@ static void write_callback(struct SoundIoOutStream *outstream,
} }
int main(int argc, char **argv) { int main(int argc, char **argv) {
int err; logger::initialize_raw();
SoundIOBackendHandler::initialize_all();
SoundIOBackendHandler::connect_all();
const auto print_device = [](const std::shared_ptr<SoundIODevice>& device) {
std::cout << " - " << device->id() << " (" << device->name() << ")";
if(device->is_output_default() || device->is_input_default())
std::cout << " [Default]";
std::cout << "\n";
};
for(auto& backend : tc::audio::SoundIOBackendHandler::all_backends()) {
std::cout << "Backend " << backend->name() << ":\n";
std::cout << " Input devices: (" << backend->input_devices().size() << "): \n";
for(auto& device : backend->input_devices()) print_device(device);
std::cout << " Output devices: (" << backend->input_devices().size() << "): \n";
for(auto& device : backend->input_devices()) print_device(device);
}
SoundIOBackendHandler::shutdown_all();
return 0;
int err;
struct SoundIo *soundio = soundio_create(); struct SoundIo *soundio = soundio_create();
if (!soundio) { if (!soundio) {
fprintf(stderr, "out of memory\n"); fprintf(stderr, "out of memory\n");
@ -85,6 +107,7 @@ int main(int argc, char **argv) {
for(int i = 0; i < soundio_input_device_count(soundio); i++) { for(int i = 0; i < soundio_input_device_count(soundio); i++) {
auto dev = soundio_get_input_device(soundio, i); auto dev = soundio_get_input_device(soundio, i);
cout << dev->name << " - " << dev->id << endl; cout << dev->name << " - " << dev->id << endl;
soundio_device_unref(dev);
} }

112
native/serverconnection/test/js/audio.ts
View File

@ -1,114 +1,48 @@
/// <reference path="../../exports/exports.d.ts" /> /// <reference path="../../exports/exports.d.ts" />
console.log("HELLO WORLD"); console.log("Starting app");
module.paths.push("../../build/linux_x64"); module.paths.push("../../build/linux_x64");
module.paths.push("../../build/win32_64"); module.paths.push("../../build/win32_64");
//LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5
const os = require('os');
//process.dlopen(module, '/usr/lib/x86_64-linux-gnu/libasan.so.5',
// os.constants.dlopen.RTLD_NOW);
import * as fs from "fs";
const original_require = require; const original_require = require;
require = (module => original_require(__dirname + "/../../../build/linux_x64/" + module + ".node")) as any; require = (module => original_require(__dirname + "/../../../build/linux_x64/" + module + ".node")) as any;
import * as handle from "teaclient_connection"; import * as handle from "teaclient_connection";
require = original_require; require = original_require;
const connection_list = []; console.dir(handle.audio);
const connection = handle.spawn_server_connection(); handle.audio.initialize(() => {
const client_list = []; console.log("Audio initialized");
console.dir(handle);
console.log("Query devices..."); console.log("Query devices...");
console.log("Devices: %o", handle.audio.available_devices()); console.log("Devices: %o", handle.audio.available_devices());
console.log("Current playback device: %o", handle.audio.playback.current_device()); console.log("Current playback device: %o", handle.audio.playback.current_device());
//handle.audio.playback.set_device(14);
//console.log("Current playback device: %o", handle.audio.playback.current_device());
const stream = handle.audio.playback.create_stream(); const stream = handle.audio.playback.create_stream();
console.log("Own stream: %o", stream); console.log("Own stream: %o", stream);
for(let i = 0; i < 12; i++) {
const recorder = handle.audio.record.create_recorder(); const recorder = handle.audio.record.create_recorder();
for(const device of handle.audio.available_devices()) { const default_input = handle.audio.available_devices().find(e => e.input_default);
if(!device.input_supported) console.log(default_input);
continue; console.log(handle.audio.available_devices().find(e => e.device_id == handle.audio.playback.current_device()));
if(device.name != "pulse") recorder.set_device(default_input.device_id, () => {
continue;
console.log("Found pulse at %o", device.device_index);
recorder.set_device(device.device_index, () => {
recorder.start(flag => console.log("X: " + flag));
const consumer = recorder.create_consumer(); const consumer = recorder.create_consumer();
consumer.create_filter_threshold(2); consumer.callback_data = buffer => {
stream.write_data(buffer.buffer, true);
};
recorder.start(result => {
console.log("Start result: %o", result);
});
}); });
break;
}
}
/* -1 => default device */
const recorder = handle.audio.record.create_recorder();
console.log("Have device: %o", recorder);
console.log("Device: %o", recorder.get_device());
if(recorder.get_device() == -1) {
console.log("Looking for devices");
for(const device of handle.audio.available_devices()) {
if(!device.input_supported)
continue;
if(device.name != "pulse")
continue;
console.log("Found pulse at %o", device.device_index);
recorder.set_device(device.device_index, () => {});
}
}
console.log("Device: %o", recorder.get_device());
recorder.start(() => {});
console.log("Started: %o", recorder.started());
const consumer = recorder.create_consumer();
{
const filter = consumer.create_filter_threshold(.5);
filter.set_margin_frames(10);
/*
filter.set_analyze_filter(value => {
console.log(value);
})
*/
}
{
//const filter = consumer.create_filter_vad(3);
//console.log("Filter name: %s; Filter level: %d; Filter margin: %d", filter.get_name(), filter.get_level(), filter.get_margin_frames());
}
{
const consume = consumer.create_filter_state();
setTimeout(() => {
console.log("Silence now!");
consume.set_consuming(true);
setTimeout(() => {
console.log("Speak now!");
consume.set_consuming(false);
}, 1000);
}, 1000);
}
setInterval(() => { setInterval(() => {
if("gc" in global) { const elements = handle.audio.available_devices().filter(e => e.input_supported);
console.log("GC"); const dev = elements[Math.floor(Math.random() * elements.length)];
global.gc(); recorder.set_device(dev.device_id, () => {
} console.log("Dev updated: %o", dev);
}, 1000);
let a_map = [consumer, recorder]; recorder.start(() => {
/* keep the object alive */ console.log("Started");
setTimeout(() => { });
connection.connected(); });
a_map = a_map.filter(e => true);
}, 1000); }, 1000);
});