diff --git a/modules/renderer/audio/AudioPlayer.ts b/modules/renderer/audio/AudioPlayer.ts index 7bb6f38..139475d 100644 --- a/modules/renderer/audio/AudioPlayer.ts +++ b/modules/renderer/audio/AudioPlayer.ts @@ -3,6 +3,8 @@ window["require_setup"](module); import {audio as naudio} from "teaclient_connection"; namespace audio.player { + //FIXME: Native audio initialize handle! + export interface Device { device_id: string; name: string; @@ -15,6 +17,7 @@ namespace audio.player { let _initialized_callbacks: (() => any)[] = []; export let _initialized = false; + export let _initializing = false; export let _audioContext: AudioContext; export let _processor: ScriptProcessorNode; export let _output_stream: naudio.playback.OwnedAudioOutputStream; @@ -44,59 +47,65 @@ namespace audio.player { } export function initialize() { - _output_stream = naudio.playback.create_stream(); - _output_stream.set_buffer_max_latency(0.4); - _output_stream.set_buffer_latency(0.02); + if(_initializing) return; + _initializing = true; - _output_stream.callback_overflow = () => { - console.warn("Main audio overflow"); - _output_stream.clear(); - }; - _output_stream.callback_underflow = () => { - console.warn("Main audio underflow"); - }; + naudio.initialize(() => { + _output_stream = naudio.playback.create_stream(); + _output_stream.set_buffer_max_latency(0.4); + _output_stream.set_buffer_latency(0.02); - _audioContext = new AudioContext(); - _processor = _audioContext.createScriptProcessor(1024 * 8, _output_stream.channels, _output_stream.channels); + _output_stream.callback_overflow = () => { + console.warn("Main audio overflow"); + _output_stream.clear(); + }; + _output_stream.callback_underflow = () => { + console.warn("Main audio underflow"); + }; - _processor.onaudioprocess = function(event) { - const buffer = event.inputBuffer; - //console.log("Received %d channels of %d with a rate of %d", buffer.numberOfChannels, buffer.length, buffer.sampleRate); - const target_buffer = new Float32Array(buffer.numberOfChannels * buffer.length); + _audioContext = new AudioContext(); + _processor = _audioContext.createScriptProcessor(1024 * 8, _output_stream.channels, _output_stream.channels); - for(let channel = 0; channel < buffer.numberOfChannels; channel++) { - const channel_data = buffer.getChannelData(channel); - target_buffer.set(channel_data, channel * buffer.length); - } - _output_stream.write_data_rated(target_buffer.buffer, false, buffer.sampleRate); - }; - _processor.connect(_audioContext.destination); + _processor.onaudioprocess = function(event) { + const buffer = event.inputBuffer; + //console.log("Received %d channels of %d with a rate of %d", buffer.numberOfChannels, buffer.length, buffer.sampleRate); + const target_buffer = new Float32Array(buffer.numberOfChannels * buffer.length); + + for(let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channel_data = buffer.getChannelData(channel); + target_buffer.set(channel_data, channel * buffer.length); + } + _output_stream.write_data_rated(target_buffer.buffer, false, buffer.sampleRate); + }; + _processor.connect(_audioContext.destination); - _initialized = true; - for(const callback of _initialized_callbacks) - callback(); - _initialized_callbacks = []; + _initialized = true; + for(const callback of _initialized_callbacks) + callback(); + _initialized_callbacks = []; + }); return true; } export async function available_devices() : Promise { - return naudio.available_devices().filter(e => e.output_supported || e.output_default).map(e => { - return { - device_id: e.device_id, - name: e.name - } - }); + return naudio.available_devices().filter(e => e.output_supported || e.output_default); } export async function set_device(device_id?: string) : Promise { const dev = naudio.available_devices().filter(e => e.device_id == device_id); if(dev.length == 0) { - console.warn("Missing audio device with is %s", device_id) + console.warn("Missing audio device with is %s", device_id); throw "invalid device id"; } - await naudio.playback.set_device(dev[0].device_index); + try { + naudio.playback.set_device(dev[0].device_id); + } catch(error) { + if(error instanceof Error) + throw error.message; + throw error; + } _current_device = dev[0]; } diff --git a/modules/renderer/audio/AudioRecorder.ts b/modules/renderer/audio/AudioRecorder.ts index aae649c..354a77d 100644 --- a/modules/renderer/audio/AudioRecorder.ts +++ b/modules/renderer/audio/AudioRecorder.ts @@ -1,8 +1,8 @@ window["require_setup"](module); import {audio as naudio} from "teaclient_connection"; -//import {audio, tr} from "../imports/imports_shared"; -/// +// +/// export namespace _audio.recorder { import InputDevice = audio.recorder.InputDevice; @@ -14,6 +14,9 @@ export namespace _audio.recorder { let _device_cache: NativeDevice[] = undefined; export function devices() : InputDevice[] { + //TODO: Handle device updates! + if(!naudio.initialized()) return []; + return _device_cache || (_device_cache = naudio.available_devices().filter(e => e.input_supported || e.input_default).map(e => { return { unique_id: e.device_id, @@ -22,8 +25,7 @@ export namespace _audio.recorder { supported: e.input_supported, name: e.name, driver: e.driver, - sample_rate: 44100, /* TODO! */ - device_index: e.device_index, + sample_rate: 48000, /* TODO! */ } as NativeDevice })); } @@ -293,22 +295,13 @@ export namespace _audio.recorder { const device = _device as NativeDevice; /* TODO: test for? */ this._current_device = _device; try { - await new Promise((resolve, reject) => { - this.handle.set_device(device ? device.device_index : -1, flag => { - if(typeof(flag) === "boolean" && flag) - resolve(); - else - reject("failed to set device" + (typeof(flag) === "string" ? (": " + flag) : "")); - }); - }); - if(!device) return; - + await new Promise(resolve => this.handle.set_device(this._current_device.unique_id, resolve)); await new Promise((resolve, reject) => { this.handle.start(flag => { - if(flag) + if(typeof flag === "boolean" && flag) resolve(); else - reject("start failed"); + reject(typeof flag === "string" ? flag : "failed to start"); }); }); } catch(error) { @@ -453,20 +446,13 @@ export namespace _audio.recorder { this._filter.set_attack_smooth(.75); this._filter.set_release_smooth(.75); - await new Promise((resolve, reject) => { - this._recorder.set_device(this._device.device_index, flag => { - if(typeof(flag) === "boolean" && flag) - resolve(); - else - reject("initialize failed" + (typeof(flag) === "string" ? (": " + flag) : "")); - }); - }); + await new Promise(resolve => this._recorder.set_device(this._device.unique_id, resolve)); await new Promise((resolve, reject) => { this._recorder.start(flag => { - if(flag) + if(typeof flag === "boolean" && flag) resolve(); else - reject("start failed"); + reject(typeof flag === "string" ? flag : "failed to start"); }); }); } catch(error) { @@ -490,7 +476,7 @@ export namespace _audio.recorder { if(this._consumer) this._recorder.delete_consumer(this._consumer); this._recorder.stop(); - this._recorder.set_device(-1, () => {}); /* -1 := No device */ + this._recorder.set_device(undefined, () => {}); /* -1 := No device */ this._recorder = undefined; this._consumer = undefined; this._filter = undefined; @@ -506,5 +492,4 @@ export namespace _audio.recorder { } } -Object.assign(window["audio"] || (window["audio"] = {} as any), _audio); -_audio.recorder.devices(); /* query devices */ \ No newline at end of file +Object.assign(window["audio"] || (window["audio"] = {} as any), _audio); \ No newline at end of file diff --git a/native/dist/ext_nan/include/NanEventCallback.h b/native/dist/ext_nan/include/NanEventCallback.h index 1c511ee..e9b1dae 100644 --- a/native/dist/ext_nan/include/NanEventCallback.h +++ b/native/dist/ext_nan/include/NanEventCallback.h @@ -84,7 +84,7 @@ namespace Nan { struct callback_wrap { std::shared_ptr> handle; - void call_cpy(Args... args, bool no_throw = false) { + void call_cpy(Args... args, bool no_throw = false) const { if(!this->handle) { if(no_throw) return; @@ -93,7 +93,7 @@ namespace Nan { handle->callback(std::forward(args)...); } - void call(Args&&... args, bool no_throw = false) { + void call(Args&&... args, bool no_throw = false) const { if(!this->handle) { if(no_throw) return; @@ -102,11 +102,6 @@ namespace Nan { handle->callback(std::forward(args)...); } - void operator()(Args&&... args) { - if(!this->handle) - throw std::bad_function_call(); - handle->callback(std::forward(args)...); - } void operator()(Args&&... args) const { if(!this->handle) throw std::bad_function_call(); diff --git a/native/serverconnection/CMakeLists.txt b/native/serverconnection/CMakeLists.txt index d979e35..f3245eb 100644 --- a/native/serverconnection/CMakeLists.txt +++ b/native/serverconnection/CMakeLists.txt @@ -5,12 +5,12 @@ set(SOURCE_FILES src/logger.cpp src/EventLoop.cpp src/hwuid.cpp + src/ring_buffer.cpp src/connection/ft/FileTransferManager.cpp src/connection/ft/FileTransferObject.cpp src/audio/AudioSamples.cpp - src/audio/AudioDevice.cpp src/audio/AudioMerger.cpp src/audio/AudioOutput.cpp src/audio/AudioInput.cpp @@ -23,6 +23,11 @@ set(SOURCE_FILES src/audio/codec/Converter.cpp src/audio/codec/OpusConverter.cpp + + src/audio/driver/AudioDriver.cpp + src/audio/driver/SoundIO.cpp + src/audio/driver/SoundIOPlayback.cpp + src/audio/driver/SoundIORecord.cpp ) set(NODEJS_SOURCE_FILES @@ -83,6 +88,8 @@ include_directories(${StringVariable_INCLUDE_DIR}) find_package(Ed25519 REQUIRED) include_directories(${ed25519_INCLUDE_DIR}) +find_package(soundio REQUIRED) + find_package(ThreadPool REQUIRED) include_directories(${ThreadPool_INCLUDE_DIR}) if (WIN32) @@ -95,9 +102,6 @@ endif () find_package(Soxr REQUIRED) include_directories(${soxr_INCLUDE_DIR}) -find_package(PortAudio REQUIRED) -include_directories(${PortAudio_INCLUDE_DIR}) - find_package(fvad REQUIRED) include_directories(${fvad_INCLUDE_DIR}) @@ -118,11 +122,11 @@ set(REQUIRED_LIBRARIES ${DataPipes_LIBRARIES_STATIC} #Needs to be static because something causes ca bad function call when loaded in electron ${ThreadPool_LIBRARIES_STATIC} ${soxr_LIBRARIES_STATIC} - ${PortAudio_LIBRARIES_STATIC} ${fvad_LIBRARIES_STATIC} ${opus_LIBRARIES_STATIC} ${ed25519_LIBRARIES_STATIC} + soundio::static spdlog::spdlog_header_only Nan::Helpers @@ -144,10 +148,10 @@ target_link_libraries(${MODULE_NAME} ${REQUIRED_LIBRARIES}) target_compile_definitions(${MODULE_NAME} PUBLIC -DNODEJS_API) add_executable(Audio-Test ${SOURCE_FILES} test/audio/main.cpp) -target_link_libraries(Audio-Test ${REQUIRED_LIBRARIES}) +target_link_libraries(Audio-Test ${REQUIRED_LIBRARIES} soundio.a) add_executable(Audio-Test-2 ${SOURCE_FILES} test/audio/sio.cpp) -target_link_libraries(Audio-Test-2 ${REQUIRED_LIBRARIES} soundio.a pulse) +target_link_libraries(Audio-Test-2 ${REQUIRED_LIBRARIES} soundio.a) add_executable(HW-UID-Test src/hwuid.cpp) target_link_libraries(HW-UID-Test diff --git a/native/serverconnection/exports/exports.d.ts b/native/serverconnection/exports/exports.d.ts index c60d42c..9e0e0a6 100644 --- a/native/serverconnection/exports/exports.d.ts +++ b/native/serverconnection/exports/exports.d.ts @@ -144,8 +144,6 @@ declare module "teaclient_connection" { input_default: boolean; output_default: boolean; - - device_index: number; } export namespace playback { @@ -174,8 +172,8 @@ declare module "teaclient_connection" { delete(); } - export function set_device(device: number); - export function current_device() : number; + export function set_device(device_id: string); + export function current_device() : string; export function create_stream() : OwnedAudioOutputStream; @@ -234,10 +232,10 @@ declare module "teaclient_connection" { } export interface AudioRecorder { - get_device() : number; - set_device(device: number, callback: (flag: boolean | string) => void); /* Recorder needs to be started afterwards */ + get_device() : string; + set_device(device_id: string, callback: () => void); /* Recorder needs to be started afterwards */ - start(callback: (flag: boolean) => void); + start(callback: (result: boolean | string) => void); started() : boolean; stop(); @@ -252,7 +250,8 @@ declare module "teaclient_connection" { export function create_recorder() : AudioRecorder; } - export function initialize(); + export function initialize(callback: () => any); + export function initialized() : boolean; export function available_devices() : AudioDevice[]; } } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioDevice.cpp b/native/serverconnection/src/audio/AudioDevice.cpp deleted file mode 100644 index a4b7fc8..0000000 --- a/native/serverconnection/src/audio/AudioDevice.cpp +++ /dev/null @@ -1,102 +0,0 @@ -#include "AudioDevice.h" -#include "../logger.h" - -using namespace std; -using namespace tc; -using namespace tc::audio; - -extern bool devices_cached(); /* if the result is false then the call to devices() may take a while */ -extern std::deque> devices(); - -bool _devices_cached = false; -std::mutex _audio_devices_lock; -std::deque> _audio_devices{}; - -bool audio::devices_cached() { - return _devices_cached; -} - -void audio::clear_device_cache() { - std::lock_guard lock(_audio_devices_lock); - _audio_devices.clear(); - _devices_cached = false; -} - -deque> audio::devices() { - std::lock_guard lock(_audio_devices_lock); - if(_devices_cached) - return _audio_devices; - - /* query devices */ - auto device_count = Pa_GetDeviceCount(); - if(device_count < 0) { - log_error(category::audio, tr("Pa_GetDeviceCount() returned {}"), device_count); - return {}; - } - - auto default_input_device = Pa_GetDefaultInputDevice(); - auto default_output_device = Pa_GetDefaultOutputDevice(); - - for(PaDeviceIndex device_index = 0; device_index < device_count; device_index++) { - auto device_info = Pa_GetDeviceInfo(device_index); - if(!device_info) { - log_warn(category::audio, tr("Pa_GetDeviceInfo(...) failed for device {}"), device_index); - continue; - } - - auto device_host_info = Pa_GetHostApiInfo(device_info->hostApi); - if(!device_host_info) { - log_warn(category::audio, tr("Pa_GetHostApiInfo(...) failed for device {} with host api {}"), device_index, device_info->hostApi); - continue; - } - - auto info = make_shared(); - info->device_id = device_index; - info->name = device_info->name; - - info->max_inputs = device_info->maxInputChannels; - info->input_supported = device_info->maxInputChannels > 0; - - info->max_outputs = device_info->maxOutputChannels; - info->output_supported = device_info->maxOutputChannels > 0; - - info->is_default_input = device_index == default_input_device; - info->is_default_output = device_index == default_output_device; - - info->driver = device_host_info->name; - info->is_default_driver_input = device_index == device_host_info->defaultInputDevice; - info->is_default_driver_output = device_index == device_host_info->defaultOutputDevice; - - PaStreamParameters test_parameters{}; - test_parameters.device = device_index; - test_parameters.sampleFormat = paFloat32; - test_parameters.suggestedLatency = 0; /* ignored by Pa_IsFormatSupported() */ - test_parameters.hostApiSpecificStreamInfo = nullptr; - - /* - if(info->input_supported) { - test_parameters.channelCount = device_info->maxInputChannels; - for(size_t index = 0; standard_sample_rates[index] > 0; index++) { - auto rate = standard_sample_rates[index]; - auto err = Pa_IsFormatSupported(&test_parameters, nullptr, rate); - if(err == paFormatIsSupported) - info->supported_input_rates.push_back(rate); - } - } - if(info->output_supported) { - test_parameters.channelCount = device_info->maxOutputChannels; - for(size_t index = 0; standard_sample_rates[index] > 0; index++) { - auto rate = standard_sample_rates[index]; - auto err = Pa_IsFormatSupported(nullptr, &test_parameters, rate); - if(err == paFormatIsSupported) - info->supported_output_rates.push_back(rate); - } - } - */ - - _audio_devices.push_back(info); - } - - _devices_cached = true; - return _audio_devices; -} \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioDevice.h b/native/serverconnection/src/audio/AudioDevice.h deleted file mode 100644 index 91d7fa2..0000000 --- a/native/serverconnection/src/audio/AudioDevice.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace tc { - namespace audio { - static constexpr double standard_sample_rates[] = { - 8000.0, 9600.0, 11025.0, 12000.0, 16000.0, 22050.0, 24000.0, 32000.0, - 44100.0, 48000.0, 88200.0, 96000.0, 192000.0, -1 /* negative terminated list */ - }; - - struct AudioDevice { - PaDeviceIndex device_id; - - bool is_default_output; - bool is_default_input; - - bool is_default_driver_output; - bool is_default_driver_input; - - bool input_supported; - bool output_supported; - std::string name; - std::string driver; - - /* - std::vector supported_input_rates; - std::vector supported_output_rates; - */ - - int max_inputs; - int max_outputs; - }; - - extern void clear_device_cache(); - extern bool devices_cached(); /* if the result is false then the call to devices() may take a while */ - extern std::deque> devices(); - } -} \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioInput.cpp b/native/serverconnection/src/audio/AudioInput.cpp index 9e017e1..545a04d 100644 --- a/native/serverconnection/src/audio/AudioInput.cpp +++ b/native/serverconnection/src/audio/AudioInput.cpp @@ -9,6 +9,7 @@ using namespace std; using namespace tc; using namespace tc::audio; +#if false class AudioInputSource { public: constexpr static auto kChannelCount{2}; @@ -121,7 +122,7 @@ class AudioInputSource { std::lock_guard lock{input_source->registered_inputs_lock}; for(auto& client : input_source->registered_inputs) - client->audio_callback(input, frameCount, timeInfo, statusFlags); + client->consume(input, frameCount, 2); return 0; } @@ -152,6 +153,7 @@ std::shared_ptr get_input_source(PaDeviceIndex device_index, b input_sources.push_back(std::make_shared(device_index)); return input; } +#endif AudioConsumer::AudioConsumer(tc::audio::AudioInput *handle, size_t channel_count, size_t sample_rate, size_t frame_size) : handle(handle), @@ -189,50 +191,57 @@ AudioInput::~AudioInput() { consumer->handle = nullptr; } -PaDeviceIndex AudioInput::current_device() { +void AudioInput::set_device(const std::shared_ptr &device) { lock_guard lock(this->input_source_lock); - return this->input_source ? this->input_source->device_index : paNoDevice; -} + if(device == this->input_device) return; -bool AudioInput::open_device(std::string& error, PaDeviceIndex index) { - lock_guard lock(this->input_source_lock); - - if(index == (this->input_source ? this->input_source->device_index : paNoDevice)) - return true; - - this->close_device(); - if(index == paNoDevice) - return true; - - this->input_source = get_input_source(index, true); - this->input_source->register_consumer(this); - return this->input_source->begin_recording(error); + this->close_device(); + this->input_device = device; } void AudioInput::close_device() { lock_guard lock(this->input_source_lock); - if(this->input_source) { - this->input_source->remove_consumer(this); - this->input_source->stop_recording_if_possible(); - this->input_source.reset(); + if(this->input_recorder) { + this->input_recorder->remove_consumer(this); + this->input_recorder->stop_if_possible(); + this->input_recorder.reset(); } - this->input_recording = false; + this->input_device = nullptr; } -bool AudioInput::record() { +bool AudioInput::record(std::string& error) { lock_guard lock(this->input_source_lock); - if(!this->input_source) return false; + if(!this->input_device) { + error = "no device"; + return false; + } + if(this->input_recorder) return true; - this->input_recording = true; + this->input_recorder = this->input_device->record(); + if(!this->input_recorder) { + error = "failed to get recorder"; + return false; + } + + this->input_recorder->register_consumer(this); + if(!this->input_recorder->start(error)) { + this->input_recorder->remove_consumer(this); + this->input_recorder.reset(); + return false; + } return true; } bool AudioInput::recording() { - return this->input_recording; + return !!this->input_recorder; } void AudioInput::stop() { - this->input_recording = false; + if(!this->input_recorder) return; + + this->input_recorder->remove_consumer(this); + this->input_recorder->stop_if_possible(); + this->input_recorder.reset(); } std::shared_ptr AudioInput::create_consumer(size_t frame_length) { @@ -255,9 +264,7 @@ void AudioInput::delete_consumer(const std::shared_ptr &source) { source->handle = nullptr; } -void AudioInput::audio_callback(const void *input, unsigned long frameCount, const PaStreamCallbackTimeInfo* timeInfo, PaStreamCallbackFlags statusFlags) { - if(!this->input_recording) return; - +void AudioInput::consume(const void *input, unsigned long frameCount, size_t /* channels */) { if(this->_volume != 1 && false) { auto ptr = (float*) input; auto left = frameCount * this->_channel_count; diff --git a/native/serverconnection/src/audio/AudioInput.h b/native/serverconnection/src/audio/AudioInput.h index 9bb4e17..877b16a 100644 --- a/native/serverconnection/src/audio/AudioInput.h +++ b/native/serverconnection/src/audio/AudioInput.h @@ -5,9 +5,9 @@ #include #include #include -#include #include #include "AudioSamples.h" +#include "driver/AudioDriver.h" class AudioInputSource; namespace tc { @@ -31,25 +31,22 @@ namespace tc { AudioConsumer(AudioInput* handle, size_t channel_count, size_t sample_rate, size_t frame_size); std::unique_ptr reframer; - std::mutex buffer_lock; - size_t buffered_samples = 0; - std::deque> sample_buffers; void process_data(const void* /* buffer */, size_t /* samples */); void handle_framed_data(const void* /* buffer */, size_t /* samples */); }; - class AudioInput { + class AudioInput : public AudioDeviceRecord::Consumer { friend class ::AudioInputSource; public: AudioInput(size_t /* channels */, size_t /* rate */); virtual ~AudioInput(); - [[nodiscard]] bool open_device(std::string& /* error */, PaDeviceIndex); - [[nodiscard]] PaDeviceIndex current_device(); + void set_device(const std::shared_ptr& /* device */); + [[nodiscard]] std::shared_ptr current_device() const { return this->input_device; } void close_device(); - [[nodiscard]] bool record(); + [[nodiscard]] bool record(std::string& /* error */); [[nodiscard]] bool recording(); void stop(); @@ -67,7 +64,7 @@ namespace tc { inline float volume() { return this->_volume; } inline void set_volume(float value) { this->_volume = value; } private: - void audio_callback(const void *, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags); + void consume(const void *, unsigned long, size_t) override; size_t const _channel_count; size_t const _sample_rate; @@ -76,10 +73,10 @@ namespace tc { std::deque> _consumers; std::recursive_mutex input_source_lock; - bool input_recording{false}; - std::shared_ptr<::AudioInputSource> input_source{}; - + std::shared_ptr input_device{}; float _volume = 1.f; - }; + + std::shared_ptr input_recorder{}; + }; } } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioOutput.cpp b/native/serverconnection/src/audio/AudioOutput.cpp index 05d2012..82cebbf 100644 --- a/native/serverconnection/src/audio/AudioOutput.cpp +++ b/native/serverconnection/src/audio/AudioOutput.cpp @@ -10,110 +10,127 @@ using namespace tc; using namespace tc::audio; void AudioOutputSource::clear() { - lock_guard lock(this->buffer_lock); - this->sample_buffers.clear(); - this->buffered_samples = 0; + this->buffer.clear(); + this->buffering = true; } ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) { - auto sample_count = samples; + size_t written{0}, written_bytes{0}; - _retest: - { - lock_guard lock(this->buffer_lock); - if(this->buffering) { - if(this->buffered_samples > this->min_buffer) { - this->buffering = false; - } else { - return 0; - } - } - while(sample_count > 0 && !this->sample_buffers.empty()) { - auto buf = this->sample_buffers[0]; - auto sc = min((size_t) (buf->sample_size - buf->sample_index), (size_t) sample_count); - if(sc > 0 && buffer) { /* just to ensure */ - memcpy(buffer, (char *) buf->sample_data + this->channel_count * buf->sample_index * 4, sc * this->channel_count * 4); - } else { -#ifndef WIN32 - /* for my debugger */ - __asm__("nop"); -#endif - } + load_buffer: + auto available_bytes = this->buffer.fill_count(); + if(available_bytes < sizeof(float) * this->channel_count) return written; - sample_count -= sc; - buf->sample_index += (uint16_t) sc; - if(buf->sample_index == buf->sample_size) - this->sample_buffers.pop_front(); + auto available_samples = available_bytes / sizeof(float) / this->channel_count; + //log_trace(category::audio, tr("Min: {}, Max: {}, Current: {}, Buffering: {}"), this->min_buffered_samples, this->max_buffered_samples, available_samples, this->buffering); + if(this->buffering && available_samples < this->min_buffered_samples) return -2; - if(buffer) - buffer = (char*) buffer + sc * this->channel_count * 4; - } - this->buffered_samples -= samples - sample_count; - } + this->buffering = false; + if(available_samples >= samples - written) { + const auto byte_length = (samples - written) * sizeof(float) * this->channel_count; + memcpy((char*) buffer + written_bytes, this->buffer.read_ptr(), byte_length); + this->buffer.advance_read_ptr(byte_length); + return samples; + } else { + const auto byte_length = available_samples * sizeof(float) * this->channel_count; + memcpy((char*) buffer + written_bytes, this->buffer.read_ptr(), byte_length); + this->buffer.advance_read_ptr(byte_length); + written += available_samples; + written_bytes += byte_length; + } - if(sample_count > 0) { - if(this->on_underflow) { - if(this->on_underflow()) { - goto _retest; - } - } - this->buffering = true; - } + if(auto fn = this->on_underflow; fn) + if(fn()) + goto load_buffer; + this->buffering = true; if(this->on_read) this->on_read(); - return samples - sample_count; /* return the written samples */ + return written; /* return the written samples */ } ssize_t AudioOutputSource::enqueue_samples(const void *buffer, size_t samples) { - auto buf = SampleBuffer::allocate((uint8_t) this->channel_count, (uint16_t) samples); - if(!buf) - return -1; + size_t enqueued{0}; - buf->sample_index = 0; - memcpy(buf->sample_data, buffer, this->channel_count * samples * 4); + auto free_bytes = this->buffer.free_count(); + auto free_samples = free_bytes / sizeof(float) / this->channel_count; + if(this->max_buffered_samples && free_samples > this->max_buffered_samples) free_samples = this->max_buffered_samples; - return this->enqueue_samples(buf); + if(free_samples >= samples) { + const auto byte_length = samples * sizeof(float) * this->channel_count; + memcpy(this->buffer.write_ptr(), buffer, byte_length); + this->buffer.advance_write_ptr(byte_length); + return samples; + } else { + const auto byte_length = free_samples * sizeof(float) * this->channel_count; + memcpy(this->buffer.write_ptr(), buffer, byte_length); + this->buffer.advance_write_ptr(byte_length); + enqueued += free_samples; + } + + if(auto fn = this->on_overflow; fn) + fn(samples - enqueued); + + switch (this->overflow_strategy) { + case overflow_strategy::discard_input: + return -2; + case overflow_strategy::discard_buffer_all: + this->buffer.clear(); + break; + case overflow_strategy::discard_buffer_half: + this->buffer.advance_read_ptr(this->buffer.fill_count() / 2); + break; + case overflow_strategy::ignore: + break; + } + + return enqueued; } -ssize_t AudioOutputSource::enqueue_samples(const std::shared_ptr &buf) { - if(!buf) return 0; +ssize_t AudioOutputSource::enqueue_samples_no_interleave(const void *buffer, size_t samples) { + auto free_bytes = this->buffer.free_count(); + auto free_samples = free_bytes / sizeof(float) / this->channel_count; + if(this->max_buffered_samples && free_samples > this->max_buffered_samples) free_samples = this->max_buffered_samples; - { - unique_lock lock(this->buffer_lock); - if(this->max_latency > 0 && this->buffered_samples + buf->sample_size > this->max_latency) { - /* overflow! */ - auto overflow_length = this->buffered_samples + buf->sample_size - this->max_latency; - if(this->on_overflow) { - lock.unlock(); - this->on_overflow(overflow_length); - lock.lock(); - } + auto samples_to_write{samples}; + if(samples_to_write > free_samples) samples_to_write = free_samples; + const auto enqueued{samples_to_write}; + { + auto src_buffer = (const float*) buffer; + auto target_buffer = (float*) this->buffer.write_ptr(); - switch (this->overflow_strategy) { - case overflow_strategy::discard_input: - return -2; - case overflow_strategy::discard_buffer_all: - this->sample_buffers.clear(); - break; - case overflow_strategy::discard_buffer_half: - this->sample_buffers.erase(this->sample_buffers.begin(), this->sample_buffers.begin() + (int) ceil(this->sample_buffers.size() / 2)); - break; - case overflow_strategy::ignore: - break; - } - } + while (samples_to_write-- > 0) { + *target_buffer = *src_buffer; + *(target_buffer + 1) = *(src_buffer + samples); - this->sample_buffers.push_back(buf); - this->buffered_samples += buf->sample_size; - } + target_buffer += 2; + src_buffer++; + } + } - return buf->sample_size; + if(auto fn = this->on_overflow; fn) + fn(samples - enqueued); + + switch (this->overflow_strategy) { + case overflow_strategy::discard_input: + return -2; + case overflow_strategy::discard_buffer_all: + this->buffer.clear(); + break; + case overflow_strategy::discard_buffer_half: + this->buffer.advance_read_ptr(this->buffer.fill_count() / 2); + break; + case overflow_strategy::ignore: + break; + } + + return enqueued; } -AudioOutput::AudioOutput(size_t channels, size_t rate) : _channel_count(channels), _sample_rate(rate) {} +AudioOutput::AudioOutput(size_t channels, size_t rate) : _channel_count(channels), _sample_rate(rate) { } + AudioOutput::~AudioOutput() { this->close_device(); this->cleanup_buffers(); @@ -152,136 +169,115 @@ void AudioOutput::cleanup_buffers() { this->source_buffer_length = 0; } -int AudioOutput::_audio_callback(const void *a, void *b, unsigned long c, const PaStreamCallbackTimeInfo* d, PaStreamCallbackFlags e, void *_ptr_audio_output) { - return reinterpret_cast(_ptr_audio_output)->audio_callback(a, b, c, d, e); -} - -int AudioOutput::audio_callback(const void *input, void *output, unsigned long frameCount, const PaStreamCallbackTimeInfo* timeInfo, PaStreamCallbackFlags statusFlags) { - if(!output) /* hmmm.. suspicious */ - return 0; - +void AudioOutput::fill_buffer(void *output, unsigned long frameCount, size_t channels) { lock_guard buffer_lock(this->buffer_lock); + if(this->_volume <= 0) { + for(auto& source : this->_sources) + source->pop_samples(nullptr, frameCount); + memset(output, 0, sizeof(frameCount) * channels * sizeof(float)); + return; + } + size_t buffer_length = frameCount * 4 * this->_channel_count; size_t sources = 0; size_t actual_sources = 0; - auto volume = this->_volume; { lock_guard lock(this->sources_lock); sources = this->_sources.size(); actual_sources = sources; if(sources > 0) { - if(volume > 0) { /* allocate the required space */ - auto source_buffer_length = buffer_length * sources; - auto source_merge_buffer_length = sizeof(void*) * sources; + /* allocate the required space */ + auto source_buffer_length = buffer_length * sources; + auto source_merge_buffer_length = sizeof(void*) * sources; - if(this->source_buffer_length < source_buffer_length || !this->source_buffer) { - if(this->source_buffer) - free(this->source_buffer); - this->source_buffer = malloc(source_buffer_length); - this->source_buffer_length = source_buffer_length; - } - if(this->source_merge_buffer_length < source_merge_buffer_length || !this->source_merge_buffer) { - if(this->source_merge_buffer) - free(this->source_merge_buffer); - this->source_merge_buffer = (void**) malloc(source_merge_buffer_length); - this->source_merge_buffer_length = source_merge_buffer_length; - } - } + //TODO: Move this out of the loop? + { + + if(this->source_buffer_length < source_buffer_length || !this->source_buffer) { + if(this->source_buffer) + free(this->source_buffer); + this->source_buffer = malloc(source_buffer_length); + this->source_buffer_length = source_buffer_length; + } + if(this->source_merge_buffer_length < source_merge_buffer_length || !this->source_merge_buffer) { + if (this->source_merge_buffer) + free(this->source_merge_buffer); + this->source_merge_buffer = (void **) malloc(source_merge_buffer_length); + this->source_merge_buffer_length = source_merge_buffer_length; + } + } for(size_t index = 0; index < sources; index++) { auto& source = this->_sources[index]; - if(volume > 0) { - this->source_merge_buffer[index] = (char*) this->source_buffer + (buffer_length * index); - auto written_frames = this->_sources[index]->pop_samples(this->source_merge_buffer[index], frameCount); - if(written_frames != frameCount) { - if(written_frames == 0) { - this->source_merge_buffer[index] = nullptr; - actual_sources--; - } else { - /* fill up the rest with silence (0) */ - auto written = written_frames * this->_channel_count * 4; - memset((char*) this->source_merge_buffer[index] + written, 0, (frameCount - written_frames) * this->_channel_count * 4); - } - } - } else { - this->_sources[index]->pop_samples(nullptr, frameCount); - } + this->source_merge_buffer[index] = (char*) this->source_buffer + (buffer_length * index); + auto written_frames = this->_sources[index]->pop_samples(this->source_merge_buffer[index], frameCount); + if(written_frames != frameCount) { + if(written_frames <= 0) { + this->source_merge_buffer[index] = nullptr; + actual_sources--; + } else { + /* fill up the rest with silence (0) */ + auto written = written_frames * this->_channel_count * 4; + memset((char*) this->source_merge_buffer[index] + written, 0, (frameCount - written_frames) * this->_channel_count * 4); + } + } } } } - if(actual_sources > 0 && volume > 0) { - if(!merge::merge_n_sources(output, this->source_merge_buffer, sources, this->_channel_count, frameCount)) { + if(actual_sources > 0) { + if(!merge::merge_n_sources(output, this->source_merge_buffer, sources, this->_channel_count, frameCount)) log_warn(category::audio, tr("failed to merge buffers!")); + + auto volume = this->_volume; + if(volume != 1) { + auto float_length = this->_channel_count * frameCount; + auto data = (float*) output; + while(float_length-- > 0) + *data++ *= volume; } - auto float_length = this->_channel_count * frameCount; - auto data = (float*) output; - while(float_length-- > 0) - *data++ *= volume; } else { - memset(output, 0, this->_channel_count * 4 * frameCount); + memset(output, 0, this->_channel_count * sizeof(float) * frameCount); } - - return 0; } -bool AudioOutput::open_device(std::string& error, PaDeviceIndex index) { - lock_guard lock(this->output_stream_lock); +void AudioOutput::set_device(const std::shared_ptr &device) { + lock_guard lock(this->device_lock); + if(this->device == device) return; - if(index == this->_current_device_index) - return true; - - this->close_device(); - this->_current_device_index = index; - this->_current_device = Pa_GetDeviceInfo(index); - - if(!this->_current_device) { - this->_current_device_index = paNoDevice; - error = "failed to get device info"; - return false; - } - - PaStreamParameters output_parameters{}; - memset(&output_parameters, 0, sizeof(output_parameters)); - output_parameters.channelCount = (int) this->_channel_count; - output_parameters.device = this->_current_device_index; - output_parameters.sampleFormat = paFloat32; - output_parameters.suggestedLatency = this->_current_device->defaultLowOutputLatency; - - auto err = Pa_OpenStream(&output_stream, nullptr, &output_parameters, (double) this->_sample_rate, paFramesPerBufferUnspecified, paClipOff, &AudioOutput::_audio_callback, this); - if(err != paNoError) { - error = to_string(err) + "/" + Pa_GetErrorText(err); - return false; - } - - return true; -} - -bool AudioOutput::playback() { - lock_guard lock(this->output_stream_lock); - if(!this->output_stream) - return false; - - auto err = Pa_StartStream(this->output_stream); - if(err != paNoError && err != paStreamIsNotStopped) { - log_error(category::audio, tr("Pa_StartStream returned {}"), err); - return false; - } - - return true; + this->close_device(); + this->device = device; } void AudioOutput::close_device() { - lock_guard lock(this->output_stream_lock); - if(this->output_stream) { - /* TODO: Test for errors */ - Pa_StopStream(this->output_stream); - Pa_CloseStream(this->output_stream); + lock_guard lock(this->device_lock); + if(this->_playback) { + this->_playback->remove_source(this); + this->_playback->stop_if_possible(); + this->_playback.reset(); + } - this->output_stream = nullptr; + this->device = nullptr; +} + +bool AudioOutput::playback(std::string& error) { + lock_guard lock(this->device_lock); + if(!this->device) { + error = "invalid device handle"; + return false; } + if(this->_playback) return true; + + this->_playback = this->device->playback(); + if(!this->_playback) { + error = "failed to allocate memory"; + return false; + } + + this->_playback->register_source(this); + return this->_playback->start(error); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/AudioOutput.h b/native/serverconnection/src/audio/AudioOutput.h index 64b11be..c23ef3b 100644 --- a/native/serverconnection/src/audio/AudioOutput.h +++ b/native/serverconnection/src/audio/AudioOutput.h @@ -5,15 +5,15 @@ #include #include #include -#include -#include "AudioSamples.h" +#include "./AudioSamples.h" +#include "./driver/AudioDriver.h" +#include "../ring_buffer.h" #ifdef WIN32 #define ssize_t int64_t #endif -namespace tc { - namespace audio { +namespace tc::audio { class AudioOutput; namespace overflow_strategy { @@ -32,13 +32,21 @@ namespace tc { size_t const channel_count = 0; size_t const sample_rate = 0; - bool buffering = true; - size_t min_buffer = 0; + [[nodiscard]] inline size_t max_supported_latency() const { + return this->buffer.capacity() / this->channel_count / sizeof(float); + } + + [[nodiscard]] inline size_t max_latency() const { + const auto max_samples = this->max_supported_latency(); + if(this->max_buffered_samples && this->max_buffered_samples <= max_samples) return this->max_buffered_samples; + + return max_samples; + } + + bool buffering{true}; + size_t min_buffered_samples{0}; + size_t max_buffered_samples{0}; - /* For stream set it to the max latency in samples you want. - * Zero means unlimited - */ - size_t max_latency = 0; overflow_strategy::value overflow_strategy = overflow_strategy::discard_buffer_half; /* if it returns true then the it means that the buffer has been refilled, we have to test again */ @@ -49,24 +57,24 @@ namespace tc { void clear(); ssize_t pop_samples(void* /* output buffer */, size_t /* sample count */); ssize_t enqueue_samples(const void * /* input buffer */, size_t /* sample count */); - ssize_t enqueue_samples(const std::shared_ptr& /* buffer */); + ssize_t enqueue_samples_no_interleave(const void * /* input buffer */, size_t /* sample count */); private: - AudioOutputSource(AudioOutput* handle, size_t channel_count, size_t sample_rate) : handle(handle), channel_count(channel_count), sample_rate(sample_rate) {} + AudioOutputSource(AudioOutput* handle, size_t channel_count, size_t sample_rate) : + handle(handle), channel_count(channel_count), sample_rate(sample_rate), buffer{channel_count * sample_rate * sizeof(float)} { + } - std::mutex buffer_lock; - size_t buffered_samples = 0; - std::deque> sample_buffers; + tc::ring_buffer buffer; }; - class AudioOutput { + class AudioOutput : public AudioDevicePlayback::Source { public: AudioOutput(size_t /* channels */, size_t /* rate */); virtual ~AudioOutput(); - bool open_device(std::string& /* error */, PaDeviceIndex); - bool playback(); + void set_device(const std::shared_ptr& /* device */); + bool playback(std::string& /* error */); void close_device(); - PaDeviceIndex current_device() { return this->_current_device_index; } + std::shared_ptr current_device() { return this->device; } std::deque> sources() { std::lock_guard lock(this->sources_lock); @@ -82,8 +90,7 @@ namespace tc { inline float volume() { return this->_volume; } inline void set_volume(float value) { this->_volume = value; } private: - static int _audio_callback(const void *, void *, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags, void*); - int audio_callback(const void *, void *, unsigned long, const PaStreamCallbackTimeInfo*, PaStreamCallbackFlags); + void fill_buffer(void *, size_t frames, size_t channels) override ; size_t const _channel_count; size_t const _sample_rate; @@ -91,10 +98,9 @@ namespace tc { std::mutex sources_lock; std::deque> _sources; - std::recursive_mutex output_stream_lock; - const PaDeviceInfo* _current_device = nullptr; - PaDeviceIndex _current_device_index = paNoDevice; - PaStream* output_stream = nullptr; + std::recursive_mutex device_lock; + std::shared_ptr device{nullptr}; + std::shared_ptr _playback{nullptr}; std::mutex buffer_lock; /* not required, but why not. Usually only used within audio_callback! */ void* source_buffer = nullptr; @@ -106,5 +112,4 @@ namespace tc { float _volume = 1.f; }; - } -} \ No newline at end of file + } \ No newline at end of file diff --git a/native/serverconnection/src/audio/driver/AudioDriver.cpp b/native/serverconnection/src/audio/driver/AudioDriver.cpp new file mode 100644 index 0000000..5c7f808 --- /dev/null +++ b/native/serverconnection/src/audio/driver/AudioDriver.cpp @@ -0,0 +1,219 @@ +// +// Created by wolverindev on 07.02.20. +// + +#include +#include +#include +#include "AudioDriver.h" +#include "SoundIO.h" +#include "../../logger.h" +#include "../AudioMerger.h" + +using namespace tc::audio; + +namespace tc::audio { + std::deque> devices() { + std::deque> result{}; + for(auto& backend : SoundIOBackendHandler::all_backends()) { + auto input_devices = backend->input_devices(); + auto output_devices = backend->output_devices(); + + result.insert(result.end(), input_devices.begin(), input_devices.end()); + result.insert(result.end(), output_devices.begin(), output_devices.end()); + } + return result; + } + + std::shared_ptr find_device_by_id(const std::string_view& id, bool input) { + for(auto& backend : SoundIOBackendHandler::all_backends()) { + for(auto& dev : input ? backend->input_devices() : backend->output_devices()) + if(dev->id() == id) + return dev; + } + return nullptr; + } + + std::mutex initialize_lock{}; + std::deque initialize_callbacks{}; + int initialize_state{0}; /* 0 := not initialized | 1 := initialized | 2 := initializing */ + + void _initialize() { + SoundIOBackendHandler::initialize_all(); + SoundIOBackendHandler::connect_all(); + } + + void _finalize() { + SoundIOBackendHandler::shutdown_all(); + } + + void initialize(const initialize_callback_t& callback) { + { + std::unique_lock init_lock{initialize_lock}; + if(initialize_state == 2) { + if(callback) + initialize_callbacks.push_back(callback); + return; + } else if(initialize_state == 1) { + init_lock.unlock(); + callback(); + return; + } else if(initialize_state != 0) { + init_lock.unlock(); + callback(); + log_warn(category::audio, tr("Invalid initialize state ({})"), initialize_state); + return; + } + + initialize_state = 2; + } + std::thread init_thread([]{ + _initialize(); + + std::unique_lock lock{initialize_lock}; + auto callbacks = std::move(initialize_callbacks); + initialize_state = 1; + lock.unlock(); + + for(auto& callback : callbacks) + callback(); + }); + threads::name(init_thread, tr("audio init")); + init_thread.detach(); + } + + void await_initialized() { + std::condition_variable cv{}; + std::mutex m{}; + + std::unique_lock init_lock{initialize_lock}; + if(initialize_state != 2) return; + initialize_callbacks.emplace_back([&]{ cv.notify_all(); }); + init_lock.unlock(); + + std::unique_lock m_lock{m}; + cv.wait(m_lock); + } + + bool initialized() { + std::unique_lock init_lock{initialize_lock}; + return initialize_state == 1; + } + + void finalize() { + await_initialized(); + _finalize(); + } + + bool AudioDevicePlayback::start(std::string &error) { + std::lock_guard lock{this->state_lock}; + if(this->running) return true; + + if(!this->impl_start(error)) { + log_error(category::audio, tr("Failed to start playback: {}"), error); + return false; + } + this->running = true; + return true; + } + + void AudioDevicePlayback::stop_if_possible() { + std::lock_guard lock{this->state_lock}; + { + std::lock_guard s_lock{this->source_lock}; + if(!this->_sources.empty()) return; + } + + this->impl_stop(); + this->running = false; + } + + void AudioDevicePlayback::stop() { + std::lock_guard lock{this->state_lock}; + if(this->running) return; + + this->impl_stop(); + this->running = false; + } + + void AudioDevicePlayback::register_source(Source* source) { + std::lock_guard s_lock{this->source_lock}; + this->_sources.push_back(source); + } + + void AudioDevicePlayback::remove_source(Source* source) { + std::lock_guard s_lock{this->source_lock}; + auto index = find(this->_sources.begin(), this->_sources.end(), source); + if(index == this->_sources.end()) return; + + this->_sources.erase(index); + } + +#define TMP_BUFFER_SIZE 8096 + void AudioDevicePlayback::fill_buffer(void *buffer, size_t samples, size_t channels) { + std::lock_guard lock{this->source_lock}; + + const auto size = this->_sources.size(); + if(size == 1) { + this->_sources.front()->fill_buffer(buffer, samples, channels); + } else if(size > 1) { + this->_sources.front()->fill_buffer(buffer, samples, channels); + uint8_t tmp_buffer[TMP_BUFFER_SIZE]; + if(sizeof(float) * samples * channels > TMP_BUFFER_SIZE) { + log_warn(category::audio, tr("Dropping input source data because of too small merge buffer")); + return; + } + + for(auto it = this->_sources.begin() + 1; it != this->_sources.end(); it++) { + (*it)->fill_buffer(tmp_buffer, samples, channels); + merge::merge_sources(buffer, buffer, tmp_buffer, channels, samples); + } + } else { + memset(buffer, 0, samples * channels * sizeof(float)); + } + } + + bool AudioDeviceRecord::start(std::string &error) { + std::lock_guard lock{this->state_lock}; + if(this->running) return true; + + if(!this->impl_start(error)) { + log_error(category::audio, tr("Failed to start record: {}"), error); + return false; + } + this->running = true; + return true; + } + + void AudioDeviceRecord::stop_if_possible() { + std::lock_guard lock{this->state_lock}; + { + std::lock_guard s_lock{this->consumer_lock}; + if(!this->_consumers.empty()) return; + } + + this->impl_stop(); + this->running = false; + } + + void AudioDeviceRecord::stop() { + std::lock_guard lock{this->state_lock}; + if(this->running) return; + + this->impl_stop(); + this->running = false; + } + + void AudioDeviceRecord::register_consumer(Consumer* source) { + std::lock_guard s_lock{this->consumer_lock}; + this->_consumers.push_back(source); + } + + void AudioDeviceRecord::remove_consumer(Consumer* source) { + std::lock_guard s_lock{this->consumer_lock}; + auto index = find(this->_consumers.begin(), this->_consumers.end(), source); + if(index == this->_consumers.end()) return; + + this->_consumers.erase(index); + } +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/driver/AudioDriver.h b/native/serverconnection/src/audio/driver/AudioDriver.h new file mode 100644 index 0000000..aabbec0 --- /dev/null +++ b/native/serverconnection/src/audio/driver/AudioDriver.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace tc::audio { + class AudioDeviceRecord { + public: + class Consumer { + public: + virtual void consume(const void* /* buffer */, size_t /* samples */, size_t /* channel count */) = 0; + }; + + [[nodiscard]] bool start(std::string& /* error */); + void stop_if_possible(); + void stop(); + + [[nodiscard]] inline std::vector consumer() { + std::lock_guard lock{this->consumer_lock}; + return this->_consumers; + } + void register_consumer(Consumer* /* source */); + void remove_consumer(Consumer* /* source */); + + protected: + virtual bool impl_start(std::string& /* error */) = 0; + virtual void impl_stop() = 0; + + std::mutex state_lock{}; + bool running{false}; + + std::mutex consumer_lock{}; + std::vector _consumers{}; + }; + + class AudioDevicePlayback { + public: + class Source { + public: + virtual void fill_buffer(void* /* target */, size_t /* samples */, size_t /* channel count */) = 0; + }; + + [[nodiscard]] bool start(std::string& /* error */); + void stop_if_possible(); + void stop(); + + [[nodiscard]] inline std::vector sources() { + std::lock_guard lock{this->source_lock}; + return this->_sources; + } + void register_source(Source* /* source */); + void remove_source(Source* /* source */); + + protected: + virtual bool impl_start(std::string& /* error */) = 0; + virtual void impl_stop() = 0; + + void fill_buffer(void* /* target */, size_t /* samples */, size_t /* channel count */); + + std::mutex state_lock{}; + bool running{false}; + + std::mutex source_lock{}; + std::vector _sources{}; + }; + + class AudioDevice { + public: + /* information */ + [[nodiscard]] virtual std::string id() const = 0; + [[nodiscard]] virtual std::string name() const = 0; + [[nodiscard]] virtual std::string driver() const = 0; + + [[nodiscard]] virtual bool is_input_supported() const = 0; + [[nodiscard]] virtual bool is_output_supported() const = 0; + + [[nodiscard]] virtual bool is_input_default() const = 0; + [[nodiscard]] virtual bool is_output_default() const = 0; + + [[nodiscard]] virtual std::shared_ptr playback() = 0; + [[nodiscard]] virtual std::shared_ptr record() = 0; + }; + + typedef std::function initialize_callback_t; + + extern void finalize(); + extern void initialize(const initialize_callback_t& /* callback */ = []{}); + extern void await_initialized(); + extern bool initialized(); + + extern std::deque> devices(); + extern std::shared_ptr find_device_by_id(const std::string_view& /* id */, bool /* input */); +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/driver/SoundIO.cpp b/native/serverconnection/src/audio/driver/SoundIO.cpp new file mode 100644 index 0000000..fed7dda --- /dev/null +++ b/native/serverconnection/src/audio/driver/SoundIO.cpp @@ -0,0 +1,260 @@ +// +// Created by wolverindev on 07.02.20. +// + +#include "SoundIO.h" +#include +#include "../../logger.h" + +using namespace tc::audio; + +std::mutex SoundIOBackendHandler::backend_lock{}; +std::vector> SoundIOBackendHandler::backends{}; + +std::shared_ptr SoundIOBackendHandler::get_backend(SoundIoBackend backend_type) { + std::lock_guard lock{backend_lock}; + for(auto& backend : SoundIOBackendHandler::backends) + if(backend->backend == backend_type) + return backend; + + return nullptr; +} + +void SoundIOBackendHandler::initialize_all() { + std::lock_guard lock{backend_lock}; + + for(const auto& backend : { + SoundIoBackendJack, + SoundIoBackendPulseAudio, + SoundIoBackendAlsa, + SoundIoBackendCoreAudio, + SoundIoBackendWasapi, + SoundIoBackendDummy + }) { + if(!soundio_have_backend(backend)) { + log_debug(category::audio, tr("Skipping audio backend {} because its not supported on this platform."), soundio_backend_name(backend)); + continue; + } + + auto handler = std::make_shared(backend); + if(std::string error{}; !handler->initialize(error)) { + log_error(category::audio, tr("Failed to initialize sound backed {}: {}"), soundio_backend_name(backend), error); + continue; + } + + backends.push_back(handler); + } + + std::stable_sort(backends.begin(), backends.end(), [](const auto& a, const auto& b) { return a->priority() > b->priority(); }); +} + +void SoundIOBackendHandler::connect_all() { + std::string error{}; + + std::lock_guard lock{backend_lock}; + for(const auto& backend : backends) + if(!backend->connect(error)) + log_error(category::audio, tr("Failed to connect to audio backend {}: {}"), backend->name(), error); + +} + +void SoundIOBackendHandler::shutdown_all() { + std::lock_guard lock{backend_lock}; + for(auto& entry : backends) + entry->shutdown(); + backends.clear(); +} + +SoundIOBackendHandler::SoundIOBackendHandler(SoundIoBackend backed) : backend{backed} {} +SoundIOBackendHandler::~SoundIOBackendHandler() { + this->shutdown(); +} + +bool SoundIOBackendHandler::initialize(std::string &error) { + assert(!this->soundio_handle); + + this->soundio_handle = soundio_create(); + if(!this->soundio_handle) { + error = "out of memory"; + return false; + } + + this->soundio_handle->userdata = this; + this->soundio_handle->on_devices_change = [](auto handle){ + reinterpret_cast(handle->userdata)->handle_device_change(); + }; + this->soundio_handle->on_backend_disconnect = [](auto handle, auto err){ + reinterpret_cast(handle->userdata)->handle_backend_disconnect(err); + }; + + return true; +} + +void SoundIOBackendHandler::shutdown() { + if(!this->soundio_handle) return; + + soundio_destroy(this->soundio_handle); + this->soundio_handle = nullptr; +} + +bool SoundIOBackendHandler::connect(std::string &error, bool enforce) { + if(!this->soundio_handle) { + error = "invalid handle"; + return false; + } + + if(this->_connected && !enforce) { + error = "already connected"; + return false; + } + + auto err = soundio_connect_backend(this->soundio_handle, this->backend); + if(err) { + error = soundio_strerror(err); + return false; + } + + this->soundio_handle->app_name = "TeaClient"; + this->_connected = true; + + { + auto begin = std::chrono::system_clock::now(); + soundio_flush_events(this->soundio_handle); + auto end = std::chrono::system_clock::now(); + log_debug(category::audio, tr("Flushed connect events within {}ms for backend {}"), + std::chrono::ceil(end - begin).count(), + this->name()); + } + return true; +} + +void SoundIOBackendHandler::disconnect() { + if(!this->soundio_handle || !this->_connected) return; + + soundio_disconnect(this->soundio_handle); +} + +void SoundIOBackendHandler::handle_device_change() { + log_debug(category::audio, tr("Device list changed for backend {}. Reindexing devices."), this->name()); + + std::lock_guard lock{this->device_lock}; + this->_default_output_device.reset(); + this->_default_input_device.reset(); + this->cached_input_devices.clear(); + this->cached_output_devices.clear(); + + if(!this->_connected || !this->soundio_handle) return; + + size_t input_devices{0}, output_devices{0}; + auto default_input_device{soundio_default_input_device_index(this->soundio_handle)}; + for(int i = 0; i < soundio_input_device_count(this->soundio_handle); i++) { + auto dev = soundio_get_input_device(this->soundio_handle, i); + if(!dev) { + log_warn(category::audio, tr("Failed to get input device at index {} for backend {}."), i, this->name()); + continue; + } + if(dev->probe_error) { + log_trace(category::audio, tr("Skipping input device {} ({}) for backend {} because of probe error: {}"), dev->id, dev->name, this->name(), soundio_strerror(dev->probe_error)); + soundio_device_unref(dev); + continue; + } + + auto device = std::make_shared(dev, this->name(), i == default_input_device, true); + log_trace(category::audio, tr("Found input device {} ({})."), dev->id, dev->name); + this->cached_input_devices.push_back(device); + if(i == default_input_device) + this->_default_input_device = device; + input_devices++; + } + + auto default_output_device{soundio_default_output_device_index(this->soundio_handle)}; + for(int i = 0; i < soundio_output_device_count(this->soundio_handle); i++) { + auto dev = soundio_get_output_device(this->soundio_handle, i); + if(!dev) { + log_warn(category::audio, tr("Failed to get output device at index {} for backend {}."), i, this->name()); + continue; + } + if(dev->probe_error) { + log_trace(category::audio, tr("Skipping output device {} ({}) for backend {} because of probe error: {}"), dev->id, dev->name, this->name(), soundio_strerror(dev->probe_error)); + soundio_device_unref(dev); + continue; + } + + auto device = std::make_shared(dev, this->name(), i == default_output_device, true); + log_trace(category::audio, tr("Found output device {} ({})."), dev->id, dev->name); + this->cached_output_devices.push_back(device); + if(i == default_output_device) + this->_default_output_device = device; + output_devices++; + } + + log_info(category::audio, tr("Queried devices for backend {}, resulting in {} input and {} output devices."), + this->name(), + input_devices, + output_devices + ); +} + +void SoundIOBackendHandler::handle_backend_disconnect(int error) { + log_info(category::audio, tr("Backend {} disconnected with error {}."), this->name(), soundio_strerror(error)); +} + +SoundIODevice::SoundIODevice(struct ::SoundIoDevice *dev, std::string driver, bool default_, bool owned) : device_handle{dev}, driver_name{std::move(driver)}, _default{default_} { + if(!owned) soundio_device_ref(dev); +} + +SoundIODevice::~SoundIODevice() { + soundio_device_unref(this->device_handle); +} + +std::string SoundIODevice::id() const { + return this->device_handle->id; +} + +std::string SoundIODevice::name() const { + return this->device_handle->name; +} + +std::string SoundIODevice::driver() const { + return this->driver_name; /* we do not use this->device_handle->soundio->current_backend because the soundio could be null */ +} + +bool SoundIODevice::is_input_supported() const { + return this->device_handle->aim == SoundIoDeviceAimInput; +} + +bool SoundIODevice::is_output_supported() const { + return this->device_handle->aim == SoundIoDeviceAimOutput; +} + +bool SoundIODevice::is_input_default() const { + return this->_default && this->is_input_supported(); +} + +bool SoundIODevice::is_output_default() const { + return this->_default && this->is_output_supported(); +} + +std::shared_ptr SoundIODevice::playback() { + if(!this->is_output_supported()) { + log_warn(category::audio, tr("Tried to create playback manager for device which does not supports it.")); + return nullptr; + } + + std::lock_guard lock{this->io_lock}; + if(!this->_playback) + this->_playback = std::make_shared(this->device_handle); + return this->_playback; +} + +std::shared_ptr SoundIODevice::record() { + if(!this->is_input_supported()) { + log_warn(category::audio, tr("Tried to create record manager for device which does not supports it.")); + return nullptr; + } + + std::lock_guard lock{this->io_lock}; + if(!this->_record) + this->_record = std::make_shared(this->device_handle); + return this->_record; +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/driver/SoundIO.h b/native/serverconnection/src/audio/driver/SoundIO.h new file mode 100644 index 0000000..5e6f20d --- /dev/null +++ b/native/serverconnection/src/audio/driver/SoundIO.h @@ -0,0 +1,158 @@ +#pragma once + +#include +#include +#include +#include +#include "./AudioDriver.h" + +namespace tc::audio { + struct BackendPriority { + static constexpr std::array mapping{ + /* SoundIoBackendNone */ -100, + /* SoundIoBackendJack */ 100, + /* SoundIoBackendPulseAudio */ 90, + /* SoundIoBackendAlsa */ 50, + /* SoundIoBackendCoreAudio */ 100, + /* SoundIoBackendWasapi */ 100, + /* SoundIoBackendDummy */ 0 + }; + + [[nodiscard]] static constexpr auto priority(SoundIoBackend backend) { + if(backend >= mapping.size()) + return 0; + return mapping[backend]; + } + }; + + class SoundIOPlayback : public AudioDevicePlayback { + public: + constexpr static auto kChunkSize{960}; + + explicit SoundIOPlayback(struct ::SoundIoDevice* /* handle */); + virtual ~SoundIOPlayback(); + + protected: + bool impl_start(std::string& /* error */) override; + void impl_stop() override; + + private: + bool stream_invalid{false}; + struct ::SoundIoDevice* device_handle{nullptr}; + struct ::SoundIoOutStream* stream{nullptr}; + + struct ::SoundIoRingBuffer* buffer{nullptr}; + + void write_callback(int frame_count_min, int frame_count_max); + }; + + class SoundIORecord : public AudioDeviceRecord { + public: + constexpr static auto kChunkSize{960}; + + explicit SoundIORecord(struct ::SoundIoDevice* /* handle */); + virtual ~SoundIORecord(); + + protected: + bool impl_start(std::string& /* error */) override; + void impl_stop() override; + + private: + bool stream_invalid{false}; + struct ::SoundIoDevice* device_handle{nullptr}; + struct ::SoundIoInStream* stream{nullptr}; + + struct ::SoundIoRingBuffer* buffer{nullptr}; + + void read_callback(int frame_count_min, int frame_count_max); + }; + + class SoundIODevice : public AudioDevice { + public: + explicit SoundIODevice(struct ::SoundIoDevice* /* handle */, std::string /* driver */, bool /* default */, bool /* owned */); + virtual ~SoundIODevice(); + + [[nodiscard]] std::string id() const override; + [[nodiscard]] std::string name() const override; + [[nodiscard]] std::string driver() const override; + + [[nodiscard]] bool is_input_supported() const override; + [[nodiscard]] bool is_output_supported() const override; + + [[nodiscard]] bool is_input_default() const override; + [[nodiscard]] bool is_output_default() const override; + + [[nodiscard]] std::shared_ptr playback() override; + [[nodiscard]] std::shared_ptr record() override; + private: + std::string driver_name{}; + struct ::SoundIoDevice* device_handle{nullptr}; + bool _default{false}; + + std::mutex io_lock{}; + std::shared_ptr _playback; + std::shared_ptr _record; + }; + + class SoundIOBackendHandler { + public: + /* its sorted by priority */ + static std::vector> all_backends() { + std::lock_guard lock{backend_lock}; + return backends;; + } + static std::shared_ptr get_backend(SoundIoBackend backend); + + static void initialize_all(); + static void connect_all(); + static void shutdown_all(); + + explicit SoundIOBackendHandler(SoundIoBackend backed); + virtual ~SoundIOBackendHandler(); + + bool initialize(std::string& error); + void shutdown(); + + bool connect(std::string& /* error */, bool /* enforce */ = false); + [[nodiscard]] inline bool connected() const { return this->_connected; } + void disconnect(); + + [[nodiscard]] inline int priority() const { return BackendPriority::priority(this->backend); } + [[nodiscard]] inline const char* name() const { return soundio_backend_name(this->backend); } + + [[nodiscard]] inline std::vector> input_devices() const { + std::lock_guard lock{this->device_lock}; + return this->cached_input_devices; + } + + [[nodiscard]] inline std::vector> output_devices() const { + std::lock_guard lock{this->device_lock}; + return this->cached_output_devices; + } + + [[nodiscard]] inline std::shared_ptr default_input_device() const { + return this->_default_input_device; + } + + [[nodiscard]] inline std::shared_ptr default_output_device() const { + return this->_default_output_device; + } + + const SoundIoBackend backend; + private: + static std::mutex backend_lock; + static std::vector> backends; + + void handle_backend_disconnect(int /* error */); + void handle_device_change(); + + bool _connected{false}; + struct SoundIo* soundio_handle{nullptr}; + + mutable std::mutex device_lock{}; + std::vector> cached_input_devices{}; + std::vector> cached_output_devices{}; + std::shared_ptr _default_output_device{nullptr}; + std::shared_ptr _default_input_device{nullptr}; + }; +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/driver/SoundIOPlayback.cpp b/native/serverconnection/src/audio/driver/SoundIOPlayback.cpp new file mode 100644 index 0000000..89f41ae --- /dev/null +++ b/native/serverconnection/src/audio/driver/SoundIOPlayback.cpp @@ -0,0 +1,145 @@ +// +// Created by wolverindev on 07.02.20. +// + +#include "SoundIO.h" +#include +#include "../../logger.h" + +using namespace tc::audio; + +SoundIOPlayback::SoundIOPlayback(struct ::SoundIoDevice *device) : device_handle{device} { + soundio_device_ref(device); +} + +SoundIOPlayback::~SoundIOPlayback() { + soundio_device_unref(this->device_handle); +} + +bool SoundIOPlayback::impl_start(std::string &error) { + assert(this->device_handle); + + this->buffer = soundio_ring_buffer_create(nullptr, kChunkSize * sizeof(float) * 2); /* 2 channels */ + if(!buffer) { + error = "failed to allocate the buffer"; + return false; + } + soundio_ring_buffer_clear(this->buffer); + + this->stream = soundio_outstream_create(this->device_handle); + if(!this->stream) { + error = "out of memory"; + return false; + } + + this->stream->userdata = this; + this->stream->format = SoundIoFormatFloat32LE; + this->stream->software_latency = 0.02; + + this->stream->underflow_callback = [](auto str) { + auto handle = reinterpret_cast(str->userdata); + log_info(category::audio, tr("Having an underflow on {}"), handle->device_handle->id); + }; + + this->stream->error_callback = [](auto str, int err) { + auto handle = reinterpret_cast(str->userdata); + log_info(category::audio, tr("Having an error on {}: {}. Aborting playback."), handle->device_handle->id, soundio_strerror(err)); + handle->stream_invalid = true; + }; + + this->stream->write_callback = [](struct SoundIoOutStream *str, int frame_count_min, int frame_count_max) { + auto handle = reinterpret_cast(str->userdata); + handle->write_callback(frame_count_min, frame_count_max); + }; + + if(auto err = soundio_outstream_open(this->stream); err) { + error = soundio_strerror(err) + std::string{" (open)"}; + soundio_outstream_destroy(this->stream); + this->stream = nullptr; + return false; + } + + if(false && this->stream->layout_error) { + error = std::string{"failed to set audio layout: "} + soundio_strerror(this->stream->layout_error); + soundio_outstream_destroy(this->stream); + this->stream = nullptr; + return false; + } + + if(auto err = soundio_outstream_start(this->stream); err) { + error = soundio_strerror(err) + std::string{" (start)"}; + soundio_outstream_destroy(this->stream); + this->stream = nullptr; + return false; + } + + //TODO: Test for interleaved channel layout! + + return true; +} + +void SoundIOPlayback::impl_stop() { + if(!this->stream) return; + + soundio_outstream_destroy(this->stream); + this->stream = nullptr; + + soundio_ring_buffer_destroy(this->buffer); + this->buffer = nullptr; +} + +void SoundIOPlayback::write_callback(int frame_count_min, int frame_count_max) { + const struct SoundIoChannelLayout *layout = &this->stream->layout; + + struct SoundIoChannelArea *areas; + int frames_left{frame_count_min}, err; + + if(frames_left < 120) + frames_left = 120; + if(frames_left > frame_count_max) + frames_left = frame_count_max; + + while(frames_left > 0) { + int frame_count{frames_left}; + auto buffered = soundio_ring_buffer_fill_count(this->buffer) / (sizeof(float) * layout->channel_count); + if(frame_count > buffered) { + if(buffered == 0) { + const auto length = sizeof(float) * frame_count * layout->channel_count; + this->fill_buffer(soundio_ring_buffer_write_ptr(this->buffer), frame_count, layout->channel_count); + soundio_ring_buffer_advance_write_ptr(this->buffer, length); + } else + frame_count = buffered; + } + if((err = soundio_outstream_begin_write(this->stream, &areas, &frame_count))) { + log_warn(category::audio, tr("Failed to begin a write to the soundio buffer: {}"), err); + return; + } + + /* test for interleaved */ + { + char* begin = areas[0].ptr - sizeof(float); + for(size_t channel{0}; channel < layout->channel_count; channel++) { + if((begin += sizeof(float)) != areas[channel].ptr) { + log_error(category::audio, tr("Expected interleaved buffer, which it isn't")); + return; + } + + if(areas[channel].step != sizeof(float) * layout->channel_count) { + log_error(category::audio, tr("Invalid step size for channel {}"), channel); + return; + } + } + } + + const auto length = sizeof(float) * frame_count * layout->channel_count; + memcpy(areas[0].ptr, soundio_ring_buffer_read_ptr(this->buffer), length); + soundio_ring_buffer_advance_read_ptr(this->buffer, length); + + if((err = soundio_outstream_end_write(this->stream))) { + log_warn(category::audio, tr("Failed to end a write to the soundio buffer: {}"), err); + return; + } + + frames_left -= frame_count; + } +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/driver/SoundIORecord.cpp b/native/serverconnection/src/audio/driver/SoundIORecord.cpp new file mode 100644 index 0000000..2029fec --- /dev/null +++ b/native/serverconnection/src/audio/driver/SoundIORecord.cpp @@ -0,0 +1,145 @@ +// +// Created by wolverindev on 07.02.20. +// + +#include "SoundIO.h" +#include +#include "../../logger.h" + +using namespace tc::audio; + +SoundIORecord::SoundIORecord(struct ::SoundIoDevice *device) : device_handle{device} { + soundio_device_ref(device); +} + +SoundIORecord::~SoundIORecord() { + soundio_device_unref(this->device_handle); +} + +bool SoundIORecord::impl_start(std::string &error) { + assert(this->device_handle); + + this->buffer = soundio_ring_buffer_create(nullptr, kChunkSize * sizeof(float) * 2); //2 Channels + if(!buffer) { + error = "failed to allocate the buffer"; + return false; + } + soundio_ring_buffer_clear(this->buffer); + + this->stream = soundio_instream_create(this->device_handle); + if(!this->stream) { + error = "out of memory"; + return false; + } + + this->stream->userdata = this; + this->stream->format = SoundIoFormatFloat32LE; + this->stream->software_latency = 0.02; + + this->stream->overflow_callback = [](auto str) { + auto handle = reinterpret_cast(str->userdata); + log_info(category::audio, tr("Having an overflow on {}"), handle->device_handle->id); + }; + + this->stream->error_callback = [](auto str, int err) { + auto handle = reinterpret_cast(str->userdata); + log_info(category::audio, tr("Having an error on {}: {}. Aborting recording."), handle->device_handle->id, soundio_strerror(err)); + handle->stream_invalid = true; + }; + + this->stream->read_callback = [](struct SoundIoInStream *str, int frame_count_min, int frame_count_max) { + auto handle = reinterpret_cast(str->userdata); + handle->read_callback(frame_count_min, frame_count_max); + }; + + if(auto err = soundio_instream_open(this->stream); err) { + error = soundio_strerror(err) + std::string{" (open)"}; + soundio_instream_destroy(this->stream); + this->stream = nullptr; + return false; + } + + if(false && this->stream->layout_error) { + error = std::string{"failed to set audio layout: "} + soundio_strerror(this->stream->layout_error); + soundio_instream_destroy(this->stream); + this->stream = nullptr; + return false; + } + + if(auto err = soundio_instream_start(this->stream); err) { + error = soundio_strerror(err) + std::string{" (start)"}; + soundio_instream_destroy(this->stream); + this->stream = nullptr; + return false; + } + + //TODO: Test for interleaved channel layout! + + return true; +} + +void SoundIORecord::impl_stop() { + if(!this->stream) return; + + soundio_instream_destroy(this->stream); + this->stream = nullptr; + + soundio_ring_buffer_destroy(this->buffer); + this->buffer = nullptr; +} + +void SoundIORecord::read_callback(int frame_count_min, int frame_count_max) { + const struct SoundIoChannelLayout *layout = &this->stream->layout; + + struct SoundIoChannelArea *areas; + + int frames_left{frame_count_max}; + int buffer_samples = soundio_ring_buffer_free_count(this->buffer) / (sizeof(float) * layout->channel_count); + + while(frames_left > 0) { + int frame_count{frames_left}; + if(frame_count > buffer_samples) + frame_count = buffer_samples; + if(auto err = soundio_instream_begin_read(this->stream, &areas, &frame_count); err) { + log_error(category::audio, tr("Failed to begin read from input stream buffer: {}"), soundio_strerror(err)); + return; + } + + /* test for interleaved */ + { + char* begin = areas[0].ptr - sizeof(float); + for(size_t channel{0}; channel < layout->channel_count; channel++) { + if((begin += sizeof(float)) != areas[channel].ptr) { + log_error(category::audio, tr("Expected interleaved buffer, which it isn't")); + return; + } + + if(areas[channel].step != sizeof(float) * layout->channel_count) { + log_error(category::audio, tr("Invalid step size for channel {}"), channel); + return; + } + } + } + + const auto length = sizeof(float) * layout->channel_count * frame_count; + memcpy(soundio_ring_buffer_write_ptr(this->buffer), areas[0].ptr, length); + soundio_ring_buffer_advance_write_ptr(this->buffer, length); + buffer_samples -= frame_count; + frames_left -= frame_count; + + if(buffer_samples == 0) { + std::lock_guard consumer{this->consumer_lock}; + const auto byte_count = soundio_ring_buffer_fill_count(this->buffer); + const auto frame_count = byte_count / (sizeof(float) * layout->channel_count); + for(auto& consumer : this->_consumers) + consumer->consume(soundio_ring_buffer_read_ptr(this->buffer), frame_count, layout->channel_count); + soundio_ring_buffer_advance_read_ptr(this->buffer, byte_count); + buffer_samples = frame_count; + } + + if(auto err = soundio_instream_end_read(this->stream); err) { + log_error(category::audio, tr("Failed to close input stream buffer: {}"), soundio_strerror(err)); + return; + } + } +} \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioConsumer.cpp b/native/serverconnection/src/audio/js/AudioConsumer.cpp index a8e6d7b..b91b758 100644 --- a/native/serverconnection/src/audio/js/AudioConsumer.cpp +++ b/native/serverconnection/src/audio/js/AudioConsumer.cpp @@ -311,7 +311,7 @@ NAN_METHOD(AudioConsumerWrapper::_create_filter_state) { assert(consumer); /* should never be null! */ string error; - auto filter = make_shared(consumer->channel_count,consumer->sample_rate,consumer->frame_size); + auto filter = make_shared(consumer->channel_count, consumer->sample_rate, consumer->frame_size); if(!filter->initialize(error)) { Nan::ThrowError(Nan::New("failed to initialize filter (" + error + ")").ToLocalChecked()); return; diff --git a/native/serverconnection/src/audio/js/AudioOutputStream.cpp b/native/serverconnection/src/audio/js/AudioOutputStream.cpp index 04d53b2..97b254f 100644 --- a/native/serverconnection/src/audio/js/AudioOutputStream.cpp +++ b/native/serverconnection/src/audio/js/AudioOutputStream.cpp @@ -121,25 +121,10 @@ NAN_METHOD(AudioOutputStreamWrapper::_delete) { } ssize_t AudioOutputStreamWrapper::write_data(const std::shared_ptr& handle, void *source, size_t samples, bool interleaved) { - ssize_t result = 0; - if(interleaved) { - result = handle->enqueue_samples(source, samples); - } else { - auto buffer = SampleBuffer::allocate(handle->channel_count, samples); - auto src_buffer = (float*) source; - auto target_buffer = (float*) buffer->sample_data; - - while (samples-- > 0) { - *target_buffer = *src_buffer; - *(target_buffer + 1) = *(src_buffer + buffer->sample_size); - - target_buffer += 2; - src_buffer++; - } - - result = handle->enqueue_samples(buffer); - } - return result; + if(interleaved) + return handle->enqueue_samples(source, samples); + else + return handle->enqueue_samples_no_interleave(source, samples); } NAN_METHOD(AudioOutputStreamWrapper::_write_data) { @@ -198,6 +183,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_write_data_rated) { return; } + //TODO: Use a tmp preallocated buffer here! ssize_t target_samples = client->_resampler->estimated_output_size(samples); auto buffer = SampleBuffer::allocate(handle->channel_count, max((size_t) samples, (size_t) target_samples)); auto source_buffer = js_buffer.Data(); @@ -224,7 +210,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_write_data_rated) { buffer->sample_index = 0; buffer->sample_size = target_samples; - info.GetReturnValue().Set((int32_t) handle->enqueue_samples(buffer)); + info.GetReturnValue().Set((int32_t) handle->enqueue_samples(buffer->sample_data, target_samples)); } } @@ -237,7 +223,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_get_buffer_latency) { return; } - info.GetReturnValue().Set((float) handle->min_buffer / (float) handle->sample_rate); + info.GetReturnValue().Set((float) handle->min_buffered_samples / (float) handle->sample_rate); } NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_latency) { @@ -254,7 +240,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_latency) { return; } - handle->min_buffer = (size_t) ceil(handle->sample_rate * info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0)); + handle->min_buffered_samples = (size_t) ceil(handle->sample_rate * info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0)); } NAN_METHOD(AudioOutputStreamWrapper::_get_buffer_max_latency) { @@ -266,7 +252,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_get_buffer_max_latency) { return; } - info.GetReturnValue().Set((float) handle->max_latency / (float) handle->sample_rate); + info.GetReturnValue().Set((float) handle->max_latency() / (float) handle->sample_rate); } NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_max_latency) { @@ -283,7 +269,7 @@ NAN_METHOD(AudioOutputStreamWrapper::_set_buffer_max_latency) { return; } - handle->max_latency = (size_t) ceil(handle->sample_rate * info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0)); + handle->max_buffered_samples = (size_t) ceil(handle->sample_rate * info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0)); } NAN_METHOD(AudioOutputStreamWrapper::_flush_buffer) { diff --git a/native/serverconnection/src/audio/js/AudioPlayer.cpp b/native/serverconnection/src/audio/js/AudioPlayer.cpp index f4b224b..6100abf 100644 --- a/native/serverconnection/src/audio/js/AudioPlayer.cpp +++ b/native/serverconnection/src/audio/js/AudioPlayer.cpp @@ -1,10 +1,9 @@ -#include #include #include #include "AudioPlayer.h" #include "../AudioOutput.h" -#include "../AudioDevice.h" #include "AudioOutputStream.h" +#include "../../logger.h" using namespace tc; using namespace tc::audio; @@ -22,6 +21,11 @@ NAN_MODULE_INIT(player::init_js) { } NAN_METHOD(audio::available_devices) { + if(!audio::initialized()) { + Nan::ThrowError(tr("audio hasn't been initialized!")); + return; + } + auto devices = audio::devices(); auto result = Nan::New(devices.size()); @@ -29,17 +33,15 @@ NAN_METHOD(audio::available_devices) { auto device_info = Nan::New(); auto device = devices[index]; - Nan::Set(device_info, Nan::LocalString("name"), Nan::LocalStringUTF8(device->name)); - Nan::Set(device_info, Nan::New("driver").ToLocalChecked(), Nan::New(device->driver).ToLocalChecked()); - Nan::Set(device_info, Nan::New("device_id").ToLocalChecked(), Nan::New(base64::encode(digest::sha1(device->name + device->driver))).ToLocalChecked()); + Nan::Set(device_info, Nan::LocalString("name"), Nan::LocalStringUTF8(device->name())); + Nan::Set(device_info, Nan::New("driver").ToLocalChecked(), Nan::New(device->driver()).ToLocalChecked()); + Nan::Set(device_info, Nan::New("device_id").ToLocalChecked(), Nan::New(device->id()).ToLocalChecked()); - Nan::Set(device_info, Nan::New("input_supported").ToLocalChecked(), Nan::New(device->input_supported)); - Nan::Set(device_info, Nan::New("output_supported").ToLocalChecked(), Nan::New(device->output_supported)); + Nan::Set(device_info, Nan::New("input_supported").ToLocalChecked(), Nan::New(device->is_input_supported())); + Nan::Set(device_info, Nan::New("output_supported").ToLocalChecked(), Nan::New(device->is_output_supported())); - Nan::Set(device_info, Nan::New("input_default").ToLocalChecked(), Nan::New(device->is_default_input)); - Nan::Set(device_info, Nan::New("output_default").ToLocalChecked(), Nan::New(device->is_default_output)); - - Nan::Set(device_info, Nan::New("device_index").ToLocalChecked(), Nan::New(device->device_id)); + Nan::Set(device_info, Nan::New("input_default").ToLocalChecked(), Nan::New(device->is_input_default())); + Nan::Set(device_info, Nan::New("output_default").ToLocalChecked(), Nan::New(device->is_output_default())); Nan::Set(result, index, device_info); } @@ -47,13 +49,48 @@ NAN_METHOD(audio::available_devices) { info.GetReturnValue().Set(result); } +NAN_METHOD(audio::await_initialized_js) { + if(info.Length() != 1 || !info[0]->IsFunction()) { + Nan::ThrowError(tr("Invalid arguments")); + return; + } + + if(audio::initialized()) { + (void) info[0].As()->Call(Nan::GetCurrentContext(), Nan::Undefined(), 0, nullptr); + } else { + auto _callback = std::make_unique>(info[0].As()); + + auto _async_callback = Nan::async_callback([call = std::move(_callback)] { + Nan::HandleScope scope; + auto callback_function = call->Get(Nan::GetCurrentContext()->GetIsolate()); + + (void) callback_function->Call(Nan::GetCurrentContext(), Nan::Undefined(), 0, nullptr); + call->Reset(); + }).option_destroyed_execute(true); + + audio::initialize([_async_callback] { + _async_callback.call(); + }); + } +} + + +NAN_METHOD(audio::initialized_js) { + info.GetReturnValue().Set(audio::initialized()); +} + NAN_METHOD(player::current_playback_device) { if(!global_audio_output) { - info.GetReturnValue().Set(paNoDevice); + info.GetReturnValue().Set(Nan::Undefined()); return; } - info.GetReturnValue().Set(global_audio_output->current_device()); + auto device = global_audio_output->current_device(); + if(!device) { + info.GetReturnValue().Set(Nan::Undefined()); + return; + } + info.GetReturnValue().Set(Nan::LocalString(device->id())); } NAN_METHOD(player::set_playback_device) { @@ -62,24 +99,33 @@ NAN_METHOD(player::set_playback_device) { return; } - if(info.Length() != 1 || !info[0]->IsNumber()) { + const auto null_device = info[0]->IsNullOrUndefined(); + if(info.Length() != 1 || !(info[0]->IsString() || null_device)) { Nan::ThrowError("invalid arguments"); return; } - std::string error; - if(!global_audio_output->open_device(error, info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0))) { - Nan::ThrowError(Nan::New("failed to open device (" + error + ")").ToLocalChecked()); - return; - } + auto device = null_device ? nullptr : audio::find_device_by_id(*Nan::Utf8String(info[0]), false); + if(!device && !null_device) { + Nan::ThrowError("invalid device id"); + return; + } - if(!global_audio_output->playback()) { - Nan::ThrowError("failed to open playback stream"); + std::string error; + + global_audio_output->set_device(device); + if(!global_audio_output->playback(error)) { + Nan::ThrowError(error.c_str()); return; } } NAN_METHOD(player::create_stream) { + if(!audio::initialized()) { + Nan::ThrowError(tr("audio hasn't been initialized yet")); + return; + } + if(!global_audio_output) { Nan::ThrowError("Global audio output hasn't been yet initialized!"); return; diff --git a/native/serverconnection/src/audio/js/AudioPlayer.h b/native/serverconnection/src/audio/js/AudioPlayer.h index 4194d74..34f55ee 100644 --- a/native/serverconnection/src/audio/js/AudioPlayer.h +++ b/native/serverconnection/src/audio/js/AudioPlayer.h @@ -2,32 +2,32 @@ #include -namespace tc { - namespace audio { - extern NAN_METHOD(available_devices); +namespace tc::audio { + extern NAN_METHOD(available_devices); + extern NAN_METHOD(await_initialized_js); + extern NAN_METHOD(initialized_js); - namespace player { - extern NAN_MODULE_INIT(init_js); + namespace player { + extern NAN_MODULE_INIT(init_js); - extern NAN_METHOD(current_playback_device); - extern NAN_METHOD(set_playback_device); + extern NAN_METHOD(current_playback_device); + extern NAN_METHOD(set_playback_device); - extern NAN_METHOD(create_stream); + extern NAN_METHOD(create_stream); - extern NAN_METHOD(get_master_volume); - extern NAN_METHOD(set_master_volume); - /* - export function get_master_volume() : number; - export function set_master_volume(volume: number); + extern NAN_METHOD(get_master_volume); + extern NAN_METHOD(set_master_volume); + /* + export function get_master_volume() : number; + export function set_master_volume(volume: number); - export function set_device(device: AudioDevice) : Promise; - export function current_device() : AudioDevice; + export function set_device(device: AudioDevice) : Promise; + export function current_device() : AudioDevice; - export function available_devices() : AudioDevice[]; + export function available_devices() : AudioDevice[]; - export function create_stream() : OwnedAudioOutputStream; - export function delete_stream(stream: OwnedAudioOutputStream) : number; - */ - } - } + export function create_stream() : OwnedAudioOutputStream; + export function delete_stream(stream: OwnedAudioOutputStream) : number; + */ + } } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioRecorder.cpp b/native/serverconnection/src/audio/js/AudioRecorder.cpp index 1153070..32a8c4b 100644 --- a/native/serverconnection/src/audio/js/AudioRecorder.cpp +++ b/native/serverconnection/src/audio/js/AudioRecorder.cpp @@ -1,4 +1,5 @@ #include +#include #include "AudioRecorder.h" #include "AudioConsumer.h" @@ -14,6 +15,10 @@ NAN_MODULE_INIT(recorder::init_js) { } NAN_METHOD(recorder::create_recorder) { + if(!audio::initialized()) { + Nan::ThrowError(tr("audio hasn't been initialized yet")); + return; + } auto input = make_shared(2, 48000); auto wrapper = new AudioRecorderWrapper(input); auto js_object = Nan::NewInstance(Nan::New(AudioRecorderWrapper::constructor())).ToLocalChecked(); @@ -122,42 +127,50 @@ NAN_METHOD(AudioRecorderWrapper::_get_device) { auto handle = ObjectWrap::Unwrap(info.Holder()); auto input = handle->_input; - info.GetReturnValue().Set(input->current_device()); + auto device = input->current_device(); + if(device) + info.GetReturnValue().Set(Nan::LocalString(device->id())); + else + info.GetReturnValue().Set(Nan::Undefined()); } NAN_METHOD(AudioRecorderWrapper::_set_device) { auto handle = ObjectWrap::Unwrap(info.Holder()); auto input = handle->_input; - if(info.Length() != 2 || !info[0]->IsNumber() || !info[1]->IsFunction()) { + const auto is_null_device = info[0]->IsNullOrUndefined(); + if(info.Length() != 2 || !(is_null_device || info[0]->IsString()) || !info[1]->IsFunction()) { Nan::ThrowError("invalid arguments"); return; } - auto device_id = info[0]->Int32Value(Nan::GetCurrentContext()).FromMaybe(0); + if(!audio::initialized()) { + Nan::ThrowError("audio hasn't been initialized yet"); + return; + } + + auto device = is_null_device ? nullptr : audio::find_device_by_id(*Nan::Utf8String(info[0]), true); + if(!device && !is_null_device) { + Nan::ThrowError("invalid device id"); + return; + } unique_ptr> _callback = make_unique>(info[1].As()); unique_ptr> _recorder = make_unique>(info.Holder()); - auto _async_callback = Nan::async_callback([call = std::move(_callback), recorder = move(_recorder)](bool result, std::string error) mutable { + auto _async_callback = Nan::async_callback([call = std::move(_callback), recorder = move(_recorder)] { Nan::HandleScope scope; auto callback_function = call->Get(Nan::GetCurrentContext()->GetIsolate()); - v8::Local argv[1]; - if(result) - argv[0] = v8::Boolean::New(Nan::GetCurrentContext()->GetIsolate(), result); - else - argv[0] = Nan::NewOneByteString((uint8_t*) error.data(), error.length()).ToLocalChecked(); - callback_function->Call(Nan::GetCurrentContext(), Nan::Undefined(), 1, argv); + (void) callback_function->Call(Nan::GetCurrentContext(), Nan::Undefined(), 0, nullptr); recorder->Reset(); call->Reset(); }).option_destroyed_execute(true); - std::thread([_async_callback, input, device_id]{ - string error; - auto flag = input->open_device(error, device_id); - _async_callback(std::forward(flag), std::forward(error)); + std::thread([_async_callback, input, device]{ + input->set_device(device); + _async_callback(); }).detach(); } @@ -173,9 +186,13 @@ NAN_METHOD(AudioRecorderWrapper::_start) { } auto input = ObjectWrap::Unwrap(info.Holder())->_input; + std::string error{}; v8::Local argv[1]; - argv[0] = v8::Boolean::New(Nan::GetCurrentContext()->GetIsolate(), input->record()); + if(input->record(error)) + argv[0] = Nan::New(true); + else + argv[0] = Nan::LocalString(error); (void) info[0].As()->Call(Nan::GetCurrentContext(), Nan::Undefined(), 1, argv); } diff --git a/native/serverconnection/src/bindings.cpp b/native/serverconnection/src/bindings.cpp index 13c3f48..0a38c8b 100644 --- a/native/serverconnection/src/bindings.cpp +++ b/native/serverconnection/src/bindings.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "logger.h" #include "include/NanException.h" @@ -15,7 +16,7 @@ #include "connection/ft/FileTransferManager.h" #include "connection/ft/FileTransferObject.h" #include "audio/AudioOutput.h" -#include "audio/AudioDevice.h" +#include "audio/driver/AudioDriver.h" #include "audio/js/AudioOutputStream.h" #include "audio/js/AudioPlayer.h" #include "audio/js/AudioRecorder.h" @@ -104,11 +105,7 @@ NAN_MODULE_INIT(init) { } string error; - //TODO here - //PaJack_SetClientName("TeaClient"); - Pa_Initialize(); - - std::thread(audio::devices).detach(); /* cache the devices */ + tc::audio::initialize(); //TODO: Notify JS when initialized? logger::info(category::general, "Loading crypt modules"); std::string descriptors = "LTGE"; @@ -141,20 +138,32 @@ NAN_MODULE_INIT(init) { #endif tc::audio::init_event_loops(); - /* TODO: Test error codes and make the audi playback device configurable */ - global_audio_output = new tc::audio::AudioOutput(2, 48000); //48000 44100 - if(!global_audio_output->open_device(error, Pa_GetDefaultOutputDevice())) { - logger::error(category::audio, "Failed to initialize default audio playback: {}", error); - } else { - if(!global_audio_output->playback()) { - logger::error(category::audio, "Failed to start audio playback"); - } - } + tc::audio::initialize([]{ + std::string error{}; + + std::shared_ptr default_output{}; + for(auto& device : tc::audio::devices()) { + if(device->is_output_default()) { + default_output = device; + break; + } + } + + /* TODO: Test error codes and make the audi playback device configurable */ + global_audio_output = new tc::audio::AudioOutput(2, 48000); + global_audio_output->set_device(default_output); + if(!global_audio_output->playback(error)) { + logger::error(category::audio, "Failed to start audio playback: {}", error); + } + }); { auto namespace_audio = Nan::New(); - Nan::Set(namespace_audio, Nan::New("available_devices").ToLocalChecked(), Nan::GetFunction(Nan::New(audio::available_devices)).ToLocalChecked()); - { + Nan::Set(namespace_audio, Nan::LocalString("available_devices"), Nan::GetFunction(Nan::New(audio::available_devices)).ToLocalChecked()); + Nan::Set(namespace_audio, Nan::LocalString("initialize"), Nan::GetFunction(Nan::New(audio::await_initialized_js)).ToLocalChecked()); + Nan::Set(namespace_audio, Nan::LocalString("initialized"), Nan::GetFunction(Nan::New(audio::initialized_js)).ToLocalChecked()); + + { auto namespace_playback = Nan::New(); audio::player::init_js(namespace_playback); audio::AudioOutputStreamWrapper::Init(namespace_playback); diff --git a/native/serverconnection/src/connection/audio/VoiceClient.cpp b/native/serverconnection/src/connection/audio/VoiceClient.cpp index cf793c8..c573fd7 100644 --- a/native/serverconnection/src/connection/audio/VoiceClient.cpp +++ b/native/serverconnection/src/connection/audio/VoiceClient.cpp @@ -211,8 +211,8 @@ VoiceClientWrap::~VoiceClientWrap() {} VoiceClient::VoiceClient(const std::shared_ptr&, uint16_t client_id) : _client_id(client_id) { this->output_source = global_audio_output->create_source(); this->output_source->overflow_strategy = audio::overflow_strategy::ignore; - this->output_source->max_latency = (size_t) ceil(this->output_source->sample_rate * 0.5); - this->output_source->min_buffer = (size_t) ceil(this->output_source->sample_rate * 0.04); + this->output_source->max_buffered_samples = (size_t) ceil(this->output_source->sample_rate * 0.5); + this->output_source->min_buffered_samples = (size_t) ceil(this->output_source->sample_rate * 0.04); this->output_source->on_underflow = [&]{ if(this->_state == state::stopping) @@ -533,8 +533,9 @@ void VoiceClient::event_execute(const std::chrono::system_clock::time_point &sch if(replay_head->reset_decoder) audio_codec.converter->reset_decoder(); + //TODO: Use statically allocated buffer? auto decoded = this->decode_buffer(audio_codec.codec, replay_head->buffer); - this->output_source->enqueue_samples(decoded); + this->output_source->enqueue_samples(decoded->sample_data, decoded->sample_size); this->set_state(state::playing); } diff --git a/native/serverconnection/src/connection/ft/FileTransferManager.cpp b/native/serverconnection/src/connection/ft/FileTransferManager.cpp index 0875aed..fd075c2 100644 --- a/native/serverconnection/src/connection/ft/FileTransferManager.cpp +++ b/native/serverconnection/src/connection/ft/FileTransferManager.cpp @@ -1,6 +1,5 @@ #include "FileTransferManager.h" #include "FileTransferObject.h" -#include #include #include @@ -593,6 +592,7 @@ void FileTransferManager::remove_transfer(tc::ft::Transfer *transfer) { } #ifdef NODEJS_API +#include NAN_MODULE_INIT(JSTransfer::Init) { auto klass = Nan::New(JSTransfer::NewInstance); diff --git a/native/serverconnection/src/logger.cpp b/native/serverconnection/src/logger.cpp index 1cf469f..b03125c 100644 --- a/native/serverconnection/src/logger.cpp +++ b/native/serverconnection/src/logger.cpp @@ -1,5 +1,4 @@ #include -#include #include "logger.h" /* Basic */ @@ -9,6 +8,7 @@ void force_log_raw(logger::category::value, spdlog::level::level_enum level, con void force_log_node(logger::category::value, spdlog::level::level_enum, const std::string_view &); #ifdef NODEJS_API +#include #include #include @@ -109,7 +109,6 @@ void force_log_node(logger::category::value category, spdlog::level::level_enum } log_messages_callback(); } - #endif void logger::initialize_raw() { diff --git a/native/serverconnection/src/ring_buffer.cpp b/native/serverconnection/src/ring_buffer.cpp new file mode 100644 index 0000000..1726b9f --- /dev/null +++ b/native/serverconnection/src/ring_buffer.cpp @@ -0,0 +1,49 @@ +// +// Created by wolverindev on 07.02.20. +// + +#include "ring_buffer.h" +#include + + +using namespace tc; + +ring_buffer::ring_buffer(size_t cap) { + this->handle = soundio_ring_buffer_create(nullptr, cap); +} + +ring_buffer::~ring_buffer() { + soundio_ring_buffer_destroy((SoundIoRingBuffer*) this->handle); +} + +size_t ring_buffer::capacity() const { + return soundio_ring_buffer_capacity((SoundIoRingBuffer*) this->handle); +} + +size_t ring_buffer::free_count() const { + return soundio_ring_buffer_free_count((SoundIoRingBuffer*) this->handle); +} + +size_t ring_buffer::fill_count() const { + return soundio_ring_buffer_fill_count((SoundIoRingBuffer*) this->handle); +} + +char* ring_buffer::write_ptr() { + return soundio_ring_buffer_write_ptr((SoundIoRingBuffer*) this->handle); +} + +void ring_buffer::advance_write_ptr(size_t bytes) { + soundio_ring_buffer_advance_write_ptr((SoundIoRingBuffer*) this->handle, bytes); +} + +const void* ring_buffer::read_ptr() const { + return soundio_ring_buffer_read_ptr((SoundIoRingBuffer*) this->handle); +} + +void ring_buffer::advance_read_ptr(size_t bytes) { + soundio_ring_buffer_advance_read_ptr((SoundIoRingBuffer*) this->handle, bytes); +} + +void ring_buffer::clear() { + soundio_ring_buffer_clear((SoundIoRingBuffer*) this->handle); +} \ No newline at end of file diff --git a/native/serverconnection/src/ring_buffer.h b/native/serverconnection/src/ring_buffer.h new file mode 100644 index 0000000..6a357e0 --- /dev/null +++ b/native/serverconnection/src/ring_buffer.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace tc { + class ring_buffer { + public: + /* Attention: Actual size may be larger than given capacity */ + explicit ring_buffer(size_t /* minimum capacity */); + ~ring_buffer(); + + [[nodiscard]] size_t capacity() const; + [[nodiscard]] size_t fill_count() const; + [[nodiscard]] size_t free_count() const; + + /* do not write more than the capacity! */ + char* write_ptr(); + void advance_write_ptr(size_t /* count */); + + /* do not read more than the capacity! */ + [[nodiscard]] const void* read_ptr() const; + void advance_read_ptr(size_t /* count */); + + void clear(); + private: + void* handle{nullptr}; + }; +} \ No newline at end of file diff --git a/native/serverconnection/test/audio/main.cpp b/native/serverconnection/test/audio/main.cpp index ce95a62..b8afae0 100644 --- a/native/serverconnection/test/audio/main.cpp +++ b/native/serverconnection/test/audio/main.cpp @@ -1,17 +1,14 @@ -#include -#include #include #include #include #include #include -#include #include "../../src/audio/AudioOutput.h" #include "../../src/audio/AudioInput.h" #include "../../src/audio/filter/FilterVad.h" #include "../../src/audio/filter/FilterThreshold.h" -#include "../../src/audio/Audio.h" +#include "../../src/logger.h" #ifdef WIN32 #include @@ -21,61 +18,85 @@ using namespace std; using namespace tc; int main() { - string error; - if(!tc::audio::initialize(error)) { - cerr << "Failed to initialize audio: " << error << endl; - return 1; - } - auto playback_manager = audio::AudioOutput(2, 48000); - if(!playback_manager.open_device(error, Pa_GetDefaultOutputDevice())) { - cerr << "Failed to open output device (" << error << ")" << endl; - return 1; - } - if(!playback_manager.playback()) { - cerr << "failed to start playback" << endl; - return 1; - } + std::string error{}; - auto input = audio::AudioInput(2, 48000); - if(!input.open_device(error, Pa_GetDefaultInputDevice())) { - cerr << "Failed to open input device (" << error << ")" << endl; - return 1; - } - if(!input.record()) { - cerr << "failed to start record" << endl; - return 1; - } + Pa_Initialize(); - { - auto consumer = input.create_consumer(960); - auto target_stream = playback_manager.create_source(); - auto vad_handler = make_shared(2, 48000, 960); - if(!vad_handler->initialize(error, 3, 4)) { - cerr << "failed to initialize vad handler (" << error << ")"; - return 1; - } + logger::initialize_raw(); + tc::audio::initialize(); + tc::audio::await_initialized(); - auto threshold_filter = make_shared(2, 48000, 960); - if(!threshold_filter->initialize(error, .5, 5)) { - cerr << "failed to initialize threashold handler (" << error << ")"; - return 1; - } + std::shared_ptr default_playback{nullptr}, default_record{nullptr}; + for(auto& device : tc::audio::devices()) { + if(device->is_output_default()) + default_playback = device; + if(device->is_input_default()) + default_record = device; + } + assert(default_record); + assert(default_playback); - consumer->on_read = [target_stream, vad_handler, threshold_filter](const void* buffer, size_t samples) { - target_stream->enqueue_samples(buffer, samples); + for(auto& dev : tc::audio::devices()) { + if(!dev->is_input_supported()) continue; - cout << "T: " << threshold_filter->analyze(buffer, 0) << endl; - if(vad_handler->process(buffer)) { - cout << "Read " << samples << endl; - target_stream->enqueue_samples(buffer, samples); - } else { - cout << "Drop " << samples << endl; - } - }; - cout << "Read started" << endl; - } + auto playback_manager = std::make_unique(2, 48000); + if(!playback_manager->set_device(error, default_playback)) { + cerr << "Failed to open output device (" << error << ")" << endl; + return 1; + } + if(!playback_manager->playback()) { + cerr << "failed to start playback" << endl; + return 1; + } - this_thread::sleep_for(chrono::seconds(60)); + auto input = std::make_unique(2, 48000); + if(!input->set_device(error, dev)) { + cerr << "Failed to open input device (" << error << "): " << dev->id() << " (" << dev->name() << ")" << endl; + continue; + } + if(!input->record()) { + cerr << "failed to start record for " << dev->id() << " (" << dev->name() << ")" << endl; + continue; + } + + { + auto target_stream = playback_manager->create_source(); + + auto consumer = input->create_consumer(960); + auto vad_handler = make_shared(2, 48000, 960); + if(!vad_handler->initialize(error, 3, 4)) { + cerr << "failed to initialize vad handler (" << error << ")"; + return 1; + } + + auto threshold_filter = make_shared(2, 48000, 960); + if(!threshold_filter->initialize(error, .5, 5)) { + cerr << "failed to initialize threashold handler (" << error << ")"; + return 1; + } + + consumer->on_read = [target_stream, vad_handler, threshold_filter](const void* buffer, size_t samples) { //Do not capture consumer! + target_stream->enqueue_samples(buffer, samples); + + /* + const auto analize_value = threshold_filter->analyze(buffer, 0); + if(vad_handler->process(buffer)) { + cout << "Read " << samples << " (" << analize_value << ")" << endl; + target_stream->enqueue_samples(buffer, samples); + } else { + cout << "Drop " << samples << " (" << analize_value << ")" << endl; + } + */ + }; + + input.release(); + cout << "Read started" << endl; + } + + playback_manager.release(); //FIXME: Memory leak! + } + + this_thread::sleep_for(chrono::seconds(360)); /* while(true) { diff --git a/native/serverconnection/test/audio/sio.cpp b/native/serverconnection/test/audio/sio.cpp index 09d38fb..1dcf3be 100644 --- a/native/serverconnection/test/audio/sio.cpp +++ b/native/serverconnection/test/audio/sio.cpp @@ -1,13 +1,16 @@ #include +#include #include -#include -#include #include -#include #include +#include +#include "../../src/logger.h" +#include "../../src/audio/driver/SoundIO.h" using namespace std; +using namespace tc::audio; + static const float PI = 3.1415926535f; static float seconds_offset = 0.0f; @@ -63,8 +66,27 @@ static void write_callback(struct SoundIoOutStream *outstream, } int main(int argc, char **argv) { - int err; + logger::initialize_raw(); + SoundIOBackendHandler::initialize_all(); + SoundIOBackendHandler::connect_all(); + const auto print_device = [](const std::shared_ptr& device) { + std::cout << " - " << device->id() << " (" << device->name() << ")"; + if(device->is_output_default() || device->is_input_default()) + std::cout << " [Default]"; + std::cout << "\n"; + }; + for(auto& backend : tc::audio::SoundIOBackendHandler::all_backends()) { + std::cout << "Backend " << backend->name() << ":\n"; + std::cout << " Input devices: (" << backend->input_devices().size() << "): \n"; + for(auto& device : backend->input_devices()) print_device(device); + std::cout << " Output devices: (" << backend->input_devices().size() << "): \n"; + for(auto& device : backend->input_devices()) print_device(device); + } + + SoundIOBackendHandler::shutdown_all(); + return 0; + int err; struct SoundIo *soundio = soundio_create(); if (!soundio) { fprintf(stderr, "out of memory\n"); @@ -85,6 +107,7 @@ int main(int argc, char **argv) { for(int i = 0; i < soundio_input_device_count(soundio); i++) { auto dev = soundio_get_input_device(soundio, i); cout << dev->name << " - " << dev->id << endl; + soundio_device_unref(dev); } diff --git a/native/serverconnection/test/js/audio.ts b/native/serverconnection/test/js/audio.ts index 016d171..1e0227e 100644 --- a/native/serverconnection/test/js/audio.ts +++ b/native/serverconnection/test/js/audio.ts @@ -1,114 +1,48 @@ /// -console.log("HELLO WORLD"); +console.log("Starting app"); module.paths.push("../../build/linux_x64"); module.paths.push("../../build/win32_64"); -//LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 -const os = require('os'); -//process.dlopen(module, '/usr/lib/x86_64-linux-gnu/libasan.so.5', -// os.constants.dlopen.RTLD_NOW); -import * as fs from "fs"; - const original_require = require; require = (module => original_require(__dirname + "/../../../build/linux_x64/" + module + ".node")) as any; import * as handle from "teaclient_connection"; require = original_require; -const connection_list = []; -const connection = handle.spawn_server_connection(); -const client_list = []; +console.dir(handle.audio); +handle.audio.initialize(() => { + console.log("Audio initialized"); -console.dir(handle); -console.log("Query devices..."); -console.log("Devices: %o", handle.audio.available_devices()); -console.log("Current playback device: %o", handle.audio.playback.current_device()); -//handle.audio.playback.set_device(14); -//console.log("Current playback device: %o", handle.audio.playback.current_device()); + console.log("Query devices..."); + console.log("Devices: %o", handle.audio.available_devices()); + console.log("Current playback device: %o", handle.audio.playback.current_device()); -const stream = handle.audio.playback.create_stream(); -console.log("Own stream: %o", stream); + const stream = handle.audio.playback.create_stream(); + console.log("Own stream: %o", stream); - -for(let i = 0; i < 12; i++) { const recorder = handle.audio.record.create_recorder(); - for(const device of handle.audio.available_devices()) { - if(!device.input_supported) - continue; + const default_input = handle.audio.available_devices().find(e => e.input_default); + console.log(default_input); + console.log(handle.audio.available_devices().find(e => e.device_id == handle.audio.playback.current_device())); - if(device.name != "pulse") - continue; - - console.log("Found pulse at %o", device.device_index); - recorder.set_device(device.device_index, () => { - recorder.start(flag => console.log("X: " + flag)); - const consumer = recorder.create_consumer(); - consumer.create_filter_threshold(2); + recorder.set_device(default_input.device_id, () => { + const consumer = recorder.create_consumer(); + consumer.callback_data = buffer => { + stream.write_data(buffer.buffer, true); + }; + recorder.start(result => { + console.log("Start result: %o", result); }); - break; - } -} + }); -/* -1 => default device */ -const recorder = handle.audio.record.create_recorder(); -console.log("Have device: %o", recorder); -console.log("Device: %o", recorder.get_device()); -if(recorder.get_device() == -1) { - console.log("Looking for devices"); - for(const device of handle.audio.available_devices()) { - if(!device.input_supported) - continue; + setInterval(() => { + const elements = handle.audio.available_devices().filter(e => e.input_supported); + const dev = elements[Math.floor(Math.random() * elements.length)]; + recorder.set_device(dev.device_id, () => { + console.log("Dev updated: %o", dev); - if(device.name != "pulse") - continue; - - console.log("Found pulse at %o", device.device_index); - recorder.set_device(device.device_index, () => {}); - } -} -console.log("Device: %o", recorder.get_device()); -recorder.start(() => {}); -console.log("Started: %o", recorder.started()); - -const consumer = recorder.create_consumer(); - -{ - const filter = consumer.create_filter_threshold(.5); - filter.set_margin_frames(10); - /* - filter.set_analyze_filter(value => { - console.log(value); - }) - */ -} - -{ - //const filter = consumer.create_filter_vad(3); - //console.log("Filter name: %s; Filter level: %d; Filter margin: %d", filter.get_name(), filter.get_level(), filter.get_margin_frames()); -} - -{ - const consume = consumer.create_filter_state(); - setTimeout(() => { - console.log("Silence now!"); - consume.set_consuming(true); - - setTimeout(() => { - console.log("Speak now!"); - consume.set_consuming(false); - }, 1000); + recorder.start(() => { + console.log("Started"); + }); + }); }, 1000); -} - -setInterval(() => { - if("gc" in global) { - console.log("GC"); - global.gc(); - } -}, 1000); - -let a_map = [consumer, recorder]; -/* keep the object alive */ -setTimeout(() => { - connection.connected(); - a_map = a_map.filter(e => true); -}, 1000); +});