Some updates
This commit is contained in:
parent
ea375bc07e
commit
8bd8c74613
@ -11,7 +11,7 @@ file_paths=(
|
|||||||
#TODO Windows path
|
#TODO Windows path
|
||||||
)
|
)
|
||||||
files=(
|
files=(
|
||||||
"exports.d.ts;imports_shared.d.ts"
|
"exports_app.d.ts;imports_shared.d.ts"
|
||||||
# "exports_loader.d.ts;imports_shared_loader.d.ts"
|
# "exports_loader.d.ts;imports_shared_loader.d.ts"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -45,7 +45,7 @@ namespace audio.player {
|
|||||||
|
|
||||||
export function initialize() {
|
export function initialize() {
|
||||||
_output_stream = naudio.playback.create_stream();
|
_output_stream = naudio.playback.create_stream();
|
||||||
_output_stream.set_buffer_max_latency(0.08);
|
_output_stream.set_buffer_max_latency(0.4);
|
||||||
_output_stream.set_buffer_latency(0.02);
|
_output_stream.set_buffer_latency(0.02);
|
||||||
|
|
||||||
_output_stream.callback_overflow = () => {
|
_output_stream.callback_overflow = () => {
|
||||||
@ -57,7 +57,7 @@ namespace audio.player {
|
|||||||
};
|
};
|
||||||
|
|
||||||
_audioContext = new AudioContext();
|
_audioContext = new AudioContext();
|
||||||
_processor = _audioContext.createScriptProcessor(1024, _output_stream.channels, _output_stream.channels);
|
_processor = _audioContext.createScriptProcessor(1024 * 8, _output_stream.channels, _output_stream.channels);
|
||||||
|
|
||||||
_processor.onaudioprocess = function(event) {
|
_processor.onaudioprocess = function(event) {
|
||||||
const buffer = event.inputBuffer;
|
const buffer = event.inputBuffer;
|
||||||
|
@ -250,6 +250,25 @@ export namespace _audio.recorder {
|
|||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.handle = naudio.record.create_recorder();
|
this.handle = naudio.record.create_recorder();
|
||||||
|
|
||||||
|
this.consumer = this.handle.create_consumer();
|
||||||
|
this.consumer.callback_ended = () => {
|
||||||
|
if(this._current_state !== audio.recorder.InputState.RECORDING)
|
||||||
|
return;
|
||||||
|
|
||||||
|
this._current_state = audio.recorder.InputState.DRY;
|
||||||
|
if(this.callback_end)
|
||||||
|
this.callback_end();
|
||||||
|
};
|
||||||
|
this.consumer.callback_started = () => {
|
||||||
|
if(this._current_state !== audio.recorder.InputState.DRY)
|
||||||
|
return;
|
||||||
|
|
||||||
|
this._current_state = audio.recorder.InputState.RECORDING;
|
||||||
|
if(this.callback_begin)
|
||||||
|
this.callback_begin();
|
||||||
|
};
|
||||||
|
|
||||||
this._current_state = audio.recorder.InputState.PAUSED;
|
this._current_state = audio.recorder.InputState.PAUSED;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -370,20 +389,6 @@ export namespace _audio.recorder {
|
|||||||
|
|
||||||
this._current_state = audio.recorder.InputState.DRY;
|
this._current_state = audio.recorder.InputState.DRY;
|
||||||
try {
|
try {
|
||||||
if(!this.consumer) {
|
|
||||||
this.consumer = this.handle.create_consumer();
|
|
||||||
this.consumer.callback_ended = () => {
|
|
||||||
this._current_state = audio.recorder.InputState.RECORDING;
|
|
||||||
if(this.callback_end)
|
|
||||||
this.callback_end();
|
|
||||||
};
|
|
||||||
this.consumer.callback_started = () => {
|
|
||||||
this._current_state = audio.recorder.InputState.DRY;
|
|
||||||
if(this.callback_begin)
|
|
||||||
this.callback_begin();
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
await new Promise((resolve, reject) => {
|
await new Promise((resolve, reject) => {
|
||||||
this.handle.start(flag => {
|
this.handle.start(flag => {
|
||||||
if(flag)
|
if(flag)
|
||||||
|
@ -88,10 +88,99 @@ namespace _transfer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class NativeFileUpload implements RequestFileUpload {
|
||||||
|
readonly transfer_key: transfer.UploadKey;
|
||||||
|
private _handle: native.ft.NativeFileTransfer;
|
||||||
|
|
||||||
|
private _result: Promise<void>;
|
||||||
|
|
||||||
|
private _result_success: () => any;
|
||||||
|
private _result_error: (error: any) => any;
|
||||||
|
|
||||||
|
constructor(key: transfer.UploadKey) {
|
||||||
|
this.transfer_key = key;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async put_data(data: BlobPart | File) : Promise<void> {
|
||||||
|
if(this._result) {
|
||||||
|
await this._result;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let buffer: native.ft.FileTransferSource;
|
||||||
|
|
||||||
|
if(data instanceof File) {
|
||||||
|
if(data.size != this.transfer_key.total_size)
|
||||||
|
throw "invalid size";
|
||||||
|
|
||||||
|
throw "files arent yet supported";
|
||||||
|
} else if(typeof(data) === "string") {
|
||||||
|
if(data.length != this.transfer_key.total_size)
|
||||||
|
throw "invalid size";
|
||||||
|
|
||||||
|
buffer = native.ft.upload_transfer_object_from_buffer(str2ab8(data));
|
||||||
|
} else {
|
||||||
|
let buf = <BufferSource>data;
|
||||||
|
if(buf.byteLength != this.transfer_key.total_size)
|
||||||
|
throw "invalid size";
|
||||||
|
|
||||||
|
if(ArrayBuffer.isView(buf))
|
||||||
|
buf = buf.buffer.slice(buf.byteOffset);
|
||||||
|
|
||||||
|
buffer = native.ft.upload_transfer_object_from_buffer(buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
this._handle = native.ft.spawn_connection({
|
||||||
|
client_transfer_id: this.transfer_key.client_transfer_id,
|
||||||
|
server_transfer_id: this.transfer_key.server_transfer_id,
|
||||||
|
|
||||||
|
remote_address: this.transfer_key.peer.hosts[0],
|
||||||
|
remote_port: this.transfer_key.peer.port,
|
||||||
|
|
||||||
|
transfer_key: this.transfer_key.key,
|
||||||
|
|
||||||
|
object: buffer
|
||||||
|
});
|
||||||
|
|
||||||
|
await (this._result = new Promise((resolve, reject) => {
|
||||||
|
this._result_error = (error) => {
|
||||||
|
this._result_error = undefined;
|
||||||
|
this._result_success = undefined;
|
||||||
|
reject(error);
|
||||||
|
};
|
||||||
|
this._result_success = () => {
|
||||||
|
this._result_error = undefined;
|
||||||
|
this._result_success = undefined;
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
|
||||||
|
this._handle.callback_failed = this._result_error;
|
||||||
|
this._handle.callback_finished = aborted => {
|
||||||
|
if(aborted)
|
||||||
|
this._result_error("aborted");
|
||||||
|
else
|
||||||
|
this._result_success();
|
||||||
|
};
|
||||||
|
|
||||||
|
this._handle.start();
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private try_put(data: FormData, url: string): Promise<void> {
|
||||||
|
throw "what the hell?";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
export function spawn_download_transfer(key: transfer.DownloadKey) : transfer.DownloadTransfer {
|
export function spawn_download_transfer(key: transfer.DownloadKey) : transfer.DownloadTransfer {
|
||||||
return new NativeFileDownload(key);
|
return new NativeFileDownload(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export function spawn_upload_transfer(key: transfer.UploadKey) : transfer.NativeFileUpload {
|
||||||
|
return new NativeFileUpload(key);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Object.assign(window["transfer"] || (window["transfer"] = {}), _transfer);
|
Object.assign(window["transfer"] || (window["transfer"] = {}), _transfer);
|
@ -184,6 +184,8 @@ export namespace _connection {
|
|||||||
|
|
||||||
callback: error => {
|
callback: error => {
|
||||||
if(error != 0) {
|
if(error != 0) {
|
||||||
|
/* required to notify the handle, just a promise reject does not work */
|
||||||
|
this.client.handleDisconnect(DisconnectReason.CONNECT_FAILURE, error);
|
||||||
reject(this._native_handle.error_message(error));
|
reject(this._native_handle.error_message(error));
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
|
2237
modules/renderer/imports/.copy_imports_shared.d.ts
vendored
2237
modules/renderer/imports/.copy_imports_shared.d.ts
vendored
File diff suppressed because it is too large
Load Diff
@ -22,7 +22,7 @@ message("Module path: ${CMAKE_MODULE_PATH}")
|
|||||||
function(setup_nodejs)
|
function(setup_nodejs)
|
||||||
set(NodeJS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
|
set(NodeJS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
|
||||||
set(NODEJS_URL "https://atom.io/download/atom-shell")
|
set(NODEJS_URL "https://atom.io/download/atom-shell")
|
||||||
set(NODEJS_VERSION "v5.0.6")
|
set(NODEJS_VERSION "v6.0.7")
|
||||||
|
|
||||||
#set(NODEJS_URL "https://nodejs.org/download/release/")
|
#set(NODEJS_URL "https://nodejs.org/download/release/")
|
||||||
#set(NODEJS_VERSION "v12.7.0")
|
#set(NODEJS_VERSION "v12.7.0")
|
||||||
|
@ -100,8 +100,10 @@ bool merge::merge_channels_interleaved(void *target, size_t target_channels, con
|
|||||||
auto source_array = (float*) src;
|
auto source_array = (float*) src;
|
||||||
auto target_array = (float*) target;
|
auto target_array = (float*) target;
|
||||||
|
|
||||||
while(samples-- > 0)
|
while(samples-- > 0) {
|
||||||
*(target_array++) = merge_ab(*(source_array++), *(source_array++));
|
*(target_array++) = merge_ab(*(source_array), *(source_array + 1));
|
||||||
|
source_array += 2;
|
||||||
|
}
|
||||||
} else
|
} else
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
|
@ -18,6 +18,7 @@ void AudioOutputSource::clear() {
|
|||||||
ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) {
|
ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) {
|
||||||
auto sample_count = samples;
|
auto sample_count = samples;
|
||||||
|
|
||||||
|
_retest:
|
||||||
{
|
{
|
||||||
lock_guard lock(this->buffer_lock);
|
lock_guard lock(this->buffer_lock);
|
||||||
while(sample_count > 0 && !this->sample_buffers.empty()) {
|
while(sample_count > 0 && !this->sample_buffers.empty()) {
|
||||||
@ -44,8 +45,11 @@ ssize_t AudioOutputSource::pop_samples(void *buffer, size_t samples) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if(sample_count > 0) {
|
if(sample_count > 0) {
|
||||||
if(this->on_underflow)
|
if(this->on_underflow) {
|
||||||
this->on_underflow();
|
if(this->on_underflow()) {
|
||||||
|
goto _retest;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
this->buffering = true;
|
this->buffering = true;
|
||||||
}
|
}
|
||||||
@ -68,6 +72,8 @@ ssize_t AudioOutputSource::enqueue_samples(const void *buffer, size_t samples) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ssize_t AudioOutputSource::enqueue_samples(const std::shared_ptr<tc::audio::SampleBuffer> &buf) {
|
ssize_t AudioOutputSource::enqueue_samples(const std::shared_ptr<tc::audio::SampleBuffer> &buf) {
|
||||||
|
if(!buf) return 0;
|
||||||
|
|
||||||
{
|
{
|
||||||
unique_lock lock(this->buffer_lock);
|
unique_lock lock(this->buffer_lock);
|
||||||
if(this->max_latency > 0 && this->buffered_samples + buf->sample_size > this->max_latency) {
|
if(this->max_latency > 0 && this->buffered_samples + buf->sample_size > this->max_latency) {
|
||||||
|
@ -41,7 +41,8 @@ namespace tc {
|
|||||||
size_t max_latency = 0;
|
size_t max_latency = 0;
|
||||||
overflow_strategy::value overflow_strategy = overflow_strategy::discard_buffer_half;
|
overflow_strategy::value overflow_strategy = overflow_strategy::discard_buffer_half;
|
||||||
|
|
||||||
std::function<void()> on_underflow;
|
/* if it returns true then the it means that the buffer has been refilled, we have to test again */
|
||||||
|
std::function<bool()> on_underflow;
|
||||||
std::function<void(size_t /* sample count */)> on_overflow;
|
std::function<void(size_t /* sample count */)> on_overflow;
|
||||||
std::function<void()> on_read; /* will be invoked after sample read, e.g. for buffer fullup */
|
std::function<void()> on_read; /* will be invoked after sample read, e.g. for buffer fullup */
|
||||||
|
|
||||||
|
@ -117,6 +117,12 @@ bool OpusConverter::_initialize_encoder(std::string &error) {
|
|||||||
error = "failed to create encoder (" + to_string(error_id) + ")";
|
error = "failed to create encoder (" + to_string(error_id) + ")";
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
error_id = opus_encoder_ctl(encoder, OPUS_SET_BITRATE(64000));
|
||||||
|
if(error_id) {
|
||||||
|
error = "failed to set bitrate (" + to_string(error_id) + ")";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -90,7 +90,7 @@ void AudioOutputStreamWrapper::do_wrap(const v8::Local<v8::Object> &obj) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
this->_own_handle->on_overflow = [&](size_t){ this->call_overflow(); };
|
this->_own_handle->on_overflow = [&](size_t){ this->call_overflow(); };
|
||||||
this->_own_handle->on_underflow = [&]{ this->call_underflow(); };
|
this->_own_handle->on_underflow = [&]{ this->call_underflow(); return false; };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -499,6 +499,7 @@ void ServerConnection::send_voice_data(const void *buffer, size_t buffer_length,
|
|||||||
|
|
||||||
if(head) /* head packet */
|
if(head) /* head packet */
|
||||||
packet->enable_flag(ts::protocol::PacketFlag::Compressed);
|
packet->enable_flag(ts::protocol::PacketFlag::Compressed);
|
||||||
|
packet->enable_flag(ts::protocol::PacketFlag::Unencrypted);
|
||||||
|
|
||||||
//#define FUZZ_VOICE
|
//#define FUZZ_VOICE
|
||||||
#ifdef FUZZ_VOICE
|
#ifdef FUZZ_VOICE
|
||||||
|
@ -3,6 +3,7 @@
|
|||||||
#include "../ServerConnection.h"
|
#include "../ServerConnection.h"
|
||||||
#include "../../logger.h"
|
#include "../../logger.h"
|
||||||
#include "AudioEventLoop.h"
|
#include "AudioEventLoop.h"
|
||||||
|
#include "../../audio/AudioMerger.h"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
using namespace tc;
|
using namespace tc;
|
||||||
@ -17,7 +18,7 @@ VoiceSender::~VoiceSender() {
|
|||||||
this->clear_buffer(); /* buffer might be accessed within encode_raw_frame, but this could not be trigered while this will be deallocated! */
|
this->clear_buffer(); /* buffer might be accessed within encode_raw_frame, but this could not be trigered while this will be deallocated! */
|
||||||
}
|
}
|
||||||
|
|
||||||
bool VoiceSender::initialize_codec(std::string& error, connection::codec::value codec, size_t channels, size_t rate, bool reset_decoder) {
|
bool VoiceSender::initialize_codec(std::string& error, connection::codec::value codec, size_t channels, size_t rate, bool reset_encoder) {
|
||||||
auto& data = this->codec[codec];
|
auto& data = this->codec[codec];
|
||||||
bool new_allocated = !data;
|
bool new_allocated = !data;
|
||||||
if(new_allocated) {
|
if(new_allocated) {
|
||||||
@ -33,16 +34,17 @@ bool VoiceSender::initialize_codec(std::string& error, connection::codec::value
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!data->converter || data->converter->channels() != channels || data->converter->sample_rate() != rate) {
|
if(!data->converter) {
|
||||||
data->converter = info->new_converter(error);
|
data->converter = info->new_converter(error);
|
||||||
if(!data->converter)
|
if(!data->converter)
|
||||||
return false;
|
return false;
|
||||||
} else if(reset_decoder) {
|
} else if(reset_encoder) {
|
||||||
data->converter->reset_encoder();
|
data->converter->reset_encoder();
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!data->resampler || data->resampler->input_rate() != rate)
|
if(!data->resampler || data->resampler->input_rate() != rate)
|
||||||
data->resampler = make_shared<AudioResampler>(rate, data->converter->sample_rate(), data->converter->channels());
|
data->resampler = make_shared<AudioResampler>(rate, data->converter->sample_rate(), data->converter->channels());
|
||||||
|
|
||||||
if(!data->resampler->valid()) {
|
if(!data->resampler->valid()) {
|
||||||
error = "resampler is invalid";
|
error = "resampler is invalid";
|
||||||
return false;
|
return false;
|
||||||
@ -111,10 +113,12 @@ void VoiceSender::event_execute(const std::chrono::system_clock::time_point &poi
|
|||||||
unique_lock buffer_lock(this->raw_audio_buffer_lock);
|
unique_lock buffer_lock(this->raw_audio_buffer_lock);
|
||||||
if(this->raw_audio_buffers.empty())
|
if(this->raw_audio_buffers.empty())
|
||||||
break;
|
break;
|
||||||
|
|
||||||
if(chrono::system_clock::now() - now > max_time) {
|
if(chrono::system_clock::now() - now > max_time) {
|
||||||
reschedule = true;
|
reschedule = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto entry = move(this->raw_audio_buffers.front());
|
auto entry = move(this->raw_audio_buffers.front());
|
||||||
this->raw_audio_buffers.pop_front();
|
this->raw_audio_buffers.pop_front();
|
||||||
buffer_lock.unlock();
|
buffer_lock.unlock();
|
||||||
@ -168,15 +172,34 @@ void VoiceSender::encode_raw_frame(const std::unique_ptr<AudioFrame> &frame) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* TODO: May test for channel and sample rate? */
|
auto merged_channel_byte_size = codec_data->converter->channels() * (frame->buffer.length() / frame->channels);
|
||||||
this->ensure_buffer(codec_data->resampler->estimated_output_size(frame->buffer.length()));
|
auto estimated_resampled_byte_size = codec_data->resampler->estimated_output_size(merged_channel_byte_size);
|
||||||
auto resampled_samples = codec_data->resampler->process(this->_buffer, frame->buffer.data_ptr(), frame->buffer.length() / frame->channels / 4);
|
this->ensure_buffer(max(estimated_resampled_byte_size, merged_channel_byte_size));
|
||||||
|
|
||||||
|
auto codec_channels = codec_data->converter->channels();
|
||||||
|
if(!audio::merge::merge_channels_interleaved(this->_buffer, codec_channels, frame->buffer.data_ptr(), frame->channels, frame->buffer.length() / frame->channels / 4)) {
|
||||||
|
log_warn(category::voice_connection, tr("Failed to merge channels to output stream channel count! Dropping local voice packet"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto resampled_samples = codec_data->resampler->process(this->_buffer, this->_buffer, merged_channel_byte_size / codec_channels / 4);
|
||||||
if(resampled_samples <= 0) {
|
if(resampled_samples <= 0) {
|
||||||
log_error(category::voice_connection, tr("Resampler returned {}"), resampled_samples);
|
log_error(category::voice_connection, tr("Resampler returned {}"), resampled_samples);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto encoded_bytes = codec_data->converter->encode(error, this->_buffer, this->_buffer, this->_buffer_size);
|
if(resampled_samples * codec_channels * 4 != codec_data->converter->bytes_per_frame()) {
|
||||||
|
log_error(category::voice_connection,
|
||||||
|
tr("Could not encode audio frame. Frame length is not equal to code frame length! Codec: {}, Packet: {}"),
|
||||||
|
codec_data->converter->bytes_per_frame(), resampled_samples * codec_channels * 4
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
log_trace(category::voice_connection, tr("Encode buffer size: {}"), this->_buffer_size);
|
||||||
|
|
||||||
|
char _packet_buffer[512];
|
||||||
|
auto encoded_bytes = codec_data->converter->encode(error, this->_buffer, _packet_buffer, 512);
|
||||||
if(encoded_bytes <= 0) {
|
if(encoded_bytes <= 0) {
|
||||||
log_error(category::voice_connection, tr("Failed to encode voice: {}"), error);
|
log_error(category::voice_connection, tr("Failed to encode voice: {}"), error);
|
||||||
return;
|
return;
|
||||||
@ -190,6 +213,6 @@ void VoiceSender::encode_raw_frame(const std::unique_ptr<AudioFrame> &frame) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
auto server = this->handle->handle();
|
auto server = this->handle->handle();
|
||||||
server->send_voice_data(this->_buffer, encoded_bytes, codec, flag_head);
|
server->send_voice_data(_packet_buffer, encoded_bytes, codec, flag_head);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -226,6 +226,10 @@ VoiceClient::VoiceClient(const std::shared_ptr<VoiceConnection>&, uint16_t clien
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
play_premature_packets = true; /* try to replay any premature packets because we assume that the other packets got lost */
|
||||||
|
audio::decode_event_loop->schedule(static_pointer_cast<event::EventEntry>(this->ref()));
|
||||||
|
return false;
|
||||||
};
|
};
|
||||||
this->output_source->on_overflow = [&](size_t count){
|
this->output_source->on_overflow = [&](size_t count){
|
||||||
log_warn(category::audio, tr("Client {} has a audio buffer overflow of {}."), this->_client_id, count);
|
log_warn(category::audio, tr("Client {} has a audio buffer overflow of {}."), this->_client_id, count);
|
||||||
@ -286,7 +290,6 @@ void VoiceClient::process_packet(uint16_t packet_id, const pipes::buffer_view& b
|
|||||||
this->audio_decode_queue.push_back(move(encoded_buffer));
|
this->audio_decode_queue.push_back(move(encoded_buffer));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ‘tc::event::EventEntry’ is an inaccessible base of ‘tc::connection::VoiceClient’ => So we need a */
|
|
||||||
audio::decode_event_loop->schedule(static_pointer_cast<event::EventEntry>(this->ref()));
|
audio::decode_event_loop->schedule(static_pointer_cast<event::EventEntry>(this->ref()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -303,12 +306,42 @@ void VoiceClient::event_execute(const std::chrono::system_clock::time_point &sch
|
|||||||
auto now = chrono::system_clock::now();
|
auto now = chrono::system_clock::now();
|
||||||
while(true) {
|
while(true) {
|
||||||
unique_lock buffer_lock(this->audio_decode_queue_lock);
|
unique_lock buffer_lock(this->audio_decode_queue_lock);
|
||||||
|
if(this->play_premature_packets) {
|
||||||
|
this->play_premature_packets = false;
|
||||||
|
|
||||||
|
for(auto& codec_data : this->codec) {
|
||||||
|
if(!codec_data) continue;
|
||||||
|
|
||||||
|
if(!codec_data->premature_packets.empty()) {
|
||||||
|
size_t play_count = 0;
|
||||||
|
while(!codec_data->premature_packets.empty()) {
|
||||||
|
auto& packet = codec_data->premature_packets.front();
|
||||||
|
|
||||||
|
//Test if we're able to replay stuff again
|
||||||
|
if((uint16_t) (codec_data->last_packet_id + 1) < packet.packet_id && play_count > 0) //Only check for the order if we replayed one already
|
||||||
|
break; //Nothing new
|
||||||
|
|
||||||
|
this->output_source->enqueue_samples(packet.buffer);
|
||||||
|
codec_data->last_packet_id = packet.packet_id;
|
||||||
|
codec_data->premature_packets.pop_front();
|
||||||
|
play_count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(play_count > 0)
|
||||||
|
log_debug(category::audio, tr("Replayed {} premature packets for client {}"), play_count, this->_client_id);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if(this->audio_decode_queue.empty())
|
if(this->audio_decode_queue.empty())
|
||||||
break;
|
break;
|
||||||
|
|
||||||
if(chrono::system_clock::now() - now > max_time) {
|
if(chrono::system_clock::now() - now > max_time) {
|
||||||
reschedule = true;
|
reschedule = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto entry = move(this->audio_decode_queue.front());
|
auto entry = move(this->audio_decode_queue.front());
|
||||||
this->audio_decode_queue.pop_front();
|
this->audio_decode_queue.pop_front();
|
||||||
buffer_lock.unlock();
|
buffer_lock.unlock();
|
||||||
@ -323,6 +356,7 @@ void VoiceClient::event_execute(const std::chrono::system_clock::time_point &sch
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//FIXME premature packets dont work
|
||||||
#define MAX_LOST_PACKETS (6)
|
#define MAX_LOST_PACKETS (6)
|
||||||
void VoiceClient::process_encoded_buffer(const std::unique_ptr<EncodedBuffer> &buffer) {
|
void VoiceClient::process_encoded_buffer(const std::unique_ptr<EncodedBuffer> &buffer) {
|
||||||
string error;
|
string error;
|
||||||
@ -362,26 +396,39 @@ void VoiceClient::process_encoded_buffer(const std::unique_ptr<EncodedBuffer> &b
|
|||||||
}
|
}
|
||||||
|
|
||||||
uint16_t diff;
|
uint16_t diff;
|
||||||
if(codec_data->last_packet_id > buffer->packet_id) {
|
bool premature = false;
|
||||||
auto local_index = (uint16_t) (codec_data->last_packet_id + MAX_LOST_PACKETS);
|
|
||||||
if(local_index < buffer->packet_id)
|
|
||||||
diff = 0xFF;
|
|
||||||
else
|
|
||||||
diff = static_cast<uint16_t>(MAX_LOST_PACKETS - (local_index - buffer->packet_id));
|
|
||||||
} else {
|
|
||||||
diff = buffer->packet_id - codec_data->last_packet_id;
|
|
||||||
}
|
|
||||||
|
|
||||||
if(codec_data->last_packet_timestamp + chrono::seconds(1) < buffer->receive_timestamp || this->_state >= state::stopping)
|
if(codec_data->last_packet_timestamp + chrono::seconds(1) < buffer->receive_timestamp || this->_state >= state::stopping) {
|
||||||
diff = 0xFFFF;
|
diff = 0xFFFF;
|
||||||
|
} else {
|
||||||
|
if(codec_data->last_packet_id > buffer->packet_id) {
|
||||||
|
auto local_index = (uint16_t) (codec_data->last_packet_id + MAX_LOST_PACKETS);
|
||||||
|
if(local_index < buffer->packet_id)
|
||||||
|
diff = 0xFF; /* we got in a new generation */
|
||||||
|
else {
|
||||||
|
log_warn(category::audio,
|
||||||
|
tr("Received voice packet for client {} with is older than the last we received (Current index: {}, Packet index: {}). Dropping packet."),
|
||||||
|
this->_client_id, buffer->packet_id, codec_data->last_packet_id
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
diff = buffer->packet_id - codec_data->last_packet_id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const auto old_packet_id = codec_data->last_packet_id;
|
const auto old_packet_id = codec_data->last_packet_id;
|
||||||
codec_data->last_packet_timestamp = buffer->receive_timestamp;
|
codec_data->last_packet_timestamp = buffer->receive_timestamp;
|
||||||
codec_data->last_packet_id = buffer->packet_id;
|
|
||||||
|
|
||||||
if(buffer->buffer.empty()) {
|
if(buffer->buffer.empty()) {
|
||||||
/* lets playpack the last samples and we're done */
|
/* lets playback the last samples and we're done */
|
||||||
this->set_state(state::stopping);
|
this->set_state(state::stopping);
|
||||||
|
|
||||||
|
/* enqueue all premature packets (list should be already ordered!) */
|
||||||
|
for(const auto& packet : codec_data->premature_packets)
|
||||||
|
this->output_source->enqueue_samples(packet.buffer);
|
||||||
|
codec_data->premature_packets.clear();
|
||||||
|
|
||||||
log_trace(category::voice_connection, tr("Stopping replay for client {}. Empty buffer!"), this->_client_id);
|
log_trace(category::voice_connection, tr("Stopping replay for client {}. Empty buffer!"), this->_client_id);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -395,10 +442,14 @@ void VoiceClient::process_encoded_buffer(const std::unique_ptr<EncodedBuffer> &b
|
|||||||
|
|
||||||
if(diff <= MAX_LOST_PACKETS) {
|
if(diff <= MAX_LOST_PACKETS) {
|
||||||
if(diff > 0) {
|
if(diff > 0) {
|
||||||
log_debug(category::voice_connection, tr("Client {} dropped one or more audio packets. Old packet id: {}, New packet id: {}, Diff: {}"), this->_client_id, old_packet_id, buffer->packet_id, diff);
|
log_debug(category::voice_connection,
|
||||||
auto status = codec_data->converter->decode_lost(error, diff);
|
tr("Client {} dropped one or more audio packets. Old packet id: {}, New packet id: {}, Diff: {}"),
|
||||||
if(status < 0)
|
this->_client_id, old_packet_id, buffer->packet_id, diff);
|
||||||
log_warn(category::voice_connection, tr("Failed to decode (skip) dropped packets. Return code {} => {}"), status, error);
|
/* lets first handle packet as "lost", even thou we're enqueueing it as premature */
|
||||||
|
//auto status = codec_data->converter->decode_lost(error, diff);
|
||||||
|
//if(status < 0)
|
||||||
|
// log_warn(category::voice_connection, tr("Failed to decode (skip) dropped packets. Return code {} => {}"), status, error);
|
||||||
|
premature = this->state() != state::stopped;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log_debug(category::voice_connection, tr("Client {} resetted decoder. Old packet id: {}, New packet id: {}, diff: {}"), this->_client_id, old_packet_id, buffer->packet_id, diff);
|
log_debug(category::voice_connection, tr("Client {} resetted decoder. Old packet id: {}, New packet id: {}, diff: {}"), this->_client_id, old_packet_id, buffer->packet_id, diff);
|
||||||
@ -409,6 +460,30 @@ void VoiceClient::process_encoded_buffer(const std::unique_ptr<EncodedBuffer> &b
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(!premature) {
|
||||||
|
codec_data->last_packet_id = buffer->packet_id;
|
||||||
|
|
||||||
|
/* test if any premature got its original place */
|
||||||
|
{
|
||||||
|
size_t play_count = 0;
|
||||||
|
while(!codec_data->premature_packets.empty()) {
|
||||||
|
auto& packet = codec_data->premature_packets.front();
|
||||||
|
|
||||||
|
//Test if we're able to replay stuff again
|
||||||
|
if((uint16_t) (codec_data->last_packet_id + 1) < packet.packet_id)
|
||||||
|
break; //Nothing new
|
||||||
|
|
||||||
|
this->output_source->enqueue_samples(packet.buffer);
|
||||||
|
codec_data->last_packet_id = packet.packet_id;
|
||||||
|
codec_data->premature_packets.pop_front();
|
||||||
|
play_count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(play_count > 0)
|
||||||
|
log_debug(category::audio, tr("Replayed {} premature packets for client {}"), play_count, this->_client_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
char target_buffer[target_buffer_length];
|
char target_buffer[target_buffer_length];
|
||||||
if(target_buffer_length < codec_data->converter->expected_decoded_length(buffer->buffer.data_ptr(), buffer->buffer.length())) {
|
if(target_buffer_length < codec_data->converter->expected_decoded_length(buffer->buffer.data_ptr(), buffer->buffer.length())) {
|
||||||
log_warn(category::voice_connection, tr("Failed to decode audio data. Target buffer is smaller then expected bytes ({} < {})"), target_buffer_length, codec_data->converter->expected_decoded_length(buffer->buffer.data_ptr(), buffer->buffer.length()));
|
log_warn(category::voice_connection, tr("Failed to decode audio data. Target buffer is smaller then expected bytes ({} < {})"), target_buffer_length, codec_data->converter->expected_decoded_length(buffer->buffer.data_ptr(), buffer->buffer.length()));
|
||||||
@ -442,8 +517,29 @@ void VoiceClient::process_encoded_buffer(const std::unique_ptr<EncodedBuffer> &b
|
|||||||
*(buf++) *= this->_volume;
|
*(buf++) *= this->_volume;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto enqueued = this->output_source->enqueue_samples(target_buffer, resampled_samples);
|
if(premature) {
|
||||||
if(enqueued != resampled_samples)
|
auto audio_buffer = audio::SampleBuffer::allocate((uint8_t) this->output_source->channel_count, (uint16_t) resampled_samples);
|
||||||
log_warn(category::voice_connection, tr("Failed to enqueue all samples for client {}. Enqueued {} of {}"), this->_client_id, enqueued, resampled_samples);
|
|
||||||
this->set_state(state::playing);
|
audio_buffer->sample_index = 0;
|
||||||
|
memcpy(audio_buffer->sample_data, target_buffer, this->output_source->channel_count * resampled_samples * 4);
|
||||||
|
|
||||||
|
auto it = codec_data->premature_packets.begin();
|
||||||
|
for(; it != codec_data->premature_packets.end(); it++) {
|
||||||
|
if(it->packet_id > buffer->packet_id) {
|
||||||
|
break; /* it is set to the right position */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
codec_data->premature_packets.insert(it, {
|
||||||
|
buffer->packet_id,
|
||||||
|
audio_buffer
|
||||||
|
});
|
||||||
|
std::stable_sort(codec_data->premature_packets.begin(), codec_data->premature_packets.end(), [](const PrematureAudioPacket& a, const PrematureAudioPacket& b) {
|
||||||
|
return a.packet_id < b.packet_id;
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
auto enqueued = this->output_source->enqueue_samples(target_buffer, resampled_samples);
|
||||||
|
if(enqueued != resampled_samples)
|
||||||
|
log_warn(category::voice_connection, tr("Failed to enqueue all samples for client {}. Enqueued {} of {}"), this->_client_id, enqueued, resampled_samples);
|
||||||
|
this->set_state(state::playing);
|
||||||
|
}
|
||||||
}
|
}
|
@ -88,6 +88,11 @@ namespace tc {
|
|||||||
|
|
||||||
inline std::shared_ptr<audio::AudioOutputSource> output_stream() { return this->output_source; }
|
inline std::shared_ptr<audio::AudioOutputSource> output_stream() { return this->output_source; }
|
||||||
private:
|
private:
|
||||||
|
struct PrematureAudioPacket {
|
||||||
|
uint16_t packet_id = 0;
|
||||||
|
std::shared_ptr<tc::audio::SampleBuffer> buffer;
|
||||||
|
};
|
||||||
|
|
||||||
struct AudioCodec {
|
struct AudioCodec {
|
||||||
uint16_t last_packet_id = 0;
|
uint16_t last_packet_id = 0;
|
||||||
std::chrono::system_clock::time_point last_packet_timestamp;
|
std::chrono::system_clock::time_point last_packet_timestamp;
|
||||||
@ -95,6 +100,8 @@ namespace tc {
|
|||||||
bool successfully_initialized;
|
bool successfully_initialized;
|
||||||
std::shared_ptr<audio::codec::Converter> converter;
|
std::shared_ptr<audio::codec::Converter> converter;
|
||||||
std::shared_ptr<audio::AudioResampler> resampler;
|
std::shared_ptr<audio::AudioResampler> resampler;
|
||||||
|
|
||||||
|
std::deque<PrematureAudioPacket> premature_packets;
|
||||||
};
|
};
|
||||||
|
|
||||||
std::array<std::unique_ptr<AudioCodec>, codec::MAX + 1> codec{
|
std::array<std::unique_ptr<AudioCodec>, codec::MAX + 1> codec{
|
||||||
@ -112,6 +119,8 @@ namespace tc {
|
|||||||
uint16_t _client_id;
|
uint16_t _client_id;
|
||||||
float _volume = 1.f;
|
float _volume = 1.f;
|
||||||
|
|
||||||
|
bool play_premature_packets = false;
|
||||||
|
|
||||||
std::chrono::system_clock::time_point _last_received_packet;
|
std::chrono::system_clock::time_point _last_received_packet;
|
||||||
state::value _state = state::stopped;
|
state::value _state = state::stopped;
|
||||||
inline void set_state(state::value value) {
|
inline void set_state(state::value value) {
|
||||||
|
@ -149,12 +149,6 @@ NAN_METHOD(VoiceConnectionWrap::unregister_client) {
|
|||||||
|
|
||||||
NAN_METHOD(VoiceConnectionWrap::_audio_source) {
|
NAN_METHOD(VoiceConnectionWrap::_audio_source) {
|
||||||
auto client = ObjectWrap::Unwrap<VoiceConnectionWrap>(info.Holder());
|
auto client = ObjectWrap::Unwrap<VoiceConnectionWrap>(info.Holder());
|
||||||
|
|
||||||
if(info.Length() != 1) {
|
|
||||||
Nan::ThrowError("invalid argument count");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
info.GetReturnValue().Set(client->_voice_recoder_handle.Get(info.GetIsolate()));
|
info.GetReturnValue().Set(client->_voice_recoder_handle.Get(info.GetIsolate()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -192,7 +186,10 @@ NAN_METHOD(VoiceConnectionWrap::set_audio_source) {
|
|||||||
lock_guard read_lock(this->_voice_recoder_ptr->native_read_callback_lock);
|
lock_guard read_lock(this->_voice_recoder_ptr->native_read_callback_lock);
|
||||||
this->_voice_recoder_ptr->native_read_callback = [weak_handle, sample_rate, channels](const void* buffer, size_t length) {
|
this->_voice_recoder_ptr->native_read_callback = [weak_handle, sample_rate, channels](const void* buffer, size_t length) {
|
||||||
auto handle = weak_handle.lock();
|
auto handle = weak_handle.lock();
|
||||||
if(!handle) return;
|
if(!handle) {
|
||||||
|
log_warn(category::audio, tr("Missing voice connection handle. Dropping input!"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
shared_ptr<VoiceSender> sender = handle->voice_sender();
|
shared_ptr<VoiceSender> sender = handle->voice_sender();
|
||||||
if(sender) {
|
if(sender) {
|
||||||
@ -200,6 +197,9 @@ NAN_METHOD(VoiceConnectionWrap::set_audio_source) {
|
|||||||
sender->send_data(buffer, length, sample_rate, channels);
|
sender->send_data(buffer, length, sample_rate, channels);
|
||||||
else
|
else
|
||||||
sender->send_stop();
|
sender->send_stop();
|
||||||
|
} else {
|
||||||
|
log_warn(category::audio, tr("Missing voice connection audio sender. Dropping input!"));
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -10,6 +10,11 @@ using namespace std;
|
|||||||
|
|
||||||
TransferJSBufferTarget::TransferJSBufferTarget() {
|
TransferJSBufferTarget::TransferJSBufferTarget() {
|
||||||
log_allocate("TransferJSBufferTarget", this);
|
log_allocate("TransferJSBufferTarget", this);
|
||||||
|
|
||||||
|
if(!this->_js_buffer.IsEmpty()) {
|
||||||
|
assert(v8::Isolate::GetCurrent());
|
||||||
|
this->_js_buffer.Reset();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
TransferJSBufferTarget::~TransferJSBufferTarget() {
|
TransferJSBufferTarget::~TransferJSBufferTarget() {
|
||||||
@ -74,6 +79,11 @@ NAN_METHOD(TransferJSBufferTarget::create_from_buffer) {
|
|||||||
|
|
||||||
TransferJSBufferSource::~TransferJSBufferSource() {
|
TransferJSBufferSource::~TransferJSBufferSource() {
|
||||||
log_free("TransferJSBufferSource", this);
|
log_free("TransferJSBufferSource", this);
|
||||||
|
|
||||||
|
if(!this->_js_buffer.IsEmpty()) {
|
||||||
|
assert(v8::Isolate::GetCurrent());
|
||||||
|
this->_js_buffer.Reset();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
TransferJSBufferSource::TransferJSBufferSource() {
|
TransferJSBufferSource::TransferJSBufferSource() {
|
||||||
log_allocate("TransferJSBufferSource", this);
|
log_allocate("TransferJSBufferSource", this);
|
||||||
|
@ -79,7 +79,7 @@ namespace tc {
|
|||||||
TransferJSBufferSource();
|
TransferJSBufferSource();
|
||||||
virtual ~TransferJSBufferSource();
|
virtual ~TransferJSBufferSource();
|
||||||
|
|
||||||
std::string name() const override { return "TransferJSBufferTarget"; }
|
std::string name() const override { return "TransferJSBufferSource"; }
|
||||||
bool initialize(std::string &string) override;
|
bool initialize(std::string &string) override;
|
||||||
|
|
||||||
void finalize() override;
|
void finalize() override;
|
||||||
|
@ -46,7 +46,7 @@
|
|||||||
"assert-plus": "^1.0.0",
|
"assert-plus": "^1.0.0",
|
||||||
"aws-sign2": "^0.7.0",
|
"aws-sign2": "^0.7.0",
|
||||||
"aws4": "^1.8.0",
|
"aws4": "^1.8.0",
|
||||||
"electron": "5.0.6",
|
"electron": "6.0.7",
|
||||||
"electron-installer-windows": "^1.1.1",
|
"electron-installer-windows": "^1.1.1",
|
||||||
"electron-navigation": "^1.5.8",
|
"electron-navigation": "^1.5.8",
|
||||||
"electron-rebuild": "^1.8.5",
|
"electron-rebuild": "^1.8.5",
|
||||||
|
Loading…
x
Reference in New Issue
Block a user