From 7087514df1b21701c34b04d88c953dd5252971ba Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Thu, 1 Oct 2020 10:56:22 +0200 Subject: [PATCH] Adding rnnoise suppression --- github | 2 +- jenkins/create_build.sh | 4 +- main.ts | 2 +- modules/core/AppInstance.ts | 47 +++++ ...nce_handler.ts => MultiInstanceHandler.ts} | 21 ++- modules/core/app-updater/index.ts | 18 +- .../{main_window.ts => main-window/index.ts} | 152 +++++++---------- modules/core/main-window/preload.ts | 12 ++ modules/core/main.ts | 69 ++++---- modules/core/render-backend/ExternalModal.ts | 5 +- modules/core/render-backend/index.ts | 8 +- modules/core/ui-loader/graphical.ts | 6 +- modules/core/ui-loader/loader.ts | 12 +- modules/renderer-manifest/index.ts | 18 +- modules/renderer-manifest/preload.ts | 12 ++ modules/renderer/ContextMenu.ts | 64 +++++++ modules/renderer/IconHelper.ts | 2 +- modules/renderer/MenuBarHandler.ts | 4 +- modules/renderer/RequireProxy.ts | 14 +- modules/renderer/UnloadHandler.ts | 4 +- modules/renderer/audio/AudioRecorder.ts | 85 +++++---- .../renderer/connection/VoiceConnection.ts | 22 +-- modules/renderer/context-menu.ts | 128 -------------- modules/renderer/hooks/AudioInput.ts | 9 + modules/renderer/index.ts | 33 ++-- modules/shared/process-arguments/index.ts | 20 +-- native/serverconnection/CMakeLists.txt | 10 +- native/serverconnection/exports/exports.d.ts | 9 +- .../serverconnection/src/audio/AudioInput.cpp | 15 +- .../src/audio/js/AudioConsumer.cpp | 141 ++++++++++++++- .../src/audio/js/AudioConsumer.h | 161 ++++++++++-------- .../src/audio/js/AudioFilter.h | 96 +++++------ .../src/audio/js/AudioRecorder.cpp | 50 +++--- .../src/audio/js/AudioRecorder.h | 11 +- .../src/connection/audio/AudioSender.cpp | 7 +- .../src/connection/audio/VoiceConnection.cpp | 2 +- native/serverconnection/test/js/main.ts | 3 + package.json | 2 +- 38 files changed, 727 insertions(+), 553 deletions(-) create mode 100644 modules/core/AppInstance.ts rename modules/core/{instance_handler.ts => MultiInstanceHandler.ts} (52%) rename modules/core/{main_window.ts => main-window/index.ts} (51%) create mode 100644 modules/core/main-window/preload.ts create mode 100644 modules/renderer-manifest/preload.ts create mode 100644 modules/renderer/ContextMenu.ts delete mode 100644 modules/renderer/context-menu.ts diff --git a/github b/github index 4fa1ab2..9c3cc6d 160000 --- a/github +++ b/github @@ -1 +1 @@ -Subproject commit 4fa1ab237cd12b53de46fe82d31c942513c619bd +Subproject commit 9c3cc6d05838a03a5827836b300f8bc8e71b26d2 diff --git a/jenkins/create_build.sh b/jenkins/create_build.sh index 08ef2e3..d96f108 100755 --- a/jenkins/create_build.sh +++ b/jenkins/create_build.sh @@ -122,5 +122,5 @@ function deploy_client() { #install_npm #compile_scripts #compile_native -#package_client -deploy_client +package_client +#deploy_client diff --git a/main.ts b/main.ts index a432384..ec07e59 100644 --- a/main.ts +++ b/main.ts @@ -2,7 +2,7 @@ import * as electron from "electron"; import * as crash_handler from "./modules/crash_handler"; import * as child_process from "child_process"; import {app} from "electron"; -import * as Sentry from "@sentry/electron"; +//import * as Sentry from "@sentry/electron"; /* Sentry.init({ diff --git a/modules/core/AppInstance.ts b/modules/core/AppInstance.ts new file mode 100644 index 0000000..995d40b --- /dev/null +++ b/modules/core/AppInstance.ts @@ -0,0 +1,47 @@ +import {app} from "electron"; +import * as crash_handler from "../crash_handler"; +import * as loader from "./ui-loader/graphical"; + +let appReferences = 0; + +/** + * Normally the app closes when all windows have been closed. + * If you're holding an app reference, it will not terminate when all windows have been closed. + */ +export function referenceApp() { + appReferences++; +} + +export function dereferenceApp() { + appReferences--; + testAppState(); +} + + +function testAppState() { + if(appReferences > 0) { return; } + + console.log("All windows have been closed, closing app."); + app.quit(); +} + +function initializeAppListeners() { + app.on('quit', () => { + console.debug("Shutting down app."); + crash_handler.finalize_handler(); + loader.ui.cleanup(); + console.log("App has been finalized."); + }); + + + app.on('window-all-closed', () => { + console.log("All windows have been closed. Manual app reference count: %d", appReferences); + testAppState(); + }); + + app.on('activate', () => { + // On macOS it's common to re-create a window in the app when the + // dock icon is clicked and there are no other windows open. + }); +} +initializeAppListeners(); \ No newline at end of file diff --git a/modules/core/instance_handler.ts b/modules/core/MultiInstanceHandler.ts similarity index 52% rename from modules/core/instance_handler.ts rename to modules/core/MultiInstanceHandler.ts index 027cee5..d83bebc 100644 --- a/modules/core/instance_handler.ts +++ b/modules/core/MultiInstanceHandler.ts @@ -1,14 +1,16 @@ -import * as main_window from "./main_window"; +import { app } from "electron"; +import { mainWindow } from "./main-window"; -export function handle_second_instance_call(argv: string[], work_dir: string) { +export function handleSecondInstanceCall(argv: string[], _workingDirectory: string) { const original_args = argv.slice(1).filter(e => !e.startsWith("--original-process-start-time=") && e != "--allow-file-access-from-files"); console.log("Second instance: %o", original_args); - if(!main_window.main_window) { + if(!mainWindow) { console.warn("Ignoring second instance call because we haven't yet started"); return; } - main_window.main_window.focus(); + + mainWindow.focus(); execute_connect_urls(original_args); } @@ -16,6 +18,15 @@ export function execute_connect_urls(argv: string[]) { const connect_urls = argv.filter(e => e.startsWith("teaclient://")); for(const url of connect_urls) { console.log("Received connect url: %s", url); - main_window.main_window.webContents.send('connect', url); + mainWindow.webContents.send('connect', url); } +} + +export function initializeSingleInstance() : boolean { + if(!app.requestSingleInstanceLock()) { + return false; + } + + app.on('second-instance', (event, argv, workingDirectory) => handleSecondInstanceCall(argv, workingDirectory)); + return true; } \ No newline at end of file diff --git a/modules/core/app-updater/index.ts b/modules/core/app-updater/index.ts index ad4a756..6ed0e33 100644 --- a/modules/core/app-updater/index.ts +++ b/modules/core/app-updater/index.ts @@ -16,18 +16,18 @@ import {parse_version, Version} from "../../shared/version"; import Timer = NodeJS.Timer; import MessageBoxOptions = Electron.MessageBoxOptions; import {Headers} from "tar-stream"; -import {Arguments, process_args} from "../../shared/process-arguments"; +import {Arguments, processArguments} from "../../shared/process-arguments"; import * as electron from "electron"; import {PassThrough} from "stream"; import ErrnoException = NodeJS.ErrnoException; -import {reference_app} from "../main_window"; import * as url from "url"; import {loadWindowBounds, startTrackWindowBounds} from "../../shared/window"; +import {referenceApp} from "../AppInstance"; const is_debug = false; export function server_url() : string { const default_path = is_debug ? "http://localhost/home/TeaSpeak/TeaSpeak/Web-Client/client-api/environment/" : "http://clientapi.teaspeak.de/"; - return process_args.has_value(...Arguments.SERVER_URL) ? process_args.value(...Arguments.SERVER_URL) : default_path; + return processArguments.has_value(...Arguments.SERVER_URL) ? processArguments.value(...Arguments.SERVER_URL) : default_path; } export interface UpdateVersion { @@ -647,8 +647,8 @@ export async function execute_update(update_file: string, restart_callback: (cal } export async function current_version() : Promise { - if(process_args.has_value(Arguments.UPDATER_LOCAL_VERSION)) - return parse_version(process_args.value(Arguments.UPDATER_LOCAL_VERSION)); + if(processArguments.has_value(Arguments.UPDATER_LOCAL_VERSION)) + return parse_version(processArguments.value(Arguments.UPDATER_LOCAL_VERSION)); let parent_path = app.getAppPath(); if(parent_path.endsWith(".asar")) { @@ -679,7 +679,7 @@ export let update_restart_pending = false; export async function execute_graphical(channel: string, ask_install: boolean) : Promise { const electron = require('electron'); - const ui_debug = process_args.has_flag(Arguments.UPDATER_UI_DEBUG); + const ui_debug = processArguments.has_flag(Arguments.UPDATER_UI_DEBUG); const window = new electron.BrowserWindow({ show: false, width: ui_debug ? 1200 : 600, @@ -736,7 +736,7 @@ export async function execute_graphical(channel: string, ask_install: boolean) : set_text("Loading data"); let version: UpdateVersion; try { - version = await minawait(newest_version(process_args.has_flag(Arguments.UPDATER_ENFORCE) ? undefined : current_vers, channel), 3000); + version = await minawait(newest_version(processArguments.has_flag(Arguments.UPDATER_ENFORCE) ? undefined : current_vers, channel), 3000); } catch (error) { set_error("Failed to get newest information:
" + error); await await_exit(); @@ -815,7 +815,7 @@ export async function execute_graphical(channel: string, ask_install: boolean) : try { await execute_update(update_path, callback => { - reference_app(); /* we'll never delete this reference, but we'll call app.quit() manually */ + referenceApp(); /* we'll never delete this reference, but we'll call app.quit() manually */ update_restart_pending = true; window.close(); callback(); @@ -873,5 +873,5 @@ export function stop_auto_update_check() { } export async function selected_channel() : Promise { - return process_args.has_value(Arguments.UPDATER_CHANNEL) ? process_args.value(Arguments.UPDATER_CHANNEL) : "release"; + return processArguments.has_value(Arguments.UPDATER_CHANNEL) ? processArguments.value(Arguments.UPDATER_CHANNEL) : "release"; } \ No newline at end of file diff --git a/modules/core/main_window.ts b/modules/core/main-window/index.ts similarity index 51% rename from modules/core/main_window.ts rename to modules/core/main-window/index.ts index e5d9fcc..57b6042 100644 --- a/modules/core/main_window.ts +++ b/modules/core/main-window/index.ts @@ -1,35 +1,32 @@ -import {BrowserWindow, Menu, MenuItem, MessageBoxOptions, app, dialog} from "electron"; +import {BrowserWindow, app, dialog} from "electron"; import * as path from "path"; -let app_references = 0; -export function reference_app() { - app_references++; -} - -export function unreference_app() { - app_references--; - test_app_should_exit(); -} - export let is_debug: boolean; export let allow_dev_tools: boolean; -import {Arguments, parse_arguments, process_args} from "../shared/process-arguments"; -import * as updater from "./app-updater"; -import * as loader from "./ui-loader"; -import * as crash_handler from "../crash_handler"; +import {Arguments, processArguments} from "../../shared/process-arguments"; +import * as updater from "./../app-updater"; +import * as loader from "./../ui-loader"; import * as url from "url"; -import {loadWindowBounds, startTrackWindowBounds} from "../shared/window"; +import {loadWindowBounds, startTrackWindowBounds} from "../../shared/window"; +import {referenceApp, dereferenceApp} from "../AppInstance"; // Keep a global reference of the window object, if you don't, the window will // be closed automatically when the JavaScript object is garbage collected. -export let main_window: BrowserWindow = null; +export let mainWindow: BrowserWindow = null; + +function spawnMainWindow(rendererEntryPoint: string) { + app.on('certificate-error', (event, webContents, url, error, certificate, callback) => { + console.log("Allowing untrusted certificate for %o", url); + event.preventDefault(); + callback(true); + }); -function spawn_main_window(entry_point: string) { // Create the browser window. console.log("Spawning main window"); - reference_app(); /* main browser window references the app */ - main_window = new BrowserWindow({ + + referenceApp(); /* main browser window references the app */ + mainWindow = new BrowserWindow({ width: 800, height: 600, @@ -40,38 +37,42 @@ function spawn_main_window(entry_point: string) { webPreferences: { webSecurity: false, nodeIntegrationInWorker: true, - nodeIntegration: true + nodeIntegration: true, + preload: path.join(__dirname, "preload.js") }, - icon: path.join(__dirname, "..", "..", "resources", "logo.ico") + icon: path.join(__dirname, "..", "..", "resources", "logo.ico"), }); - main_window.webContents.on('devtools-closed', event => { + mainWindow.webContents.on('devtools-closed', () => { console.log("Dev tools destroyed!"); }); - main_window.on('closed', () => { + mainWindow.on('closed', () => { app.releaseSingleInstanceLock(); - require("./url-preview").close(); - main_window = null; + require("../url-preview").close(); + mainWindow = null; - unreference_app(); + dereferenceApp(); }); - main_window.loadURL(url.pathToFileURL(loader.ui.preloading_page(entry_point)).toString()); + mainWindow.loadURL(url.pathToFileURL(loader.ui.preloading_page(rendererEntryPoint)).toString()).catch(error => { + console.error("Failed to load UI entry point: %o", error); + handleUILoadingError("UI entry point failed to load"); + }); - main_window.once('ready-to-show', () => { - main_window.show(); - loadWindowBounds('main-window', main_window).then(() => { - startTrackWindowBounds('main-window', main_window); + mainWindow.once('ready-to-show', () => { + mainWindow.show(); + loadWindowBounds('main-window', mainWindow).then(() => { + startTrackWindowBounds('main-window', mainWindow); - main_window.focus(); + mainWindow.focus(); loader.ui.cleanup(); - if(allow_dev_tools && !main_window.webContents.isDevToolsOpened()) - main_window.webContents.openDevTools(); + if(allow_dev_tools && !mainWindow.webContents.isDevToolsOpened()) + mainWindow.webContents.openDevTools(); }); }); - main_window.webContents.on('new-window', (event, url_str, frameName, disposition, options, additionalFeatures) => { + mainWindow.webContents.on('new-window', (event, url_str, frameName, disposition, options, additionalFeatures) => { if(frameName.startsWith("__modal_external__")) { return; } @@ -101,79 +102,44 @@ function spawn_main_window(entry_point: string) { } }); - main_window.webContents.on('crashed', event => { + mainWindow.webContents.on('crashed', () => { console.error("UI thread crashed! Closing app!"); - if(!process_args.has_flag(Arguments.DEBUG)) - main_window.close(); + if(!processArguments.has_flag(Arguments.DEBUG)) { + mainWindow.close(); + } }); } -function handle_ui_load_error(message: string) { +function handleUILoadingError(message: string) { + referenceApp(); + console.log("Caught loading error: %s", message); - //"A critical error happened while loading TeaClient!", "A critical error happened while loading TeaClient!
" + message - reference_app(); + if(mainWindow) { + mainWindow.close(); + mainWindow = undefined; + } + dialog.showMessageBox({ type: "error", buttons: ["exit"], title: "A critical error happened while loading TeaClient!", message: (message || "no error").toString() - }).then(unreference_app); + }).then(dereferenceApp); loader.ui.cancel(); } -function test_app_should_exit() { - if(app_references > 0) return; - - console.log("All windows have been closed, closing app."); - app.quit(); -} - -function init_listener() { - app.on('quit', () => { - console.debug("Shutting down app."); - crash_handler.finalize_handler(); - loader.ui.cleanup(); - console.log("App has been finalized."); - }); - - - app.on('window-all-closed', () => { - console.log("All windows have been closed. App reference count: %d", app_references); - test_app_should_exit(); - }); - - app.on('activate', () => { - // On macOS it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (main_window === null) { - //spawn_loading_screen(); - //createWindow() - } - }); - - app.on('certificate-error', (event, webContents, url, error, certificate, callback) => { - console.log("Allowing untrusted certificate for %o", url); - event.preventDefault(); - callback(true); - }); -} - export function execute() { console.log("Main app executed!"); - parse_arguments(); - is_debug = process_args.has_flag(...Arguments.DEBUG); - allow_dev_tools = process_args.has_flag(...Arguments.DEV_TOOLS); + is_debug = processArguments.has_flag(...Arguments.DEBUG); + allow_dev_tools = processArguments.has_flag(...Arguments.DEV_TOOLS); if(is_debug) { console.log("Enabled debug!"); - console.log("Arguments: %o", process_args); + console.log("Arguments: %o", processArguments); } - Menu.setApplicationMenu(null); - init_listener(); - console.log("Setting up render backend"); - require("./render-backend"); + require("../render-backend"); console.log("Spawn loading screen"); loader.ui.execute_loader().then(async (entry_point: string) => { @@ -193,14 +159,14 @@ export function execute() { return entry_point; }).then((entry_point: string) => { - reference_app(); /* because we've no windows when we close the loader UI */ + referenceApp(); /* because we've no windows when we close the loader UI */ loader.ui.cleanup(); /* close the window */ if(entry_point) //has not been canceled - spawn_main_window(entry_point); + spawnMainWindow(entry_point); else { - handle_ui_load_error("Missing UI entry point"); + handleUILoadingError("Missing UI entry point"); } - unreference_app(); - }).catch(handle_ui_load_error); + dereferenceApp(); + }).catch(handleUILoadingError); } diff --git a/modules/core/main-window/preload.ts b/modules/core/main-window/preload.ts new file mode 100644 index 0000000..20b6814 --- /dev/null +++ b/modules/core/main-window/preload.ts @@ -0,0 +1,12 @@ +/* preloaded script, init hook will be called before the loader will be executed */ +declare global { + interface Window { + __native_client_init_hook: () => void; + __native_client_init_shared: (webpackRequire: any) => void; + } +} + +window.__native_client_init_hook = () => require("../../renderer/index"); +window.__native_client_init_shared = webpackRequire => window["shared-require"] = webpackRequire; + +export = {}; \ No newline at end of file diff --git a/modules/core/main.ts b/modules/core/main.ts index 6989c14..55be823 100644 --- a/modules/core/main.ts +++ b/modules/core/main.ts @@ -1,26 +1,29 @@ -// Quit when all windows are closed. import * as electron from "electron"; import * as app_updater from "./app-updater"; -import * as instance_handler from "./instance_handler"; -import { app } from "electron"; +import {app, Menu} from "electron"; import MessageBoxOptions = electron.MessageBoxOptions; -import {process_args, parse_arguments, Arguments} from "../shared/process-arguments"; +import {processArguments, parseProcessArguments, Arguments} from "../shared/process-arguments"; import {open as open_changelog} from "./app-updater/changelog"; import * as crash_handler from "../crash_handler"; +import {initializeSingleInstance} from "./MultiInstanceHandler"; -async function execute_app() { - if(process_args.has_value("update-execute")) { - console.log("Executing update " + process_args.value("update-execute")); - await app_updater.execute_update(process_args.value("update-execute"), callback => { +import "./AppInstance"; + +async function handleAppReady() { + Menu.setApplicationMenu(null); + + if(processArguments.has_value("update-execute")) { + console.log("Executing update " + processArguments.value("update-execute")); + await app_updater.execute_update(processArguments.value("update-execute"), callback => { console.log("Update preconfig successful. Extracting update. (The client should start automatically)"); app.quit(); setImmediate(callback); }); return; - } else if(process_args.has_value("update-failed-new") || process_args.has_value("update-succeed-new")) { - const success = process_args.has_value("update-succeed-new"); + } else if(processArguments.has_value("update-failed-new") || processArguments.has_value("update-succeed-new")) { + const success = processArguments.has_value("update-succeed-new"); let data: { parse_success: boolean; log_file?: string; @@ -30,7 +33,7 @@ async function execute_app() { parse_success: false }; try { - let encoded_data = Buffer.from(process_args.value("update-failed-new") || process_args.value("update-succeed-new"), "base64").toString(); + let encoded_data = Buffer.from(processArguments.value("update-failed-new") || processArguments.value("update-succeed-new"), "base64").toString(); for(const part of encoded_data.split(";")) { const index = part.indexOf(':'); if(index == -1) @@ -45,9 +48,9 @@ async function execute_app() { } console.log("Update success: %o. Update data: %o", success, data); - let title = ""; - let type = ""; - let message = ""; + let title; + let type; + let message; const buttons: ({ key: string, @@ -129,22 +132,22 @@ async function execute_app() { /* register client a teaclient:// handler */ if(app.getAppPath().endsWith(".asar")) { - if(!app.setAsDefaultProtocolClient("teaclient", app.getPath("exe"))) + if(!app.setAsDefaultProtocolClient("teaclient", app.getPath("exe"))) { console.error("Failed to setup default teaclient protocol handler"); + } } try { - { - const version = await app_updater.current_version(); - global["app_version_client"] = version.toString(); - } - const main = require("./main_window"); + const version = await app_updater.current_version(); + global["app_version_client"] = version.toString(); + + const main = require("./main-window"); main.execute(); app_updater.start_auto_update_check(); } catch (error) { - console.dir(error); - const result = electron.dialog.showMessageBox({ + console.error(error); + await electron.dialog.showMessageBox({ type: "error", message: "Failed to execute app main!\n" + error, title: "Main execution failed!", @@ -155,26 +158,26 @@ async function execute_app() { } function main() { - //setTimeout(() => process.crash(), 1000); - - if('allowRendererProcessReuse' in app) + if('allowRendererProcessReuse' in app) { app.allowRendererProcessReuse = false; + } - parse_arguments(); - if(process_args.has_value(Arguments.DISABLE_HARDWARE_ACCELERATION)) + parseProcessArguments(); + if(processArguments.has_value(Arguments.DISABLE_HARDWARE_ACCELERATION)) { app.disableHardwareAcceleration(); + } - if(process_args.has_value(Arguments.DUMMY_CRASH_MAIN)) + if(processArguments.has_value(Arguments.DUMMY_CRASH_MAIN)) { crash_handler.handler.crash(); + } - if(!process_args.has_value(Arguments.DEBUG) && !process_args.has_value(Arguments.NO_SINGLE_INSTANCE)) { - if(!app.requestSingleInstanceLock()) { + if(!processArguments.has_value(Arguments.DEBUG) && !processArguments.has_value(Arguments.NO_SINGLE_INSTANCE)) { + if(!initializeSingleInstance()) { console.log("Another instance is already running. Closing this instance"); app.exit(0); } - - app.on('second-instance', (event, argv, workingDirectory) => instance_handler.handle_second_instance_call(argv, workingDirectory)); } - app.on('ready', execute_app); + + app.on('ready', handleAppReady); } export const execute = main; \ No newline at end of file diff --git a/modules/core/render-backend/ExternalModal.ts b/modules/core/render-backend/ExternalModal.ts index 84d9549..a202058 100644 --- a/modules/core/render-backend/ExternalModal.ts +++ b/modules/core/render-backend/ExternalModal.ts @@ -3,7 +3,7 @@ import {ExternalModal, kIPCChannelExternalModal} from "../../shared/ipc/External import {ProxiedClass} from "../../shared/proxy/Definitions"; import {BrowserWindow, dialog} from "electron"; import {loadWindowBounds, startTrackWindowBounds} from "../../shared/window"; -import {Arguments, process_args} from "../../shared/process-arguments"; +import {Arguments, processArguments} from "../../shared/process-arguments"; import {open_preview} from "../url-preview"; import * as path from "path"; @@ -33,6 +33,7 @@ class ProxyImplementation extends ProxiedClass implements Externa webPreferences: { nodeIntegration: true, + preload: path.join(__dirname, "..", "..", "renderer-manifest", "preload.js") }, icon: path.join(__dirname, "..", "..", "resources", "logo.ico"), minWidth: 600, @@ -77,7 +78,7 @@ class ProxyImplementation extends ProxiedClass implements Externa } }); - if(process_args.has_flag(Arguments.DEV_TOOLS)) + if(processArguments.has_flag(Arguments.DEV_TOOLS)) this.windowInstance.webContents.openDevTools(); try { diff --git a/modules/core/render-backend/index.ts b/modules/core/render-backend/index.ts index 279086e..7f2b67c 100644 --- a/modules/core/render-backend/index.ts +++ b/modules/core/render-backend/index.ts @@ -6,10 +6,8 @@ import BrowserWindow = electron.BrowserWindow; import {open as open_changelog} from "../app-updater/changelog"; import * as updater from "../app-updater"; -import {execute_connect_urls} from "../instance_handler"; -import {process_args} from "../../shared/process-arguments"; -import {open_preview} from "../url-preview"; -import {dialog} from "electron"; +import {execute_connect_urls} from "../MultiInstanceHandler"; +import {processArguments} from "../../shared/process-arguments"; import "./ExternalModal"; @@ -17,7 +15,7 @@ ipcMain.on('basic-action', (event, action, ...args: any[]) => { const window = BrowserWindow.fromWebContents(event.sender); if(action == "parse-connect-arguments") { - execute_connect_urls(process_args["_"] || []); + execute_connect_urls(processArguments["_"] || []); } else if(action === "open-changelog") { open_changelog(); } else if(action === "check-native-update") { diff --git a/modules/core/ui-loader/graphical.ts b/modules/core/ui-loader/graphical.ts index 6f24e94..59ca93c 100644 --- a/modules/core/ui-loader/graphical.ts +++ b/modules/core/ui-loader/graphical.ts @@ -2,7 +2,7 @@ import * as electron from "electron"; import * as path from "path"; import {screen} from "electron"; -import {Arguments, process_args} from "../../shared/process-arguments"; +import {Arguments, processArguments} from "../../shared/process-arguments"; import * as loader from "./loader"; import * as updater from "../app-updater"; import * as url from "url"; @@ -60,7 +60,7 @@ export namespace ui { console.error("Received error from loader after it had been closed... Error: %o", error); }; }; - if(!process_args.has_flag(...Arguments.DISABLE_ANIMATION)) + if(!processArguments.has_flag(...Arguments.DISABLE_ANIMATION)) setTimeout(resolved, 250); else setImmediate(resolved); @@ -133,7 +133,7 @@ export namespace ui { startTrackWindowBounds('ui-load-window', gui); const call_loader = () => load_files().catch(reject); - if(!process_args.has_flag(...Arguments.DISABLE_ANIMATION)) + if(!processArguments.has_flag(...Arguments.DISABLE_ANIMATION)) setTimeout(call_loader, 1000); else setImmediate(call_loader); diff --git a/modules/core/ui-loader/loader.ts b/modules/core/ui-loader/loader.ts index dd2b339..7e8dc29 100644 --- a/modules/core/ui-loader/loader.ts +++ b/modules/core/ui-loader/loader.ts @@ -1,4 +1,4 @@ -import {is_debug} from "../main_window"; +import {is_debug} from "../main-window"; import * as moment from "moment"; import * as request from "request"; import * as querystring from "querystring"; @@ -8,7 +8,7 @@ const UUID = require('pure-uuid'); import * as path from "path"; import * as zlib from "zlib"; import * as tar from "tar-stream"; -import {Arguments, process_args} from "../../shared/process-arguments"; +import {Arguments, processArguments} from "../../shared/process-arguments"; import {parse_version} from "../../shared/version"; import * as electron from "electron"; @@ -27,7 +27,7 @@ const remote_url: RemoteURL = () => { if(remote_url.cached) return remote_url.cached; const default_path = is_debug ? "http://localhost/home/TeaSpeak/Web-Client/client-api/environment/" : "https://clientapi.teaspeak.de/"; - return remote_url.cached = (process_args.has_value(...Arguments.SERVER_URL) ? process_args.value(...Arguments.SERVER_URL) : default_path); + return remote_url.cached = (processArguments.has_value(...Arguments.SERVER_URL) ? processArguments.value(...Arguments.SERVER_URL) : default_path); }; export interface VersionedFile { @@ -82,7 +82,7 @@ function get_raw_app_files() : Promise { } if(response.statusCode != 200) { setImmediate(reject, "invalid status code " + response.statusCode + " for " + url); return; } - if(parseInt(response.headers["info-version"] as string) != 1 && !process_args.has_flag(Arguments.UPDATER_UI_IGNORE_VERSION)) { setImmediate(reject, "Invalid response version (" + response.headers["info-version"] + "). Update your app manually!"); return; } + if(parseInt(response.headers["info-version"] as string) != 1 && !processArguments.has_flag(Arguments.UPDATER_UI_IGNORE_VERSION)) { setImmediate(reject, "Invalid response version (" + response.headers["info-version"] + "). Update your app manually!"); return; } if(!body) { setImmediate(reject, "invalid body. (Missing)"); return; @@ -461,7 +461,7 @@ async function load_cached_or_remote_ui_pack(channel: string, stats_update: (mes const required_version = parse_version(e.pack_info.min_client_version); return client_version.in_dev() || client_version.newer_than(required_version) || client_version.equals(required_version); }); - if(process_args.has_flag(Arguments.UPDATER_UI_NO_CACHE)) { + if(processArguments.has_flag(Arguments.UPDATER_UI_NO_CACHE)) { console.log("Ignoring local UI cache"); available_versions = []; } @@ -574,7 +574,7 @@ enum UILoaderMethod { } export async function load_files(channel: string, stats_update: (message: string, index: number) => any) : Promise { - let enforced_loading_method = parseInt(process_args.has_value(Arguments.UPDATER_UI_LOAD_TYPE) ? process_args.value(Arguments.UPDATER_UI_LOAD_TYPE) : "-1") as UILoaderMethod; + let enforced_loading_method = parseInt(processArguments.has_value(Arguments.UPDATER_UI_LOAD_TYPE) ? processArguments.value(Arguments.UPDATER_UI_LOAD_TYPE) : "-1") as UILoaderMethod; if(typeof UILoaderMethod[enforced_loading_method] !== "undefined") { switch (enforced_loading_method) { diff --git a/modules/renderer-manifest/index.ts b/modules/renderer-manifest/index.ts index 9918f7a..baa2a99 100644 --- a/modules/renderer-manifest/index.ts +++ b/modules/renderer-manifest/index.ts @@ -1,19 +1,19 @@ /* --------------- bootstrap --------------- */ import * as RequireProxy from "../renderer/RequireProxy"; import * as path from "path"; -RequireProxy.initialize(path.join(__dirname, "backend-impl")); +RequireProxy.initialize(path.join(__dirname, "backend-impl"), "modal-external"); /* --------------- entry point --------------- */ import * as loader from "tc-loader"; import {Stage} from "tc-loader"; -import {Arguments, process_args} from "../shared/process-arguments"; +import {Arguments, processArguments} from "../shared/process-arguments"; import {remote} from "electron"; -export function initialize(manifestTarget: string) { - console.log("Initializing native client for manifest target %s", manifestTarget); +export function initialize() { + console.log("Initializing native client"); const _impl = message => { - if(!process_args.has_flag(Arguments.DEBUG)) { + if(!processArguments.has_flag(Arguments.DEBUG)) { console.error("Displaying critical error: %o", message); message = message.replace(/
/i, "\n"); @@ -33,10 +33,11 @@ export function initialize(manifestTarget: string) { } }; - if(window.impl_display_critical_error) + if(window.impl_display_critical_error) { window.impl_display_critical_error = _impl; - else + } else { window.displayCriticalError = _impl; + } loader.register_task(loader.Stage.JAVASCRIPT, { name: "teaclient jquery", @@ -51,10 +52,11 @@ export function initialize(manifestTarget: string) { loader.register_task(Stage.JAVASCRIPT_INITIALIZING, { name: "handler initialize", - priority: 100, + priority: 80, function: async () => { await import("../renderer/Logger"); await import("../renderer/PersistentLocalStorage"); + await import("../renderer/ContextMenu"); } }) } \ No newline at end of file diff --git a/modules/renderer-manifest/preload.ts b/modules/renderer-manifest/preload.ts new file mode 100644 index 0000000..94683f7 --- /dev/null +++ b/modules/renderer-manifest/preload.ts @@ -0,0 +1,12 @@ +/* preloaded script, init hook will be called before the loader will be executed */ +declare global { + interface Window { + __native_client_init_hook: () => void; + __native_client_init_shared: (webpackRequire: any) => void; + } +} + +window.__native_client_init_hook = () => require("./index").initialize(); +window.__native_client_init_shared = webpackRequire => window["shared-require"] = webpackRequire; + +export = {}; \ No newline at end of file diff --git a/modules/renderer/ContextMenu.ts b/modules/renderer/ContextMenu.ts new file mode 100644 index 0000000..ed000a7 --- /dev/null +++ b/modules/renderer/ContextMenu.ts @@ -0,0 +1,64 @@ +import {ContextMenuEntry, ContextMenuFactory, setGlobalContextMenuFactory} from "tc-shared/ui/ContextMenu"; +import * as electron from "electron"; +import {MenuItemConstructorOptions} from "electron"; +import {clientIconClassToImage} from "./IconHelper"; +const {Menu} = electron.remote; + +let currentMenu: electron.Menu; + +function mapMenuEntry(entry: ContextMenuEntry) : MenuItemConstructorOptions { + switch (entry.type) { + case "normal": + return { + type: "normal", + label: typeof entry.label === "string" ? entry.label : entry.label.text, + enabled: entry.enabled, + visible: entry.visible, + click: entry.click, + icon: typeof entry.icon === "string" ? clientIconClassToImage(entry.icon) : undefined, + id: entry.uniqueId, + submenu: entry.subMenu ? entry.subMenu.map(mapMenuEntry).filter(e => !!e) : undefined + }; + + case "checkbox": + return { + type: "normal", + label: typeof entry.label === "string" ? entry.label : entry.label.text, + enabled: entry.enabled, + visible: entry.visible, + click: entry.click, + id: entry.uniqueId, + + checked: entry.checked + }; + + case "separator": + return { + type: "separator" + }; + + default: + return undefined; + } +} + +setGlobalContextMenuFactory(new class implements ContextMenuFactory { + closeContextMenu() { + currentMenu?.closePopup(); + currentMenu = undefined; + } + + spawnContextMenu(position: { pageX: number; pageY: number }, entries: ContextMenuEntry[], callbackClose?: () => void) { + this.closeContextMenu(); + currentMenu = Menu.buildFromTemplate(entries.map(mapMenuEntry).filter(e => !!e)); + currentMenu.popup({ + callback: () => { + callbackClose(); + currentMenu = undefined; + }, + x: position.pageX, + y: position.pageY, + window: electron.remote.BrowserWindow.getFocusedWindow() + }); + } +}); \ No newline at end of file diff --git a/modules/renderer/IconHelper.ts b/modules/renderer/IconHelper.ts index 6a54e6c..f491b57 100644 --- a/modules/renderer/IconHelper.ts +++ b/modules/renderer/IconHelper.ts @@ -29,7 +29,7 @@ loader.register_task(Stage.JAVASCRIPT_INITIALIZING, { name: "native icon sprite loader", function: async () => { const image = new Image(); - image.src = kClientSpriteUrl; + image.src = loader.config.baseUrl + kClientSpriteUrl; await new Promise((resolve, reject) => { image.onload = resolve; image.onerror = () => reject("failed to load client icon sprite"); diff --git a/modules/renderer/MenuBarHandler.ts b/modules/renderer/MenuBarHandler.ts index f7e58ac..5c23550 100644 --- a/modules/renderer/MenuBarHandler.ts +++ b/modules/renderer/MenuBarHandler.ts @@ -1,7 +1,7 @@ import {clientIconClassToImage} from "./IconHelper"; import * as electron from "electron"; import * as mbar from "tc-shared/ui/frames/MenuBar"; -import {Arguments, process_args} from "../shared/process-arguments"; +import {Arguments, processArguments} from "../shared/process-arguments"; import ipcRenderer = electron.ipcRenderer; import {LocalIcon} from "tc-shared/file/Icons"; @@ -239,7 +239,7 @@ mbar.native_actions = { call_basic_action("reload-window") }, - show_dev_tools() { return process_args.has_flag(Arguments.DEV_TOOLS); } + show_dev_tools() { return processArguments.has_flag(Arguments.DEV_TOOLS); } }; diff --git a/modules/renderer/RequireProxy.ts b/modules/renderer/RequireProxy.ts index 76d83e5..3ee0538 100644 --- a/modules/renderer/RequireProxy.ts +++ b/modules/renderer/RequireProxy.ts @@ -49,8 +49,10 @@ namespace proxied_load { } let backend_root: string; -export function initialize(backend_root_: string) { +let app_module: string; +export function initialize(backend_root_: string, app_module_: string) { backend_root = backend_root_; + app_module = app_module_; proxied_load.original_load = Module._load; Module._load = proxied_load; @@ -95,13 +97,17 @@ overrides.push({ }); function resolveModuleMapping(context: string, resource: string) { - if(context.endsWith("/")) + if(context.endsWith("/")) { context = context.substring(0, context.length - 1); + } const loader = require("tc-loader"); - const mapping = loader.module_mapping().find(e => e.application === "client-app"); //FIXME: Variable name! - if(!mapping) throw "missing mapping"; + const mapping = loader.module_mapping().find(e => e.application === app_module); //FIXME: Variable name! + if(!mapping) { + debugger; + throw "missing ui module mapping"; + } const entries = mapping.modules.filter(e => e.context === context); if(!entries.length) { diff --git a/modules/renderer/UnloadHandler.ts b/modules/renderer/UnloadHandler.ts index 1d469f3..eed546b 100644 --- a/modules/renderer/UnloadHandler.ts +++ b/modules/renderer/UnloadHandler.ts @@ -1,6 +1,6 @@ import {Settings, settings} from "tc-shared/settings"; import {tr} from "tc-shared/i18n/localize"; -import {Arguments, process_args} from "../shared/process-arguments"; +import {Arguments, processArguments} from "../shared/process-arguments"; import {remote} from "electron"; import {server_connections} from "tc-shared/ConnectionManager"; @@ -36,7 +36,7 @@ window.onbeforeunload = event => { } }; - if(process_args.has_flag(Arguments.DEBUG)) { + if(processArguments.has_flag(Arguments.DEBUG)) { do_exit(false); return; } diff --git a/modules/renderer/audio/AudioRecorder.ts b/modules/renderer/audio/AudioRecorder.ts index a2cf0d2..b0eb294 100644 --- a/modules/renderer/audio/AudioRecorder.ts +++ b/modules/renderer/audio/AudioRecorder.ts @@ -18,10 +18,12 @@ import {LogCategory, logWarn} from "tc-shared/log"; import NativeFilterMode = audio.record.FilterMode; export class NativeInput implements AbstractInput { + static readonly instances = [] as NativeInput[]; + readonly events: Registry; - private nativeHandle: audio.record.AudioRecorder; - private nativeConsumer: audio.record.AudioConsumer; + readonly nativeHandle: audio.record.AudioRecorder; + readonly nativeConsumer: audio.record.AudioConsumer; private state: InputState; private deviceId: string | undefined; @@ -35,6 +37,9 @@ export class NativeInput implements AbstractInput { this.nativeHandle = audio.record.create_recorder(); this.nativeConsumer = this.nativeHandle.create_consumer(); + this.nativeConsumer.toggle_rnnoise(true); + (window as any).consumer = this.nativeConsumer; /* FIXME! */ + this.nativeConsumer.callback_ended = () => { this.filtered = true; this.events.fire("notify_voice_end"); @@ -45,6 +50,14 @@ export class NativeInput implements AbstractInput { }; this.state = InputState.PAUSED; + NativeInput.instances.push(this); + } + + destroy() { + const index = NativeInput.instances.indexOf(this); + if(index !== -1) { + NativeInput.instances.splice(index, 1); + } } async start(): Promise { @@ -214,29 +227,33 @@ export class NativeInput implements AbstractInput { } export class NativeLevelMeter implements LevelMeter { - readonly _device: IDevice; + static readonly instances: NativeLevelMeter[] = []; + readonly targetDevice: IDevice; - private _callback: (num: number) => any; - private _recorder: audio.record.AudioRecorder; - private _consumer: audio.record.AudioConsumer; - private _filter: audio.record.ThresholdConsumeFilter; + public nativeRecorder: audio.record.AudioRecorder; + public nativeConsumer: audio.record.AudioConsumer; + + private callback: (num: number) => any; + private nativeFilter: audio.record.ThresholdConsumeFilter; constructor(device: IDevice) { - this._device = device; + this.targetDevice = device; } async initialize() { try { - this._recorder = audio.record.create_recorder(); - this._consumer = this._recorder.create_consumer(); + this.nativeRecorder = audio.record.create_recorder(); + this.nativeConsumer = this.nativeRecorder.create_consumer(); - this._filter = this._consumer.create_filter_threshold(.5); - this._filter.set_attack_smooth(.75); - this._filter.set_release_smooth(.75); + this.nativeConsumer.toggle_rnnoise(true); /* FIXME! */ - await new Promise(resolve => this._recorder.set_device(this._device.deviceId, resolve)); + this.nativeFilter = this.nativeConsumer.create_filter_threshold(.5); + this.nativeFilter.set_attack_smooth(.75); + this.nativeFilter.set_release_smooth(.75); + + await new Promise(resolve => this.nativeRecorder.set_device(this.targetDevice.deviceId, resolve)); await new Promise((resolve, reject) => { - this._recorder.start(flag => { + this.nativeRecorder.start(flag => { if (typeof flag === "boolean" && flag) resolve(); else @@ -246,40 +263,46 @@ export class NativeLevelMeter implements LevelMeter { } catch (error) { if (typeof (error) === "string") throw error; - console.warn(tr("Failed to initialize levelmeter for device %o: %o"), this._device, error); + console.warn(tr("Failed to initialize levelmeter for device %o: %o"), this.targetDevice, error); throw "initialize failed (lookup console)"; } /* references this variable, needs a destory() call, else memory leak */ - this._filter.set_analyze_filter(value => { - (this._callback || (() => { - }))(value); + this.nativeFilter.set_analyze_filter(value => { + if(this.callback) this.callback(value); }); + + NativeLevelMeter.instances.push(this); } destroy() { - if (this._filter) { - this._filter.set_analyze_filter(undefined); - this._consumer.unregister_filter(this._filter); + const index = NativeLevelMeter.instances.indexOf(this); + if(index !== -1) { + NativeLevelMeter.instances.splice(index, 1); } - if (this._consumer) { - this._recorder.delete_consumer(this._consumer); + if (this.nativeFilter) { + this.nativeFilter.set_analyze_filter(undefined); + this.nativeConsumer.unregister_filter(this.nativeFilter); } - if(this._recorder) { - this._recorder.stop(); + if (this.nativeConsumer) { + this.nativeRecorder.delete_consumer(this.nativeConsumer); } - this._recorder = undefined; - this._consumer = undefined; - this._filter = undefined; + + if(this.nativeRecorder) { + this.nativeRecorder.stop(); + } + this.nativeRecorder = undefined; + this.nativeConsumer = undefined; + this.nativeFilter = undefined; } getDevice(): IDevice { - return this._device; + return this.targetDevice; } setObserver(callback: (value: number) => any) { - this._callback = callback; + this.callback = callback; } } \ No newline at end of file diff --git a/modules/renderer/connection/VoiceConnection.ts b/modules/renderer/connection/VoiceConnection.ts index 69f42af..8a32788 100644 --- a/modules/renderer/connection/VoiceConnection.ts +++ b/modules/renderer/connection/VoiceConnection.ts @@ -12,7 +12,7 @@ import {NativeInput} from "../audio/AudioRecorder"; import {ConnectionState} from "tc-shared/ConnectionHandler"; import {VoicePlayerEvents, VoicePlayerLatencySettings, VoicePlayerState} from "tc-shared/voice/VoicePlayer"; import {Registry} from "tc-shared/events"; -import {LogCategory, logInfo, logWarn} from "tc-shared/log"; +import {LogCategory, logDebug, logInfo, logWarn} from "tc-shared/log"; import {tr} from "tc-shared/i18n/localize"; export class NativeVoiceConnectionWrapper extends AbstractVoiceConnection { @@ -83,11 +83,15 @@ export class NativeVoiceConnectionWrapper extends AbstractVoiceConnection { } if(this.currentRecorder) { + this.currentRecorder.callback_unmount = undefined; + this.native.set_audio_source(undefined); + + this.handleVoiceEndEvent(); await this.currentRecorder.unmount(); + this.currentRecorder = undefined; } - this.handleVoiceEndEvent(); - + await recorder?.unmount(); this.currentRecorder = recorder; try { @@ -97,14 +101,10 @@ export class NativeVoiceConnectionWrapper extends AbstractVoiceConnection { throw "Recorder input must be an instance of NativeInput!"; } - await recorder.unmount(); - recorder.current_handler = this.connection.client; - recorder.callback_unmount = () => { - this.currentRecorder = undefined; - this.native.set_audio_source(undefined); - this.handleVoiceEndEvent(); + logDebug(LogCategory.VOICE, tr("Lost voice recorder...")); + this.acquireVoiceRecorder(undefined); }; recorder.callback_start = this.handleVoiceStartEvent.bind(this); @@ -116,7 +116,7 @@ export class NativeVoiceConnectionWrapper extends AbstractVoiceConnection { this.currentRecorder = undefined; throw error; } - this.events.fire("notify_recorder_changed", {}) + this.events.fire("notify_recorder_changed", {}); } voiceRecorder(): RecorderProfile { @@ -259,6 +259,8 @@ class NativeVoiceClientWrapper implements VoiceClient { break; } } + + this.resetLatencySettings(); } destroy() { diff --git a/modules/renderer/context-menu.ts b/modules/renderer/context-menu.ts deleted file mode 100644 index 4e3f803..0000000 --- a/modules/renderer/context-menu.ts +++ /dev/null @@ -1,128 +0,0 @@ -import {clientIconClassToImage} from "./IconHelper"; -import * as contextmenu from "tc-shared/ui/elements/ContextMenu"; -import * as electron from "electron"; -const remote = electron.remote; -const {Menu, MenuItem} = remote; - -class ElectronContextMenu implements contextmenu.ContextMenuProvider { - private _close_listeners: (() => any)[] = []; - private _current_menu: electron.Menu; - - private _div: JQuery; - - despawn_context_menu() { - if(!this._current_menu) - return; - this._current_menu.closePopup(); - this._current_menu = undefined; - - for(const listener of this._close_listeners) { - if(listener) { - try { - listener(); - } catch (e) { - console.error("Failed to call context menu close listener: %o", e); - } - } - } - this._close_listeners = []; - } - - finalize() { - if(this._div) this._div.detach(); - this._div = undefined; - } - - initialize() { - } - - - private _entry_id = 0; - private build_menu(entry: contextmenu.MenuEntry) : electron.MenuItem { - if(entry.type == contextmenu.MenuEntryType.CLOSE) { - this._close_listeners.push(entry.callback); - return undefined; - } - - const click_callback = () => { - if(entry.callback) - entry.callback(); - this.despawn_context_menu(); - }; - const _id = "entry_" + (this._entry_id++); - if(entry.type == contextmenu.MenuEntryType.ENTRY) { - return new MenuItem({ - id: _id, - label: (typeof entry.name === "function" ? (entry.name as (() => string))() : entry.name) as string, - type: "normal", - click: click_callback, - icon: clientIconClassToImage(entry.icon_class), - visible: entry.visible, - enabled: !entry.disabled - }); - } else if(entry.type == contextmenu.MenuEntryType.HR) { - if(typeof(entry.visible) === "boolean" && !entry.visible) - return undefined; - - return new MenuItem({ - id: _id, - type: "separator", - label: '', - click: click_callback - }) - } else if(entry.type == contextmenu.MenuEntryType.CHECKBOX) { - return new MenuItem({ - id: _id, - label: (typeof entry.name === "function" ? (entry.name as (() => string))() : entry.name) as string, - type: "checkbox", - checked: !!entry.checkbox_checked, - click: click_callback, - icon: clientIconClassToImage(entry.icon_class), - visible: entry.visible, - enabled: !entry.disabled - }); - } else if (entry.type == contextmenu.MenuEntryType.SUB_MENU) { - const sub_menu = new Menu(); - for(const e of entry.sub_menu) { - const build = this.build_menu(e); - if(!build) - continue; - sub_menu.append(build); - } - return new MenuItem({ - id: _id, - label: (typeof entry.name === "function" ? (entry.name as (() => string))() : entry.name) as string, - type: "submenu", - submenu: sub_menu, - click: click_callback, - icon: clientIconClassToImage(entry.icon_class), - visible: entry.visible, - enabled: !entry.disabled - }); - } - return undefined; - } - - spawn_context_menu(x: number, y: number, ...entries: contextmenu.MenuEntry[]) { - this.despawn_context_menu(); - - this._current_menu = new Menu(); - for(const entry of entries) { - const build = this.build_menu(entry); - if(!build) - continue; - this._current_menu.append(build); - } - - this._current_menu.popup({ - window: remote.getCurrentWindow(), - x: x, - y: y, - callback: () => this.despawn_context_menu() - }); - } - - html_format_enabled() { return false; } -} - -contextmenu.set_provider(new ElectronContextMenu()); \ No newline at end of file diff --git a/modules/renderer/hooks/AudioInput.ts b/modules/renderer/hooks/AudioInput.ts index ecd07b0..e568fcd 100644 --- a/modules/renderer/hooks/AudioInput.ts +++ b/modules/renderer/hooks/AudioInput.ts @@ -17,4 +17,13 @@ setRecorderBackend(new class implements AudioRecorderBacked { getDeviceList(): DeviceList { return inputDeviceList; } + + isRnNoiseSupported(): boolean { + return true; + } + + toggleRnNoise(target: boolean) { + NativeLevelMeter.instances.forEach(input => input.nativeConsumer.toggle_rnnoise(target)); + NativeInput.instances.forEach(input => input.nativeConsumer.toggle_rnnoise(target)); + } }); \ No newline at end of file diff --git a/modules/renderer/index.ts b/modules/renderer/index.ts index f3c55da..4303ca1 100644 --- a/modules/renderer/index.ts +++ b/modules/renderer/index.ts @@ -1,13 +1,13 @@ /* --------------- bootstrap --------------- */ import * as RequireProxy from "./RequireProxy"; import * as crash_handler from "../crash_handler"; -import {Arguments, parse_arguments, process_args} from "../shared/process-arguments"; +import {Arguments, parseProcessArguments, processArguments} from "../shared/process-arguments"; import * as path from "path"; -import * as Sentry from "@sentry/electron"; import * as electron from "electron"; import {remote} from "electron"; /* +import * as Sentry from "@sentry/electron"; Sentry.init({ dsn: "https://72b9f40ce6894b179154e7558f1aeb87@o437344.ingest.sentry.io/5399791", appName: "TeaSpeak - Client", @@ -16,13 +16,13 @@ Sentry.init({ */ /* first of all setup crash handler */ -parse_arguments(); -if(!process_args.has_flag(Arguments.NO_CRASH_RENDERER)) { +parseProcessArguments(); +if(!processArguments.has_flag(Arguments.NO_CRASH_RENDERER)) { const is_electron_run = process.argv[0].endsWith("electron") || process.argv[0].endsWith("electron.exe"); crash_handler.initialize_handler("renderer", is_electron_run); } -RequireProxy.initialize(path.join(__dirname, "backend-impl")); +RequireProxy.initialize(path.join(__dirname, "backend-impl"), "client-app"); /* --------------- main initialize --------------- */ import * as loader from "tc-loader"; @@ -73,7 +73,7 @@ loader.register_task(loader.Stage.INITIALIZING, { name: "teaclient initialize error", function: async () => { const _impl = message => { - if(!process_args.has_flag(Arguments.DEBUG)) { + if(!processArguments.has_flag(Arguments.DEBUG)) { console.error("Displaying critical error: %o", message); message = message.replace(/
/i, "\n"); @@ -92,10 +92,11 @@ loader.register_task(loader.Stage.INITIALIZING, { } }; - if(window.impl_display_critical_error) + if(window.impl_display_critical_error) { window.impl_display_critical_error = _impl; - else + } else { window.displayCriticalError = _impl; + } }, priority: 100 }); @@ -103,14 +104,14 @@ loader.register_task(loader.Stage.INITIALIZING, { loader.register_task(loader.Stage.INITIALIZING, { name: "teaclient initialize arguments", function: async () => { - if(process_args.has_value(Arguments.DUMMY_CRASH_RENDERER)) + if(processArguments.has_value(Arguments.DUMMY_CRASH_RENDERER)) crash_handler.handler.crash(); /* loader url setup */ { - const baseUrl = process_args.value(Arguments.SERVER_URL); - console.error(process_args.value(Arguments.UPDATER_UI_LOAD_TYPE)); - if(typeof baseUrl === "string" && parseFloat((process_args.value(Arguments.UPDATER_UI_LOAD_TYPE)?.toString() || "").trim()) === 3) { + const baseUrl = processArguments.value(Arguments.SERVER_URL); + console.error(processArguments.value(Arguments.UPDATER_UI_LOAD_TYPE)); + if(typeof baseUrl === "string" && parseFloat((processArguments.value(Arguments.UPDATER_UI_LOAD_TYPE)?.toString() || "").trim()) === 3) { loader.config.baseUrl = baseUrl; } } @@ -121,7 +122,7 @@ loader.register_task(loader.Stage.INITIALIZING, { loader.register_task(loader.Stage.INITIALIZING, { name: 'gdb-waiter', function: async () => { - if(process_args.has_flag(Arguments.DEV_TOOLS_GDB)) { + if(processArguments.has_flag(Arguments.DEV_TOOLS_GDB)) { console.log("Process ID: %d", process.pid); await new Promise(resolve => { console.log("Waiting for continue!"); @@ -154,7 +155,7 @@ loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { try { await import("./version"); await import("./MenuBarHandler"); - await import("./context-menu"); + await import("./ContextMenu"); await import("./SingleInstanceHandler"); await import("./IconHelper"); await import("./connection/FileTransfer"); @@ -174,6 +175,4 @@ loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { remote.getCurrentWindow().on('focus', () => remote.getCurrentWindow().flashFrame(false)); }, priority: 60 -}); - -export async function initialize() { } \ No newline at end of file +}); \ No newline at end of file diff --git a/modules/shared/process-arguments/index.ts b/modules/shared/process-arguments/index.ts index 3fcd2b4..28e409e 100644 --- a/modules/shared/process-arguments/index.ts +++ b/modules/shared/process-arguments/index.ts @@ -29,12 +29,12 @@ export interface Window { process_args: Arguments; } -export const process_args: Arguments = {} as Arguments; +export const processArguments: Arguments = {} as Arguments; -export function parse_arguments() { +export function parseProcessArguments() { if(!process || !process.type || process.type === 'renderer') { - Object.assign(process_args, electron.remote.getGlobal("process_arguments")); - (window as any).process_args = process_args; + Object.assign(processArguments, electron.remote.getGlobal("process_arguments")); + (window as any).process_args = processArguments; } else { const is_electron_run = process.argv[0].endsWith("electron") || process.argv[0].endsWith("electron.exe"); @@ -46,15 +46,15 @@ export function parse_arguments() { }) as Arguments; args.has_flag = (...keys) => { for(const key of [].concat(...Array.of(...keys).map(e => Array.isArray(e) ? Array.of(...e) : [e]))) - if(typeof process_args[key as any as string] === "boolean") - return process_args[key as any as string]; + if(typeof processArguments[key as any as string] === "boolean") + return processArguments[key as any as string]; return false; }; args.value = (...keys) => { for(const key of [].concat(...Array.of(...keys).map(e => Array.isArray(e) ? Array.of(...e) : [e]))) - if(typeof process_args[key] !== "undefined") - return process_args[key]; + if(typeof processArguments[key] !== "undefined") + return processArguments[key]; return undefined; }; @@ -78,11 +78,11 @@ export function parse_arguments() { } } console.log("Parsed CMD arguments %o as %o", process.argv, args); - Object.assign(process_args, args); + Object.assign(processArguments, args); Object.assign(global["process_arguments"] = {}, args); } - if(process_args.has_flag("help", "h")) { + if(processArguments.has_flag("help", "h")) { console.log("TeaClient command line help page"); console.log(" -h --help => Displays this page"); console.log(" -d --debug => Enabled the application debug"); diff --git a/native/serverconnection/CMakeLists.txt b/native/serverconnection/CMakeLists.txt index 6fcdb09..ddcffbd 100644 --- a/native/serverconnection/CMakeLists.txt +++ b/native/serverconnection/CMakeLists.txt @@ -109,11 +109,14 @@ include_directories(${TeaSpeak_SharedLib_INCLUDE_DIR}) find_package(StringVariable REQUIRED) include_directories(${StringVariable_INCLUDE_DIR}) -find_package(Ed25519 REQUIRED) +find_package(ed25519 REQUIRED) include_directories(${ed25519_INCLUDE_DIR}) find_package(ThreadPool REQUIRED) include_directories(${ThreadPool_INCLUDE_DIR}) + +find_package(rnnoise REQUIRED) + if (WIN32) add_compile_options(/NODEFAULTLIB:ThreadPoolStatic) add_definitions(-DWINDOWS) #Required by ThreadPool @@ -121,13 +124,13 @@ if (WIN32) add_definitions(-D_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING) # For the FMT library endif () -find_package(Soxr REQUIRED) +find_package(soxr REQUIRED) include_directories(${soxr_INCLUDE_DIR}) find_package(fvad REQUIRED) include_directories(${fvad_INCLUDE_DIR}) -find_package(Opus REQUIRED) +find_package(opus REQUIRED) include_directories(${opus_INCLUDE_DIR}) find_package(spdlog REQUIRED) @@ -148,6 +151,7 @@ set(REQUIRED_LIBRARIES ${opus_LIBRARIES_STATIC} ${ed25519_LIBRARIES_STATIC} + rnnoise spdlog::spdlog_header_only Nan::Helpers diff --git a/native/serverconnection/exports/exports.d.ts b/native/serverconnection/exports/exports.d.ts index c45c77b..60affdd 100644 --- a/native/serverconnection/exports/exports.d.ts +++ b/native/serverconnection/exports/exports.d.ts @@ -223,9 +223,9 @@ declare module "tc-native/connection" { } export interface AudioConsumer { - sample_rate: number; - channels: number; - frame_size: number; + readonly sampleRate: number; + readonly channelCount: number; + readonly frameSize: number; /* TODO add some kind of order to may improve CPU performance (Some filters are more intense then others) */ get_filters() : ConsumeFilter[]; @@ -238,6 +238,9 @@ declare module "tc-native/connection" { set_filter_mode(mode: FilterMode); get_filter_mode() : FilterMode; + toggle_rnnoise(enabled: boolean); + rnnoise_enabled() : boolean; + callback_data: (buffer: Float32Array) => any; callback_ended: () => any; callback_started: () => any; diff --git a/native/serverconnection/src/audio/AudioInput.cpp b/native/serverconnection/src/audio/AudioInput.cpp index 1ad5570..7ce46c5 100644 --- a/native/serverconnection/src/audio/AudioInput.cpp +++ b/native/serverconnection/src/audio/AudioInput.cpp @@ -33,10 +33,11 @@ void AudioConsumer::handle_framed_data(const void *buffer, size_t samples) { } void AudioConsumer::process_data(const void *buffer, size_t samples) { - if(this->reframer) - this->reframer->process(buffer, samples); - else - this->handle_framed_data(buffer, samples); + if(this->reframer) { + this->reframer->process(buffer, samples); + } else { + this->handle_framed_data(buffer, samples); + } } AudioInput::AudioInput(size_t channels, size_t rate) : _channel_count(channels), _sample_rate(rate) {} @@ -51,7 +52,7 @@ AudioInput::~AudioInput() { } void AudioInput::set_device(const std::shared_ptr &device) { - std::lock_guard lock(this->input_source_lock); + std::lock_guard lock{this->input_source_lock}; if(device == this->input_device) return; this->close_device(); @@ -59,7 +60,7 @@ void AudioInput::set_device(const std::shared_ptr &device) { } void AudioInput::close_device() { - std::lock_guard lock(this->input_source_lock); + std::lock_guard lock{this->input_source_lock}; if(this->input_recorder) { this->input_recorder->remove_consumer(this); this->input_recorder->stop_if_possible(); @@ -70,7 +71,7 @@ void AudioInput::close_device() { } bool AudioInput::record(std::string& error) { - std::lock_guard lock(this->input_source_lock); + std::lock_guard lock{this->input_source_lock}; if(!this->input_device) { error = "no device"; return false; diff --git a/native/serverconnection/src/audio/js/AudioConsumer.cpp b/native/serverconnection/src/audio/js/AudioConsumer.cpp index b5dd62a..b0295a8 100644 --- a/native/serverconnection/src/audio/js/AudioConsumer.cpp +++ b/native/serverconnection/src/audio/js/AudioConsumer.cpp @@ -1,3 +1,4 @@ +#include #include "AudioConsumer.h" #include "AudioRecorder.h" #include "AudioFilter.h" @@ -12,6 +13,10 @@ using namespace std; using namespace tc::audio; using namespace tc::audio::recorder; +inline v8::PropertyAttribute operator|(const v8::PropertyAttribute& a, const v8::PropertyAttribute& b) { + return (v8::PropertyAttribute) ((unsigned) a | (unsigned) b); +} + NAN_MODULE_INIT(AudioConsumerWrapper::Init) { auto klass = Nan::New(AudioConsumerWrapper::NewInstance); klass->SetClassName(Nan::New("AudioConsumer").ToLocalChecked()); @@ -27,6 +32,9 @@ NAN_MODULE_INIT(AudioConsumerWrapper::Init) { Nan::SetPrototypeMethod(klass, "get_filter_mode", AudioConsumerWrapper::_get_filter_mode); Nan::SetPrototypeMethod(klass, "set_filter_mode", AudioConsumerWrapper::_set_filter_mode); + Nan::SetPrototypeMethod(klass, "rnnoise_enabled", AudioConsumerWrapper::rnnoise_enabled); + Nan::SetPrototypeMethod(klass, "toggle_rnnoise", AudioConsumerWrapper::toggle_rnnoise); + constructor_template().Reset(klass); constructor().Reset(Nan::GetFunction(klass).ToLocalChecked()); } @@ -39,7 +47,7 @@ NAN_METHOD(AudioConsumerWrapper::NewInstance) { AudioConsumerWrapper::AudioConsumerWrapper(AudioRecorderWrapper* h, const std::shared_ptr &handle) : _handle(handle), _recorder(h) { log_allocate("AudioConsumerWrapper", this); { - lock_guard read_lock(handle->on_read_lock); + lock_guard read_lock{handle->on_read_lock}; handle->on_read = [&](const void* buffer, size_t length){ this->process_data(buffer, length); }; } @@ -51,16 +59,32 @@ AudioConsumerWrapper::AudioConsumerWrapper(AudioRecorderWrapper* h, const std::s AudioConsumerWrapper::~AudioConsumerWrapper() { log_free("AudioConsumerWrapper", this); - lock_guard lock(this->execute_lock); + lock_guard lock{this->execute_mutex}; this->unbind(); if(this->_handle->handle) { this->_handle->handle->delete_consumer(this->_handle); this->_handle = nullptr; } + for(auto& instance : this->rnnoise_processor) { + if(!instance) { continue; } + + rnnoise_destroy((DenoiseState*) instance); + instance = nullptr; + } + + for(auto index{0}; index < kInternalFrameBufferCount; index++) { + if(!this->internal_frame_buffer[index]) { continue; } + + free(this->internal_frame_buffer[index]); + this->internal_frame_buffer[index] = nullptr; + this->internal_frame_buffer_size[index] = 0; + } + #ifdef DO_DEADLOCK_REF - if(this->_recorder) + if(this->_recorder) { this->_recorder->js_unref(); + } #endif } @@ -119,20 +143,88 @@ void AudioConsumerWrapper::do_wrap(const v8::Local &obj) { (void) callback_function.As()->Call(Nan::GetCurrentContext(), Nan::Undefined(), 0, nullptr); }); - Nan::Set(this->handle(), Nan::New("frame_size").ToLocalChecked(), Nan::New((uint32_t) this->_handle->frame_size)); - Nan::Set(this->handle(), Nan::New("sample_rate").ToLocalChecked(), Nan::New((uint32_t) this->_handle->sample_rate)); - Nan::Set(this->handle(), Nan::New("channels").ToLocalChecked(), Nan::New((uint32_t) this->_handle->channel_count)); + Nan::DefineOwnProperty(this->handle(), Nan::New("frameSize").ToLocalChecked(), Nan::New((uint32_t) this->_handle->frame_size), v8::ReadOnly | v8::DontDelete); + Nan::DefineOwnProperty(this->handle(), Nan::New("sampleRate").ToLocalChecked(), Nan::New((uint32_t) this->_handle->sample_rate), v8::ReadOnly | v8::DontDelete); + Nan::DefineOwnProperty(this->handle(), Nan::New("channelCount").ToLocalChecked(), Nan::New((uint32_t) this->_handle->channel_count), v8::ReadOnly | v8::DontDelete); } void AudioConsumerWrapper::unbind() { if(this->_handle) { - lock_guard lock(this->_handle->on_read_lock); + lock_guard lock{this->_handle->on_read_lock}; this->_handle->on_read = nullptr; } } +static const float kRnNoiseScale = -INT16_MIN; void AudioConsumerWrapper::process_data(const void *buffer, size_t samples) { - lock_guard lock(this->execute_lock); + if(samples != 960) { + logger::error(logger::category::audio, tr("Received audio frame with invalid sample count (Expected 960, Received {})"), samples); + return; + } + + lock_guard lock{this->execute_mutex}; + if(this->filter_mode_ == FilterMode::BLOCK) { return; } + + /* apply input modifiers */ + if(this->rnnoise) { + /* TODO: don't call reserve_internal_buffer every time and assume the buffers are initialized */ + /* TODO: Maybe find out if the microphone is some kind of pseudo stero so we can handle it as mono? */ + + if(this->_handle->channel_count > 1) { + auto channel_count = this->_handle->channel_count; + this->reserve_internal_buffer(0, samples * channel_count * sizeof(float)); + this->reserve_internal_buffer(1, samples * channel_count * sizeof(float)); + + for(size_t channel{0}; channel < channel_count; channel++) { + auto target_buffer = (float*) this->internal_frame_buffer[1]; + auto source_buffer = (const float*) buffer + channel; + + for(size_t index{0}; index < samples; index++) { + *target_buffer = *source_buffer * kRnNoiseScale; + source_buffer += channel_count; + target_buffer++; + } + + /* rnnoise uses a frame size of 480 */ + this->initialize_rnnoise(channel); + rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[channel], (float*) this->internal_frame_buffer[0] + channel * samples, (const float*) this->internal_frame_buffer[1]); + rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[channel], (float*) this->internal_frame_buffer[0] + channel * samples + 480, (const float*) this->internal_frame_buffer[1] + 480); + } + + const float* channel_buffer_ptr[kMaxChannelCount]; + for(size_t channel{0}; channel < channel_count; channel++) { + channel_buffer_ptr[channel] = (const float*) this->internal_frame_buffer[0] + channel * samples; + } + + /* now back again to interlanced */ + auto target_buffer = (float*) this->internal_frame_buffer[1]; + for(size_t index{0}; index < samples; index++) { + for(size_t channel{0}; channel < channel_count; channel++) { + *target_buffer = *(channel_buffer_ptr[channel]++) / kRnNoiseScale; + target_buffer++; + } + } + + buffer = this->internal_frame_buffer[1]; + } else { + /* rnnoise uses a frame size of 480 */ + this->reserve_internal_buffer(0, samples * sizeof(float)); + + auto target_buffer = (float*) this->internal_frame_buffer[0]; + for(size_t index{0}; index < samples; index++) { + target_buffer[index] = ((float*) buffer)[index] * kRnNoiseScale; + } + + this->initialize_rnnoise(0); + rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[0], target_buffer, target_buffer); + rnnoise_process_frame((DenoiseState*) this->rnnoise_processor[0], &target_buffer[480], &target_buffer[480]); + + buffer = target_buffer; + for(size_t index{0}; index < samples; index++) { + target_buffer[index] /= kRnNoiseScale; + } + } + } bool should_process{true}; if(this->filter_mode_ == FilterMode::FILTER) { @@ -243,11 +335,26 @@ void AudioConsumerWrapper::delete_filter(const AudioFilterWrapper* filter) { } { - lock_guard lock(this->execute_lock); /* ensure that the filter isn't used right now */ + lock_guard lock(this->execute_mutex); /* ensure that the filter isn't used right now */ handle->_filter = nullptr; } } +void AudioConsumerWrapper::reserve_internal_buffer(int index, size_t target) { + assert(index < kInternalFrameBufferCount); + if(this->internal_frame_buffer_size[index] < target) { + if(this->internal_frame_buffer_size[index]) { ::free(this->internal_frame_buffer[index]); } + + this->internal_frame_buffer[index] = malloc(target); + this->internal_frame_buffer_size[index] = target; + } +} + +void AudioConsumerWrapper::initialize_rnnoise(int channel) { + if(!this->rnnoise_processor[channel]) { + this->rnnoise_processor[channel] = rnnoise_create(nullptr); + } +} NAN_METHOD(AudioConsumerWrapper::_get_filters) { auto handle = ObjectWrap::Unwrap(info.Holder()); @@ -352,4 +459,20 @@ NAN_METHOD(AudioConsumerWrapper::_set_filter_mode) { auto value = info[0].As()->ToInteger()->Value(); handle->filter_mode_ = (FilterMode) value; +} + +NAN_METHOD(AudioConsumerWrapper::rnnoise_enabled) { + auto handle = ObjectWrap::Unwrap(info.Holder()); + info.GetReturnValue().Set(handle->rnnoise); +} + +NAN_METHOD(AudioConsumerWrapper::toggle_rnnoise) { + auto handle = ObjectWrap::Unwrap(info.Holder()); + + if(info.Length() != 1 || !info[0]->IsBoolean()) { + Nan::ThrowError("invalid argument"); + return; + } + + handle->rnnoise = info[0]->BooleanValue(); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioConsumer.h b/native/serverconnection/src/audio/js/AudioConsumer.h index 45bd2b3..a31302c 100644 --- a/native/serverconnection/src/audio/js/AudioConsumer.h +++ b/native/serverconnection/src/audio/js/AudioConsumer.h @@ -1,104 +1,117 @@ #pragma once +#include #include #include #include #include -namespace tc { - namespace audio { - class AudioInput; - class AudioConsumer; +namespace tc::audio { + class AudioInput; + class AudioConsumer; - namespace filter { - class Filter; - } + namespace filter { + class Filter; + } - namespace recorder { - class AudioFilterWrapper; - class AudioRecorderWrapper; + namespace recorder { + class AudioFilterWrapper; + class AudioRecorderWrapper; - enum FilterMode { - BYPASS, - FILTER, - BLOCK - }; + enum FilterMode { + BYPASS, + FILTER, + BLOCK + }; - class AudioConsumerWrapper : public Nan::ObjectWrap { - friend class AudioRecorderWrapper; - public: - static NAN_MODULE_INIT(Init); - static NAN_METHOD(NewInstance); - static inline Nan::Persistent & constructor() { - static Nan::Persistent my_constructor; - return my_constructor; - } + class AudioConsumerWrapper : public Nan::ObjectWrap { + friend class AudioRecorderWrapper; + constexpr static auto kMaxChannelCount{2}; + public: + static NAN_MODULE_INIT(Init); + static NAN_METHOD(NewInstance); + static inline Nan::Persistent & constructor() { + static Nan::Persistent my_constructor; + return my_constructor; + } - static inline Nan::Persistent & constructor_template() { - static Nan::Persistent my_constructor_template; - return my_constructor_template; - } + static inline Nan::Persistent & constructor_template() { + static Nan::Persistent my_constructor_template; + return my_constructor_template; + } - AudioConsumerWrapper(AudioRecorderWrapper*, const std::shared_ptr& /* handle */); - ~AudioConsumerWrapper() override; + AudioConsumerWrapper(AudioRecorderWrapper*, const std::shared_ptr& /* handle */); + ~AudioConsumerWrapper() override; - static NAN_METHOD(_get_filters); - static NAN_METHOD(_unregister_filter); + static NAN_METHOD(_get_filters); + static NAN_METHOD(_unregister_filter); - static NAN_METHOD(_create_filter_vad); - static NAN_METHOD(_create_filter_threshold); - static NAN_METHOD(_create_filter_state); + static NAN_METHOD(_create_filter_vad); + static NAN_METHOD(_create_filter_threshold); + static NAN_METHOD(_create_filter_state); - static NAN_METHOD(_get_filter_mode); - static NAN_METHOD(_set_filter_mode); + static NAN_METHOD(_get_filter_mode); + static NAN_METHOD(_set_filter_mode); - std::shared_ptr create_filter(const std::string& /* name */, const std::shared_ptr& /* filter impl */); - void delete_filter(const AudioFilterWrapper*); + static NAN_METHOD(toggle_rnnoise); + static NAN_METHOD(rnnoise_enabled); - inline std::deque> filters() { - std::lock_guard lock(this->filter_mutex_); - return this->filter_; - } + std::shared_ptr create_filter(const std::string& /* name */, const std::shared_ptr& /* filter impl */); + void delete_filter(const AudioFilterWrapper*); - inline FilterMode filter_mode() const { return this->filter_mode_; } + inline std::deque> filters() { + std::lock_guard lock(this->filter_mutex_); + return this->filter_; + } - inline std::shared_ptr native_consumer() { return this->_handle; } + inline FilterMode filter_mode() const { return this->filter_mode_; } + inline std::shared_ptr native_consumer() { return this->_handle; } - std::mutex native_read_callback_lock; - std::function native_read_callback; - private: - AudioRecorderWrapper* _recorder; + std::mutex native_read_callback_lock; + std::function native_read_callback; + private: + AudioRecorderWrapper* _recorder; - std::mutex execute_lock; - std::shared_ptr _handle; + /* preprocessors */ + bool rnnoise{false}; + std::array rnnoise_processor{nullptr}; - std::mutex filter_mutex_; - std::deque> filter_; - FilterMode filter_mode_{FilterMode::FILTER}; - bool last_consumed = false; + std::mutex execute_mutex; + std::shared_ptr _handle; - void do_wrap(const v8::Local& /* object */); + std::mutex filter_mutex_; + std::deque> filter_; + FilterMode filter_mode_{FilterMode::FILTER}; + bool last_consumed = false; - void unbind(); /* called with execute_lock locked */ - void process_data(const void* /* buffer */, size_t /* samples */); + constexpr static auto kInternalFrameBufferCount{2}; + void* internal_frame_buffer[kInternalFrameBufferCount]{nullptr}; + size_t internal_frame_buffer_size[kInternalFrameBufferCount]{0}; - struct DataEntry { - void* buffer = nullptr; - size_t sample_count = 0; + void do_wrap(const v8::Local& /* object */); - ~DataEntry() { - if(buffer) - free(buffer); - } - }; + void unbind(); /* called with execute_lock locked */ + void process_data(const void* /* buffer */, size_t /* samples */); - std::mutex _data_lock; - std::deque> _data_entries; + void reserve_internal_buffer(int /* buffer */, size_t /* bytes */); + void initialize_rnnoise(int /* channel */); - Nan::callback_t<> _call_data; - Nan::callback_t<> _call_ended; - Nan::callback_t<> _call_started; - }; - } - } + struct DataEntry { + void* buffer = nullptr; + size_t sample_count = 0; + + ~DataEntry() { + if(buffer) + free(buffer); + } + }; + + std::mutex _data_lock; + std::deque> _data_entries; + + Nan::callback_t<> _call_data; + Nan::callback_t<> _call_ended; + Nan::callback_t<> _call_started; + }; + } } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioFilter.h b/native/serverconnection/src/audio/js/AudioFilter.h index fafa435..13b2ee7 100644 --- a/native/serverconnection/src/audio/js/AudioFilter.h +++ b/native/serverconnection/src/audio/js/AudioFilter.h @@ -3,67 +3,65 @@ #include #include -namespace tc { - namespace audio { - namespace filter { - class Filter; - } +namespace tc::audio { + namespace filter { + class Filter; + } - namespace recorder { - class AudioConsumerWrapper; + namespace recorder { + class AudioConsumerWrapper; - class AudioFilterWrapper : public Nan::ObjectWrap { - friend class AudioConsumerWrapper; - public: - static NAN_MODULE_INIT(Init); - static NAN_METHOD(NewInstance); - static inline Nan::Persistent & constructor() { - static Nan::Persistent my_constructor; - return my_constructor; - } - static inline Nan::Persistent & constructor_template() { - static Nan::Persistent my_constructor_template; - return my_constructor_template; - } + class AudioFilterWrapper : public Nan::ObjectWrap { + friend class AudioConsumerWrapper; + public: + static NAN_MODULE_INIT(Init); + static NAN_METHOD(NewInstance); + static inline Nan::Persistent & constructor() { + static Nan::Persistent my_constructor; + return my_constructor; + } + static inline Nan::Persistent & constructor_template() { + static Nan::Persistent my_constructor_template; + return my_constructor_template; + } - AudioFilterWrapper(const std::string& name, const std::shared_ptr& /* handle */); - ~AudioFilterWrapper() override; + AudioFilterWrapper(const std::string& name, const std::shared_ptr& /* handle */); + ~AudioFilterWrapper() override; - static NAN_METHOD(_get_name); + static NAN_METHOD(_get_name); - /* VAD and Threshold */ - static NAN_METHOD(_get_margin_time); - static NAN_METHOD(_set_margin_time); + /* VAD and Threshold */ + static NAN_METHOD(_get_margin_time); + static NAN_METHOD(_set_margin_time); - /* VAD relevant */ - static NAN_METHOD(_get_level); + /* VAD relevant */ + static NAN_METHOD(_get_level); - /* threshold filter relevant */ - static NAN_METHOD(_get_threshold); - static NAN_METHOD(_set_threshold); + /* threshold filter relevant */ + static NAN_METHOD(_get_threshold); + static NAN_METHOD(_set_threshold); - static NAN_METHOD(_get_attack_smooth); - static NAN_METHOD(_set_attack_smooth); + static NAN_METHOD(_get_attack_smooth); + static NAN_METHOD(_set_attack_smooth); - static NAN_METHOD(_get_release_smooth); - static NAN_METHOD(_set_release_smooth); + static NAN_METHOD(_get_release_smooth); + static NAN_METHOD(_set_release_smooth); - static NAN_METHOD(_set_analyze_filter); + static NAN_METHOD(_set_analyze_filter); - /* consume filter */ - static NAN_METHOD(_is_consuming); - static NAN_METHOD(_set_consuming); + /* consume filter */ + static NAN_METHOD(_is_consuming); + static NAN_METHOD(_set_consuming); - inline std::shared_ptr filter() { return this->_filter; } - private: - std::shared_ptr _filter; - std::string _name; + inline std::shared_ptr filter() { return this->_filter; } + private: + std::shared_ptr _filter; + std::string _name; - void do_wrap(const v8::Local& /* object */); + void do_wrap(const v8::Local& /* object */); - Nan::callback_t _call_analyzed; - Nan::Persistent _callback_analyzed; - }; - } - } + Nan::callback_t _call_analyzed; + Nan::Persistent _callback_analyzed; + }; + } } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioRecorder.cpp b/native/serverconnection/src/audio/js/AudioRecorder.cpp index d356cc7..b4703b7 100644 --- a/native/serverconnection/src/audio/js/AudioRecorder.cpp +++ b/native/serverconnection/src/audio/js/AudioRecorder.cpp @@ -19,7 +19,7 @@ NAN_METHOD(recorder::create_recorder) { Nan::ThrowError(tr("audio hasn't been initialized yet")); return; } - auto input = make_shared(2, 48000); + auto input = std::make_shared(2, 48000); auto wrapper = new AudioRecorderWrapper(input); auto js_object = Nan::NewInstance(Nan::New(AudioRecorderWrapper::constructor())).ToLocalChecked(); wrapper->do_wrap(js_object); @@ -55,24 +55,24 @@ NAN_METHOD(AudioRecorderWrapper::NewInstance) { } -AudioRecorderWrapper::AudioRecorderWrapper(std::shared_ptr handle) : _input(std::move(handle)) { +AudioRecorderWrapper::AudioRecorderWrapper(std::shared_ptr handle) : input_(std::move(handle)) { log_allocate("AudioRecorderWrapper", this); } AudioRecorderWrapper::~AudioRecorderWrapper() { - if(this->_input) { - this->_input->stop(); - this->_input->close_device(); - this->_input = nullptr; + if(this->input_) { + this->input_->stop(); + this->input_->close_device(); + this->input_ = nullptr; } { - lock_guard lock(this->_consumer_lock); - this->_consumers.clear(); + lock_guard lock{this->consumer_mutex}; + this->consumer_.clear(); } log_free("AudioRecorderWrapper", this); } std::shared_ptr AudioRecorderWrapper::create_consumer() { - auto result = shared_ptr(new AudioConsumerWrapper(this, this->_input->create_consumer(960)), [](AudioConsumerWrapper* ptr) { + auto result = shared_ptr(new AudioConsumerWrapper(this, this->input_->create_consumer(960)), [](AudioConsumerWrapper* ptr) { assert(v8::Isolate::GetCurrent()); ptr->Unref(); }); @@ -85,8 +85,8 @@ std::shared_ptr AudioRecorderWrapper::create_consumer() { } { - lock_guard lock(this->_consumer_lock); - this->_consumers.push_back(result); + lock_guard lock(this->consumer_mutex); + this->consumer_.push_back(result); } return result; @@ -95,8 +95,8 @@ std::shared_ptr AudioRecorderWrapper::create_consumer() { void AudioRecorderWrapper::delete_consumer(const AudioConsumerWrapper* consumer) { shared_ptr handle; /* need to keep the handle 'till everything has been finished */ { - lock_guard lock(this->_consumer_lock); - for(auto& c : this->_consumers) { + lock_guard lock(this->consumer_mutex); + for(auto& c : this->consumer_) { if(&*c == consumer) { handle = c; break; @@ -106,16 +106,16 @@ void AudioRecorderWrapper::delete_consumer(const AudioConsumerWrapper* consumer) return; { - auto it = find(this->_consumers.begin(), this->_consumers.end(), handle); - if(it != this->_consumers.end()) - this->_consumers.erase(it); + auto it = find(this->consumer_.begin(), this->consumer_.end(), handle); + if(it != this->consumer_.end()) + this->consumer_.erase(it); } } { - lock_guard lock(handle->execute_lock); /* if we delete the consumer while executing strange stuff could happen */ + lock_guard lock(handle->execute_mutex); /* if we delete the consumer while executing strange stuff could happen */ handle->unbind(); - this->_input->delete_consumer(handle->_handle); + this->input_->delete_consumer(handle->_handle); } } @@ -125,7 +125,7 @@ void AudioRecorderWrapper::do_wrap(const v8::Local &obj) { NAN_METHOD(AudioRecorderWrapper::_get_device) { auto handle = ObjectWrap::Unwrap(info.Holder()); - auto input = handle->_input; + auto input = handle->input_; auto device = input->current_device(); if(device) @@ -136,7 +136,7 @@ NAN_METHOD(AudioRecorderWrapper::_get_device) { NAN_METHOD(AudioRecorderWrapper::_set_device) { auto handle = ObjectWrap::Unwrap(info.Holder()); - auto input = handle->_input; + auto input = handle->input_; const auto is_null_device = info[0]->IsNullOrUndefined(); if(info.Length() != 2 || !(is_null_device || info[0]->IsString()) || !info[1]->IsFunction()) { @@ -190,7 +190,7 @@ NAN_METHOD(AudioRecorderWrapper::_start) { return; } - auto input = ObjectWrap::Unwrap(info.Holder())->_input; + auto input = ObjectWrap::Unwrap(info.Holder())->input_; std::string error{}; v8::Local argv[1]; @@ -204,14 +204,14 @@ NAN_METHOD(AudioRecorderWrapper::_start) { NAN_METHOD(AudioRecorderWrapper::_started) { auto handle = ObjectWrap::Unwrap(info.Holder()); - auto input = handle->_input; + auto input = handle->input_; info.GetReturnValue().Set(input->recording()); } NAN_METHOD(AudioRecorderWrapper::_stop) { auto handle = ObjectWrap::Unwrap(info.Holder()); - auto input = handle->_input; + auto input = handle->input_; input->stop(); } @@ -265,10 +265,10 @@ NAN_METHOD(AudioRecorderWrapper::_set_volume) { return; } - handle->_input->set_volume((float) info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0)); + handle->input_->set_volume((float) info[0]->NumberValue(Nan::GetCurrentContext()).FromMaybe(0)); } NAN_METHOD(AudioRecorderWrapper::_get_volume) { auto handle = ObjectWrap::Unwrap(info.Holder()); - info.GetReturnValue().Set(handle->_input->volume()); + info.GetReturnValue().Set(handle->input_->volume()); } \ No newline at end of file diff --git a/native/serverconnection/src/audio/js/AudioRecorder.h b/native/serverconnection/src/audio/js/AudioRecorder.h index be99bfe..25e7844 100644 --- a/native/serverconnection/src/audio/js/AudioRecorder.h +++ b/native/serverconnection/src/audio/js/AudioRecorder.h @@ -44,8 +44,8 @@ namespace tc::audio { void delete_consumer(const AudioConsumerWrapper*); inline std::deque> consumers() { - std::lock_guard lock(this->_consumer_lock); - return this->_consumers; + std::lock_guard lock{this->consumer_mutex}; + return this->consumer_; } void do_wrap(const v8::Local& /* obj */); @@ -53,10 +53,11 @@ namespace tc::audio { inline void js_ref() { this->Ref(); } inline void js_unref() { this->Unref(); } private: - std::shared_ptr _input; + std::shared_ptr input_; - std::mutex _consumer_lock; - std::deque> _consumers; + /* javascript consumer */ + std::mutex consumer_mutex; + std::deque> consumer_; }; } } \ No newline at end of file diff --git a/native/serverconnection/src/connection/audio/AudioSender.cpp b/native/serverconnection/src/connection/audio/AudioSender.cpp index c914bd7..d9f68ad 100644 --- a/native/serverconnection/src/connection/audio/AudioSender.cpp +++ b/native/serverconnection/src/connection/audio/AudioSender.cpp @@ -42,8 +42,9 @@ bool VoiceSender::initialize_codec(std::string& error, connection::codec::value data->converter->reset_encoder(); } - if(!data->resampler || data->resampler->input_rate() != rate) - data->resampler = make_shared(rate, data->converter->sample_rate(), data->converter->channels()); + if(!data->resampler || data->resampler->input_rate() != rate) { + data->resampler = make_shared(rate, data->converter->sample_rate(), data->converter->channels()); + } if(!data->resampler->valid()) { error = "resampler is invalid"; @@ -58,7 +59,7 @@ void VoiceSender::set_voice_send_enabled(bool flag) { } void VoiceSender::send_data(const void *data, size_t samples, size_t rate, size_t channels) { - unique_lock lock(this->_execute_lock); + unique_lock lock{this->_execute_lock}; if(!this->handle) { log_warn(category::voice_connection, tr("Dropping raw audio frame because of an invalid handle.")); return; diff --git a/native/serverconnection/src/connection/audio/VoiceConnection.cpp b/native/serverconnection/src/connection/audio/VoiceConnection.cpp index f60a68e..3f09b33 100644 --- a/native/serverconnection/src/connection/audio/VoiceConnection.cpp +++ b/native/serverconnection/src/connection/audio/VoiceConnection.cpp @@ -182,7 +182,7 @@ NAN_METHOD(VoiceConnectionWrap::set_audio_source) { auto sample_rate = native_consumer->sample_rate; auto channels = native_consumer->channel_count; - lock_guard read_lock(connection->_voice_recoder_ptr->native_read_callback_lock); + lock_guard read_lock{connection->_voice_recoder_ptr->native_read_callback_lock}; connection->_voice_recoder_ptr->native_read_callback = [weak_handle, sample_rate, channels](const void* buffer, size_t length) { auto handle = weak_handle.lock(); if(!handle) { diff --git a/native/serverconnection/test/js/main.ts b/native/serverconnection/test/js/main.ts index 21c85de..afee24a 100644 --- a/native/serverconnection/test/js/main.ts +++ b/native/serverconnection/test/js/main.ts @@ -151,6 +151,8 @@ const do_connect = (connection) => { connection._voice_connection.register_client(7); }; do_connect(connection); + +/* let _connections = []; let i = 0; let ii = setInterval(() => { @@ -160,6 +162,7 @@ let ii = setInterval(() => { _connections.push(c); do_connect(c); }, 500); +*/ connection.callback_voice_data = (buffer, client_id, codec_id, flag_head, packet_id) => { console.log("Received voice of length %d from client %d in codec %d (Head: %o | ID: %d)", buffer.byteLength, client_id, codec_id, flag_head, packet_id); diff --git a/package.json b/package.json index 7852d61..c6eaf7e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "TeaClient", - "version": "1.4.11", + "version": "1.4.12", "description": "", "main": "main.js", "scripts": {